From d435a11acf62ba9bd82ee883b5b8c60c5b440d75 Mon Sep 17 00:00:00 2001 From: Wenyao Gao Date: Fri, 11 Sep 2026 15:18:06 -0700 Subject: [PATCH 1/2] feat: expose PowerX coverage and verify publication MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 中文：展示 PowerX 指标覆盖并核验发布结果。保留功耗审计信息，核对 8K/1K 产物、数据库与 API，补充指标可用性和 %TDP Pareto 支持。 --- .github/workflows/apply-run-overrides.yml | 2 + .github/workflows/ingest-results.yml | 16 +- docs/d3-charts.md | 4 +- docs/data-pipeline.md | 34 +++ .../app/cypress/component/gpu-graph.cy.tsx | 14 +- .../power-metric-availability.cy.tsx | 88 ++++++ .../cypress/component/scatter-graph.cy.tsx | 13 +- .../app/cypress/e2e/inference-chart.cy.ts | 25 ++ packages/app/cypress/support/mock-data.ts | 1 + .../src/app/api/unofficial-run/route.test.ts | 58 ++++ .../app/src/app/api/unofficial-run/route.ts | 2 + .../components/inference/InferenceContext.tsx | 2 + .../measured-power-direction.test.ts | 13 +- .../inference/metric-registry.test.ts | 4 +- .../components/inference/metric-registry.ts | 1 + .../app/src/components/inference/types.ts | 6 +- .../components/inference/ui/ChartControls.tsx | 7 + .../inference/ui/PowerMetricAvailability.tsx | 233 +++++++++++++++ .../utils/power-metric-availability.test.ts | 61 ++++ .../utils/power-metric-availability.ts | 68 +++++ .../inference/utils/powerCurves.test.ts | 34 +++ .../inference/utils/tooltip-utils.test.ts | 76 +++++ .../inference/utils/tooltipUtils.ts | 19 ++ .../src/components/ui/searchable-select.tsx | 2 +- .../app/src/hooks/api/ai-chart-data.test.ts | 2 +- .../src/lib/api-documentation.power.test.ts | 7 +- packages/app/src/lib/api-documentation.ts | 13 +- packages/app/src/lib/api-route-catalog.ts | 4 +- .../app/src/lib/benchmark-api-view.test.ts | 9 + packages/app/src/lib/benchmark-api-view.ts | 9 +- .../app/src/lib/benchmark-transform.test.ts | 19 ++ packages/app/src/lib/benchmark-transform.ts | 6 + packages/constants/src/metric-keys.test.ts | 6 +- packages/constants/src/metric-keys.ts | 2 + .../db/migrations/015_power_provenance.sql | 56 ++++ packages/db/src/etl/benchmark-ingest.test.ts | 73 +++++ packages/db/src/etl/benchmark-ingest.ts | 28 +- packages/db/src/etl/benchmark-mapper.test.ts | 265 +++++++++++++++++- packages/db/src/etl/benchmark-mapper.ts | 123 ++++++++ .../etl/fixtures/power-processor/README.md | 7 + .../power-processor/missing-power.json | 47 ++++ .../fixtures/power-processor/valid-power.json | 61 ++++ packages/db/src/etl/power-publication.test.ts | 111 ++++++++ packages/db/src/etl/power-publication.ts | 127 +++++++++ packages/db/src/ingest-ci-run.ts | 47 +++- packages/db/src/ingest-supplemental.ts | 18 +- .../src/queries/benchmark-snapshots.test.ts | 28 ++ packages/db/src/queries/benchmarks.test.ts | 40 +++ packages/db/src/queries/benchmarks.ts | 17 +- packages/db/src/verify-power-publication.ts | 89 ++++++ 50 files changed, 1957 insertions(+), 40 deletions(-) create mode 100644 packages/app/cypress/component/power-metric-availability.cy.tsx create mode 100644 packages/app/src/components/inference/ui/PowerMetricAvailability.tsx create mode 100644 packages/app/src/components/inference/utils/power-metric-availability.test.ts create mode 100644 packages/app/src/components/inference/utils/power-metric-availability.ts create mode 100644 packages/db/migrations/015_power_provenance.sql create mode 100644 packages/db/src/etl/fixtures/power-processor/README.md create mode 100644 packages/db/src/etl/fixtures/power-processor/missing-power.json create mode 100644 packages/db/src/etl/fixtures/power-processor/valid-power.json create mode 100644 packages/db/src/etl/power-publication.test.ts create mode 100644 packages/db/src/etl/power-publication.ts create mode 100644 packages/db/src/verify-power-publication.ts diff --git a/.github/workflows/apply-run-overrides.yml b/.github/workflows/apply-run-overrides.yml index 03e8b18b2..749880e80 100644 --- a/.github/workflows/apply-run-overrides.yml +++ b/.github/workflows/apply-run-overrides.yml @@ -5,6 +5,8 @@ on: branches: [main, master] paths: - 'packages/db/src/etl/run-overrides.ts' + - 'packages/db/src/etl/power-p90-backfills.ts' + - 'docs/data/power-p90-backfill.json' - '.github/workflows/apply-run-overrides.yml' workflow_dispatch: inputs: diff --git a/.github/workflows/ingest-results.yml b/.github/workflows/ingest-results.yml index 692382ab7..04b253e34 100644 --- a/.github/workflows/ingest-results.yml +++ b/.github/workflows/ingest-results.yml @@ -75,6 +75,7 @@ jobs: INGEST_RUN_ATTEMPT: ${{ steps.artifacts.outputs.merge-run-attempt }} INGEST_ARTIFACTS_PATH: ${{ github.workspace }}/artifacts INGEST_REPO: SemiAnalysisAI/InferenceX + POWER_PUBLICATION_MANIFEST: ${{ github.workspace }}/power-publication.json UNMAPPED_ENTITIES_OUTPUT: ${{ github.workspace }}/unmapped-entities.json run: bun run admin:db:ingest:ci @@ -94,7 +95,20 @@ jobs: VERCEL_INVALIDATE_SECRET: ${{ secrets.VERCEL_INVALIDATE_SECRET }} run: | curl -sSf -X POST "https://inferencex.semianalysis.com/api/v1/invalidate" \ - -H "Authorization: Bearer $VERCEL_INVALIDATE_SECRET" || true + -H "Authorization: Bearer $VERCEL_INVALIDATE_SECRET" + + - name: Verify PowerX source, database and public API + env: + DATABASE_WRITE_URL: ${{ secrets.DATABASE_WRITE_URL }} + run: bun packages/db/src/verify-power-publication.ts power-publication.json + + - name: Retain PowerX publication receipt + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: power-publication-${{ steps.artifacts.outputs.source-run-id }} + path: power-publication.json* + if-no-files-found: warn - name: Check for unmapped entities if: always() diff --git a/docs/d3-charts.md b/docs/d3-charts.md index 2d1902d15..953902330 100644 --- a/docs/d3-charts.md +++ b/docs/d3-charts.md @@ -79,7 +79,9 @@ Power axes use the same Pareto directions as other metrics while **Optimal Only* Power-boundary views show only the measurements forming that boundary by default. **Show all measurements** reveals the remaining dots without changing the curves, axis domains or zoom. This display preference is independent of **Optimal Only**, defaults off, and is shared through `i_allpoints=1`. Historical rings remain attached to visible historical points; tables and data exports retain their existing selection rules. -Measured power as a percentage of TDP has no preferred direction, so it always uses the upper boundary; its optimal switch is hidden. Charts preserve both display preferences when switching metrics. Energy per token retains its existing Pareto behavior. Power boundaries disable gradient strategy labels and the performance ruler. +Measured power as a percentage of TDP uses the same lower-is-better power-demand direction as watts. Dividing watts by one hardware's positive, constant TDP preserves its Pareto membership. Optimal Only therefore works on both metrics and retains the saved preference when switching between them. A lower percentage across different chips is not, by itself, an energy-efficiency comparison. Energy per token retains its existing Pareto behavior. + +Upper boundaries use monotone interpolation between unique-X vertices, including after zoom, and disable gradient strategy labels and the performance ruler. The same grouping applies to unofficial-run overlays; unrelated dates and runs never share a curve. ## Gradient Roofline Labels diff --git a/docs/data-pipeline.md b/docs/data-pipeline.md index f514339b3..5b52b05b5 100644 --- a/docs/data-pipeline.md +++ b/docs/data-pipeline.md @@ -491,3 +491,37 @@ requires a known, matching run attempt and leaves the row unchanged if those source values differ. Database recovery rejects the mismatch before writing. P90 replays require numeric `power_valid: 1`, schema version 2, and the exact original average power, including when checking an already-applied correction. + +### Power Audit Provenance (`power_invalid_reasons`, `power_audit`) + +Producers (`aggregate_power.py`) annotate every aggregate result row with two optional provenance fields alongside the `power_valid` verdict: + +- **`power_invalid_reasons`** — array of snake_case reason-code strings explaining a withheld verdict (emitted when `power_valid == 0`), e.g. `sampling_gap_exceeded`, `expected_gpu_count_mismatch`. +- **`power_audit`** — compact measurement-window audit object with optional fields: `window_start_unix`, `window_end_unix`, `expected_gpu_count`, `observed_gpu_count`, `sample_count`, `max_sample_gap_s`, `producer_sha`, `exporter_image_sha256`, `source`, `observed_gpu_ids`. `source` is a safe relative path inside the original artifact bundle, and device identifiers may be indices on older collectors. Present on valid and invalid rows alike. + +`mapBenchmarkRow()` narrows them defensively (`extractPowerInvalidReasons` / `extractPowerAudit`): reason codes must match `/^[a-z][a-z0-9_]*$/` (≤ 64 chars, deduplicated, capped at 32), audit numerics must be finite (counts: non-negative safe integers), shas collapse to `null` unless a non-empty string ≤ 128 chars, and unknown audit keys are dropped. A failed `benchmark_outcome.status` is rejected as a performance point, retaining its original artifact as evidence. An empty result maps to `undefined`, so the dedicated `benchmark_results.power_invalid_reasons` / `power_audit` JSONB columns (migration 015, mirroring the `workers` precedent from migration 006) store SQL NULL — never `[]` or `{}`. Legacy artifacts without the fields flow through every layer as NULL/undefined. + +Reads are **permanently tolerant**: `queries/benchmarks.ts` selects the columns as `to_jsonb(br) -> 'power_invalid_reasons'` (and `lb` on the matview branch) rather than bare column references. A bare reference fails during query planning until the next ingest workflow applies the migration, because migrations run in the ingest workflows rather than at Vercel deploy. The key lookup degrades to NULL while the column is missing and is byte-identical once it exists, making deploy order irrelevant. + +### PowerX publication receipts + +The normal CI importer writes `POWER_PUBLICATION_MANIFEST` when configured. Each +8K/1K point records the mapped, override-adjusted metric contract, all configuration +dimensions, original source run/attempt, structured audit, and input file SHA-256. +Reused sweeps keep their original source identity. Failed or unmapped explicit 8K/1K +results and database errors remain in the manifest; they cannot pass verification. + +After ingestion and cache invalidation, `bun packages/db/src/verify-power-publication.ts +power-publication.json` compares every expected point against the exact database +run attempt and public `runId=…&exactRun=true` response. It checks missing values and +withheld telemetry as well as numbers; legacy missing measurements remain missing. +A `matched` receipt establishes transport fidelity, not collection coverage. An +empty receipt says `no_8k1k_points`, never that power coverage was validated. +The workflow retains both the input manifest and verification receipt. Cache +invalidation errors fail the workflow instead of being swallowed. Imported P75/P90 +ledger edits trigger the existing reviewed override workflow. + +The dashboard availability panel uses scoped points before Y-metric filtering, +including visible unofficial overlays. It distinguishes schema-2 validation, +other validated data, missing verdicts, withheld measurements, unavailable metrics, +and non-applicable separate-pool metrics without filling missing values. diff --git a/packages/app/cypress/component/gpu-graph.cy.tsx b/packages/app/cypress/component/gpu-graph.cy.tsx index a0c938357..f191427dc 100644 --- a/packages/app/cypress/component/gpu-graph.cy.tsx +++ b/packages/app/cypress/component/gpu-graph.cy.tsx @@ -1,4 +1,5 @@ import GPUGraph from '@/components/inference/ui/GPUGraph'; +import { chartDefinitions } from '@/components/inference/metric-registry'; import { InferenceContextsProvider } from '@/components/inference/InferenceContext'; import { useState } from 'react'; import { mountWithProviders } from '../support/test-utils'; @@ -549,6 +550,8 @@ describe('GPU comparison power envelopes', () => { chartType: latency ? 'e2e' : 'interactivity', y_measuredAvgPower_roofline: latency ? 'lower_left' : 'lower_right', y_measuredJPerOutputToken_roofline: latency ? 'lower_left' : 'lower_right', + y_measuredPowerPercentTdp_roofline: + chartDefinitions[latency ? 1 : 0].y_measuredPowerPercentTdp_roofline, })} /> @@ -625,20 +628,27 @@ describe('GPU comparison power envelopes', () => { .and('contain.text', 'not efficiency frontiers'); }); - it('localizes the %TDP measurement toggle without changing the saved Optimal Only preference for energy', () => { + it('applies %TDP Pareto filtering and localizes its independent measurement toggle', () => { mountWithProviders( , ); cy.contains('button', 'Percent TDP').click(); - cy.get('#gpu-hide-non-optimal').should('not.exist'); + cy.get('#gpu-hide-non-optimal').should('have.attr', 'data-state', 'checked'); + cy.get('#gpu-power-curves .dot-group').should('have.length', 2); + cy.get('#gpu-power-curves .roofline-path').should('not.exist'); + cy.get('[data-testid="power-curve-description"]').should('contain', '只有一个点'); + cy.get('#gpu-show-all-measurements').should('not.exist'); + cy.get('#gpu-hide-non-optimal').click({ force: true }); cy.get('#gpu-power-curves .dot-group').should('have.length', 6); cy.contains('显示全部测量点').should('be.visible'); + cy.get('#gpu-show-all-measurements').should('have.attr', 'data-state', 'unchecked'); cy.get('#gpu-show-all-measurements').click({ force: true }); cy.get('#gpu-power-curves .dot-group').should('have.length', 12); cy.get('#gpu-power-curves .roofline-path').should('have.length', 2); cy.get('[data-testid="power-curve-description"]').should('contain', '不代表能效 Pareto 前沿'); + cy.get('#gpu-hide-non-optimal').click({ force: true }); cy.contains('button', 'Energy').click(); cy.get('#gpu-show-all-measurements').should('not.exist'); cy.get('#gpu-hide-non-optimal').should('have.attr', 'data-state', 'checked'); diff --git a/packages/app/cypress/component/power-metric-availability.cy.tsx b/packages/app/cypress/component/power-metric-availability.cy.tsx new file mode 100644 index 000000000..6a6c7b1a6 --- /dev/null +++ b/packages/app/cypress/component/power-metric-availability.cy.tsx @@ -0,0 +1,88 @@ +import { useState } from 'react'; +import { PathnameContext } from 'next/dist/shared/lib/hooks-client-context.shared-runtime'; +import { + PowerMetricAvailability, + PowerMetricAvailabilityPanel, +} from '@/components/inference/ui/PowerMetricAvailability'; +import { createMockInferenceData } from '../support/mock-data'; +import { mountWithProviders } from '../support/test-utils'; +import { Precision, Model, Sequence } from '@/lib/data-mappings'; + +const source = 'https://github.com/SemiAnalysisAI/InferenceX/actions/runs/123/attempts/2'; +const base = { model: Model.Qwen3_5, precision: Precision.FP8, run_url: source }; +const missing = createMockInferenceData({ ...base, hwKey: 'gb200' }); +const invalid = createMockInferenceData({ + ...base, + hwKey: 'mi355x', + power_valid: 0, + power_invalid_reasons: ['sampling_gap_exceeded'], +}); +const measured = createMockInferenceData({ + ...base, + hwKey: 'b200', + power_valid: 1, + power_metric_schema_version: 2, + measuredAvgPower: { y: 640, roof: false }, +}); +function Panel() { + const [metric, select] = useState('y_measuredAvgPower'); + return ( + + ); +} + +describe('PowerX metric availability', () => { + it('explains missing GB and withheld AMD data, exposes source links, and switches metrics', () => { + cy.mount(); + cy.contains('1 of 3 points have this metric'); + cy.contains('Validation failed: 1'); + cy.contains('Metric not reported: 1'); + cy.contains('summary', 'source details').click(); + cy.contains('sampling_gap_exceeded'); + cy.contains('a', 'Source run').should('have.attr', 'href', source); + cy.contains('summary', 'all measured metrics').click(); + cy.contains('button', 'Measured Prefill Power per Chip').click(); + cy.contains('No separate worker pools: 3'); + cy.contains('0 of 3 points have this metric'); + }); + it('uses Chinese copy for the same coverage states', () => { + cy.mount( + + + , + ); + cy.contains('3 个数据点中有 1 个提供此指标'); + cy.contains('验证失败: 1'); + cy.contains('summary', '所有实测指标的可用性').click(); + cy.contains('缺失值不会被替换为零或 TDP 估算值'); + }); + it('counts filtered unofficial rows before the metric filter and respects hidden overlay hardware', () => { + mountWithProviders( + , + { + inference: { + selectionPoints: [missing], + activeHwTypes: new Set(['gb200']), + selectedModel: Model.Qwen3_5, + selectedSequence: Sequence.EightK_OneK, + selectedPrecisions: [Precision.FP8], + }, + unofficial: { + isUnofficialRun: true, + activeOverlayHwTypes: new Set(['b200']), + getOverlayData: () => ({ data: [measured, invalid], hardwareConfig: {} }), + }, + }, + ); + cy.contains('1 of 2 points have this metric'); + cy.contains('Metric not reported: 1'); + cy.get('[data-testid="power-metric-availability"]').should( + 'not.contain', + 'Validation failed: 1', + ); + }); +}); diff --git a/packages/app/cypress/component/scatter-graph.cy.tsx b/packages/app/cypress/component/scatter-graph.cy.tsx index 9670c360c..ff996c0df 100644 --- a/packages/app/cypress/component/scatter-graph.cy.tsx +++ b/packages/app/cypress/component/scatter-graph.cy.tsx @@ -6,6 +6,7 @@ import { UnofficialRunContext, } from '@/components/unofficial-run-provider'; import ScatterGraph from '@/components/inference/ui/ScatterGraph'; +import { chartDefinitions } from '@/components/inference/metric-registry'; import ChartDisplay from '@/components/inference/ui/ChartDisplay'; import { mountWithProviders } from '../support/test-utils'; import { expandLegendAdvanced } from '../support/legend-advanced'; @@ -2255,6 +2256,7 @@ describe('Power envelopes', () => { chartType: 'interactivity', y_measuredAvgPower_roofline: 'lower_right', y_measuredJPerOutputToken_roofline: 'lower_right', + y_measuredPowerPercentTdp_roofline: chartDefinitions[0].y_measuredPowerPercentTdp_roofline, }); return ( @@ -2319,7 +2321,10 @@ describe('Power envelopes', () => { cy.get('#power-sweep .roofline-path').should('not.exist'); cy.get('#scatter-show-all-measurements').should('not.exist'); cy.contains('button', 'Percent TDP').click(); - cy.get('#scatter-hide-non-optimal').should('not.exist'); + cy.get('#scatter-hide-non-optimal').should('have.attr', 'data-state', 'checked'); + cy.get('#power-sweep .roofline-path').should('not.exist'); + cy.get('[data-testid="power-curve-description"]').should('contain', 'single point'); + cy.get('#scatter-hide-non-optimal').click({ force: true }); cy.get('#power-sweep .roofline-path[data-curve-kind="power-envelope"]') .should('have.length', 1) .invoke('attr', 'd') @@ -2327,6 +2332,12 @@ describe('Power envelopes', () => { cy.get('#power-sweep .dot-group') .filter((_, element) => element.style.opacity !== '0') .should('have.length', 3); + cy.get('#scatter-show-all-measurements').should('have.attr', 'data-state', 'unchecked'); + cy.get('#scatter-show-all-measurements').click({ force: true }); + cy.get('#power-sweep .dot-group') + .should('have.length', 4) + .each(($point) => cy.wrap($point).should('have.css', 'opacity', '1')); + cy.get('#scatter-hide-non-optimal').click({ force: true }); cy.contains('button', 'Energy').click(); cy.get('#scatter-hide-non-optimal').should('have.attr', 'data-state', 'checked'); cy.get('#power-sweep .roofline-path[data-curve-kind="pareto"]').should('have.length', 1); diff --git a/packages/app/cypress/e2e/inference-chart.cy.ts b/packages/app/cypress/e2e/inference-chart.cy.ts index 43c4f4fa9..7c1a55547 100644 --- a/packages/app/cypress/e2e/inference-chart.cy.ts +++ b/packages/app/cypress/e2e/inference-chart.cy.ts @@ -490,3 +490,28 @@ describe('AgentX replaces a complete curve while preserving an unofficial compar cy.get('[data-testid="inference-chart-display"] svg .dot-group').should('have.length', 6); }); }); + +it('hydrates a direct PowerX metric link and shows availability for the selected workload', () => { + cy.visit('/inference/qwen-3-5?i_seq=8k%2F1k&i_prec=fp8&i_metric=y_measuredPowerPercentTdp', { + onBeforeLoad(win) { + win.localStorage.setItem('inferencex-star-modal-dismissed', String(Date.now())); + unlockAgenticGate(win); + cy.spy(win.console, 'error').as('powerLinkConsoleErrors'); + }, + }); + cy.get('[data-testid="yaxis-metric-selector"]').should('contain', 'Percent of TDP'); + cy.get('[data-testid="power-metric-availability"]').should( + 'contain', + 'Current workload and hardware selection', + ); + cy.contains('summary', 'Availability of all measured metrics').click(); + cy.get('[data-testid="power-metric-availability"]').within(() => { + cy.contains('button', 'Measured P75 Fleet Power per Chip').should('contain', '/'); + cy.contains('button', 'Measured Joules per Output Token').click(); + }); + cy.get('[data-testid="yaxis-metric-selector"]').should( + 'contain', + 'Measured Joules per Output Token', + ); + cy.get('@powerLinkConsoleErrors').should('not.be.calledWithMatch', /hydrat/i); +}); diff --git a/packages/app/cypress/support/mock-data.ts b/packages/app/cypress/support/mock-data.ts index 400d9cca0..25b852620 100644 --- a/packages/app/cypress/support/mock-data.ts +++ b/packages/app/cypress/support/mock-data.ts @@ -175,6 +175,7 @@ export function createMockInferenceContextValues( ): MockInferenceContextValues { const hwConfig = createMockHardwareConfig(); return { + selectionPoints: [], activeHwTypes: new Set(['h100', 'b200', 'b200_trt', 'mi300x', 'h200']), hwTypesWithData: new Set(['h100', 'b200', 'b200_trt', 'mi300x', 'h200']), toggleHwType: namedStub('toggleHwType'), diff --git a/packages/app/src/app/api/unofficial-run/route.test.ts b/packages/app/src/app/api/unofficial-run/route.test.ts index d4a8e5f4d..d4a647f58 100644 --- a/packages/app/src/app/api/unofficial-run/route.test.ts +++ b/packages/app/src/app/api/unofficial-run/route.test.ts @@ -193,6 +193,40 @@ describe('normalizeArtifactRows', () => { }, ); + it('carries power audit provenance on overlay rows', () => { + const audit = { + window_start_unix: 1756174800, + window_end_unix: 1756175400, + expected_gpu_count: 8, + observed_gpu_count: 8, + sample_count: 4800, + max_sample_gap_s: 1.013, + producer_sha: null, + exporter_image_sha256: null, + }; + const [row] = normalizeArtifactRows( + [ + rawRow({ + power_valid: 0, + power_invalid_reasons: ['sampling_gap_exceeded'], + power_audit: audit, + }), + ], + '2026-03-01', + ); + + expect(row.power_invalid_reasons).toEqual(['sampling_gap_exceeded']); + expect(row.power_audit).toEqual(audit); + expect(row.metrics).not.toHaveProperty('power_invalid_reasons'); + expect(row.metrics).not.toHaveProperty('power_audit'); + }); + + it('leaves provenance keys undefined for rows without the contract fields', () => { + const [row] = normalizeArtifactRows([rawRow()], '2026-03-01'); + expect(row.power_invalid_reasons).toBeUndefined(); + expect(row.power_audit).toBeUndefined(); + }); + it('preserves recipe identity for unofficial overlays', () => { const rows = normalizeArtifactRows( [ @@ -937,3 +971,27 @@ describe('GET /api/unofficial-run', () => { expect(body.error).toContain('999'); }); }); + +// Retained outputs of the real producer CLI, with synthetic input traces. +import validProcessorPower from '../../../../../db/src/etl/fixtures/power-processor/valid-power.json'; +import missingProcessorPower from '../../../../../db/src/etl/fixtures/power-processor/missing-power.json'; +import { transformBenchmarkRows } from '@/lib/benchmark-transform'; + +it('carries the actual processor audit and power through API mapping into measured chart fields', () => { + const rows = normalizeArtifactRows([validProcessorPower, missingProcessorPower], '2026-09-11'); + expect(rows[0].power_audit).toMatchObject({ + source: 'power_validation_benchmark_result.json', + observed_gpu_ids: ['0', '1'], + sample_count: 34, + }); + expect(rows[1].power_invalid_reasons).toEqual(['telemetry_file_missing']); + expect(rows[1].metrics).not.toHaveProperty('avg_power_w'); + const points = transformBenchmarkRows(rows).chartData[0]; + expect(points[0].measuredAvgPower?.y).toBe(500); + expect(points[0].measuredP75Power?.y).toBe(500); + expect(points[0].measuredP90Power?.y).toBe(500); + expect(points[0].measuredWhPerSuccessfulQuery?.y).toBeCloseTo(100 / 3600); + expect(points[0].measuredPrefillAvgPower).toBeUndefined(); + expect(points[1].measuredAvgPower).toBeUndefined(); + expect(points[1].power_audit?.observed_gpu_count).toBe(0); +}); diff --git a/packages/app/src/app/api/unofficial-run/route.ts b/packages/app/src/app/api/unofficial-run/route.ts index 69d06f2da..fb6fa684f 100644 --- a/packages/app/src/app/api/unofficial-run/route.ts +++ b/packages/app/src/app/api/unofficial-run/route.ts @@ -83,6 +83,8 @@ export function normalizeArtifactRows( // Surface the same per-worker payload the DB path emits so unofficial // overlays carry the multinode measured-power breakdown too. workers: params.workers, + power_invalid_reasons: params.powerInvalidReasons, + power_audit: params.powerAudit, date, run_url: runUrl, }); diff --git a/packages/app/src/components/inference/InferenceContext.tsx b/packages/app/src/components/inference/InferenceContext.tsx index bd549df9c..d8d37181f 100644 --- a/packages/app/src/components/inference/InferenceContext.tsx +++ b/packages/app/src/components/inference/InferenceContext.tsx @@ -1724,6 +1724,7 @@ export function InferenceProvider({ hwTypesWithData, hardwareConfig, graphs, + selectionPoints, loading, refreshing, error, @@ -1741,6 +1742,7 @@ export function InferenceProvider({ hwTypesWithData, hardwareConfig, graphs, + selectionPoints, loading, refreshing, error, diff --git a/packages/app/src/components/inference/measured-power-direction.test.ts b/packages/app/src/components/inference/measured-power-direction.test.ts index 8e4aaaf19..44d2e4ff0 100644 --- a/packages/app/src/components/inference/measured-power-direction.test.ts +++ b/packages/app/src/components/inference/measured-power-direction.test.ts @@ -28,6 +28,7 @@ const MEASURED_POWER_METRICS = [ 'y_measuredAvgPower', 'y_measuredPrefillAvgPower', 'y_measuredDecodeAvgPower', + 'y_measuredPowerPercentTdp', ] as const; const QUERY_ENERGY_METRICS = [ @@ -164,16 +165,16 @@ describe('measured-power Pareto direction', () => { } }); - it('leaves %TDP without a Pareto direction on either block', () => { - // %TDP is a utilization gauge, not an efficiency frontier: a config running - // hotter is not "worse" along an axis the roofline can order, so declaring a - // corner would draw a frontier with no meaning. The axis still ships as a - // plottable, bilingual metric — it just never anchors a roofline. + it('keeps %TDP bilingual while using the same per-hardware frontier as watts', () => { + // A fixed hardware TDP rescales watts without changing dominance within + // that hardware series. The shared sweep tests above exercise both axes. for (const chartDef of [interactivityDef, e2eDef]) { expect(chartDef.y_measuredPowerPercentTdp).toMatch(/\.y$/u); expect(chartDef['y_measuredPowerPercentTdp_label']).toBeTruthy(); expect(chartDef['y_measuredPowerPercentTdp_labelZh']).toBeTruthy(); - expect(declaredDirection(chartDef, 'y_measuredPowerPercentTdp')).toBeUndefined(); + expect(declaredDirection(chartDef, 'y_measuredPowerPercentTdp')).toBe( + declaredDirection(chartDef, 'y_measuredAvgPower'), + ); } }); }); diff --git a/packages/app/src/components/inference/metric-registry.test.ts b/packages/app/src/components/inference/metric-registry.test.ts index 71f1bd7a4..28045728e 100644 --- a/packages/app/src/components/inference/metric-registry.test.ts +++ b/packages/app/src/components/inference/metric-registry.test.ts @@ -37,8 +37,8 @@ describe('metric registry', () => { expect(e2e.y_tokensPerDollarH_roofline).toBe('upper_right'); expect(interactivity.y_costh_roofline).toBe('lower_right'); expect(e2e.y_costh_roofline).toBe('lower_left'); - expect(interactivity.y_measuredPowerPercentTdp_roofline).toBeUndefined(); - expect(e2e.y_measuredPowerPercentTdp_roofline).toBeUndefined(); + expect(interactivity.y_measuredPowerPercentTdp_roofline).toBe('lower_right'); + expect(e2e.y_measuredPowerPercentTdp_roofline).toBe('lower_left'); }); it('preserves metric-specific x overrides and bilingual labels', () => { diff --git a/packages/app/src/components/inference/metric-registry.ts b/packages/app/src/components/inference/metric-registry.ts index 762ca02bb..ccba918b1 100644 --- a/packages/app/src/components/inference/metric-registry.ts +++ b/packages/app/src/components/inference/metric-registry.ts @@ -384,6 +384,7 @@ export const METRIC_REGISTRY = { labelZh: '实测平均功耗（TDP 占比）', title: 'Measured Average Power as Percent of TDP', titleZh: '实测平均功耗占 TDP 百分比', + polarity: 'lower', }, } as const satisfies Record; diff --git a/packages/app/src/components/inference/types.ts b/packages/app/src/components/inference/types.ts index fc5033125..5af962e2a 100644 --- a/packages/app/src/components/inference/types.ts +++ b/packages/app/src/components/inference/types.ts @@ -1,5 +1,5 @@ import type React from 'react'; -import type { WorkerPower } from '@semianalysisai/inferencex-db/queries/benchmarks'; +import type { PowerAudit, WorkerPower } from '@semianalysisai/inferencex-db/queries/benchmarks'; import type { HardwareEntry } from '@/lib/constants'; import type { Model, Sequence } from '@/lib/data-mappings'; @@ -120,6 +120,8 @@ export interface AggDataEntry { // Measured GPU telemetry (emitted by runner's aggregate_power.py). // Optional because historical runs predate the fields. power_valid?: number; + power_invalid_reasons?: string[]; + power_audit?: PowerAudit; power_metric_schema_version?: number; /** * Certification tier for the measured power telemetry, derived by @@ -593,6 +595,8 @@ export interface InferenceDataContextType { hwTypesWithData: Set; hardwareConfig: HardwareConfig; graphs: RenderableGraph[]; + /** Missing metrics must remain countable after chart filtering hides their points. */ + selectionPoints: InferenceData[]; loading: boolean; /** True while `graphs` shows previous-key data (placeholder) or a background * refetch is in flight — i.e. content is visible but about to update. */ diff --git a/packages/app/src/components/inference/ui/ChartControls.tsx b/packages/app/src/components/inference/ui/ChartControls.tsx index 6e881dc6b..cbd24da8d 100644 --- a/packages/app/src/components/inference/ui/ChartControls.tsx +++ b/packages/app/src/components/inference/ui/ChartControls.tsx @@ -58,6 +58,7 @@ import { import { useOpenDropdown } from '@/hooks/useOpenDropdown'; import { ModelArchitectureInfoLink } from './ModelArchitectureInfoLink'; import { MetricExplanation } from './MetricExplanation'; +import { PowerMetricAvailability } from './PowerMetricAvailability'; import { XAxisModeSelector } from './XAxisModeSelector'; import { showsTcoBasisSelector, Sequence, type Model, type Percentile } from '@/lib/data-mappings'; import { useLocale } from '@/lib/use-locale'; @@ -508,6 +509,12 @@ export default function ChartControls({ noResultsLabel={locale === 'zh' ? '无结果' : undefined} clearSearchLabel={locale === 'zh' ? '清除搜索' : undefined} /> + {mounted && ( + + )} {tcoVisible && ( diff --git a/packages/app/src/components/inference/ui/PowerMetricAvailability.tsx b/packages/app/src/components/inference/ui/PowerMetricAvailability.tsx new file mode 100644 index 000000000..a3dd548b0 --- /dev/null +++ b/packages/app/src/components/inference/ui/PowerMetricAvailability.tsx @@ -0,0 +1,233 @@ +'use client'; + +import { useMemo } from 'react'; +import { useInferenceData, useInferenceFilters } from '../InferenceContext'; +import { isMeasuredEnergyConfigKey, metricOptionTitle, type MetricKey } from '../metric-registry'; +import type { InferenceData } from '../types'; +import { matchesQuickFilters } from '../utils/quickFilters'; +import { + powerMetricAvailability, + powerMetricState, + type PowerAvailabilityState, +} from '../utils/power-metric-availability'; +import { useUnofficialRun } from '@/components/unofficial-run-provider'; +import { hardwareKeyMatchesAnyBase } from '@/lib/constants'; +import { useLocale } from '@/lib/use-locale'; +import { track } from '@/lib/analytics'; + +const STRINGS = { + en: { + title: 'PowerX availability', + scope: + 'Current workload and hardware selection, before Optimal Only. Includes visible unofficial runs.', + loading: 'Updating measurement availability…', + none: 'No benchmark points match this selection.', + summary: (available: number, total: number) => + `${available} of ${total} points have this metric`, + labels: { + strict: 'Validated · schema 2', + validated: 'Validated · other or missing schema', + unverified: 'No validation verdict', + invalid: 'Validation failed', + inapplicable: 'No separate worker pools', + ambiguous: 'Whole-deployment energy schema unavailable', + missing: 'Metric not reported', + }, + note: 'A missing verdict does not establish age or validity. Prefill/decode metrics measure separate worker pools. Missing values are never replaced with zero or TDP estimates.', + all: 'Availability of all measured metrics', + evidence: 'Selected metric: source details', + run: 'Source run', + }, + zh: { + title: 'PowerX 指标可用性', + scope: '按当前工作负载与硬件选择统计，不受“仅最优”影响，包含已显示的非官方运行。', + loading: '正在更新指标可用性…', + none: '当前选择没有匹配的基准测试数据点。', + summary: (available: number, total: number) => + `${total} 个数据点中有 ${available} 个提供此指标`, + labels: { + strict: '已验证 · schema 2', + validated: '已验证 · 其他或未标注 schema', + unverified: '未提供验证结论', + invalid: '验证失败', + inapplicable: '无独立 worker 池', + ambiguous: '缺少整个部署的能耗 schema', + missing: '未提供此指标', + }, + note: '缺少验证结论不能判断数据新旧或有效性。prefill/decode 指标仅衡量独立 worker 池。缺失值不会被替换为零或 TDP 估算值。', + all: '所有实测指标的可用性', + evidence: '当前指标的来源详情', + run: '来源运行', + }, +} as const; + +export function PowerMetricAvailabilityPanel({ + points, + metric, + onSelect, + loading = false, +}: { + points: readonly InferenceData[]; + metric: string; + onSelect: (metric: string) => void; + loading?: boolean; +}) { + const locale = useLocale(); + const t = STRINGS[locale]; + const availability = useMemo(() => powerMetricAvailability(points), [points]); + const selected = availability.find((entry) => entry.metric === metric); + if (!selected) return null; + const sources = new Map< + string, + { point: InferenceData; state: PowerAvailabilityState; count: number } + >(); + for (const point of points) { + const state = powerMetricState(point, metric); + if (state === 'strict') continue; + const key = JSON.stringify([point.hwKey, point.run_url, state, point.power_invalid_reasons]); + const group = sources.get(key); + if (group) group.count++; + else sources.set(key, { point, state, count: 1 }); + } + return ( +
+

+ {t.title}:{' '} + {loading + ? t.loading + : points.length > 0 + ? t.summary(selected.available, selected.total) + : t.none} +

+ {!loading && ( + <> +

{t.scope}

+

+ {Object.entries(selected.counts) + .filter(([, count]) => count > 0) + .map(([state, count]) => `${t.labels[state as PowerAvailabilityState]}: ${count}`) + .join(' · ')} +

+
{ + if (event.currentTarget.open) track('inference_power_availability_opened'); + }} + > + {t.all} +
    + {availability.map((entry) => ( +
  • + +
    • + ))} +
+

{t.note}

+
+ {sources.size > 0 && ( +
+ {t.evidence} +
    + {[...sources.values()].map(({ point, state, count }, index) => ( +
  • + {point.hwKey}: {t.labels[state]} ({count}) + {point.power_invalid_reasons?.length + ? ` · ${point.power_invalid_reasons.join(', ')}` + : ''} + {point.power_audit?.source ? ` · ${point.power_audit.source}` : ''} + {point.run_url && + /^https:\/\/github\.com\/[\w.-]+\/[\w.-]+\/actions\/runs\/\d+(?:\/attempts\/\d+)?$/u.test( + point.run_url, + ) && ( + <> + {' '} + ·{' '} + + {t.run} + + + )} +
    • + ))} +
+
+ )} + + )} +
+ ); +} + +/** Reuse pre-metric rows, including overlays: absent power must not erase its own explanation. */ +export function PowerMetricAvailability({ + metric, + onSelect, +}: { + metric: string; + onSelect: (metric: string) => void; +}) { + const { selectionPoints = [], loading, refreshing } = useInferenceData(); + const { + selectedModel, + selectedSequence, + selectedPrecisions, + activeHwTypes, + quickFilters, + compareGpuPair, + } = useInferenceFilters(); + const { getOverlayData, activeOverlayHwTypes, isUnofficialRun, localOfficialOverride } = + useUnofficialRun(); + const points = useMemo(() => { + const officialHw = isUnofficialRun ? (localOfficialOverride ?? activeHwTypes) : activeHwTypes; + const official = selectionPoints.filter( + (point) => + selectedPrecisions.includes(point.precision) && officialHw.has(String(point.hwKey)), + ); + const overlay = getOverlayData?.(selectedModel, selectedSequence, 'interactivity')?.data ?? []; + return [ + ...official, + ...overlay.filter( + (point) => + selectedPrecisions.includes(point.precision) && + activeOverlayHwTypes.has(String(point.hwKey)) && + matchesQuickFilters(point, quickFilters) && + (!compareGpuPair || hardwareKeyMatchesAnyBase(String(point.hwKey), compareGpuPair)), + ), + ]; + }, [ + selectionPoints, + selectedPrecisions, + activeHwTypes, + getOverlayData, + selectedModel, + selectedSequence, + activeOverlayHwTypes, + isUnofficialRun, + localOfficialOverride, + quickFilters, + compareGpuPair, + ]); + if (!isMeasuredEnergyConfigKey(metric)) return null; + return ( + + ); +} diff --git a/packages/app/src/components/inference/utils/power-metric-availability.test.ts b/packages/app/src/components/inference/utils/power-metric-availability.test.ts new file mode 100644 index 000000000..d63c225ce --- /dev/null +++ b/packages/app/src/components/inference/utils/power-metric-availability.test.ts @@ -0,0 +1,61 @@ +import { describe, expect, it } from 'vitest'; +import type { InferenceData } from '../types'; +import { powerMetricAvailability, powerMetricState } from './power-metric-availability'; +import { createMockInferenceData } from '../../../../cypress/support/mock-data'; + +function point(overrides: Partial = {}) { + return createMockInferenceData(overrides); +} +describe('PowerX availability', () => { + it('separates strict, validated unversioned, no verdict, invalid and missing without hiding zero', () => { + const values = { measuredAvgPower: { y: 0, roof: false } }; + const rows = [ + point({ ...values, power_valid: 1, power_metric_schema_version: 2 }), + point({ ...values, power_valid: 1 }), + point(values), + point({ power_valid: 0, ...values }), + point(), + ]; + expect( + powerMetricAvailability(rows).find((row) => row.metric === 'y_measuredAvgPower'), + ).toMatchObject({ + available: 3, + total: 5, + counts: { strict: 1, validated: 1, unverified: 1, invalid: 1, missing: 1 }, + }); + }); + it('retains real role metrics while distinguishing shared pools and ambiguous old deployment energy', () => { + expect(powerMetricState(point(), 'y_measuredPrefillAvgPower')).toBe('inapplicable'); + expect( + powerMetricState( + point({ + disagg: true, + power_valid: 1, + power_metric_schema_version: 2, + measuredPrefillJPerInputToken: { y: 2, roof: false }, + }), + 'y_measuredPrefillJPerInputToken', + ), + ).toBe('strict'); + expect( + powerMetricState(point({ disagg: true, power_valid: 1 }), 'y_measuredJPerInputToken'), + ).toBe('ambiguous'); + }); + it('counts the actual conversion and percentile fields independently', () => { + const rows = [ + point({ + power_valid: 1, + power_metric_schema_version: 2, + measuredWhPerSuccessfulQuery: { y: 1, roof: false }, + measuredPowerPercentTdp: { y: 40, roof: false }, + }), + ]; + const byMetric = Object.fromEntries( + powerMetricAvailability(rows).map((row) => [row.metric, row.available]), + ); + expect(byMetric.y_measuredWhPerSuccessfulQuery).toBe(1); + expect(byMetric.y_measuredPowerPercentTdp).toBe(1); + expect(byMetric.y_measuredP75Power).toBe(0); + expect(byMetric.y_measuredP90Power).toBe(0); + }); +}); diff --git a/packages/app/src/components/inference/utils/power-metric-availability.ts b/packages/app/src/components/inference/utils/power-metric-availability.ts new file mode 100644 index 000000000..6a2e7882f --- /dev/null +++ b/packages/app/src/components/inference/utils/power-metric-availability.ts @@ -0,0 +1,68 @@ +import { MEASURED_ENERGY_METRIC_CONFIG_KEYS, isMetricKey } from '../metric-registry'; +import type { InferenceData } from '../types'; + +export const POWER_AVAILABILITY_STATES = [ + 'strict', + 'validated', + 'unverified', + 'invalid', + 'inapplicable', + 'ambiguous', + 'missing', +] as const; +export type PowerAvailabilityState = (typeof POWER_AVAILABILITY_STATES)[number]; +export type PowerAvailabilityCounts = Record; +const ROLE_METRICS = new Set([ + 'y_measuredPrefillAvgPower', + 'y_measuredDecodeAvgPower', + 'y_measuredPrefillJPerInputToken', + 'y_measuredDecodeJPerOutputToken', +]); +const WHOLE_ENERGY_METRICS = new Set([ + 'y_measuredJPerInputToken', + 'y_measuredJPerOutputToken', + 'y_measuredJPerTotalToken', + 'y_measuredJPerSuccessfulQuery', + 'y_measuredWhPerSuccessfulQuery', +]); + +/** Uses the chart's actual admitted field, so conversions and semantic gates agree. */ +export function powerMetricState(point: InferenceData, configKey: string): PowerAvailabilityState { + if (ROLE_METRICS.has(configKey) && !point.disagg) return 'inapplicable'; + if (point.power_valid === 0) return 'invalid'; + const key = configKey.replace(/^y_/u, ''); + const value = isMetricKey(key) ? point[key] : undefined; + if ( + value && + typeof value === 'object' && + 'y' in value && + typeof value.y === 'number' && + Number.isFinite(value.y) + ) { + if (point.power_valid === 1) + return point.power_metric_schema_version === 2 ? 'strict' : 'validated'; + return 'unverified'; + } + if ( + WHOLE_ENERGY_METRICS.has(configKey) && + point.disagg && + point.power_metric_schema_version !== 2 + ) + return 'ambiguous'; + return 'missing'; +} + +export function powerMetricAvailability(points: readonly InferenceData[]) { + return [...MEASURED_ENERGY_METRIC_CONFIG_KEYS].map((metric) => { + const counts = Object.fromEntries( + POWER_AVAILABILITY_STATES.map((state) => [state, 0]), + ) as PowerAvailabilityCounts; + for (const point of points) counts[powerMetricState(point, metric)]++; + return { + metric, + counts, + available: counts.strict + counts.validated + counts.unverified, + total: points.length, + }; + }); +} diff --git a/packages/app/src/components/inference/utils/powerCurves.test.ts b/packages/app/src/components/inference/utils/powerCurves.test.ts index 99f5d2711..a71f8bde2 100644 --- a/packages/app/src/components/inference/utils/powerCurves.test.ts +++ b/packages/app/src/components/inference/utils/powerCurves.test.ts @@ -1,6 +1,8 @@ import { describe, expect, it } from 'vitest'; import type { InferenceData } from '@/components/inference/types'; +import { chartDefinitions } from '@/components/inference/metric-registry'; +import type { ParetoDirection } from '@/lib/chart-utils'; import { chartFrontier, isPowerCurveMetric, upperPowerEnvelope } from './powerCurves'; @@ -24,6 +26,38 @@ function point(conc: number, x: number, y: number, overrides: Partial { + it.each(chartDefinitions)( + 'keeps watts and percent TDP on the same $chartType frontier within a hardware series', + (definition) => { + const samples = [ + point(1, 200, 600), + point(4, 160, 400), + point(8, 120, 500), + point(16, 100, 300), + ].map((sample) => ({ + ...sample, + x: definition.chartType === 'e2e' ? 1000 / sample.x : sample.x, + })); + const watts = chartFrontier( + samples, + definition.y_measuredAvgPower_roofline as ParetoDirection | undefined, + ); + expect(watts.map((sample) => sample.conc).toSorted((a, b) => a - b)).toEqual([1, 4, 16]); + // TDP is fixed within each hardware series. Converting its watts to a + // percentage must neither admit the dominated point nor drop a tradeoff. + for (const tdp of [700, 1000, 1400]) { + const percentages = samples.map((sample) => ({ ...sample, y: (sample.y / tdp) * 100 })); + const percentFrontier = chartFrontier( + percentages, + definition.y_measuredPowerPercentTdp_roofline as ParetoDirection | undefined, + ); + expect(percentFrontier.map((sample) => sample.conc)).toEqual( + watts.map((sample) => sample.conc), + ); + } + }, + ); + it('selects power gauges without changing energy chart semantics', () => { expect(isPowerCurveMetric('y_measuredAvgPower')).toBe(true); expect(isPowerCurveMetric('y_measuredP75Power')).toBe(true); diff --git a/packages/app/src/components/inference/utils/tooltip-utils.test.ts b/packages/app/src/components/inference/utils/tooltip-utils.test.ts index f7f222753..3579779fd 100644 --- a/packages/app/src/components/inference/utils/tooltip-utils.test.ts +++ b/packages/app/src/components/inference/utils/tooltip-utils.test.ts @@ -970,6 +970,82 @@ describe('generateGPUGraphTooltipContent', () => { }); }); +describe('measured-power withheld tooltip line', () => { + const reasons = ['sampling_gap_exceeded', 'expected_gpu_count_mismatch']; + + it('renders the withheld line with humanized codes (en)', () => { + const html = generateTooltipContent( + tooltipConfig({ data: pt({ power_valid: 0, power_invalid_reasons: reasons }) }), + ); + expect(html).toContain('Measured power withheld'); + expect(html).toContain('sampling gap exceeded'); + expect(html).toContain('expected gpu count mismatch'); + }); + + it('renders the withheld line in Chinese on /zh surfaces', () => { + const html = generateTooltipContent( + tooltipConfig({ + data: pt({ power_valid: 0, power_invalid_reasons: reasons }), + locale: 'zh', + }), + ); + expect(html).toContain('实测功耗未采信'); + expect(html).toContain('sampling gap exceeded'); + }); + + it('filters malformed codes before HTML interpolation (defense in depth)', () => { + const html = generateTooltipContent( + tooltipConfig({ + data: pt({ + power_valid: 0, + power_invalid_reasons: ['', 'sampling_gap_exceeded', 'UPPER'], + }), + }), + ); + expect(html).not.toContain(''); + expect(html).not.toContain('UPPER'); + expect(html).toContain('sampling gap exceeded'); + }); + + it('omits the line entirely when every code is malformed', () => { + const html = generateTooltipContent( + tooltipConfig({ data: pt({ power_valid: 0, power_invalid_reasons: [''] }) }), + ); + expect(html).not.toContain('Measured power withheld'); + }); + + it.each([ + ['absent reasons', pt({ power_valid: 0 })], + ['empty reasons', pt({ power_valid: 0, power_invalid_reasons: [] })], + ['valid row', pt({ power_valid: 1 })], + ])('omits the line for %s', (_name, data) => { + const html = generateTooltipContent(tooltipConfig({ data })); + expect(html).not.toContain('Measured power withheld'); + }); + + it('gives unofficial overlay tooltips the same line', () => { + const html = generateOverlayTooltipContent({ + ...tooltipConfig({ data: pt({ power_valid: 0, power_invalid_reasons: reasons }) }), + overlayData: { + label: 'feature-branch', + hardwareConfig: mockHardwareConfig, + data: [], + runUrl: 'https://example.com', + } as any, + } as OverlayTooltipConfig); + expect(html).toContain('Measured power withheld'); + expect(html).toContain('sampling gap exceeded'); + }); + + it('gives GPU comparison tooltips the same line', () => { + const html = generateGPUGraphTooltipContent( + tooltipConfig({ data: pt({ power_valid: 0, power_invalid_reasons: reasons }) }), + ); + expect(html).toContain('Measured power withheld'); + expect(html).toContain('sampling gap exceeded'); + }); +}); + describe('worker power drilldown', () => { const workers = [ { role: 'frontend', worker_idx: 0, hosts: ['fe0'], num_gpus: 0, avg_power_w: 120 }, diff --git a/packages/app/src/components/inference/utils/tooltipUtils.ts b/packages/app/src/components/inference/utils/tooltipUtils.ts index 4e17da53c..28dd66b82 100644 --- a/packages/app/src/components/inference/utils/tooltipUtils.ts +++ b/packages/app/src/components/inference/utils/tooltipUtils.ts @@ -155,6 +155,7 @@ const TOOLTIP_STRINGS = { powerData: 'Power Measurement', powerCertified: 'Validated (current PowerX method)', powerLegacy: 'Historical (not validated under the current method)', + powerWithheld: 'Measured power withheld', }, zh: { dismiss: '点击其他区域关闭', @@ -173,6 +174,7 @@ const TOOLTIP_STRINGS = { powerData: '功耗测量', powerCertified: '已验证（采用当前 PowerX 方法）', powerLegacy: '历史测量（尚未按当前方法验证）', + powerWithheld: '实测功耗未采信', }, } as const; @@ -643,6 +645,20 @@ const generateParallelismHTML = (d: InferenceData, locale: Locale = 'en'): strin ${tooltipLine(t.dpAttention, d.dp_attention ? t.yes : t.no)}`; }; +const POWER_REASON_CODE_RE = /^[a-z][a-z0-9_]*$/u; + +/** Raw tooltip HTML must not trust producer-supplied reason codes. */ +const powerWithheldHTML = (d: InferenceData, locale: Locale): string => { + if (!Array.isArray(d.power_invalid_reasons) || d.power_invalid_reasons.length === 0) return ''; + const codes = d.power_invalid_reasons + .filter( + (code) => typeof code === 'string' && code.length <= 64 && POWER_REASON_CODE_RE.test(code), + ) + .map((code) => code.replaceAll('_', ' ')); + if (codes.length === 0) return ''; + return tooltipLine(TOOLTIP_STRINGS[locale].powerWithheld, codes.join(', ')); +}; + /** * Generates HTML content for official data point tooltips. * @@ -697,6 +713,7 @@ export const generateTooltipContent = (config: TooltipConfig): string => { ${tooltipLine(t.concurrency, `${d.conc}`)} ${tooltipLine(t.precision, `${d.precision.toUpperCase()}`)} ${generateCacheMetadataHTML(d, locale)} + ${powerWithheldHTML(d, locale)} ${generateAgenticHTML(d, locale)} ${generateWorkerPowerHTML(d, isPinned, locale)} ${runLinkHTML(runUrl, locale)} @@ -740,6 +757,7 @@ export const generateOverlayTooltipContent = (config: OverlayTooltipConfig): str ${tooltipLine(t.concurrency, `${d.conc}`)} ${tooltipLine(t.precision, `${d.precision.toUpperCase()}`)} ${generateCacheMetadataHTML(d, locale)} + ${powerWithheldHTML(d, locale)} ${generateAgenticHTML(d, locale)} ${generateWorkerPowerHTML(d, isPinned, locale)} @@ -800,6 +818,7 @@ export const generateGPUGraphTooltipContent = (config: TooltipConfig): string => ${tooltipLine(t.concurrency, `${d.conc}`)} ${tooltipLine(t.precision, `${d.precision.toUpperCase()}`)} ${generateCacheMetadataHTML(d, locale)} + ${powerWithheldHTML(d, locale)} ${generateAgenticHTML(d, locale)} ${generateWorkerPowerHTML(d, isPinned, locale)} ${runLinkHTML(runUrl, locale)} diff --git a/packages/app/src/components/ui/searchable-select.tsx b/packages/app/src/components/ui/searchable-select.tsx index 54b45723c..ac91199f2 100644 --- a/packages/app/src/components/ui/searchable-select.tsx +++ b/packages/app/src/components/ui/searchable-select.tsx @@ -223,7 +223,7 @@ export function SearchableSelect({ data-testid={triggerTestId} data-slot="select-trigger" data-size={size} - data-value={value} + data-value={mounted ? value : undefined} role="combobox" aria-expanded={isOpen} aria-haspopup={hasOptionHelp ? 'grid' : 'listbox'} diff --git a/packages/app/src/hooks/api/ai-chart-data.test.ts b/packages/app/src/hooks/api/ai-chart-data.test.ts index bfc9702d1..52d6d644d 100644 --- a/packages/app/src/hooks/api/ai-chart-data.test.ts +++ b/packages/app/src/hooks/api/ai-chart-data.test.ts @@ -67,7 +67,7 @@ describe('getAiMetricDirection', () => { }); it('defaults a directionless metric to higher-is-better', () => { - expect(getAiMetricDirection('y_measuredPowerPercentTdp', chartDefinition)).toBe('higher'); + expect(getAiMetricDirection('y_measuredPowerPercentTdp', {})).toBe('higher'); }); }); diff --git a/packages/app/src/lib/api-documentation.power.test.ts b/packages/app/src/lib/api-documentation.power.test.ts index d5d80bdd0..4ecb6ebdb 100644 --- a/packages/app/src/lib/api-documentation.power.test.ts +++ b/packages/app/src/lib/api-documentation.power.test.ts @@ -23,14 +23,18 @@ describe('measured-power API documentation', () => { } }); - it('reserves optional power_invalid_reasons and power_audit row fields', () => { + it('documents optional power_invalid_reasons and power_audit row fields', () => { const reasons = benchmarkRowSchema?.properties?.power_invalid_reasons; expect(reasons?.type).toBe('array'); expect(reasons?.items).toEqual({ type: 'string' }); + expect(reasons?.description).not.toMatch(/reserved|forthcoming/iu); const audit = benchmarkRowSchema?.properties?.power_audit; + expect(audit?.description).not.toMatch(/reserved|forthcoming/iu); expect(Object.keys(audit?.properties ?? {}).toSorted()).toEqual( [ + 'source', + 'observed_gpu_ids', 'window_start_unix', 'window_end_unix', 'expected_gpu_count', @@ -41,7 +45,6 @@ describe('measured-power API documentation', () => { 'exporter_image_sha256', ].toSorted(), ); - // Producers may emit partial audits, so individual audit fields remain optional. expect(audit?.required).toBeUndefined(); expect(benchmarkRowSchema?.required).not.toContain('power_invalid_reasons'); diff --git a/packages/app/src/lib/api-documentation.ts b/packages/app/src/lib/api-documentation.ts index 7aac2f664..d4fc0a99c 100644 --- a/packages/app/src/lib/api-documentation.ts +++ b/packages/app/src/lib/api-documentation.ts @@ -203,6 +203,9 @@ const powerMetricDescriptions: Readonly { ]); }); + it('strips workers and the power audit provenance from the payload-trimmed view', () => { + const [row] = toCalculatorBenchmarkRows(rows, '1k/1k'); + expect(row).not.toHaveProperty('workers'); + expect(row).not.toHaveProperty('power_invalid_reasons'); + expect(row).not.toHaveProperty('power_audit'); + }); + it('keeps all three cache tiers — the trim cannot know which one a row will use', () => { // `measuredCacheHitRate` picks between external and CPU per row, so the allowlist // has to pass all three through or the choice is made for it by the trim. diff --git a/packages/app/src/lib/benchmark-api-view.ts b/packages/app/src/lib/benchmark-api-view.ts index 372db5752..c6ef2a8b4 100644 --- a/packages/app/src/lib/benchmark-api-view.ts +++ b/packages/app/src/lib/benchmark-api-view.ts @@ -31,6 +31,8 @@ interface BenchmarkViewRow { osl: number | null; metrics: Record; workers?: unknown; + power_invalid_reasons?: unknown; + power_audit?: unknown; } /** @@ -45,7 +47,12 @@ export function toCalculatorBenchmarkRows( return rows .filter((row) => rowToSequence(row) === sequence) .map((row) => { - const { workers: _workers, ...rest } = row; + const { + workers: _workers, + power_invalid_reasons: _powerInvalidReasons, + power_audit: _powerAudit, + ...rest + } = row; return { ...rest, metrics: Object.fromEntries( diff --git a/packages/app/src/lib/benchmark-transform.test.ts b/packages/app/src/lib/benchmark-transform.test.ts index 680bf8e7c..401d0e157 100644 --- a/packages/app/src/lib/benchmark-transform.test.ts +++ b/packages/app/src/lib/benchmark-transform.test.ts @@ -301,6 +301,25 @@ describe('rowToAggDataEntry', () => { expect(entry.joules_per_output_token).toBe(8.4); }); + it('passes through producer power_invalid_reasons on withheld rows', () => { + const entry = rowToAggDataEntry( + makeRow({ + metrics: { power_valid: 0 }, + power_invalid_reasons: ['thermal_throttle', 'sample_gap'], + }), + ); + expect(entry.power_invalid_reasons).toEqual(['thermal_throttle', 'sample_gap']); + }); + + it.each([ + ['legacy row without the field', {}], + ['API null (SQL NULL column)', { power_invalid_reasons: null }], + ['empty array', { power_invalid_reasons: [] }], + ])('leaves power_invalid_reasons undefined for %s', (_name, overrides) => { + const entry = rowToAggDataEntry(makeRow({ metrics: {}, ...overrides })); + expect(entry.power_invalid_reasons).toBeUndefined(); + }); + it('passes through versioned whole-deployment joules per successful query', () => { const entry = rowToAggDataEntry( makeRow({ diff --git a/packages/app/src/lib/benchmark-transform.ts b/packages/app/src/lib/benchmark-transform.ts index cda0ffc5b..cb01c31e8 100644 --- a/packages/app/src/lib/benchmark-transform.ts +++ b/packages/app/src/lib/benchmark-transform.ts @@ -225,6 +225,12 @@ export function rowToAggDataEntry(row: BenchmarkRow): AggDataEntry { // rows predating the field so downstream chart code can distinguish // "no measurement" from "0 W" via createChartDataPoint's typeof guard. power_valid: m.power_valid, + power_audit: row.power_audit ?? undefined, + // SQL NULL and omitted legacy fields both mean no diagnostic was supplied. + power_invalid_reasons: + Array.isArray(row.power_invalid_reasons) && row.power_invalid_reasons.length > 0 + ? row.power_invalid_reasons + : undefined, power_metric_schema_version: m.power_metric_schema_version, modeledSystemPower: modelSystemPower(row), power_tier: resolvePowerTier({ diff --git a/packages/constants/src/metric-keys.test.ts b/packages/constants/src/metric-keys.test.ts index 277dd9769..a524a87b4 100644 --- a/packages/constants/src/metric-keys.test.ts +++ b/packages/constants/src/metric-keys.test.ts @@ -18,6 +18,8 @@ describe('MEASURED_POWER_METRIC_KEYS', () => { expect(new Set(MEASURED_POWER_METRIC_KEY_LIST)).toEqual( new Set([ 'avg_power_w', + 'avg_total_gpu_power_w', + 'total_gpu_energy_j', 'p75_power_w', 'p75_total_gpu_power_w', 'p90_power_w', @@ -36,7 +38,7 @@ describe('MEASURED_POWER_METRIC_KEYS', () => { 'avg_mem_used_mb', ]), ); - expect(MEASURED_POWER_METRIC_KEYS.size).toBe(17); + expect(MEASURED_POWER_METRIC_KEYS.size).toBe(19); }); it('never contains the contract discriminators or invalid-verdict companion fields', () => { @@ -69,6 +71,6 @@ describe('POWER_METRIC_KEYS', () => { expect(new Set(POWER_METRIC_KEYS)).toEqual( new Set(['power_valid', 'power_metric_schema_version', ...MEASURED_POWER_METRIC_KEY_LIST]), ); - expect(POWER_METRIC_KEYS).toHaveLength(19); + expect(POWER_METRIC_KEYS).toHaveLength(21); }); }); diff --git a/packages/constants/src/metric-keys.ts b/packages/constants/src/metric-keys.ts index d69667dd6..3c983ccb3 100644 --- a/packages/constants/src/metric-keys.ts +++ b/packages/constants/src/metric-keys.ts @@ -14,6 +14,8 @@ export const MEASURED_POWER_METRIC_KEY_LIST = [ // — cluster-wide; workload-shape-fair view that // doesn't treat prompt as free. 'avg_power_w', + 'avg_total_gpu_power_w', + 'total_gpu_energy_j', // Time-weighted percentiles of synchronized fleet draw; per-chip divides by GPU count. 'p75_power_w', 'p75_total_gpu_power_w', diff --git a/packages/db/migrations/015_power_provenance.sql b/packages/db/migrations/015_power_provenance.sql new file mode 100644 index 000000000..60f45ce69 --- /dev/null +++ b/packages/db/migrations/015_power_provenance.sql @@ -0,0 +1,56 @@ +-- Power audit provenance lives in dedicated JSONB columns so `metrics` stays +-- a flat Record. NULL does not establish age or validity. + +-- Earlier previews may already have the audit columns under the old filename. +alter table benchmark_results + add column if not exists power_invalid_reasons jsonb; + +alter table benchmark_results + add column if not exists power_audit jsonb; + +-- Re-create the view so `br.*` includes the audit columns while preserving +-- migration 014's shared AgentX curve scope and append-only snapshot semantics. + +drop materialized view latest_benchmarks; +create materialized view latest_benchmarks as +with recursive ranked_runs as ( + select benchmark_curve_runs.*, + row_number() over ( + partition by curve_scope + order by date desc, run_started_at desc nulls last, workflow_run_id desc + ) as run_rank + from benchmark_curve_runs +), curve_runs as ( + select ranked_runs.*, image as root_image, + date as snapshot_date, workflow_run_id as snapshot_workflow_run_id + from ranked_runs where run_rank = 1 + + union all + + select older.*, current.root_image, current.snapshot_date, current.snapshot_workflow_run_id + from curve_runs current + join ranked_runs older + on older.curve_scope = current.curve_scope + and older.run_rank = current.run_rank + 1 + where current.append_only + and current.image_count = 1 and current.images_complete + and older.image_count = 1 and older.images_complete + and older.image = current.root_image +) +select distinct on ( + br.config_id, br.benchmark_type, br.isl, br.osl, br.offload_mode, br.recipe_fingerprint, br.conc +) + br.*, cr.snapshot_date, cr.snapshot_workflow_run_id +from curve_runs cr +join benchmark_results br on br.workflow_run_id = cr.workflow_run_id +join configs c on c.id = br.config_id + and benchmark_curve_scope(c.model, c.hardware, c.framework, c.precision, + br.benchmark_type, br.isl, br.osl, c.spec_method, c.disagg, br.offload_mode) = cr.curve_scope +where br.error is null +order by br.config_id, br.benchmark_type, br.isl, br.osl, br.offload_mode, + br.recipe_fingerprint, br.conc, cr.run_rank; + +create unique index latest_benchmarks_pk + on latest_benchmarks (config_id, conc, isl, osl, benchmark_type, offload_mode, recipe_fingerprint) + nulls not distinct; +create index latest_benchmarks_model_idx on latest_benchmarks (config_id); diff --git a/packages/db/src/etl/benchmark-ingest.test.ts b/packages/db/src/etl/benchmark-ingest.test.ts index d1a1db6a3..e97792ba6 100644 --- a/packages/db/src/etl/benchmark-ingest.test.ts +++ b/packages/db/src/etl/benchmark-ingest.test.ts @@ -7,8 +7,10 @@ import { describe, expect, it, vi } from 'vitest'; import type { Sql } from './db-utils'; import { benchmarkPointIngestKey, + bulkIngestBenchmarkRows, insertServerLogFilePaths, insertServerLogFiles, + type BenchmarkPersistenceInput, } from './benchmark-ingest'; const point = (recipeFingerprint: string | null) => ({ @@ -55,6 +57,77 @@ function fakeTransactionSql(linkedId: number | null) { return { sql: tag as Sql, calls }; } +function captureInsertSql() { + const calls: { text: string; values: unknown[] }[] = []; + const tag = vi.fn((strings: TemplateStringsArray, ...values: unknown[]) => { + calls.push({ text: strings.join('?').replaceAll(/\s+/gu, ' ').trim(), values }); + return Promise.resolve([]); + }) as any; + tag.array = (value: unknown) => value; + return { sql: tag as Sql, calls }; +} + +describe('bulkIngestBenchmarkRows — power audit provenance lanes', () => { + const provenancedRow: BenchmarkPersistenceInput = { + configId: 7, + benchmarkType: 'single_turn', + isl: 1024, + osl: 1024, + conc: 64, + offloadMode: 'off', + image: 'img', + recipeFingerprint: null, + metrics: { power_valid: 0 }, + powerInvalidReasons: ['sampling_gap_exceeded'], + powerAudit: { + window_start_unix: 1756174800, + window_end_unix: 1756175400, + expected_gpu_count: 8, + observed_gpu_count: 8, + sample_count: 4800, + max_sample_gap_s: 1.013, + producer_sha: null, + exporter_image_sha256: null, + }, + }; + const legacyRow: BenchmarkPersistenceInput = { + configId: 8, + benchmarkType: 'single_turn', + isl: 1024, + osl: 1024, + conc: 128, + offloadMode: 'off', + image: 'img', + recipeFingerprint: null, + metrics: { tput_per_gpu: 100 }, + }; + + it('names both columns, adds two jsonb lanes, and refreshes both on conflict', async () => { + const { sql, calls } = captureInsertSql(); + await bulkIngestBenchmarkRows(sql, [provenancedRow, legacyRow], 42, '2026-08-27'); + + const { text } = calls[0]; + expect(text).toContain('metrics, workers, power_invalid_reasons, power_audit )'); + expect(text.match(/::jsonb\[\]/gu)).toHaveLength(4); + expect(text).toContain('power_invalid_reasons = excluded.power_invalid_reasons'); + expect(text).toContain('power_audit = excluded.power_audit'); + }); + + it('serializes present fields and contributes null lanes for absent ones', async () => { + const { sql, calls } = captureInsertSql(); + await bulkIngestBenchmarkRows(sql, [provenancedRow, legacyRow], 42, '2026-08-27'); + + const { values } = calls[0]; + expect(values[10]).toEqual([ + JSON.stringify(provenancedRow.metrics), + JSON.stringify(legacyRow.metrics), + ]); + expect(values[11]).toEqual([null, null]); + expect(values[12]).toEqual([JSON.stringify(provenancedRow.powerInvalidReasons), null]); + expect(values[13]).toEqual([JSON.stringify(provenancedRow.powerAudit), null]); + }); +}); + describe('insertServerLogFiles', () => { const files = [ { fileName: 'results/benchmark.log', logText: 'benchmark' }, diff --git a/packages/db/src/etl/benchmark-ingest.ts b/packages/db/src/etl/benchmark-ingest.ts index 617c51efe..122965e08 100644 --- a/packages/db/src/etl/benchmark-ingest.ts +++ b/packages/db/src/etl/benchmark-ingest.ts @@ -7,7 +7,7 @@ import path from 'node:path'; import type postgres from 'postgres'; import { cleanLogText, type ServerLogFile, type ServerLogFilePath } from './server-log-artifacts'; -import type { BenchmarkType, WorkerPower } from './benchmark-mapper'; +import type { BenchmarkType, PowerAudit, WorkerPower } from './benchmark-mapper'; import { kvCachePoolTokensFromServerLog } from './server-log-metrics'; type Sql = ReturnType; @@ -23,6 +23,8 @@ export interface BenchmarkPersistenceInput { recipeFingerprint: string | null; metrics: Record; workers?: WorkerPower[]; + powerInvalidReasons?: string[]; + powerAudit?: PowerAudit; } type BenchmarkPointIdentity = Pick< @@ -82,17 +84,23 @@ export async function bulkIngestBenchmarkRows( const images = deduped.map((r) => r.image); const recipeFingerprints = deduped.map((r) => r.recipeFingerprint); const metricsJsons = deduped.map((r) => JSON.stringify(r.metrics)); - // workers is optional — encode missing values as JSON null so the JSONB - // unnest input has a homogeneous type (jsonb[]) and stores SQL NULL in the - // column for rows that didn't emit a per-worker breakdown. + // Optional JSONB lanes use JSON null so each jsonb[] input remains + // homogeneous and missing payloads persist as SQL NULL. const workersJsons = deduped.map((r) => r.workers === undefined ? null : JSON.stringify(r.workers), ); + const powerInvalidReasonsJsons = deduped.map((r) => + r.powerInvalidReasons === undefined ? null : JSON.stringify(r.powerInvalidReasons), + ); + const powerAuditJsons = deduped.map((r) => + r.powerAudit === undefined ? null : JSON.stringify(r.powerAudit), + ); const result = await sql<{ inserted: boolean; id: number }[]>` insert into benchmark_results ( workflow_run_id, config_id, benchmark_type, offload_mode, date, - isl, osl, conc, image, recipe_fingerprint, metrics, workers + isl, osl, conc, image, recipe_fingerprint, metrics, workers, + power_invalid_reasons, power_audit ) select ${workflowRunId}, @@ -106,7 +114,9 @@ export async function bulkIngestBenchmarkRows( unnest(${sql.array(images)}), unnest(${sql.array(recipeFingerprints)}), unnest(${sql.array(metricsJsons)}::jsonb[]), - unnest(${sql.array(workersJsons)}::jsonb[]) + unnest(${sql.array(workersJsons)}::jsonb[]), + unnest(${sql.array(powerInvalidReasonsJsons)}::jsonb[]), + unnest(${sql.array(powerAuditJsons)}::jsonb[]) on conflict ( workflow_run_id, config_id, benchmark_type, isl, osl, conc, offload_mode, recipe_fingerprint @@ -120,7 +130,11 @@ export async function bulkIngestBenchmarkRows( jsonb_build_object('kv_cache_pool_tokens', benchmark_results.metrics->'kv_cache_pool_tokens') ), image = excluded.image, - workers = excluded.workers + workers = excluded.workers, + -- Like workers, the fresh artifact is authoritative for provenance: a + -- re-ingest from an artifact without the fields deliberately nulls them. + power_invalid_reasons = excluded.power_invalid_reasons, + power_audit = excluded.power_audit returning (xmax = 0) as inserted, id `; diff --git a/packages/db/src/etl/benchmark-mapper.test.ts b/packages/db/src/etl/benchmark-mapper.test.ts index 5fa8287bc..962f5de16 100644 --- a/packages/db/src/etl/benchmark-mapper.test.ts +++ b/packages/db/src/etl/benchmark-mapper.test.ts @@ -1,6 +1,8 @@ -import { describe, it, expect } from 'vitest'; +import { describe, it, expect, vi } from 'vitest'; import { MEASURED_POWER_METRIC_KEYS } from '@semianalysisai/inferencex-constants'; import { + extractPowerAudit, + extractPowerInvalidReasons, extractWorkers, mapBenchmarkRow, normalizePowerContractMetrics, @@ -439,6 +441,13 @@ describe('mapBenchmarkRow', () => { } expect(result!.metrics).not.toHaveProperty('power_invalid_reasons'); expect(result!.metrics).not.toHaveProperty('power_audit'); + expect(result!.powerInvalidReasons).toEqual(['window_too_short']); + expect(result!.powerAudit).toEqual({ + window_start_unix: 1, + window_end_unix: 2, + producer_sha: null, + exporter_image_sha256: null, + }); }); }); @@ -892,6 +901,30 @@ describe('scrubWithheldPowerMetrics (direct — supplemental ingest path)', () = expect(metrics).not.toHaveProperty(key); } }); + + it('recovers provenance companions nested under metrics and leaves the record flat', () => { + const metrics = supplementalMetrics({ + power_valid: 0, + power_invalid_reasons: ['sampling_gap_exceeded', 'sampling_gap_exceeded', ''], + power_audit: { sample_count: 12, producer_sha: 'abc123', unknown_key: true }, + }); + normalizePowerContractMetrics(metrics, metrics); + scrubWithheldPowerMetrics(metrics); + const reasons = extractPowerInvalidReasons(metrics.power_invalid_reasons); + const audit = extractPowerAudit(metrics.power_audit); + delete metrics.power_invalid_reasons; + delete metrics.power_audit; + + expect(reasons).toEqual(['sampling_gap_exceeded']); + expect(audit).toEqual({ + sample_count: 12, + producer_sha: 'abc123', + exporter_image_sha256: null, + }); + expect(metrics).not.toHaveProperty('power_invalid_reasons'); + expect(metrics).not.toHaveProperty('power_audit'); + expect(metrics.tput_per_gpu).toBe(567.8); + }); }); describe('extractWorkers', () => { @@ -978,6 +1011,236 @@ describe('extractWorkers', () => { }); }); +describe('extractPowerInvalidReasons', () => { + it('keeps valid snake_case codes in first-seen order', () => { + expect( + extractPowerInvalidReasons(['sampling_gap_exceeded', 'expected_gpu_count_mismatch']), + ).toEqual(['sampling_gap_exceeded', 'expected_gpu_count_mismatch']); + }); + + it('deduplicates preserving first-seen order', () => { + expect( + extractPowerInvalidReasons([ + 'telemetry_file_missing', + 'no_usable_power_samples', + 'telemetry_file_missing', + ]), + ).toEqual(['telemetry_file_missing', 'no_usable_power_samples']); + }); + + it.each([ + ['non-string entry', [42]], + ['empty string', ['']], + ['hyphenated code', ['Bad-Reason']], + ['uppercase code', ['UPPER']], + ['leading digit', ['9lives']], + ['65-char code', ['a'.repeat(65)]], + ])('silently drops %s', (_name, raw) => { + expect(extractPowerInvalidReasons(raw)).toBeUndefined(); + }); + + it('drops malformed entries while keeping valid siblings', () => { + expect(extractPowerInvalidReasons([42, 'sampling_gap_exceeded', '', null])).toEqual([ + 'sampling_gap_exceeded', + ]); + }); + + it('keeps a 64-char code (boundary)', () => { + const code = 'a'.repeat(64); + expect(extractPowerInvalidReasons([code])).toEqual([code]); + }); + + it('caps the result at 32 codes', () => { + const raw = Array.from({ length: 40 }, (_v, i) => `reason_${i}`); + const result = extractPowerInvalidReasons(raw); + expect(result).toHaveLength(32); + expect(result![0]).toBe('reason_0'); + expect(result![31]).toBe('reason_31'); + }); + + it.each([ + ['empty array', []], + ['non-array object', { reason: 'x' }], + ['string', 'sampling_gap_exceeded'], + ['null', null], + ['undefined', undefined], + ])('returns undefined (never []) for %s', (_name, raw) => { + expect(extractPowerInvalidReasons(raw)).toBeUndefined(); + }); +}); + +describe('extractPowerAudit', () => { + const fullAudit = { + window_start_unix: 1756174800.25, + window_end_unix: 1756175400.75, + expected_gpu_count: 16, + observed_gpu_count: 16, + sample_count: 9600, + max_sample_gap_s: 1.013, + producer_sha: '887a6cb7c2ec174e5e2b977468a12ab34cd56ef7', + exporter_image_sha256: + 'sha256:0b7f1a2c3d4e5f60718293a4b5c6d7e8f9012a3b4c5d6e7f8091a2b3c4d5e6f7', + }; + + it('round-trips a full valid object across all 8 fields', () => { + expect(extractPowerAudit(fullAudit)).toEqual(fullAudit); + }); + + it('omits Infinity / NaN / junk-string numerics (partial audit beats none)', () => { + expect( + extractPowerAudit({ + ...fullAudit, + window_start_unix: Number.POSITIVE_INFINITY, + window_end_unix: Number.NaN, + max_sample_gap_s: 'garbage', + }), + ).toEqual({ + expected_gpu_count: 16, + observed_gpu_count: 16, + sample_count: 9600, + producer_sha: fullAudit.producer_sha, + exporter_image_sha256: fullAudit.exporter_image_sha256, + }); + }); + + it('rejects negative and non-safe-integer counts', () => { + expect( + extractPowerAudit({ + ...fullAudit, + expected_gpu_count: -1, + observed_gpu_count: Number.MAX_SAFE_INTEGER + 1, + sample_count: Number.POSITIVE_INFINITY, + }), + ).toEqual({ + window_start_unix: fullAudit.window_start_unix, + window_end_unix: fullAudit.window_end_unix, + max_sample_gap_s: fullAudit.max_sample_gap_s, + producer_sha: fullAudit.producer_sha, + exporter_image_sha256: fullAudit.exporter_image_sha256, + }); + }); + + it('keeps shas trimmed and collapses null / number / oversized shas to null', () => { + expect( + extractPowerAudit({ + sample_count: 1, + producer_sha: ' abc123 ', + exporter_image_sha256: null, + }), + ).toEqual({ sample_count: 1, producer_sha: 'abc123', exporter_image_sha256: null }); + expect( + extractPowerAudit({ + sample_count: 1, + producer_sha: 42, + exporter_image_sha256: 'x'.repeat(129), + }), + ).toEqual({ sample_count: 1, producer_sha: null, exporter_image_sha256: null }); + }); + + it('nulls the explicit-null numeric fields a producer emits without a benchmark window', () => { + expect( + extractPowerAudit({ + window_start_unix: null, + window_end_unix: null, + expected_gpu_count: 8, + observed_gpu_count: 0, + sample_count: 0, + max_sample_gap_s: null, + producer_sha: null, + exporter_image_sha256: null, + }), + ).toEqual({ + expected_gpu_count: 8, + observed_gpu_count: 0, + sample_count: 0, + producer_sha: null, + exporter_image_sha256: null, + }); + }); + + it('drops unknown keys (fixed 8-key shape bounds the stored object)', () => { + expect(extractPowerAudit({ sample_count: 3, integration_method: 'trapezoid' })).toEqual({ + sample_count: 3, + producer_sha: null, + exporter_image_sha256: null, + }); + }); + + it.each([ + ['string', 'audit'], + ['array', [1, 2]], + ['null', null], + ['undefined', undefined], + ['number', 7], + ])('returns undefined for non-object input: %s', (_name, raw) => { + expect(extractPowerAudit(raw)).toBeUndefined(); + }); + + it('returns undefined for an empty husk (no numerics, both shas null)', () => { + expect(extractPowerAudit({})).toBeUndefined(); + expect(extractPowerAudit({ producer_sha: null, exporter_image_sha256: 42 })).toBeUndefined(); + expect(extractPowerAudit({ window_start_unix: 'junk' })).toBeUndefined(); + }); +}); + +describe('mapBenchmarkRow — power audit provenance', () => { + const reasons = ['sampling_gap_exceeded', 'expected_gpu_count_mismatch']; + const audit = { + window_start_unix: 1756174800, + window_end_unix: 1756175400, + expected_gpu_count: 8, + observed_gpu_count: 8, + sample_count: 4800, + max_sample_gap_s: 1.013, + producer_sha: null, + exporter_image_sha256: null, + }; + + it.each([ + ['v1', makeV1Row], + ['v2', makeV2Row], + ['agentic', makeAgenticRow], + ])('lands the contract fields on BenchmarkParams for %s rows', (_name, makeRow) => { + const tracker = createSkipTracker(); + const result = mapBenchmarkRow( + makeRow({ power_valid: 0, power_invalid_reasons: reasons, power_audit: audit }), + tracker, + ); + expect(result!.powerInvalidReasons).toEqual(reasons); + expect(result!.powerAudit).toEqual(audit); + }); + + it('stores provenance from valid rows too (tolerance in both directions)', () => { + const tracker = createSkipTracker(); + const result = mapBenchmarkRow(makeV2Row({ power_valid: 1, power_audit: audit }), tracker); + expect(result!.metrics.power_valid).toBe(1); + expect(result!.powerAudit).toEqual(audit); + expect(result!.powerInvalidReasons).toBeUndefined(); + }); + + it('leaves both fields undefined on legacy rows', () => { + const tracker = createSkipTracker(); + const result = mapBenchmarkRow(makeV2Row(), tracker); + expect(result!.powerInvalidReasons).toBeUndefined(); + expect(result!.powerAudit).toBeUndefined(); + }); + + it("never captures a malformed ['5'] reasons array as a numeric metric", () => { + const tracker = createSkipTracker(); + const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}); + try { + const result = mapBenchmarkRow(makeV2Row({ power_invalid_reasons: ['5'] }), tracker); + expect(result!.metrics).not.toHaveProperty('power_invalid_reasons'); + expect(result!.powerInvalidReasons).toBeUndefined(); + expect(warn.mock.calls.map((call) => String(call[0]))).not.toContainEqual( + expect.stringContaining('power_invalid_reasons'), + ); + } finally { + warn.mockRestore(); + } + }); +}); + describe('mapBenchmarkRow — agentic interactivity normalization', () => { it('derives *_intvty from 1/*_itl, discarding the artifact value', () => { const tracker = createSkipTracker(); diff --git a/packages/db/src/etl/benchmark-mapper.ts b/packages/db/src/etl/benchmark-mapper.ts index 2dccff9bb..060b39412 100644 --- a/packages/db/src/etl/benchmark-mapper.ts +++ b/packages/db/src/etl/benchmark-mapper.ts @@ -66,6 +66,7 @@ const NON_METRIC_KEYS = new Set([ 'num_gpus', 'num_prefill_gpu', 'num_decode_gpu', + 'num_aggregate_gpu', // agentic scenario 'scenario_type', 'users', @@ -89,6 +90,11 @@ const NON_METRIC_KEYS = new Set([ // sibling of the metrics JSONB by mapBenchmarkRow so the metrics column // stays Record for the index signature on BenchmarkRow. 'workers', + // Keep structured provenance outside flat numeric metrics. Explicitly + // excluding the keys also blocks Number(['5']) coercion. + 'power_invalid_reasons', + 'power_audit', + 'benchmark_outcome', ]); /** @@ -129,6 +135,25 @@ export interface WorkerPower { avg_mem_used_mb?: number; } +/** + * Narrowed measurement-window audit. Fields are optional because malformed + * numerics are omitted; producer identity is null when unavailable. + */ +export interface PowerAudit { + window_start_unix?: number; + window_end_unix?: number; + expected_gpu_count?: number; + observed_gpu_count?: number; + sample_count?: number; + max_sample_gap_s?: number; + producer_sha?: string | null; + exporter_image_sha256?: string | null; + /** Relative path within the source run artifact bundle. */ + source?: string; + /** Producer device identifiers; not necessarily physical UUIDs on older traces. */ + observed_gpu_ids?: string[]; +} + export interface BenchmarkParams { config: ConfigParams; benchmarkType: BenchmarkType; @@ -151,6 +176,8 @@ export interface BenchmarkParams { * predating the multinode patch. */ workers?: WorkerPower[]; + powerInvalidReasons?: string[]; + powerAudit?: PowerAudit; } /** @@ -181,6 +208,11 @@ export function mapBenchmarkRow( // first so the rest of the mapper (auto-capture, intvty invariant, guards) // is version-agnostic. No-op for v1/v2 rows. row = normalizeLegacyTpuRow(flattenAgenticAggRow(row), runId); + // Failed-client JSON is retained as evidence, never as a performance point. + if (row.benchmark_outcome?.status === 'failed') { + tracker.skips.failedRun++; + return null; + } const modelKey = resolveModelKey(row); if (!modelKey) { @@ -361,6 +393,9 @@ export function mapBenchmarkRow( // narrowing — anything other than a non-empty array of objects is dropped, // and a withheld power verdict drops the payload entirely. const workers = powerWithheld ? undefined : extractWorkers(row.workers); + // Audit metadata is independent of the verdict so valid-row provenance is retained. + const powerInvalidReasons = extractPowerInvalidReasons(row.power_invalid_reasons); + const powerAudit = extractPowerAudit(row.power_audit); return { config: { @@ -382,6 +417,8 @@ export function mapBenchmarkRow( recipeFingerprint, metrics, workers, + powerInvalidReasons, + powerAudit, }; } @@ -587,3 +624,89 @@ export function extractWorkers(raw: unknown): WorkerPower[] | undefined { } return out.length > 0 ? out : undefined; } + +const POWER_REASON_CODE_RE = /^[a-z][a-z0-9_]*$/u; +const MAX_POWER_REASON_CODES = 32; +const MAX_POWER_REASON_LENGTH = 64; +const MAX_POWER_AUDIT_SHA_LENGTH = 128; + +/** + * Empty results become undefined so persistence stores SQL NULL, not `[]`. + */ +export function extractPowerInvalidReasons(raw: unknown): string[] | undefined { + if (!Array.isArray(raw)) return undefined; + const out: string[] = []; + const seen = new Set(); + for (const entry of raw) { + if (typeof entry !== 'string') continue; + const code = entry.trim(); + if (code.length === 0 || code.length > MAX_POWER_REASON_LENGTH) continue; + if (!POWER_REASON_CODE_RE.test(code) || seen.has(code)) continue; + seen.add(code); + out.push(code); + if (out.length >= MAX_POWER_REASON_CODES) break; + } + return out.length > 0 ? out : undefined; +} + +/** parseNum accepts Infinity, which cannot describe a measurement window. */ +function auditFiniteNum(v: unknown): number | undefined { + const n = parseNum(v); + return n !== undefined && Number.isFinite(n) ? n : undefined; +} + +function auditCount(v: unknown): number | undefined { + const n = parseInt2(v); + return n !== undefined && Number.isSafeInteger(n) && n >= 0 ? n : undefined; +} + +function auditSha(v: unknown): string | null { + if (typeof v !== 'string') return null; + const s = v.trim(); + return s.length > 0 && s.length <= MAX_POWER_AUDIT_SHA_LENGTH ? s : null; +} + +/** + * Missing or malformed audit values must not become a fabricated measurement; + * SQL NULL distinguishes absent evidence from an empty recorded object. + */ +export function extractPowerAudit(raw: unknown): PowerAudit | undefined { + if (!raw || typeof raw !== 'object' || Array.isArray(raw)) return undefined; + const e = raw as Record; + const audit: PowerAudit = {}; + + const window_start_unix = auditFiniteNum(e.window_start_unix); + if (window_start_unix !== undefined) audit.window_start_unix = window_start_unix; + const window_end_unix = auditFiniteNum(e.window_end_unix); + if (window_end_unix !== undefined) audit.window_end_unix = window_end_unix; + const max_sample_gap_s = auditFiniteNum(e.max_sample_gap_s); + if (max_sample_gap_s !== undefined) audit.max_sample_gap_s = max_sample_gap_s; + const expected_gpu_count = auditCount(e.expected_gpu_count); + if (expected_gpu_count !== undefined) audit.expected_gpu_count = expected_gpu_count; + const observed_gpu_count = auditCount(e.observed_gpu_count); + if (observed_gpu_count !== undefined) audit.observed_gpu_count = observed_gpu_count; + const sample_count = auditCount(e.sample_count); + if (sample_count !== undefined) audit.sample_count = sample_count; + if ( + typeof e.source === 'string' && + e.source.length <= 512 && + /^[a-zA-Z0-9_./-]+$/u.test(e.source) && + !e.source.startsWith('/') && + !e.source.split('/').includes('..') + ) + audit.source = e.source; + if (Array.isArray(e.observed_gpu_ids)) { + const ids = e.observed_gpu_ids.filter( + (id): id is string => typeof id === 'string' && id.length > 0 && id.length <= 128, + ); + if (ids.length > 0) audit.observed_gpu_ids = [...new Set(ids)].slice(0, 1024); + } + const hasNumericField = Object.keys(audit).length > 0; + + audit.producer_sha = auditSha(e.producer_sha); + audit.exporter_image_sha256 = auditSha(e.exporter_image_sha256); + if (!hasNumericField && audit.producer_sha === null && audit.exporter_image_sha256 === null) { + return undefined; + } + return audit; +} diff --git a/packages/db/src/etl/fixtures/power-processor/README.md b/packages/db/src/etl/fixtures/power-processor/README.md new file mode 100644 index 000000000..94145d906 --- /dev/null +++ b/packages/db/src/etl/fixtures/power-processor/README.md @@ -0,0 +1,7 @@ +Synthetic local contract fixtures, not measured benchmark observations. + +Generated by the actual InferenceX processor CLI at local producer commit +`cdb071af9`, from two devices with synthetic samples and 100 completed requests. +The paired missing-power case omits telemetry while retaining valid performance. +These files exercise producer → mapper → API shape → chart transformation. +They must never be ingested into a production benchmark database. diff --git a/packages/db/src/etl/fixtures/power-processor/missing-power.json b/packages/db/src/etl/fixtures/power-processor/missing-power.json new file mode 100644 index 000000000..958098b6b --- /dev/null +++ b/packages/db/src/etl/fixtures/power-processor/missing-power.json @@ -0,0 +1,47 @@ +{ + "hw": "b200", + "conc": 4, + "image": "synthetic-local-test-fixture", + "model": "Qwen/Qwen3.5-397B-A17B", + "infmax_model_prefix": "qwen3.5", + "framework": "sglang", + "precision": "fp8", + "spec_decoding": "none", + "disagg": false, + "recipe_fingerprint": "", + "isl": 8192, + "osl": 1024, + "benchmark_outcome": { + "status": "passed", + "requested": 100, + "completed": 100, + "failed": 0, + "max_failure_rate": 0.05 + }, + "is_multinode": false, + "tp": 2, + "pp": 1, + "dcp_size": 1, + "pcp_size": 1, + "ep": 1, + "dp_attention": "false", + "tput_per_gpu": 250.0, + "output_tput_per_gpu": 50.0, + "input_tput_per_gpu": 200.0, + "mean_ttft": 0.1, + "median_ttft": 0.1, + "mean_tpot": 0.02, + "mean_intvty": 50.0, + "median_tpot": 0.02, + "median_intvty": 50.0, + "power_metric_schema_version": 2, + "power_valid": 0, + "power_invalid_reasons": ["telemetry_file_missing"], + "power_audit": { + "source": "power_validation_benchmark_result.json", + "window_start_unix": 1700000100.0, + "window_end_unix": 1700000110.0, + "expected_gpu_count": 2, + "observed_gpu_count": 0 + } +} diff --git a/packages/db/src/etl/fixtures/power-processor/valid-power.json b/packages/db/src/etl/fixtures/power-processor/valid-power.json new file mode 100644 index 000000000..fc33d33d7 --- /dev/null +++ b/packages/db/src/etl/fixtures/power-processor/valid-power.json @@ -0,0 +1,61 @@ +{ + "hw": "b200", + "conc": 4, + "image": "synthetic-local-test-fixture", + "model": "Qwen/Qwen3.5-397B-A17B", + "infmax_model_prefix": "qwen3.5", + "framework": "sglang", + "precision": "fp8", + "spec_decoding": "none", + "disagg": false, + "recipe_fingerprint": "", + "isl": 8192, + "osl": 1024, + "benchmark_outcome": { + "status": "passed", + "requested": 100, + "completed": 100, + "failed": 0, + "max_failure_rate": 0.05 + }, + "is_multinode": false, + "tp": 2, + "pp": 1, + "dcp_size": 1, + "pcp_size": 1, + "ep": 1, + "dp_attention": "false", + "tput_per_gpu": 250.0, + "output_tput_per_gpu": 50.0, + "input_tput_per_gpu": 200.0, + "mean_ttft": 0.1, + "median_ttft": 0.1, + "mean_tpot": 0.02, + "mean_intvty": 50.0, + "median_tpot": 0.02, + "median_intvty": 50.0, + "power_metric_schema_version": 2, + "power_valid": 1, + "avg_power_w": 500.0, + "p75_power_w": 500.0, + "p75_total_gpu_power_w": 1000.0, + "p90_power_w": 500.0, + "p90_total_gpu_power_w": 1000.0, + "avg_total_gpu_power_w": 1000.0, + "total_gpu_energy_j": 10000.0, + "joules_per_successful_query": 100.0, + "joules_per_input_token": 0.012207, + "joules_per_output_token": 0.097656, + "joules_per_total_token": 0.010851, + "power_invalid_reasons": [], + "power_audit": { + "source": "power_validation_benchmark_result.json", + "window_start_unix": 1700000100.0, + "window_end_unix": 1700000110.0, + "expected_gpu_count": 2, + "observed_gpu_count": 2, + "sample_count": 34, + "max_sample_gap_s": 1.0, + "observed_gpu_ids": ["0", "1"] + } +} diff --git a/packages/db/src/etl/power-publication.test.ts b/packages/db/src/etl/power-publication.test.ts new file mode 100644 index 000000000..75e5eb997 --- /dev/null +++ b/packages/db/src/etl/power-publication.test.ts @@ -0,0 +1,111 @@ +import { describe, expect, it } from 'vitest'; +import { mapBenchmarkRow } from './benchmark-mapper'; +import { createSkipTracker } from './skip-tracker'; +import { + powerPublicationPoint, + verifyPowerPublication, + type PublishedPowerRow, +} from './power-publication'; + +const raw = { + infmax_model_prefix: 'qwen3.5', + hw: 'b200', + framework: 'sglang', + precision: 'fp8', + isl: 8192, + osl: 1024, + conc: 32, + tp: 4, + ep: 1, + dp_attention: false, + avg_power_w: 642, + power_valid: 1, + power_metric_schema_version: 2, + joules_per_successful_query: 1200, + power_audit: { + window_start_unix: 100, + window_end_unix: 120, + observed_gpu_count: 4, + source: 'power_validation.json', + }, +}; +function expected(overrides = {}) { + const row = mapBenchmarkRow({ ...raw, ...overrides }, createSkipTracker()); + if (!row) throw new Error('Fixture must map'); + const point = powerPublicationPoint( + row, + 'https://github.com/SemiAnalysisAI/InferenceX/actions/runs/123/attempts/2', + { path: 'bmk_qwen3.5/results.json', sha256: 'abc' }, + ); + if (!point) throw new Error('Fixture must be 8K/1K'); + return point; +} +function actual(point = expected()): PublishedPowerRow { + return { + ...point.identity, + metrics: { ...point.metrics, tput_per_gpu: 400 }, + workers: point.workers, + power_invalid_reasons: point.power_invalid_reasons, + power_audit: point.power_audit, + }; +} + +describe('PowerX publication', () => { + it('matches the exact ordinary source and attempt through mapping, while ignoring unrelated performance metrics', () => { + const point = expected(); + expect(verifyPowerPublication([point], [actual(point)], 'API')).toEqual([]); + expect(point.power_audit).toMatchObject({ + source: 'power_validation.json', + observed_gpu_count: 4, + }); + }); + it('fails missing points, stale attempts and duplicate identities', () => { + const point = expected(); + expect(verifyPowerPublication([point], [], 'DB')[0]).toContain('found 0'); + expect( + verifyPowerPublication( + [point], + [ + { + ...actual(), + run_url: 'https://github.com/SemiAnalysisAI/InferenceX/actions/runs/123/attempts/1', + }, + ], + 'API', + )[0], + ).toContain('found 0'); + expect(verifyPowerPublication([point], [actual(), actual()], 'API')[0]).toContain('found 2'); + }); + it('fails altered watts and audit provenance, including a newer performance point with older power', () => { + const row = actual(); + row.metrics.avg_power_w = 650; + expect(verifyPowerPublication([expected()], [row], 'API')).toContainEqual( + expect.stringContaining('avg_power_w expected 642, got 650'), + ); + expect( + verifyPowerPublication([expected()], [{ ...actual(), power_audit: null }], 'API'), + ).toContainEqual(expect.stringContaining('power_audit differs')); + expect( + verifyPowerPublication( + [expected()], + [{ ...actual(), recipe_fingerprint: 'different' }], + 'API', + )[0], + ).toContain('found 0'); + }); + it('preserves invalid diagnostics and rejects leaked invalid watts', () => { + const point = expected({ power_valid: 0, power_invalid_reasons: ['sampling_gap_exceeded'] }); + expect(point.metrics).not.toHaveProperty('avg_power_w'); + expect(verifyPowerPublication([point], [actual(point)], 'API')).toEqual([]); + const row = actual(point); + row.metrics.avg_power_w = 642; + expect(verifyPowerPublication([point], [row], 'API')[0]).toContain('expected absent, got 642'); + }); + it('rejects failed-client artifacts as performance points', () => { + const tracker = createSkipTracker(); + expect( + mapBenchmarkRow({ ...raw, benchmark_outcome: { status: 'failed' } }, tracker), + ).toBeNull(); + expect(tracker.skips.failedRun).toBe(1); + }); +}); diff --git a/packages/db/src/etl/power-publication.ts b/packages/db/src/etl/power-publication.ts new file mode 100644 index 000000000..d56942376 --- /dev/null +++ b/packages/db/src/etl/power-publication.ts @@ -0,0 +1,127 @@ +import { isDeepStrictEqual } from 'node:util'; +import { MEASURED_POWER_METRIC_KEYS } from '@semianalysisai/inferencex-constants'; +import type { BenchmarkParams } from './benchmark-mapper'; + +const CONFIG_FIELDS = { + hardware: 'hardware', + framework: 'framework', + model: 'model', + precision: 'precision', + specMethod: 'spec_method', + disagg: 'disagg', + isMultinode: 'is_multinode', + prefillTp: 'prefill_tp', + prefillEp: 'prefill_ep', + prefillDpAttn: 'prefill_dp_attention', + prefillNumWorkers: 'prefill_num_workers', + decodeTp: 'decode_tp', + decodeEp: 'decode_ep', + decodeDpAttn: 'decode_dp_attention', + decodeNumWorkers: 'decode_num_workers', + numPrefillGpu: 'num_prefill_gpu', + numDecodeGpu: 'num_decode_gpu', +} as const; +const IDENTITY_FIELDS = [ + ...Object.values(CONFIG_FIELDS), + 'benchmark_type', + 'isl', + 'osl', + 'conc', + 'offload_mode', + 'recipe_fingerprint', + 'image', + 'run_url', +] as const; +const POWER_FIELDS = [...MEASURED_POWER_METRIC_KEYS, 'power_valid', 'power_metric_schema_version']; +export interface PowerPublicationPoint { + identity: Record; + metrics: Record; + workers: unknown; + power_invalid_reasons: unknown; + power_audit: unknown; + artifact: { path: string; sha256: string }; +} +export interface PowerPublicationManifest { + version: 1; + runId: number; + runAttempt: number; + points: PowerPublicationPoint[]; + ingestErrors?: string[]; +} +export interface PublishedPowerRow extends Record { + metrics: Record; +} + +export function publicationIdentity(row: Record): string { + return JSON.stringify(IDENTITY_FIELDS.map((key) => row[key] ?? null)); +} + +export function powerPublicationPoint( + row: BenchmarkParams, + runUrl: string, + artifact: PowerPublicationPoint['artifact'], +): PowerPublicationPoint | null { + if (row.benchmarkType !== 'single_turn' || row.isl !== 8192 || row.osl !== 1024) return null; + const identity: Record = Object.fromEntries( + Object.entries(CONFIG_FIELDS).map(([source, target]) => [ + target, + row.config[source as keyof typeof CONFIG_FIELDS], + ]), + ); + Object.assign(identity, { + benchmark_type: row.benchmarkType, + isl: row.isl, + osl: row.osl, + conc: row.conc, + offload_mode: row.offloadMode, + recipe_fingerprint: row.recipeFingerprint, + image: row.image, + run_url: runUrl, + }); + return { + identity, + metrics: Object.fromEntries( + POWER_FIELDS.filter((key) => Object.hasOwn(row.metrics, key)).map((key) => [ + key, + row.metrics[key], + ]), + ), + workers: row.workers ?? null, + power_invalid_reasons: row.powerInvalidReasons ?? null, + power_audit: row.powerAudit ?? null, + artifact, + }; +} + +/** Matching numbers alone would miss leaked invalid telemetry or lost audit evidence. */ +export function verifyPowerPublication( + expected: readonly PowerPublicationPoint[], + actual: readonly PublishedPowerRow[], + source: string, +): string[] { + const byIdentity = new Map(); + for (const row of actual) { + const key = publicationIdentity(row); + byIdentity.set(key, [...(byIdentity.get(key) ?? []), row]); + } + const errors: string[] = []; + for (const point of expected) { + const rows = byIdentity.get(publicationIdentity(point.identity)) ?? []; + const label = `${source}: ${point.identity.hardware}/${point.identity.model} conc=${point.identity.conc} (${point.artifact.path})`; + if (rows.length !== 1) { + errors.push(`${label}: expected one exact source point, found ${rows.length}`); + continue; + } + const row = rows[0]; + for (const key of POWER_FIELDS) { + if (!Object.is(point.metrics[key], row.metrics[key])) + errors.push( + `${label}: ${key} expected ${point.metrics[key] ?? 'absent'}, got ${row.metrics[key] ?? 'absent'}`, + ); + } + for (const key of ['workers', 'power_invalid_reasons', 'power_audit'] as const) { + if (!isDeepStrictEqual(point[key], row[key] ?? null)) errors.push(`${label}: ${key} differs`); + } + } + return errors; +} diff --git a/packages/db/src/ingest-ci-run.ts b/packages/db/src/ingest-ci-run.ts index c3eb61362..e23e26163 100644 --- a/packages/db/src/ingest-ci-run.ts +++ b/packages/db/src/ingest-ci-run.ts @@ -23,6 +23,12 @@ */ import fs from 'fs'; +import { createHash } from 'node:crypto'; +import { + powerPublicationPoint, + publicationIdentity, + type PowerPublicationPoint, +} from './etl/power-publication'; import os from 'os'; import path from 'path'; @@ -84,6 +90,9 @@ import { // ── Config ────────────────────────────────────────────────────────────────── const DEFAULT_REPO = 'SemiAnalysisAI/InferenceX'; +const powerPublicationPoints = new Map(); +const powerPublicationErrors: string[] = []; +const tracker = createSkipTracker(); const isDownloadMode = process.argv[2] === '--download'; let artifactsDir: string; @@ -269,7 +278,6 @@ function findJsonFiles(dir: string): string[] { async function main(): Promise { validateRunBackfills(); - const tracker = createSkipTracker(); const configCache = createConfigCache(sql); const { getOrCreateConfig, preloadConfigs } = configCache; const { fetchGithubRun, getOrCreateWorkflowRun } = createWorkflowRunServices(sql, GITHUB_TOKEN, [ @@ -486,11 +494,13 @@ async function main(): Promise { for (const [fileIndex, file] of allBmkFiles.entries()) { const fileStart = Date.now(); const relativeFile = path.relative(artifactsDir, file); + const artifactSha256 = createHash('sha256').update(fs.readFileSync(file)).digest('hex'); console.log( ` [${fileIndex + 1}/${allBmkFiles.length}] ${relativeFile} (${formatBytes(fileSize(file))})`, ); const data = readJson(file); if (!data) { + powerPublicationErrors.push(`Unreadable benchmark JSON: ${relativeFile}`); console.log(` skipped unreadable JSON (${elapsed(fileStart)})`); continue; } @@ -508,7 +518,13 @@ async function main(): Promise { const rows = rawRows .filter((r) => typeof r === 'object' && r !== null) - .map((r) => mapBenchmarkRow(r, tracker, undefined, runIdStr)) + .map((r) => { + const mapped = mapBenchmarkRow(r, tracker, undefined, runIdStr); + if (!mapped && Number(r.isl) === 8192 && Number(r.osl) === 1024) { + powerPublicationErrors.push(`Unmapped or failed 8K/1K result: ${relativeFile}`); + } + return mapped; + }) .filter((r): r is NonNullable => r !== null); console.log(` mapped rows: ${rows.length}`); @@ -559,6 +575,13 @@ async function main(): Promise { `config ${configId}, conc ${row.conc}`, ); } + const publication = powerPublicationPoint( + applied.point, + `https://github.com/${REPO}/actions/runs/${runIdNum}/attempts/${runAttemptNum}`, + { path: relativeFile, sha256: artifactSha256 }, + ); + if (publication) + powerPublicationPoints.set(publicationIdentity(publication.identity), publication); toInsert.push(applied.point); } console.log(` rows with resolved configs: ${toInsert.length}`); @@ -998,6 +1021,26 @@ main() process.exitCode = 1; }) .finally(() => { + const publicationPath = process.env.POWER_PUBLICATION_MANIFEST; + if (publicationPath) { + fs.writeFileSync( + publicationPath, + `${JSON.stringify( + { + version: 1, + runId: runIdNum, + runAttempt: runAttemptNum, + points: [...powerPublicationPoints.values()], + ingestErrors: [ + ...powerPublicationErrors, + ...(tracker.skips.dbError ? [`${tracker.skips.dbError} database ingest errors`] : []), + ], + }, + null, + 2, + )}\n`, + ); + } if (tempDir) { try { fs.rmSync(tempDir, { recursive: true, force: true }); diff --git a/packages/db/src/ingest-supplemental.ts b/packages/db/src/ingest-supplemental.ts index 18a2a8b55..c498cf011 100644 --- a/packages/db/src/ingest-supplemental.ts +++ b/packages/db/src/ingest-supplemental.ts @@ -27,7 +27,12 @@ import { bulkUpsertAvailability, type BenchmarkPersistenceInput, } from './etl/benchmark-ingest'; -import { normalizePowerContractMetrics, scrubWithheldPowerMetrics } from './etl/benchmark-mapper'; +import { + extractPowerAudit, + extractPowerInvalidReasons, + normalizePowerContractMetrics, + scrubWithheldPowerMetrics, +} from './etl/benchmark-mapper'; import { ingestEvalRow } from './etl/eval-ingest'; const sql = createAdminSql({ @@ -180,6 +185,8 @@ interface SupplementalBmk { is_multinode?: boolean; prefill_num_workers?: number; decode_num_workers?: number; + power_invalid_reasons?: unknown; + power_audit?: unknown; } async function ingestSupplementalBmk( @@ -271,6 +278,13 @@ async function ingestSupplementalBmk( // then strip withheld measurements when power_valid=0. normalizePowerContractMetrics(entry.metrics, entry.metrics); scrubWithheldPowerMetrics(entry.metrics); + // Supplemental data bypasses mapBenchmarkRow's numeric-only metric boundary. + const powerInvalidReasons = extractPowerInvalidReasons( + entry.power_invalid_reasons ?? entry.metrics.power_invalid_reasons, + ); + const powerAudit = extractPowerAudit(entry.power_audit ?? entry.metrics.power_audit); + delete entry.metrics.power_invalid_reasons; + delete entry.metrics.power_audit; rows.push({ configId, @@ -282,6 +296,8 @@ async function ingestSupplementalBmk( image: entry.image, recipeFingerprint: null, metrics: entry.metrics, + powerInvalidReasons, + powerAudit, }); } diff --git a/packages/db/src/queries/benchmark-snapshots.test.ts b/packages/db/src/queries/benchmark-snapshots.test.ts index 207a5f481..98af99c24 100644 --- a/packages/db/src/queries/benchmark-snapshots.test.ts +++ b/packages/db/src/queries/benchmark-snapshots.test.ts @@ -84,6 +84,7 @@ beforeAll(async () => { const legacyRows = await getLatestBenchmarks(sql, 'glm5.2'); legacyCount = legacyRows.length; await db.exec(readFileSync(new URL('014_agentic_curve_snapshots.sql', dir), 'utf8')); + await db.exec(readFileSync(new URL('015_power_provenance.sql', dir), 'utf8')); const retained = await db.query<{ count: number }>( 'SELECT count(*)::int AS count FROM benchmark_results', ); @@ -107,6 +108,33 @@ describe('AgentX curve snapshots in PostgreSQL', () => { ids(await getAllBenchmarksForHistory(sql, 'glm5.2', null, null, 'agentic_traces')), ).toEqual([1, ...currentIds]); }); + it('retains power audits in materialized and dated snapshots without reviving replaced points', async () => { + await sql`UPDATE benchmark_results SET power_invalid_reasons = '["sampling_gap_exceeded"]'::jsonb, + power_audit = '{"expected_gpu_count":16}'::jsonb WHERE id IN (1, 2)`; + await db.exec( + readFileSync(new URL('../../migrations/015_power_provenance.sql', import.meta.url), 'utf8'), + ); + for (const rows of [ + await getLatestBenchmarks(sql, 'glm5.2'), + await getLatestBenchmarks(sql, 'glm5.2', '2026-09-11'), + ]) { + expect(ids(rows)).toEqual(currentIds); + expect(rows.find((row) => Number(row.id) === 2)).toMatchObject({ + power_invalid_reasons: ['sampling_gap_exceeded'], + power_audit: { expected_gpu_count: 16 }, + }); + expect(rows.find((row) => Number(row.id) === 3)).toMatchObject({ + power_invalid_reasons: null, + power_audit: null, + }); + } + const historicalRows = await getBenchmarksForRun(sql, 'glm5.2', 33219706372); + expect(historicalRows[0]).toMatchObject({ + id: 1, + power_invalid_reasons: ['sampling_gap_exceeded'], + power_audit: { expected_gpu_count: 16 }, + }); + }); it('uses the same scope in SQL and TypeScript and preserves point attributes', async () => { const old = await getBenchmarksForRun(sql, 'glm5.2', 33219706372); for (const row of [...old, ...(await getLatestBenchmarks(sql, 'glm5.2'))]) { diff --git a/packages/db/src/queries/benchmarks.test.ts b/packages/db/src/queries/benchmarks.test.ts index d9e9befc2..4a73c7d84 100644 --- a/packages/db/src/queries/benchmarks.test.ts +++ b/packages/db/src/queries/benchmarks.test.ts @@ -113,3 +113,43 @@ describe('append-only benchmark snapshots', () => { expect(values).toEqual([['dsv4']]); }); }); + +describe('power audit provenance reads (tolerant to a not-yet-applied migration 015)', () => { + const TOLERANT_BR = [ + "to_jsonb(br) -> 'power_invalid_reasons' AS power_invalid_reasons", + "to_jsonb(br) -> 'power_audit' AS power_audit", + ]; + + it('selects both columns via to_jsonb on the exact-run path', async () => { + const captured = captureSql(); + await getBenchmarksForRun(captured.sql, 'dsv4', 123456); + const { text } = captured.query(); + for (const piece of TOLERANT_BR) expect(text).toContain(piece); + expect(text).not.toMatch(/\b(?:br|lb)\.power_(?:invalid_reasons|audit)\b/u); + }); + + it('selects both columns via to_jsonb on the dated latest path', async () => { + const captured = captureSql(); + await getLatestBenchmarks(captured.sql, 'dsv4', '2026-08-01'); + const { text } = captured.query(); + for (const piece of TOLERANT_BR) expect(text).toContain(piece); + expect(text).not.toMatch(/\b(?:br|lb)\.power_(?:invalid_reasons|audit)\b/u); + }); + + it('selects both columns via to_jsonb on the history path', async () => { + const captured = captureSql(); + await getAllBenchmarksForHistory(captured.sql, 'dsv4', 8192, 1024); + const { text } = captured.query(); + for (const piece of TOLERANT_BR) expect(text).toContain(piece); + expect(text).not.toMatch(/\b(?:br|lb)\.power_(?:invalid_reasons|audit)\b/u); + }); + + it('selects both columns via to_jsonb on the no-date matview path', async () => { + const captured = captureSql(); + await getLatestBenchmarks(captured.sql, 'dsv4'); + const { text } = captured.query(); + expect(text).toContain("to_jsonb(lb) -> 'power_invalid_reasons' AS power_invalid_reasons"); + expect(text).toContain("to_jsonb(lb) -> 'power_audit' AS power_audit"); + expect(text).not.toMatch(/\b(?:br|lb)\.power_(?:invalid_reasons|audit)\b/u); + }); +}); diff --git a/packages/db/src/queries/benchmarks.ts b/packages/db/src/queries/benchmarks.ts index af34cd39a..00d835c2d 100644 --- a/packages/db/src/queries/benchmarks.ts +++ b/packages/db/src/queries/benchmarks.ts @@ -1,6 +1,6 @@ import type { DbClient } from '../connection.js'; -import type { WorkerPower } from '../etl/benchmark-mapper.js'; -export type { WorkerPower } from '../etl/benchmark-mapper.js'; +import type { PowerAudit, WorkerPower } from '../etl/benchmark-mapper.js'; +export type { PowerAudit, WorkerPower } from '../etl/benchmark-mapper.js'; /** * One entry in `BenchmarkRow.workers` — mirrors the runner's aggregate_power.py @@ -47,6 +47,10 @@ export interface BenchmarkRow { * aggregate_power.py's multinode patch — surfaced as undefined here. */ workers?: BenchmarkWorkerRow[]; + /** Producer reason codes for withheld power; null/undefined on other rows. */ + power_invalid_reasons?: string[] | null; + /** Narrowed measurement-window audit; null/undefined on legacy rows. */ + power_audit?: PowerAudit | null; date: string; /** Producer identity and timestamp; preserved for per-point provenance. */ workflow_run_id?: number; @@ -223,6 +227,11 @@ function executeRecursiveBenchmarkQuery( br.recipe_fingerprint, ${plan.metricsExpression}, br.workers, + -- A bare br.power_* reference fails during query planning until the next + -- ingest applies migration 015. The jsonb lookup returns NULL before the + -- migration and the stored value afterward, making deploy order safe. + to_jsonb(br) -> 'power_invalid_reasons' AS power_invalid_reasons, + to_jsonb(br) -> 'power_audit' AS power_audit, br.date::text, br.workflow_run_id, wr.run_started_at::text, @@ -414,6 +423,10 @@ export async function getLatestBenchmarks( lb.recipe_fingerprint, lb.metrics, lb.workers, + -- latest_benchmarks lacks these fields until migration 015 recreates it; + -- the jsonb lookup keeps reads safe during that deploy window. + to_jsonb(lb) -> 'power_invalid_reasons' AS power_invalid_reasons, + to_jsonb(lb) -> 'power_audit' AS power_audit, lb.date::text, lb.workflow_run_id, wr.run_started_at::text, diff --git a/packages/db/src/verify-power-publication.ts b/packages/db/src/verify-power-publication.ts new file mode 100644 index 000000000..769c12a74 --- /dev/null +++ b/packages/db/src/verify-power-publication.ts @@ -0,0 +1,89 @@ +import fs from 'node:fs'; +import { DB_MODEL_TO_DISPLAY } from '@semianalysisai/inferencex-constants'; +import { createAdminSql } from './etl/db-utils'; +import { + verifyPowerPublication, + type PowerPublicationManifest, + type PublishedPowerRow, +} from './etl/power-publication'; + +const manifestPath = process.argv[2]; +if (!manifestPath) + throw new Error('Usage: verify-power-publication.ts [public-origin]'); +const manifest = JSON.parse(fs.readFileSync(manifestPath, 'utf8')) as PowerPublicationManifest; +if ( + manifest.version !== 1 || + !Number.isSafeInteger(manifest.runId) || + manifest.runId <= 0 || + !Number.isSafeInteger(manifest.runAttempt) || + manifest.runAttempt <= 0 || + !Array.isArray(manifest.points) +) + throw new Error('Invalid PowerX publication manifest'); +const origin = process.argv[3] ?? 'https://inferencex.semianalysis.com'; +const sql = createAdminSql(); +try { + const rows = await sql` + select c.*, br.id, br.benchmark_type, br.isl, br.osl, br.conc, br.offload_mode, + br.recipe_fingerprint, br.image, br.metrics, br.workers, + to_jsonb(br) -> 'power_invalid_reasons' as power_invalid_reasons, + to_jsonb(br) -> 'power_audit' as power_audit, + wr.html_url || '/attempts/' || wr.run_attempt as run_url + from benchmark_results br join configs c on c.id = br.config_id + join workflow_runs wr on wr.id = br.workflow_run_id + where wr.github_run_id = ${manifest.runId} and wr.run_attempt = ${manifest.runAttempt} + and br.benchmark_type = 'single_turn' and br.isl = 8192 and br.osl = 1024 + `; + const errors = [ + ...(manifest.ingestErrors ?? []), + ...verifyPowerPublication(manifest.points, rows as unknown as PublishedPowerRow[], 'database'), + ]; + const publicRows: PublishedPowerRow[] = []; + const models = [ + ...new Set(manifest.points.map((point) => DB_MODEL_TO_DISPLAY[String(point.identity.model)])), + ]; + for (const model of models) { + if (!model) throw new Error('Manifest contains an unmapped public model'); + const url = new URL('/api/v1/benchmarks', origin); + url.search = new URLSearchParams({ + model, + runId: String(manifest.runId), + exactRun: 'true', + }).toString(); + const response = await fetch(url, { signal: AbortSignal.timeout(30_000) }); + if (!response.ok) + throw new Error(`Public PowerX verification returned HTTP ${response.status}: ${url}`); + const body: unknown = await response.json(); + if ( + !Array.isArray(body) || + body.some( + (row) => !row || typeof row !== 'object' || !row.metrics || typeof row.metrics !== 'object', + ) + ) + throw new Error(`Invalid benchmark response: ${url}`); + publicRows.push(...body); + } + errors.push(...verifyPowerPublication(manifest.points, publicRows, 'public API')); + const counts = { strict: 0, invalid: 0, other: 0 }; + for (const point of manifest.points) { + if (point.metrics.power_valid === 1 && point.metrics.power_metric_schema_version === 2) + counts.strict++; + else if (point.metrics.power_valid === 0) counts.invalid++; + else counts.other++; + } + const receipt = { + runId: manifest.runId, + runAttempt: manifest.runAttempt, + checkedAt: new Date().toISOString(), + points: manifest.points.length, + counts, + status: + errors.length > 0 ? 'failed' : manifest.points.length > 0 ? 'matched' : 'no_8k1k_points', + errors, + }; + fs.writeFileSync(`${manifestPath}.verification.json`, `${JSON.stringify(receipt, null, 2)}\n`); + console.log(JSON.stringify(receipt, null, 2)); + if (errors.length > 0) process.exitCode = 1; +} finally { + await sql.end(); +} From c7e60a016166f85c39ab604a2ba860213b24ee27 Mon Sep 17 00:00:00 2001 From: Wenyao Gao Date: Fri, 11 Sep 2026 18:19:33 -0700 Subject: [PATCH 2/2] test: provide unofficial context to chart controls MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 中文：为图表控件测试补充非官方运行上下文，匹配生产页面的 Provider 结构；保留全部原有断言。修复后相关组件测试 55 项全部通过，未启用重试。 --- .../component/inference-chart-controls.cy.tsx | 28 ++++++++++++++----- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/packages/app/cypress/component/inference-chart-controls.cy.tsx b/packages/app/cypress/component/inference-chart-controls.cy.tsx index c8b8656a4..a51f0f3f7 100644 --- a/packages/app/cypress/component/inference-chart-controls.cy.tsx +++ b/packages/app/cypress/component/inference-chart-controls.cy.tsx @@ -74,12 +74,12 @@ describe('Modeled system-power table', () => { */ function mountWithPowerGroupsUnlocked() { cy.window().then((win) => win.localStorage.setItem('inferencex-feature-gate', '1')); - mountWithProviders(, { inference: {} }); + mountWithProviders(, { inference: {}, unofficial: {} }); } describe('Inference ChartControls', () => { beforeEach(() => { - mountWithProviders(, { inference: {} }); + mountWithProviders(, { inference: {}, unofficial: {} }); }); afterEach(() => { @@ -320,7 +320,10 @@ describe('Inference ChartControls', () => { it('keeps benchmark and chart settings in one row when history comparison is omitted', () => { cy.viewport(1280, 900); - mountWithProviders(, { inference: {} }); + mountWithProviders(, { + inference: {}, + unofficial: {}, + }); cy.get('[data-testid="x-axis-mode-selector"]').should('not.exist'); cy.get('fieldset') .should('have.length', 2) @@ -344,7 +347,7 @@ describe('Inference ChartControls', () => { it('keeps primary controls visible while secondary controls collapse on mobile', () => { cy.viewport(390, 844); - mountWithProviders(, { inference: {} }); + mountWithProviders(, { inference: {}, unofficial: {} }); cy.get('#model-select').should('be.visible'); cy.get('[data-testid="inference-secondary-controls"] > button') @@ -362,7 +365,7 @@ describe('Inference ChartControls', () => { it('shows secondary controls by default on desktop', () => { cy.viewport(1280, 900); - mountWithProviders(, { inference: {} }); + mountWithProviders(, { inference: {}, unofficial: {} }); cy.get('[data-testid="inference-secondary-controls"] > button').should('not.be.visible'); cy.get('[data-testid="yaxis-metric-selector"]').should('be.visible'); @@ -374,7 +377,7 @@ describe('Inference ChartControls', () => { , - { inference: {} }, + { inference: {}, unofficial: {} }, ); // The count is derived from actual non-default settings, not merely present controls. cy.get('[data-testid="inference-secondary-controls"] > button') @@ -386,6 +389,7 @@ describe('Inference ChartControls', () => { describe('Inference ChartControls cost metrics', () => { beforeEach(() => { mountWithProviders(, { + unofficial: {}, inference: { selectedYAxisMetric: 'y_costh', selectedModel: Model.Qwen3_5, @@ -398,6 +402,7 @@ describe('Inference ChartControls cost metrics', () => { it('hides the TCO basis toggle for other models and scenarios', () => { mountWithProviders(, { + unofficial: {}, inference: { selectedYAxisMetric: 'y_costh', selectedModel: Model.DeepSeek_V4_Pro, @@ -411,6 +416,7 @@ describe('Inference ChartControls cost metrics', () => { cy.get('[data-testid="yaxis-metric-selector"]').should('exist'); cy.get('[data-testid="tco-basis-toggle"]').should('not.exist'); mountWithProviders(, { + unofficial: {}, inference: { selectedYAxisMetric: 'y_costh', selectedModel: Model.Qwen3_5, @@ -424,6 +430,7 @@ describe('Inference ChartControls cost metrics', () => { for (const selectedYAxisMetric of ['y_tpPerGpu', 'y_tpPerMw'] as const) { it(`hides TCO for ${selectedYAxisMetric} even with visible TPU hardware`, () => { mountWithProviders(, { + unofficial: {}, inference: { selectedYAxisMetric, selectedModel: Model.Qwen3_5, @@ -438,6 +445,7 @@ describe('Inference ChartControls cost metrics', () => { it('hides TCO for a cost metric when no TPU hardware is visible', () => { mountWithProviders(, { + unofficial: {}, inference: { selectedYAxisMetric: 'y_costh', selectedModel: Model.Qwen3_5, @@ -489,6 +497,7 @@ describe('Inference ChartControls cost metrics', () => { describe('Inference ChartControls infrastructure tokens per dollar', () => { beforeEach(() => { mountWithProviders(, { + unofficial: {}, inference: { selectedYAxisMetric: 'y_tokensPerDollarR' }, globalFilters: {}, }); @@ -512,6 +521,7 @@ describe('Inference ChartControls infrastructure tokens per dollar', () => { describe('Inference ChartControls with GPUs selected', () => { it('shows the date range picker when GPUs are selected', () => { mountWithProviders(, { + unofficial: {}, inference: { selectedGPUs: ['h100'], selectedDateRange: { startDate: '', endDate: '' }, @@ -523,6 +533,7 @@ describe('Inference ChartControls with GPUs selected', () => { it('leaves the optional date range unflagged for a selected current config', () => { mountWithProviders(, { + unofficial: {}, inference: { selectedGPUs: ['h100'], selectedDateRange: { startDate: '', endDate: '' }, @@ -537,6 +548,7 @@ describe('Inference ChartControls with GPUs selected', () => { it('leaves the date range unflagged when exact comparison entries are pinned', () => { mountWithProviders(, { + unofficial: {}, inference: { selectedGPUs: ['b200_sglang', 'b200_vllm'], selectedDateRange: { startDate: '', endDate: '' }, @@ -551,6 +563,7 @@ describe('Inference ChartControls with GPUs selected', () => { describe('Inference ChartControls with hideGpuComparison', () => { it('hides GPU config selector when hideGpuComparison is true', () => { mountWithProviders(, { + unofficial: {}, inference: {}, }); @@ -567,6 +580,7 @@ describe('Inference axis selector — Chinese Agentic controls', () => { , { + unofficial: {}, inference: { selectedSequence: Sequence.AgenticTraces, selectedXAxisMode: 'interactivity' }, }, ); @@ -586,7 +600,7 @@ describe('Inference axis selector — Chinese Agentic controls', () => { describe('Axis option help', () => { beforeEach(() => { - mountWithProviders(, { inference: {} }); + mountWithProviders(, { inference: {}, unofficial: {} }); }); for (const searchable of [true, false]) {