From f387fe099fee259595b5c95f83d72167477b1aef Mon Sep 17 00:00:00 2001 From: Wenyao Gao Date: Thu, 10 Sep 2026 12:33:22 -0700 Subject: [PATCH 1/2] feat: add validated P90 GPU power metrics MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 中文:新增经过验证的 GPU P90 功耗指标。先按时间对齐并汇总设备功耗,再计算时间加权 P90;单节点和多节点均复用正式测量窗口,验证失败时不发布。 --- docs/results-and-ingestion.md | 13 ++++++ docs/results-and-ingestion_zh.md | 10 +++++ infx/results/power/__init__.py | 2 + infx/results/power/common.py | 56 +++++++++++++++++++++++++ infx/results/power/multinode.py | 8 ++++ infx/results/power/single_node.py | 15 +++++++ utils/test_aggregate_power.py | 38 +++++++++++++++++ utils/test_aggregate_power_multinode.py | 4 ++ 8 files changed, 146 insertions(+) diff --git a/docs/results-and-ingestion.md b/docs/results-and-ingestion.md index 3225e6d985..45c00e4be1 100644 --- a/docs/results-and-ingestion.md +++ b/docs/results-and-ingestion.md @@ -353,6 +353,19 @@ printf 'temporary inspection directory: %s\n' "$tmp" rm -rf -- "$tmp" ``` +## P90 measured GPU power + +Validated single-node SMI and multinode DCGM results also emit `p90_total_gpu_power_w` +and `p90_power_w`. The former is the time-weighted 90th percentile of the sum of +all participating GPU-board power curves during the same formal benchmark window +used for energy integration. Device samples are aligned with piecewise-linear +interpolation before summing; elapsed time, rather than sample count, weights the +percentile. The latter divides this fleet percentile by the participating GPU count. +It is not an individual GPU's percentile or the average of device percentiles. +Both values are withheld when telemetry validation fails. Older results remain +missing until their original raw traces can be replayed; average watts cannot +supply P90. The validation sidecar records `power_percentile_method`. + ## Verification and stop conditions A handoff is verified only when all applicable checks pass. diff --git a/docs/results-and-ingestion_zh.md b/docs/results-and-ingestion_zh.md index 54673f639f..a10012e859 100644 --- a/docs/results-and-ingestion_zh.md +++ b/docs/results-and-ingestion_zh.md @@ -352,6 +352,16 @@ printf 'temporary inspection directory: %s\n' "$tmp" rm -rf -- "$tmp" ``` +## GPU 实测功耗 P90 + +通过验证的单节点 SMI 和多节点 DCGM 结果还会输出 `p90_total_gpu_power_w` +与 `p90_power_w`。前者使用与能耗积分相同的正式基准测试窗口,对参与测量的所有 +GPU 板卡功耗之和计算按时间加权的第 90 百分位数。各设备采样通过分段线性插值 +按时间对齐后求和,分位数按持续时间加权,而不是按采样数量加权。后者再将这个 +整组 GPU 的 P90 除以参与测量的 GPU 数量,因此既不是单个 GPU 的 P90,也不是 +各设备 P90 的平均值。遥测验证失败时,两项指标都不发布。旧结果需要使用原始 +遥测重新计算;不能从平均功耗推算 P90。验证 sidecar 会记录 `power_percentile_method`。 + ## 验证和停止条件 只有全部适用检查通过,交接才算验证完成。 diff --git a/infx/results/power/__init__.py b/infx/results/power/__init__.py index c2c04b35b6..551bbac01e 100644 --- a/infx/results/power/__init__.py +++ b/infx/results/power/__init__.py @@ -15,6 +15,8 @@ WHOLE_METRIC_KEYS = ( "avg_power_w", + "p90_power_w", + "p90_total_gpu_power_w", "avg_total_gpu_power_w", "total_gpu_energy_j", "joules_per_successful_query", diff --git a/infx/results/power/common.py b/infx/results/power/common.py index 82748f9287..13c09394d6 100644 --- a/infx/results/power/common.py +++ b/infx/results/power/common.py @@ -76,6 +76,62 @@ def _integrate_device( return energy_j +def _p90_total_power( + device_samples: list[list[tuple[float, float]]], + *, + start_unix: float, + end_unix: float, +) -> float: + """Time-weighted P90 of synchronized fleet power with linear interpolation. + + Sum device curves before taking the percentile. Each linear segment's + distribution is uniform over its power range, weighted by elapsed time; + constant segments contribute a point mass. Sampling cadence cannot bias + the result. Call only after all streams and their shared window validate. + """ + slope_changes: dict[float, float] = {start_unix: 0.0, end_unix: 0.0} + total_power = 0.0 + for samples in device_samples: + first = _interpolate_power(samples, start_unix) + total_power += first + clipped = [(start_unix, first)] + clipped.extend((t, p) for t, p in samples if start_unix < t < end_unix) + clipped.append((end_unix, _interpolate_power(samples, end_unix))) + for (left_t, left_p), (right_t, right_p) in zip(clipped, clipped[1:]): + slope = (right_p - left_p) / (right_t - left_t) + slope_changes[left_t] = slope_changes.get(left_t, 0.0) + slope + slope_changes[right_t] = slope_changes.get(right_t, 0.0) - slope + + # The union of all device timestamps gives the exact knots of their sum. + segments: list[tuple[float, float, float]] = [] + slope = 0.0 + times = sorted(slope_changes) + for left_t, right_t in zip(times, times[1:]): + slope += slope_changes[left_t] + next_power = total_power + slope * (right_t - left_t) + segments.append(( + min(total_power, next_power), max(total_power, next_power), right_t - left_t + )) + total_power = next_power + lower = min(low for low, _, _ in segments) + upper = max(high for _, high, _ in segments) + target_time = (end_unix - start_unix) * 0.9 + # Bisection includes point masses without averaging device percentiles. + for _ in range(60): + value = lower + (upper - lower) / 2 + time_below = sum( + duration if value >= high + else duration * (value - low) / (high - low) if value > low + else 0.0 + for low, high, duration in segments + ) + if time_below >= target_time: + upper = value + else: + lower = value + return upper + + def _load_benchmark_data( bench_result_path: Path, ) -> tuple[BenchmarkData | None, list[str]]: diff --git a/infx/results/power/multinode.py b/infx/results/power/multinode.py index 6f0bde4442..e3a9e7a257 100644 --- a/infx/results/power/multinode.py +++ b/infx/results/power/multinode.py @@ -43,6 +43,7 @@ BenchmarkData, _append_reason, _integrate_device, + _p90_total_power, _load_benchmark_data, _write_json_atomic, audit_metrics, @@ -1062,8 +1063,14 @@ def validate_and_integrate( duration_s = window.end_unix - window.start_unix total_energy = sum(per_gpu_energy.values()) total_tokens = benchmark.total_input_tokens + benchmark.total_output_tokens + p90_total = _p90_total_power( + [sorted(per_key_samples[device.key]) for device in expected_devices], + start_unix=window.start_unix, end_unix=window.end_unix, + ) metrics = { "avg_power_w": total_energy / duration_s / len(expected_devices), + "p90_power_w": p90_total / len(expected_devices), + "p90_total_gpu_power_w": p90_total, "avg_total_gpu_power_w": total_energy / duration_s, "total_gpu_energy_j": total_energy, "joules_per_successful_query": total_energy / benchmark.completed, @@ -1190,6 +1197,7 @@ def _sidecar_payload( "benchmark_window": benchmark_window_payload(benchmark), "selected_window": audit.window, "integration_method": _INTEGRATION_METHOD, + "power_percentile_method": "time_weighted_synchronized_total_piecewise_linear", "producer": { "producer_git_commit": audit.producer_git_commit, "expected_producer_git_commit": audit.expected_producer_git_commit, diff --git a/infx/results/power/single_node.py b/infx/results/power/single_node.py index ff9bd115e4..58de815c63 100644 --- a/infx/results/power/single_node.py +++ b/infx/results/power/single_node.py @@ -32,6 +32,7 @@ BenchmarkData, _append_reason, _integrate_device, + _p90_total_power, _interpolate_power, _load_benchmark_data, _write_json_atomic, @@ -65,6 +66,8 @@ class PowerIntegration: per_gpu_energy_j: dict[str, float] device_issues: dict[str, list[str]] avg_power_w: float | None = None + p90_power_w: float | None = None + p90_total_gpu_power_w: float | None = None avg_total_gpu_power_w: float | None = None total_gpu_energy_j: float | None = None @@ -382,12 +385,14 @@ def integrate_power( per_gpu_max_sample_gap_s: dict[str, float] = {} per_gpu_energy_j: dict[str, float] = {} device_issues: dict[str, list[str]] = {} + device_samples: list[list[tuple[float, float]]] = [] for gpu_id in observed_gpu_ids: timestamp_values = raw_samples[gpu_id] samples = sorted( (timestamp, mean(values)) for timestamp, values in timestamp_values.items() ) + device_samples.append(samples) per_gpu_sample_counts[gpu_id] = len(samples) issues: list[str] = [] @@ -428,6 +433,9 @@ def integrate_power( avg_total_gpu_power_w = total_gpu_energy_j / duration_s avg_power_w = avg_total_gpu_power_w / len(observed_gpu_ids) + p90_total = None if reasons else _p90_total_power( + device_samples, start_unix=start_unix, end_unix=end_unix + ) return PowerIntegration( power_valid=not reasons, invalid_reasons=tuple(reasons), @@ -438,6 +446,8 @@ def integrate_power( per_gpu_energy_j=per_gpu_energy_j, device_issues=device_issues, avg_power_w=avg_power_w, + p90_power_w=p90_total / len(observed_gpu_ids) if p90_total is not None else None, + p90_total_gpu_power_w=p90_total, avg_total_gpu_power_w=avg_total_gpu_power_w, total_gpu_energy_j=total_gpu_energy_j, ) @@ -595,6 +605,8 @@ def _derived_metrics( """Return whole-deployment energy metrics for a valid measurement.""" if ( integration.avg_power_w is None + or integration.p90_power_w is None + or integration.p90_total_gpu_power_w is None or integration.avg_total_gpu_power_w is None or integration.total_gpu_energy_j is None ): @@ -605,6 +617,8 @@ def _derived_metrics( total_tokens = benchmark.total_input_tokens + benchmark.total_output_tokens return { "avg_power_w": avg_power_w, + "p90_power_w": integration.p90_power_w, + "p90_total_gpu_power_w": integration.p90_total_gpu_power_w, "avg_total_gpu_power_w": avg_total_gpu_power_w, "total_gpu_energy_j": energy, "joules_per_successful_query": energy / benchmark.completed, @@ -646,6 +660,7 @@ def _validation_payload( "benchmark_result": str(bench_result), "benchmark_window": benchmark_window_payload(benchmark), "integration_method": _INTEGRATION_METHOD, + "power_percentile_method": "time_weighted_synchronized_total_piecewise_linear", "expected_gpu_count": integration.expected_num_gpus, "observed_gpu_count": integration.observed_num_gpus, "observed_gpu_ids": list(integration.observed_gpu_ids), diff --git a/utils/test_aggregate_power.py b/utils/test_aggregate_power.py index 5b74134982..e87504b115 100644 --- a/utils/test_aggregate_power.py +++ b/utils/test_aggregate_power.py @@ -1417,3 +1417,41 @@ def test_packaged_power_runs_without_legacy_scripts(power_artifacts, tmp_path): assert result.returncode == 0, result.stderr assert power_artifacts["package"].agg()["total_gpu_energy_j"] == 84000 assert power_artifacts["package"].sidecar()["power_valid"] is True + + +def test_p90_power_uses_synchronized_total_not_device_percentiles(tmp_path): + csv_path = tmp_path / "power.csv" + # Opposing device ramps keep the fleet draw constant at 600 W. + _write_amd_csv(csv_path, [(0, 0, 100), (1, 0, 500), (0, 1, 500), (1, 1, 100)]) + result = integrate_power(csv_path, start_unix=0, end_unix=1, expected_num_gpus=2) + assert result.power_valid + assert result.p90_total_gpu_power_w == pytest.approx(600) + assert result.p90_power_w == pytest.approx(300) + + +def test_p90_power_weights_time_and_clips_the_validated_window(tmp_path): + csv_path = tmp_path / "power.csv" + # Dense readings near the high end must not bias a uniform linear ramp. + _write_amd_csv(csv_path, [(0, 0, 0), (1, 0, 100), (1.9, 0, 190), (2, 0, 200)]) + result = integrate_power(csv_path, start_unix=0.5, end_unix=1.5, expected_num_gpus=1) + assert result.power_valid + assert result.p90_power_w == pytest.approx(140) + + +def test_p90_power_is_withheld_for_invalid_telemetry(tmp_path): + csv_path = tmp_path / "power.csv" + _write_amd_csv(csv_path, [(0, 0, 100), (10, 0, 500)]) + result = integrate_power(csv_path, start_unix=0, end_unix=10, expected_num_gpus=1) + assert not result.power_valid + assert result.p90_power_w is None + + +def test_p90_power_aligns_asynchronous_gpu_samples(tmp_path): + csv_path = tmp_path / "power.csv" + _write_amd_csv(csv_path, [ + (0, 0, 100), (1, 0, 300), (-0.5, 1, 600), (0.5, 1, 400), (1.5, 1, 200) + ]) + result = integrate_power(csv_path, start_unix=0, end_unix=1, expected_num_gpus=2) + assert result.power_valid + assert result.p90_total_gpu_power_w == pytest.approx(600) + assert result.p90_power_w == pytest.approx(300) diff --git a/utils/test_aggregate_power_multinode.py b/utils/test_aggregate_power_multinode.py index 6dfb9136d9..850382bdb6 100644 --- a/utils/test_aggregate_power_multinode.py +++ b/utils/test_aggregate_power_multinode.py @@ -246,6 +246,8 @@ def test_emits_all_metrics_exactly(self, tmp_path): assert agg["power_metric_schema_version"] == 2 assert agg["power_valid"] == 1 assert agg["avg_power_w"] == 350.0 + assert agg["p90_power_w"] == 350.0 + assert agg["p90_total_gpu_power_w"] == 1400.0 assert agg["avg_total_gpu_power_w"] == 1400.0 assert agg["total_gpu_energy_j"] == 84000.0 assert agg["joules_per_successful_query"] == 10500.0 @@ -291,6 +293,8 @@ def ramp(host, idx, ts): assert agg["decode_avg_power_w"] == pytest.approx(316.0) assert agg["avg_total_gpu_power_w"] == pytest.approx(1432.0) assert agg["avg_power_w"] == pytest.approx(358.0) + assert agg["p90_total_gpu_power_w"] == pytest.approx(1456.0) + assert agg["p90_power_w"] == pytest.approx(364.0) def test_strict_mode_passes_on_valid_package(self, tmp_path): pkg = build_package(tmp_path) From 4b70d21287b20415039d4aa0bc0bcf29114b1a6e Mon Sep 17 00:00:00 2001 From: Wenyao Gao Date: Thu, 10 Sep 2026 16:32:24 -0700 Subject: [PATCH 2/2] feat: add P75 to validated GPU power percentiles MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 中文:在已验证的 GPU 功耗指标中新增 P75,复用按时间加权的整组功耗分位数计算,同时保留 P90。单节点、多节点输出及无效数据清理同步更新。 --- docs/results-and-ingestion.md | 13 +++++++------ docs/results-and-ingestion_zh.md | 19 ++++++++++--------- infx/results/power/__init__.py | 2 ++ infx/results/power/common.py | 7 ++++--- infx/results/power/multinode.py | 13 +++++++++---- infx/results/power/single_node.py | 17 ++++++++++++++--- utils/test_aggregate_power.py | 15 +++++++++++---- utils/test_aggregate_power_multinode.py | 4 ++++ 8 files changed, 61 insertions(+), 29 deletions(-) diff --git a/docs/results-and-ingestion.md b/docs/results-and-ingestion.md index 45c00e4be1..95d951d66c 100644 --- a/docs/results-and-ingestion.md +++ b/docs/results-and-ingestion.md @@ -353,18 +353,19 @@ printf 'temporary inspection directory: %s\n' "$tmp" rm -rf -- "$tmp" ``` -## P90 measured GPU power +## P75 and P90 measured GPU power -Validated single-node SMI and multinode DCGM results also emit `p90_total_gpu_power_w` -and `p90_power_w`. The former is the time-weighted 90th percentile of the sum of +Validated single-node SMI and multinode DCGM results also emit `p75_total_gpu_power_w`, +`p75_power_w`, `p90_total_gpu_power_w`, and `p90_power_w`. The total fields are +the time-weighted 75th and 90th percentiles of the sum of all participating GPU-board power curves during the same formal benchmark window used for energy integration. Device samples are aligned with piecewise-linear interpolation before summing; elapsed time, rather than sample count, weights the -percentile. The latter divides this fleet percentile by the participating GPU count. +percentile. Each per-chip field divides its fleet percentile by the participating GPU count. It is not an individual GPU's percentile or the average of device percentiles. -Both values are withheld when telemetry validation fails. Older results remain +All four values are withheld when telemetry validation fails. Older results remain missing until their original raw traces can be replayed; average watts cannot -supply P90. The validation sidecar records `power_percentile_method`. +supply P75 or P90. The validation sidecar records `power_percentile_method`. ## Verification and stop conditions diff --git a/docs/results-and-ingestion_zh.md b/docs/results-and-ingestion_zh.md index a10012e859..ec98c51680 100644 --- a/docs/results-and-ingestion_zh.md +++ b/docs/results-and-ingestion_zh.md @@ -352,15 +352,16 @@ printf 'temporary inspection directory: %s\n' "$tmp" rm -rf -- "$tmp" ``` -## GPU 实测功耗 P90 - -通过验证的单节点 SMI 和多节点 DCGM 结果还会输出 `p90_total_gpu_power_w` -与 `p90_power_w`。前者使用与能耗积分相同的正式基准测试窗口,对参与测量的所有 -GPU 板卡功耗之和计算按时间加权的第 90 百分位数。各设备采样通过分段线性插值 -按时间对齐后求和,分位数按持续时间加权,而不是按采样数量加权。后者再将这个 -整组 GPU 的 P90 除以参与测量的 GPU 数量,因此既不是单个 GPU 的 P90,也不是 -各设备 P90 的平均值。遥测验证失败时,两项指标都不发布。旧结果需要使用原始 -遥测重新计算;不能从平均功耗推算 P90。验证 sidecar 会记录 `power_percentile_method`。 +## GPU 实测功耗 P75 和 P90 + +通过验证的单节点 SMI 和多节点 DCGM 结果还会输出 `p75_total_gpu_power_w`、 +`p75_power_w`、`p90_total_gpu_power_w` 与 `p90_power_w`。两个整组指标使用与能耗 +积分相同的正式基准测试窗口,对参与测量的所有 GPU 板卡功耗之和计算按时间加权的 +第 75 和第 90 百分位数。各设备采样通过分段线性插值按时间对齐后求和,分位数按 +持续时间加权,而不是按采样数量加权。两个按芯片均摊的指标分别将对应的整组 GPU +功耗分位数除以参与测量的 GPU 数量,因此既不是单个 GPU 的分位数,也不是 +各设备分位数的平均值。遥测验证失败时,四项指标都不发布。旧结果需要使用原始 +遥测重新计算;不能从平均功耗推算 P75 或 P90。验证 sidecar 会记录 `power_percentile_method`。 ## 验证和停止条件 diff --git a/infx/results/power/__init__.py b/infx/results/power/__init__.py index 551bbac01e..f0e1da95cc 100644 --- a/infx/results/power/__init__.py +++ b/infx/results/power/__init__.py @@ -15,6 +15,8 @@ WHOLE_METRIC_KEYS = ( "avg_power_w", + "p75_power_w", + "p75_total_gpu_power_w", "p90_power_w", "p90_total_gpu_power_w", "avg_total_gpu_power_w", diff --git a/infx/results/power/common.py b/infx/results/power/common.py index 13c09394d6..8886744687 100644 --- a/infx/results/power/common.py +++ b/infx/results/power/common.py @@ -76,13 +76,14 @@ def _integrate_device( return energy_j -def _p90_total_power( +def _percentile_total_power( device_samples: list[list[tuple[float, float]]], *, start_unix: float, end_unix: float, + quantile: float, ) -> float: - """Time-weighted P90 of synchronized fleet power with linear interpolation. + """Time-weighted quantile of synchronized fleet power with linear interpolation. Sum device curves before taking the percentile. Each linear segment's distribution is uniform over its power range, weighted by elapsed time; @@ -115,7 +116,7 @@ def _p90_total_power( total_power = next_power lower = min(low for low, _, _ in segments) upper = max(high for _, high, _ in segments) - target_time = (end_unix - start_unix) * 0.9 + target_time = (end_unix - start_unix) * quantile # Bisection includes point masses without averaging device percentiles. for _ in range(60): value = lower + (upper - lower) / 2 diff --git a/infx/results/power/multinode.py b/infx/results/power/multinode.py index e3a9e7a257..7899b067d3 100644 --- a/infx/results/power/multinode.py +++ b/infx/results/power/multinode.py @@ -43,7 +43,7 @@ BenchmarkData, _append_reason, _integrate_device, - _p90_total_power, + _percentile_total_power, _load_benchmark_data, _write_json_atomic, audit_metrics, @@ -1063,12 +1063,17 @@ def validate_and_integrate( duration_s = window.end_unix - window.start_unix total_energy = sum(per_gpu_energy.values()) total_tokens = benchmark.total_input_tokens + benchmark.total_output_tokens - p90_total = _p90_total_power( - [sorted(per_key_samples[device.key]) for device in expected_devices], - start_unix=window.start_unix, end_unix=window.end_unix, + device_samples = [sorted(per_key_samples[device.key]) for device in expected_devices] + p75_total = _percentile_total_power( + device_samples, start_unix=window.start_unix, end_unix=window.end_unix, quantile=0.75, + ) + p90_total = _percentile_total_power( + device_samples, start_unix=window.start_unix, end_unix=window.end_unix, quantile=0.9, ) metrics = { "avg_power_w": total_energy / duration_s / len(expected_devices), + "p75_power_w": p75_total / len(expected_devices), + "p75_total_gpu_power_w": p75_total, "p90_power_w": p90_total / len(expected_devices), "p90_total_gpu_power_w": p90_total, "avg_total_gpu_power_w": total_energy / duration_s, diff --git a/infx/results/power/single_node.py b/infx/results/power/single_node.py index 58de815c63..54669a1a80 100644 --- a/infx/results/power/single_node.py +++ b/infx/results/power/single_node.py @@ -32,7 +32,7 @@ BenchmarkData, _append_reason, _integrate_device, - _p90_total_power, + _percentile_total_power, _interpolate_power, _load_benchmark_data, _write_json_atomic, @@ -66,6 +66,8 @@ class PowerIntegration: per_gpu_energy_j: dict[str, float] device_issues: dict[str, list[str]] avg_power_w: float | None = None + p75_power_w: float | None = None + p75_total_gpu_power_w: float | None = None p90_power_w: float | None = None p90_total_gpu_power_w: float | None = None avg_total_gpu_power_w: float | None = None @@ -433,8 +435,11 @@ def integrate_power( avg_total_gpu_power_w = total_gpu_energy_j / duration_s avg_power_w = avg_total_gpu_power_w / len(observed_gpu_ids) - p90_total = None if reasons else _p90_total_power( - device_samples, start_unix=start_unix, end_unix=end_unix + p75_total = None if reasons else _percentile_total_power( + device_samples, start_unix=start_unix, end_unix=end_unix, quantile=0.75 + ) + p90_total = None if reasons else _percentile_total_power( + device_samples, start_unix=start_unix, end_unix=end_unix, quantile=0.9 ) return PowerIntegration( power_valid=not reasons, @@ -446,6 +451,8 @@ def integrate_power( per_gpu_energy_j=per_gpu_energy_j, device_issues=device_issues, avg_power_w=avg_power_w, + p75_power_w=p75_total / len(observed_gpu_ids) if p75_total is not None else None, + p75_total_gpu_power_w=p75_total, p90_power_w=p90_total / len(observed_gpu_ids) if p90_total is not None else None, p90_total_gpu_power_w=p90_total, avg_total_gpu_power_w=avg_total_gpu_power_w, @@ -605,6 +612,8 @@ def _derived_metrics( """Return whole-deployment energy metrics for a valid measurement.""" if ( integration.avg_power_w is None + or integration.p75_power_w is None + or integration.p75_total_gpu_power_w is None or integration.p90_power_w is None or integration.p90_total_gpu_power_w is None or integration.avg_total_gpu_power_w is None @@ -617,6 +626,8 @@ def _derived_metrics( total_tokens = benchmark.total_input_tokens + benchmark.total_output_tokens return { "avg_power_w": avg_power_w, + "p75_power_w": integration.p75_power_w, + "p75_total_gpu_power_w": integration.p75_total_gpu_power_w, "p90_power_w": integration.p90_power_w, "p90_total_gpu_power_w": integration.p90_total_gpu_power_w, "avg_total_gpu_power_w": avg_total_gpu_power_w, diff --git a/utils/test_aggregate_power.py b/utils/test_aggregate_power.py index e87504b115..6a62f780ae 100644 --- a/utils/test_aggregate_power.py +++ b/utils/test_aggregate_power.py @@ -1419,39 +1419,46 @@ def test_packaged_power_runs_without_legacy_scripts(power_artifacts, tmp_path): assert power_artifacts["package"].sidecar()["power_valid"] is True -def test_p90_power_uses_synchronized_total_not_device_percentiles(tmp_path): +def test_power_percentiles_uses_synchronized_total_not_device_percentiles(tmp_path): csv_path = tmp_path / "power.csv" # Opposing device ramps keep the fleet draw constant at 600 W. _write_amd_csv(csv_path, [(0, 0, 100), (1, 0, 500), (0, 1, 500), (1, 1, 100)]) result = integrate_power(csv_path, start_unix=0, end_unix=1, expected_num_gpus=2) assert result.power_valid + assert result.p75_total_gpu_power_w == pytest.approx(600) + assert result.p75_power_w == pytest.approx(300) assert result.p90_total_gpu_power_w == pytest.approx(600) assert result.p90_power_w == pytest.approx(300) -def test_p90_power_weights_time_and_clips_the_validated_window(tmp_path): +def test_power_percentiles_weights_time_and_clips_the_validated_window(tmp_path): csv_path = tmp_path / "power.csv" # Dense readings near the high end must not bias a uniform linear ramp. _write_amd_csv(csv_path, [(0, 0, 0), (1, 0, 100), (1.9, 0, 190), (2, 0, 200)]) result = integrate_power(csv_path, start_unix=0.5, end_unix=1.5, expected_num_gpus=1) assert result.power_valid + assert result.p75_power_w == pytest.approx(125) assert result.p90_power_w == pytest.approx(140) -def test_p90_power_is_withheld_for_invalid_telemetry(tmp_path): +def test_power_percentiles_is_withheld_for_invalid_telemetry(tmp_path): csv_path = tmp_path / "power.csv" _write_amd_csv(csv_path, [(0, 0, 100), (10, 0, 500)]) result = integrate_power(csv_path, start_unix=0, end_unix=10, expected_num_gpus=1) assert not result.power_valid + assert result.p75_power_w is None + assert result.p75_total_gpu_power_w is None assert result.p90_power_w is None -def test_p90_power_aligns_asynchronous_gpu_samples(tmp_path): +def test_power_percentiles_aligns_asynchronous_gpu_samples(tmp_path): csv_path = tmp_path / "power.csv" _write_amd_csv(csv_path, [ (0, 0, 100), (1, 0, 300), (-0.5, 1, 600), (0.5, 1, 400), (1.5, 1, 200) ]) result = integrate_power(csv_path, start_unix=0, end_unix=1, expected_num_gpus=2) assert result.power_valid + assert result.p75_total_gpu_power_w == pytest.approx(600) + assert result.p75_power_w == pytest.approx(300) assert result.p90_total_gpu_power_w == pytest.approx(600) assert result.p90_power_w == pytest.approx(300) diff --git a/utils/test_aggregate_power_multinode.py b/utils/test_aggregate_power_multinode.py index 850382bdb6..15c4de2b04 100644 --- a/utils/test_aggregate_power_multinode.py +++ b/utils/test_aggregate_power_multinode.py @@ -246,6 +246,8 @@ def test_emits_all_metrics_exactly(self, tmp_path): assert agg["power_metric_schema_version"] == 2 assert agg["power_valid"] == 1 assert agg["avg_power_w"] == 350.0 + assert agg["p75_power_w"] == 350.0 + assert agg["p75_total_gpu_power_w"] == 1400.0 assert agg["p90_power_w"] == 350.0 assert agg["p90_total_gpu_power_w"] == 1400.0 assert agg["avg_total_gpu_power_w"] == 1400.0 @@ -293,6 +295,8 @@ def ramp(host, idx, ts): assert agg["decode_avg_power_w"] == pytest.approx(316.0) assert agg["avg_total_gpu_power_w"] == pytest.approx(1432.0) assert agg["avg_power_w"] == pytest.approx(358.0) + assert agg["p75_total_gpu_power_w"] == pytest.approx(1447.0) + assert agg["p75_power_w"] == pytest.approx(361.75) assert agg["p90_total_gpu_power_w"] == pytest.approx(1456.0) assert agg["p90_power_w"] == pytest.approx(364.0)