diff --git a/docs/results-and-ingestion.md b/docs/results-and-ingestion.md index 3225e6d985..95d951d66c 100644 --- a/docs/results-and-ingestion.md +++ b/docs/results-and-ingestion.md @@ -353,6 +353,20 @@ printf 'temporary inspection directory: %s\n' "$tmp" rm -rf -- "$tmp" ``` +## P75 and P90 measured GPU power + +Validated single-node SMI and multinode DCGM results also emit `p75_total_gpu_power_w`, +`p75_power_w`, `p90_total_gpu_power_w`, and `p90_power_w`. The total fields are +the time-weighted 75th and 90th percentiles of the sum of +all participating GPU-board power curves during the same formal benchmark window +used for energy integration. Device samples are aligned with piecewise-linear +interpolation before summing; elapsed time, rather than sample count, weights the +percentile. Each per-chip field divides its fleet percentile by the participating GPU count. +It is not an individual GPU's percentile or the average of device percentiles. +All four values are withheld when telemetry validation fails. Older results remain +missing until their original raw traces can be replayed; average watts cannot +supply P75 or P90. The validation sidecar records `power_percentile_method`. + ## Verification and stop conditions A handoff is verified only when all applicable checks pass. diff --git a/docs/results-and-ingestion_zh.md b/docs/results-and-ingestion_zh.md index 54673f639f..ec98c51680 100644 --- a/docs/results-and-ingestion_zh.md +++ b/docs/results-and-ingestion_zh.md @@ -352,6 +352,17 @@ printf 'temporary inspection directory: %s\n' "$tmp" rm -rf -- "$tmp" ``` +## GPU 实测功耗 P75 和 P90 + +通过验证的单节点 SMI 和多节点 DCGM 结果还会输出 `p75_total_gpu_power_w`、 +`p75_power_w`、`p90_total_gpu_power_w` 与 `p90_power_w`。两个整组指标使用与能耗 +积分相同的正式基准测试窗口，对参与测量的所有 GPU 板卡功耗之和计算按时间加权的 +第 75 和第 90 百分位数。各设备采样通过分段线性插值按时间对齐后求和，分位数按 +持续时间加权，而不是按采样数量加权。两个按芯片均摊的指标分别将对应的整组 GPU +功耗分位数除以参与测量的 GPU 数量，因此既不是单个 GPU 的分位数，也不是 +各设备分位数的平均值。遥测验证失败时，四项指标都不发布。旧结果需要使用原始 +遥测重新计算；不能从平均功耗推算 P75 或 P90。验证 sidecar 会记录 `power_percentile_method`。 + ## 验证和停止条件 只有全部适用检查通过，交接才算验证完成。 diff --git a/infx/results/power/__init__.py b/infx/results/power/__init__.py index c2c04b35b6..f0e1da95cc 100644 --- a/infx/results/power/__init__.py +++ b/infx/results/power/__init__.py @@ -15,6 +15,10 @@ WHOLE_METRIC_KEYS = ( "avg_power_w", + "p75_power_w", + "p75_total_gpu_power_w", + "p90_power_w", + "p90_total_gpu_power_w", "avg_total_gpu_power_w", "total_gpu_energy_j", "joules_per_successful_query", diff --git a/infx/results/power/common.py b/infx/results/power/common.py index 82748f9287..8886744687 100644 --- a/infx/results/power/common.py +++ b/infx/results/power/common.py @@ -76,6 +76,63 @@ def _integrate_device( return energy_j +def _percentile_total_power( + device_samples: list[list[tuple[float, float]]], + *, + start_unix: float, + end_unix: float, + quantile: float, +) -> float: + """Time-weighted quantile of synchronized fleet power with linear interpolation. + + Sum device curves before taking the percentile. Each linear segment's + distribution is uniform over its power range, weighted by elapsed time; + constant segments contribute a point mass. Sampling cadence cannot bias + the result. Call only after all streams and their shared window validate. + """ + slope_changes: dict[float, float] = {start_unix: 0.0, end_unix: 0.0} + total_power = 0.0 + for samples in device_samples: + first = _interpolate_power(samples, start_unix) + total_power += first + clipped = [(start_unix, first)] + clipped.extend((t, p) for t, p in samples if start_unix < t < end_unix) + clipped.append((end_unix, _interpolate_power(samples, end_unix))) + for (left_t, left_p), (right_t, right_p) in zip(clipped, clipped[1:]): + slope = (right_p - left_p) / (right_t - left_t) + slope_changes[left_t] = slope_changes.get(left_t, 0.0) + slope + slope_changes[right_t] = slope_changes.get(right_t, 0.0) - slope + + # The union of all device timestamps gives the exact knots of their sum. + segments: list[tuple[float, float, float]] = [] + slope = 0.0 + times = sorted(slope_changes) + for left_t, right_t in zip(times, times[1:]): + slope += slope_changes[left_t] + next_power = total_power + slope * (right_t - left_t) + segments.append(( + min(total_power, next_power), max(total_power, next_power), right_t - left_t + )) + total_power = next_power + lower = min(low for low, _, _ in segments) + upper = max(high for _, high, _ in segments) + target_time = (end_unix - start_unix) * quantile + # Bisection includes point masses without averaging device percentiles. + for _ in range(60): + value = lower + (upper - lower) / 2 + time_below = sum( + duration if value >= high + else duration * (value - low) / (high - low) if value > low + else 0.0 + for low, high, duration in segments + ) + if time_below >= target_time: + upper = value + else: + lower = value + return upper + + def _load_benchmark_data( bench_result_path: Path, ) -> tuple[BenchmarkData | None, list[str]]: diff --git a/infx/results/power/multinode.py b/infx/results/power/multinode.py index 6f0bde4442..7899b067d3 100644 --- a/infx/results/power/multinode.py +++ b/infx/results/power/multinode.py @@ -43,6 +43,7 @@ BenchmarkData, _append_reason, _integrate_device, + _percentile_total_power, _load_benchmark_data, _write_json_atomic, audit_metrics, @@ -1062,8 +1063,19 @@ def validate_and_integrate( duration_s = window.end_unix - window.start_unix total_energy = sum(per_gpu_energy.values()) total_tokens = benchmark.total_input_tokens + benchmark.total_output_tokens + device_samples = [sorted(per_key_samples[device.key]) for device in expected_devices] + p75_total = _percentile_total_power( + device_samples, start_unix=window.start_unix, end_unix=window.end_unix, quantile=0.75, + ) + p90_total = _percentile_total_power( + device_samples, start_unix=window.start_unix, end_unix=window.end_unix, quantile=0.9, + ) metrics = { "avg_power_w": total_energy / duration_s / len(expected_devices), + "p75_power_w": p75_total / len(expected_devices), + "p75_total_gpu_power_w": p75_total, + "p90_power_w": p90_total / len(expected_devices), + "p90_total_gpu_power_w": p90_total, "avg_total_gpu_power_w": total_energy / duration_s, "total_gpu_energy_j": total_energy, "joules_per_successful_query": total_energy / benchmark.completed, @@ -1190,6 +1202,7 @@ def _sidecar_payload( "benchmark_window": benchmark_window_payload(benchmark), "selected_window": audit.window, "integration_method": _INTEGRATION_METHOD, + "power_percentile_method": "time_weighted_synchronized_total_piecewise_linear", "producer": { "producer_git_commit": audit.producer_git_commit, "expected_producer_git_commit": audit.expected_producer_git_commit, diff --git a/infx/results/power/single_node.py b/infx/results/power/single_node.py index ff9bd115e4..54669a1a80 100644 --- a/infx/results/power/single_node.py +++ b/infx/results/power/single_node.py @@ -32,6 +32,7 @@ BenchmarkData, _append_reason, _integrate_device, + _percentile_total_power, _interpolate_power, _load_benchmark_data, _write_json_atomic, @@ -65,6 +66,10 @@ class PowerIntegration: per_gpu_energy_j: dict[str, float] device_issues: dict[str, list[str]] avg_power_w: float | None = None + p75_power_w: float | None = None + p75_total_gpu_power_w: float | None = None + p90_power_w: float | None = None + p90_total_gpu_power_w: float | None = None avg_total_gpu_power_w: float | None = None total_gpu_energy_j: float | None = None @@ -382,12 +387,14 @@ def integrate_power( per_gpu_max_sample_gap_s: dict[str, float] = {} per_gpu_energy_j: dict[str, float] = {} device_issues: dict[str, list[str]] = {} + device_samples: list[list[tuple[float, float]]] = [] for gpu_id in observed_gpu_ids: timestamp_values = raw_samples[gpu_id] samples = sorted( (timestamp, mean(values)) for timestamp, values in timestamp_values.items() ) + device_samples.append(samples) per_gpu_sample_counts[gpu_id] = len(samples) issues: list[str] = [] @@ -428,6 +435,12 @@ def integrate_power( avg_total_gpu_power_w = total_gpu_energy_j / duration_s avg_power_w = avg_total_gpu_power_w / len(observed_gpu_ids) + p75_total = None if reasons else _percentile_total_power( + device_samples, start_unix=start_unix, end_unix=end_unix, quantile=0.75 + ) + p90_total = None if reasons else _percentile_total_power( + device_samples, start_unix=start_unix, end_unix=end_unix, quantile=0.9 + ) return PowerIntegration( power_valid=not reasons, invalid_reasons=tuple(reasons), @@ -438,6 +451,10 @@ def integrate_power( per_gpu_energy_j=per_gpu_energy_j, device_issues=device_issues, avg_power_w=avg_power_w, + p75_power_w=p75_total / len(observed_gpu_ids) if p75_total is not None else None, + p75_total_gpu_power_w=p75_total, + p90_power_w=p90_total / len(observed_gpu_ids) if p90_total is not None else None, + p90_total_gpu_power_w=p90_total, avg_total_gpu_power_w=avg_total_gpu_power_w, total_gpu_energy_j=total_gpu_energy_j, ) @@ -595,6 +612,10 @@ def _derived_metrics( """Return whole-deployment energy metrics for a valid measurement.""" if ( integration.avg_power_w is None + or integration.p75_power_w is None + or integration.p75_total_gpu_power_w is None + or integration.p90_power_w is None + or integration.p90_total_gpu_power_w is None or integration.avg_total_gpu_power_w is None or integration.total_gpu_energy_j is None ): @@ -605,6 +626,10 @@ def _derived_metrics( total_tokens = benchmark.total_input_tokens + benchmark.total_output_tokens return { "avg_power_w": avg_power_w, + "p75_power_w": integration.p75_power_w, + "p75_total_gpu_power_w": integration.p75_total_gpu_power_w, + "p90_power_w": integration.p90_power_w, + "p90_total_gpu_power_w": integration.p90_total_gpu_power_w, "avg_total_gpu_power_w": avg_total_gpu_power_w, "total_gpu_energy_j": energy, "joules_per_successful_query": energy / benchmark.completed, @@ -646,6 +671,7 @@ def _validation_payload( "benchmark_result": str(bench_result), "benchmark_window": benchmark_window_payload(benchmark), "integration_method": _INTEGRATION_METHOD, + "power_percentile_method": "time_weighted_synchronized_total_piecewise_linear", "expected_gpu_count": integration.expected_num_gpus, "observed_gpu_count": integration.observed_num_gpus, "observed_gpu_ids": list(integration.observed_gpu_ids), diff --git a/utils/test_aggregate_power.py b/utils/test_aggregate_power.py index 5b74134982..6a62f780ae 100644 --- a/utils/test_aggregate_power.py +++ b/utils/test_aggregate_power.py @@ -1417,3 +1417,48 @@ def test_packaged_power_runs_without_legacy_scripts(power_artifacts, tmp_path): assert result.returncode == 0, result.stderr assert power_artifacts["package"].agg()["total_gpu_energy_j"] == 84000 assert power_artifacts["package"].sidecar()["power_valid"] is True + + +def test_power_percentiles_uses_synchronized_total_not_device_percentiles(tmp_path): + csv_path = tmp_path / "power.csv" + # Opposing device ramps keep the fleet draw constant at 600 W. + _write_amd_csv(csv_path, [(0, 0, 100), (1, 0, 500), (0, 1, 500), (1, 1, 100)]) + result = integrate_power(csv_path, start_unix=0, end_unix=1, expected_num_gpus=2) + assert result.power_valid + assert result.p75_total_gpu_power_w == pytest.approx(600) + assert result.p75_power_w == pytest.approx(300) + assert result.p90_total_gpu_power_w == pytest.approx(600) + assert result.p90_power_w == pytest.approx(300) + + +def test_power_percentiles_weights_time_and_clips_the_validated_window(tmp_path): + csv_path = tmp_path / "power.csv" + # Dense readings near the high end must not bias a uniform linear ramp. + _write_amd_csv(csv_path, [(0, 0, 0), (1, 0, 100), (1.9, 0, 190), (2, 0, 200)]) + result = integrate_power(csv_path, start_unix=0.5, end_unix=1.5, expected_num_gpus=1) + assert result.power_valid + assert result.p75_power_w == pytest.approx(125) + assert result.p90_power_w == pytest.approx(140) + + +def test_power_percentiles_is_withheld_for_invalid_telemetry(tmp_path): + csv_path = tmp_path / "power.csv" + _write_amd_csv(csv_path, [(0, 0, 100), (10, 0, 500)]) + result = integrate_power(csv_path, start_unix=0, end_unix=10, expected_num_gpus=1) + assert not result.power_valid + assert result.p75_power_w is None + assert result.p75_total_gpu_power_w is None + assert result.p90_power_w is None + + +def test_power_percentiles_aligns_asynchronous_gpu_samples(tmp_path): + csv_path = tmp_path / "power.csv" + _write_amd_csv(csv_path, [ + (0, 0, 100), (1, 0, 300), (-0.5, 1, 600), (0.5, 1, 400), (1.5, 1, 200) + ]) + result = integrate_power(csv_path, start_unix=0, end_unix=1, expected_num_gpus=2) + assert result.power_valid + assert result.p75_total_gpu_power_w == pytest.approx(600) + assert result.p75_power_w == pytest.approx(300) + assert result.p90_total_gpu_power_w == pytest.approx(600) + assert result.p90_power_w == pytest.approx(300) diff --git a/utils/test_aggregate_power_multinode.py b/utils/test_aggregate_power_multinode.py index 6dfb9136d9..15c4de2b04 100644 --- a/utils/test_aggregate_power_multinode.py +++ b/utils/test_aggregate_power_multinode.py @@ -246,6 +246,10 @@ def test_emits_all_metrics_exactly(self, tmp_path): assert agg["power_metric_schema_version"] == 2 assert agg["power_valid"] == 1 assert agg["avg_power_w"] == 350.0 + assert agg["p75_power_w"] == 350.0 + assert agg["p75_total_gpu_power_w"] == 1400.0 + assert agg["p90_power_w"] == 350.0 + assert agg["p90_total_gpu_power_w"] == 1400.0 assert agg["avg_total_gpu_power_w"] == 1400.0 assert agg["total_gpu_energy_j"] == 84000.0 assert agg["joules_per_successful_query"] == 10500.0 @@ -291,6 +295,10 @@ def ramp(host, idx, ts): assert agg["decode_avg_power_w"] == pytest.approx(316.0) assert agg["avg_total_gpu_power_w"] == pytest.approx(1432.0) assert agg["avg_power_w"] == pytest.approx(358.0) + assert agg["p75_total_gpu_power_w"] == pytest.approx(1447.0) + assert agg["p75_power_w"] == pytest.approx(361.75) + assert agg["p90_total_gpu_power_w"] == pytest.approx(1456.0) + assert agg["p90_power_w"] == pytest.approx(364.0) def test_strict_mode_passes_on_valid_package(self, tmp_path): pkg = build_package(tmp_path)