[PowerX] add validated P75 and P90 GPU power metrics / 新增经过验证的 GPU P75 和 P90 功耗指标

docs/results-and-ingestion.md

@@ -353,6 +353,20 @@ printf 'temporary inspection directory: %s\n' "$tmp"
 rm -rf -- "$tmp"
 ```
 
+## P75 and P90 measured GPU power
+
+Validated single-node SMI and multinode DCGM results also emit `p75_total_gpu_power_w`,
+`p75_power_w`, `p90_total_gpu_power_w`, and `p90_power_w`. The total fields are
+the time-weighted 75th and 90th percentiles of the sum of
+all participating GPU-board power curves during the same formal benchmark window
+used for energy integration. Device samples are aligned with piecewise-linear
+interpolation before summing; elapsed time, rather than sample count, weights the
+percentile. Each per-chip field divides its fleet percentile by the participating GPU count.
+It is not an individual GPU's percentile or the average of device percentiles.
+All four values are withheld when telemetry validation fails. Older results remain
+missing until their original raw traces can be replayed; average watts cannot
+supply P75 or P90. The validation sidecar records `power_percentile_method`.
+
 ## Verification and stop conditions
 
 A handoff is verified only when all applicable checks pass.

docs/results-and-ingestion_zh.md

@@ -352,6 +352,17 @@ printf 'temporary inspection directory: %s\n' "$tmp"
 rm -rf -- "$tmp"
 ```
 
+## GPU 实测功耗 P75 和 P90
+
+通过验证的单节点 SMI 和多节点 DCGM 结果还会输出 `p75_total_gpu_power_w`、
+`p75_power_w`、`p90_total_gpu_power_w` 与 `p90_power_w`。两个整组指标使用与能耗
+积分相同的正式基准测试窗口，对参与测量的所有 GPU 板卡功耗之和计算按时间加权的
+第 75 和第 90 百分位数。各设备采样通过分段线性插值按时间对齐后求和，分位数按
+持续时间加权，而不是按采样数量加权。两个按芯片均摊的指标分别将对应的整组 GPU
+功耗分位数除以参与测量的 GPU 数量，因此既不是单个 GPU 的分位数，也不是
+各设备分位数的平均值。遥测验证失败时，四项指标都不发布。旧结果需要使用原始
+遥测重新计算；不能从平均功耗推算 P75 或 P90。验证 sidecar 会记录 `power_percentile_method`。
+
 ## 验证和停止条件
 
 只有全部适用检查通过，交接才算验证完成。

infx/results/power/__init__.py

@@ -15,6 +15,10 @@
 
 WHOLE_METRIC_KEYS = (
     "avg_power_w",
+    "p75_power_w",
+    "p75_total_gpu_power_w",
+    "p90_power_w",
+    "p90_total_gpu_power_w",
     "avg_total_gpu_power_w",
     "total_gpu_energy_j",
     "joules_per_successful_query",

infx/results/power/common.py

@@ -76,6 +76,63 @@ def _integrate_device(
     return energy_j
 
 
+def _percentile_total_power(
+    device_samples: list[list[tuple[float, float]]],
+    *,
+    start_unix: float,
+    end_unix: float,
+    quantile: float,
+) -> float:
+    """Time-weighted quantile of synchronized fleet power with linear interpolation.
+
+    Sum device curves before taking the percentile. Each linear segment's
+    distribution is uniform over its power range, weighted by elapsed time;
+    constant segments contribute a point mass. Sampling cadence cannot bias
+    the result. Call only after all streams and their shared window validate.
+    """
+    slope_changes: dict[float, float] = {start_unix: 0.0, end_unix: 0.0}
+    total_power = 0.0
+    for samples in device_samples:
+        first = _interpolate_power(samples, start_unix)
+        total_power += first
+        clipped = [(start_unix, first)]
+        clipped.extend((t, p) for t, p in samples if start_unix < t < end_unix)
+        clipped.append((end_unix, _interpolate_power(samples, end_unix)))
+        for (left_t, left_p), (right_t, right_p) in zip(clipped, clipped[1:]):
+            slope = (right_p - left_p) / (right_t - left_t)
+            slope_changes[left_t] = slope_changes.get(left_t, 0.0) + slope
+            slope_changes[right_t] = slope_changes.get(right_t, 0.0) - slope
+
+    # The union of all device timestamps gives the exact knots of their sum.
+    segments: list[tuple[float, float, float]] = []
+    slope = 0.0
+    times = sorted(slope_changes)
+    for left_t, right_t in zip(times, times[1:]):
+        slope += slope_changes[left_t]
+        next_power = total_power + slope * (right_t - left_t)
+        segments.append((
+            min(total_power, next_power), max(total_power, next_power), right_t - left_t
+        ))
+        total_power = next_power
+    lower = min(low for low, _, _ in segments)
+    upper = max(high for _, high, _ in segments)
+    target_time = (end_unix - start_unix) * quantile
+    # Bisection includes point masses without averaging device percentiles.
+    for _ in range(60):
+        value = lower + (upper - lower) / 2
+        time_below = sum(
+            duration if value >= high
+            else duration * (value - low) / (high - low) if value > low
+            else 0.0
+            for low, high, duration in segments
+        )
+        if time_below >= target_time:
+            upper = value
+        else:
+            lower = value
+    return upper
+
+
 def _load_benchmark_data(
     bench_result_path: Path,
 ) -> tuple[BenchmarkData | None, list[str]]:

infx/results/power/multinode.py

@@ -43,6 +43,7 @@
     BenchmarkData,
     _append_reason,
     _integrate_device,
+    _percentile_total_power,
     _load_benchmark_data,
     _write_json_atomic,
     audit_metrics,
@@ -1062,8 +1063,19 @@ def validate_and_integrate(
     duration_s = window.end_unix - window.start_unix
     total_energy = sum(per_gpu_energy.values())
     total_tokens = benchmark.total_input_tokens + benchmark.total_output_tokens
+    device_samples = [sorted(per_key_samples[device.key]) for device in expected_devices]
+    p75_total = _percentile_total_power(
+        device_samples, start_unix=window.start_unix, end_unix=window.end_unix, quantile=0.75,
+    )
+    p90_total = _percentile_total_power(
+        device_samples, start_unix=window.start_unix, end_unix=window.end_unix, quantile=0.9,
+    )
     metrics = {
         "avg_power_w": total_energy / duration_s / len(expected_devices),
+        "p75_power_w": p75_total / len(expected_devices),
+        "p75_total_gpu_power_w": p75_total,
+        "p90_power_w": p90_total / len(expected_devices),
+        "p90_total_gpu_power_w": p90_total,
         "avg_total_gpu_power_w": total_energy / duration_s,
         "total_gpu_energy_j": total_energy,
         "joules_per_successful_query": total_energy / benchmark.completed,
@@ -1190,6 +1202,7 @@ def _sidecar_payload(
         "benchmark_window": benchmark_window_payload(benchmark),
         "selected_window": audit.window,
         "integration_method": _INTEGRATION_METHOD,
+        "power_percentile_method": "time_weighted_synchronized_total_piecewise_linear",
         "producer": {
             "producer_git_commit": audit.producer_git_commit,
             "expected_producer_git_commit": audit.expected_producer_git_commit,

infx/results/power/single_node.py

@@ -32,6 +32,7 @@
     BenchmarkData,
     _append_reason,
     _integrate_device,
+    _percentile_total_power,
     _interpolate_power,
     _load_benchmark_data,
     _write_json_atomic,
@@ -65,6 +66,10 @@ class PowerIntegration:
     per_gpu_energy_j: dict[str, float]
     device_issues: dict[str, list[str]]
     avg_power_w: float | None = None
+    p75_power_w: float | None = None
+    p75_total_gpu_power_w: float | None = None
+    p90_power_w: float | None = None
+    p90_total_gpu_power_w: float | None = None
     avg_total_gpu_power_w: float | None = None
     total_gpu_energy_j: float | None = None
 
@@ -382,12 +387,14 @@ def integrate_power(
     per_gpu_max_sample_gap_s: dict[str, float] = {}
     per_gpu_energy_j: dict[str, float] = {}
     device_issues: dict[str, list[str]] = {}
+    device_samples: list[list[tuple[float, float]]] = []
 
     for gpu_id in observed_gpu_ids:
         timestamp_values = raw_samples[gpu_id]
         samples = sorted(
             (timestamp, mean(values)) for timestamp, values in timestamp_values.items()
         )
+        device_samples.append(samples)
         per_gpu_sample_counts[gpu_id] = len(samples)
         issues: list[str] = []
 
@@ -428,6 +435,12 @@ def integrate_power(
         avg_total_gpu_power_w = total_gpu_energy_j / duration_s
         avg_power_w = avg_total_gpu_power_w / len(observed_gpu_ids)
 
+    p75_total = None if reasons else _percentile_total_power(
+        device_samples, start_unix=start_unix, end_unix=end_unix, quantile=0.75
+    )
+    p90_total = None if reasons else _percentile_total_power(
+        device_samples, start_unix=start_unix, end_unix=end_unix, quantile=0.9
+    )
     return PowerIntegration(
         power_valid=not reasons,
         invalid_reasons=tuple(reasons),
@@ -438,6 +451,10 @@ def integrate_power(
         per_gpu_energy_j=per_gpu_energy_j,
         device_issues=device_issues,
         avg_power_w=avg_power_w,
+        p75_power_w=p75_total / len(observed_gpu_ids) if p75_total is not None else None,
+        p75_total_gpu_power_w=p75_total,
+        p90_power_w=p90_total / len(observed_gpu_ids) if p90_total is not None else None,
+        p90_total_gpu_power_w=p90_total,
         avg_total_gpu_power_w=avg_total_gpu_power_w,
         total_gpu_energy_j=total_gpu_energy_j,
     )
@@ -595,6 +612,10 @@ def _derived_metrics(
     """Return whole-deployment energy metrics for a valid measurement."""
     if (
         integration.avg_power_w is None
+        or integration.p75_power_w is None
+        or integration.p75_total_gpu_power_w is None
+        or integration.p90_power_w is None
+        or integration.p90_total_gpu_power_w is None
         or integration.avg_total_gpu_power_w is None
         or integration.total_gpu_energy_j is None
     ):
@@ -605,6 +626,10 @@ def _derived_metrics(
     total_tokens = benchmark.total_input_tokens + benchmark.total_output_tokens
     return {
         "avg_power_w": avg_power_w,
+        "p75_power_w": integration.p75_power_w,
+        "p75_total_gpu_power_w": integration.p75_total_gpu_power_w,
+        "p90_power_w": integration.p90_power_w,
+        "p90_total_gpu_power_w": integration.p90_total_gpu_power_w,
         "avg_total_gpu_power_w": avg_total_gpu_power_w,
         "total_gpu_energy_j": energy,
         "joules_per_successful_query": energy / benchmark.completed,
@@ -646,6 +671,7 @@ def _validation_payload(
         "benchmark_result": str(bench_result),
         "benchmark_window": benchmark_window_payload(benchmark),
         "integration_method": _INTEGRATION_METHOD,
+        "power_percentile_method": "time_weighted_synchronized_total_piecewise_linear",
         "expected_gpu_count": integration.expected_num_gpus,
         "observed_gpu_count": integration.observed_num_gpus,
         "observed_gpu_ids": list(integration.observed_gpu_ids),

utils/test_aggregate_power.py

@@ -1417,3 +1417,48 @@ def test_packaged_power_runs_without_legacy_scripts(power_artifacts, tmp_path):
     assert result.returncode == 0, result.stderr
     assert power_artifacts["package"].agg()["total_gpu_energy_j"] == 84000
     assert power_artifacts["package"].sidecar()["power_valid"] is True
+
+
+def test_power_percentiles_uses_synchronized_total_not_device_percentiles(tmp_path):
+    csv_path = tmp_path / "power.csv"
+    # Opposing device ramps keep the fleet draw constant at 600 W.
+    _write_amd_csv(csv_path, [(0, 0, 100), (1, 0, 500), (0, 1, 500), (1, 1, 100)])
+    result = integrate_power(csv_path, start_unix=0, end_unix=1, expected_num_gpus=2)
+    assert result.power_valid
+    assert result.p75_total_gpu_power_w == pytest.approx(600)
+    assert result.p75_power_w == pytest.approx(300)
+    assert result.p90_total_gpu_power_w == pytest.approx(600)
+    assert result.p90_power_w == pytest.approx(300)
+
+
+def test_power_percentiles_weights_time_and_clips_the_validated_window(tmp_path):
+    csv_path = tmp_path / "power.csv"
+    # Dense readings near the high end must not bias a uniform linear ramp.
+    _write_amd_csv(csv_path, [(0, 0, 0), (1, 0, 100), (1.9, 0, 190), (2, 0, 200)])
+    result = integrate_power(csv_path, start_unix=0.5, end_unix=1.5, expected_num_gpus=1)
+    assert result.power_valid
+    assert result.p75_power_w == pytest.approx(125)
+    assert result.p90_power_w == pytest.approx(140)
+
+
+def test_power_percentiles_is_withheld_for_invalid_telemetry(tmp_path):
+    csv_path = tmp_path / "power.csv"
+    _write_amd_csv(csv_path, [(0, 0, 100), (10, 0, 500)])
+    result = integrate_power(csv_path, start_unix=0, end_unix=10, expected_num_gpus=1)
+    assert not result.power_valid
+    assert result.p75_power_w is None
+    assert result.p75_total_gpu_power_w is None
+    assert result.p90_power_w is None
+
+
+def test_power_percentiles_aligns_asynchronous_gpu_samples(tmp_path):
+    csv_path = tmp_path / "power.csv"
+    _write_amd_csv(csv_path, [
+        (0, 0, 100), (1, 0, 300), (-0.5, 1, 600), (0.5, 1, 400), (1.5, 1, 200)
+    ])
+    result = integrate_power(csv_path, start_unix=0, end_unix=1, expected_num_gpus=2)
+    assert result.power_valid
+    assert result.p75_total_gpu_power_w == pytest.approx(600)
+    assert result.p75_power_w == pytest.approx(300)
+    assert result.p90_total_gpu_power_w == pytest.approx(600)
+    assert result.p90_power_w == pytest.approx(300)

utils/test_aggregate_power_multinode.py

@@ -246,6 +246,10 @@ def test_emits_all_metrics_exactly(self, tmp_path):
         assert agg["power_metric_schema_version"] == 2
         assert agg["power_valid"] == 1
         assert agg["avg_power_w"] == 350.0
+        assert agg["p75_power_w"] == 350.0
+        assert agg["p75_total_gpu_power_w"] == 1400.0
+        assert agg["p90_power_w"] == 350.0
+        assert agg["p90_total_gpu_power_w"] == 1400.0
         assert agg["avg_total_gpu_power_w"] == 1400.0
         assert agg["total_gpu_energy_j"] == 84000.0
         assert agg["joules_per_successful_query"] == 10500.0
@@ -291,6 +295,10 @@ def ramp(host, idx, ts):
         assert agg["decode_avg_power_w"] == pytest.approx(316.0)
         assert agg["avg_total_gpu_power_w"] == pytest.approx(1432.0)
         assert agg["avg_power_w"] == pytest.approx(358.0)
+        assert agg["p75_total_gpu_power_w"] == pytest.approx(1447.0)
+        assert agg["p75_power_w"] == pytest.approx(361.75)
+        assert agg["p90_total_gpu_power_w"] == pytest.approx(1456.0)
+        assert agg["p90_power_w"] == pytest.approx(364.0)
 
     def test_strict_mode_passes_on_valid_package(self, tmp_path):
         pkg = build_package(tmp_path)