diff --git a/.github/workflows/test-changelog-gate.yml b/.github/workflows/test-changelog-gate.yml index 65bd1eebc..175c37c53 100644 --- a/.github/workflows/test-changelog-gate.yml +++ b/.github/workflows/test-changelog-gate.yml @@ -29,6 +29,7 @@ on: - "utils/test_collect_eval_results.py" - "utils/evals/validate_scores.py" - "utils/evals/test_batched_eval.py" + - "utils/evals/test_run_eval_dispatch.py" - "utils/prepare_perf_changelog_merge.py" - "utils/recover_failed_ingest.py" - "utils/changelog_gate_tests/test_prepare_perf_changelog_merge.py" @@ -83,4 +84,5 @@ jobs: utils/test_process_changelog.py \ utils/test_collect_eval_results.py \ utils/evals/test_batched_eval.py \ + utils/evals/test_run_eval_dispatch.py \ -v diff --git a/benchmarks/benchmark_lib.sh b/benchmarks/benchmark_lib.sh index 8b9b38f2e..9bab13b25 100644 --- a/benchmarks/benchmark_lib.sh +++ b/benchmarks/benchmark_lib.sh @@ -2267,7 +2267,12 @@ _write_lm_eval_meta_json() { local batch_metadata="${2:-}" local metadata_conc="${3:-${CONC:-1}}" - bridge_disagg_eval_metadata + # Single-node jobs already export TP/EP/DP_ATTENTION. The disaggregated + # bridge defaults missing per-phase DP flags to false, so applying it to + # single-node jobs would overwrite their actual DP-attention setting. + if [ "${IS_MULTINODE:-false}" = "true" ]; then + bridge_disagg_eval_metadata + fi local model_name="${MODEL_NAME:-$MODEL}" local is_multinode_json="false" diff --git a/docs/results-and-ingestion.md b/docs/results-and-ingestion.md index 1c1f5b916..3225e6d98 100644 --- a/docs/results-and-ingestion.md +++ b/docs/results-and-ingestion.md @@ -102,6 +102,16 @@ InferenceX-app treats routing fields as columns or config dimensions and stores Each eval upload is named `eval__`. Its current allowed payload includes `meta_env.json`, `results*.json`, sample JSONL, predictions, SWE-bench reports, and trajectory files. The collector uses only the metadata and lm-eval result JSON for aggregate rows. +The shared eval metadata writer preserves single-node `DP_ATTENTION` and uses it +as the default for both `prefill_dp_attention` and `decode_dp_attention`. Only +`IS_MULTINODE=true` jobs bridge the separate prefill/decode environment variables; +those jobs may have different DP-attention settings on each side. The collector +does not reconstruct topology from artifact names or server logs. Older artifacts +affected by the unconditional bridge can report `false` for a single-node +DP-attention eval. Fixing the writer does not repair those artifacts or existing +database rows: verify the original job configuration and server logs before +correcting metadata, regenerating aggregates, and re-ingesting affected results. + [`utils/collect_eval_results.py`](../utils/collect_eval_results.py) applies these rules: 1. An eval set is a root or immediate child directory containing `meta_env.json`. diff --git a/docs/results-and-ingestion_zh.md b/docs/results-and-ingestion_zh.md index 168251f45..54673f639 100644 --- a/docs/results-and-ingestion_zh.md +++ b/docs/results-and-ingestion_zh.md @@ -102,6 +102,15 @@ InferenceX-app 将路由字段作为列或配置维度,并把数值测量存 每个评测上传名为 `eval__`。当前允许的载荷包括 `meta_env.json`、`results*.json`、样本 JSONL、预测、SWE-bench 报告和轨迹文件。收集器只使用元数据和 lm-eval 结果 JSON 来生成聚合记录。 +共享评测元数据写入器保留单节点的 `DP_ATTENTION`,并将其作为 +`prefill_dp_attention` 和 `decode_dp_attention` 的默认值。只有 +`IS_MULTINODE=true` 的任务才会转换独立的 prefill/decode 环境变量; +这类任务两侧的 DP attention 设置可以不同。收集器不会根据工件名称或 +服务端日志重建拓扑。受旧版无条件转换逻辑影响的历史工件,可能将实际启用 +DP attention 的单节点评测记录为 `false`。修复写入器不会修复这些工件或 +已有数据库记录:应先核实原始任务配置和服务端日志,再更正元数据、重新生成 +聚合结果并重新摄取受影响的数据。 + [`utils/collect_eval_results.py`](../utils/collect_eval_results.py) 执行以下规则: 1. 评测集是包含 `meta_env.json` 的根目录或一级子目录。 diff --git a/utils/evals/test_run_eval_dispatch.py b/utils/evals/test_run_eval_dispatch.py index fd869efec..00d7b75a7 100644 --- a/utils/evals/test_run_eval_dispatch.py +++ b/utils/evals/test_run_eval_dispatch.py @@ -1489,13 +1489,46 @@ def _summary_metadata(tmp_path: Path, **overrides: str) -> dict: "CONC": "7", "KV_OFFLOADING": "none", } - for key in ("EVAL_COMPLETED_SUITE", "EVAL_SUITE", "EVAL_TASKS_DIR"): + for key in ( + "EVAL_COMPLETED_SUITE", "EVAL_SUITE", "EVAL_TASKS_DIR", + "IS_MULTINODE", "DP_ATTENTION", + "PREFILL_DP_ATTN", "PREFILL_DP_ATTENTION", "PREFILL_ENABLE_DP", + "DECODE_DP_ATTN", "DECODE_DP_ATTENTION", "DECODE_ENABLE_DP", + ): env.pop(key, None) env.update(overrides) subprocess.run(["bash", "-c", script], env=env, check=True) return json.loads((work_dir / "meta_env.json").read_text()) +@pytest.mark.parametrize("dp_attention, expected", [("true", True), ("false", False)]) +def test_summary_preserves_single_node_dp_attention( + tmp_path: Path, dp_attention: str, expected: bool, +) -> None: + meta = _summary_metadata( + tmp_path, IS_MULTINODE="false", TP="8", EP_SIZE="8", + DP_ATTENTION=dp_attention, + ) + assert meta["dp_attention"] is expected + assert meta["prefill_dp_attention"] is expected + assert meta["decode_dp_attention"] is expected + assert meta["tp"] == 8 + assert meta["ep"] == 8 + + +def test_summary_preserves_asymmetric_multinode_dp_attention(tmp_path: Path) -> None: + meta = _summary_metadata( + tmp_path, IS_MULTINODE="true", DP_ATTENTION="false", + PREFILL_TP="4", PREFILL_EP="4", DECODE_TP="8", DECODE_EP="8", + PREFILL_DP_ATTN="true", DECODE_DP_ATTN="false", + ) + assert meta["dp_attention"] is True + assert meta["prefill_dp_attention"] is True + assert meta["decode_dp_attention"] is False + assert meta["prefill_tp"] == 4 + assert meta["decode_tp"] == 8 + + def test_summary_stages_bfcl_upstream_archive_before_cleanup(tmp_path: Path) -> None: work_dir = tmp_path / "work" results_dir = tmp_path / "results"