diff --git a/benchmarks/benchmark_lib.sh b/benchmarks/benchmark_lib.sh index 8b9b38f2e5..e8ffd55a55 100644 --- a/benchmarks/benchmark_lib.sh +++ b/benchmarks/benchmark_lib.sh @@ -3297,7 +3297,7 @@ build_replay_cmd() { # necessarily a valid HF repo id (e.g. "Qwen3.5-397B-A17B-NVFP4-V2" vs # the real "nvidia/Qwen3.5-397B-A17B-NVFP4-V2"), which 404s tokenizer # loading. Always pass the real HF id explicitly. - REPLAY_CMD+=" --tokenizer $MODEL" + REPLAY_CMD+=" --tokenizer ${AIPERF_TOKENIZER:-$MODEL}" REPLAY_CMD+=" --concurrency $CONC" REPLAY_CMD+=" --benchmark-duration $duration" REPLAY_CMD+=" --stats-interval 30" diff --git a/benchmarks/single_node/agentic/qwen3.5_fp8_b200_sglang.sh b/benchmarks/single_node/agentic/qwen3.5_fp8_b200_sglang.sh new file mode 100644 index 0000000000..fa16ad4fed --- /dev/null +++ b/benchmarks/single_node/agentic/qwen3.5_fp8_b200_sglang.sh @@ -0,0 +1,2 @@ +#!/usr/bin/env bash +exec bash "$(dirname "$0")/qwen3.5_fp8_blackwell_powerx.sh" "$@" diff --git a/benchmarks/single_node/agentic/qwen3.5_fp8_b300_sglang.sh b/benchmarks/single_node/agentic/qwen3.5_fp8_b300_sglang.sh new file mode 100644 index 0000000000..fa16ad4fed --- /dev/null +++ b/benchmarks/single_node/agentic/qwen3.5_fp8_b300_sglang.sh @@ -0,0 +1,2 @@ +#!/usr/bin/env bash +exec bash "$(dirname "$0")/qwen3.5_fp8_blackwell_powerx.sh" "$@" diff --git a/benchmarks/single_node/agentic/qwen3.5_fp8_blackwell_powerx.sh b/benchmarks/single_node/agentic/qwen3.5_fp8_blackwell_powerx.sh new file mode 100644 index 0000000000..09fb3958ec --- /dev/null +++ b/benchmarks/single_node/agentic/qwen3.5_fp8_blackwell_powerx.sh @@ -0,0 +1,151 @@ +#!/usr/bin/env bash +set -euo pipefail +set -x + +# Controlled PowerX AgentX arm shared by B200 and B300: TP4, FP8, no MTP, +# with GPU prefix reuse and no CPU KV offload. +source "$(dirname "$0")/../../benchmark_lib.sh" + +check_env_vars TP CONC EP_SIZE KV_OFFLOADING RESULT_DIR DURATION +if [[ "$TP" != 4 || "$EP_SIZE" != 1 || "${PP_SIZE:-1}" != 1 || + "${DCP_SIZE:-1}" != 1 || "${PCP_SIZE:-1}" != 1 || + "${DP_ATTENTION:-false}" != false || "${SPEC_DECODING:-none}" != none ]]; then + echo "Error: the PowerX arm requires TP4/EP1, no context/pipeline/attention parallelism, and no MTP" >&2 + exit 1 +fi +require_agentic_kv_offload_none + +export MODEL="Qwen/Qwen3.5-397B-A17B-FP8" +export MODEL_REVISION="ea5b4f81096f3901c91dea97f81324302495781d" +export EVAL_FRAMEWORK="lm-eval" +export EVAL_TASKS_DIR="$INFERENCEX_REPO_ROOT/utils/evals/gsm8k.yaml" +export ENABLE_AGENTX_POWER=1 +export REQUIRE_POWER=1 +export AIPERF_FAILED_REQUEST_THRESHOLD=0 +export AIPERF_LIVE_FAILED_REQUEST_THRESHOLD=0 +export AIPERF_REQUIRED_SERVER_METRIC_PREFIX="sglang:" +export AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES=0 +export WEKA_LOADER_OVERRIDE=semianalysis_cc_traces_weka_062126_256k +for simulation_var in ${!SGLANG_SIMULATE_ACC_@}; do + unset "$simulation_var" +done + +mkdir -p "$RESULT_DIR" +SERVER_LOG="$RESULT_DIR/server.log" +install_agentic_deps +# Staged MODEL_PATH directories do not prove a checkpoint revision. Reuse a +# pinned HF cache snapshot when available, otherwise download that revision. +"$AIPERF_HF_CLI" download "$MODEL" --revision "$MODEL_REVISION" --dry-run \ + | tee "$RESULT_DIR/powerx_model_cache.txt" +MODEL_PATH=$("$AIPERF_HF_CLI" download "$MODEL" --revision "$MODEL_REVISION") +export MODEL_PATH +export AIPERF_TOKENIZER="$MODEL_PATH" +resolve_trace_source + +verify_trace_revision() { + "$AIPERF_PYTHON" - <<'PY' | tee -a "$RESULT_DIR/powerx_dataset_revision.txt" +from huggingface_hub import HfApi + +revision = HfApi().dataset_info("semianalysisai/cc-traces-weka-062126-256k").sha +print(revision) +if revision != "8fecd2fc56694469f758f0afbbb6335ad3043740": + raise SystemExit("PowerX AgentX dataset revision changed") +PY +} +verify_trace_revision + +{ + printf 'model=%s\nmodel_revision=%s\nmodel_path=%s\nimage=%s\n' \ + "$MODEL" "$MODEL_REVISION" "$MODEL_PATH" "${IMAGE:-unknown}" + printf 'inferencex_commit=%s\naiperf_commit=%s\n' \ + "$(git -c safe.directory="$INFERENCEX_REPO_ROOT" -C "$INFERENCEX_REPO_ROOT" rev-parse HEAD)" \ + "$(git -c safe.directory="$AIPERF_DIR" -C "$AIPERF_DIR" rev-parse HEAD)" + printf 'tp=%s\nep=%s\nconcurrency=%s\nduration=%s\nagentx_fast=%s\ncuda_visible_devices=%s\n' \ + "$TP" "$EP_SIZE" "$CONC" "$DURATION" "${AIPERF_EXPERIMENTAL_FAST:-0}" "${CUDA_VISIBLE_DEVICES:-unset}" + sha256sum "$MODEL_PATH/config.json" "$MODEL_PATH/tokenizer_config.json" + nvidia-smi --query-gpu=index,uuid,name,driver_version,power.limit --format=csv +} > "$RESULT_DIR/powerx_runtime.txt" +python3 -m pip freeze > "$RESULT_DIR/powerx_server_packages.txt" +"$AIPERF_UV_BIN" pip freeze --python "$AIPERF_PYTHON" > "$RESULT_DIR/powerx_client_packages.txt" + +# Concurrency counts session trees; retain room for subagent requests. +MAX_RUNNING_REQUESTS=$((2 * CONC)) +CUDA_GRAPH_MAX_BS="$CONC" +[ "$CUDA_GRAPH_MAX_BS" -gt 64 ] && CUDA_GRAPH_MAX_BS=64 + +export TORCH_CUDA_ARCH_LIST="10.0" +export PYTHONNOUSERSITE=1 +export NCCL_NVLS_ENABLE=1 +export SGL_ENABLE_JIT_DEEPGEMM=false +export SGLANG_ENABLE_FLASHINFER_GEMM=true +export SGLANG_TIMEOUT_KEEP_ALIVE=1800 + +SGLANG_CMD=( + python3 -m sglang.launch_server + --model-path "$MODEL_PATH" + --served-model-name "$MODEL" + --host 0.0.0.0 + --port "$PORT" + --trust-remote-code + --tp "$TP" + --dp 1 + --ep-size "$EP_SIZE" + --enable-symm-mem + --quantization fp8 + --kv-cache-dtype fp8_e4m3 + --mamba-ssm-dtype bfloat16 + --attention-backend trtllm_mha + --moe-runner-backend flashinfer_trtllm + --cuda-graph-max-bs "$CUDA_GRAPH_MAX_BS" + --max-running-requests "$MAX_RUNNING_REQUESTS" + --max-prefill-tokens 16384 + --chunked-prefill-size 16384 + --mem-fraction-static 0.80 + --stream-interval 50 + --scheduler-recv-interval 10 + --tokenizer-worker-num 6 + --tokenizer-path "$MODEL_PATH" + --reasoning-parser qwen3 + --tool-call-parser qwen3_coder + --enable-metrics + --enable-cache-report +) + +printf '%q ' "${SGLANG_CMD[@]}" | tee "$RESULT_DIR/sglang_command.txt" +printf '\n' | tee -a "$RESULT_DIR/sglang_command.txt" +SERVER_PID="" +cleanup_agentic_services() { + local exit_code=$? + trap - EXIT INT TERM + set +e + capture_cache_metrics + stop_background_process_tree "$SERVER_PID" "SGLang server" 60 + exit "$exit_code" +} +trap cleanup_agentic_services EXIT +trap 'exit 130' INT +trap 'exit 143' TERM +"${SGLANG_CMD[@]}" > "$SERVER_LOG" 2>&1 & +SERVER_PID=$! + +capture_cache_metrics() { + { + echo "=== SGLang cache metrics snapshot $(date --iso-8601=seconds) ===" + curl -fsS "http://localhost:$PORT/metrics" 2>/dev/null \ + | grep -E '^(sglang:(cache_hit_rate|cached_tokens_total|prompt_tokens_total|token_usage|num_requests_running|num_requests_waiting))' \ + || true + } >> "$SERVER_LOG" +} + +wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" +capture_cache_metrics + +if [ "${EVAL_ONLY:-false}" = "true" ]; then + run_eval --port "$PORT" +else + build_replay_cmd "$RESULT_DIR" + REPLAY_CMD+=" --apply-chat-template" + REPLAY_CMD+=" --server-metrics http://localhost:$PORT/metrics" + run_agentic_replay_and_write_outputs "$RESULT_DIR" + verify_trace_revision +fi diff --git a/benchmarks/single_node/agentic/qwen3.5_fp8_mi355x_sglang.sh b/benchmarks/single_node/agentic/qwen3.5_fp8_mi355x_sglang.sh new file mode 100755 index 0000000000..1549e48106 --- /dev/null +++ b/benchmarks/single_node/agentic/qwen3.5_fp8_mi355x_sglang.sh @@ -0,0 +1,132 @@ +#!/usr/bin/env bash +set -euo pipefail +set -x + +# Controlled PowerX AgentX replay: FP8 TP4, native decoding, HBM prefix reuse. +source "$(dirname "$0")/../../benchmark_lib.sh" + +check_env_vars MODEL TP CONC EP_SIZE KV_OFFLOADING RESULT_DIR DURATION +require_agentic_kv_offload_none +if [[ "$TP" != 4 || "$EP_SIZE" != 1 || "${SPEC_DECODING:-none}" != none ]]; then + echo "Error: PowerX requires TP4/EP1 without speculative decoding" >&2 + exit 1 +fi + +export EVAL_FRAMEWORK=lm-eval +export EVAL_TASKS_DIR="$INFERENCEX_REPO_ROOT/utils/evals/gsm8k.yaml" +export ENABLE_AGENTX_POWER=1 +export REQUIRE_POWER=1 +export AIPERF_FAILED_REQUEST_THRESHOLD=0 +export AIPERF_LIVE_FAILED_REQUEST_THRESHOLD=0 +export AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES=0 +export AIPERF_SERVER_METRICS_URLS="http://localhost:${PORT}/metrics" +export AIPERF_REQUIRED_SERVER_METRIC_PREFIX="sglang:" +export WEKA_LOADER_OVERRIDE=semianalysis_cc_traces_weka_062126_256k +unset SGLANG_SIMULATE_ACC_LEN SGLANG_SIMULATE_ACC_METHOD SGLANG_SIMULATE_ACC_TOKEN_MODE + +mkdir -p "$RESULT_DIR" +install_agentic_deps +guard_powerx_dataset_revision() { + "$AIPERF_PYTHON" - <<'PY' +from huggingface_hub import HfApi + +dataset = "semianalysisai/cc-traces-weka-062126-256k" +expected = "8fecd2fc56694469f758f0afbbb6335ad3043740" +actual = HfApi().dataset_info(dataset).sha +if actual != expected: + raise SystemExit(f"PowerX dataset revision changed: {actual} != {expected}") +print(f"{dataset}@{actual}") +PY +} +guard_powerx_dataset_revision > "$RESULT_DIR/powerx_dataset_revision.txt" +resolve_trace_source +export MODEL_REVISION=ea5b4f81096f3901c91dea97f81324302495781d +"$AIPERF_HF_CLI" download "$MODEL" --revision "$MODEL_REVISION" --dry-run \ + | tee "$RESULT_DIR/powerx_model_cache.txt" +MODEL_PATH=$("$AIPERF_HF_CLI" download "$MODEL" --revision "$MODEL_REVISION") +export MODEL_PATH +export AIPERF_TOKENIZER="$MODEL_PATH" + +SERVER_LOG="$RESULT_DIR/server.log" +{ + printf 'model=%s\nmodel_revision=%s\nmodel_path=%s\nimage=%s\n' \ + "$MODEL" "$MODEL_REVISION" "$MODEL_PATH" "${IMAGE:-unknown}" + printf 'ROCR_VISIBLE_DEVICES=%s\nHIP_VISIBLE_DEVICES=%s\nCUDA_VISIBLE_DEVICES=%s\n' \ + "${ROCR_VISIBLE_DEVICES:-}" "${HIP_VISIBLE_DEVICES:-}" "${CUDA_VISIBLE_DEVICES:-}" + git -c safe.directory="$INFERENCEX_REPO_ROOT" -C "$INFERENCEX_REPO_ROOT" rev-parse HEAD + git -c safe.directory="$AIPERF_DIR" -C "$AIPERF_DIR" rev-parse HEAD + sha256sum "$MODEL_PATH/config.json" "$MODEL_PATH/tokenizer_config.json" +} > "$RESULT_DIR/powerx_runtime.txt" +amd-smi static --json > "$RESULT_DIR/powerx_gpu_identity.json" +python3 -m pip freeze > "$RESULT_DIR/powerx_server_packages.txt" +"$AIPERF_UV_BIN" pip freeze --python "$AIPERF_PYTHON" > "$RESULT_DIR/powerx_client_packages.txt" + +SERVER_PID="" +cleanup_agentic_services() { + local exit_code=$? + trap - EXIT INT TERM + set +e + stop_background_process_tree "$SERVER_PID" "SGLang server" 60 + exit "$exit_code" +} +trap cleanup_agentic_services EXIT +trap 'exit 130' INT +trap 'exit 143' TERM + +MAX_RUNNING_REQUESTS=$((2 * CONC)) +CUDA_GRAPH_MAX_BS="$CONC" +[ "$CUDA_GRAPH_MAX_BS" -gt 64 ] && CUDA_GRAPH_MAX_BS=64 + +export PYTHONNOUSERSITE=1 +export SGLANG_USE_AITER=1 +export SGLANG_USE_AITER_UNIFIED_ATTN=1 +export SGLANG_MAMBA_SSM_DTYPE=bfloat16 +export SGLANG_TIMEOUT_KEEP_ALIVE=1800 + +SGLANG_CMD=( + python3 -m sglang.launch_server + --model-path "$MODEL_PATH" + --served-model-name "$MODEL" + --host 0.0.0.0 + --port "$PORT" + --trust-remote-code + --tp "$TP" + --dp 1 + --ep-size "$EP_SIZE" + --attention-backend aiter + --enable-aiter-allreduce-fusion + --quantization fp8 + --kv-cache-dtype fp8_e4m3 + --mamba-ssm-dtype bfloat16 + --mem-fraction-static 0.80 + --model-loader-extra-config '{"enable_multithread_load": true}' + --watchdog-timeout 1200 + --page-size 16 + --cuda-graph-max-bs "$CUDA_GRAPH_MAX_BS" + --max-running-requests "$MAX_RUNNING_REQUESTS" + --max-prefill-tokens 16384 + --chunked-prefill-size 16384 + --scheduler-recv-interval 10 + --stream-interval 50 + --tokenizer-worker-num 6 + --tokenizer-path "$MODEL_PATH" + --reasoning-parser qwen3 + --tool-call-parser qwen3_coder + --enable-metrics + --enable-cache-report +) + +printf '%q ' "${SGLANG_CMD[@]}" | tee "$RESULT_DIR/sglang_command.txt" +printf '\n' | tee -a "$RESULT_DIR/sglang_command.txt" +"${SGLANG_CMD[@]}" > "$SERVER_LOG" 2>&1 & +SERVER_PID=$! +wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" + +if [ "${EVAL_ONLY:-false}" = true ]; then + run_eval --port "$PORT" +else + build_replay_cmd "$RESULT_DIR" + REPLAY_CMD+=" --apply-chat-template" + run_agentic_replay_and_write_outputs "$RESULT_DIR" + guard_powerx_dataset_revision >> "$RESULT_DIR/powerx_dataset_revision.txt" +fi diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 0269ddfa05..68d7672423 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -200,6 +200,19 @@ qwen3.5-fp8-mi355x-sglang: search-space: - { tp: 4, ep: 1, conc-start: 4, conc-end: 256 } +qwen3.5-fp8-mi355x-sglang-agentic-powerx: + image: lmsysorg/sglang:v0.5.16-rocm720-mi35x@sha256:54ac680bad1832b8acd469533ae66f608b525cec3449bbd5f3d0238351e9b965 + model: Qwen/Qwen3.5-397B-A17B-FP8 + model-prefix: qwen3.5 + runner: cluster:mi355x-amds + precision: fp8 + framework: sglang + multinode: false + scenarios: + agentic-coding: + - search-space: + - { tp: 4, ep: 1, spec-decoding: none, kv-offloading: none, conc-list: [1, 4, 12] } + qwen3.5-fp8-mi355x-sglang-mtp: image: lmsysorg/sglang-rocm:v0.5.18-rocm720-mi35x-20260828 model: Qwen/Qwen3.5-397B-A17B-FP8 diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 0a6c86df6d..10d6c2bdd0 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -1215,6 +1215,20 @@ qwen3.5-fp8-b200-sglang: - { tp: 8, conc-start: 4, conc-end: 4 } - { tp: 4, ep: 1, conc-start: 4, conc-end: 256 } +# Controlled PowerX arm; AgentX matrix generation defaults to 3600 seconds. +qwen3.5-fp8-b200-sglang-agentic-powerx: + image: lmsysorg/sglang:v0.5.16-cu130@sha256:7b6a35df9839fd593a94a1eaee82d7777f472225d9f3ad1f8a2e0cb2bd1785d0 + model: Qwen/Qwen3.5-397B-A17B-FP8 + model-prefix: qwen3.5 + runner: cluster:b200-nscale + precision: fp8 + framework: sglang + multinode: false + scenarios: + agentic-coding: + - search-space: + - { tp: 4, ep: 1, spec-decoding: none, kv-offloading: none, conc-list: [1, 4, 12] } + qwen3.5-fp8-b200-sglang-agentic-mtp: image: lmsysorg/sglang:v0.5.16-cu130 model: Qwen/Qwen3.5-397B-A17B-FP8 @@ -7200,6 +7214,20 @@ qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg: dp-attn: false # ---------- 1k1k high-throughput (wide-EP decode, EAGLE MTP) ---------- +# Controlled PowerX arm; shares the B200 launcher and 3600-second default. +qwen3.5-fp8-b300-sglang-agentic-powerx: + image: lmsysorg/sglang:v0.5.16-cu130@sha256:7b6a35df9839fd593a94a1eaee82d7777f472225d9f3ad1f8a2e0cb2bd1785d0 + model: Qwen/Qwen3.5-397B-A17B-FP8 + model-prefix: qwen3.5 + runner: cluster:b300-dsxe + precision: fp8 + framework: sglang + multinode: false + scenarios: + agentic-coding: + - search-space: + - { tp: 4, ep: 1, spec-decoding: none, kv-offloading: none, conc-list: [1, 4, 12] } + qwen3.5-fp8-b300-sglang-agentic-mtp: image: lmsysorg/sglang:v0.5.16-cu130 model: Qwen/Qwen3.5-397B-A17B-FP8 diff --git a/docs/eval-agentx-procedures.md b/docs/eval-agentx-procedures.md index 469bd4aafd..8fb71f7b76 100644 --- a/docs/eval-agentx-procedures.md +++ b/docs/eval-agentx-procedures.md @@ -173,6 +173,8 @@ Retain `meta_env.json`, `results*.json`, and `sample*.jsonl`. Agentic SWE-bench ## 7. Run AgentX: fast feedback versus canonical evidence +The controlled PowerX keys `qwen3.5-fp8-{b200,b300,mi355x}-sglang-agentic-powerx` use TP4, native decoding, prefix reuse and no CPU offload. Qualify their serving and four-GPU power boundaries before collecting the configured one-hour runs. A recipe can set `AIPERF_TOKENIZER` to its pinned model snapshot so replay and serving use the same tokenizer; otherwise replay continues to use `MODEL`. + AgentX is AIPerf `inferencex-agentx-mvp` trace replay, not a fixed-token synthetic benchmark. The checked-in default uses ten additional warmup requests per trajectory lane and the recipe's configured profile duration. `agentx-fast` forces one warmup request per lane and a 1,200-second profile. It affects single- and multi-node AgentX throughput only. Fixed-sequence throughput and evals remain canonical. Fast runs are not eligible for artifact reuse ([workflow policy](../.github/workflows/README.md#agentx-fast-mode), [fast replay settings](../benchmarks/benchmark_lib.sh#L2104-L2128)). For multi-node srt-slurm jobs, the benchmark client may run on a different host from the frontend. `agentic_srt.sh` uses an explicit `AIPERF_SERVER_URL` when supplied, otherwise derives it from `SRT_FRONTEND_HOST` and `SRT_FRONTEND_PORT`, and falls back to `localhost:$PORT` only when no remote endpoint is available. Trace replay and inter-point drain checks must use that same resolved endpoint. @@ -338,3 +340,5 @@ Use `scancel` or process termination only with explicit approval and a concrete - Every backend/frontend and metrics source is represented in live evidence. - Fast/smoke results are labeled diagnostic. Only the canonical candidate is used for final comparison. - Workflow and artifact collection conclude green before success is reported. + +The PowerX runners convert Docker digest pins to Enroot manifest references at import time. B200 and B300 mount persistent Hugging Face caches, and the recipes record a pinned-model download dry run before loading weights. diff --git a/docs/eval-agentx-procedures_zh.md b/docs/eval-agentx-procedures_zh.md index 501344bf25..0618e2d685 100644 --- a/docs/eval-agentx-procedures_zh.md +++ b/docs/eval-agentx-procedures_zh.md @@ -173,6 +173,8 @@ gh run download "$RUN_ID" --repo SemiAnalysisAI/InferenceX \ ## 7. 运行 AgentX:快速反馈与 canonical 证据 +受控 PowerX 配置 `qwen3.5-fp8-{b200,b300,mi355x}-sglang-agentic-powerx` 使用 TP4、原生解码、前缀复用,且不启用 CPU offload。在采集配置规定的一小时结果前,先验证服务与四张 GPU 的功耗统计边界。recipe 可将 `AIPERF_TOKENIZER` 设为固定版本的模型快照路径,让 replay 与服务使用同一 tokenizer;未设置时仍使用 `MODEL`。 + AgentX 是 AIPerf `inferencex-agentx-mvp` trace replay,不是固定 token 的合成 benchmark。仓库默认设置对每条 trajectory lane 额外执行十个 warmup 请求,并使用 recipe 配置的 profile 时长。`agentx-fast` 强制每条 lane 只运行一个 warmup 请求,并将 profile 设为 1,200 秒。它只影响单节点和多节点 AgentX 吞吐量;定长序列吞吐量与 eval 保持 canonical。Fast 运行不符合 artifact reuse 条件([工作流策略](../.github/workflows/README.md#agentx-fast-mode)、[fast replay 设置](../benchmarks/benchmark_lib.sh#L2104-L2128))。 对于多节点 srt-slurm 作业,benchmark client 与 frontend 可能运行在不同主机上。`agentic_srt.sh` 会优先使用显式提供的 `AIPERF_SERVER_URL`;否则从 `SRT_FRONTEND_HOST` 和 `SRT_FRONTEND_PORT` 推导地址;仅在没有远端 endpoint 时回退到 `localhost:$PORT`。Trace replay 和并发点之间的 drain 检查必须使用同一个解析后的 endpoint。 @@ -338,3 +340,5 @@ gh run cancel --repo SemiAnalysisAI/InferenceX - 每个 backend/frontend 与 metrics source 都在实时证据中有所体现。 - Fast/smoke 结果明确标为诊断用途;只有 canonical candidate 用于最终比较。 - 在报告成功前,工作流与 artifact collection 均已得出 green 结论。 + +PowerX runner 在导入时将 Docker digest 转换为 Enroot manifest 引用。B200 和 B300 挂载持久化 Hugging Face 缓存,配置在加载权重前记录固定模型版本的下载预检查。 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 14434c76ad..9dddd51785 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6878,3 +6878,25 @@ description: - "Refresh the Kimi-K3 GB300 AgentX configurations with updated configs." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2811 + +- config-keys: + - qwen3.5-fp8-b200-sglang-agentic-powerx + - qwen3.5-fp8-b300-sglang-agentic-powerx + - qwen3.5-fp8-mi355x-sglang-agentic-powerx + scenario-type: + - agentic-coding + description: + - "Add controlled Qwen3.5 FP8 TP4 AgentX power configurations for B200, B300, and MI355X with prefix caching and no speculative decoding or CPU offload." + - "Pin serving images and model weights, require valid GPU power, and allow the replay tokenizer to use the pinned model snapshot." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2827 + +- config-keys: + - qwen3.5-fp8-b200-sglang-agentic-powerx + - qwen3.5-fp8-b300-sglang-agentic-powerx + - qwen3.5-fp8-mi355x-sglang-agentic-powerx + scenario-type: + - agentic-coding + description: + - "Fix digest-pinned image imports for the PowerX qualification runners by reusing the Enroot manifest-reference converter." + - "Mount persistent HF caches on the NVIDIA single-node runners and record pinned model cache requirements before download." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2827 diff --git a/runners/container_utils.sh b/runners/container_utils.sh new file mode 100644 index 0000000000..a1e6f209c8 --- /dev/null +++ b/runners/container_utils.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash + +# Enroot 3.x does not parse Docker's tag@digest syntax. For digest-pinned +# images, use its explicit registry syntax and pass the digest as the +# manifest reference so the import remains immutable. +enroot_uri_for_image() { + local image="$1" + local image_without_digest="$image" + local digest="" + local first_component registry repository repository_dir repository_name + + if [[ "$image" == *@sha256:* ]]; then + image_without_digest="${image%@*}" + digest="${image##*@}" + fi + + first_component="${image_without_digest%%/*}" + if [[ "$image_without_digest" == */* && ( "$first_component" == *.* || "$first_component" == *:* || "$first_component" == "localhost" ) ]]; then + registry="$first_component" + repository="${image_without_digest#*/}" + else + registry="registry-1.docker.io" + repository="$image_without_digest" + fi + + if [[ -z "$digest" ]]; then + if [[ "$registry" == "registry-1.docker.io" ]]; then + printf 'docker://%s\n' "$image" + else + printf 'docker://%s#%s\n' "$registry" "$repository" + fi + return + fi + + repository_dir="${repository%/*}" + repository_name="${repository##*/}" + repository_name="${repository_name%%:*}" + if [[ "$repository" == */* ]]; then + repository="${repository_dir}/${repository_name}" + else + repository="$repository_name" + fi + if [[ "$registry" == "registry-1.docker.io" && "$repository" != */* ]]; then + repository="library/$repository" + fi + + printf 'docker://%s#%s:%s\n' "$registry" "$repository" "$digest" +} diff --git a/runners/launch_b200-nscale-compat.sh b/runners/launch_b200-nscale-compat.sh index 15b4013aac..d93bed41ce 100644 --- a/runners/launch_b200-nscale-compat.sh +++ b/runners/launch_b200-nscale-compat.sh @@ -1,5 +1,7 @@ #!/usr/bin/bash +source "$(dirname "${BASH_SOURCE[0]}")/container_utils.sh" + # Compatibility launcher for B200 Nscale configurations that have not yet # moved to the native srt-slurm path in launch_b200-nscale-slurm.sh. SLURM_PARTITION="${SLURM_PARTITION:-batch_1}" @@ -570,11 +572,17 @@ else salloc --partition=$SLURM_PARTITION --account=$SLURM_ACCOUNT --gres=gpu:$GPU_COUNT --exclusive --mem=0 --time="$SALLOC_TIME_LIMIT" --no-shell --job-name="$RUNNER_NAME" JOB_ID=$(squeue --name="$RUNNER_NAME" -u "$USER" -h -o %A | head -n1) + # Keep pinned model snapshots and datasets across single-node jobs. + HF_HUB_CACHE_HOST_PATH="/data/home/sa-shared/gharunners/hf-hub-cache" + export HF_HUB_CACHE="${HF_HUB_CACHE:-/mnt/hf_hub_cache}" + mkdir -p "$HF_HUB_CACHE_HOST_PATH" + # Point the bench script at the resolved MODEL_PATH instead of # pulling from the HF hub cache. Bench scripts skip `hf download` when # MODEL is a local path. export MODEL="$MODEL_PATH" + ENROOT_IMAGE_URI=$(enroot_uri_for_image "$IMAGE") || exit 1 # Use flock to serialize concurrent imports to the same squash file # Override ENROOT_CACHE_PATH to avoid permission issues with system-wide cache on worker nodes srun --jobid=$JOB_ID bash -c " @@ -586,13 +594,13 @@ else echo 'Squash file already exists and is valid, skipping import' else rm -f \"$SQUASH_FILE\" - enroot import -o \"$SQUASH_FILE\" docker://$IMAGE + enroot import -o \"$SQUASH_FILE\" \"$ENROOT_IMAGE_URI\" fi " srun --jobid=$JOB_ID \ --container-image=$SQUASH_FILE \ - --container-mounts=$GITHUB_WORKSPACE:$CONTAINER_MOUNT_DIR,$MODEL_PATH:$MODEL_PATH,$AIPERF_MMAP_CACHE_HOST_PATH:/aiperf_mmap_cache \ + --container-mounts=$GITHUB_WORKSPACE:$CONTAINER_MOUNT_DIR,$MODEL_PATH:$MODEL_PATH,$HF_HUB_CACHE_HOST_PATH:$HF_HUB_CACHE,$AIPERF_MMAP_CACHE_HOST_PATH:/aiperf_mmap_cache \ --no-container-mount-home \ --container-workdir=$CONTAINER_MOUNT_DIR \ --no-container-entrypoint --export=ALL,PORT=8888,AIPERF_DATASET_MMAP_CACHE_DIR=/aiperf_mmap_cache \ diff --git a/runners/launch_b300-dsxe.sh b/runners/launch_b300-dsxe.sh index c229146bfa..3ba6f960dc 100755 --- a/runners/launch_b300-dsxe.sh +++ b/runners/launch_b300-dsxe.sh @@ -1,5 +1,7 @@ #!/usr/bin/bash +source "$(dirname "${BASH_SOURCE[0]}")/container_utils.sh" + # Launcher for the B300 DSXE Slurm cluster (dsxe-sa-b300-prd0), runners run as sa-gha-runner. # # Every cluster-specific fact lives in this block. The rest of the file is generic: @@ -87,6 +89,8 @@ import_squash_image() { local image_ref="$1" local sqsh="$2" local lock="${2}.lock" + local enroot_uri + enroot_uri=$(enroot_uri_for_image "$image_ref") || exit 1 if unsquashfs -l "$sqsh" > /dev/null 2>&1; then echo "Squash file already present, skipping import: $sqsh" @@ -102,7 +106,7 @@ import_squash_image() { exit 0 fi rm -f \"$sqsh\" - enroot import -o \"$sqsh\" \"docker://$image_ref\" + enroot import -o \"$sqsh\" \"$enroot_uri\" unsquashfs -l \"$sqsh\" > /dev/null " || { echo "Error: enroot import failed for $image_ref -> $sqsh" >&2; exit 1; } @@ -432,9 +436,10 @@ done find . -name '.nfs*' -delete 2>/dev/null || true else - # HF_HUB_CACHE is set to help with dataset download inside the container - # for eval jobs. - export HF_HUB_CACHE="$HOME/.cache/huggingface" + # Persist pinned snapshots and datasets even without the home mount. + HF_HUB_CACHE_HOST_PATH="/data/home/sa-gha-runner/hf-hub-cache" + export HF_HUB_CACHE="${HF_HUB_CACHE:-/mnt/hf_hub_cache}" + mkdir -p "$HF_HUB_CACHE_HOST_PATH" # MODEL stays the HF id for the client; MODEL_PATH is where the server reads # weights. Only the root holding MODEL_PATH is mounted -- mounting both roots @@ -499,6 +504,7 @@ else CONTAINER_MOUNTS=( "$GITHUB_WORKSPACE:$CONTAINER_MOUNT_DIR" "$MODEL_MOUNT_DIR:$MODEL_MOUNT_DIR" + "$HF_HUB_CACHE_HOST_PATH:$HF_HUB_CACHE" ) CONTAINER_MOUNTS_ARG=$(IFS=,; printf '%s' "${CONTAINER_MOUNTS[*]}") diff --git a/runners/launch_gb200-nv.sh b/runners/launch_gb200-nv.sh index 9799aa3b71..48a4049675 100755 --- a/runners/launch_gb200-nv.sh +++ b/runners/launch_gb200-nv.sh @@ -16,52 +16,7 @@ SQUASH_DIR="/mnt/lustre01/users-public/sa-shared" POWER_SRT_SLURM_URL="https://github.com/edwingao28/srt-slurm.git" POWER_SRT_SLURM_PIN="6fc1bed01a0b82dae0088a105c03ce0cfb353443" -# Enroot 3.x does not parse Docker's tag@digest syntax. For digest-pinned -# images, use its explicit registry syntax and pass the digest as the -# manifest reference so the import remains immutable. -enroot_uri_for_image() { - local image="$1" - local image_without_digest="$image" - local digest="" - local first_component registry repository repository_dir repository_name - - if [[ "$image" == *@sha256:* ]]; then - image_without_digest="${image%@*}" - digest="${image##*@}" - fi - - first_component="${image_without_digest%%/*}" - if [[ "$image_without_digest" == */* && ( "$first_component" == *.* || "$first_component" == *:* || "$first_component" == "localhost" ) ]]; then - registry="$first_component" - repository="${image_without_digest#*/}" - else - registry="registry-1.docker.io" - repository="$image_without_digest" - fi - - if [[ -z "$digest" ]]; then - if [[ "$registry" == "registry-1.docker.io" ]]; then - printf 'docker://%s\n' "$image" - else - printf 'docker://%s#%s\n' "$registry" "$repository" - fi - return - fi - - repository_dir="${repository%/*}" - repository_name="${repository##*/}" - repository_name="${repository_name%%:*}" - if [[ "$repository" == */* ]]; then - repository="${repository_dir}/${repository_name}" - else - repository="$repository_name" - fi - if [[ "$registry" == "registry-1.docker.io" && "$repository" != */* ]]; then - repository="library/$repository" - fi - - printf 'docker://%s#%s:%s\n' "$registry" "$repository" "$digest" -} +source "$(dirname "${BASH_SOURCE[0]}")/container_utils.sh" # Concurrent matrix jobs import to the same shared-FS squash path. # Serialize imports and atomically replace invalid images so readers never diff --git a/runners/launch_mi355x-amds.sh b/runners/launch_mi355x-amds.sh index 49c3af0cf5..78ec6b76c4 100644 --- a/runners/launch_mi355x-amds.sh +++ b/runners/launch_mi355x-amds.sh @@ -1,5 +1,7 @@ #!/usr/bin/env bash +source "$(dirname "${BASH_SOURCE[0]}")/container_utils.sh" + scancel_sync() { local jobid=$1 local timeout=${2:-600} @@ -271,6 +273,7 @@ else srun --jobid=$JOB_ID bash -c "docker stop \$(docker ps -a -q)" + ENROOT_IMAGE_URI=$(enroot_uri_for_image "$IMAGE") || exit 1 # Use flock to serialize concurrent imports to the same squash file srun --jobid=$JOB_ID bash -c " exec 9>\"$LOCK_FILE\" @@ -279,7 +282,7 @@ else echo 'Squash file already exists and is valid, skipping import' else rm -f \"$SQUASH_FILE\" - enroot import -o \"$SQUASH_FILE\" docker://$IMAGE + enroot import -o \"$SQUASH_FILE\" \"$ENROOT_IMAGE_URI\" fi " diff --git a/utils/test_container_utils.py b/utils/test_container_utils.py new file mode 100644 index 0000000000..b846fb403e --- /dev/null +++ b/utils/test_container_utils.py @@ -0,0 +1,53 @@ +"""Check image references against the Enroot 3.x consumer grammar.""" + +import subprocess +from pathlib import Path + +import pytest + + +HELPER = Path(__file__).resolve().parents[1] / "runners" / "container_utils.sh" +DIGEST = "sha256:" + "a" * 64 + +# Consumer grammar from NVIDIA/enroot v3.5.0 src/docker.sh:259-269. +# `tag` is passed directly to /v2//manifests/, so sha256: +# remains an immutable manifest reference without using Docker's @ syntax. +PARSE_URI = r""" +source "$1" +uri=$(enroot_uri_for_image "$2") || exit +reg_user="[[:alnum:]_.!~*\'()%\;:\&=+$,-@]+" +reg_registry="[^#]+" +reg_image="[[:lower:][:digit:]/._-]+" +reg_tag="[[:alnum:]._:-]+" +if [[ "${uri}" =~ ^docker://((${reg_user})@)?((${reg_registry})#)?(${reg_image})(:(${reg_tag}))?$ ]]; then + printf '%s\n' "${BASH_REMATCH[2]}" "${BASH_REMATCH[4]}" "${BASH_REMATCH[5]}" "${BASH_REMATCH[7]}" +else + exit 1 +fi +""" + + +@pytest.mark.parametrize( + ("image", "registry", "repository", "reference"), + [ + (f"lmsysorg/sglang:v0.5.16-cu130@{DIGEST}", "registry-1.docker.io", "lmsysorg/sglang", DIGEST), + (f"lmsysorg/sglang:v0.5.16-rocm720-mi35x@{DIGEST}", "registry-1.docker.io", "lmsysorg/sglang", DIGEST), + (f"lmsysorg/sglang@{DIGEST}", "registry-1.docker.io", "lmsysorg/sglang", DIGEST), + (f"ubuntu@{DIGEST}", "registry-1.docker.io", "library/ubuntu", DIGEST), + (f"ghcr.io/org/image:release@{DIGEST}", "ghcr.io", "org/image", DIGEST), + (f"localhost:5000/org/image:release@{DIGEST}", "localhost:5000", "org/image", DIGEST), + ("lmsysorg/sglang:v0.5.16-cu130", "", "lmsysorg/sglang", "v0.5.16-cu130"), + ("nvcr.io/nvidia/cuda:13.0", "nvcr.io", "nvidia/cuda", "13.0"), + ], +) +def test_enroot_reference_preserves_manifest_identity( + image: str, registry: str, repository: str, reference: str +) -> None: + result = subprocess.run( + ["bash", "-c", PARSE_URI, "bash", str(HELPER), image], + capture_output=True, + text=True, + check=False, + ) + assert result.returncode == 0, result.stderr + assert result.stdout.splitlines() == ["", registry, repository, reference]