feat: add controlled AgentX power experiment / 添加受控 AgentX 功耗实验

benchmarks/benchmark_lib.sh

@@ -3297,7 +3297,7 @@ build_replay_cmd() {
     # necessarily a valid HF repo id (e.g. "Qwen3.5-397B-A17B-NVFP4-V2" vs
     # the real "nvidia/Qwen3.5-397B-A17B-NVFP4-V2"), which 404s tokenizer
     # loading. Always pass the real HF id explicitly.
-    REPLAY_CMD+=" --tokenizer $MODEL"
+    REPLAY_CMD+=" --tokenizer ${AIPERF_TOKENIZER:-$MODEL}"
     REPLAY_CMD+=" --concurrency $CONC"
     REPLAY_CMD+=" --benchmark-duration $duration"
     REPLAY_CMD+=" --stats-interval 30"

benchmarks/single_node/agentic/qwen3.5_fp8_b200_sglang.sh

@@ -0,0 +1,2 @@
+#!/usr/bin/env bash
+exec bash "$(dirname "$0")/qwen3.5_fp8_blackwell_powerx.sh" "$@"

benchmarks/single_node/agentic/qwen3.5_fp8_b300_sglang.sh

@@ -0,0 +1,2 @@
+#!/usr/bin/env bash
+exec bash "$(dirname "$0")/qwen3.5_fp8_blackwell_powerx.sh" "$@"

benchmarks/single_node/agentic/qwen3.5_fp8_blackwell_powerx.sh

@@ -0,0 +1,151 @@
+#!/usr/bin/env bash
+set -euo pipefail
+set -x
+
+# Controlled PowerX AgentX arm shared by B200 and B300: TP4, FP8, no MTP,
+# with GPU prefix reuse and no CPU KV offload.
+source "$(dirname "$0")/../../benchmark_lib.sh"
+
+check_env_vars TP CONC EP_SIZE KV_OFFLOADING RESULT_DIR DURATION
+if [[ "$TP" != 4 || "$EP_SIZE" != 1 || "${PP_SIZE:-1}" != 1 ||
+      "${DCP_SIZE:-1}" != 1 || "${PCP_SIZE:-1}" != 1 ||
+      "${DP_ATTENTION:-false}" != false || "${SPEC_DECODING:-none}" != none ]]; then
+    echo "Error: the PowerX arm requires TP4/EP1, no context/pipeline/attention parallelism, and no MTP" >&2
+    exit 1
+fi
+require_agentic_kv_offload_none
+
+export MODEL="Qwen/Qwen3.5-397B-A17B-FP8"
+export MODEL_REVISION="ea5b4f81096f3901c91dea97f81324302495781d"
+export EVAL_FRAMEWORK="lm-eval"
+export EVAL_TASKS_DIR="$INFERENCEX_REPO_ROOT/utils/evals/gsm8k.yaml"
+export ENABLE_AGENTX_POWER=1
+export REQUIRE_POWER=1
+export AIPERF_FAILED_REQUEST_THRESHOLD=0
+export AIPERF_LIVE_FAILED_REQUEST_THRESHOLD=0
+export AIPERF_REQUIRED_SERVER_METRIC_PREFIX="sglang:"
+export AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES=0
+export WEKA_LOADER_OVERRIDE=semianalysis_cc_traces_weka_062126_256k
+for simulation_var in ${!SGLANG_SIMULATE_ACC_@}; do
+    unset "$simulation_var"
+done
+
+mkdir -p "$RESULT_DIR"
+SERVER_LOG="$RESULT_DIR/server.log"
+install_agentic_deps
+# Staged MODEL_PATH directories do not prove a checkpoint revision. Reuse a
+# pinned HF cache snapshot when available, otherwise download that revision.
+"$AIPERF_HF_CLI" download "$MODEL" --revision "$MODEL_REVISION" --dry-run \
+    | tee "$RESULT_DIR/powerx_model_cache.txt"
+MODEL_PATH=$("$AIPERF_HF_CLI" download "$MODEL" --revision "$MODEL_REVISION")
+export MODEL_PATH
+export AIPERF_TOKENIZER="$MODEL_PATH"
+resolve_trace_source
+
+verify_trace_revision() {
+    "$AIPERF_PYTHON" - <<'PY' | tee -a "$RESULT_DIR/powerx_dataset_revision.txt"
+from huggingface_hub import HfApi
+
+revision = HfApi().dataset_info("semianalysisai/cc-traces-weka-062126-256k").sha
+print(revision)
+if revision != "8fecd2fc56694469f758f0afbbb6335ad3043740":
+    raise SystemExit("PowerX AgentX dataset revision changed")
+PY
+}
+verify_trace_revision
+
+{
+    printf 'model=%s\nmodel_revision=%s\nmodel_path=%s\nimage=%s\n' \
+        "$MODEL" "$MODEL_REVISION" "$MODEL_PATH" "${IMAGE:-unknown}"
+    printf 'inferencex_commit=%s\naiperf_commit=%s\n' \
+        "$(git -c safe.directory="$INFERENCEX_REPO_ROOT" -C "$INFERENCEX_REPO_ROOT" rev-parse HEAD)" \
+        "$(git -c safe.directory="$AIPERF_DIR" -C "$AIPERF_DIR" rev-parse HEAD)"
+    printf 'tp=%s\nep=%s\nconcurrency=%s\nduration=%s\nagentx_fast=%s\ncuda_visible_devices=%s\n' \
+        "$TP" "$EP_SIZE" "$CONC" "$DURATION" "${AIPERF_EXPERIMENTAL_FAST:-0}" "${CUDA_VISIBLE_DEVICES:-unset}"
+    sha256sum "$MODEL_PATH/config.json" "$MODEL_PATH/tokenizer_config.json"
+    nvidia-smi --query-gpu=index,uuid,name,driver_version,power.limit --format=csv
+} > "$RESULT_DIR/powerx_runtime.txt"
+python3 -m pip freeze > "$RESULT_DIR/powerx_server_packages.txt"
+"$AIPERF_UV_BIN" pip freeze --python "$AIPERF_PYTHON" > "$RESULT_DIR/powerx_client_packages.txt"
+
+# Concurrency counts session trees; retain room for subagent requests.
+MAX_RUNNING_REQUESTS=$((2 * CONC))
+CUDA_GRAPH_MAX_BS="$CONC"
+[ "$CUDA_GRAPH_MAX_BS" -gt 64 ] && CUDA_GRAPH_MAX_BS=64
+
+export TORCH_CUDA_ARCH_LIST="10.0"
+export PYTHONNOUSERSITE=1
+export NCCL_NVLS_ENABLE=1
+export SGL_ENABLE_JIT_DEEPGEMM=false
+export SGLANG_ENABLE_FLASHINFER_GEMM=true
+export SGLANG_TIMEOUT_KEEP_ALIVE=1800
+
+SGLANG_CMD=(
+    python3 -m sglang.launch_server
+    --model-path "$MODEL_PATH"
+    --served-model-name "$MODEL"
+    --host 0.0.0.0
+    --port "$PORT"
+    --trust-remote-code
+    --tp "$TP"
+    --dp 1
+    --ep-size "$EP_SIZE"
+    --enable-symm-mem
+    --quantization fp8
+    --kv-cache-dtype fp8_e4m3
+    --mamba-ssm-dtype bfloat16
+    --attention-backend trtllm_mha
+    --moe-runner-backend flashinfer_trtllm
+    --cuda-graph-max-bs "$CUDA_GRAPH_MAX_BS"
+    --max-running-requests "$MAX_RUNNING_REQUESTS"
+    --max-prefill-tokens 16384
+    --chunked-prefill-size 16384
+    --mem-fraction-static 0.80
+    --stream-interval 50
+    --scheduler-recv-interval 10
+    --tokenizer-worker-num 6
+    --tokenizer-path "$MODEL_PATH"
+    --reasoning-parser qwen3
+    --tool-call-parser qwen3_coder
+    --enable-metrics
+    --enable-cache-report
+)
+
+printf '%q ' "${SGLANG_CMD[@]}" | tee "$RESULT_DIR/sglang_command.txt"
+printf '\n' | tee -a "$RESULT_DIR/sglang_command.txt"
+SERVER_PID=""
+cleanup_agentic_services() {
+    local exit_code=$?
+    trap - EXIT INT TERM
+    set +e
+    capture_cache_metrics
+    stop_background_process_tree "$SERVER_PID" "SGLang server" 60
+    exit "$exit_code"
+}
+trap cleanup_agentic_services EXIT
+trap 'exit 130' INT
+trap 'exit 143' TERM
+"${SGLANG_CMD[@]}" > "$SERVER_LOG" 2>&1 &
+SERVER_PID=$!
+
+capture_cache_metrics() {
+    {
+        echo "=== SGLang cache metrics snapshot $(date --iso-8601=seconds) ==="
+        curl -fsS "http://localhost:$PORT/metrics" 2>/dev/null \
+            | grep -E '^(sglang:(cache_hit_rate|cached_tokens_total|prompt_tokens_total|token_usage|num_requests_running|num_requests_waiting))' \
+            || true
+    } >> "$SERVER_LOG"
+}
+
+wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID"
+capture_cache_metrics
+
+if [ "${EVAL_ONLY:-false}" = "true" ]; then
+    run_eval --port "$PORT"
+else
+    build_replay_cmd "$RESULT_DIR"
+    REPLAY_CMD+=" --apply-chat-template"
+    REPLAY_CMD+=" --server-metrics http://localhost:$PORT/metrics"
+    run_agentic_replay_and_write_outputs "$RESULT_DIR"
+    verify_trace_revision
+fi

benchmarks/single_node/agentic/qwen3.5_fp8_mi355x_sglang.sh

@@ -0,0 +1,132 @@
+#!/usr/bin/env bash
+set -euo pipefail
+set -x
+
+# Controlled PowerX AgentX replay: FP8 TP4, native decoding, HBM prefix reuse.
+source "$(dirname "$0")/../../benchmark_lib.sh"
+
+check_env_vars MODEL TP CONC EP_SIZE KV_OFFLOADING RESULT_DIR DURATION
+require_agentic_kv_offload_none
+if [[ "$TP" != 4 || "$EP_SIZE" != 1 || "${SPEC_DECODING:-none}" != none ]]; then
+    echo "Error: PowerX requires TP4/EP1 without speculative decoding" >&2
+    exit 1
+fi
+
+export EVAL_FRAMEWORK=lm-eval
+export EVAL_TASKS_DIR="$INFERENCEX_REPO_ROOT/utils/evals/gsm8k.yaml"
+export ENABLE_AGENTX_POWER=1
+export REQUIRE_POWER=1
+export AIPERF_FAILED_REQUEST_THRESHOLD=0
+export AIPERF_LIVE_FAILED_REQUEST_THRESHOLD=0
+export AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES=0
+export AIPERF_SERVER_METRICS_URLS="http://localhost:${PORT}/metrics"
+export AIPERF_REQUIRED_SERVER_METRIC_PREFIX="sglang:"
+export WEKA_LOADER_OVERRIDE=semianalysis_cc_traces_weka_062126_256k
+unset SGLANG_SIMULATE_ACC_LEN SGLANG_SIMULATE_ACC_METHOD SGLANG_SIMULATE_ACC_TOKEN_MODE
+
+mkdir -p "$RESULT_DIR"
+install_agentic_deps
+guard_powerx_dataset_revision() {
+    "$AIPERF_PYTHON" - <<'PY'
+from huggingface_hub import HfApi
+
+dataset = "semianalysisai/cc-traces-weka-062126-256k"
+expected = "8fecd2fc56694469f758f0afbbb6335ad3043740"
+actual = HfApi().dataset_info(dataset).sha
+if actual != expected:
+    raise SystemExit(f"PowerX dataset revision changed: {actual} != {expected}")
+print(f"{dataset}@{actual}")
+PY
+}
+guard_powerx_dataset_revision > "$RESULT_DIR/powerx_dataset_revision.txt"
+resolve_trace_source
+export MODEL_REVISION=ea5b4f81096f3901c91dea97f81324302495781d
+"$AIPERF_HF_CLI" download "$MODEL" --revision "$MODEL_REVISION" --dry-run \
+    | tee "$RESULT_DIR/powerx_model_cache.txt"
+MODEL_PATH=$("$AIPERF_HF_CLI" download "$MODEL" --revision "$MODEL_REVISION")
+export MODEL_PATH
+export AIPERF_TOKENIZER="$MODEL_PATH"
+
+SERVER_LOG="$RESULT_DIR/server.log"
+{
+    printf 'model=%s\nmodel_revision=%s\nmodel_path=%s\nimage=%s\n' \
+        "$MODEL" "$MODEL_REVISION" "$MODEL_PATH" "${IMAGE:-unknown}"
+    printf 'ROCR_VISIBLE_DEVICES=%s\nHIP_VISIBLE_DEVICES=%s\nCUDA_VISIBLE_DEVICES=%s\n' \
+        "${ROCR_VISIBLE_DEVICES:-}" "${HIP_VISIBLE_DEVICES:-}" "${CUDA_VISIBLE_DEVICES:-}"
+    git -c safe.directory="$INFERENCEX_REPO_ROOT" -C "$INFERENCEX_REPO_ROOT" rev-parse HEAD
+    git -c safe.directory="$AIPERF_DIR" -C "$AIPERF_DIR" rev-parse HEAD
+    sha256sum "$MODEL_PATH/config.json" "$MODEL_PATH/tokenizer_config.json"
+} > "$RESULT_DIR/powerx_runtime.txt"
+amd-smi static --json > "$RESULT_DIR/powerx_gpu_identity.json"
+python3 -m pip freeze > "$RESULT_DIR/powerx_server_packages.txt"
+"$AIPERF_UV_BIN" pip freeze --python "$AIPERF_PYTHON" > "$RESULT_DIR/powerx_client_packages.txt"
+
+SERVER_PID=""
+cleanup_agentic_services() {
+    local exit_code=$?
+    trap - EXIT INT TERM
+    set +e
+    stop_background_process_tree "$SERVER_PID" "SGLang server" 60
+    exit "$exit_code"
+}
+trap cleanup_agentic_services EXIT
+trap 'exit 130' INT
+trap 'exit 143' TERM
+
+MAX_RUNNING_REQUESTS=$((2 * CONC))
+CUDA_GRAPH_MAX_BS="$CONC"
+[ "$CUDA_GRAPH_MAX_BS" -gt 64 ] && CUDA_GRAPH_MAX_BS=64
+
+export PYTHONNOUSERSITE=1
+export SGLANG_USE_AITER=1
+export SGLANG_USE_AITER_UNIFIED_ATTN=1
+export SGLANG_MAMBA_SSM_DTYPE=bfloat16
+export SGLANG_TIMEOUT_KEEP_ALIVE=1800
+
+SGLANG_CMD=(
+    python3 -m sglang.launch_server
+    --model-path "$MODEL_PATH"
+    --served-model-name "$MODEL"
+    --host 0.0.0.0
+    --port "$PORT"
+    --trust-remote-code
+    --tp "$TP"
+    --dp 1
+    --ep-size "$EP_SIZE"
+    --attention-backend aiter
+    --enable-aiter-allreduce-fusion
+    --quantization fp8
+    --kv-cache-dtype fp8_e4m3
+    --mamba-ssm-dtype bfloat16
+    --mem-fraction-static 0.80
+    --model-loader-extra-config '{"enable_multithread_load": true}'
+    --watchdog-timeout 1200
+    --page-size 16
+    --cuda-graph-max-bs "$CUDA_GRAPH_MAX_BS"
+    --max-running-requests "$MAX_RUNNING_REQUESTS"
+    --max-prefill-tokens 16384
+    --chunked-prefill-size 16384
+    --scheduler-recv-interval 10
+    --stream-interval 50
+    --tokenizer-worker-num 6
+    --tokenizer-path "$MODEL_PATH"
+    --reasoning-parser qwen3
+    --tool-call-parser qwen3_coder
+    --enable-metrics
+    --enable-cache-report
+)
+
+printf '%q ' "${SGLANG_CMD[@]}" | tee "$RESULT_DIR/sglang_command.txt"
+printf '\n' | tee -a "$RESULT_DIR/sglang_command.txt"
+"${SGLANG_CMD[@]}" > "$SERVER_LOG" 2>&1 &
+SERVER_PID=$!
+wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID"
+
+if [ "${EVAL_ONLY:-false}" = true ]; then
+    run_eval --port "$PORT"
+else
+    build_replay_cmd "$RESULT_DIR"
+    REPLAY_CMD+=" --apply-chat-template"
+    run_agentic_replay_and_write_outputs "$RESULT_DIR"
+    guard_powerx_dataset_revision >> "$RESULT_DIR/powerx_dataset_revision.txt"
+fi

configs/amd-master.yaml

@@ -200,6 +200,19 @@ qwen3.5-fp8-mi355x-sglang:
       search-space:
       - { tp: 4, ep: 1, conc-start: 4, conc-end: 256 }
 
+qwen3.5-fp8-mi355x-sglang-agentic-powerx:
+  image: lmsysorg/sglang:v0.5.16-rocm720-mi35x@sha256:54ac680bad1832b8acd469533ae66f608b525cec3449bbd5f3d0238351e9b965
+  model: Qwen/Qwen3.5-397B-A17B-FP8
+  model-prefix: qwen3.5
+  runner: cluster:mi355x-amds
+  precision: fp8
+  framework: sglang
+  multinode: false
+  scenarios:
+    agentic-coding:
+    - search-space:
+      - { tp: 4, ep: 1, spec-decoding: none, kv-offloading: none, conc-list: [1, 4, 12] }
+
 qwen3.5-fp8-mi355x-sglang-mtp:
   image: lmsysorg/sglang-rocm:v0.5.18-rocm720-mi35x-20260828
   model: Qwen/Qwen3.5-397B-A17B-FP8

configs/nvidia-master.yaml

@@ -1215,6 +1215,20 @@ qwen3.5-fp8-b200-sglang:
       - { tp: 8, conc-start: 4, conc-end: 4 }
       - { tp: 4, ep: 1, conc-start: 4, conc-end: 256 }
 
+# Controlled PowerX arm; AgentX matrix generation defaults to 3600 seconds.
+qwen3.5-fp8-b200-sglang-agentic-powerx:
+  image: lmsysorg/sglang:v0.5.16-cu130@sha256:7b6a35df9839fd593a94a1eaee82d7777f472225d9f3ad1f8a2e0cb2bd1785d0
+  model: Qwen/Qwen3.5-397B-A17B-FP8
+  model-prefix: qwen3.5
+  runner: cluster:b200-nscale
+  precision: fp8
+  framework: sglang
+  multinode: false
+  scenarios:
+    agentic-coding:
+    - search-space:
+      - { tp: 4, ep: 1, spec-decoding: none, kv-offloading: none, conc-list: [1, 4, 12] }
+
 qwen3.5-fp8-b200-sglang-agentic-mtp:
   image: lmsysorg/sglang:v0.5.16-cu130
   model: Qwen/Qwen3.5-397B-A17B-FP8
@@ -7200,6 +7214,20 @@ qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg:
           dp-attn: false
 
     # ---------- 1k1k high-throughput (wide-EP decode, EAGLE MTP) ----------
+# Controlled PowerX arm; shares the B200 launcher and 3600-second default.
+qwen3.5-fp8-b300-sglang-agentic-powerx:
+  image: lmsysorg/sglang:v0.5.16-cu130@sha256:7b6a35df9839fd593a94a1eaee82d7777f472225d9f3ad1f8a2e0cb2bd1785d0
+  model: Qwen/Qwen3.5-397B-A17B-FP8
+  model-prefix: qwen3.5
+  runner: cluster:b300-dsxe
+  precision: fp8
+  framework: sglang
+  multinode: false
+  scenarios:
+    agentic-coding:
+    - search-space:
+      - { tp: 4, ep: 1, spec-decoding: none, kv-offloading: none, conc-list: [1, 4, 12] }
+
 qwen3.5-fp8-b300-sglang-agentic-mtp:
   image: lmsysorg/sglang:v0.5.16-cu130
   model: Qwen/Qwen3.5-397B-A17B-FP8

docs/eval-agentx-procedures.md

@@ -173,6 +173,8 @@ Retain `meta_env.json`, `results*.json`, and `sample*.jsonl`. Agentic SWE-bench
 
 ## 7. Run AgentX: fast feedback versus canonical evidence
 
+The controlled PowerX keys `qwen3.5-fp8-{b200,b300,mi355x}-sglang-agentic-powerx` use TP4, native decoding, prefix reuse and no CPU offload. Qualify their serving and four-GPU power boundaries before collecting the configured one-hour runs. A recipe can set `AIPERF_TOKENIZER` to its pinned model snapshot so replay and serving use the same tokenizer; otherwise replay continues to use `MODEL`.
+
 AgentX is AIPerf `inferencex-agentx-mvp` trace replay, not a fixed-token synthetic benchmark. The checked-in default uses ten additional warmup requests per trajectory lane and the recipe's configured profile duration. `agentx-fast` forces one warmup request per lane and a 1,200-second profile. It affects single- and multi-node AgentX throughput only. Fixed-sequence throughput and evals remain canonical. Fast runs are not eligible for artifact reuse ([workflow policy](../.github/workflows/README.md#agentx-fast-mode), [fast replay settings](../benchmarks/benchmark_lib.sh#L2104-L2128)).
 
 For multi-node srt-slurm jobs, the benchmark client may run on a different host from the frontend. `agentic_srt.sh` uses an explicit `AIPERF_SERVER_URL` when supplied, otherwise derives it from `SRT_FRONTEND_HOST` and `SRT_FRONTEND_PORT`, and falls back to `localhost:$PORT` only when no remote endpoint is available. Trace replay and inter-point drain checks must use that same resolved endpoint.
@@ -338,3 +340,5 @@ Use `scancel` or process termination only with explicit approval and a concrete
 - Every backend/frontend and metrics source is represented in live evidence.
 - Fast/smoke results are labeled diagnostic. Only the canonical candidate is used for final comparison.
 - Workflow and artifact collection conclude green before success is reported.
+
+The PowerX runners convert Docker digest pins to Enroot manifest references at import time. B200 and B300 mount persistent Hugging Face caches, and the recipes record a pinned-model download dry run before loading weights.

docs/eval-agentx-procedures_zh.md

@@ -173,6 +173,8 @@ gh run download "$RUN_ID" --repo SemiAnalysisAI/InferenceX \
 
 ## 7. 运行 AgentX：快速反馈与 canonical 证据
 
+受控 PowerX 配置 `qwen3.5-fp8-{b200,b300,mi355x}-sglang-agentic-powerx` 使用 TP4、原生解码、前缀复用，且不启用 CPU offload。在采集配置规定的一小时结果前，先验证服务与四张 GPU 的功耗统计边界。recipe 可将 `AIPERF_TOKENIZER` 设为固定版本的模型快照路径，让 replay 与服务使用同一 tokenizer；未设置时仍使用 `MODEL`。
+
 AgentX 是 AIPerf `inferencex-agentx-mvp` trace replay，不是固定 token 的合成 benchmark。仓库默认设置对每条 trajectory lane 额外执行十个 warmup 请求，并使用 recipe 配置的 profile 时长。`agentx-fast` 强制每条 lane 只运行一个 warmup 请求，并将 profile 设为 1,200 秒。它只影响单节点和多节点 AgentX 吞吐量；定长序列吞吐量与 eval 保持 canonical。Fast 运行不符合 artifact reuse 条件（[工作流策略](../.github/workflows/README.md#agentx-fast-mode)、[fast replay 设置](../benchmarks/benchmark_lib.sh#L2104-L2128)）。
 
 对于多节点 srt-slurm 作业，benchmark client 与 frontend 可能运行在不同主机上。`agentic_srt.sh` 会优先使用显式提供的 `AIPERF_SERVER_URL`；否则从 `SRT_FRONTEND_HOST` 和 `SRT_FRONTEND_PORT` 推导地址；仅在没有远端 endpoint 时回退到 `localhost:$PORT`。Trace replay 和并发点之间的 drain 检查必须使用同一个解析后的 endpoint。
@@ -338,3 +340,5 @@ gh run cancel <RUN_ID> --repo SemiAnalysisAI/InferenceX
 - 每个 backend/frontend 与 metrics source 都在实时证据中有所体现。
 - Fast/smoke 结果明确标为诊断用途；只有 canonical candidate 用于最终比较。
 - 在报告成功前，工作流与 artifact collection 均已得出 green 结论。
+
+PowerX runner 在导入时将 Docker digest 转换为 Enroot manifest 引用。B200 和 B300 挂载持久化 Hugging Face 缓存，配置在加载权重前记录固定模型版本的下载预检查。