diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep8-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep8-mtp-agentic.yaml index 221215b33f..4c5b5fe02a 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep8-mtp-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep8-mtp-agentic.yaml @@ -50,7 +50,7 @@ backend: type: vllm connector: null dp_launch_mode: per_node - prefill_environment: &worker_environment + decode_environment: &worker_environment HF_HUB_CACHE: /hf_hub_cache VLLM_ENGINE_READY_TIMEOUT_S: "3600" VLLM_RPC_TIMEOUT: "600000" @@ -66,7 +66,9 @@ backend: UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" UCX_TLS: "rc,cuda_copy" NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - decode_environment: *worker_environment + prefill_environment: + <<: *worker_environment + PYTORCH_CUDA_ALLOC_CONF: "expandable_segments:True" vllm_config: prefill: &dep8_config kv-transfer-config: '{"kv_connector":"NixlConnector","kv_role":"kv_both"}' @@ -116,7 +118,8 @@ benchmark: IS_MULTINODE: "true" AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_HTTP_TCP_USER_TIMEOUT: "900000" AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache HF_HUB_CACHE: /hf_hub_cache WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:kv_cache_usage_perc" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p1d-dep8-dep8-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p1d-dep8-dep8-mtp-agentic.yaml index 8b3ddfe146..0cb058b4df 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p1d-dep8-dep8-mtp-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p1d-dep8-dep8-mtp-agentic.yaml @@ -13,7 +13,7 @@ identity: dynamo: { version: "1.3.1", install: true } setup_script: vllm-container-deps.sh environment: { ETCD_LEASE_TTL: "7200" } -slurm: { time_limit: "8:00:00" } +slurm: { time_limit: "12:00:00" } health_check: { max_attempts: 2160, interval_seconds: 10 } resources: @@ -45,7 +45,7 @@ backend: type: vllm connector: null dp_launch_mode: per_node - prefill_environment: &worker_environment + decode_environment: &worker_environment HF_HUB_CACHE: /hf_hub_cache VLLM_ENGINE_READY_TIMEOUT_S: "3600" VLLM_RPC_TIMEOUT: "600000" @@ -61,7 +61,9 @@ backend: UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" UCX_TLS: "rc,cuda_copy" NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - decode_environment: *worker_environment + prefill_environment: + <<: *worker_environment + PYTORCH_CUDA_ALLOC_CONF: "expandable_segments:True" vllm_config: prefill: &dep8_config kv-transfer-config: '{"kv_connector":"NixlConnector","kv_role":"kv_both"}' @@ -111,7 +113,8 @@ benchmark: IS_MULTINODE: "true" AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_HTTP_TCP_USER_TIMEOUT: "900000" AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache HF_HUB_CACHE: /hf_hub_cache WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:kv_cache_usage_perc" diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 551dfd8a0c..31526dae4f 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -7978,6 +7978,7 @@ dsv4-fp4-gb200-dynamo-vllm-agentic-mtp-disagg: - search-space: # One two-tray DEP8 prefill and decode worker covers the balanced region. - spec-decoding: mtp + kv-offloading: none conc-list: [32, 64, 96, 128, 192, 256] prefill: num-worker: 1 @@ -7995,6 +7996,7 @@ dsv4-fp4-gb200-dynamo-vllm-agentic-mtp-disagg: dp-attn: true # A second DEP8 prefill worker extends the high-concurrency region. - spec-decoding: mtp + kv-offloading: none conc-list: [192, 256, 384, 512] prefill: num-worker: 2 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 73f2b38f7e..5a8991ac09 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6868,3 +6868,11 @@ description: - "Refresh the DeepSeek-V4-Pro FP4 B200 vLLM image and search space, pin the Nscale NVFP4 checkpoint, use the FlashInfer CuTeDSL MoE backend for expert-parallel runs with EPLB disabled, set GPU memory utilization to 0.95, and cap the model length at 12288." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2534 + +- config-keys: + - dsv4-fp4-gb200-dynamo-vllm-agentic-mtp-disagg + scenario-type: + - agentic-coding + description: + - "Refresh: we're refreshing this run with the correct metrics." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2822 diff --git a/runners/launch_gb200-nv.sh b/runners/launch_gb200-nv.sh index 9799aa3b71..c728cc09dd 100755 --- a/runners/launch_gb200-nv.sh +++ b/runners/launch_gb200-nv.sh @@ -420,6 +420,14 @@ elif [[ "$IS_AGENTIC" == "1" && "$MODEL_PREFIX" == "minimaxm3" && "$PRECISION" = mkdir -p recipes/vllm/minimax-m3/gb200-fp4/agentic cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic" \ recipes/vllm/minimax-m3/gb200-fp4/agentic +# Collect both nodes' DP metrics for resident DeepSeek-V4 disaggregation. +elif [[ "$IS_AGENTIC" == "1" && "$MODEL_PREFIX" == "dsv4" && "$PRECISION" == "fp4" && "$FRAMEWORK" == "dynamo-vllm" && "$DISAGG" == "true" && "$KV_OFFLOADING" == "none" ]]; then + git clone --branch fix/custom-vllm-dp-metrics --single-branch https://github.com/SemiAnalysisAI/srt-slurm.git "$SRT_REPO_DIR" || exit 1 + cd "$SRT_REPO_DIR" || exit 1 + git checkout bde8ca8a7a14e124531085cf0d908d3907b11ad6 || exit 1 + mkdir -p recipes/vllm/deepseek-v4/agentic + cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic" \ + recipes/vllm/deepseek-v4/agentic # These AgentX submissions use released srt-slurm custom-benchmark metrics # discovery so AIPerf receives every logical worker endpoint. elif [[ "$IS_AGENTIC" == "1" && (( "$MODEL_PREFIX" == "qwen3.5" && "$PRECISION" == "fp4" && "$FRAMEWORK" == "dynamo-sglang" ) || ( "$MODEL_PREFIX" == "dsv4" && "$PRECISION" == "fp4" && "$FRAMEWORK" == "dynamo-vllm" )) ]]; then