From abc34805f6d5ed860902b90e0e549b9286b16988 Mon Sep 17 00:00:00 2001 From: Po-Han Huang Date: Thu, 3 Sep 2026 18:55:49 -0700 Subject: [PATCH 1/2] feat(sglang): use DSpark for DSV4 B200 AgentX MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Move the DeepSeek-V4-Pro B200 SGLang AgentX configuration to the 0813 checkpoint and DSpark6, update the DP-attention flags, and use golden acceptance length 3.77.\n\n将 DeepSeek-V4-Pro B200 SGLang AgentX 配置更新到 0813 checkpoint 和 DSpark6，并更新 DP-attention 参数及 3.77 的黄金接受长度。 --- .../agentic/dsv4_fp4_b200_sglang_mtp.sh | 15 ++++++++------- configs/nvidia-master.yaml | 8 ++++---- perf-changelog.yaml | 11 +++++++++++ runners/launch_b200-nscale-compat.sh | 4 +++- 4 files changed, 26 insertions(+), 12 deletions(-) diff --git a/benchmarks/single_node/agentic/dsv4_fp4_b200_sglang_mtp.sh b/benchmarks/single_node/agentic/dsv4_fp4_b200_sglang_mtp.sh index 27a52dc325..060b590ae7 100755 --- a/benchmarks/single_node/agentic/dsv4_fp4_b200_sglang_mtp.sh +++ b/benchmarks/single_node/agentic/dsv4_fp4_b200_sglang_mtp.sh @@ -2,7 +2,7 @@ set -eo pipefail set -x -# Agentic trace replay for DeepSeek-V4-Pro FP4 on B200 with native EAGLE MTP. +# Agentic trace replay for DeepSeek-V4-Pro-0813 FP4 on B200 with DSpark K=6. # Throughput uses the committed golden synthetic AL; eval retains real target # verification. # @@ -110,8 +110,6 @@ CHUNKED_PREFILL_SIZE=8192 SWA_FULL_TOKENS_RATIO=0.1 MEM_FRACTION_STATIC=0.90 if [ "$DP_ATTENTION" = "true" ]; then - export SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS=1 - export SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND=1 export SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK=8320 # Leave HBM headroom for the FP4 indexer's context-dependent workspace. @@ -132,12 +130,14 @@ if [ "$DP_ATTENTION" = "true" ]; then --tokenizer-worker-num "$TP" --prefill-decode-interval "$PREFILL_DECODE_INTERVAL" --enable-dp-attention + --enable-dp-lm-head --enable-dp-attention-local-control-broadcast --incremental-streaming-output --stream-interval 20 --dist-init-addr "127.0.0.1:$((PORT + 2000))" --ep-size "$EP_SIZE" --moe-a2a-backend megamoe + --enable-w4a4-mxfp4-megamoe --enable-deepseek-v4-fp4-indexer --disable-shared-experts-fusion --disable-flashinfer-autotune @@ -188,7 +188,7 @@ export SGLANG_OPT_USE_JIT_INDEXER_METADATA=1 export SGLANG_OPT_USE_TOPK_V2=1 export SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2=1 if [ "${EVAL_ONLY}" != "true" ]; then - export SGLANG_SIMULATE_ACC_LEN=2.49 + export SGLANG_SIMULATE_ACC_LEN=3.77 export SGLANG_SIMULATE_ACC_METHOD=match-expected export SGLANG_SIMULATE_ACC_TOKEN_MODE=real-draft-token fi @@ -218,10 +218,11 @@ SGLANG_CMD=( --reasoning-parser deepseek-v4 --chat-template "$SCRIPT_DIR/../chat_templates/deepseek_v4_thinking.jinja" --watchdog-timeout 1800 - --speculative-algorithm EAGLE - --speculative-num-steps 3 + --speculative-algorithm DSPARK + --speculative-dspark-block-size 6 + --speculative-num-steps 1 --speculative-eagle-topk 1 - --speculative-num-draft-tokens 4 + --speculative-num-draft-tokens 7 # The B200 checkpoint lives on Lustre. Partition sequential prefetching # across local ranks so post-load weight repacking reads from page cache # instead of issuing redundant fragmented mmap faults from every rank. diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 551dfd8a0c..5659cfdd4b 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -930,7 +930,7 @@ dsv4-fp4-b200-sglang: dsv4-fp4-b200-sglang-agentic-hicache-mtp: image: lmsysorg/sglang:nightly-dev-20260827-20621aa1 - model: deepseek-ai/DeepSeek-V4-Pro + model: deepseek-ai/DeepSeek-V4-Pro-0813 model-prefix: dsv4 runner: cluster:b200-nscale precision: fp4 @@ -940,9 +940,9 @@ dsv4-fp4-b200-sglang-agentic-hicache-mtp: agentic-coding: - dram-utilization: 0.80 search-space: - - { tp: 8, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 3, 4, 5] } - - { tp: 8, kv-offloading: dram, kv-offload-backend: { name: hicache }, spec-decoding: mtp, conc-list: [8, 10, 16] } - - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: hicache }, spec-decoding: mtp, conc-list: [64, 96, 128, 160], router: { name: sglang-router, version: "0.3.2" } } + - { tp: 8, kv-offloading: none, spec-decoding: draft_model, conc-list: [1, 2, 3, 4, 5] } + - { tp: 8, kv-offloading: dram, kv-offload-backend: { name: hicache }, spec-decoding: draft_model, conc-list: [8, 10, 16] } + - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: hicache }, spec-decoding: draft_model, conc-list: [64, 96, 128, 160], router: { name: sglang-router, version: "0.3.2" } } dsv4-fp4-b200-vllm: image: vllm/vllm-openai:nightly-7c5dc571cbd1064ecc8a9b1045637ff647aa22cb diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 73f2b38f7e..004180e676 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6868,3 +6868,14 @@ description: - "Refresh the DeepSeek-V4-Pro FP4 B200 vLLM image and search space, pin the Nscale NVFP4 checkpoint, use the FlashInfer CuTeDSL MoE backend for expert-parallel runs with EPLB disabled, set GPU memory utilization to 0.95, and cap the model length at 12288." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2534 + +- config-keys: + - dsv4-fp4-b200-sglang-agentic-hicache-mtp + scenario-type: + - agentic-coding + description: + - "Replace the B200 DeepSeek-V4-Pro AgentX native EAGLE/MTP serving path with DSpark block size 6 on the DeepSeek-V4-Pro-0813 checkpoint." + - "Use one speculative step, seven draft tokens, and thinking-on golden synthetic acceptance length 3.77 for throughput; eval retains real DSpark verification." + - "Use SGLang's current W4A4 MegaMoE and DP LM-head flags for DP-attention instead of the deprecated MegaMoE environment variables." + - "Resolve draft-model jobs through the existing B200 SGLang speculative recipe and the staged DeepSeek-V4-Pro-0813 checkpoint." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX diff --git a/runners/launch_b200-nscale-compat.sh b/runners/launch_b200-nscale-compat.sh index 15b4013aac..b6f722bead 100644 --- a/runners/launch_b200-nscale-compat.sh +++ b/runners/launch_b200-nscale-compat.sh @@ -20,6 +20,8 @@ if [[ $MODEL_PREFIX == "dsr1" && $PRECISION == "fp4" ]]; then elif [[ $MODEL_PREFIX == "dsr1" && $PRECISION == "fp8" ]]; then export MODEL_PATH="/scratch/models/DeepSeek-R1-0528" export SRT_SLURM_MODEL_PREFIX="dsr1-fp8" +elif [[ $MODEL_PREFIX == "dsv4" && $PRECISION == "fp4" && $MODEL == "deepseek-ai/DeepSeek-V4-Pro-0813" ]]; then + export MODEL_PATH="${MODEL_PATH:-/scratch/models/DeepSeek-V4-Pro-0813}" elif [[ $MODEL_PREFIX == "dsv4" && $PRECISION == "fp4" ]]; then # Node-local weights are not visible on the runner/login node. export MODEL_PATH="/scratch/models/DeepSeek-V4-Pro-NVFP4" @@ -540,7 +542,7 @@ else SQUASH_FILE="/data/home/sa-shared/containers/$(echo "$IMAGE" | sed 's/[\/:@#]/_/g').sqsh" FRAMEWORK_SUFFIX=$([[ "$FRAMEWORK" == "trt" ]] && printf '_trt' || printf '') - SPEC_SUFFIX=$([[ "$SPEC_DECODING" == "mtp" ]] && printf '_mtp' || printf '') + SPEC_SUFFIX=$([[ "$SPEC_DECODING" == "mtp" || "$SPEC_DECODING" == "draft_model" ]] && printf '_mtp' || printf '') # Prefer a framework-tagged script (e.g. dsv4_fp4_b200_vllm.sh) so models # with multiple inference engines can coexist; fall back to the historical # name without an engine suffix (`_trt` for trt, bare for everyone else). From 874e8a66297b79ff38fcb9b608396c35430e24cd Mon Sep 17 00:00:00 2001 From: Po-Han Huang Date: Thu, 3 Sep 2026 18:59:08 -0700 Subject: [PATCH 2/2] chore: link perf changelog to #2821 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Backfill the public pull request URL in the DSV4 DSpark changelog entry.\n\n在 DSV4 DSpark changelog 条目中回填公开 pull request URL。 --- perf-changelog.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 004180e676..a7e34f722c 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6878,4 +6878,4 @@ - "Use one speculative step, seven draft tokens, and thinking-on golden synthetic acceptance length 3.77 for throughput; eval retains real DSpark verification." - "Use SGLang's current W4A4 MegaMoE and DP LM-head flags for DP-attention instead of the deprecated MegaMoE environment variables." - "Resolve draft-model jobs through the existing B200 SGLang speculative recipe and the staged DeepSeek-V4-Pro-0813 checkpoint." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2821