diff --git a/benchmarks/single_node/agentic/glm5.2_fp4_b200_sglang_mtp.sh b/benchmarks/single_node/agentic/glm5.2_fp4_b200_sglang_mtp.sh index f3b6821d7..e37289864 100755 --- a/benchmarks/single_node/agentic/glm5.2_fp4_b200_sglang_mtp.sh +++ b/benchmarks/single_node/agentic/glm5.2_fp4_b200_sglang_mtp.sh @@ -19,9 +19,10 @@ set -x # DP_ATTENTION=false -> low-latency arm (TP8, fp8 KV, cutedsl bf16 GEMM) # DP_ATTENTION=true -> high-throughput DEP arm (TP8 + DP8 attention-DP + # EP_SIZE expert-parallel MoE via --ep-size) -# Only the low-latency arm is wired into the master config for this MTP recipe -# (see the entry comment on glm5.2-fp4-b200-sglang-agentic-mtp); the DEP branch -# is kept intact so the throughput arm can be added without re-deriving it. +# The master config wires three arms (see the entry comment on +# glm5.2-fp4-b200-sglang-agentic-mtp): low-latency TP8 through conc 20, then +# EP8 (EP_SIZE=8, DP_ATTENTION=false) and DEP8 (EP_SIZE=8, DP_ATTENTION=true) +# from conc 20 to 48. # # Required env vars: # MODEL, TP, CONC, KV_OFFLOADING, TOTAL_CPU_DRAM_GB, RESULT_DIR, DURATION, @@ -120,9 +121,11 @@ if require_agentic_kv_offload_backend hicache; then # whole history; the host tier restores those hits at C2C bandwidth. # GLM-5.2 is DSA/MLA-family (attention_backend=dsa): every TP rank holds # complete per-token KV. The ratio 0.75 gives the main target host pool - # only 1,257,728 token slots. Use a 169 GB/rank absolute target pool at - # c12/c16 and keep ratio mode at the lower concurrencies, where the - # smaller working set does not need the larger pinned allocation. + # only 1,257,728 token slots. Use a 169 GB/rank absolute target pool from + # c12 upward (c12 through c48, on every arm) and keep ratio mode at the + # lower concurrencies, where the smaller working set does not need the + # larger pinned allocation. The per-rank budget below is the same with + # attention-DP: each rank then pins the pool for its own sessions only. # # cluster:b200-nscale advertises 2,063,920 MiB and this config exposes 80%, # giving the benchmark 1,731 GB. A 169 GB/rank packed target+MTP pool plus @@ -132,7 +135,7 @@ if require_agentic_kv_offload_backend hicache; then DEFAULT_HICACHE_RATIO=0.75 DEFAULT_HICACHE_SIZE=0 case "$CONC" in - 12|16) DEFAULT_HICACHE_SIZE=169 ;; + 12|16|20|24|28|32|40|48) DEFAULT_HICACHE_SIZE=169 ;; esac MAX_HICACHE_SIZE=270 HICACHE_SIZE="${HICACHE_SIZE:-$DEFAULT_HICACHE_SIZE}" @@ -208,6 +211,16 @@ if [ "$DP_ATTENTION" = "true" ]; then --tokenizer-worker-num "$TP" --dist-init-addr "127.0.0.1:$((PORT + 2000))" ) +else + # Cookbook low-latency levers; the DP-attention cell omits them. + PARALLEL_ARGS+=( + --kv-cache-dtype fp8_e4m3 + --bf16-gemm-backend cutedsl + --max-prefill-tokens 8192 + ) +fi + +if [ "$EP_SIZE" -gt 1 ]; then # GLM-5.2-NVFP4 leaves the MTP/nextn layer unquantized (hf_quant_config # excludes model.layers.78*), so the EAGLE draft MoE is bf16 and # UnquantizedFusedMoEMethod pins it to the triton runner core. Inheriting @@ -216,19 +229,12 @@ if [ "$DP_ATTENTION" = "true" ]; then # flashinfer to triton is not registered". SGLang handles this in # _deepseek_spec_moe_resolution but gates the hook on is_hip(), so on CUDA # the draft silently inherits; set upstream's own ROCm values explicitly. - # Only needed once expert parallelism puts an a2a in the MoE path -- the - # plain-TP arm below has none. + # Only needed once expert parallelism puts an a2a in the MoE path, i.e. on + # the EP8 and DEP8 arms -- the plain-TP arm (EP_SIZE=1) has none. SPEC_ARGS+=( --speculative-moe-a2a-backend none --speculative-moe-runner-backend triton ) -else - # Cookbook low-latency levers; the DP-attention cell omits them. - PARALLEL_ARGS+=( - --kv-cache-dtype fp8_e4m3 - --bf16-gemm-backend cutedsl - --max-prefill-tokens 8192 - ) fi # AgentX concurrency counts live session trees, not individual requests. diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 0a6c86df6..0ba686157 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -9011,11 +9011,14 @@ glm5.2-fp4-b300-sglang-agentic-mtp: # nightly includes FlashInfer 0.6.18's BF16 TRTLLM MoE allocation fix, which is # required by GLM-5.2's unquantized EAGLE draft head at CUDA-graph capture. # -# Same single arm and concurrency grid as glm5.2-fp4-b300-sglang-agentic-mtp: -# cookbook low-latency TP8 with HiCache host-DRAM offload, conc -# [1, 4, 8, 12, 16] (steps of at least 2, hard stop at 16). TP8-only for -# memory as well as comparability -- the ~433 GB NVFP4 -# checkpoint needs ~54 GB/GPU across 8 B200s and does not fit below 8. +# Three arms, all TP8 with HiCache host-DRAM offload (steps of at least 2 +# through 48). The cookbook low-latency TP8 arm shares its grid with +# glm5.2-fp4-b300-sglang-agentic-mtp through conc 16 and stops at 20: past that +# point TP8 no longer moves the Pareto front. Conc 20 through 48 continue on the +# EP8 arm (expert-parallel MoE, TP attention) and the DEP8 arm (EP8 plus +# attention-DP behind sglang-router). 8 GPUs for memory as well as +# comparability -- the ~433 GB NVFP4 checkpoint needs ~54 GB/GPU across 8 B200s +# and does not fit below 8. glm5.2-fp4-b200-sglang-agentic-mtp: image: lmsysorg/sglang:nightly-dev-cu13-20260901-07c8f729 model: nvidia/GLM-5.2-NVFP4 @@ -9028,7 +9031,9 @@ glm5.2-fp4-b200-sglang-agentic-mtp: agentic-coding: - dram-utilization: 0.80 search-space: - - { tp: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [1, 4, 8, 12, 16] } + - { tp: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [1, 4, 8, 12, 16, 20] } + - { tp: 8, ep: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [20, 24, 28, 32, 40, 48] } + - { tp: 8, ep: 8, dp-attn: true, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [20, 24, 28, 32, 40, 48], router: { name: sglang-router, version: "0.3.2" } } glm5.2-fp4-gb200-dynamo-sglang-agentic-agg: image: lmsysorg/sglang:v0.5.17-cu130 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 912728b2e..3a9e73efc 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6902,3 +6902,12 @@ - "Lower mem-fraction-static from 0.89 to 0.86 on every arm: TP4, TP8 and TP8 with DP attention. swa-full-tokens-ratio becomes per-arm: 0.10 on the tensor-parallel arms as before, 0.15 under DP attention." - "Drop concurrency 2 and 10 from the TP4 arm, leaving [1, 4, 8], and drop concurrency 16 from the TP8 hicache arm, leaving [32, 48]. Concurrency 16 remains on the TP8 no-offload arm." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2800 + +- config-keys: + - glm5.2-fp4-b200-sglang-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Extend the GLM-5.2 NVFP4 B200 SGLang AgentX MTP HiCache curve from concurrency 16 to 48: the TP8 arm gains concurrency 20, and new EP8 and DEP8 (EP8 with attention-DP behind sglang-router) arms cover 20, 24, 28, 32, 40, and 48; existing points are unchanged." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2828 + append-only: true