[Klaud Cold] glm5.2-fp4-b200-sglang-agentic-mtp: extend the AgentX curve to concurrency 48 with EP8/DEP8 arms / 以 EP8/DEP8 分支将 GLM-5.2 B200 SGLang AgentX 曲线扩展至并发度 48

benchmarks/single_node/agentic/glm5.2_fp4_b200_sglang_mtp.sh

@@ -19,9 +19,10 @@ set -x
 #   DP_ATTENTION=false -> low-latency arm (TP8, fp8 KV, cutedsl bf16 GEMM)
 #   DP_ATTENTION=true  -> high-throughput DEP arm (TP8 + DP8 attention-DP +
 #                         EP_SIZE expert-parallel MoE via --ep-size)
-# Only the low-latency arm is wired into the master config for this MTP recipe
-# (see the entry comment on glm5.2-fp4-b200-sglang-agentic-mtp); the DEP branch
-# is kept intact so the throughput arm can be added without re-deriving it.
+# The master config wires three arms (see the entry comment on
+# glm5.2-fp4-b200-sglang-agentic-mtp): low-latency TP8 through conc 20, then
+# EP8 (EP_SIZE=8, DP_ATTENTION=false) and DEP8 (EP_SIZE=8, DP_ATTENTION=true)
+# from conc 20 to 48.
 #
 # Required env vars:
 #   MODEL, TP, CONC, KV_OFFLOADING, TOTAL_CPU_DRAM_GB, RESULT_DIR, DURATION,
@@ -120,9 +121,11 @@ if require_agentic_kv_offload_backend hicache; then
     # whole history; the host tier restores those hits at C2C bandwidth.
     # GLM-5.2 is DSA/MLA-family (attention_backend=dsa): every TP rank holds
     # complete per-token KV. The ratio 0.75 gives the main target host pool
-    # only 1,257,728 token slots. Use a 169 GB/rank absolute target pool at
-    # c12/c16 and keep ratio mode at the lower concurrencies, where the
-    # smaller working set does not need the larger pinned allocation.
+    # only 1,257,728 token slots. Use a 169 GB/rank absolute target pool from
+    # c12 upward (c12 through c48, on every arm) and keep ratio mode at the
+    # lower concurrencies, where the smaller working set does not need the
+    # larger pinned allocation. The per-rank budget below is the same with
+    # attention-DP: each rank then pins the pool for its own sessions only.
     #
     # cluster:b200-nscale advertises 2,063,920 MiB and this config exposes 80%,
     # giving the benchmark 1,731 GB. A 169 GB/rank packed target+MTP pool plus
@@ -132,7 +135,7 @@ if require_agentic_kv_offload_backend hicache; then
     DEFAULT_HICACHE_RATIO=0.75
     DEFAULT_HICACHE_SIZE=0
     case "$CONC" in
-        12|16) DEFAULT_HICACHE_SIZE=169 ;;
+        12|16|20|24|28|32|40|48) DEFAULT_HICACHE_SIZE=169 ;;
     esac
     MAX_HICACHE_SIZE=270
     HICACHE_SIZE="${HICACHE_SIZE:-$DEFAULT_HICACHE_SIZE}"
@@ -208,6 +211,16 @@ if [ "$DP_ATTENTION" = "true" ]; then
         --tokenizer-worker-num "$TP"
         --dist-init-addr "127.0.0.1:$((PORT + 2000))"
     )
+else
+    # Cookbook low-latency levers; the DP-attention cell omits them.
+    PARALLEL_ARGS+=(
+        --kv-cache-dtype fp8_e4m3
+        --bf16-gemm-backend cutedsl
+        --max-prefill-tokens 8192
+    )
+fi
+
+if [ "$EP_SIZE" -gt 1 ]; then
     # GLM-5.2-NVFP4 leaves the MTP/nextn layer unquantized (hf_quant_config
     # excludes model.layers.78*), so the EAGLE draft MoE is bf16 and
     # UnquantizedFusedMoEMethod pins it to the triton runner core. Inheriting
@@ -216,19 +229,12 @@ if [ "$DP_ATTENTION" = "true" ]; then
     # flashinfer to triton is not registered". SGLang handles this in
     # _deepseek_spec_moe_resolution but gates the hook on is_hip(), so on CUDA
     # the draft silently inherits; set upstream's own ROCm values explicitly.
-    # Only needed once expert parallelism puts an a2a in the MoE path -- the
-    # plain-TP arm below has none.
+    # Only needed once expert parallelism puts an a2a in the MoE path, i.e. on
+    # the EP8 and DEP8 arms -- the plain-TP arm (EP_SIZE=1) has none.
     SPEC_ARGS+=(
         --speculative-moe-a2a-backend none
         --speculative-moe-runner-backend triton
     )
-else
-    # Cookbook low-latency levers; the DP-attention cell omits them.
-    PARALLEL_ARGS+=(
-        --kv-cache-dtype fp8_e4m3
-        --bf16-gemm-backend cutedsl
-        --max-prefill-tokens 8192
-    )
 fi
 
 # AgentX concurrency counts live session trees, not individual requests.

configs/nvidia-master.yaml

@@ -9011,11 +9011,14 @@ glm5.2-fp4-b300-sglang-agentic-mtp:
 # nightly includes FlashInfer 0.6.18's BF16 TRTLLM MoE allocation fix, which is
 # required by GLM-5.2's unquantized EAGLE draft head at CUDA-graph capture.
 #
-# Same single arm and concurrency grid as glm5.2-fp4-b300-sglang-agentic-mtp:
-# cookbook low-latency TP8 with HiCache host-DRAM offload, conc
-# [1, 4, 8, 12, 16] (steps of at least 2, hard stop at 16). TP8-only for
-# memory as well as comparability -- the ~433 GB NVFP4
-# checkpoint needs ~54 GB/GPU across 8 B200s and does not fit below 8.
+# Three arms, all TP8 with HiCache host-DRAM offload (steps of at least 2
+# through 48). The cookbook low-latency TP8 arm shares its grid with
+# glm5.2-fp4-b300-sglang-agentic-mtp through conc 16 and stops at 20: past that
+# point TP8 no longer moves the Pareto front. Conc 20 through 48 continue on the
+# EP8 arm (expert-parallel MoE, TP attention) and the DEP8 arm (EP8 plus
+# attention-DP behind sglang-router). 8 GPUs for memory as well as
+# comparability -- the ~433 GB NVFP4 checkpoint needs ~54 GB/GPU across 8 B200s
+# and does not fit below 8.
 glm5.2-fp4-b200-sglang-agentic-mtp:
   image: lmsysorg/sglang:nightly-dev-cu13-20260901-07c8f729
   model: nvidia/GLM-5.2-NVFP4
@@ -9028,7 +9031,9 @@ glm5.2-fp4-b200-sglang-agentic-mtp:
     agentic-coding:
     - dram-utilization: 0.80
       search-space:
-      - { tp: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [1, 4, 8, 12, 16] }
+      - { tp: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [1, 4, 8, 12, 16, 20] }
+      - { tp: 8, ep: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [20, 24, 28, 32, 40, 48] }
+      - { tp: 8, ep: 8, dp-attn: true, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [20, 24, 28, 32, 40, 48], router: { name: sglang-router, version: "0.3.2" } }
 
 glm5.2-fp4-gb200-dynamo-sglang-agentic-agg:
   image: lmsysorg/sglang:v0.5.17-cu130

perf-changelog.yaml

@@ -6902,3 +6902,12 @@
     - "Lower mem-fraction-static from 0.89 to 0.86 on every arm: TP4, TP8 and TP8 with DP attention. swa-full-tokens-ratio becomes per-arm: 0.10 on the tensor-parallel arms as before, 0.15 under DP attention."
     - "Drop concurrency 2 and 10 from the TP4 arm, leaving [1, 4, 8], and drop concurrency 16 from the TP8 hicache arm, leaving [32, 48]. Concurrency 16 remains on the TP8 no-offload arm."
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2800
+
+- config-keys:
+    - glm5.2-fp4-b200-sglang-agentic-mtp
+  scenario-type:
+    - agentic-coding
+  description:
+    - "Extend the GLM-5.2 NVFP4 B200 SGLang AgentX MTP HiCache curve from concurrency 16 to 48: the TP8 arm gains concurrency 20, and new EP8 and DEP8 (EP8 with attention-DP behind sglang-router) arms cover 20, 24, 28, 32, 40, and 48; existing points are unchanged."
+  pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2828
+  append-only: true