From 64e97b2db0fe565eb3676b218967b9382c9c432d Mon Sep 17 00:00:00 2001 From: functionstackx <47992694+functionstackx@users.noreply.github.com> Date: Fri, 4 Sep 2026 18:06:27 -0400 Subject: [PATCH 1/3] feat(config): extend GLM-5.2 B200 SGLang AgentX curve to concurrency 48 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Append concurrency points 20, 24, 28, 32, 40, and 48 to the TP8 HiCache MTP arm of glm5.2-fp4-b200-sglang-agentic-mtp. The new points reuse the 169 GB/rank HiCache target pool that c12 and c16 already select; the c1/c4/c8 ratio-mode points and the c12/c16 points are unchanged, so the entry is append-only. 为 glm5.2-fp4-b200-sglang-agentic-mtp 的 TP8 HiCache MTP 分支追加并发度 20、24、28、32、40、48。新增点沿用 c12/c16 已使用的每 rank 169 GB HiCache 目标池；c1/c4/c8 的比例模式及 c12/c16 保持不变，因此为 append-only 条目。 Co-Authored-By: Claude Fable 5.1 --- .../single_node/agentic/glm5.2_fp4_b200_sglang_mtp.sh | 9 +++++---- configs/nvidia-master.yaml | 6 +++--- perf-changelog.yaml | 9 +++++++++ 3 files changed, 17 insertions(+), 7 deletions(-) diff --git a/benchmarks/single_node/agentic/glm5.2_fp4_b200_sglang_mtp.sh b/benchmarks/single_node/agentic/glm5.2_fp4_b200_sglang_mtp.sh index f3b6821d7..5654a961e 100755 --- a/benchmarks/single_node/agentic/glm5.2_fp4_b200_sglang_mtp.sh +++ b/benchmarks/single_node/agentic/glm5.2_fp4_b200_sglang_mtp.sh @@ -120,9 +120,10 @@ if require_agentic_kv_offload_backend hicache; then # whole history; the host tier restores those hits at C2C bandwidth. # GLM-5.2 is DSA/MLA-family (attention_backend=dsa): every TP rank holds # complete per-token KV. The ratio 0.75 gives the main target host pool - # only 1,257,728 token slots. Use a 169 GB/rank absolute target pool at - # c12/c16 and keep ratio mode at the lower concurrencies, where the - # smaller working set does not need the larger pinned allocation. + # only 1,257,728 token slots. Use a 169 GB/rank absolute target pool from + # c12 upward (c12 through c48) and keep ratio mode at the lower + # concurrencies, where the smaller working set does not need the larger + # pinned allocation. # # cluster:b200-nscale advertises 2,063,920 MiB and this config exposes 80%, # giving the benchmark 1,731 GB. A 169 GB/rank packed target+MTP pool plus @@ -132,7 +133,7 @@ if require_agentic_kv_offload_backend hicache; then DEFAULT_HICACHE_RATIO=0.75 DEFAULT_HICACHE_SIZE=0 case "$CONC" in - 12|16) DEFAULT_HICACHE_SIZE=169 ;; + 12|16|20|24|28|32|40|48) DEFAULT_HICACHE_SIZE=169 ;; esac MAX_HICACHE_SIZE=270 HICACHE_SIZE="${HICACHE_SIZE:-$DEFAULT_HICACHE_SIZE}" diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 0a6c86df6..f77659b17 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -9013,8 +9013,8 @@ glm5.2-fp4-b300-sglang-agentic-mtp: # # Same single arm and concurrency grid as glm5.2-fp4-b300-sglang-agentic-mtp: # cookbook low-latency TP8 with HiCache host-DRAM offload, conc -# [1, 4, 8, 12, 16] (steps of at least 2, hard stop at 16). TP8-only for -# memory as well as comparability -- the ~433 GB NVFP4 +# [1, 4, 8, 12, 16, 20, 24, 28, 32, 40, 48] (steps of at least 2 through 48). +# TP8-only for memory as well as comparability -- the ~433 GB NVFP4 # checkpoint needs ~54 GB/GPU across 8 B200s and does not fit below 8. glm5.2-fp4-b200-sglang-agentic-mtp: image: lmsysorg/sglang:nightly-dev-cu13-20260901-07c8f729 @@ -9028,7 +9028,7 @@ glm5.2-fp4-b200-sglang-agentic-mtp: agentic-coding: - dram-utilization: 0.80 search-space: - - { tp: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [1, 4, 8, 12, 16] } + - { tp: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [1, 4, 8, 12, 16, 20, 24, 28, 32, 40, 48] } glm5.2-fp4-gb200-dynamo-sglang-agentic-agg: image: lmsysorg/sglang:v0.5.17-cu130 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 14434c76a..5a890712d 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6878,3 +6878,12 @@ description: - "Refresh the Kimi-K3 GB300 AgentX configurations with updated configs." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2811 + +- config-keys: + - glm5.2-fp4-b200-sglang-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Extend the GLM-5.2 NVFP4 B200 SGLang AgentX MTP TP8 HiCache curve from concurrency 16 to 48 by appending points at 20, 24, 28, 32, 40, and 48, which reuse the 169 GB/rank HiCache target pool already used at 12 and 16; existing points are unchanged." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX + append-only: true From 136d9e21fcd77d3e0b755b707dc22f0e6e18bfa0 Mon Sep 17 00:00:00 2001 From: functionstackx <47992694+functionstackx@users.noreply.github.com> Date: Fri, 4 Sep 2026 18:08:26 -0400 Subject: [PATCH 2/3] chore(changelog): fill the PR link for #2828 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 补全 perf-changelog 条目中的 PR 链接（#2828）。 Co-Authored-By: Claude Fable 5.1 --- perf-changelog.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 5a890712d..6e194e10b 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6885,5 +6885,5 @@ - agentic-coding description: - "Extend the GLM-5.2 NVFP4 B200 SGLang AgentX MTP TP8 HiCache curve from concurrency 16 to 48 by appending points at 20, 24, 28, 32, 40, and 48, which reuse the 169 GB/rank HiCache target pool already used at 12 and 16; existing points are unchanged." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2828 append-only: true From e41c9342a2707a7c24ed4ebdefc447a7af110449 Mon Sep 17 00:00:00 2001 From: functionstackx <47992694+functionstackx@users.noreply.github.com> Date: Sat, 5 Sep 2026 00:37:25 -0400 Subject: [PATCH 3/3] feat(config): cap the GLM-5.2 B200 TP8 arm at conc 20 and add EP8/DEP8 arms MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Past concurrency 20 the low-latency TP8 arm no longer moves the Pareto front, so it now stops at 20 and concurrency 20 through 48 run on two new arms: EP8 (expert-parallel MoE with TP attention) and DEP8 (EP8 with attention-DP behind sglang-router). The recipe script already carried the DP-attention branch; the EAGLE draft-MoE backend override it needed is now gated on EP_SIZE > 1 so the plain EP8 arm gets it too. The TP8 c1-c16 points are byte-identical, so the entry stays append-only. TP8 低延迟分支在并发度 20 之后不再改善帕累托前沿，因此止于 20；并发度 20 至 48 改由两个新分支承担：EP8（专家并行 MoE，TP 注意力）与 DEP8（EP8 加注意力 DP， 经 sglang-router 路由）。脚本已有 DP 注意力分支；其所需的 EAGLE 草稿 MoE 后端 覆盖现改为按 EP_SIZE > 1 触发，使纯 EP8 分支同样生效。TP8 的 c1–c16 点逐字节 不变，条目保持 append-only。 Co-Authored-By: Claude Fable 5.1 --- .../agentic/glm5.2_fp4_b200_sglang_mtp.sh | 35 +++++++++++-------- configs/nvidia-master.yaml | 17 +++++---- perf-changelog.yaml | 2 +- 3 files changed, 32 insertions(+), 22 deletions(-) diff --git a/benchmarks/single_node/agentic/glm5.2_fp4_b200_sglang_mtp.sh b/benchmarks/single_node/agentic/glm5.2_fp4_b200_sglang_mtp.sh index 5654a961e..e37289864 100755 --- a/benchmarks/single_node/agentic/glm5.2_fp4_b200_sglang_mtp.sh +++ b/benchmarks/single_node/agentic/glm5.2_fp4_b200_sglang_mtp.sh @@ -19,9 +19,10 @@ set -x # DP_ATTENTION=false -> low-latency arm (TP8, fp8 KV, cutedsl bf16 GEMM) # DP_ATTENTION=true -> high-throughput DEP arm (TP8 + DP8 attention-DP + # EP_SIZE expert-parallel MoE via --ep-size) -# Only the low-latency arm is wired into the master config for this MTP recipe -# (see the entry comment on glm5.2-fp4-b200-sglang-agentic-mtp); the DEP branch -# is kept intact so the throughput arm can be added without re-deriving it. +# The master config wires three arms (see the entry comment on +# glm5.2-fp4-b200-sglang-agentic-mtp): low-latency TP8 through conc 20, then +# EP8 (EP_SIZE=8, DP_ATTENTION=false) and DEP8 (EP_SIZE=8, DP_ATTENTION=true) +# from conc 20 to 48. # # Required env vars: # MODEL, TP, CONC, KV_OFFLOADING, TOTAL_CPU_DRAM_GB, RESULT_DIR, DURATION, @@ -121,9 +122,10 @@ if require_agentic_kv_offload_backend hicache; then # GLM-5.2 is DSA/MLA-family (attention_backend=dsa): every TP rank holds # complete per-token KV. The ratio 0.75 gives the main target host pool # only 1,257,728 token slots. Use a 169 GB/rank absolute target pool from - # c12 upward (c12 through c48) and keep ratio mode at the lower - # concurrencies, where the smaller working set does not need the larger - # pinned allocation. + # c12 upward (c12 through c48, on every arm) and keep ratio mode at the + # lower concurrencies, where the smaller working set does not need the + # larger pinned allocation. The per-rank budget below is the same with + # attention-DP: each rank then pins the pool for its own sessions only. # # cluster:b200-nscale advertises 2,063,920 MiB and this config exposes 80%, # giving the benchmark 1,731 GB. A 169 GB/rank packed target+MTP pool plus @@ -209,6 +211,16 @@ if [ "$DP_ATTENTION" = "true" ]; then --tokenizer-worker-num "$TP" --dist-init-addr "127.0.0.1:$((PORT + 2000))" ) +else + # Cookbook low-latency levers; the DP-attention cell omits them. + PARALLEL_ARGS+=( + --kv-cache-dtype fp8_e4m3 + --bf16-gemm-backend cutedsl + --max-prefill-tokens 8192 + ) +fi + +if [ "$EP_SIZE" -gt 1 ]; then # GLM-5.2-NVFP4 leaves the MTP/nextn layer unquantized (hf_quant_config # excludes model.layers.78*), so the EAGLE draft MoE is bf16 and # UnquantizedFusedMoEMethod pins it to the triton runner core. Inheriting @@ -217,19 +229,12 @@ if [ "$DP_ATTENTION" = "true" ]; then # flashinfer to triton is not registered". SGLang handles this in # _deepseek_spec_moe_resolution but gates the hook on is_hip(), so on CUDA # the draft silently inherits; set upstream's own ROCm values explicitly. - # Only needed once expert parallelism puts an a2a in the MoE path -- the - # plain-TP arm below has none. + # Only needed once expert parallelism puts an a2a in the MoE path, i.e. on + # the EP8 and DEP8 arms -- the plain-TP arm (EP_SIZE=1) has none. SPEC_ARGS+=( --speculative-moe-a2a-backend none --speculative-moe-runner-backend triton ) -else - # Cookbook low-latency levers; the DP-attention cell omits them. - PARALLEL_ARGS+=( - --kv-cache-dtype fp8_e4m3 - --bf16-gemm-backend cutedsl - --max-prefill-tokens 8192 - ) fi # AgentX concurrency counts live session trees, not individual requests. diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index f77659b17..0ba686157 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -9011,11 +9011,14 @@ glm5.2-fp4-b300-sglang-agentic-mtp: # nightly includes FlashInfer 0.6.18's BF16 TRTLLM MoE allocation fix, which is # required by GLM-5.2's unquantized EAGLE draft head at CUDA-graph capture. # -# Same single arm and concurrency grid as glm5.2-fp4-b300-sglang-agentic-mtp: -# cookbook low-latency TP8 with HiCache host-DRAM offload, conc -# [1, 4, 8, 12, 16, 20, 24, 28, 32, 40, 48] (steps of at least 2 through 48). -# TP8-only for memory as well as comparability -- the ~433 GB NVFP4 -# checkpoint needs ~54 GB/GPU across 8 B200s and does not fit below 8. +# Three arms, all TP8 with HiCache host-DRAM offload (steps of at least 2 +# through 48). The cookbook low-latency TP8 arm shares its grid with +# glm5.2-fp4-b300-sglang-agentic-mtp through conc 16 and stops at 20: past that +# point TP8 no longer moves the Pareto front. Conc 20 through 48 continue on the +# EP8 arm (expert-parallel MoE, TP attention) and the DEP8 arm (EP8 plus +# attention-DP behind sglang-router). 8 GPUs for memory as well as +# comparability -- the ~433 GB NVFP4 checkpoint needs ~54 GB/GPU across 8 B200s +# and does not fit below 8. glm5.2-fp4-b200-sglang-agentic-mtp: image: lmsysorg/sglang:nightly-dev-cu13-20260901-07c8f729 model: nvidia/GLM-5.2-NVFP4 @@ -9028,7 +9031,9 @@ glm5.2-fp4-b200-sglang-agentic-mtp: agentic-coding: - dram-utilization: 0.80 search-space: - - { tp: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [1, 4, 8, 12, 16, 20, 24, 28, 32, 40, 48] } + - { tp: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [1, 4, 8, 12, 16, 20] } + - { tp: 8, ep: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [20, 24, 28, 32, 40, 48] } + - { tp: 8, ep: 8, dp-attn: true, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [20, 24, 28, 32, 40, 48], router: { name: sglang-router, version: "0.3.2" } } glm5.2-fp4-gb200-dynamo-sglang-agentic-agg: image: lmsysorg/sglang:v0.5.17-cu130 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 6e194e10b..651f3c726 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6884,6 +6884,6 @@ scenario-type: - agentic-coding description: - - "Extend the GLM-5.2 NVFP4 B200 SGLang AgentX MTP TP8 HiCache curve from concurrency 16 to 48 by appending points at 20, 24, 28, 32, 40, and 48, which reuse the 169 GB/rank HiCache target pool already used at 12 and 16; existing points are unchanged." + - "Extend the GLM-5.2 NVFP4 B200 SGLang AgentX MTP HiCache curve from concurrency 16 to 48: the TP8 arm gains concurrency 20, and new EP8 and DEP8 (EP8 with attention-DP behind sglang-router) arms cover 20, 24, 28, 32, 40, and 48; existing points are unchanged." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2828 append-only: true