From fc3d0790d05cb8ea509abbf077fefb205b46880a Mon Sep 17 00:00:00 2001 From: Sahithi Chigurupati Date: Tue, 18 Aug 2026 10:23:23 -0700 Subject: [PATCH 01/13] Add GLM-5.2-NVFP4 B300 AgentX agentic benchmarks (agg + disagg) B300 Dynamo-SGLang agentic-coding sweep for GLM-5.2-NVFP4, 7 points across two config keys: - glm5.2-fp4-b300-dynamo-sglang-agentic-agg: aggregated single-node TP8 (conc 2/4) - glm5.2-fp4-b300-dynamo-sglang-agentic-disagg: low-latency 1P2D/1P3D/1P4D/1P6D TP4-decode (conc 63/61/59/58) and throughput 2P1D DEP8-decode (conc 139) --- .../b300-fp4/agentic/glm5.2-agentx-agg.yaml | 112 +++++++++++ .../b300-fp4/agentic/glm5.2-agentx.yaml | 180 ++++++++++++++++++ configs/nvidia-master.yaml | 135 +++++++++++++ perf-changelog.yaml | 11 ++ runners/launch_b300-nv.sh | 13 +- 5 files changed, 450 insertions(+), 1 deletion(-) create mode 100644 benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/b300-fp4/agentic/glm5.2-agentx-agg.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/b300-fp4/agentic/glm5.2-agentx.yaml diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/b300-fp4/agentic/glm5.2-agentx-agg.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/b300-fp4/agentic/glm5.2-agentx-agg.yaml new file mode 100644 index 0000000000..f77303daab --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/b300-fp4/agentic/glm5.2-agentx-agg.yaml @@ -0,0 +1,112 @@ +# Agentic-coding SGLang aggregated recipe for GLM-5.2-NVFP4 on B300 +# (single aggregated worker, TP8 on one 8-GPU B300 node, EAGLE MTP + HiCache). +# +# Flat single-variant schema the agentic CI flow expects: applied via +# CONFIG_FILE= with no zip_override selector, so it must NOT be +# base:-wrapped. Concurrency is not a recipe field; the GHA matrix fans out +# one job per concurrency from the master-config conc-list into agentic_srt.sh. +name: b300-fp4-glm5.2-agentx-agg +model: + path: glm-5.2-fp4 + container: dynamo-sglang + precision: fp4 +resources: + gpu_type: b300 + gpus_per_node: 8 + agg_nodes: 1 + agg_workers: 1 + gpus_per_agg: 8 +frontend: + type: dynamo + env: + PIP_BREAK_SYSTEM_PACKAGES: '1' + args: + router-mode: kv + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None + router-session-affinity-ttl-secs: 3600 +dynamo: + hash: 71eb001e17fa73c742f0afe1a6ed96836cb135fd + install: true +backend: + type: sglang + sglang_config: + aggregated: + served-model-name: nvidia/GLM-5.2-NVFP4 + trust-remote-code: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 8 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-dp-attention: false + enable-dp-lm-head: false + max-running-requests: 10 + cuda-graph-max-bs: 10 + chunked-prefill-size: 8192 + max-prefill-tokens: 8192 + context-length: 1048576 + speculative-algorithm: EAGLE + speculative-num-steps: 4 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 5 + nsa-prefill-backend: trtllm + nsa-decode-backend: trtllm + moe-runner-backend: flashinfer_trtllm + fp4-gemm-backend: flashinfer_trtllm + disable-shared-experts-fusion: true + enable-flashinfer-allreduce-fusion: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + mem-fraction-static: 0.8 + enable-hierarchical-cache: true + hicache-write-policy: write_back + hicache-size: 270 + hicache-io-backend: direct + enable-metrics: true + enable-cache-report: true + aggregated_environment: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + MC_TE_METRIC: 'true' + NCCL_CUMEM_ENABLE: '1' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_ENABLE_THINKING: '1' + SGLANG_REASONING_EFFORT: max + SGLANG_ENABLE_UNIFIED_RADIX_TREE: '1' + SGLANG_HICACHE_DEBUG_LOG: '1' + SGLANG_HICACHE_DEBUG_SAMPLE_RATE: '16384' + SGLANG_MOE_NVFP4_DISPATCH: '1' + SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' + SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: '8' + SGLANG_SIMULATE_ACC_LEN: '3.33' + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token + PIP_BREAK_SYSTEM_PACKAGES: '1' +health_check: + max_attempts: 1440 + interval_seconds: 10 +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: "8000" + IS_MULTINODE: "false" + TP: "8" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 64 +sbatch_directives: + mem: '0' + cpus-per-task: '192' +srun_options: + mem: '0' + container-remap-root: '' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/b300-fp4/agentic/glm5.2-agentx.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/b300-fp4/agentic/glm5.2-agentx.yaml new file mode 100644 index 0000000000..a0c6965832 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/b300-fp4/agentic/glm5.2-agentx.yaml @@ -0,0 +1,180 @@ +base: + name: b300-fp4-glm5.2-agentx + model: + path: glm-5.2-fp4 + container: dynamo-sglang + precision: fp4 + resources: + gpu_type: b300 + gpus_per_node: 8 + frontend: + type: dynamo + nginx_session_affinity: true + nginx_session_affinity_header: X-Dynamo-Session-ID + env: + PIP_BREAK_SYSTEM_PACKAGES: '1' + args: + router-mode: kv + router-session-affinity-ttl-secs: '3600' + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None + dynamo: + hash: 71eb001e17fa73c742f0afe1a6ed96836cb135fd + install: true + backend: + type: sglang + prefill_environment: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + NCCL_CUMEM_ENABLE: '1' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_ENABLE_THINKING: '1' + SGLANG_ENABLE_UNIFIED_RADIX_TREE: '1' + SGLANG_HICACHE_DEBUG_LOG: '1' + SGLANG_HICACHE_DEBUG_SAMPLE_RATE: '16384' + SGLANG_REASONING_EFFORT: max + PIP_BREAK_SYSTEM_PACKAGES: '1' + SGLANG_SIMULATE_ACC_LEN: '2.5' + decode_environment: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + NCCL_CUMEM_ENABLE: '1' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_MOE_NVFP4_DISPATCH: '1' + SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' + SGLANG_ENABLE_THINKING: '1' + SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: '8' + SGLANG_REASONING_EFFORT: max + PIP_BREAK_SYSTEM_PACKAGES: '1' + SGLANG_SIMULATE_ACC_LEN: '2.5' + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token + sglang_config: + prefill: + served-model-name: nvidia/GLM-5.2-NVFP4 + trust-remote-code: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + disaggregation-mode: prefill + disaggregation-transfer-backend: nixl + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 4 + enable-dp-attention: true + enable-dp-lm-head: true + load-balance-method: total_tokens + chunked-prefill-size: 65536 + max-prefill-tokens: 16384 + max-running-requests: 16 + cuda-graph-max-bs: 16 + disable-cuda-graph: true + nsa-prefill-backend: trtllm + nsa-decode-backend: trtllm + moe-runner-backend: flashinfer_cutlass + fp4-gemm-backend: flashinfer_cutlass + enable-flashinfer-allreduce-fusion: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + mem-fraction-static: 0.8 + context-length: 1048576 + enable-hierarchical-cache: true + hicache-write-policy: write_back + hicache-size: 270 + hicache-io-backend: direct + speculative-algorithm: EAGLE + speculative-num-steps: 1 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 2 + enable-metrics: true + enable-cache-report: true + decode: + served-model-name: nvidia/GLM-5.2-NVFP4 + trust-remote-code: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + disaggregation-mode: decode + disaggregation-transfer-backend: nixl + disable-radix-cache: true + speculative-algorithm: EAGLE + speculative-num-steps: 2 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 3 + tensor-parallel-size: 4 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-dp-attention: false + enable-dp-lm-head: false + max-running-requests: 16 + cuda-graph-max-bs: 16 + chunked-prefill-size: 64 + context-length: 1048576 + nsa-prefill-backend: trtllm + nsa-decode-backend: trtllm + moe-runner-backend: flashinfer_trtllm + fp4-gemm-backend: flashinfer_cutlass + skip-tokenizer-init: true + stream-interval: 30 + enable-flashinfer-allreduce-fusion: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + mem-fraction-static: 0.92 + disaggregation-decode-extra-slots: 0 + enable-metrics: true + enable-cache-report: true + health_check: + max_attempts: 1440 + interval_seconds: 10 + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 64 + sbatch_directives: + mem: '0' + cpus-per-task: '192' + srun_options: + mem: '0' + container-remap-root: '' +# ################# agentx ################# +zip_override_mtp_agentx_lowlat: + name: [agentx-1p2d_tp4-c63, agentx-1p3d_tp4-c61, agentx-1p4d_tp4-c59, agentx-1p6d_tp4-c58, agentx-2p1d_dtp8-c139] + backend: + sglang_config: + decode: + cuda-graph-max-bs: [16, 16, 16, 16, 139] + data-parallel-size: [1, 1, 1, 1, 8] + enable-dp-attention: [false, false, false, false, true] + enable-dp-lm-head: [false, false, false, false, true] + max-running-requests: [16, 16, 16, 16, 139] + tensor-parallel-size: [4, 4, 4, 4, 8] + frontend: + enable_multiple_frontends: true + resources: + decode_nodes: [1, 2, 2, 3, 1] + decode_workers: [2, 3, 4, 6, 1] + gpus_per_decode: [4, 4, 4, 4, 8] + gpus_per_prefill: 4 + prefill_nodes: 1 + prefill_workers: [1, 1, 1, 1, 2] diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index c8d55ad5d5..dd328cbb3b 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -8319,6 +8319,141 @@ glm5.2-fp4-gb200-dynamo-sglang-agentic-disagg: ep: 1 dp-attn: false +glm5.2-fp4-b300-dynamo-sglang-agentic-agg: + image: lmsysorg/sglang:nightly-dev-cu13-20260805-211ee642 + model: nvidia/GLM-5.2-NVFP4 + model-prefix: glm5.2 + runner: cluster:b300-nv + precision: fp4 + framework: dynamo-sglang + router: { name: dynamo-router, version: "71eb001e17fa73c742f0afe1a6ed96836cb135fd" } + multinode: true + disagg: false + scenarios: + agentic-coding: + - dram-utilization: 0.80 + search-space: + # aggregated 1-worker TP8 (prefill+decode on one 8-GPU B300 node) + - spec-decoding: mtp + conc-list: [2, 4] + kv-offloading: dram + kv-offload-backend: { name: hicache } + prefill: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5.2/b300-fp4/agentic/glm5.2-agentx-agg.yaml" + # Aggregated worker serves prefill and decode on the same GPUs. + # Keep decode at zero to avoid double-counting the TP8 allocation. + decode: + num-worker: 0 + tp: 8 + ep: 1 + dp-attn: false + +glm5.2-fp4-b300-dynamo-sglang-agentic-disagg: + image: lmsysorg/sglang:nightly-dev-cu13-20260805-211ee642 + model: nvidia/GLM-5.2-NVFP4 + model-prefix: glm5.2 + runner: cluster:b300-nv + precision: fp4 + framework: dynamo-sglang + router: { name: dynamo-router, version: "71eb001e17fa73c742f0afe1a6ed96836cb135fd" } + kv-p2p-transfer: nixl + multinode: true + disagg: true + scenarios: + agentic-coding: + - dram-utilization: 0.80 + search-space: + # low-latency 1P2D TP4-prefill / TP4-decode (2 decode workers) + - spec-decoding: mtp + conc-list: [63] + kv-offloading: dram + kv-offload-backend: { name: hicache } + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5.2/b300-fp4/agentic/glm5.2-agentx.yaml:zip_override_mtp_agentx_lowlat[0]" + decode: + num-worker: 2 + tp: 4 + ep: 1 + dp-attn: false + # low-latency 1P3D TP4-prefill / TP4-decode (3 decode workers) + - spec-decoding: mtp + conc-list: [61] + kv-offloading: dram + kv-offload-backend: { name: hicache } + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5.2/b300-fp4/agentic/glm5.2-agentx.yaml:zip_override_mtp_agentx_lowlat[1]" + decode: + num-worker: 3 + tp: 4 + ep: 1 + dp-attn: false + # low-latency 1P4D TP4-prefill / TP4-decode (4 decode workers) + - spec-decoding: mtp + conc-list: [59] + kv-offloading: dram + kv-offload-backend: { name: hicache } + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5.2/b300-fp4/agentic/glm5.2-agentx.yaml:zip_override_mtp_agentx_lowlat[2]" + decode: + num-worker: 4 + tp: 4 + ep: 1 + dp-attn: false + # low-latency 1P6D TP4-prefill / TP4-decode (6 decode workers) + - spec-decoding: mtp + conc-list: [58] + kv-offloading: dram + kv-offload-backend: { name: hicache } + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5.2/b300-fp4/agentic/glm5.2-agentx.yaml:zip_override_mtp_agentx_lowlat[3]" + decode: + num-worker: 6 + tp: 4 + ep: 1 + dp-attn: false + # throughput 2P1D DEP8-decode (DP8/TP8 decode, dp-attention) + - spec-decoding: mtp + conc-list: [139] + kv-offloading: dram + kv-offload-backend: { name: hicache } + prefill: + num-worker: 2 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5.2/b300-fp4/agentic/glm5.2-agentx.yaml:zip_override_mtp_agentx_lowlat[4]" + decode: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: true + glm5.2-fp4-gb300-dynamo-sglang-agentic-agg: image: lmsysorg/sglang:nightly-dev-cu13-20260805-211ee642 model: nvidia/GLM-5.2-NVFP4 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 73648d9b99..e027306529 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6076,3 +6076,14 @@ - "Add GB200 DeepSeek-V4-Pro FP4 Dynamo-vLLM AgentX mirroring the GB300 PR #2571 MTP tuning, with every GB300 4-GPU worker sized to 8 GPUs on GB200." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2636 + +- config-keys: + - glm5.2-fp4-b300-dynamo-sglang-agentic-agg + - glm5.2-fp4-b300-dynamo-sglang-agentic-disagg + scenario-type: + - agentic-coding + description: + - "Add GLM-5.2-NVFP4 B300 AgentX (agentic-coding) Dynamo-SGLang benchmarks over a 7-point sweep: aggregated single-node TP8 (conc 2/4), disaggregated low-latency TP4-prefill/TP4-decode 1P2D (conc 63), 1P3D (conc 61), 1P4D (conc 59), 1P6D (conc 58), and throughput 2P1D DEP8-decode (conc 139)" + - "EAGLE speculative decoding, hierarchical KV cache (hicache-size 270), MTP, decode mem-fraction-static 0.92, nixl KV transfer, and X-Dynamo-Session-ID session affinity; driven from the NVIDIA/srt-slurm base+zip_override recipes via CONFIG_FILE selectors" + - "Image: lmsysorg/sglang:nightly-dev-cu13-20260805-211ee642; runner: b300-nv" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX diff --git a/runners/launch_b300-nv.sh b/runners/launch_b300-nv.sh index f7878a8ad8..05d02a962c 100644 --- a/runners/launch_b300-nv.sh +++ b/runners/launch_b300-nv.sh @@ -56,8 +56,11 @@ elif [[ $MODEL_PREFIX == "minimaxm3" && $PRECISION == "fp4" && $FRAMEWORK == "dy elif [[ $MODEL_PREFIX == "minimaxm3" && $PRECISION == "fp8" && $FRAMEWORK == "dynamo-vllm" ]]; then export MODEL_PATH="/data/models/MiniMax-M3-MXFP8" export SRT_SLURM_MODEL_PREFIX="MiniMaxAI/MiniMax-M3-MXFP8" +elif [[ $MODEL_PREFIX == "glm5.2" && $PRECISION == "fp4" ]]; then + export MODEL_PATH="${MODEL_PATH:-/scratch/models/GLM-5.2-NVFP4}" + export SRT_SLURM_MODEL_PREFIX="glm-5.2-fp4" else - echo "Unsupported model: $MODEL_PREFIX-$PRECISION. Supported models are: dsr1-fp4, dsr1-fp8, dsv4-fp4 with dynamo-vllm or dynamo-sglang, minimaxm2.5-fp4 with dynamo-vllm, minimaxm2.5-fp8 with dynamo-vllm, minimaxm3-fp4 with dynamo-vllm, minimaxm3-fp8 with dynamo-vllm" + echo "Unsupported model: $MODEL_PREFIX-$PRECISION. Supported models are: dsr1-fp4, dsr1-fp8, dsv4-fp4 with dynamo-vllm or dynamo-sglang, minimaxm2.5-fp4 with dynamo-vllm, minimaxm2.5-fp8 with dynamo-vllm, minimaxm3-fp4 with dynamo-vllm, minimaxm3-fp8 with dynamo-vllm, glm5.2-fp4 with dynamo-sglang" exit 1 fi @@ -85,6 +88,14 @@ elif [[ $FRAMEWORK == "dynamo-sglang" && $MODEL_PREFIX == "dsv4" && $PRECISION = git checkout c180328b98c3793ca84a1e24a030f90545eb7d5d || exit 1 mkdir -p recipes/sglang/deepseek-v4 cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4" recipes/sglang/deepseek-v4 +elif [[ $FRAMEWORK == "dynamo-sglang" && $MODEL_PREFIX == "glm5.2" && $PRECISION == "fp4" ]]; then + # GLM-5.2 B300 sglang AgentX: srt-slurm main carries the agentx-mvp scenario, + # session-affinity frontend, and custom benchmark schema these recipes need. + git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" + cd "$SRT_REPO_DIR" || exit 1 + git checkout main + mkdir -p recipes/sglang/glm5.2/b300-fp4 + cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/b300-fp4" recipes/sglang/glm5.2/b300-fp4 elif [[ $FRAMEWORK == "dynamo-vllm" && $MODEL_PREFIX == "minimaxm3" && $PRECISION == "fp4" && "$CONFIG_FILE" == recipes/vllm/minimax-m3/b300-fp4/8k1k/mtp/*.yaml ]]; then git clone --branch main --single-branch https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" cd "$SRT_REPO_DIR" || exit 1 From fb7e871816c8a185ca3f31a0af273166a7e0fb9d Mon Sep 17 00:00:00 2001 From: Sahithi Chigurupati Date: Tue, 18 Aug 2026 10:42:54 -0700 Subject: [PATCH 02/13] changelog: point GLM-5.2 B300 AgentX entry at PR #2654 --- perf-changelog.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index e027306529..d23853a155 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6086,4 +6086,4 @@ - "Add GLM-5.2-NVFP4 B300 AgentX (agentic-coding) Dynamo-SGLang benchmarks over a 7-point sweep: aggregated single-node TP8 (conc 2/4), disaggregated low-latency TP4-prefill/TP4-decode 1P2D (conc 63), 1P3D (conc 61), 1P4D (conc 59), 1P6D (conc 58), and throughput 2P1D DEP8-decode (conc 139)" - "EAGLE speculative decoding, hierarchical KV cache (hicache-size 270), MTP, decode mem-fraction-static 0.92, nixl KV transfer, and X-Dynamo-Session-ID session affinity; driven from the NVIDIA/srt-slurm base+zip_override recipes via CONFIG_FILE selectors" - "Image: lmsysorg/sglang:nightly-dev-cu13-20260805-211ee642; runner: b300-nv" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2654 From 6f27e8310b98e2fbb195327ce31d71329da8cf14 Mon Sep 17 00:00:00 2001 From: Sahithi Chigurupati Date: Tue, 18 Aug 2026 10:47:52 -0700 Subject: [PATCH 03/13] launch_b300-nv.sh: make GLM-5.2 agentic clone branch reachable The srt-slurm clone chain opens with a bare `if [[ "$IS_AGENTIC" == "1" ]]` catch-all, so the glm5.2 dynamo-sglang elif never ran for agentic jobs (all 7 GLM-5.2 B300 points hit the catch-all, cloned the wrong fork, and failed the CONFIG_PATH check). Promote the glm5.2 branch to the opening `if` with IS_AGENTIC==1 ANDed in and demote the generic catch-all to elif, matching the ordering in launch_gb300-nv.sh / launch_gb200-nv.sh. --- runners/launch_b300-nv.sh | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/runners/launch_b300-nv.sh b/runners/launch_b300-nv.sh index 05d02a962c..871705dbce 100644 --- a/runners/launch_b300-nv.sh +++ b/runners/launch_b300-nv.sh @@ -73,7 +73,16 @@ if [ -d "$SRT_REPO_DIR" ]; then fi # TODO(CJQ): make first class upon srt-slurm upstream refactor -if [[ "$IS_AGENTIC" == "1" ]]; then +if [[ "$IS_AGENTIC" == "1" && $FRAMEWORK == "dynamo-sglang" && $MODEL_PREFIX == "glm5.2" && $PRECISION == "fp4" ]]; then + # GLM-5.2 B300 sglang AgentX: srt-slurm main carries the agentx-mvp scenario, + # session-affinity frontend, and custom benchmark schema these recipes need. + # Must precede the generic IS_AGENTIC catch-all below (it clones a different fork). + git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" + cd "$SRT_REPO_DIR" || exit 1 + git checkout main + mkdir -p recipes/sglang/glm5.2/b300-fp4 + cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/b300-fp4" recipes/sglang/glm5.2/b300-fp4 +elif [[ "$IS_AGENTIC" == "1" ]]; then git clone --branch cam/sa-submission-q2-2026 --single-branch https://github.com/cquil11/srt-slurm-nv.git "$SRT_REPO_DIR" cd "$SRT_REPO_DIR" || exit 1 elif [[ $FRAMEWORK == "dynamo-vllm" && $MODEL_PREFIX == "dsv4" ]]; then @@ -88,14 +97,6 @@ elif [[ $FRAMEWORK == "dynamo-sglang" && $MODEL_PREFIX == "dsv4" && $PRECISION = git checkout c180328b98c3793ca84a1e24a030f90545eb7d5d || exit 1 mkdir -p recipes/sglang/deepseek-v4 cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4" recipes/sglang/deepseek-v4 -elif [[ $FRAMEWORK == "dynamo-sglang" && $MODEL_PREFIX == "glm5.2" && $PRECISION == "fp4" ]]; then - # GLM-5.2 B300 sglang AgentX: srt-slurm main carries the agentx-mvp scenario, - # session-affinity frontend, and custom benchmark schema these recipes need. - git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" - cd "$SRT_REPO_DIR" || exit 1 - git checkout main - mkdir -p recipes/sglang/glm5.2/b300-fp4 - cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/b300-fp4" recipes/sglang/glm5.2/b300-fp4 elif [[ $FRAMEWORK == "dynamo-vllm" && $MODEL_PREFIX == "minimaxm3" && $PRECISION == "fp4" && "$CONFIG_FILE" == recipes/vllm/minimax-m3/b300-fp4/8k1k/mtp/*.yaml ]]; then git clone --branch main --single-branch https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" cd "$SRT_REPO_DIR" || exit 1 From 4b45366dc579552bd8c75669f4df2304a4aff6cf Mon Sep 17 00:00:00 2001 From: Sahithi Chigurupati Date: Tue, 18 Aug 2026 11:58:49 -0700 Subject: [PATCH 04/13] launch_b300-nv.sh: skip srtctl preflight for GLM-5.2 dynamo-sglang GLM-5.2-NVFP4 is staged under compute-node-local /scratch/models, which the GHA login host running srtctl preflight cannot stat, so preflight fails with "path is unavailable" even though the workers can load it. Extend the existing dsv4 dynamo-sglang --no-preflight bypass to glm5.2 (runtime model loading still validates the path). --- runners/launch_b300-nv.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runners/launch_b300-nv.sh b/runners/launch_b300-nv.sh index 871705dbce..1a95e25d07 100644 --- a/runners/launch_b300-nv.sh +++ b/runners/launch_b300-nv.sh @@ -227,7 +227,7 @@ SRTCTL_APPLY_ARGS=( if [[ $FRAMEWORK == "dynamo-vllm" && $MODEL_PREFIX == "minimaxm3" && $PRECISION == "fp4" && ( "$CONFIG_FILE" == recipes/vllm/minimax-m3/b300-fp4/8k1k/mtp/*.yaml || "$CONFIG_FILE" == recipes/vllm/minimax-m3/b300-fp4/8k1k/*-tp1-*.yaml ) ]]; then SRTCTL_APPLY_ARGS+=(--no-preflight) fi -if [[ $FRAMEWORK == "dynamo-sglang" && $MODEL_PREFIX == "dsv4" && "$MODEL_PATH" == /scratch/models/* ]]; then +if [[ $FRAMEWORK == "dynamo-sglang" && ( $MODEL_PREFIX == "dsv4" || $MODEL_PREFIX == "glm5.2" ) && "$MODEL_PATH" == /scratch/models/* ]]; then SRTCTL_APPLY_ARGS+=(--no-preflight) fi if [[ -n "$SRTCTL_SETUP_SCRIPT" ]]; then From b42d4dc7f860fff9418ab0273203314c417812dd Mon Sep 17 00:00:00 2001 From: Sahithi Chigurupati Date: Tue, 18 Aug 2026 12:51:13 -0700 Subject: [PATCH 05/13] glm5.2 b300 disagg: add UCX cuda_copy transport for nixl KV transfer The disagg decode worker's NIXL KV manager crashed at buffer registration ("VRAM memory is detected as host by UCX ... VRAM registration cannot proceed" -> NIXL_ERR_BACKEND) because the recipe shipped no UCX config, so UCX had no CUDA memory transport. Add UCX_TLS=cuda_copy,rc + MC_FORCE_MNNVL + NCCL_MNNVL_ENABLE to prefill/decode env, matching the proven dsv4 B300 nixl disagg recipes. --- .../sglang/glm5.2/b300-fp4/agentic/glm5.2-agentx.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/b300-fp4/agentic/glm5.2-agentx.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/b300-fp4/agentic/glm5.2-agentx.yaml index a0c6965832..e5160a71ae 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/b300-fp4/agentic/glm5.2-agentx.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/b300-fp4/agentic/glm5.2-agentx.yaml @@ -33,6 +33,9 @@ base: SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' MC_TE_METRIC: 'true' NCCL_CUMEM_ENABLE: '1' + NCCL_MNNVL_ENABLE: '1' + MC_FORCE_MNNVL: '1' + UCX_TLS: cuda_copy,rc SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' SGLANG_ENABLE_THINKING: '1' @@ -51,6 +54,9 @@ base: SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' MC_TE_METRIC: 'true' NCCL_CUMEM_ENABLE: '1' + NCCL_MNNVL_ENABLE: '1' + MC_FORCE_MNNVL: '1' + UCX_TLS: cuda_copy,rc SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' SGLANG_MOE_NVFP4_DISPATCH: '1' From 55209a8f7c823c94258f69e2a4ec8311f8484e95 Mon Sep 17 00:00:00 2001 From: Sahithi Chigurupati Date: Wed, 19 Aug 2026 10:16:52 -0700 Subject: [PATCH 06/13] fix: update B300 GLM-5.2 AgentX UCX transports Use the requested UCX_TLS override for both prefill and decode environments. --- .../sglang/glm5.2/b300-fp4/agentic/glm5.2-agentx.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/b300-fp4/agentic/glm5.2-agentx.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/b300-fp4/agentic/glm5.2-agentx.yaml index e5160a71ae..747e7a735b 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/b300-fp4/agentic/glm5.2-agentx.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/b300-fp4/agentic/glm5.2-agentx.yaml @@ -35,7 +35,7 @@ base: NCCL_CUMEM_ENABLE: '1' NCCL_MNNVL_ENABLE: '1' MC_FORCE_MNNVL: '1' - UCX_TLS: cuda_copy,rc + UCX_TLS: rc,cuda_copy,cuda_ipc,sm,self SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' SGLANG_ENABLE_THINKING: '1' @@ -56,7 +56,7 @@ base: NCCL_CUMEM_ENABLE: '1' NCCL_MNNVL_ENABLE: '1' MC_FORCE_MNNVL: '1' - UCX_TLS: cuda_copy,rc + UCX_TLS: rc,cuda_copy,cuda_ipc,sm,self SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' SGLANG_MOE_NVFP4_DISPATCH: '1' From e0d3952223d96028af7484654fa3b9d0e8f707d6 Mon Sep 17 00:00:00 2001 From: hshrivastava-droid Date: Thu, 20 Aug 2026 12:34:58 -0700 Subject: [PATCH 07/13] chore(glm5.2): enable UCX debug logging on B300 AgentX disagg recipe MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 将 B300 AgentX 分离式 SGLang 侧的 prefill 与 decode 环境变量中固定的 UCX_TLS 显式设置替换为 UCX_LOG_LEVEL: debug，恢复由 UCX 自行选择传输通道，同时打开 UCX 侧调试日志。 --- .../sglang/glm5.2/b300-fp4/agentic/glm5.2-agentx.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/b300-fp4/agentic/glm5.2-agentx.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/b300-fp4/agentic/glm5.2-agentx.yaml index 747e7a735b..678702a66e 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/b300-fp4/agentic/glm5.2-agentx.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/b300-fp4/agentic/glm5.2-agentx.yaml @@ -35,7 +35,7 @@ base: NCCL_CUMEM_ENABLE: '1' NCCL_MNNVL_ENABLE: '1' MC_FORCE_MNNVL: '1' - UCX_TLS: rc,cuda_copy,cuda_ipc,sm,self + UCX_LOG_LEVEL: debug SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' SGLANG_ENABLE_THINKING: '1' @@ -56,7 +56,7 @@ base: NCCL_CUMEM_ENABLE: '1' NCCL_MNNVL_ENABLE: '1' MC_FORCE_MNNVL: '1' - UCX_TLS: rc,cuda_copy,cuda_ipc,sm,self + UCX_LOG_LEVEL: debug SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' SGLANG_MOE_NVFP4_DISPATCH: '1' From 6bd24245512885b50de6ac00e4b3be280e1d594b Mon Sep 17 00:00:00 2001 From: hshrivastava-droid Date: Thu, 20 Aug 2026 18:04:02 -0700 Subject: [PATCH 08/13] fix(glm5.2): unset UCX_TLS for dynamo-sglang srtctl apply --- runners/launch_b300-nv.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/runners/launch_b300-nv.sh b/runners/launch_b300-nv.sh index 1a95e25d07..b3b0a32bb0 100644 --- a/runners/launch_b300-nv.sh +++ b/runners/launch_b300-nv.sh @@ -233,7 +233,11 @@ fi if [[ -n "$SRTCTL_SETUP_SCRIPT" ]]; then SRTCTL_APPLY_ARGS+=(--setup-script "$SRTCTL_SETUP_SCRIPT") fi -SRTCTL_OUTPUT=$(srtctl apply "${SRTCTL_APPLY_ARGS[@]}" 2>&1) +if [[ "$MODEL_PREFIX" == "glm5.2" && "$FRAMEWORK" == "dynamo-sglang" ]]; then + SRTCTL_OUTPUT=$(env -u UCX_TLS srtctl apply "${SRTCTL_APPLY_ARGS[@]}" 2>&1) +else + SRTCTL_OUTPUT=$(srtctl apply "${SRTCTL_APPLY_ARGS[@]}" 2>&1) +fi echo "$SRTCTL_OUTPUT" # Extract JOB_ID from srtctl output From e8c0a18bf7f090f1dad2e77e08729e979eba2e30 Mon Sep 17 00:00:00 2001 From: hshrivastava-droid Date: Tue, 25 Aug 2026 03:04:08 -0700 Subject: [PATCH 09/13] recipe(glm5.2-b300-agentx): adopt upstream srt-slurm recipes, bump image MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 将 GLM-5.2 B300 AgentX 的两份 recipe 替换为上游 srt-slurm 版本（base + zip_override_mtp_agentx_lowlat），将 sglang 镜像升级到 nightly-dev-cu13-20260824-95f5ecd3，并把 nvidia-master.yaml 的 -agg 条目拆分为对应两个 zip_override 索引（c2 / c4）。 --- .../b300-fp4/agentic/glm5.2-agentx-agg.yaml | 274 +++++++++++------- .../b300-fp4/agentic/glm5.2-agentx.yaml | 83 ++++-- configs/nvidia-master.yaml | 27 +- 3 files changed, 249 insertions(+), 135 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/b300-fp4/agentic/glm5.2-agentx-agg.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/b300-fp4/agentic/glm5.2-agentx-agg.yaml index f77303daab..1a2867ca7f 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/b300-fp4/agentic/glm5.2-agentx-agg.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/b300-fp4/agentic/glm5.2-agentx-agg.yaml @@ -1,112 +1,162 @@ -# Agentic-coding SGLang aggregated recipe for GLM-5.2-NVFP4 on B300 -# (single aggregated worker, TP8 on one 8-GPU B300 node, EAGLE MTP + HiCache). -# -# Flat single-variant schema the agentic CI flow expects: applied via -# CONFIG_FILE= with no zip_override selector, so it must NOT be -# base:-wrapped. Concurrency is not a recipe field; the GHA matrix fans out -# one job per concurrency from the master-config conc-list into agentic_srt.sh. -name: b300-fp4-glm5.2-agentx-agg -model: - path: glm-5.2-fp4 - container: dynamo-sglang - precision: fp4 -resources: - gpu_type: b300 - gpus_per_node: 8 - agg_nodes: 1 - agg_workers: 1 - gpus_per_agg: 8 -frontend: - type: dynamo - env: - PIP_BREAK_SYSTEM_PACKAGES: '1' - args: - router-mode: kv - active-decode-blocks-threshold: None - active-prefill-tokens-threshold: None - active-prefill-tokens-threshold-frac: None - router-session-affinity-ttl-secs: 3600 -dynamo: - hash: 71eb001e17fa73c742f0afe1a6ed96836cb135fd - install: true -backend: - type: sglang - sglang_config: - aggregated: - served-model-name: nvidia/GLM-5.2-NVFP4 - trust-remote-code: true - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - tensor-parallel-size: 8 - data-parallel-size: 1 - expert-parallel-size: 1 - enable-dp-attention: false - enable-dp-lm-head: false - max-running-requests: 10 - cuda-graph-max-bs: 10 - chunked-prefill-size: 8192 - max-prefill-tokens: 8192 - context-length: 1048576 - speculative-algorithm: EAGLE - speculative-num-steps: 4 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 5 - nsa-prefill-backend: trtllm - nsa-decode-backend: trtllm - moe-runner-backend: flashinfer_trtllm - fp4-gemm-backend: flashinfer_trtllm - disable-shared-experts-fusion: true - enable-flashinfer-allreduce-fusion: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - mem-fraction-static: 0.8 - enable-hierarchical-cache: true - hicache-write-policy: write_back - hicache-size: 270 - hicache-io-backend: direct - enable-metrics: true - enable-cache-report: true - aggregated_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - PYTHONUNBUFFERED: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - MC_TE_METRIC: 'true' - NCCL_CUMEM_ENABLE: '1' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - SGLANG_ENABLE_THINKING: '1' - SGLANG_REASONING_EFFORT: max - SGLANG_ENABLE_UNIFIED_RADIX_TREE: '1' - SGLANG_HICACHE_DEBUG_LOG: '1' - SGLANG_HICACHE_DEBUG_SAMPLE_RATE: '16384' - SGLANG_MOE_NVFP4_DISPATCH: '1' - SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' - SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: '8' - SGLANG_SIMULATE_ACC_LEN: '3.33' - SGLANG_SIMULATE_ACC_METHOD: match-expected - SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token - PIP_BREAK_SYSTEM_PACKAGES: '1' -health_check: - max_attempts: 1440 - interval_seconds: 10 -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: "8000" - IS_MULTINODE: "false" - TP: "8" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache -infra: - etcd_nats_dedicated_node: false - nats_max_payload_mb: 64 -sbatch_directives: - mem: '0' - cpus-per-task: '192' -srun_options: - mem: '0' - container-remap-root: '' +base: + name: b300-fp4-glm5.2-agentx-agg + model: + path: glm-5.2-fp4 + container: nightly-20260824 + precision: fp4 + identity: + model: + repo: nvidia/GLM-5.2-NVFP4 + revision: aec724e + frameworks: + sglang: nightly-dev-cu13-20260824-95f5ecd3 + resources: + gpu_type: b300 + gpus_per_node: 8 + dynamo: + hash: 71eb001e17fa73c742f0afe1a6ed96836cb135fd + install: true + backend: + type: sglang + sglang_config: + aggregated: + served-model-name: GLM-5.2-FP4 + trust-remote-code: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 8 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-dp-attention: false + enable-dp-lm-head: false + max-running-requests: 10 + cuda-graph-max-bs: 10 + chunked-prefill-size: 8192 + max-prefill-tokens: 8192 + context-length: 1048576 + speculative-algorithm: EAGLE + speculative-num-steps: 4 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 5 + nsa-prefill-backend: trtllm + nsa-decode-backend: trtllm + moe-runner-backend: flashinfer_trtllm + fp4-gemm-backend: flashinfer_trtllm + disable-shared-experts-fusion: true + enable-flashinfer-allreduce-fusion: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + mem-fraction-static: 0.8 + enable-hierarchical-cache: true + hicache-write-policy: write_back + hicache-size: 270 + hicache-io-backend: direct + enable-metrics: true + enable-cache-report: true + aggregated_environment: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + MC_TE_METRIC: 'true' + NCCL_CUMEM_ENABLE: '1' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_ENABLE_THINKING: '1' + SGLANG_REASONING_EFFORT: max + SGLANG_ENABLE_UNIFIED_RADIX_TREE: '1' + SGLANG_HICACHE_DEBUG_LOG: '1' + SGLANG_HICACHE_DEBUG_SAMPLE_RATE: '16384' + SGLANG_MOE_NVFP4_DISPATCH: '1' + SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' + SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: '8' + SGLANG_SIMULATE_ACC_LEN: '3.33' + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token + PIP_BREAK_SYSTEM_PACKAGES: '1' + benchmark: + type: custom + env: + CONCURRENCY: '2' + DURATION: '3600' + AIPERF_HF_HOME: /hf_home + HF_HOME: /hf_home + HF_HUB_CACHE: /hf_home/hub + HF_DATASETS_CACHE: /hf_home/datasets + HF_HUB_OFFLINE: '1' + HF_DATASETS_OFFLINE: '1' + HF_HUB_DISABLE_XET: '1' + XDG_CACHE_HOME: /tmp/cache + AIPERF_DATASET_MMAP_CACHE_DIR: /hf_home/mmap_cache + AIPERF_DATASET_CONFIGURATION_TIMEOUT: '1800' + AIPERF_SERVICE_PROFILE_CONFIGURE_TIMEOUT: '1800' + AIPERF_FAILED_REQUEST_THRESHOLD: '0.1' + AIPERF_RECORD_STRIP_PAYLOAD_BYTES: 'true' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' + command: | + set -euo pipefail + export HF_HOME="${AIPERF_HF_HOME:-/hf_home}" + export HF_HUB_CACHE="${HF_HOME}/hub" + export HF_DATASETS_CACHE="${HF_HOME}/datasets" + mkdir -p "${XDG_CACHE_HOME}" "${AIPERF_DATASET_MMAP_CACHE_DIR}" /logs/aiperf + /aiperf-venv/bin/aiperf profile \ + --scenario inferencex-agentx-mvp \ + --url http://localhost:8000 \ + --endpoint /v1/chat/completions \ + --endpoint-type chat \ + --streaming \ + --model GLM-5.2-FP4 \ + --tokenizer /model \ + --tokenizer-trust-remote-code \ + --concurrency "${CONCURRENCY}" \ + --benchmark-duration "${DURATION}" \ + --random-seed 42 \ + --failed-request-threshold "${AIPERF_FAILED_REQUEST_THRESHOLD}" \ + --stats-interval 30 \ + --warmup-requests-per-lane 10 \ + --warmup-grace-period 1800 \ + --trajectory-start-min-ratio 0.25 \ + --trajectory-start-max-ratio 0.75 \ + --use-server-token-count \ + --no-gpu-telemetry \ + --num-dataset-entries 393 \ + --slice-duration 1.0 \ + --public-dataset semianalysis_cc_traces_weka_062126 \ + --trace-idle-gap-cap-seconds 300 \ + --unsafe-override \ + --ui simple \ + --server-metrics-formats json \ + --output-artifact-dir /logs/aiperf + rm -f /logs/aiperf/inputs.json + health_check: + max_attempts: 1440 + interval_seconds: 10 + infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 64 + frontend: + type: dynamo + env: + PIP_BREAK_SYSTEM_PACKAGES: '1' + args: + router-mode: kv + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None + router-session-affinity-ttl-secs: 3600 + sbatch_directives: + mem: '0' + srun_options: + mem: '0' + container-remap-root: '' +# ################# agentx ################# +zip_override_mtp_agentx_lowlat: + name: [agentx-agg1_tp8-c2, agentx-agg1_tp8-c4] + benchmark: + env: + CONCURRENCY: ['2', '4'] + resources: + agg_nodes: 1 + agg_workers: 1 + gpus_per_decode: 8 + gpus_per_prefill: 8 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/b300-fp4/agentic/glm5.2-agentx.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/b300-fp4/agentic/glm5.2-agentx.yaml index 678702a66e..e3e36f7235 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/b300-fp4/agentic/glm5.2-agentx.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/b300-fp4/agentic/glm5.2-agentx.yaml @@ -2,8 +2,15 @@ base: name: b300-fp4-glm5.2-agentx model: path: glm-5.2-fp4 - container: dynamo-sglang + container: nightly-20260824 precision: fp4 + identity: + model: + repo: nvidia/GLM-5.2-NVFP4 + revision: aec724e + frameworks: + dynamo: 1.2.1 + sglang: nightly-dev-cu13-20260824-95f5ecd3 resources: gpu_type: b300 gpus_per_node: 8 @@ -33,9 +40,6 @@ base: SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' MC_TE_METRIC: 'true' NCCL_CUMEM_ENABLE: '1' - NCCL_MNNVL_ENABLE: '1' - MC_FORCE_MNNVL: '1' - UCX_LOG_LEVEL: debug SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' SGLANG_ENABLE_THINKING: '1' @@ -54,9 +58,6 @@ base: SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' MC_TE_METRIC: 'true' NCCL_CUMEM_ENABLE: '1' - NCCL_MNNVL_ENABLE: '1' - MC_FORCE_MNNVL: '1' - UCX_LOG_LEVEL: debug SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' SGLANG_MOE_NVFP4_DISPATCH: '1' @@ -70,7 +71,7 @@ base: SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token sglang_config: prefill: - served-model-name: nvidia/GLM-5.2-NVFP4 + served-model-name: GLM-5.2-FP4 trust-remote-code: true quantization: modelopt_fp4 kv-cache-dtype: fp8_e4m3 @@ -107,7 +108,7 @@ base: enable-metrics: true enable-cache-report: true decode: - served-model-name: nvidia/GLM-5.2-NVFP4 + served-model-name: GLM-5.2-FP4 trust-remote-code: true quantization: modelopt_fp4 kv-cache-dtype: fp8_e4m3 @@ -145,21 +146,64 @@ base: interval_seconds: 10 benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache + CONCURRENCY: '63' + DURATION: '3600' + AIPERF_HF_HOME: /hf_home + HF_HOME: /hf_home + HF_HUB_CACHE: /hf_home/hub + HF_DATASETS_CACHE: /hf_home/datasets + HF_HUB_OFFLINE: '1' + HF_DATASETS_OFFLINE: '1' + HF_HUB_DISABLE_XET: '1' + XDG_CACHE_HOME: /tmp/cache + AIPERF_DATASET_MMAP_CACHE_DIR: /hf_home/mmap_cache + AIPERF_DATASET_CONFIGURATION_TIMEOUT: '1800' + AIPERF_SERVICE_PROFILE_CONFIGURE_TIMEOUT: '1800' + AIPERF_FAILED_REQUEST_THRESHOLD: '0.1' + AIPERF_RECORD_STRIP_PAYLOAD_BYTES: 'true' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' + command: | + set -euo pipefail + export HF_HOME="${AIPERF_HF_HOME:-/hf_home}" + export HF_HUB_CACHE="${HF_HOME}/hub" + export HF_DATASETS_CACHE="${HF_HOME}/datasets" + mkdir -p "${XDG_CACHE_HOME}" "${AIPERF_DATASET_MMAP_CACHE_DIR}" /logs/aiperf + /aiperf-venv/bin/aiperf profile \ + --scenario inferencex-agentx-mvp \ + --url http://localhost:8000 \ + --endpoint /v1/chat/completions \ + --endpoint-type chat \ + --streaming \ + --model GLM-5.2-FP4 \ + --tokenizer /model \ + --tokenizer-trust-remote-code \ + --concurrency "${CONCURRENCY}" \ + --benchmark-duration "${DURATION}" \ + --random-seed 42 \ + --failed-request-threshold "${AIPERF_FAILED_REQUEST_THRESHOLD}" \ + --stats-interval 30 \ + --warmup-requests-per-lane 10 \ + --warmup-grace-period 1800 \ + --trajectory-start-min-ratio 0.25 \ + --trajectory-start-max-ratio 0.75 \ + --use-server-token-count \ + --no-gpu-telemetry \ + --num-dataset-entries 393 \ + --slice-duration 1.0 \ + --public-dataset semianalysis_cc_traces_weka_062126 \ + --trace-idle-gap-cap-seconds 300 \ + --unsafe-override \ + --ui simple \ + --server-metrics-formats json \ + --output-artifact-dir /logs/aiperf + rm -f /logs/aiperf/inputs.json infra: etcd_nats_dedicated_node: false nats_max_payload_mb: 64 sbatch_directives: mem: '0' - cpus-per-task: '192' srun_options: mem: '0' container-remap-root: '' @@ -175,6 +219,9 @@ zip_override_mtp_agentx_lowlat: enable-dp-lm-head: [false, false, false, false, true] max-running-requests: [16, 16, 16, 16, 139] tensor-parallel-size: [4, 4, 4, 4, 8] + benchmark: + env: + CONCURRENCY: ['63', '61', '59', '58', '139'] frontend: enable_multiple_frontends: true resources: diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 85e3e43dbf..2ce45dfd92 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -8710,7 +8710,7 @@ glm5.2-fp4-gb200-dynamo-sglang-agentic-mtp-agg: dp-attn: false glm5.2-fp4-b300-dynamo-sglang-agentic-agg: - image: lmsysorg/sglang:nightly-dev-cu13-20260805-211ee642 + image: lmsysorg/sglang:nightly-dev-cu13-20260824-95f5ecd3 model: nvidia/GLM-5.2-NVFP4 model-prefix: glm5.2 runner: cluster:b300-nv @@ -8723,9 +8723,9 @@ glm5.2-fp4-b300-dynamo-sglang-agentic-agg: agentic-coding: - dram-utilization: 0.80 search-space: - # aggregated 1-worker TP8 (prefill+decode on one 8-GPU B300 node) + # aggregated 1-worker TP8 (prefill+decode on one 8-GPU B300 node), conc 2 - spec-decoding: mtp - conc-list: [2, 4] + conc-list: [2] kv-offloading: dram kv-offload-backend: { name: hicache } prefill: @@ -8734,7 +8734,7 @@ glm5.2-fp4-b300-dynamo-sglang-agentic-agg: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5.2/b300-fp4/agentic/glm5.2-agentx-agg.yaml" + - "CONFIG_FILE=recipes/sglang/glm5.2/b300-fp4/agentic/glm5.2-agentx-agg.yaml:zip_override_mtp_agentx_lowlat[0]" # Aggregated worker serves prefill and decode on the same GPUs. # Keep decode at zero to avoid double-counting the TP8 allocation. decode: @@ -8742,9 +8742,26 @@ glm5.2-fp4-b300-dynamo-sglang-agentic-agg: tp: 8 ep: 1 dp-attn: false + # aggregated 1-worker TP8 (prefill+decode on one 8-GPU B300 node), conc 4 + - spec-decoding: mtp + conc-list: [4] + kv-offloading: dram + kv-offload-backend: { name: hicache } + prefill: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5.2/b300-fp4/agentic/glm5.2-agentx-agg.yaml:zip_override_mtp_agentx_lowlat[1]" + decode: + num-worker: 0 + tp: 8 + ep: 1 + dp-attn: false glm5.2-fp4-b300-dynamo-sglang-agentic-disagg: - image: lmsysorg/sglang:nightly-dev-cu13-20260805-211ee642 + image: lmsysorg/sglang:nightly-dev-cu13-20260824-95f5ecd3 model: nvidia/GLM-5.2-NVFP4 model-prefix: glm5.2 runner: cluster:b300-nv From a11f922ae046c19b8805870be2835b2b83dbea66 Mon Sep 17 00:00:00 2001 From: hshrivastava-droid Date: Tue, 25 Aug 2026 16:45:38 -0700 Subject: [PATCH 10/13] fix(glm5.2): resolve B300 AgentX recipe image MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 修复 GLM-5.2 B300 AgentX 配方镜像解析。 --- .../sglang/glm5.2/b300-fp4/agentic/glm5.2-agentx-agg.yaml | 2 +- .../sglang/glm5.2/b300-fp4/agentic/glm5.2-agentx.yaml | 2 +- perf-changelog.yaml | 2 +- runners/launch_b300-nv.sh | 4 +++- 4 files changed, 6 insertions(+), 4 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/b300-fp4/agentic/glm5.2-agentx-agg.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/b300-fp4/agentic/glm5.2-agentx-agg.yaml index 1a2867ca7f..adb7225531 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/b300-fp4/agentic/glm5.2-agentx-agg.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/b300-fp4/agentic/glm5.2-agentx-agg.yaml @@ -2,7 +2,7 @@ base: name: b300-fp4-glm5.2-agentx-agg model: path: glm-5.2-fp4 - container: nightly-20260824 + container: lmsysorg/sglang:nightly-dev-cu13-20260824-95f5ecd3 precision: fp4 identity: model: diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/b300-fp4/agentic/glm5.2-agentx.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/b300-fp4/agentic/glm5.2-agentx.yaml index e3e36f7235..d6dd47d1ae 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/b300-fp4/agentic/glm5.2-agentx.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/b300-fp4/agentic/glm5.2-agentx.yaml @@ -2,7 +2,7 @@ base: name: b300-fp4-glm5.2-agentx model: path: glm-5.2-fp4 - container: nightly-20260824 + container: lmsysorg/sglang:nightly-dev-cu13-20260824-95f5ecd3 precision: fp4 identity: model: diff --git a/perf-changelog.yaml b/perf-changelog.yaml index c511e0df8f..76a1774e1b 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6418,5 +6418,5 @@ description: - "Add GLM-5.2-NVFP4 B300 AgentX (agentic-coding) Dynamo-SGLang benchmarks over a 7-point sweep: aggregated single-node TP8 (conc 2/4), disaggregated low-latency TP4-prefill/TP4-decode 1P2D (conc 63), 1P3D (conc 61), 1P4D (conc 59), 1P6D (conc 58), and throughput 2P1D DEP8-decode (conc 139)" - "EAGLE speculative decoding, hierarchical KV cache (hicache-size 270), MTP, decode mem-fraction-static 0.92, nixl KV transfer, and X-Dynamo-Session-ID session affinity; driven from the NVIDIA/srt-slurm base+zip_override recipes via CONFIG_FILE selectors" - - "Image: lmsysorg/sglang:nightly-dev-cu13-20260805-211ee642; runner: b300-nv" + - "Image: lmsysorg/sglang:nightly-dev-cu13-20260824-95f5ecd3; runner: b300-nv" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2654 diff --git a/runners/launch_b300-nv.sh b/runners/launch_b300-nv.sh index b3b0a32bb0..45644a3084 100644 --- a/runners/launch_b300-nv.sh +++ b/runners/launch_b300-nv.sh @@ -79,7 +79,9 @@ if [[ "$IS_AGENTIC" == "1" && $FRAMEWORK == "dynamo-sglang" && $MODEL_PREFIX == # Must precede the generic IS_AGENTIC catch-all below (it clones a different fork). git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" cd "$SRT_REPO_DIR" || exit 1 - git checkout main + # NVIDIA/srt-slurm#313 is still open, so pin its reviewed recipe contract. + git fetch origin pull/313/head || exit 1 + git checkout --detach 93fae852166a30f3cc054c8616228bf62c69c48c || exit 1 mkdir -p recipes/sglang/glm5.2/b300-fp4 cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/b300-fp4" recipes/sglang/glm5.2/b300-fp4 elif [[ "$IS_AGENTIC" == "1" ]]; then From 6f5d148090f5932eb351f8df56aa40d9923dabfc Mon Sep 17 00:00:00 2001 From: hshrivastava-droid Date: Thu, 10 Sep 2026 14:33:04 -0700 Subject: [PATCH 11/13] feat(glm5.2): route B300 AgentX agg/disagg to cluster:b300-dsxe Co-Authored-By: Claude Sonnet 5 --- configs/nvidia-master.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 2306893734..41a2a425cf 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -9117,7 +9117,7 @@ glm5.2-fp4-b300-dynamo-sglang-agentic-agg: image: lmsysorg/sglang:nightly-dev-cu13-20260824-95f5ecd3 model: nvidia/GLM-5.2-NVFP4 model-prefix: glm5.2 - runner: cluster:b300-nv + runner: cluster:b300-dsxe precision: fp4 framework: dynamo-sglang router: { name: dynamo-router, version: "71eb001e17fa73c742f0afe1a6ed96836cb135fd" } @@ -9168,7 +9168,7 @@ glm5.2-fp4-b300-dynamo-sglang-agentic-disagg: image: lmsysorg/sglang:nightly-dev-cu13-20260824-95f5ecd3 model: nvidia/GLM-5.2-NVFP4 model-prefix: glm5.2 - runner: cluster:b300-nv + runner: cluster:b300-dsxe precision: fp4 framework: dynamo-sglang router: { name: dynamo-router, version: "71eb001e17fa73c742f0afe1a6ed96836cb135fd" } From 0c044132685e7d9e66f9cfae0a5294f4da82eb92 Mon Sep 17 00:00:00 2001 From: hshrivastava-droid Date: Thu, 10 Sep 2026 14:37:30 -0700 Subject: [PATCH 12/13] feat(glm5.2): port B300 AgentX launcher logic to launch_b300-dsxe.sh launch_b300-nv.sh was retired on main (#2826) in favor of the B300 DSXE cluster, so PR #2654's runner move to cluster:b300-dsxe needs the GLM-5.2 AgentX special-casing carried over: the pinned NVIDIA/srt-slurm#313 fork checkout (scoped to just the glm5.2/b300-fp4 recipes), and the UCX_TLS-unset wrapper around srtctl apply for dynamo-sglang. Co-Authored-By: Claude Sonnet 5 --- runners/launch_b300-dsxe.sh | 58 +++++++++++++++++++++++++------------ 1 file changed, 39 insertions(+), 19 deletions(-) diff --git a/runners/launch_b300-dsxe.sh b/runners/launch_b300-dsxe.sh index 70254deb65..315d3cb4cc 100755 --- a/runners/launch_b300-dsxe.sh +++ b/runners/launch_b300-dsxe.sh @@ -159,27 +159,43 @@ select_srt_slurm_version() { SRT_REPO_DIR="srt-slurm" rm -rf "$SRT_REPO_DIR" -if [[ "$USES_DCGM_POWER" == "1" ]]; then - SRT_SLURM_REPO="$POWER_SRT_SLURM_URL" - SRT_SLURM_REF="$POWER_SRT_SLURM_PIN" +if [[ "$IS_AGENTIC" == "1" && $FRAMEWORK == "dynamo-sglang" && $MODEL_PREFIX == "glm5.2" && $PRECISION == "fp4" ]]; then + # GLM-5.2 B300 sglang AgentX: srt-slurm main carries the agentx-mvp scenario, + # session-affinity frontend, and custom benchmark schema these recipes need. + # Only glm5.2/b300-fp4 recipes get copied onto this pinned fork checkout -- + # its recipes/ tree follows a contract other models haven't adopted yet. + echo "Cloning srt-slurm (https://github.com/NVIDIA/srt-slurm.git, pinned to open PR #313)..." + git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" || exit 1 + cd "$SRT_REPO_DIR" || exit 1 + # NVIDIA/srt-slurm#313 is still open, so pin its reviewed recipe contract. + git fetch origin pull/313/head || exit 1 + git checkout --detach 93fae852166a30f3cc054c8616228bf62c69c48c || exit 1 + git rev-parse HEAD > "$GITHUB_WORKSPACE/srt-slurm-sha.txt" + mkdir -p recipes/sglang/glm5.2/b300-fp4 + cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/b300-fp4" recipes/sglang/glm5.2/b300-fp4 || exit 1 else - select_srt_slurm_version -fi + if [[ "$USES_DCGM_POWER" == "1" ]]; then + SRT_SLURM_REPO="$POWER_SRT_SLURM_URL" + SRT_SLURM_REF="$POWER_SRT_SLURM_PIN" + else + select_srt_slurm_version + fi -echo "Cloning srt-slurm ($SRT_SLURM_REPO @ $SRT_SLURM_REF)..." -git clone "$SRT_SLURM_REPO" "$SRT_REPO_DIR" || exit 1 -cd "$SRT_REPO_DIR" || exit 1 -git checkout --quiet "$SRT_SLURM_REF" || exit 1 -git rev-parse HEAD > "$GITHUB_WORKSPACE/srt-slurm-sha.txt" -if [[ "$USES_DCGM_POWER" == "1" ]]; then - test "$(git rev-parse HEAD)" = "$POWER_SRT_SLURM_PIN" \ - || { echo "Error: srt-slurm HEAD does not match POWER_SRT_SLURM_PIN=$POWER_SRT_SLURM_PIN" >&2; exit 1; } - cp "$GITHUB_WORKSPACE/srt-slurm-sha.txt" "$GITHUB_WORKSPACE/power-producer-sha.txt" -fi + echo "Cloning srt-slurm ($SRT_SLURM_REPO @ $SRT_SLURM_REF)..." + git clone "$SRT_SLURM_REPO" "$SRT_REPO_DIR" || exit 1 + cd "$SRT_REPO_DIR" || exit 1 + git checkout --quiet "$SRT_SLURM_REF" || exit 1 + git rev-parse HEAD > "$GITHUB_WORKSPACE/srt-slurm-sha.txt" + if [[ "$USES_DCGM_POWER" == "1" ]]; then + test "$(git rev-parse HEAD)" = "$POWER_SRT_SLURM_PIN" \ + || { echo "Error: srt-slurm HEAD does not match POWER_SRT_SLURM_PIN=$POWER_SRT_SLURM_PIN" >&2; exit 1; } + cp "$GITHUB_WORKSPACE/srt-slurm-sha.txt" "$GITHUB_WORKSPACE/power-producer-sha.txt" + fi -# Recipes live in this repo; overlay all of them onto the checkout's recipes/ dir. -mkdir -p recipes -cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes" recipes || exit 1 + # Recipes live in this repo; overlay all of them onto the checkout's recipes/ dir. + mkdir -p recipes + cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes" recipes || exit 1 +fi if [[ "${EVAL_FRAMEWORK:-lm-eval}" != "lm-eval" ]]; then python3 "$GITHUB_WORKSPACE/runners/patch_srt_eval_dispatch.py" "$(pwd)" || exit 1 @@ -293,7 +309,11 @@ SRTCTL_APPLY_ARGS=( --no-preflight --tags "b300,${MODEL_PREFIX},${PRECISION},${ISL}x${OSL},infmax-$(date +%Y%m%d)" ) -SRTCTL_OUTPUT=$(srtctl apply "${SRTCTL_APPLY_ARGS[@]}" 2>&1) +if [[ "$MODEL_PREFIX" == "glm5.2" && "$FRAMEWORK" == "dynamo-sglang" ]]; then + SRTCTL_OUTPUT=$(env -u UCX_TLS srtctl apply "${SRTCTL_APPLY_ARGS[@]}" 2>&1) +else + SRTCTL_OUTPUT=$(srtctl apply "${SRTCTL_APPLY_ARGS[@]}" 2>&1) +fi echo "$SRTCTL_OUTPUT" # Extract JOB_ID from srtctl output From 6588cb230974364b199e26b09d56c62ad116e479 Mon Sep 17 00:00:00 2001 From: hshrivastava-droid Date: Thu, 10 Sep 2026 14:39:25 -0700 Subject: [PATCH 13/13] fix(glm5.2): migrate B300 agg entry to the current worker: schema Master config now rejects disagg=false entries that use prefill/decode instead of a single worker: block (validated by infx.matrix.validation.MultiNodeMasterConfigEntry, which landed on main after this PR's branch point). Convert glm5.2-fp4-b300-dynamo-sglang-agentic-agg's two search-space points to worker:/num-nodes: to match, mirroring the sibling gb200-mtp-agg entry. Co-Authored-By: Claude Sonnet 5 --- configs/nvidia-master.yaml | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index d6bafb3878..5804468870 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -9349,37 +9349,27 @@ glm5.2-fp4-b300-dynamo-sglang-agentic-agg: conc-list: [2] kv-offloading: dram kv-offload-backend: { name: hicache } - prefill: + num-nodes: 1 + worker: num-worker: 1 tp: 8 ep: 1 dp-attn: false additional-settings: - "CONFIG_FILE=recipes/sglang/glm5.2/b300-fp4/agentic/glm5.2-agentx-agg.yaml:zip_override_mtp_agentx_lowlat[0]" - # Aggregated worker serves prefill and decode on the same GPUs. - # Keep decode at zero to avoid double-counting the TP8 allocation. - decode: - num-worker: 0 - tp: 8 - ep: 1 - dp-attn: false # aggregated 1-worker TP8 (prefill+decode on one 8-GPU B300 node), conc 4 - spec-decoding: mtp conc-list: [4] kv-offloading: dram kv-offload-backend: { name: hicache } - prefill: + num-nodes: 1 + worker: num-worker: 1 tp: 8 ep: 1 dp-attn: false additional-settings: - "CONFIG_FILE=recipes/sglang/glm5.2/b300-fp4/agentic/glm5.2-agentx-agg.yaml:zip_override_mtp_agentx_lowlat[1]" - decode: - num-worker: 0 - tp: 8 - ep: 1 - dp-attn: false glm5.2-fp4-b300-dynamo-sglang-agentic-disagg: image: lmsysorg/sglang:nightly-dev-cu13-20260824-95f5ecd3