diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/b300-fp4/agentic/glm5.2-agentx-agg.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/b300-fp4/agentic/glm5.2-agentx-agg.yaml new file mode 100644 index 0000000000..adb7225531 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/b300-fp4/agentic/glm5.2-agentx-agg.yaml @@ -0,0 +1,162 @@ +base: + name: b300-fp4-glm5.2-agentx-agg + model: + path: glm-5.2-fp4 + container: lmsysorg/sglang:nightly-dev-cu13-20260824-95f5ecd3 + precision: fp4 + identity: + model: + repo: nvidia/GLM-5.2-NVFP4 + revision: aec724e + frameworks: + sglang: nightly-dev-cu13-20260824-95f5ecd3 + resources: + gpu_type: b300 + gpus_per_node: 8 + dynamo: + hash: 71eb001e17fa73c742f0afe1a6ed96836cb135fd + install: true + backend: + type: sglang + sglang_config: + aggregated: + served-model-name: GLM-5.2-FP4 + trust-remote-code: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 8 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-dp-attention: false + enable-dp-lm-head: false + max-running-requests: 10 + cuda-graph-max-bs: 10 + chunked-prefill-size: 8192 + max-prefill-tokens: 8192 + context-length: 1048576 + speculative-algorithm: EAGLE + speculative-num-steps: 4 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 5 + nsa-prefill-backend: trtllm + nsa-decode-backend: trtllm + moe-runner-backend: flashinfer_trtllm + fp4-gemm-backend: flashinfer_trtllm + disable-shared-experts-fusion: true + enable-flashinfer-allreduce-fusion: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + mem-fraction-static: 0.8 + enable-hierarchical-cache: true + hicache-write-policy: write_back + hicache-size: 270 + hicache-io-backend: direct + enable-metrics: true + enable-cache-report: true + aggregated_environment: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + MC_TE_METRIC: 'true' + NCCL_CUMEM_ENABLE: '1' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_ENABLE_THINKING: '1' + SGLANG_REASONING_EFFORT: max + SGLANG_ENABLE_UNIFIED_RADIX_TREE: '1' + SGLANG_HICACHE_DEBUG_LOG: '1' + SGLANG_HICACHE_DEBUG_SAMPLE_RATE: '16384' + SGLANG_MOE_NVFP4_DISPATCH: '1' + SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' + SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: '8' + SGLANG_SIMULATE_ACC_LEN: '3.33' + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token + PIP_BREAK_SYSTEM_PACKAGES: '1' + benchmark: + type: custom + env: + CONCURRENCY: '2' + DURATION: '3600' + AIPERF_HF_HOME: /hf_home + HF_HOME: /hf_home + HF_HUB_CACHE: /hf_home/hub + HF_DATASETS_CACHE: /hf_home/datasets + HF_HUB_OFFLINE: '1' + HF_DATASETS_OFFLINE: '1' + HF_HUB_DISABLE_XET: '1' + XDG_CACHE_HOME: /tmp/cache + AIPERF_DATASET_MMAP_CACHE_DIR: /hf_home/mmap_cache + AIPERF_DATASET_CONFIGURATION_TIMEOUT: '1800' + AIPERF_SERVICE_PROFILE_CONFIGURE_TIMEOUT: '1800' + AIPERF_FAILED_REQUEST_THRESHOLD: '0.1' + AIPERF_RECORD_STRIP_PAYLOAD_BYTES: 'true' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' + command: | + set -euo pipefail + export HF_HOME="${AIPERF_HF_HOME:-/hf_home}" + export HF_HUB_CACHE="${HF_HOME}/hub" + export HF_DATASETS_CACHE="${HF_HOME}/datasets" + mkdir -p "${XDG_CACHE_HOME}" "${AIPERF_DATASET_MMAP_CACHE_DIR}" /logs/aiperf + /aiperf-venv/bin/aiperf profile \ + --scenario inferencex-agentx-mvp \ + --url http://localhost:8000 \ + --endpoint /v1/chat/completions \ + --endpoint-type chat \ + --streaming \ + --model GLM-5.2-FP4 \ + --tokenizer /model \ + --tokenizer-trust-remote-code \ + --concurrency "${CONCURRENCY}" \ + --benchmark-duration "${DURATION}" \ + --random-seed 42 \ + --failed-request-threshold "${AIPERF_FAILED_REQUEST_THRESHOLD}" \ + --stats-interval 30 \ + --warmup-requests-per-lane 10 \ + --warmup-grace-period 1800 \ + --trajectory-start-min-ratio 0.25 \ + --trajectory-start-max-ratio 0.75 \ + --use-server-token-count \ + --no-gpu-telemetry \ + --num-dataset-entries 393 \ + --slice-duration 1.0 \ + --public-dataset semianalysis_cc_traces_weka_062126 \ + --trace-idle-gap-cap-seconds 300 \ + --unsafe-override \ + --ui simple \ + --server-metrics-formats json \ + --output-artifact-dir /logs/aiperf + rm -f /logs/aiperf/inputs.json + health_check: + max_attempts: 1440 + interval_seconds: 10 + infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 64 + frontend: + type: dynamo + env: + PIP_BREAK_SYSTEM_PACKAGES: '1' + args: + router-mode: kv + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None + router-session-affinity-ttl-secs: 3600 + sbatch_directives: + mem: '0' + srun_options: + mem: '0' + container-remap-root: '' +# ################# agentx ################# +zip_override_mtp_agentx_lowlat: + name: [agentx-agg1_tp8-c2, agentx-agg1_tp8-c4] + benchmark: + env: + CONCURRENCY: ['2', '4'] + resources: + agg_nodes: 1 + agg_workers: 1 + gpus_per_decode: 8 + gpus_per_prefill: 8 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/b300-fp4/agentic/glm5.2-agentx.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/b300-fp4/agentic/glm5.2-agentx.yaml new file mode 100644 index 0000000000..d6dd47d1ae --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/b300-fp4/agentic/glm5.2-agentx.yaml @@ -0,0 +1,233 @@ +base: + name: b300-fp4-glm5.2-agentx + model: + path: glm-5.2-fp4 + container: lmsysorg/sglang:nightly-dev-cu13-20260824-95f5ecd3 + precision: fp4 + identity: + model: + repo: nvidia/GLM-5.2-NVFP4 + revision: aec724e + frameworks: + dynamo: 1.2.1 + sglang: nightly-dev-cu13-20260824-95f5ecd3 + resources: + gpu_type: b300 + gpus_per_node: 8 + frontend: + type: dynamo + nginx_session_affinity: true + nginx_session_affinity_header: X-Dynamo-Session-ID + env: + PIP_BREAK_SYSTEM_PACKAGES: '1' + args: + router-mode: kv + router-session-affinity-ttl-secs: '3600' + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None + dynamo: + hash: 71eb001e17fa73c742f0afe1a6ed96836cb135fd + install: true + backend: + type: sglang + prefill_environment: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + NCCL_CUMEM_ENABLE: '1' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_ENABLE_THINKING: '1' + SGLANG_ENABLE_UNIFIED_RADIX_TREE: '1' + SGLANG_HICACHE_DEBUG_LOG: '1' + SGLANG_HICACHE_DEBUG_SAMPLE_RATE: '16384' + SGLANG_REASONING_EFFORT: max + PIP_BREAK_SYSTEM_PACKAGES: '1' + SGLANG_SIMULATE_ACC_LEN: '2.5' + decode_environment: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + NCCL_CUMEM_ENABLE: '1' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_MOE_NVFP4_DISPATCH: '1' + SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' + SGLANG_ENABLE_THINKING: '1' + SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: '8' + SGLANG_REASONING_EFFORT: max + PIP_BREAK_SYSTEM_PACKAGES: '1' + SGLANG_SIMULATE_ACC_LEN: '2.5' + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token + sglang_config: + prefill: + served-model-name: GLM-5.2-FP4 + trust-remote-code: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + disaggregation-mode: prefill + disaggregation-transfer-backend: nixl + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 4 + enable-dp-attention: true + enable-dp-lm-head: true + load-balance-method: total_tokens + chunked-prefill-size: 65536 + max-prefill-tokens: 16384 + max-running-requests: 16 + cuda-graph-max-bs: 16 + disable-cuda-graph: true + nsa-prefill-backend: trtllm + nsa-decode-backend: trtllm + moe-runner-backend: flashinfer_cutlass + fp4-gemm-backend: flashinfer_cutlass + enable-flashinfer-allreduce-fusion: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + mem-fraction-static: 0.8 + context-length: 1048576 + enable-hierarchical-cache: true + hicache-write-policy: write_back + hicache-size: 270 + hicache-io-backend: direct + speculative-algorithm: EAGLE + speculative-num-steps: 1 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 2 + enable-metrics: true + enable-cache-report: true + decode: + served-model-name: GLM-5.2-FP4 + trust-remote-code: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + disaggregation-mode: decode + disaggregation-transfer-backend: nixl + disable-radix-cache: true + speculative-algorithm: EAGLE + speculative-num-steps: 2 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 3 + tensor-parallel-size: 4 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-dp-attention: false + enable-dp-lm-head: false + max-running-requests: 16 + cuda-graph-max-bs: 16 + chunked-prefill-size: 64 + context-length: 1048576 + nsa-prefill-backend: trtllm + nsa-decode-backend: trtllm + moe-runner-backend: flashinfer_trtllm + fp4-gemm-backend: flashinfer_cutlass + skip-tokenizer-init: true + stream-interval: 30 + enable-flashinfer-allreduce-fusion: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + mem-fraction-static: 0.92 + disaggregation-decode-extra-slots: 0 + enable-metrics: true + enable-cache-report: true + health_check: + max_attempts: 1440 + interval_seconds: 10 + benchmark: + type: custom + env: + CONCURRENCY: '63' + DURATION: '3600' + AIPERF_HF_HOME: /hf_home + HF_HOME: /hf_home + HF_HUB_CACHE: /hf_home/hub + HF_DATASETS_CACHE: /hf_home/datasets + HF_HUB_OFFLINE: '1' + HF_DATASETS_OFFLINE: '1' + HF_HUB_DISABLE_XET: '1' + XDG_CACHE_HOME: /tmp/cache + AIPERF_DATASET_MMAP_CACHE_DIR: /hf_home/mmap_cache + AIPERF_DATASET_CONFIGURATION_TIMEOUT: '1800' + AIPERF_SERVICE_PROFILE_CONFIGURE_TIMEOUT: '1800' + AIPERF_FAILED_REQUEST_THRESHOLD: '0.1' + AIPERF_RECORD_STRIP_PAYLOAD_BYTES: 'true' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' + command: | + set -euo pipefail + export HF_HOME="${AIPERF_HF_HOME:-/hf_home}" + export HF_HUB_CACHE="${HF_HOME}/hub" + export HF_DATASETS_CACHE="${HF_HOME}/datasets" + mkdir -p "${XDG_CACHE_HOME}" "${AIPERF_DATASET_MMAP_CACHE_DIR}" /logs/aiperf + /aiperf-venv/bin/aiperf profile \ + --scenario inferencex-agentx-mvp \ + --url http://localhost:8000 \ + --endpoint /v1/chat/completions \ + --endpoint-type chat \ + --streaming \ + --model GLM-5.2-FP4 \ + --tokenizer /model \ + --tokenizer-trust-remote-code \ + --concurrency "${CONCURRENCY}" \ + --benchmark-duration "${DURATION}" \ + --random-seed 42 \ + --failed-request-threshold "${AIPERF_FAILED_REQUEST_THRESHOLD}" \ + --stats-interval 30 \ + --warmup-requests-per-lane 10 \ + --warmup-grace-period 1800 \ + --trajectory-start-min-ratio 0.25 \ + --trajectory-start-max-ratio 0.75 \ + --use-server-token-count \ + --no-gpu-telemetry \ + --num-dataset-entries 393 \ + --slice-duration 1.0 \ + --public-dataset semianalysis_cc_traces_weka_062126 \ + --trace-idle-gap-cap-seconds 300 \ + --unsafe-override \ + --ui simple \ + --server-metrics-formats json \ + --output-artifact-dir /logs/aiperf + rm -f /logs/aiperf/inputs.json + infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 64 + sbatch_directives: + mem: '0' + srun_options: + mem: '0' + container-remap-root: '' +# ################# agentx ################# +zip_override_mtp_agentx_lowlat: + name: [agentx-1p2d_tp4-c63, agentx-1p3d_tp4-c61, agentx-1p4d_tp4-c59, agentx-1p6d_tp4-c58, agentx-2p1d_dtp8-c139] + backend: + sglang_config: + decode: + cuda-graph-max-bs: [16, 16, 16, 16, 139] + data-parallel-size: [1, 1, 1, 1, 8] + enable-dp-attention: [false, false, false, false, true] + enable-dp-lm-head: [false, false, false, false, true] + max-running-requests: [16, 16, 16, 16, 139] + tensor-parallel-size: [4, 4, 4, 4, 8] + benchmark: + env: + CONCURRENCY: ['63', '61', '59', '58', '139'] + frontend: + enable_multiple_frontends: true + resources: + decode_nodes: [1, 2, 2, 3, 1] + decode_workers: [2, 3, 4, 6, 1] + gpus_per_decode: [4, 4, 4, 4, 8] + gpus_per_prefill: 4 + prefill_nodes: 1 + prefill_workers: [1, 1, 1, 1, 2] diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 526fbec480..5804468870 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -9329,6 +9329,149 @@ glm5.2-fp4-gb200-dynamo-sglang-agentic-mtp-agg: dp-attn: false additional-settings: - CONFIG_FILE=recipes/sglang/glm5.2/gb200-fp4/agentic/agg-gb200-tp8-c8-mtp.yaml + +glm5.2-fp4-b300-dynamo-sglang-agentic-agg: + image: lmsysorg/sglang:nightly-dev-cu13-20260824-95f5ecd3 + model: nvidia/GLM-5.2-NVFP4 + model-prefix: glm5.2 + runner: cluster:b300-dsxe + precision: fp4 + framework: dynamo-sglang + router: { name: dynamo-router, version: "71eb001e17fa73c742f0afe1a6ed96836cb135fd" } + multinode: true + disagg: false + scenarios: + agentic-coding: + - dram-utilization: 0.80 + search-space: + # aggregated 1-worker TP8 (prefill+decode on one 8-GPU B300 node), conc 2 + - spec-decoding: mtp + conc-list: [2] + kv-offloading: dram + kv-offload-backend: { name: hicache } + num-nodes: 1 + worker: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5.2/b300-fp4/agentic/glm5.2-agentx-agg.yaml:zip_override_mtp_agentx_lowlat[0]" + # aggregated 1-worker TP8 (prefill+decode on one 8-GPU B300 node), conc 4 + - spec-decoding: mtp + conc-list: [4] + kv-offloading: dram + kv-offload-backend: { name: hicache } + num-nodes: 1 + worker: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5.2/b300-fp4/agentic/glm5.2-agentx-agg.yaml:zip_override_mtp_agentx_lowlat[1]" + +glm5.2-fp4-b300-dynamo-sglang-agentic-disagg: + image: lmsysorg/sglang:nightly-dev-cu13-20260824-95f5ecd3 + model: nvidia/GLM-5.2-NVFP4 + model-prefix: glm5.2 + runner: cluster:b300-dsxe + precision: fp4 + framework: dynamo-sglang + router: { name: dynamo-router, version: "71eb001e17fa73c742f0afe1a6ed96836cb135fd" } + kv-p2p-transfer: nixl + multinode: true + disagg: true + scenarios: + agentic-coding: + - dram-utilization: 0.80 + search-space: + # low-latency 1P2D TP4-prefill / TP4-decode (2 decode workers) + - spec-decoding: mtp + conc-list: [63] + kv-offloading: dram + kv-offload-backend: { name: hicache } + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5.2/b300-fp4/agentic/glm5.2-agentx.yaml:zip_override_mtp_agentx_lowlat[0]" + decode: + num-worker: 2 + tp: 4 + ep: 1 + dp-attn: false + # low-latency 1P3D TP4-prefill / TP4-decode (3 decode workers) + - spec-decoding: mtp + conc-list: [61] + kv-offloading: dram + kv-offload-backend: { name: hicache } + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5.2/b300-fp4/agentic/glm5.2-agentx.yaml:zip_override_mtp_agentx_lowlat[1]" + decode: + num-worker: 3 + tp: 4 + ep: 1 + dp-attn: false + # low-latency 1P4D TP4-prefill / TP4-decode (4 decode workers) + - spec-decoding: mtp + conc-list: [59] + kv-offloading: dram + kv-offload-backend: { name: hicache } + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5.2/b300-fp4/agentic/glm5.2-agentx.yaml:zip_override_mtp_agentx_lowlat[2]" + decode: + num-worker: 4 + tp: 4 + ep: 1 + dp-attn: false + # low-latency 1P6D TP4-prefill / TP4-decode (6 decode workers) + - spec-decoding: mtp + conc-list: [58] + kv-offloading: dram + kv-offload-backend: { name: hicache } + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5.2/b300-fp4/agentic/glm5.2-agentx.yaml:zip_override_mtp_agentx_lowlat[3]" + decode: + num-worker: 6 + tp: 4 + ep: 1 + dp-attn: false + # throughput 2P1D DEP8-decode (DP8/TP8 decode, dp-attention) + - spec-decoding: mtp + conc-list: [139] + kv-offloading: dram + kv-offload-backend: { name: hicache } + prefill: + num-worker: 2 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5.2/b300-fp4/agentic/glm5.2-agentx.yaml:zip_override_mtp_agentx_lowlat[4]" + decode: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: true + glm5.2-fp4-gb300-dynamo-sglang-agentic-agg: image: lmsysorg/sglang:nightly-dev-cu13-20260805-211ee642 model: nvidia/GLM-5.2-NVFP4 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index b0dc6a14d4..0086f5e0d1 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -7145,3 +7145,14 @@ - "Update SGLang image from lmsysorg/sglang:nightly-dev-cu13-20260907-30705c00 (2026-09-07 cu13 dev nightly, build commit sgl-project/sglang@30705c00) to the v0.5.19 release image lmsysorg/sglang:v0.5.19-cu130 (digest sha256:d6e7288627be8b02be88e4bba38e73f6d50e2826869f753c13a4c4385ab3eda9, build commit sgl-project/sglang@0bcd822377da7b5718e674eaf9c870d349424dd1, Docker Hub last pushed 2026-09-04T22:50:19Z)." - "The release image ships the same CUDA 13.0.3, FlashInfer 0.6.18 and sgl-kernel 0.4.6.post1 as the nightly. benchmarks/single_node/agentic/qwen3.5_fp8_h200_mtp.sh is unchanged: SGLANG_ENABLE_SPEC_V2 EAGLE MTP at 3 steps, golden acceptance length 3.39, flashinfer attention with allreduce fusion, fp8 quantization and fp8_e4m3 KV, HiCache kernel IO / page_first layout. TP8/EP1 DRAM HiCache concurrency 2 through 24 unchanged." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2966 + +- config-keys: + - glm5.2-fp4-b300-dynamo-sglang-agentic-agg + - glm5.2-fp4-b300-dynamo-sglang-agentic-disagg + scenario-type: + - agentic-coding + description: + - "Add GLM-5.2-NVFP4 B300 AgentX (agentic-coding) Dynamo-SGLang benchmarks over a 7-point sweep: aggregated single-node TP8 (conc 2/4), disaggregated low-latency TP4-prefill/TP4-decode 1P2D (conc 63), 1P3D (conc 61), 1P4D (conc 59), 1P6D (conc 58), and throughput 2P1D DEP8-decode (conc 139)" + - "EAGLE speculative decoding, hierarchical KV cache (hicache-size 270), MTP, decode mem-fraction-static 0.92, nixl KV transfer, and X-Dynamo-Session-ID session affinity; driven from the NVIDIA/srt-slurm base+zip_override recipes via CONFIG_FILE selectors" + - "Image: lmsysorg/sglang:nightly-dev-cu13-20260824-95f5ecd3; runner: b300-dsxe" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2654 diff --git a/runners/launch_b300-dsxe.sh b/runners/launch_b300-dsxe.sh index 70254deb65..315d3cb4cc 100755 --- a/runners/launch_b300-dsxe.sh +++ b/runners/launch_b300-dsxe.sh @@ -159,27 +159,43 @@ select_srt_slurm_version() { SRT_REPO_DIR="srt-slurm" rm -rf "$SRT_REPO_DIR" -if [[ "$USES_DCGM_POWER" == "1" ]]; then - SRT_SLURM_REPO="$POWER_SRT_SLURM_URL" - SRT_SLURM_REF="$POWER_SRT_SLURM_PIN" +if [[ "$IS_AGENTIC" == "1" && $FRAMEWORK == "dynamo-sglang" && $MODEL_PREFIX == "glm5.2" && $PRECISION == "fp4" ]]; then + # GLM-5.2 B300 sglang AgentX: srt-slurm main carries the agentx-mvp scenario, + # session-affinity frontend, and custom benchmark schema these recipes need. + # Only glm5.2/b300-fp4 recipes get copied onto this pinned fork checkout -- + # its recipes/ tree follows a contract other models haven't adopted yet. + echo "Cloning srt-slurm (https://github.com/NVIDIA/srt-slurm.git, pinned to open PR #313)..." + git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" || exit 1 + cd "$SRT_REPO_DIR" || exit 1 + # NVIDIA/srt-slurm#313 is still open, so pin its reviewed recipe contract. + git fetch origin pull/313/head || exit 1 + git checkout --detach 93fae852166a30f3cc054c8616228bf62c69c48c || exit 1 + git rev-parse HEAD > "$GITHUB_WORKSPACE/srt-slurm-sha.txt" + mkdir -p recipes/sglang/glm5.2/b300-fp4 + cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/b300-fp4" recipes/sglang/glm5.2/b300-fp4 || exit 1 else - select_srt_slurm_version -fi + if [[ "$USES_DCGM_POWER" == "1" ]]; then + SRT_SLURM_REPO="$POWER_SRT_SLURM_URL" + SRT_SLURM_REF="$POWER_SRT_SLURM_PIN" + else + select_srt_slurm_version + fi -echo "Cloning srt-slurm ($SRT_SLURM_REPO @ $SRT_SLURM_REF)..." -git clone "$SRT_SLURM_REPO" "$SRT_REPO_DIR" || exit 1 -cd "$SRT_REPO_DIR" || exit 1 -git checkout --quiet "$SRT_SLURM_REF" || exit 1 -git rev-parse HEAD > "$GITHUB_WORKSPACE/srt-slurm-sha.txt" -if [[ "$USES_DCGM_POWER" == "1" ]]; then - test "$(git rev-parse HEAD)" = "$POWER_SRT_SLURM_PIN" \ - || { echo "Error: srt-slurm HEAD does not match POWER_SRT_SLURM_PIN=$POWER_SRT_SLURM_PIN" >&2; exit 1; } - cp "$GITHUB_WORKSPACE/srt-slurm-sha.txt" "$GITHUB_WORKSPACE/power-producer-sha.txt" -fi + echo "Cloning srt-slurm ($SRT_SLURM_REPO @ $SRT_SLURM_REF)..." + git clone "$SRT_SLURM_REPO" "$SRT_REPO_DIR" || exit 1 + cd "$SRT_REPO_DIR" || exit 1 + git checkout --quiet "$SRT_SLURM_REF" || exit 1 + git rev-parse HEAD > "$GITHUB_WORKSPACE/srt-slurm-sha.txt" + if [[ "$USES_DCGM_POWER" == "1" ]]; then + test "$(git rev-parse HEAD)" = "$POWER_SRT_SLURM_PIN" \ + || { echo "Error: srt-slurm HEAD does not match POWER_SRT_SLURM_PIN=$POWER_SRT_SLURM_PIN" >&2; exit 1; } + cp "$GITHUB_WORKSPACE/srt-slurm-sha.txt" "$GITHUB_WORKSPACE/power-producer-sha.txt" + fi -# Recipes live in this repo; overlay all of them onto the checkout's recipes/ dir. -mkdir -p recipes -cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes" recipes || exit 1 + # Recipes live in this repo; overlay all of them onto the checkout's recipes/ dir. + mkdir -p recipes + cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes" recipes || exit 1 +fi if [[ "${EVAL_FRAMEWORK:-lm-eval}" != "lm-eval" ]]; then python3 "$GITHUB_WORKSPACE/runners/patch_srt_eval_dispatch.py" "$(pwd)" || exit 1 @@ -293,7 +309,11 @@ SRTCTL_APPLY_ARGS=( --no-preflight --tags "b300,${MODEL_PREFIX},${PRECISION},${ISL}x${OSL},infmax-$(date +%Y%m%d)" ) -SRTCTL_OUTPUT=$(srtctl apply "${SRTCTL_APPLY_ARGS[@]}" 2>&1) +if [[ "$MODEL_PREFIX" == "glm5.2" && "$FRAMEWORK" == "dynamo-sglang" ]]; then + SRTCTL_OUTPUT=$(env -u UCX_TLS srtctl apply "${SRTCTL_APPLY_ARGS[@]}" 2>&1) +else + SRTCTL_OUTPUT=$(srtctl apply "${SRTCTL_APPLY_ARGS[@]}" 2>&1) +fi echo "$SRTCTL_OUTPUT" # Extract JOB_ID from srtctl output