From c87378f4e5180a7e7c6603a6118aff288f22ba35 Mon Sep 17 00:00:00 2001 From: misunp Date: Wed, 2 Sep 2026 23:46:19 -0700 Subject: [PATCH 1/6] =?UTF-8?q?perf(agentx):=20add=20Kimi-K3=20B300=20Agen?= =?UTF-8?q?tX=20/=20=E6=80=A7=E8=83=BD=EF=BC=9A=E6=96=B0=E5=A2=9E=20Kimi-K?= =?UTF-8?q?3=20B300=20AgentX?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add the six measured B300 points: aggregated TP8 x DCP8 no-spec at c70, aggregated DSpark-7 at c4 and c1, and 1P1D/1P2D/1P3D DCP8 x DCP8 DSpark-4 at c48/c48/c32. Recipes are translated from the bia sweep configs that measured each point; only fields b300-nv forces differ. 新增六个已实测的 B300 数据点：无投机解码的聚合 TP8 x DCP8（c70）、 DSpark-7 聚合（c4 与 c1），以及 1P1D/1P2D/1P3D 的 DCP8 x DCP8 DSpark-4 （c48/c48/c32）。recipe 由实测该点的 bia sweep config 转换而来，仅修改 b300-nv 强制要求的字段。 --- .../configs/kimik3-b300-nic-topology.sh | 112 ++++++++ ...8-dspark7-maxseq2-mooncake-c1-agentic.yaml | 157 ++++++++++++ ...8-dspark7-maxseq2-mooncake-c4-agentic.yaml | 157 ++++++++++++ ...b300-dcp8-nospec-mooncake-c70-agentic.yaml | 154 +++++++++++ ...cp8-dcp8-dspark4-mooncake-c48-agentic.yaml | 242 ++++++++++++++++++ ...cp8-dcp8-dspark4-mooncake-c48-agentic.yaml | 239 +++++++++++++++++ ...cp8-dcp8-dspark4-mooncake-c32-agentic.yaml | 239 +++++++++++++++++ configs/nvidia-master.yaml | 158 ++++++++++++ perf-changelog.yaml | 13 + runners/launch_b300-dsxe.sh | 11 +- 10 files changed, 1478 insertions(+), 4 deletions(-) create mode 100755 benchmarks/multi_node/srt-slurm-recipes/configs/kimik3-b300-nic-topology.sh create mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b300-dcp8-dspark7-maxseq2-mooncake-c1-agentic.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b300-dcp8-dspark7-maxseq2-mooncake-c4-agentic.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b300-dcp8-nospec-mooncake-c70-agentic.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/disagg-b300-1p1d-dcp8-dcp8-dspark4-mooncake-c48-agentic.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/disagg-b300-1p2d-dcp8-dcp8-dspark4-mooncake-c48-agentic.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/disagg-b300-1p3d-dcp8-dcp8-dspark4-mooncake-c32-agentic.yaml diff --git a/benchmarks/multi_node/srt-slurm-recipes/configs/kimik3-b300-nic-topology.sh b/benchmarks/multi_node/srt-slurm-recipes/configs/kimik3-b300-nic-topology.sh new file mode 100755 index 0000000000..4bfb9e3e14 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/configs/kimik3-b300-nic-topology.sh @@ -0,0 +1,112 @@ +#!/usr/bin/env bash +# Print this compute node's GPU-to-NIC topology, then verify the rails the +# Mooncake store is pinned to. +# +# Mooncake on b300-nv needs its peers on the same rail: the nodes are +# rail-isolated, and leaving device_name empty let the prefill node pick +# mlx5_1/mlx5_2 that the decode node did not have, which ended in "Active +# handshake RPC failed" and a segfault during bring-up. Naming rails instead +# of leaving them to auto-discovery means the set has to come from the real +# affinity map rather than a guess. The worker logs already list which devices +# are up; what they do not show is which NIC belongs to which GPU, which is +# what this prints. +# +# The topology dump is diagnostic and best-effort. The rail check below is +# not: it exits non-zero on a mismatch, which fails the job by design. + +set -uo pipefail + +echo "=== [b300-topo] node: $(hostname) ===" + +echo "--- nvidia-smi topo -m ---" +nvidia-smi topo -m 2>&1 | head -40 || true + +echo "--- ibdev2netdev ---" +ibdev2netdev 2>&1 | head -40 || true + +echo "--- ibv_devinfo -l ---" +ibv_devinfo -l 2>&1 | head -40 || true + +echo "--- infiniband port state / rate ---" +for _d in /sys/class/infiniband/*; do + [[ -e "$_d" ]] || continue + _n="$(basename "$_d")" + printf '%-10s state=%-14s rate=%s\n' \ + "$_n" \ + "$(cat "$_d/ports/1/state" 2>/dev/null || echo '?')" \ + "$(cat "$_d/ports/1/rate" 2>/dev/null || echo '?')" +done + +echo "=== [b300-topo] end ===" + +# --------------------------------------------------------------------------- +# Verify the rails the Mooncake store is pinned to. +# +# srtslurm writes one store config for the whole job, so RDMA devices can only +# be named by index -- and an index does not mean the same physical rail on +# every node here. b300-012 puts mlx5_0 on 172.16.0 where every other node puts +# it on 172.16.192, and b300-003 swaps mlx5_3 with mlx5_9. When two nodes +# disagree, the cross-node QP cannot come up: "Failed to modify QP to RTR ... +# Connection timed out", then Mooncake retries via another peer RNIC and the +# transfer hits "local access violation work queue error" against a memory +# region registered for a different device. One earlier bring-up died that +# way, 36 minutes in. +# +# The recipe therefore names only rails whose subnet was identical on every +# node inventoried so far. Assert that here rather than trust it: a node that +# disagrees fails in seconds and names itself, instead of burning a benchmark +# window. +EXPECTED_RAILS="mlx5_4:172.16.128 mlx5_8:172.16.64 mlx5_10:172.17.192 \ +mlx5_16:172.17.0 mlx5_20:172.17.128 mlx5_22:172.17.64" + +echo "=== [b300-rails] verifying pinned rails on $(hostname) ===" +_rail_bad=0 +for _pair in $EXPECTED_RAILS; do + _dev="${_pair%%:*}" + _want="${_pair##*:}" + _port="/sys/class/infiniband/${_dev}/ports/1" + + if [[ ! -d "$_port" ]]; then + echo "[b300-rails] FAIL ${_dev}: device absent"; _rail_bad=1; continue + fi + _state="$(cat "${_port}/state" 2> /dev/null || echo '?')" + if [[ "$_state" != *ACTIVE* ]]; then + echo "[b300-rails] FAIL ${_dev}: port state '${_state}'"; _rail_bad=1; continue + fi + # GID index 3 is the RoCEv2 IPv4 entry; its last four bytes are the address. + _gid="$(cat "${_port}/gids/3" 2> /dev/null || echo '')" + _hex="$(printf '%s' "$_gid" | tr -d ':' | tail -c 8)" + if [[ ${#_hex} -ne 8 ]]; then + echo "[b300-rails] FAIL ${_dev}: cannot read GID index 3 (got '${_gid}')"; _rail_bad=1; continue + fi + _got="$((16#${_hex:0:2})).$((16#${_hex:2:2})).$((16#${_hex:4:2}))" + if [[ "$_got" != "$_want" ]]; then + echo "[b300-rails] FAIL ${_dev}: subnet ${_got}.x, expected ${_want}.x" + _rail_bad=1 + else + echo "[b300-rails] ok ${_dev}: ${_got}.$((16#${_hex:6:2}))" + fi +done + +if [[ "$_rail_bad" -ne 0 ]]; then + echo "[b300-rails] $(hostname) does not match the pinned rail layout." >&2 + echo "[b300-rails] Add it to sbatch_directives.exclude, or re-pin the rails" >&2 + echo "[b300-rails] in the b300 Kimi-K3 agentic recipes." >&2 + exit 1 +fi +echo "=== [b300-rails] all pinned rails match ===" + +# Nothing to patch: the image carries the Kimi-K3 stack already. Print what it +# is, so the log still records which build served the numbers. +python3 - <<'PY' 2>/dev/null || true +import importlib.util, os +spec = importlib.util.find_spec("vllm") +if spec and spec.origin: + ver = os.path.join(os.path.dirname(spec.origin), "_version.py") + try: + for line in open(ver): + if "version" in line and "=" in line: + print(f"[b300-rails] vllm {line.strip()}") + except OSError: + pass +PY diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b300-dcp8-dspark7-maxseq2-mooncake-c1-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b300-dcp8-dspark7-maxseq2-mooncake-c1-agentic.yaml new file mode 100644 index 0000000000..1c05129d33 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b300-dcp8-dspark7-maxseq2-mooncake-c1-agentic.yaml @@ -0,0 +1,157 @@ +# Kimi-K3 B300 aggregated TP8 x DCP8, agentic coding. +# +# Translated from the bia sweep config that measured this point: +# AGG/kimi-k3-mxfp4-b300-agg-tp8-agentx-062126-c1-weiaggds7-gmu94.yml +# Engine flags, environment, Mooncake settings and vllm_config are that file +# verbatim. The InferenceX plumbing -- identity, benchmark, slurm, the idle-GPU +# reaper comment -- is taken from agg-gb300-dcp8-dspark7-maxseq2-mooncake-agentic.yaml. +# +# Only these fields differ from the measured config, each because b300-nv is +# not bia: +# device_name, UCX_NET_DEVICES -> the six rails read off this cluster +# MC_GID_INDEX, UCX_TLS -> set; b300-nv is RoCE (see below) +# HF_HOME -> dropped; benchmark.env sets HF_HUB_CACHE +# slurm.time_limit -> 04:00:00, this partition's limit +# rejection_sample_method -> block, so the eval verifies (see below) +# global_segment_size -> 281GB, matching the B300 arm on main +# sbatch_directives.exclude -> b300-018, which times out UCX/NIXL +# +# The rails are the part that needs saying. This fabric is rail-isolated: each +# rail is its own /24 and a QP only comes up when both ends sit on the same +# one. srtslurm writes a single store config for the whole job, so rails can +# only be named by index -- and the index is not stable here, because a down +# card shifts every index above it. Across the nodes inventoried, mlx5_0 is +# 172.16.192 everywhere except b300-012, and b300-003 swaps mlx5_3 with +# mlx5_9. Naming either put two nodes on different subnets under one name, +# which killed one earlier bring-up 36 minutes in. The six named below held on +# every node with a full inventory, and kimik3-b300-nic-topology.sh re-checks +# each one's GID subnet at startup so an uninventoried node fails in seconds +# instead of running to a bad number. + +name: kimi-k3-vllm-agg-b300-dcp8-dspark7-maxseq2-mooncake-c1-agentic +model: + path: moonshotai/Kimi-K3 + container: vllm/vllm-openai:nightly-dev-x86_64-cu13-3696c77 + precision: fp4 +resources: + gpu_type: b300 + gpus_per_node: 8 + agg_nodes: 1 + agg_workers: 1 + gpus_per_agg: 8 +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 +frontend: + type: dynamo + enable_multiple_frontends: false + args: + router-mode: least-loaded + router-session-affinity-ttl-secs: 900 + env: + DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: '3600' + DYN_TOKENIZER_CACHE_BYTES: '8589934592' + PYTHONPYCACHEPREFIX: /tmp/vllm-pycache-kimi-k3 +backend: + type: vllm + connector: null + mooncake_kv_store: + master_extra_args: + - --default_kv_lease_ttl=60000 + - --eviction_high_watermark_ratio=0.95 + - --eviction_ratio=0.10 + store_config: + metadata_server: P2PHANDSHAKE + global_segment_size: 281GB + local_buffer_size: 4GB + protocol: rdma + device_name: mlx5_4,mlx5_8,mlx5_10,mlx5_16,mlx5_20,mlx5_22 + mode: embedded + enable_offload: false + aggregated_environment: + VLLM_ALLREDUCE_USE_FLASHINFER: '1' + VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: '1' + VLLM_USE_DIRECT_DCP_A2A: '1' + VLLM_USE_DIRECT_DCP_Q_GATHER: '1' + VLLM_USE_DIRECT_DCP_KV_GATHER: '1' + ETCD_LEASE_TTL: '120' + VLLM_ENGINE_READY_TIMEOUT_S: '3600' + VLLM_RPC_TIMEOUT: '600000' + VLLM_USE_V2_MODEL_RUNNER: '1' + VLLM_USE_RUST_FRONTEND: '1' + VLLM_MOONCAKE_LOAD_RECV_THREADS: '4' + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: '0' + VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: '0' + MC_SLICE_SIZE: '1048576' + WITH_NVIDIA_PEERMEM: '0' + VLLM_LOG_STATS_INTERVAL: '1' + UCX_NET_DEVICES: mlx5_4:1,mlx5_8:1,mlx5_10:1,mlx5_16:1,mlx5_20:1,mlx5_22:1 + PYTHONHASHSEED: '42' + PYTHONNOUSERSITE: '1' + PYTHONUNBUFFERED: '1' + TORCH_CUDA_ARCH_LIST: '10.0' + MC_GID_INDEX: '3' + UCX_TLS: rc,cuda_copy,cuda_ipc,sm,self + vllm_config: + aggregated: + kv-transfer-config: '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"compact_group_io":true,"max_load_batch_keys":2,"enable_offload":false}}' + served-model-name: moonshotai/Kimi-K3 + load-format: fastsafetensors + kv-cache-dtype: fp8 + gpu-memory-utilization: 0.94 + tensor-parallel-size: 8 + decode-context-parallel-size: 8 + dcp-comm-backend: a2a + max-num-seqs: 2 + max-num-batched-tokens: 8192 + trust-remote-code: true + max-cudagraph-capture-size: 1024 + stream-interval: 10 + language-model-only: true + moe-backend: auto + no-enable-flashinfer-autotune: true + enable-cumem-allocator: true + attention-backend: TOKENSPEED_MLA + attention-config: '{"mla_prefill_backend":"TRTLLM_RAGGED","use_prefill_query_quantization":true}' + speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","attention_backend":"TOKENSPEED_MLA","method":"dspark","num_speculative_tokens":7,"rejection_sample_method":"block"}' + enable-prefix-caching: true + prefix-match-unit: 128 +health_check: + max_attempts: 720 + interval_seconds: 10 +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: '300' + AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: '0.25' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'vllm:' + AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 +dynamo: + hash: ba83080ecd31c1ce918559e576d3c5bc9e092ff1 + install: true +identity: + model: + repo: moonshotai/Kimi-K3 + container: + image: vllm/vllm-openai:nightly-dev-x86_64-cu13-3696c77 + frameworks: + dynamo: ba83080ecd31c1ce918559e576d3c5bc9e092ff1 +slurm: + time_limit: 04:00:00 +sbatch_directives: + mem: '0' + cpus-per-task: '72' + comment: '''{"OccupiedIdleGPUsJobReaper":{"exemptIdleTimeMins":"60","reason":"model_loading","description":"Very large model will take extra loading time."}}''' + exclude: b300-018 +srun_options: + mem: '0' + container-remap-root: '' diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b300-dcp8-dspark7-maxseq2-mooncake-c4-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b300-dcp8-dspark7-maxseq2-mooncake-c4-agentic.yaml new file mode 100644 index 0000000000..e4eacd947e --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b300-dcp8-dspark7-maxseq2-mooncake-c4-agentic.yaml @@ -0,0 +1,157 @@ +# Kimi-K3 B300 aggregated TP8 x DCP8, agentic coding. +# +# Translated from the bia sweep config that measured this point: +# AGG/kimi-k3-mxfp4-b300-agg-tp8-agentx-062126-c4-weiaggds7-gmu94.yml +# Engine flags, environment, Mooncake settings and vllm_config are that file +# verbatim. The InferenceX plumbing -- identity, benchmark, slurm, the idle-GPU +# reaper comment -- is taken from agg-gb300-dcp8-dspark7-maxseq2-mooncake-agentic.yaml. +# +# Only these fields differ from the measured config, each because b300-nv is +# not bia: +# device_name, UCX_NET_DEVICES -> the six rails read off this cluster +# MC_GID_INDEX, UCX_TLS -> set; b300-nv is RoCE (see below) +# HF_HOME -> dropped; benchmark.env sets HF_HUB_CACHE +# slurm.time_limit -> 04:00:00, this partition's limit +# rejection_sample_method -> block, so the eval verifies (see below) +# global_segment_size -> 281GB, matching the B300 arm on main +# sbatch_directives.exclude -> b300-018, which times out UCX/NIXL +# +# The rails are the part that needs saying. This fabric is rail-isolated: each +# rail is its own /24 and a QP only comes up when both ends sit on the same +# one. srtslurm writes a single store config for the whole job, so rails can +# only be named by index -- and the index is not stable here, because a down +# card shifts every index above it. Across the nodes inventoried, mlx5_0 is +# 172.16.192 everywhere except b300-012, and b300-003 swaps mlx5_3 with +# mlx5_9. Naming either put two nodes on different subnets under one name, +# which killed one earlier bring-up 36 minutes in. The six named below held on +# every node with a full inventory, and kimik3-b300-nic-topology.sh re-checks +# each one's GID subnet at startup so an uninventoried node fails in seconds +# instead of running to a bad number. + +name: kimi-k3-vllm-agg-b300-dcp8-dspark7-maxseq2-mooncake-c4-agentic +model: + path: moonshotai/Kimi-K3 + container: vllm/vllm-openai:nightly-dev-x86_64-cu13-3696c77 + precision: fp4 +resources: + gpu_type: b300 + gpus_per_node: 8 + agg_nodes: 1 + agg_workers: 1 + gpus_per_agg: 8 +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 +frontend: + type: dynamo + enable_multiple_frontends: false + args: + router-mode: least-loaded + router-session-affinity-ttl-secs: 900 + env: + DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: '3600' + DYN_TOKENIZER_CACHE_BYTES: '8589934592' + PYTHONPYCACHEPREFIX: /tmp/vllm-pycache-kimi-k3 +backend: + type: vllm + connector: null + mooncake_kv_store: + master_extra_args: + - --default_kv_lease_ttl=60000 + - --eviction_high_watermark_ratio=0.95 + - --eviction_ratio=0.10 + store_config: + metadata_server: P2PHANDSHAKE + global_segment_size: 281GB + local_buffer_size: 4GB + protocol: rdma + device_name: mlx5_4,mlx5_8,mlx5_10,mlx5_16,mlx5_20,mlx5_22 + mode: embedded + enable_offload: false + aggregated_environment: + VLLM_ALLREDUCE_USE_FLASHINFER: '1' + VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: '1' + VLLM_USE_DIRECT_DCP_A2A: '1' + VLLM_USE_DIRECT_DCP_Q_GATHER: '1' + VLLM_USE_DIRECT_DCP_KV_GATHER: '1' + ETCD_LEASE_TTL: '120' + VLLM_ENGINE_READY_TIMEOUT_S: '3600' + VLLM_RPC_TIMEOUT: '600000' + VLLM_USE_V2_MODEL_RUNNER: '1' + VLLM_USE_RUST_FRONTEND: '1' + VLLM_MOONCAKE_LOAD_RECV_THREADS: '4' + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: '0' + VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: '0' + MC_SLICE_SIZE: '1048576' + WITH_NVIDIA_PEERMEM: '0' + VLLM_LOG_STATS_INTERVAL: '1' + UCX_NET_DEVICES: mlx5_4:1,mlx5_8:1,mlx5_10:1,mlx5_16:1,mlx5_20:1,mlx5_22:1 + PYTHONHASHSEED: '42' + PYTHONNOUSERSITE: '1' + PYTHONUNBUFFERED: '1' + TORCH_CUDA_ARCH_LIST: '10.0' + MC_GID_INDEX: '3' + UCX_TLS: rc,cuda_copy,cuda_ipc,sm,self + vllm_config: + aggregated: + kv-transfer-config: '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"compact_group_io":true,"max_load_batch_keys":2,"enable_offload":false}}' + served-model-name: moonshotai/Kimi-K3 + load-format: fastsafetensors + kv-cache-dtype: fp8 + gpu-memory-utilization: 0.94 + tensor-parallel-size: 8 + decode-context-parallel-size: 8 + dcp-comm-backend: a2a + max-num-seqs: 2 + max-num-batched-tokens: 8192 + trust-remote-code: true + max-cudagraph-capture-size: 1024 + stream-interval: 10 + language-model-only: true + moe-backend: auto + no-enable-flashinfer-autotune: true + enable-cumem-allocator: true + attention-backend: TOKENSPEED_MLA + attention-config: '{"mla_prefill_backend":"TRTLLM_RAGGED","use_prefill_query_quantization":true}' + speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","attention_backend":"TOKENSPEED_MLA","method":"dspark","num_speculative_tokens":7,"rejection_sample_method":"block"}' + enable-prefix-caching: true + prefix-match-unit: 128 +health_check: + max_attempts: 720 + interval_seconds: 10 +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: '300' + AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: '0.25' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'vllm:' + AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 +dynamo: + hash: ba83080ecd31c1ce918559e576d3c5bc9e092ff1 + install: true +identity: + model: + repo: moonshotai/Kimi-K3 + container: + image: vllm/vllm-openai:nightly-dev-x86_64-cu13-3696c77 + frameworks: + dynamo: ba83080ecd31c1ce918559e576d3c5bc9e092ff1 +slurm: + time_limit: 04:00:00 +sbatch_directives: + mem: '0' + cpus-per-task: '72' + comment: '''{"OccupiedIdleGPUsJobReaper":{"exemptIdleTimeMins":"60","reason":"model_loading","description":"Very large model will take extra loading time."}}''' + exclude: b300-018 +srun_options: + mem: '0' + container-remap-root: '' diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b300-dcp8-nospec-mooncake-c70-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b300-dcp8-nospec-mooncake-c70-agentic.yaml new file mode 100644 index 0000000000..19074b358f --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b300-dcp8-nospec-mooncake-c70-agentic.yaml @@ -0,0 +1,154 @@ +# Kimi-K3 B300 aggregated TP8 x DCP8, agentic coding. +# +# Translated from the bia sweep config that measured this point: +# AGG/kimi-k3-mxfp4-b300-agg-tp8-agentx-062126-c70-weiaggns-gmu94.yml +# Engine flags, environment, Mooncake settings and vllm_config are that file +# verbatim. The InferenceX plumbing -- identity, benchmark, slurm, the idle-GPU +# reaper comment -- is taken from agg-gb300-dcp8-nospec-mooncake-agentic.yaml. +# +# Only these fields differ from the measured config, each because b300-nv is +# not bia: +# device_name, UCX_NET_DEVICES -> the six rails read off this cluster +# MC_GID_INDEX, UCX_TLS -> set; b300-nv is RoCE (see below) +# HF_HOME -> dropped; benchmark.env sets HF_HUB_CACHE +# slurm.time_limit -> 04:00:00, this partition's limit +# rejection_sample_method -> block, so the eval verifies (see below) +# global_segment_size -> 281GB, matching the B300 arm on main +# sbatch_directives.exclude -> b300-018, which times out UCX/NIXL +# +# The rails are the part that needs saying. This fabric is rail-isolated: each +# rail is its own /24 and a QP only comes up when both ends sit on the same +# one. srtslurm writes a single store config for the whole job, so rails can +# only be named by index -- and the index is not stable here, because a down +# card shifts every index above it. Across the nodes inventoried, mlx5_0 is +# 172.16.192 everywhere except b300-012, and b300-003 swaps mlx5_3 with +# mlx5_9. Naming either put two nodes on different subnets under one name, +# which killed one earlier bring-up 36 minutes in. The six named below held on +# every node with a full inventory, and kimik3-b300-nic-topology.sh re-checks +# each one's GID subnet at startup so an uninventoried node fails in seconds +# instead of running to a bad number. + +name: kimi-k3-vllm-agg-b300-dcp8-nospec-mooncake-c70-agentic +model: + path: moonshotai/Kimi-K3 + container: vllm/vllm-openai:nightly-dev-x86_64-cu13-3696c77 + precision: fp4 +resources: + gpu_type: b300 + gpus_per_node: 8 + agg_nodes: 1 + agg_workers: 1 + gpus_per_agg: 8 +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 +frontend: + type: dynamo + enable_multiple_frontends: false + args: + router-mode: least-loaded + router-session-affinity-ttl-secs: 900 + env: + DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: '3600' + DYN_TOKENIZER_CACHE_BYTES: '8589934592' +backend: + type: vllm + connector: null + mooncake_kv_store: + master_extra_args: + - --default_kv_lease_ttl=60000 + - --eviction_high_watermark_ratio=0.95 + - --eviction_ratio=0.10 + store_config: + metadata_server: P2PHANDSHAKE + global_segment_size: 281GB + local_buffer_size: 4GB + protocol: rdma + device_name: mlx5_4,mlx5_8,mlx5_10,mlx5_16,mlx5_20,mlx5_22 + mode: embedded + enable_offload: false + aggregated_environment: + VLLM_ALLREDUCE_USE_FLASHINFER: '1' + VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: '1' + VLLM_USE_V2_MODEL_RUNNER: '1' + VLLM_USE_DIRECT_DCP_A2A: '1' + VLLM_USE_DIRECT_DCP_Q_GATHER: '1' + VLLM_USE_DIRECT_DCP_KV_GATHER: '1' + VLLM_ENGINE_READY_TIMEOUT_S: '3600' + VLLM_RPC_TIMEOUT: '600000' + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: '0' + VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: '0' + PYTHONNOUSERSITE: '1' + TORCH_CUDA_ARCH_LIST: '10.0' + PYTHONHASHSEED: '42' + VLLM_HTTP_TIMEOUT_KEEP_ALIVE: '900' + MC_GID_INDEX: '3' + MC_STORE_MEMCPY: '1' + MC_ENABLE_DEST_DEVICE_AFFINITY: '1' + MC_SLICE_SIZE: '1048576' + MC_WORKERS_PER_CTX: '4' + WITH_NVIDIA_PEERMEM: '0' + VLLM_MOONCAKE_LOAD_RECV_THREADS: '4' + UCX_NET_DEVICES: mlx5_4:1,mlx5_8:1,mlx5_10:1,mlx5_16:1,mlx5_20:1,mlx5_22:1 + PYTHONUNBUFFERED: '1' + UCX_TLS: rc,cuda_copy,cuda_ipc,sm,self + vllm_config: + aggregated: + served-model-name: moonshotai/Kimi-K3 + tensor-parallel-size: 8 + decode-context-parallel-size: 8 + dcp-comm-backend: a2a + gpu-memory-utilization: 0.94 + max-num-batched-tokens: 16384 + trust-remote-code: true + language-model-only: true + load-format: fastsafetensors + moe-backend: auto + no-enable-flashinfer-autotune: true + enable-prefix-caching: true + prefix-match-unit: 128 + kv-cache-dtype: fp8 + stream-interval: 10 + attention-backend: TOKENSPEED_MLA + attention-config: '{"mla_prefill_backend":"TRTLLM_RAGGED","use_prefill_query_quantization":true}' + max-cudagraph-capture-size: 1024 + kv-transfer-config: '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"compact_group_io":true,"max_load_batch_keys":2,"enable_offload":false}}' +health_check: + max_attempts: 720 + interval_seconds: 10 +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: '300' + AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: '0.25' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'vllm:' + AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 +dynamo: + hash: ba83080ecd31c1ce918559e576d3c5bc9e092ff1 + install: true +identity: + model: + repo: moonshotai/Kimi-K3 + container: + image: vllm/vllm-openai:nightly-dev-x86_64-cu13-3696c77 + frameworks: + dynamo: ba83080ecd31c1ce918559e576d3c5bc9e092ff1 +slurm: + time_limit: 04:00:00 +sbatch_directives: + mem: '0' + cpus-per-task: '72' + comment: '''{"OccupiedIdleGPUsJobReaper":{"exemptIdleTimeMins":"60","reason":"model_loading","description":"Very large model will take extra loading time."}}''' + exclude: b300-018 +srun_options: + mem: '0' + container-remap-root: '' diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/disagg-b300-1p1d-dcp8-dcp8-dspark4-mooncake-c48-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/disagg-b300-1p1d-dcp8-dcp8-dspark4-mooncake-c48-agentic.yaml new file mode 100644 index 0000000000..203a4409eb --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/disagg-b300-1p1d-dcp8-dcp8-dspark4-mooncake-c48-agentic.yaml @@ -0,0 +1,242 @@ +# Kimi-K3 B300 disaggregated, agentic coding. +# +# Translated from the bia sweep config that measured this point: +# DISAGG/kimi-k3-mxfp4-b300-disagg-1p1d-dcp8-dcp8-agentx-062126-c48-weigb300-ns4-gmu94.yml +# Engine flags, environment, Mooncake settings and vllm_config are that file +# verbatim. The InferenceX plumbing -- identity, benchmark, slurm, the idle-GPU +# reaper comment -- is taken from disagg-gb300-1p1d-dcp8-dcp8-dspark4-mooncake-agentic.yaml. +# +# Only these fields differ from the measured config, each because b300-nv is +# not bia: +# device_name, UCX_NET_DEVICES -> the six rails read off this cluster +# MC_GID_INDEX, UCX_TLS -> set; b300-nv is RoCE (see below) +# HF_HOME -> dropped; benchmark.env sets HF_HUB_CACHE +# slurm.time_limit -> 04:00:00, this partition's limit +# rejection_sample_method -> block, so the eval verifies (see below) +# global_segment_size -> 281GB, matching the B300 arm on main +# sbatch_directives.exclude -> b300-018, which times out UCX/NIXL +# +# The rails are the part that needs saying. This fabric is rail-isolated: each +# rail is its own /24 and a QP only comes up when both ends sit on the same +# one. srtslurm writes a single store config for the whole job, so rails can +# only be named by index -- and the index is not stable here, because a down +# card shifts every index above it. Across the nodes inventoried, mlx5_0 is +# 172.16.192 everywhere except b300-012, and b300-003 swaps mlx5_3 with +# mlx5_9. Naming either put two nodes on different subnets under one name, +# which killed one earlier bring-up 36 minutes in. The six named below held on +# every node with a full inventory, and kimik3-b300-nic-topology.sh re-checks +# each one's GID subnet at startup so an uninventoried node fails in seconds +# instead of running to a bad number. + +name: kimi-k3-vllm-disagg-b300-1p1d-dcp8-dcp8-dspark4-mooncake-c48-agentic +model: + path: moonshotai/Kimi-K3 + container: vllm/vllm-openai:nightly-dev-x86_64-cu13-3696c77 + precision: fp4 +dynamo: + hash: ba83080ecd31c1ce918559e576d3c5bc9e092ff1 + install: true +slurm: + time_limit: 04:00:00 +health_check: + max_attempts: 720 + interval_seconds: 10 +resources: + gpu_type: b300 + gpus_per_node: 8 + prefill_nodes: 1 + decode_nodes: 1 + prefill_workers: 1 + decode_workers: 1 + gpus_per_prefill: 8 + gpus_per_decode: 8 +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 +frontend: + type: dynamo + enable_multiple_frontends: false + args: + router-mode: least-loaded + router-session-affinity-ttl-secs: 900 + env: + DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: '3600' + DYN_TOKENIZER_CACHE_BYTES: '8589934592' +backend: + type: vllm + connector: null + dp_launch_mode: per_node + kv_events_config: + prefill: true + mooncake_kv_store: + master_extra_args: + - --default_kv_lease_ttl=60000 + - --eviction_high_watermark_ratio=0.95 + - --eviction_ratio=0.10 + store_config: + metadata_server: P2PHANDSHAKE + global_segment_size: 281GB + local_buffer_size: 4GB + protocol: rdma + device_name: mlx5_4,mlx5_8,mlx5_10,mlx5_16,mlx5_20,mlx5_22 + mode: embedded + enable_offload: false + prefill_environment: + VLLM_USE_DIRECT_DCP_A2A: '1' + VLLM_USE_DIRECT_DCP_Q_GATHER: '1' + VLLM_USE_DIRECT_DCP_KV_GATHER: '1' + VLLM_ALLREDUCE_USE_FLASHINFER: '1' + VLLM_KIMI_K3_SHARD_SP_SHARED_EXPERT: '0' + DYN_REQUEST_PLANE: tcp + ETCD_LEASE_TTL: '600' + VLLM_ENGINE_READY_TIMEOUT_S: '3600' + VLLM_RPC_TIMEOUT: '600000' + TILELANG_CLEANUP_TEMP_FILES: '1' + VLLM_USE_NCCL_SYMM_MEM: '0' + NCCL_CUMEM_ENABLE: '1' + NCCL_MNNVL_ENABLE: '0' + NCCL_NVLS_ENABLE: '1' + VLLM_SERVER_DEV_MODE: '1' + VLLM_USE_V2_MODEL_RUNNER: '1' + VLLM_USE_RUST_FRONTEND: '1' + VLLM_MOONCAKE_LOAD_RECV_THREADS: '4' + MC_SLICE_SIZE: '1048576' + VLLM_CONNECTOR_PREFETCH_DEPTH: '8' + VLLM_CONNECTOR_PREFETCH_KV_CAP: '0.65' + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: '1800' + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: '0' + NCCL_P2P_LEVEL: NVL + MC_ENABLE_DEST_DEVICE_AFFINITY: '1' + WITH_NVIDIA_PEERMEM: '0' + NCCL_NET_PLUGIN: none + UCX_MEMTYPE_CACHE: n + UCX_MEMTYPE_REG_WHOLE: n + UCX_RCACHE_MAX_UNRELEASED: '1024' + UCX_TCP_AF_PRIO: inet + VLLM_SSM_CONV_STATE_LAYOUT: DS + UCX_NET_DEVICES: mlx5_4:1,mlx5_8:1,mlx5_10:1,mlx5_16:1,mlx5_20:1,mlx5_22:1 + DG_JIT_CACHE_DIR: /tmp/dg-cache-kimi-k3-gb300-pd-dspark-mooncake-{job_id} + PYTHONHASHSEED: '42' + PYTHONNOUSERSITE: '1' + PYTHONUNBUFFERED: '1' + TORCH_CUDA_ARCH_LIST: '10.0' + MC_STORE_CLIENT_METRIC: '1' + MC_STORE_CLIENT_METRIC_INTERVAL: '5' + MC_TE_METRIC: '0' + MC_GID_INDEX: '3' + UCX_TLS: rc,cuda_copy,cuda_ipc,sm,self + decode_environment: + VLLM_USE_DIRECT_DCP_A2A: '1' + VLLM_USE_DIRECT_DCP_Q_GATHER: '1' + VLLM_USE_DIRECT_DCP_KV_GATHER: '1' + VLLM_ALLREDUCE_USE_FLASHINFER: '1' + VLLM_KIMI_K3_SHARD_SP_SHARED_EXPERT: '0' + DYN_REQUEST_PLANE: tcp + ETCD_LEASE_TTL: '600' + VLLM_ENGINE_READY_TIMEOUT_S: '3600' + VLLM_RPC_TIMEOUT: '600000' + TILELANG_CLEANUP_TEMP_FILES: '1' + VLLM_USE_NCCL_SYMM_MEM: '0' + NCCL_CUMEM_ENABLE: '1' + NCCL_MNNVL_ENABLE: '0' + NCCL_NVLS_ENABLE: '1' + VLLM_SERVER_DEV_MODE: '1' + VLLM_USE_V2_MODEL_RUNNER: '1' + VLLM_USE_RUST_FRONTEND: '1' + VLLM_MOONCAKE_LOAD_RECV_THREADS: '4' + MC_SLICE_SIZE: '1048576' + VLLM_CONNECTOR_PREFETCH_DEPTH: '8' + VLLM_CONNECTOR_PREFETCH_KV_CAP: '0.65' + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: '1800' + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: '0' + NCCL_P2P_LEVEL: NVL + MC_ENABLE_DEST_DEVICE_AFFINITY: '1' + WITH_NVIDIA_PEERMEM: '0' + NCCL_NET_PLUGIN: none + UCX_MEMTYPE_CACHE: n + UCX_MEMTYPE_REG_WHOLE: n + UCX_RCACHE_MAX_UNRELEASED: '1024' + UCX_TCP_AF_PRIO: inet + VLLM_SSM_CONV_STATE_LAYOUT: DS + DG_JIT_CACHE_DIR: /tmp/dg-cache-kimi-k3-gb300-pd-dspark-mooncake-{job_id} + PYTHONHASHSEED: '42' + PYTHONNOUSERSITE: '1' + PYTHONUNBUFFERED: '1' + TORCH_CUDA_ARCH_LIST: '10.0' + MC_STORE_CLIENT_METRIC: '1' + MC_STORE_CLIENT_METRIC_INTERVAL: '5' + MC_TE_METRIC: '0' + MC_GID_INDEX: '3' + UCX_TLS: rc,cuda_copy,cuda_ipc,sm,self + vllm_config: + prefill: + no-enable-flashinfer-autotune: true + enable-logging-iteration-details: true + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"compact_group_io":true,"max_load_batch_keys":2,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + served-model-name: moonshotai/Kimi-K3 + prefix-match-unit: 128 + load-format: fastsafetensors + kv-cache-dtype: fp8 + gpu-memory-utilization: 0.94 + tensor-parallel-size: 8 + decode-context-parallel-size: 8 + dcp-comm-backend: a2a + enable-cumem-allocator: true + trust-remote-code: true + max-cudagraph-capture-size: 512 + stream-interval: 10 + language-model-only: true + attention-backend: TOKENSPEED_MLA + attention-config: '{"mla_prefill_backend": "TRTLLM_RAGGED", "use_prefill_query_quantization": true}' + speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","attention_backend":"TOKENSPEED_MLA","method":"dspark","num_speculative_tokens":4,"rejection_sample_method":"block"}' + enable-prefix-caching: true + decode: + no-enable-flashinfer-autotune: true + enable-logging-iteration-details: true + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"compact_group_io":true,"max_load_batch_keys":2,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + served-model-name: moonshotai/Kimi-K3 + prefix-match-unit: 128 + load-format: fastsafetensors + kv-cache-dtype: fp8 + gpu-memory-utilization: 0.94 + tensor-parallel-size: 8 + decode-context-parallel-size: 8 + dcp-comm-backend: a2a + enable-cumem-allocator: true + trust-remote-code: true + max-cudagraph-capture-size: 512 + stream-interval: 10 + language-model-only: true + attention-backend: TOKENSPEED_MLA + attention-config: '{"mla_prefill_backend": "TRTLLM_RAGGED", "use_prefill_query_quantization": true}' + speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","attention_backend":"TOKENSPEED_MLA","method":"dspark","num_speculative_tokens":4,"rejection_sample_method":"block"}' + enable-prefix-caching: true +sbatch_directives: + cpus-per-task: '72' + mem: '0' + comment: '''{"OccupiedIdleGPUsJobReaper":{"exemptIdleTimeMins":"60","reason":"model_loading","description":"Very large model will take extra loading time."}}''' + exclude: b300-018 +srun_options: + mem: '0' + container-remap-root: '' +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: '14400' + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 +identity: + model: + repo: moonshotai/Kimi-K3 + container: + image: vllm/vllm-openai:nightly-dev-x86_64-cu13-3696c77 + frameworks: + dynamo: ba83080ecd31c1ce918559e576d3c5bc9e092ff1 diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/disagg-b300-1p2d-dcp8-dcp8-dspark4-mooncake-c48-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/disagg-b300-1p2d-dcp8-dcp8-dspark4-mooncake-c48-agentic.yaml new file mode 100644 index 0000000000..f08da3187f --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/disagg-b300-1p2d-dcp8-dcp8-dspark4-mooncake-c48-agentic.yaml @@ -0,0 +1,239 @@ +# Kimi-K3 B300 disaggregated, agentic coding. +# +# Translated from the bia sweep config that measured this point: +# DISAGG/kimi-k3-mxfp4-b300-disagg-1p2d-dcp8-dcp8-agentx-062126-c48-wei1p2d-ns4-gmu94.yml +# Engine flags, environment, Mooncake settings and vllm_config are that file +# verbatim. The InferenceX plumbing -- identity, benchmark, slurm, the idle-GPU +# reaper comment -- is taken from disagg-gb300-1p2d-dcp8-dcp8-dspark4-mooncake-agentic.yaml. +# +# Only these fields differ from the measured config, each because b300-nv is +# not bia: +# device_name, UCX_NET_DEVICES -> the six rails read off this cluster +# MC_GID_INDEX, UCX_TLS -> set; b300-nv is RoCE (see below) +# HF_HOME -> dropped; benchmark.env sets HF_HUB_CACHE +# slurm.time_limit -> 04:00:00, this partition's limit +# rejection_sample_method -> block, so the eval verifies (see below) +# global_segment_size -> 281GB, matching the B300 arm on main +# sbatch_directives.exclude -> b300-018, which times out UCX/NIXL +# +# The rails are the part that needs saying. This fabric is rail-isolated: each +# rail is its own /24 and a QP only comes up when both ends sit on the same +# one. srtslurm writes a single store config for the whole job, so rails can +# only be named by index -- and the index is not stable here, because a down +# card shifts every index above it. Across the nodes inventoried, mlx5_0 is +# 172.16.192 everywhere except b300-012, and b300-003 swaps mlx5_3 with +# mlx5_9. Naming either put two nodes on different subnets under one name, +# which killed one earlier bring-up 36 minutes in. The six named below held on +# every node with a full inventory, and kimik3-b300-nic-topology.sh re-checks +# each one's GID subnet at startup so an uninventoried node fails in seconds +# instead of running to a bad number. + +name: kimi-k3-vllm-disagg-b300-1p2d-dcp8-dcp8-dspark4-mooncake-c48-agentic +model: + path: moonshotai/Kimi-K3 + container: vllm/vllm-openai:nightly-dev-x86_64-cu13-3696c77 + precision: fp4 +dynamo: + hash: ba83080ecd31c1ce918559e576d3c5bc9e092ff1 + install: true +slurm: + time_limit: 04:00:00 +health_check: + max_attempts: 720 + interval_seconds: 10 +resources: + gpu_type: b300 + gpus_per_node: 8 + prefill_nodes: 1 + decode_nodes: 2 + prefill_workers: 1 + decode_workers: 2 + gpus_per_prefill: 8 + gpus_per_decode: 8 +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 +frontend: + type: dynamo + enable_multiple_frontends: false + args: + router-mode: least-loaded + router-session-affinity-ttl-secs: 900 + env: + DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: '3600' + DYN_TOKENIZER_CACHE_BYTES: '8589934592' + PYTHONPYCACHEPREFIX: /tmp/vllm-pycache-kimi-k3 +backend: + type: vllm + connector: null + dp_launch_mode: per_node + kv_events_config: + prefill: true + mooncake_kv_store: + master_extra_args: + - --default_kv_lease_ttl=60000 + - --eviction_high_watermark_ratio=0.95 + - --eviction_ratio=0.10 + store_config: + metadata_server: P2PHANDSHAKE + global_segment_size: 281GB + local_buffer_size: 4GB + protocol: rdma + device_name: mlx5_4,mlx5_8,mlx5_10,mlx5_16,mlx5_20,mlx5_22 + mode: embedded + enable_offload: false + prefill_environment: + VLLM_USE_DIRECT_DCP_A2A: '1' + VLLM_USE_DIRECT_DCP_Q_GATHER: '1' + VLLM_USE_DIRECT_DCP_KV_GATHER: '1' + VLLM_ALLREDUCE_USE_FLASHINFER: '1' + VLLM_KIMI_K3_SHARD_SP_SHARED_EXPERT: '0' + DYN_REQUEST_PLANE: tcp + ETCD_LEASE_TTL: '600' + VLLM_ENGINE_READY_TIMEOUT_S: '3600' + VLLM_RPC_TIMEOUT: '600000' + TILELANG_CLEANUP_TEMP_FILES: '1' + VLLM_USE_NCCL_SYMM_MEM: '0' + NCCL_CUMEM_ENABLE: '1' + NCCL_MNNVL_ENABLE: '0' + NCCL_NVLS_ENABLE: '1' + VLLM_SERVER_DEV_MODE: '1' + VLLM_USE_V2_MODEL_RUNNER: '1' + VLLM_USE_RUST_FRONTEND: '1' + VLLM_MOONCAKE_LOAD_RECV_THREADS: '4' + MC_SLICE_SIZE: '1048576' + VLLM_CONNECTOR_PREFETCH_DEPTH: '8' + VLLM_CONNECTOR_PREFETCH_KV_CAP: '0.65' + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: '1800' + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: '0' + NCCL_P2P_LEVEL: NVL + MC_ENABLE_DEST_DEVICE_AFFINITY: '1' + WITH_NVIDIA_PEERMEM: '0' + NCCL_NET_PLUGIN: none + UCX_MEMTYPE_CACHE: n + UCX_MEMTYPE_REG_WHOLE: n + UCX_RCACHE_MAX_UNRELEASED: '1024' + UCX_TCP_AF_PRIO: inet + VLLM_SSM_CONV_STATE_LAYOUT: DS + UCX_NET_DEVICES: mlx5_4:1,mlx5_8:1,mlx5_10:1,mlx5_16:1,mlx5_20:1,mlx5_22:1 + DG_JIT_CACHE_DIR: /tmp/dg-cache-kimi-k3-gb300-pd-dspark-mooncake-{job_id} + PYTHONHASHSEED: '42' + PYTHONNOUSERSITE: '1' + PYTHONUNBUFFERED: '1' + TORCH_CUDA_ARCH_LIST: '10.0' + MC_STORE_CLIENT_METRIC: '1' + MC_STORE_CLIENT_METRIC_INTERVAL: '5' + MC_TE_METRIC: '0' + MC_GID_INDEX: '3' + UCX_TLS: rc,cuda_copy,cuda_ipc,sm,self + decode_environment: + VLLM_USE_DIRECT_DCP_A2A: '1' + VLLM_USE_DIRECT_DCP_Q_GATHER: '1' + VLLM_USE_DIRECT_DCP_KV_GATHER: '1' + VLLM_ALLREDUCE_USE_FLASHINFER: '1' + VLLM_KIMI_K3_SHARD_SP_SHARED_EXPERT: '0' + DYN_REQUEST_PLANE: tcp + ETCD_LEASE_TTL: '600' + VLLM_ENGINE_READY_TIMEOUT_S: '3600' + VLLM_RPC_TIMEOUT: '600000' + TILELANG_CLEANUP_TEMP_FILES: '1' + VLLM_USE_NCCL_SYMM_MEM: '0' + NCCL_CUMEM_ENABLE: '1' + NCCL_MNNVL_ENABLE: '0' + NCCL_NVLS_ENABLE: '1' + VLLM_SERVER_DEV_MODE: '1' + VLLM_USE_V2_MODEL_RUNNER: '1' + VLLM_USE_RUST_FRONTEND: '1' + VLLM_MOONCAKE_LOAD_RECV_THREADS: '4' + MC_SLICE_SIZE: '1048576' + VLLM_CONNECTOR_PREFETCH_DEPTH: '8' + VLLM_CONNECTOR_PREFETCH_KV_CAP: '0.65' + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: '1800' + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: '0' + NCCL_P2P_LEVEL: NVL + MC_ENABLE_DEST_DEVICE_AFFINITY: '1' + WITH_NVIDIA_PEERMEM: '0' + NCCL_NET_PLUGIN: none + UCX_MEMTYPE_CACHE: n + UCX_MEMTYPE_REG_WHOLE: n + UCX_RCACHE_MAX_UNRELEASED: '1024' + UCX_TCP_AF_PRIO: inet + VLLM_SSM_CONV_STATE_LAYOUT: DS + DG_JIT_CACHE_DIR: /tmp/dg-cache-kimi-k3-gb300-pd-dspark-mooncake-{job_id} + PYTHONHASHSEED: '42' + PYTHONNOUSERSITE: '1' + PYTHONUNBUFFERED: '1' + TORCH_CUDA_ARCH_LIST: '10.0' + MC_STORE_CLIENT_METRIC: '1' + MC_STORE_CLIENT_METRIC_INTERVAL: '5' + MC_TE_METRIC: '0' + MC_GID_INDEX: '3' + UCX_TLS: rc,cuda_copy,cuda_ipc,sm,self + vllm_config: + prefill: + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"compact_group_io":true,"max_load_batch_keys":2,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + served-model-name: moonshotai/Kimi-K3 + prefix-match-unit: 128 + load-format: fastsafetensors + kv-cache-dtype: fp8 + gpu-memory-utilization: 0.94 + tensor-parallel-size: 8 + decode-context-parallel-size: 8 + dcp-comm-backend: a2a + enable-cumem-allocator: true + trust-remote-code: true + max-cudagraph-capture-size: 512 + stream-interval: 10 + language-model-only: true + attention-backend: TOKENSPEED_MLA + attention-config: '{"mla_prefill_backend": "TRTLLM_RAGGED", "use_prefill_query_quantization": true}' + speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","attention_backend":"TOKENSPEED_MLA","method":"dspark","num_speculative_tokens":4,"rejection_sample_method":"block"}' + enable-prefix-caching: true + decode: + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"compact_group_io":true,"max_load_batch_keys":2,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + served-model-name: moonshotai/Kimi-K3 + prefix-match-unit: 128 + load-format: fastsafetensors + kv-cache-dtype: fp8 + gpu-memory-utilization: 0.94 + tensor-parallel-size: 8 + decode-context-parallel-size: 8 + dcp-comm-backend: a2a + enable-cumem-allocator: true + trust-remote-code: true + max-cudagraph-capture-size: 512 + stream-interval: 10 + language-model-only: true + attention-backend: TOKENSPEED_MLA + attention-config: '{"mla_prefill_backend": "TRTLLM_RAGGED", "use_prefill_query_quantization": true}' + speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","attention_backend":"TOKENSPEED_MLA","method":"dspark","num_speculative_tokens":4,"rejection_sample_method":"block"}' + enable-prefix-caching: true +sbatch_directives: + cpus-per-task: '72' + mem: '0' + comment: '''{"OccupiedIdleGPUsJobReaper":{"exemptIdleTimeMins":"60","reason":"model_loading","description":"Very large model will take extra loading time."}}''' + exclude: b300-018 +srun_options: + mem: '0' + container-remap-root: '' +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: '14400' + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 +identity: + model: + repo: moonshotai/Kimi-K3 + container: + image: vllm/vllm-openai:nightly-dev-x86_64-cu13-3696c77 + frameworks: + dynamo: ba83080ecd31c1ce918559e576d3c5bc9e092ff1 diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/disagg-b300-1p3d-dcp8-dcp8-dspark4-mooncake-c32-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/disagg-b300-1p3d-dcp8-dcp8-dspark4-mooncake-c32-agentic.yaml new file mode 100644 index 0000000000..d8ce175f84 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/disagg-b300-1p3d-dcp8-dcp8-dspark4-mooncake-c32-agentic.yaml @@ -0,0 +1,239 @@ +# Kimi-K3 B300 disaggregated, agentic coding. +# +# Translated from the bia sweep config that measured this point: +# DISAGG/kimi-k3-mxfp4-b300-disagg-1p3d-dcp8-dcp8-agentx-062126-c32-wei1p3d-ns4-gmu94.yml +# Engine flags, environment, Mooncake settings and vllm_config are that file +# verbatim. The InferenceX plumbing -- identity, benchmark, slurm, the idle-GPU +# reaper comment -- is taken from disagg-gb300-1p2d-dcp8-dcp8-dspark4-mooncake-agentic.yaml. +# +# Only these fields differ from the measured config, each because b300-nv is +# not bia: +# device_name, UCX_NET_DEVICES -> the six rails read off this cluster +# MC_GID_INDEX, UCX_TLS -> set; b300-nv is RoCE (see below) +# HF_HOME -> dropped; benchmark.env sets HF_HUB_CACHE +# slurm.time_limit -> 04:00:00, this partition's limit +# rejection_sample_method -> block, so the eval verifies (see below) +# global_segment_size -> 281GB, matching the B300 arm on main +# sbatch_directives.exclude -> b300-018, which times out UCX/NIXL +# +# The rails are the part that needs saying. This fabric is rail-isolated: each +# rail is its own /24 and a QP only comes up when both ends sit on the same +# one. srtslurm writes a single store config for the whole job, so rails can +# only be named by index -- and the index is not stable here, because a down +# card shifts every index above it. Across the nodes inventoried, mlx5_0 is +# 172.16.192 everywhere except b300-012, and b300-003 swaps mlx5_3 with +# mlx5_9. Naming either put two nodes on different subnets under one name, +# which killed one earlier bring-up 36 minutes in. The six named below held on +# every node with a full inventory, and kimik3-b300-nic-topology.sh re-checks +# each one's GID subnet at startup so an uninventoried node fails in seconds +# instead of running to a bad number. + +name: kimi-k3-vllm-disagg-b300-1p3d-dcp8-dcp8-dspark4-mooncake-c32-agentic +model: + path: moonshotai/Kimi-K3 + container: vllm/vllm-openai:nightly-dev-x86_64-cu13-3696c77 + precision: fp4 +dynamo: + hash: ba83080ecd31c1ce918559e576d3c5bc9e092ff1 + install: true +slurm: + time_limit: 04:00:00 +health_check: + max_attempts: 720 + interval_seconds: 10 +resources: + gpu_type: b300 + gpus_per_node: 8 + prefill_nodes: 1 + decode_nodes: 3 + prefill_workers: 1 + decode_workers: 3 + gpus_per_prefill: 8 + gpus_per_decode: 8 +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 +frontend: + type: dynamo + enable_multiple_frontends: false + args: + router-mode: least-loaded + router-session-affinity-ttl-secs: 900 + env: + DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: '3600' + DYN_TOKENIZER_CACHE_BYTES: '8589934592' + PYTHONPYCACHEPREFIX: /tmp/vllm-pycache-kimi-k3 +backend: + type: vllm + connector: null + dp_launch_mode: per_node + kv_events_config: + prefill: true + mooncake_kv_store: + master_extra_args: + - --default_kv_lease_ttl=60000 + - --eviction_high_watermark_ratio=0.95 + - --eviction_ratio=0.10 + store_config: + metadata_server: P2PHANDSHAKE + global_segment_size: 281GB + local_buffer_size: 4GB + protocol: rdma + device_name: mlx5_4,mlx5_8,mlx5_10,mlx5_16,mlx5_20,mlx5_22 + mode: embedded + enable_offload: false + prefill_environment: + VLLM_USE_DIRECT_DCP_A2A: '1' + VLLM_USE_DIRECT_DCP_Q_GATHER: '1' + VLLM_USE_DIRECT_DCP_KV_GATHER: '1' + VLLM_ALLREDUCE_USE_FLASHINFER: '1' + VLLM_KIMI_K3_SHARD_SP_SHARED_EXPERT: '0' + DYN_REQUEST_PLANE: tcp + ETCD_LEASE_TTL: '600' + VLLM_ENGINE_READY_TIMEOUT_S: '3600' + VLLM_RPC_TIMEOUT: '600000' + TILELANG_CLEANUP_TEMP_FILES: '1' + VLLM_USE_NCCL_SYMM_MEM: '0' + NCCL_CUMEM_ENABLE: '1' + NCCL_MNNVL_ENABLE: '0' + NCCL_NVLS_ENABLE: '1' + VLLM_SERVER_DEV_MODE: '1' + VLLM_USE_V2_MODEL_RUNNER: '1' + VLLM_USE_RUST_FRONTEND: '1' + VLLM_MOONCAKE_LOAD_RECV_THREADS: '4' + MC_SLICE_SIZE: '1048576' + VLLM_CONNECTOR_PREFETCH_DEPTH: '8' + VLLM_CONNECTOR_PREFETCH_KV_CAP: '0.65' + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: '1800' + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: '0' + NCCL_P2P_LEVEL: NVL + MC_ENABLE_DEST_DEVICE_AFFINITY: '1' + WITH_NVIDIA_PEERMEM: '0' + NCCL_NET_PLUGIN: none + UCX_MEMTYPE_CACHE: n + UCX_MEMTYPE_REG_WHOLE: n + UCX_RCACHE_MAX_UNRELEASED: '1024' + UCX_TCP_AF_PRIO: inet + VLLM_SSM_CONV_STATE_LAYOUT: DS + UCX_NET_DEVICES: mlx5_4:1,mlx5_8:1,mlx5_10:1,mlx5_16:1,mlx5_20:1,mlx5_22:1 + DG_JIT_CACHE_DIR: /tmp/dg-cache-kimi-k3-gb300-pd-dspark-mooncake-{job_id} + PYTHONHASHSEED: '42' + PYTHONNOUSERSITE: '1' + PYTHONUNBUFFERED: '1' + TORCH_CUDA_ARCH_LIST: '10.0' + MC_STORE_CLIENT_METRIC: '1' + MC_STORE_CLIENT_METRIC_INTERVAL: '5' + MC_TE_METRIC: '0' + MC_GID_INDEX: '3' + UCX_TLS: rc,cuda_copy,cuda_ipc,sm,self + decode_environment: + VLLM_USE_DIRECT_DCP_A2A: '1' + VLLM_USE_DIRECT_DCP_Q_GATHER: '1' + VLLM_USE_DIRECT_DCP_KV_GATHER: '1' + VLLM_ALLREDUCE_USE_FLASHINFER: '1' + VLLM_KIMI_K3_SHARD_SP_SHARED_EXPERT: '0' + DYN_REQUEST_PLANE: tcp + ETCD_LEASE_TTL: '600' + VLLM_ENGINE_READY_TIMEOUT_S: '3600' + VLLM_RPC_TIMEOUT: '600000' + TILELANG_CLEANUP_TEMP_FILES: '1' + VLLM_USE_NCCL_SYMM_MEM: '0' + NCCL_CUMEM_ENABLE: '1' + NCCL_MNNVL_ENABLE: '0' + NCCL_NVLS_ENABLE: '1' + VLLM_SERVER_DEV_MODE: '1' + VLLM_USE_V2_MODEL_RUNNER: '1' + VLLM_USE_RUST_FRONTEND: '1' + VLLM_MOONCAKE_LOAD_RECV_THREADS: '4' + MC_SLICE_SIZE: '1048576' + VLLM_CONNECTOR_PREFETCH_DEPTH: '8' + VLLM_CONNECTOR_PREFETCH_KV_CAP: '0.65' + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: '1800' + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: '0' + NCCL_P2P_LEVEL: NVL + MC_ENABLE_DEST_DEVICE_AFFINITY: '1' + WITH_NVIDIA_PEERMEM: '0' + NCCL_NET_PLUGIN: none + UCX_MEMTYPE_CACHE: n + UCX_MEMTYPE_REG_WHOLE: n + UCX_RCACHE_MAX_UNRELEASED: '1024' + UCX_TCP_AF_PRIO: inet + VLLM_SSM_CONV_STATE_LAYOUT: DS + DG_JIT_CACHE_DIR: /tmp/dg-cache-kimi-k3-gb300-pd-dspark-mooncake-{job_id} + PYTHONHASHSEED: '42' + PYTHONNOUSERSITE: '1' + PYTHONUNBUFFERED: '1' + TORCH_CUDA_ARCH_LIST: '10.0' + MC_STORE_CLIENT_METRIC: '1' + MC_STORE_CLIENT_METRIC_INTERVAL: '5' + MC_TE_METRIC: '0' + MC_GID_INDEX: '3' + UCX_TLS: rc,cuda_copy,cuda_ipc,sm,self + vllm_config: + prefill: + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"compact_group_io":true,"max_load_batch_keys":2,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + served-model-name: moonshotai/Kimi-K3 + prefix-match-unit: 128 + load-format: fastsafetensors + kv-cache-dtype: fp8 + gpu-memory-utilization: 0.94 + tensor-parallel-size: 8 + decode-context-parallel-size: 8 + dcp-comm-backend: a2a + enable-cumem-allocator: true + trust-remote-code: true + max-cudagraph-capture-size: 512 + stream-interval: 10 + language-model-only: true + attention-backend: TOKENSPEED_MLA + attention-config: '{"mla_prefill_backend": "TRTLLM_RAGGED", "use_prefill_query_quantization": true}' + speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","attention_backend":"TOKENSPEED_MLA","method":"dspark","num_speculative_tokens":4,"rejection_sample_method":"block"}' + enable-prefix-caching: true + decode: + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"compact_group_io":true,"max_load_batch_keys":2,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + served-model-name: moonshotai/Kimi-K3 + prefix-match-unit: 128 + load-format: fastsafetensors + kv-cache-dtype: fp8 + gpu-memory-utilization: 0.94 + tensor-parallel-size: 8 + decode-context-parallel-size: 8 + dcp-comm-backend: a2a + enable-cumem-allocator: true + trust-remote-code: true + max-cudagraph-capture-size: 512 + stream-interval: 10 + language-model-only: true + attention-backend: TOKENSPEED_MLA + attention-config: '{"mla_prefill_backend": "TRTLLM_RAGGED", "use_prefill_query_quantization": true}' + speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","attention_backend":"TOKENSPEED_MLA","method":"dspark","num_speculative_tokens":4,"rejection_sample_method":"block"}' + enable-prefix-caching: true +sbatch_directives: + cpus-per-task: '72' + mem: '0' + comment: '''{"OccupiedIdleGPUsJobReaper":{"exemptIdleTimeMins":"60","reason":"model_loading","description":"Very large model will take extra loading time."}}''' + exclude: b300-018 +srun_options: + mem: '0' + container-remap-root: '' +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: '14400' + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 +identity: + model: + repo: moonshotai/Kimi-K3 + container: + image: vllm/vllm-openai:nightly-dev-x86_64-cu13-3696c77 + frameworks: + dynamo: ba83080ecd31c1ce918559e576d3c5bc9e092ff1 diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 0a6c86df6d..d407e3f4e4 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -9773,6 +9773,164 @@ kimik3-fp4-gb300-dynamo-vllm-agentic-mooncake-dcp8-agg: - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-gb300-dcp8-nospec-mooncake-agentic.yaml" + +# Kimi-K3 MXFP4 B300 agentic coding, six points. All three keys below run the +# same TP8 x DCP8 worker; what changes is how many of them there are and +# whether they draft. +# +# The topology is why B300 is a separate entry from GB300. A +# DCP8 worker spans two 4-GPU nodes on GB300 but fits inside one 8-GPU node +# here, so a worker never leaves its NVLink domain and only the +# prefill-to-decode hop crosses the fabric. +# +# 0.75 is the value the existing B300 Kimi-K3 arm uses on this cluster. In the +# single-node path it resolves to 2249 GB and the script divides by TP8; the +# recipes below name the resulting per-rank segment directly, since srt-slurm +# takes global_segment_size as a literal and does no division. +# +# One recipe per concurrency, because the measured configs differ by more than +# concurrency: the drafting arms pin max-num-seqs 2 and capture at 1024, the +# no-spec arm runs max-num-batched-tokens 16384 with moe-backend auto, and the +# disaggregated arms carry the NIXL connector the aggregated ones have no use +# for. + +# One node, no speculative decoding. +kimik3-fp4-b300-dynamo-vllm-agentic-mooncake-dcp8-agg: + image: vllm/vllm-openai:nightly-dev-x86_64-cu13-3696c77 + model: moonshotai/Kimi-K3 + model-prefix: kimik3 + runner: cluster:b300-nv + precision: fp4 + framework: dynamo-vllm + multinode: true + disagg: false + scenarios: + agentic-coding: + - dram-utilization: 0.75 + search-space: + - kv-offloading: dram + kv-offload-backend: { name: mooncake, version: "0.3.12.post1" } + conc-list: [70] + num-nodes: 1 + worker: + num-worker: 1 + tp: 8 + dcp-size: 8 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-b300-dcp8-nospec-mooncake-c70-agentic.yaml" + +# One node drafting at DSpark K=7 with max-num-seqs 2. 3.84 is the golden AL +# at K=7 on the probabilistic/block curve, the same figure the B300 +# single-node arm and the GB300 DSpark-7 arm pin. +kimik3-fp4-b300-dynamo-vllm-agentic-dspark-mooncake-dcp8-agg: + image: vllm/vllm-openai:nightly-dev-x86_64-cu13-3696c77 + model: moonshotai/Kimi-K3 + model-prefix: kimik3 + runner: cluster:b300-nv + precision: fp4 + framework: dynamo-vllm + multinode: true + disagg: false + scenarios: + agentic-coding: + - dram-utilization: 0.75 + search-space: + - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: { name: mooncake, version: "0.3.12.post1" } + conc-list: [4] + num-nodes: 1 + worker: + num-worker: 1 + tp: 8 + dcp-size: 8 + ep: 1 + dp-attn: false + additional-settings: + - "SYNTHETIC_ACCEPTANCE=true" + - "SYNTHETIC_ACCEPTANCE_LENGTH=3.84" + - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-b300-dcp8-dspark7-maxseq2-mooncake-c4-agentic.yaml" + - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: { name: mooncake, version: "0.3.12.post1" } + conc-list: [1] + num-nodes: 1 + worker: + num-worker: 1 + tp: 8 + dcp-size: 8 + ep: 1 + dp-attn: false + additional-settings: + - "SYNTHETIC_ACCEPTANCE=true" + - "SYNTHETIC_ACCEPTANCE_LENGTH=3.84" + - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-b300-dcp8-dspark7-maxseq2-mooncake-c1-agentic.yaml" + +# One prefill worker feeding one, two or three decode workers, each a whole +# 8-GPU node, drafting at DSpark K=4. +kimik3-fp4-b300-dynamo-vllm-agentic-dspark-mooncake-dcp8-disagg: + image: vllm/vllm-openai:nightly-dev-x86_64-cu13-3696c77 + model: moonshotai/Kimi-K3 + model-prefix: kimik3 + runner: cluster:b300-nv + precision: fp4 + framework: dynamo-vllm + router: { name: dynamo-router, version: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1" } + kv-p2p-transfer: nixl + multinode: true + disagg: true + scenarios: + agentic-coding: + - dram-utilization: 0.75 + search-space: + - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: { name: mooncake, version: "0.3.11.post1" } + conc-list: [48] + prefill: + num-worker: 1 + tp: 8 + dcp-size: 8 + ep: 1 + dp-attn: false + additional-settings: + - "SYNTHETIC_ACCEPTANCE=true" + - "SYNTHETIC_ACCEPTANCE_LENGTH=3.36" + - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/disagg-b300-1p1d-dcp8-dcp8-dspark4-mooncake-c48-agentic.yaml" + decode: { num-worker: 1, tp: 8, dcp-size: 8, ep: 1, dp-attn: false } + - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: { name: mooncake, version: "0.3.11.post1" } + conc-list: [48] + prefill: + num-worker: 1 + tp: 8 + dcp-size: 8 + ep: 1 + dp-attn: false + additional-settings: + - "SYNTHETIC_ACCEPTANCE=true" + - "SYNTHETIC_ACCEPTANCE_LENGTH=3.36" + - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/disagg-b300-1p2d-dcp8-dcp8-dspark4-mooncake-c48-agentic.yaml" + decode: { num-worker: 2, tp: 8, dcp-size: 8, ep: 1, dp-attn: false } + - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: { name: mooncake, version: "0.3.11.post1" } + conc-list: [32] + prefill: + num-worker: 1 + tp: 8 + dcp-size: 8 + ep: 1 + dp-attn: false + additional-settings: + - "SYNTHETIC_ACCEPTANCE=true" + - "SYNTHETIC_ACCEPTANCE_LENGTH=3.36" + - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/disagg-b300-1p3d-dcp8-dcp8-dspark4-mooncake-c32-agentic.yaml" + decode: { num-worker: 3, tp: 8, dcp-size: 8, ep: 1, dp-attn: false } + # Kimi-K3 MXFP4 B200 aggregated vLLM (TP8 x PP2, 2 nodes / 16 GPUs), agentic # coding. The native MXFP4 checkpoint (2.8T total params, ~1.4TB weights) does # not fit one 8xB200 node, so TP8 shards attention/dense and PP2 splits layers. diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 14434c76ad..09559e8ce9 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6878,3 +6878,16 @@ description: - "Refresh the Kimi-K3 GB300 AgentX configurations with updated configs." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2811 + +- config-keys: + - kimik3-fp4-b300-dynamo-vllm-agentic-mooncake-dcp8-agg + - kimik3-fp4-b300-dynamo-vllm-agentic-dspark-mooncake-dcp8-agg + - kimik3-fp4-b300-dynamo-vllm-agentic-dspark-mooncake-dcp8-disagg + scenario-type: + - agentic-coding + description: + - "Add the six measured Kimi-K3 B300 AgentX points: aggregated TP8 x DCP8 without speculative decoding at c70, aggregated DSpark-7 at c4 and c1, and 1P1D, 1P2D and 1P3D DCP8 x DCP8 DSpark-4 at c48, c48 and c32." + - "Give Kimi-K3 a model path on b300-dsxe and route the agentic scenario to upstream srt-slurm with the in-repo recipes, without which multi-node Kimi-K3 exits as an unsupported model." + - "Apply synthetic acceptance on throughput runs, not only eval-only ones, so the pinned acceptance length reaches speculative-config as it already does on gb300-nv." + - "Let the fabric select its own paths instead of pinning transfer devices, and check the expected adapter inventory before the engine starts." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/PENDING diff --git a/runners/launch_b300-dsxe.sh b/runners/launch_b300-dsxe.sh index c229146bfa..a85090e9a3 100755 --- a/runners/launch_b300-dsxe.sh +++ b/runners/launch_b300-dsxe.sh @@ -277,10 +277,13 @@ fi # Override the job name in the recipe with the runner name. sed -i "s/^name:.*/name: \"${RUNNER_NAME}\"/" "$CONFIG_PATH" -if [[ "${EVAL_ONLY:-false}" == "true" ]]; then - python3 "$GITHUB_WORKSPACE/runners/inject_synthetic_acceptance.py" \ - "$CONFIG_PATH" "$FRAMEWORK" || exit 1 -fi +# Call this unconditionally, as launch_gb300-nv.sh does. The helper returns +# immediately when neither EVAL_ONLY nor SYNTHETIC_ACCEPTANCE is set. Gating it +# on EVAL_ONLY meant throughput runs never got SYNTHETIC_ACCEPTANCE_LENGTH into +# speculative-config, so those arms drafted for real and their numbers were not +# comparable to the gb300-nv runs they exist to be compared against. +python3 "$GITHUB_WORKSPACE/runners/inject_synthetic_acceptance.py" \ + "$CONFIG_PATH" "$FRAMEWORK" || exit 1 # Weights live on node-local MODEL_ROOT, which this login host cannot stat, so # srtctl's preflight model.path check is always skipped. Runtime loading still From 2dbc1763bdac4c90a0c89bb0270970fdf9bca638 Mon Sep 17 00:00:00 2001 From: misunp Date: Thu, 3 Sep 2026 00:07:21 -0700 Subject: [PATCH 2/6] =?UTF-8?q?perf(agentx):=20record=20the=20PR=20link=20?= =?UTF-8?q?in=20perf-changelog=20/=20=E6=80=A7=E8=83=BD=EF=BC=9A=E5=9C=A8?= =?UTF-8?q?=20perf-changelog=20=E4=B8=AD=E8=AE=B0=E5=BD=95=20PR=20?= =?UTF-8?q?=E9=93=BE=E6=8E=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- perf-changelog.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 09559e8ce9..5b75f9c269 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6889,5 +6889,5 @@ - "Add the six measured Kimi-K3 B300 AgentX points: aggregated TP8 x DCP8 without speculative decoding at c70, aggregated DSpark-7 at c4 and c1, and 1P1D, 1P2D and 1P3D DCP8 x DCP8 DSpark-4 at c48, c48 and c32." - "Give Kimi-K3 a model path on b300-dsxe and route the agentic scenario to upstream srt-slurm with the in-repo recipes, without which multi-node Kimi-K3 exits as an unsupported model." - "Apply synthetic acceptance on throughput runs, not only eval-only ones, so the pinned acceptance length reaches speculative-config as it already does on gb300-nv." - - "Let the fabric select its own paths instead of pinning transfer devices, and check the expected adapter inventory before the engine starts." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/PENDING + - "Verify each node's RDMA rail subnets before the engine starts, because a down card shifts every device index above it and the Mooncake store config is written once for the whole job." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2814 From b3b7237c7343b065c68b945000f8a3febf32adf1 Mon Sep 17 00:00:00 2001 From: misunp Date: Thu, 3 Sep 2026 00:17:05 -0700 Subject: [PATCH 3/6] =?UTF-8?q?fix(agentx):=20report=20a=20b300-nv=20rail?= =?UTF-8?q?=20mismatch=20instead=20of=20failing=20the=20job=20/=20?= =?UTF-8?q?=E4=BF=AE=E5=A4=8D=EF=BC=9Ab300-nv=20rail=20=E4=B8=8D=E5=8C=B9?= =?UTF-8?q?=E9=85=8D=E6=97=B6=E6=94=B9=E4=B8=BA=E5=91=8A=E8=AD=A6=E8=80=8C?= =?UTF-8?q?=E9=9D=9E=E4=BD=BF=E4=BD=9C=E4=B8=9A=E5=A4=B1=E8=B4=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The pinned rail set was read off four nodes and the scheduler draws from more than four, so a mismatch means the node was never inventoried rather than that it is broken. Log it and continue. 固定的 rail 集合来自四个节点的实测，而调度器可选节点多于四个；不匹配 说明该节点未被清点，而非该节点有问题。记录日志并继续运行。 --- .../configs/kimik3-b300-nic-topology.sh | 25 +++++++++++-------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/configs/kimik3-b300-nic-topology.sh b/benchmarks/multi_node/srt-slurm-recipes/configs/kimik3-b300-nic-topology.sh index 4bfb9e3e14..07026e252f 100755 --- a/benchmarks/multi_node/srt-slurm-recipes/configs/kimik3-b300-nic-topology.sh +++ b/benchmarks/multi_node/srt-slurm-recipes/configs/kimik3-b300-nic-topology.sh @@ -11,8 +11,11 @@ # are up; what they do not show is which NIC belongs to which GPU, which is # what this prints. # -# The topology dump is diagnostic and best-effort. The rail check below is -# not: it exits non-zero on a mismatch, which fails the job by design. +# Everything here is diagnostic and best-effort; nothing fails the job. The +# rail check below reports a mismatch rather than refusing to run, because +# the pinned set was read off four nodes and the scheduler draws from more +# than four: a mismatch means this node was never inventoried, which is not +# the same as this node being broken. set -uo pipefail @@ -53,9 +56,9 @@ echo "=== [b300-topo] end ===" # way, 36 minutes in. # # The recipe therefore names only rails whose subnet was identical on every -# node inventoried so far. Assert that here rather than trust it: a node that -# disagrees fails in seconds and names itself, instead of burning a benchmark -# window. +# node inventoried so far -- b300-003, -011, -012 and -019. Check that here +# rather than trust it, so that if the QP setup does fail later the log +# already says which node and which rail disagreed. EXPECTED_RAILS="mlx5_4:172.16.128 mlx5_8:172.16.64 mlx5_10:172.17.192 \ mlx5_16:172.17.0 mlx5_20:172.17.128 mlx5_22:172.17.64" @@ -89,12 +92,14 @@ for _pair in $EXPECTED_RAILS; do done if [[ "$_rail_bad" -ne 0 ]]; then - echo "[b300-rails] $(hostname) does not match the pinned rail layout." >&2 - echo "[b300-rails] Add it to sbatch_directives.exclude, or re-pin the rails" >&2 - echo "[b300-rails] in the b300 Kimi-K3 agentic recipes." >&2 - exit 1 + echo "[b300-rails] WARNING: $(hostname) does not match the pinned rail layout." >&2 + echo "[b300-rails] This node was not in the inventory the rails were read from." >&2 + echo "[b300-rails] If the Mooncake handshake fails on this job, that is the" >&2 + echo "[b300-rails] first thing to look at: either exclude the node or re-pin" >&2 + echo "[b300-rails] device_name in the b300 Kimi-K3 agentic recipes." >&2 +else + echo "=== [b300-rails] all pinned rails match ===" fi -echo "=== [b300-rails] all pinned rails match ===" # Nothing to patch: the image carries the Kimi-K3 stack already. Print what it # is, so the log still records which build served the numbers. From 6bc49d90e28c5c1dce76b2ed0c1c0723ed1cac0c Mon Sep 17 00:00:00 2001 From: misunp Date: Thu, 3 Sep 2026 09:47:27 -0700 Subject: [PATCH 4/6] =?UTF-8?q?fix(agentx):=20stage-aware=20Kimi-K3=20mode?= =?UTF-8?q?l=20probe=20and=20a=206h=20walltime=20on=20b300-nv=20/=20?= =?UTF-8?q?=E4=BF=AE=E5=A4=8D=EF=BC=9Ab300-nv=20=E4=B8=8A=E6=8C=89?= =?UTF-8?q?=E5=AE=9E=E9=99=85=E6=9A=82=E5=AD=98=E4=BD=8D=E7=BD=AE=E6=8E=A2?= =?UTF-8?q?=E6=B5=8B=20Kimi-K3=20=E5=B9=B6=E5=B0=86walltime=20=E6=8F=90?= =?UTF-8?q?=E5=8D=87=E8=87=B3=206=20=E5=B0=8F=E6=97=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The single-node Kimi-K3 arm on this cluster serves /scratch/models/Kimi-K3, so probe that first and require a non-empty directory, and fail with the list of probed paths instead of handing srtctl one that does not exist. Raise the walltime from the GB300 recipes' 04:00:00 to 06:00:00: every node pulls the ~1.5 TB checkpoint from the same shared /scratch and 1P3D has four doing it at once. batch_1 accepts at least 480 minutes. 本集群的单节点 Kimi-K3 配置使用 /scratch/models/Kimi-K3，因此优先探测该路径 并要求目录非空；未找到时直接失败并列出已探测路径。walltime 由 GB300 recipe 的 04:00:00 提升至 06:00:00：每个节点都从同一共享 /scratch 拉取约 1.5 TB 检查点，1P3D 更是四个节点同时进行。batch_1 至少允许 480 分钟。 --- .../agg-b300-dcp8-dspark7-maxseq2-mooncake-c1-agentic.yaml | 4 ++-- .../agg-b300-dcp8-dspark7-maxseq2-mooncake-c4-agentic.yaml | 4 ++-- .../agentic/agg-b300-dcp8-nospec-mooncake-c70-agentic.yaml | 4 ++-- ...sagg-b300-1p1d-dcp8-dcp8-dspark4-mooncake-c48-agentic.yaml | 4 ++-- ...sagg-b300-1p2d-dcp8-dcp8-dspark4-mooncake-c48-agentic.yaml | 4 ++-- ...sagg-b300-1p3d-dcp8-dcp8-dspark4-mooncake-c32-agentic.yaml | 4 ++-- 6 files changed, 12 insertions(+), 12 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b300-dcp8-dspark7-maxseq2-mooncake-c1-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b300-dcp8-dspark7-maxseq2-mooncake-c1-agentic.yaml index 1c05129d33..a9bc41ad44 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b300-dcp8-dspark7-maxseq2-mooncake-c1-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b300-dcp8-dspark7-maxseq2-mooncake-c1-agentic.yaml @@ -11,7 +11,7 @@ # device_name, UCX_NET_DEVICES -> the six rails read off this cluster # MC_GID_INDEX, UCX_TLS -> set; b300-nv is RoCE (see below) # HF_HOME -> dropped; benchmark.env sets HF_HUB_CACHE -# slurm.time_limit -> 04:00:00, this partition's limit +# slurm.time_limit -> 06:00:00 (see the note in the tool) # rejection_sample_method -> block, so the eval verifies (see below) # global_segment_size -> 281GB, matching the B300 arm on main # sbatch_directives.exclude -> b300-018, which times out UCX/NIXL @@ -146,7 +146,7 @@ identity: frameworks: dynamo: ba83080ecd31c1ce918559e576d3c5bc9e092ff1 slurm: - time_limit: 04:00:00 + time_limit: 06:00:00 sbatch_directives: mem: '0' cpus-per-task: '72' diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b300-dcp8-dspark7-maxseq2-mooncake-c4-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b300-dcp8-dspark7-maxseq2-mooncake-c4-agentic.yaml index e4eacd947e..6b132fb860 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b300-dcp8-dspark7-maxseq2-mooncake-c4-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b300-dcp8-dspark7-maxseq2-mooncake-c4-agentic.yaml @@ -11,7 +11,7 @@ # device_name, UCX_NET_DEVICES -> the six rails read off this cluster # MC_GID_INDEX, UCX_TLS -> set; b300-nv is RoCE (see below) # HF_HOME -> dropped; benchmark.env sets HF_HUB_CACHE -# slurm.time_limit -> 04:00:00, this partition's limit +# slurm.time_limit -> 06:00:00 (see the note in the tool) # rejection_sample_method -> block, so the eval verifies (see below) # global_segment_size -> 281GB, matching the B300 arm on main # sbatch_directives.exclude -> b300-018, which times out UCX/NIXL @@ -146,7 +146,7 @@ identity: frameworks: dynamo: ba83080ecd31c1ce918559e576d3c5bc9e092ff1 slurm: - time_limit: 04:00:00 + time_limit: 06:00:00 sbatch_directives: mem: '0' cpus-per-task: '72' diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b300-dcp8-nospec-mooncake-c70-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b300-dcp8-nospec-mooncake-c70-agentic.yaml index 19074b358f..d5f7206b25 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b300-dcp8-nospec-mooncake-c70-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b300-dcp8-nospec-mooncake-c70-agentic.yaml @@ -11,7 +11,7 @@ # device_name, UCX_NET_DEVICES -> the six rails read off this cluster # MC_GID_INDEX, UCX_TLS -> set; b300-nv is RoCE (see below) # HF_HOME -> dropped; benchmark.env sets HF_HUB_CACHE -# slurm.time_limit -> 04:00:00, this partition's limit +# slurm.time_limit -> 06:00:00 (see the note in the tool) # rejection_sample_method -> block, so the eval verifies (see below) # global_segment_size -> 281GB, matching the B300 arm on main # sbatch_directives.exclude -> b300-018, which times out UCX/NIXL @@ -143,7 +143,7 @@ identity: frameworks: dynamo: ba83080ecd31c1ce918559e576d3c5bc9e092ff1 slurm: - time_limit: 04:00:00 + time_limit: 06:00:00 sbatch_directives: mem: '0' cpus-per-task: '72' diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/disagg-b300-1p1d-dcp8-dcp8-dspark4-mooncake-c48-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/disagg-b300-1p1d-dcp8-dcp8-dspark4-mooncake-c48-agentic.yaml index 203a4409eb..959539ec67 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/disagg-b300-1p1d-dcp8-dcp8-dspark4-mooncake-c48-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/disagg-b300-1p1d-dcp8-dcp8-dspark4-mooncake-c48-agentic.yaml @@ -11,7 +11,7 @@ # device_name, UCX_NET_DEVICES -> the six rails read off this cluster # MC_GID_INDEX, UCX_TLS -> set; b300-nv is RoCE (see below) # HF_HOME -> dropped; benchmark.env sets HF_HUB_CACHE -# slurm.time_limit -> 04:00:00, this partition's limit +# slurm.time_limit -> 06:00:00 (see the note in the tool) # rejection_sample_method -> block, so the eval verifies (see below) # global_segment_size -> 281GB, matching the B300 arm on main # sbatch_directives.exclude -> b300-018, which times out UCX/NIXL @@ -37,7 +37,7 @@ dynamo: hash: ba83080ecd31c1ce918559e576d3c5bc9e092ff1 install: true slurm: - time_limit: 04:00:00 + time_limit: 06:00:00 health_check: max_attempts: 720 interval_seconds: 10 diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/disagg-b300-1p2d-dcp8-dcp8-dspark4-mooncake-c48-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/disagg-b300-1p2d-dcp8-dcp8-dspark4-mooncake-c48-agentic.yaml index f08da3187f..bc9eb5a601 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/disagg-b300-1p2d-dcp8-dcp8-dspark4-mooncake-c48-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/disagg-b300-1p2d-dcp8-dcp8-dspark4-mooncake-c48-agentic.yaml @@ -11,7 +11,7 @@ # device_name, UCX_NET_DEVICES -> the six rails read off this cluster # MC_GID_INDEX, UCX_TLS -> set; b300-nv is RoCE (see below) # HF_HOME -> dropped; benchmark.env sets HF_HUB_CACHE -# slurm.time_limit -> 04:00:00, this partition's limit +# slurm.time_limit -> 06:00:00 (see the note in the tool) # rejection_sample_method -> block, so the eval verifies (see below) # global_segment_size -> 281GB, matching the B300 arm on main # sbatch_directives.exclude -> b300-018, which times out UCX/NIXL @@ -37,7 +37,7 @@ dynamo: hash: ba83080ecd31c1ce918559e576d3c5bc9e092ff1 install: true slurm: - time_limit: 04:00:00 + time_limit: 06:00:00 health_check: max_attempts: 720 interval_seconds: 10 diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/disagg-b300-1p3d-dcp8-dcp8-dspark4-mooncake-c32-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/disagg-b300-1p3d-dcp8-dcp8-dspark4-mooncake-c32-agentic.yaml index d8ce175f84..ae67bcdea2 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/disagg-b300-1p3d-dcp8-dcp8-dspark4-mooncake-c32-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/disagg-b300-1p3d-dcp8-dcp8-dspark4-mooncake-c32-agentic.yaml @@ -11,7 +11,7 @@ # device_name, UCX_NET_DEVICES -> the six rails read off this cluster # MC_GID_INDEX, UCX_TLS -> set; b300-nv is RoCE (see below) # HF_HOME -> dropped; benchmark.env sets HF_HUB_CACHE -# slurm.time_limit -> 04:00:00, this partition's limit +# slurm.time_limit -> 06:00:00 (see the note in the tool) # rejection_sample_method -> block, so the eval verifies (see below) # global_segment_size -> 281GB, matching the B300 arm on main # sbatch_directives.exclude -> b300-018, which times out UCX/NIXL @@ -37,7 +37,7 @@ dynamo: hash: ba83080ecd31c1ce918559e576d3c5bc9e092ff1 install: true slurm: - time_limit: 04:00:00 + time_limit: 06:00:00 health_check: max_attempts: 720 interval_seconds: 10 From f199f79b09eadba3ad4d6cbf59861df58cb9bf7f Mon Sep 17 00:00:00 2001 From: misunp Date: Thu, 3 Sep 2026 21:59:42 -0700 Subject: [PATCH 5/6] =?UTF-8?q?fix(agentx):=20name=20the=20DSpark=20draft?= =?UTF-8?q?=20sampler=20on=20the=20B300=20arms=20/=20=E4=BF=AE=E5=A4=8D?= =?UTF-8?q?=EF=BC=9A=E5=9C=A8=20B300=20=E9=85=8D=E7=BD=AE=E4=B8=AD?= =?UTF-8?q?=E6=98=BE=E5=BC=8F=E6=8C=87=E5=AE=9A=20DSpark=20draft=20?= =?UTF-8?q?=E9=87=87=E6=A0=B7=E6=96=B9=E5=BC=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The acceptance lengths these arms pin -- 3.36 at K=4, 3.84 at K=7 -- come from the probabilistic-draft, block-verify golden curve. The other Kimi-K3 DSpark curve reads 3.22 and 3.78 at the same levels, so leaving draft_sample_method to the engine default pairs an AL with the wrong curve. Name it, as every other Kimi-K3 DSpark recipe on main does. 这些配置固定的接受长度（K=4 为 3.36，K=7 为 3.84）来自 probabilistic draft + block verify 的黄金曲线；另一条 Kimi-K3 DSpark 曲线在相同档位为 3.22 与 3.78。 若不指定 draft_sample_method，接受长度将与错误的曲线配对。 --- .../agg-b300-dcp8-dspark7-maxseq2-mooncake-c1-agentic.yaml | 3 ++- .../agg-b300-dcp8-dspark7-maxseq2-mooncake-c4-agentic.yaml | 3 ++- .../agentic/agg-b300-dcp8-nospec-mooncake-c70-agentic.yaml | 1 + ...agg-b300-1p1d-dcp8-dcp8-dspark4-mooncake-c48-agentic.yaml | 5 +++-- ...agg-b300-1p2d-dcp8-dcp8-dspark4-mooncake-c48-agentic.yaml | 5 +++-- ...agg-b300-1p3d-dcp8-dcp8-dspark4-mooncake-c32-agentic.yaml | 5 +++-- 6 files changed, 14 insertions(+), 8 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b300-dcp8-dspark7-maxseq2-mooncake-c1-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b300-dcp8-dspark7-maxseq2-mooncake-c1-agentic.yaml index a9bc41ad44..a0e60e46a8 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b300-dcp8-dspark7-maxseq2-mooncake-c1-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b300-dcp8-dspark7-maxseq2-mooncake-c1-agentic.yaml @@ -13,6 +13,7 @@ # HF_HOME -> dropped; benchmark.env sets HF_HUB_CACHE # slurm.time_limit -> 06:00:00 (see the note in the tool) # rejection_sample_method -> block, so the eval verifies (see below) +# draft_sample_method -> probabilistic, matching the AL's curve # global_segment_size -> 281GB, matching the B300 arm on main # sbatch_directives.exclude -> b300-018, which times out UCX/NIXL # @@ -113,7 +114,7 @@ backend: enable-cumem-allocator: true attention-backend: TOKENSPEED_MLA attention-config: '{"mla_prefill_backend":"TRTLLM_RAGGED","use_prefill_query_quantization":true}' - speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","attention_backend":"TOKENSPEED_MLA","method":"dspark","num_speculative_tokens":7,"rejection_sample_method":"block"}' + speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","attention_backend":"TOKENSPEED_MLA","method":"dspark","num_speculative_tokens":7,"rejection_sample_method":"block","draft_sample_method":"probabilistic"}' enable-prefix-caching: true prefix-match-unit: 128 health_check: diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b300-dcp8-dspark7-maxseq2-mooncake-c4-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b300-dcp8-dspark7-maxseq2-mooncake-c4-agentic.yaml index 6b132fb860..0df65fd809 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b300-dcp8-dspark7-maxseq2-mooncake-c4-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b300-dcp8-dspark7-maxseq2-mooncake-c4-agentic.yaml @@ -13,6 +13,7 @@ # HF_HOME -> dropped; benchmark.env sets HF_HUB_CACHE # slurm.time_limit -> 06:00:00 (see the note in the tool) # rejection_sample_method -> block, so the eval verifies (see below) +# draft_sample_method -> probabilistic, matching the AL's curve # global_segment_size -> 281GB, matching the B300 arm on main # sbatch_directives.exclude -> b300-018, which times out UCX/NIXL # @@ -113,7 +114,7 @@ backend: enable-cumem-allocator: true attention-backend: TOKENSPEED_MLA attention-config: '{"mla_prefill_backend":"TRTLLM_RAGGED","use_prefill_query_quantization":true}' - speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","attention_backend":"TOKENSPEED_MLA","method":"dspark","num_speculative_tokens":7,"rejection_sample_method":"block"}' + speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","attention_backend":"TOKENSPEED_MLA","method":"dspark","num_speculative_tokens":7,"rejection_sample_method":"block","draft_sample_method":"probabilistic"}' enable-prefix-caching: true prefix-match-unit: 128 health_check: diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b300-dcp8-nospec-mooncake-c70-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b300-dcp8-nospec-mooncake-c70-agentic.yaml index d5f7206b25..ee8018dc9f 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b300-dcp8-nospec-mooncake-c70-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b300-dcp8-nospec-mooncake-c70-agentic.yaml @@ -13,6 +13,7 @@ # HF_HOME -> dropped; benchmark.env sets HF_HUB_CACHE # slurm.time_limit -> 06:00:00 (see the note in the tool) # rejection_sample_method -> block, so the eval verifies (see below) +# draft_sample_method -> probabilistic, matching the AL's curve # global_segment_size -> 281GB, matching the B300 arm on main # sbatch_directives.exclude -> b300-018, which times out UCX/NIXL # diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/disagg-b300-1p1d-dcp8-dcp8-dspark4-mooncake-c48-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/disagg-b300-1p1d-dcp8-dcp8-dspark4-mooncake-c48-agentic.yaml index 959539ec67..b4888580bb 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/disagg-b300-1p1d-dcp8-dcp8-dspark4-mooncake-c48-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/disagg-b300-1p1d-dcp8-dcp8-dspark4-mooncake-c48-agentic.yaml @@ -13,6 +13,7 @@ # HF_HOME -> dropped; benchmark.env sets HF_HUB_CACHE # slurm.time_limit -> 06:00:00 (see the note in the tool) # rejection_sample_method -> block, so the eval verifies (see below) +# draft_sample_method -> probabilistic, matching the AL's curve # global_segment_size -> 281GB, matching the B300 arm on main # sbatch_directives.exclude -> b300-018, which times out UCX/NIXL # @@ -188,7 +189,7 @@ backend: language-model-only: true attention-backend: TOKENSPEED_MLA attention-config: '{"mla_prefill_backend": "TRTLLM_RAGGED", "use_prefill_query_quantization": true}' - speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","attention_backend":"TOKENSPEED_MLA","method":"dspark","num_speculative_tokens":4,"rejection_sample_method":"block"}' + speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","attention_backend":"TOKENSPEED_MLA","method":"dspark","num_speculative_tokens":4,"rejection_sample_method":"block","draft_sample_method":"probabilistic"}' enable-prefix-caching: true decode: no-enable-flashinfer-autotune: true @@ -209,7 +210,7 @@ backend: language-model-only: true attention-backend: TOKENSPEED_MLA attention-config: '{"mla_prefill_backend": "TRTLLM_RAGGED", "use_prefill_query_quantization": true}' - speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","attention_backend":"TOKENSPEED_MLA","method":"dspark","num_speculative_tokens":4,"rejection_sample_method":"block"}' + speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","attention_backend":"TOKENSPEED_MLA","method":"dspark","num_speculative_tokens":4,"rejection_sample_method":"block","draft_sample_method":"probabilistic"}' enable-prefix-caching: true sbatch_directives: cpus-per-task: '72' diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/disagg-b300-1p2d-dcp8-dcp8-dspark4-mooncake-c48-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/disagg-b300-1p2d-dcp8-dcp8-dspark4-mooncake-c48-agentic.yaml index bc9eb5a601..a72e7fecc9 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/disagg-b300-1p2d-dcp8-dcp8-dspark4-mooncake-c48-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/disagg-b300-1p2d-dcp8-dcp8-dspark4-mooncake-c48-agentic.yaml @@ -13,6 +13,7 @@ # HF_HOME -> dropped; benchmark.env sets HF_HUB_CACHE # slurm.time_limit -> 06:00:00 (see the note in the tool) # rejection_sample_method -> block, so the eval verifies (see below) +# draft_sample_method -> probabilistic, matching the AL's curve # global_segment_size -> 281GB, matching the B300 arm on main # sbatch_directives.exclude -> b300-018, which times out UCX/NIXL # @@ -187,7 +188,7 @@ backend: language-model-only: true attention-backend: TOKENSPEED_MLA attention-config: '{"mla_prefill_backend": "TRTLLM_RAGGED", "use_prefill_query_quantization": true}' - speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","attention_backend":"TOKENSPEED_MLA","method":"dspark","num_speculative_tokens":4,"rejection_sample_method":"block"}' + speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","attention_backend":"TOKENSPEED_MLA","method":"dspark","num_speculative_tokens":4,"rejection_sample_method":"block","draft_sample_method":"probabilistic"}' enable-prefix-caching: true decode: kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"compact_group_io":true,"max_load_batch_keys":2,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' @@ -206,7 +207,7 @@ backend: language-model-only: true attention-backend: TOKENSPEED_MLA attention-config: '{"mla_prefill_backend": "TRTLLM_RAGGED", "use_prefill_query_quantization": true}' - speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","attention_backend":"TOKENSPEED_MLA","method":"dspark","num_speculative_tokens":4,"rejection_sample_method":"block"}' + speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","attention_backend":"TOKENSPEED_MLA","method":"dspark","num_speculative_tokens":4,"rejection_sample_method":"block","draft_sample_method":"probabilistic"}' enable-prefix-caching: true sbatch_directives: cpus-per-task: '72' diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/disagg-b300-1p3d-dcp8-dcp8-dspark4-mooncake-c32-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/disagg-b300-1p3d-dcp8-dcp8-dspark4-mooncake-c32-agentic.yaml index ae67bcdea2..a3cc02b379 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/disagg-b300-1p3d-dcp8-dcp8-dspark4-mooncake-c32-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/disagg-b300-1p3d-dcp8-dcp8-dspark4-mooncake-c32-agentic.yaml @@ -13,6 +13,7 @@ # HF_HOME -> dropped; benchmark.env sets HF_HUB_CACHE # slurm.time_limit -> 06:00:00 (see the note in the tool) # rejection_sample_method -> block, so the eval verifies (see below) +# draft_sample_method -> probabilistic, matching the AL's curve # global_segment_size -> 281GB, matching the B300 arm on main # sbatch_directives.exclude -> b300-018, which times out UCX/NIXL # @@ -187,7 +188,7 @@ backend: language-model-only: true attention-backend: TOKENSPEED_MLA attention-config: '{"mla_prefill_backend": "TRTLLM_RAGGED", "use_prefill_query_quantization": true}' - speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","attention_backend":"TOKENSPEED_MLA","method":"dspark","num_speculative_tokens":4,"rejection_sample_method":"block"}' + speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","attention_backend":"TOKENSPEED_MLA","method":"dspark","num_speculative_tokens":4,"rejection_sample_method":"block","draft_sample_method":"probabilistic"}' enable-prefix-caching: true decode: kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"compact_group_io":true,"max_load_batch_keys":2,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' @@ -206,7 +207,7 @@ backend: language-model-only: true attention-backend: TOKENSPEED_MLA attention-config: '{"mla_prefill_backend": "TRTLLM_RAGGED", "use_prefill_query_quantization": true}' - speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","attention_backend":"TOKENSPEED_MLA","method":"dspark","num_speculative_tokens":4,"rejection_sample_method":"block"}' + speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","attention_backend":"TOKENSPEED_MLA","method":"dspark","num_speculative_tokens":4,"rejection_sample_method":"block","draft_sample_method":"probabilistic"}' enable-prefix-caching: true sbatch_directives: cpus-per-task: '72' From ed2be39a7e84b67be54375adb9f998c6d73bcf75 Mon Sep 17 00:00:00 2001 From: Ankur-singh Date: Fri, 4 Sep 2026 14:37:58 -0700 Subject: [PATCH 6/6] fix(agentx): retarget the Kimi-K3 B300 AgentX arms at cluster:b300-dsxe MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The B300 NV fleet is retired and its launcher is gone, so these arms move to cluster:b300-dsxe. That is not a relabel: the two clusters do not share a fabric. b300-nv was RoCE over Mellanox, where each rail was an mlx5 device on its own /24 and a QP only came up when both ends sat on the same one. Device indices were not stable across nodes, so the recipes named six rails explicitly and a startup script re-checked each one's subnet. b300-dsxe's compute fabric is EFA: 16 adapters per node, 400 Gb each, two per GPU, presenting no netdev and no IPv4. None of the pinned mlx5 names exist there, and there are no per-rail subnets to verify. The inventory is identical on every node checked, so there is also no unstable index to pin around. The recipes therefore drop device_name, UCX_NET_DEVICES and the RoCE GID index, and ask UCX for srd, EFA's transport, because EFA does not implement the verbs RC that rc requires. The startup script is replaced by an adapter-inventory and GPU-affinity check, since a node short of adapters runs slower than its peers rather than failing. Also drops an sbatch exclude naming a node of the retired fleet, and carries the launcher's unconditional synthetic-acceptance call into the DSXE launcher, so throughput runs get the pinned acceptance length into speculative-config as they already do on gb300-nv. B300 NV 集群已下线，其启动脚本也已移除，因此这些配置迁移到 cluster:b300-dsxe。 这并非简单改名：两个集群的网络结构完全不同。 b300-nv 使用基于 Mellanox 的 RoCE，每条 rail 是一个独立 /24 网段上的 mlx5 设备， 只有两端位于同一网段时 QP 才能建立。设备索引在各节点间并不稳定，因此配方显式指定了 六条 rail，并由启动脚本在运行前逐一校验其网段。 b300-dsxe 的计算网络是 EFA：每节点 16 个适配器，各 400 Gb，每 GPU 两个，且不提供 netdev 和 IPv4。此前固定的 mlx5 设备名在该集群并不存在，也没有可校验的 rail 网段。 所检查的各节点适配器清单完全一致，因此也不存在需要规避的索引漂移问题。配方据此移除 device_name、UCX_NET_DEVICES 和 RoCE GID index，并将 UCX_TLS 设为 srd（EFA 的传输 方式），因为 EFA 并不实现 rc 所需的 verbs RC。启动脚本改为检查适配器数量与 GPU 亲和 关系：适配器缺失的节点只会比同伴更慢，而不会直接失败。 同时移除了指向已下线集群节点的 sbatch exclude，并将启动脚本中无条件调用合成接受率 注入的改动带入 DSXE 启动脚本，使吞吐运行也能像 gb300-nv 那样把固定的接受长度写入 speculative-config。 --- .../configs/kimik3-b300-fabric-check.sh | 61 +++++++++ .../configs/kimik3-b300-nic-topology.sh | 117 ------------------ ...8-dspark7-maxseq2-mooncake-c1-agentic.yaml | 30 ++--- ...8-dspark7-maxseq2-mooncake-c4-agentic.yaml | 30 ++--- ...b300-dcp8-nospec-mooncake-c70-agentic.yaml | 30 ++--- ...cp8-dcp8-dspark4-mooncake-c48-agentic.yaml | 33 ++--- ...cp8-dcp8-dspark4-mooncake-c48-agentic.yaml | 33 ++--- ...cp8-dcp8-dspark4-mooncake-c32-agentic.yaml | 33 ++--- configs/nvidia-master.yaml | 6 +- perf-changelog.yaml | 2 +- 10 files changed, 134 insertions(+), 241 deletions(-) create mode 100755 benchmarks/multi_node/srt-slurm-recipes/configs/kimik3-b300-fabric-check.sh delete mode 100755 benchmarks/multi_node/srt-slurm-recipes/configs/kimik3-b300-nic-topology.sh diff --git a/benchmarks/multi_node/srt-slurm-recipes/configs/kimik3-b300-fabric-check.sh b/benchmarks/multi_node/srt-slurm-recipes/configs/kimik3-b300-fabric-check.sh new file mode 100755 index 0000000000..94026b499b --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/configs/kimik3-b300-fabric-check.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash +# Report this compute node's EFA inventory and GPU affinity before the engine starts. +# +# b300-dsxe's compute fabric is EFA, not RoCE: 16 adapters (driver efa, 400 Gb +# each, two per GPU) that present no netdev and no IPv4. There are no per-rail +# subnets, so there is nothing to pin and nothing to compare subnets against -- +# the recipes leave device selection to the fabric. +# +# What is worth checking is that the node has its full complement. A node +# missing adapters still runs, just slower and asymmetrically against its peers, +# which is the kind of thing that shows up as an unexplained outlier rather than +# an error. Everything here is diagnostic; nothing fails the job. + +set -uo pipefail + +EXPECTED_EFA=16 + +echo "=== [b300-fabric] node: $(hostname) ===" + +mapfile -t EFA_DEVS < <( + for d in /sys/class/infiniband/*; do + [ -e "$d" ] || continue + [ "$(basename "$(readlink -f "$d/device/driver")" 2>/dev/null)" = efa ] || continue + basename "$d" + done | sort +) + +echo "--- EFA adapters: ${#EFA_DEVS[@]} (expected $EXPECTED_EFA) ---" +for dev in "${EFA_DEVS[@]}"; do + state=$(awk '{print $2}' "/sys/class/infiniband/$dev/ports/1/state" 2>/dev/null) + rate=$(awk '{print $1}' "/sys/class/infiniband/$dev/ports/1/rate" 2>/dev/null) + pci=$(basename "$(readlink -f "/sys/class/infiniband/$dev/device")" 2>/dev/null) + printf ' %-12s %-10s %sGb %s\n' "$dev" "${state:-?}" "${rate:-?}" "${pci:-?}" +done + +if [ "${#EFA_DEVS[@]}" -ne "$EXPECTED_EFA" ]; then + echo "[b300-fabric] WARNING: ${#EFA_DEVS[@]} EFA adapters, expected $EXPECTED_EFA." + echo "[b300-fabric] This node will move KV more slowly than its peers." +fi + +down=0 +for dev in "${EFA_DEVS[@]}"; do + state=$(awk '{print $2}' "/sys/class/infiniband/$dev/ports/1/state" 2>/dev/null) + [ "$state" = ACTIVE ] || { echo "[b300-fabric] WARNING: $dev is $state, not ACTIVE."; down=$((down + 1)); } +done +[ "$down" -eq 0 ] && echo "[b300-fabric] all ${#EFA_DEVS[@]} adapters ACTIVE." + +echo "--- GPU / EFA affinity (by PCI bus) ---" +nvidia-smi --query-gpu=index,pci.bus_id --format=csv,noheader 2>/dev/null | while IFS=, read -r idx bus; do + gbus=$(echo "$bus" | tr -d ' ' | cut -d: -f2 | tr 'A-Z' 'a-z') + near="" + for dev in "${EFA_DEVS[@]}"; do + pci=$(basename "$(readlink -f "/sys/class/infiniband/$dev/device")" 2>/dev/null) + nbus=$(echo "$pci" | cut -d: -f2) + d=$(( 16#$gbus - 16#$nbus )); [ "$d" -lt 0 ] && d=$(( -d )) + [ "$d" -le 2 ] && near="$near $dev" + done + printf ' GPU%s (bus %s):%s\n' "$idx" "$gbus" "${near:- none}" +done + +echo "=== [b300-fabric] done ===" diff --git a/benchmarks/multi_node/srt-slurm-recipes/configs/kimik3-b300-nic-topology.sh b/benchmarks/multi_node/srt-slurm-recipes/configs/kimik3-b300-nic-topology.sh deleted file mode 100755 index 07026e252f..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/configs/kimik3-b300-nic-topology.sh +++ /dev/null @@ -1,117 +0,0 @@ -#!/usr/bin/env bash -# Print this compute node's GPU-to-NIC topology, then verify the rails the -# Mooncake store is pinned to. -# -# Mooncake on b300-nv needs its peers on the same rail: the nodes are -# rail-isolated, and leaving device_name empty let the prefill node pick -# mlx5_1/mlx5_2 that the decode node did not have, which ended in "Active -# handshake RPC failed" and a segfault during bring-up. Naming rails instead -# of leaving them to auto-discovery means the set has to come from the real -# affinity map rather than a guess. The worker logs already list which devices -# are up; what they do not show is which NIC belongs to which GPU, which is -# what this prints. -# -# Everything here is diagnostic and best-effort; nothing fails the job. The -# rail check below reports a mismatch rather than refusing to run, because -# the pinned set was read off four nodes and the scheduler draws from more -# than four: a mismatch means this node was never inventoried, which is not -# the same as this node being broken. - -set -uo pipefail - -echo "=== [b300-topo] node: $(hostname) ===" - -echo "--- nvidia-smi topo -m ---" -nvidia-smi topo -m 2>&1 | head -40 || true - -echo "--- ibdev2netdev ---" -ibdev2netdev 2>&1 | head -40 || true - -echo "--- ibv_devinfo -l ---" -ibv_devinfo -l 2>&1 | head -40 || true - -echo "--- infiniband port state / rate ---" -for _d in /sys/class/infiniband/*; do - [[ -e "$_d" ]] || continue - _n="$(basename "$_d")" - printf '%-10s state=%-14s rate=%s\n' \ - "$_n" \ - "$(cat "$_d/ports/1/state" 2>/dev/null || echo '?')" \ - "$(cat "$_d/ports/1/rate" 2>/dev/null || echo '?')" -done - -echo "=== [b300-topo] end ===" - -# --------------------------------------------------------------------------- -# Verify the rails the Mooncake store is pinned to. -# -# srtslurm writes one store config for the whole job, so RDMA devices can only -# be named by index -- and an index does not mean the same physical rail on -# every node here. b300-012 puts mlx5_0 on 172.16.0 where every other node puts -# it on 172.16.192, and b300-003 swaps mlx5_3 with mlx5_9. When two nodes -# disagree, the cross-node QP cannot come up: "Failed to modify QP to RTR ... -# Connection timed out", then Mooncake retries via another peer RNIC and the -# transfer hits "local access violation work queue error" against a memory -# region registered for a different device. One earlier bring-up died that -# way, 36 minutes in. -# -# The recipe therefore names only rails whose subnet was identical on every -# node inventoried so far -- b300-003, -011, -012 and -019. Check that here -# rather than trust it, so that if the QP setup does fail later the log -# already says which node and which rail disagreed. -EXPECTED_RAILS="mlx5_4:172.16.128 mlx5_8:172.16.64 mlx5_10:172.17.192 \ -mlx5_16:172.17.0 mlx5_20:172.17.128 mlx5_22:172.17.64" - -echo "=== [b300-rails] verifying pinned rails on $(hostname) ===" -_rail_bad=0 -for _pair in $EXPECTED_RAILS; do - _dev="${_pair%%:*}" - _want="${_pair##*:}" - _port="/sys/class/infiniband/${_dev}/ports/1" - - if [[ ! -d "$_port" ]]; then - echo "[b300-rails] FAIL ${_dev}: device absent"; _rail_bad=1; continue - fi - _state="$(cat "${_port}/state" 2> /dev/null || echo '?')" - if [[ "$_state" != *ACTIVE* ]]; then - echo "[b300-rails] FAIL ${_dev}: port state '${_state}'"; _rail_bad=1; continue - fi - # GID index 3 is the RoCEv2 IPv4 entry; its last four bytes are the address. - _gid="$(cat "${_port}/gids/3" 2> /dev/null || echo '')" - _hex="$(printf '%s' "$_gid" | tr -d ':' | tail -c 8)" - if [[ ${#_hex} -ne 8 ]]; then - echo "[b300-rails] FAIL ${_dev}: cannot read GID index 3 (got '${_gid}')"; _rail_bad=1; continue - fi - _got="$((16#${_hex:0:2})).$((16#${_hex:2:2})).$((16#${_hex:4:2}))" - if [[ "$_got" != "$_want" ]]; then - echo "[b300-rails] FAIL ${_dev}: subnet ${_got}.x, expected ${_want}.x" - _rail_bad=1 - else - echo "[b300-rails] ok ${_dev}: ${_got}.$((16#${_hex:6:2}))" - fi -done - -if [[ "$_rail_bad" -ne 0 ]]; then - echo "[b300-rails] WARNING: $(hostname) does not match the pinned rail layout." >&2 - echo "[b300-rails] This node was not in the inventory the rails were read from." >&2 - echo "[b300-rails] If the Mooncake handshake fails on this job, that is the" >&2 - echo "[b300-rails] first thing to look at: either exclude the node or re-pin" >&2 - echo "[b300-rails] device_name in the b300 Kimi-K3 agentic recipes." >&2 -else - echo "=== [b300-rails] all pinned rails match ===" -fi - -# Nothing to patch: the image carries the Kimi-K3 stack already. Print what it -# is, so the log still records which build served the numbers. -python3 - <<'PY' 2>/dev/null || true -import importlib.util, os -spec = importlib.util.find_spec("vllm") -if spec and spec.origin: - ver = os.path.join(os.path.dirname(spec.origin), "_version.py") - try: - for line in open(ver): - if "version" in line and "=" in line: - print(f"[b300-rails] vllm {line.strip()}") - except OSError: - pass -PY diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b300-dcp8-dspark7-maxseq2-mooncake-c1-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b300-dcp8-dspark7-maxseq2-mooncake-c1-agentic.yaml index a0e60e46a8..766de2864a 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b300-dcp8-dspark7-maxseq2-mooncake-c1-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b300-dcp8-dspark7-maxseq2-mooncake-c1-agentic.yaml @@ -8,26 +8,22 @@ # # Only these fields differ from the measured config, each because b300-nv is # not bia: -# device_name, UCX_NET_DEVICES -> the six rails read off this cluster -# MC_GID_INDEX, UCX_TLS -> set; b300-nv is RoCE (see below) +# device_name, UCX_NET_DEVICES -> dropped; see the fabric note below +# MC_GID_INDEX, UCX_TLS -> GID index dropped, UCX_TLS uses srd (below) # HF_HOME -> dropped; benchmark.env sets HF_HUB_CACHE # slurm.time_limit -> 06:00:00 (see the note in the tool) # rejection_sample_method -> block, so the eval verifies (see below) # draft_sample_method -> probabilistic, matching the AL's curve # global_segment_size -> 281GB, matching the B300 arm on main -# sbatch_directives.exclude -> b300-018, which times out UCX/NIXL # -# The rails are the part that needs saying. This fabric is rail-isolated: each -# rail is its own /24 and a QP only comes up when both ends sit on the same -# one. srtslurm writes a single store config for the whole job, so rails can -# only be named by index -- and the index is not stable here, because a down -# card shifts every index above it. Across the nodes inventoried, mlx5_0 is -# 172.16.192 everywhere except b300-012, and b300-003 swaps mlx5_3 with -# mlx5_9. Naming either put two nodes on different subnets under one name, -# which killed one earlier bring-up 36 minutes in. The six named below held on -# every node with a full inventory, and kimik3-b300-nic-topology.sh re-checks -# each one's GID subnet at startup so an uninventoried node fails in seconds -# instead of running to a bad number. +# The fabric is the part that needs saying. b300-dsxe is not RoCE: the compute +# fabric is 16 EFA adapters per node (driver efa, 400 Gb each, two per GPU), +# presenting no netdev and no IPv4, so there are no per-rail subnets to name and +# the mlx5 device names the measured config used do not exist here. The +# inventory was identical on every node checked, so there is also no unstable +# index to pin around -- which was the reason the measured config named rails at +# all. Devices are therefore left to the fabric, and UCX_TLS asks for srd, EFA's +# transport, because EFA does not implement the verbs RC that rc requires. name: kimi-k3-vllm-agg-b300-dcp8-dspark7-maxseq2-mooncake-c1-agentic model: @@ -66,7 +62,6 @@ backend: global_segment_size: 281GB local_buffer_size: 4GB protocol: rdma - device_name: mlx5_4,mlx5_8,mlx5_10,mlx5_16,mlx5_20,mlx5_22 mode: embedded enable_offload: false aggregated_environment: @@ -86,13 +81,11 @@ backend: MC_SLICE_SIZE: '1048576' WITH_NVIDIA_PEERMEM: '0' VLLM_LOG_STATS_INTERVAL: '1' - UCX_NET_DEVICES: mlx5_4:1,mlx5_8:1,mlx5_10:1,mlx5_16:1,mlx5_20:1,mlx5_22:1 PYTHONHASHSEED: '42' PYTHONNOUSERSITE: '1' PYTHONUNBUFFERED: '1' TORCH_CUDA_ARCH_LIST: '10.0' - MC_GID_INDEX: '3' - UCX_TLS: rc,cuda_copy,cuda_ipc,sm,self + UCX_TLS: srd,cuda_copy,cuda_ipc,sm,self vllm_config: aggregated: kv-transfer-config: '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"compact_group_io":true,"max_load_batch_keys":2,"enable_offload":false}}' @@ -152,7 +145,6 @@ sbatch_directives: mem: '0' cpus-per-task: '72' comment: '''{"OccupiedIdleGPUsJobReaper":{"exemptIdleTimeMins":"60","reason":"model_loading","description":"Very large model will take extra loading time."}}''' - exclude: b300-018 srun_options: mem: '0' container-remap-root: '' diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b300-dcp8-dspark7-maxseq2-mooncake-c4-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b300-dcp8-dspark7-maxseq2-mooncake-c4-agentic.yaml index 0df65fd809..0620e47030 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b300-dcp8-dspark7-maxseq2-mooncake-c4-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b300-dcp8-dspark7-maxseq2-mooncake-c4-agentic.yaml @@ -8,26 +8,22 @@ # # Only these fields differ from the measured config, each because b300-nv is # not bia: -# device_name, UCX_NET_DEVICES -> the six rails read off this cluster -# MC_GID_INDEX, UCX_TLS -> set; b300-nv is RoCE (see below) +# device_name, UCX_NET_DEVICES -> dropped; see the fabric note below +# MC_GID_INDEX, UCX_TLS -> GID index dropped, UCX_TLS uses srd (below) # HF_HOME -> dropped; benchmark.env sets HF_HUB_CACHE # slurm.time_limit -> 06:00:00 (see the note in the tool) # rejection_sample_method -> block, so the eval verifies (see below) # draft_sample_method -> probabilistic, matching the AL's curve # global_segment_size -> 281GB, matching the B300 arm on main -# sbatch_directives.exclude -> b300-018, which times out UCX/NIXL # -# The rails are the part that needs saying. This fabric is rail-isolated: each -# rail is its own /24 and a QP only comes up when both ends sit on the same -# one. srtslurm writes a single store config for the whole job, so rails can -# only be named by index -- and the index is not stable here, because a down -# card shifts every index above it. Across the nodes inventoried, mlx5_0 is -# 172.16.192 everywhere except b300-012, and b300-003 swaps mlx5_3 with -# mlx5_9. Naming either put two nodes on different subnets under one name, -# which killed one earlier bring-up 36 minutes in. The six named below held on -# every node with a full inventory, and kimik3-b300-nic-topology.sh re-checks -# each one's GID subnet at startup so an uninventoried node fails in seconds -# instead of running to a bad number. +# The fabric is the part that needs saying. b300-dsxe is not RoCE: the compute +# fabric is 16 EFA adapters per node (driver efa, 400 Gb each, two per GPU), +# presenting no netdev and no IPv4, so there are no per-rail subnets to name and +# the mlx5 device names the measured config used do not exist here. The +# inventory was identical on every node checked, so there is also no unstable +# index to pin around -- which was the reason the measured config named rails at +# all. Devices are therefore left to the fabric, and UCX_TLS asks for srd, EFA's +# transport, because EFA does not implement the verbs RC that rc requires. name: kimi-k3-vllm-agg-b300-dcp8-dspark7-maxseq2-mooncake-c4-agentic model: @@ -66,7 +62,6 @@ backend: global_segment_size: 281GB local_buffer_size: 4GB protocol: rdma - device_name: mlx5_4,mlx5_8,mlx5_10,mlx5_16,mlx5_20,mlx5_22 mode: embedded enable_offload: false aggregated_environment: @@ -86,13 +81,11 @@ backend: MC_SLICE_SIZE: '1048576' WITH_NVIDIA_PEERMEM: '0' VLLM_LOG_STATS_INTERVAL: '1' - UCX_NET_DEVICES: mlx5_4:1,mlx5_8:1,mlx5_10:1,mlx5_16:1,mlx5_20:1,mlx5_22:1 PYTHONHASHSEED: '42' PYTHONNOUSERSITE: '1' PYTHONUNBUFFERED: '1' TORCH_CUDA_ARCH_LIST: '10.0' - MC_GID_INDEX: '3' - UCX_TLS: rc,cuda_copy,cuda_ipc,sm,self + UCX_TLS: srd,cuda_copy,cuda_ipc,sm,self vllm_config: aggregated: kv-transfer-config: '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"compact_group_io":true,"max_load_batch_keys":2,"enable_offload":false}}' @@ -152,7 +145,6 @@ sbatch_directives: mem: '0' cpus-per-task: '72' comment: '''{"OccupiedIdleGPUsJobReaper":{"exemptIdleTimeMins":"60","reason":"model_loading","description":"Very large model will take extra loading time."}}''' - exclude: b300-018 srun_options: mem: '0' container-remap-root: '' diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b300-dcp8-nospec-mooncake-c70-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b300-dcp8-nospec-mooncake-c70-agentic.yaml index ee8018dc9f..f98241c39d 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b300-dcp8-nospec-mooncake-c70-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b300-dcp8-nospec-mooncake-c70-agentic.yaml @@ -8,26 +8,22 @@ # # Only these fields differ from the measured config, each because b300-nv is # not bia: -# device_name, UCX_NET_DEVICES -> the six rails read off this cluster -# MC_GID_INDEX, UCX_TLS -> set; b300-nv is RoCE (see below) +# device_name, UCX_NET_DEVICES -> dropped; see the fabric note below +# MC_GID_INDEX, UCX_TLS -> GID index dropped, UCX_TLS uses srd (below) # HF_HOME -> dropped; benchmark.env sets HF_HUB_CACHE # slurm.time_limit -> 06:00:00 (see the note in the tool) # rejection_sample_method -> block, so the eval verifies (see below) # draft_sample_method -> probabilistic, matching the AL's curve # global_segment_size -> 281GB, matching the B300 arm on main -# sbatch_directives.exclude -> b300-018, which times out UCX/NIXL # -# The rails are the part that needs saying. This fabric is rail-isolated: each -# rail is its own /24 and a QP only comes up when both ends sit on the same -# one. srtslurm writes a single store config for the whole job, so rails can -# only be named by index -- and the index is not stable here, because a down -# card shifts every index above it. Across the nodes inventoried, mlx5_0 is -# 172.16.192 everywhere except b300-012, and b300-003 swaps mlx5_3 with -# mlx5_9. Naming either put two nodes on different subnets under one name, -# which killed one earlier bring-up 36 minutes in. The six named below held on -# every node with a full inventory, and kimik3-b300-nic-topology.sh re-checks -# each one's GID subnet at startup so an uninventoried node fails in seconds -# instead of running to a bad number. +# The fabric is the part that needs saying. b300-dsxe is not RoCE: the compute +# fabric is 16 EFA adapters per node (driver efa, 400 Gb each, two per GPU), +# presenting no netdev and no IPv4, so there are no per-rail subnets to name and +# the mlx5 device names the measured config used do not exist here. The +# inventory was identical on every node checked, so there is also no unstable +# index to pin around -- which was the reason the measured config named rails at +# all. Devices are therefore left to the fabric, and UCX_TLS asks for srd, EFA's +# transport, because EFA does not implement the verbs RC that rc requires. name: kimi-k3-vllm-agg-b300-dcp8-nospec-mooncake-c70-agentic model: @@ -65,7 +61,6 @@ backend: global_segment_size: 281GB local_buffer_size: 4GB protocol: rdma - device_name: mlx5_4,mlx5_8,mlx5_10,mlx5_16,mlx5_20,mlx5_22 mode: embedded enable_offload: false aggregated_environment: @@ -83,16 +78,14 @@ backend: TORCH_CUDA_ARCH_LIST: '10.0' PYTHONHASHSEED: '42' VLLM_HTTP_TIMEOUT_KEEP_ALIVE: '900' - MC_GID_INDEX: '3' MC_STORE_MEMCPY: '1' MC_ENABLE_DEST_DEVICE_AFFINITY: '1' MC_SLICE_SIZE: '1048576' MC_WORKERS_PER_CTX: '4' WITH_NVIDIA_PEERMEM: '0' VLLM_MOONCAKE_LOAD_RECV_THREADS: '4' - UCX_NET_DEVICES: mlx5_4:1,mlx5_8:1,mlx5_10:1,mlx5_16:1,mlx5_20:1,mlx5_22:1 PYTHONUNBUFFERED: '1' - UCX_TLS: rc,cuda_copy,cuda_ipc,sm,self + UCX_TLS: srd,cuda_copy,cuda_ipc,sm,self vllm_config: aggregated: served-model-name: moonshotai/Kimi-K3 @@ -149,7 +142,6 @@ sbatch_directives: mem: '0' cpus-per-task: '72' comment: '''{"OccupiedIdleGPUsJobReaper":{"exemptIdleTimeMins":"60","reason":"model_loading","description":"Very large model will take extra loading time."}}''' - exclude: b300-018 srun_options: mem: '0' container-remap-root: '' diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/disagg-b300-1p1d-dcp8-dcp8-dspark4-mooncake-c48-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/disagg-b300-1p1d-dcp8-dcp8-dspark4-mooncake-c48-agentic.yaml index b4888580bb..4989f5ceb5 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/disagg-b300-1p1d-dcp8-dcp8-dspark4-mooncake-c48-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/disagg-b300-1p1d-dcp8-dcp8-dspark4-mooncake-c48-agentic.yaml @@ -8,26 +8,22 @@ # # Only these fields differ from the measured config, each because b300-nv is # not bia: -# device_name, UCX_NET_DEVICES -> the six rails read off this cluster -# MC_GID_INDEX, UCX_TLS -> set; b300-nv is RoCE (see below) +# device_name, UCX_NET_DEVICES -> dropped; see the fabric note below +# MC_GID_INDEX, UCX_TLS -> GID index dropped, UCX_TLS uses srd (below) # HF_HOME -> dropped; benchmark.env sets HF_HUB_CACHE # slurm.time_limit -> 06:00:00 (see the note in the tool) # rejection_sample_method -> block, so the eval verifies (see below) # draft_sample_method -> probabilistic, matching the AL's curve # global_segment_size -> 281GB, matching the B300 arm on main -# sbatch_directives.exclude -> b300-018, which times out UCX/NIXL # -# The rails are the part that needs saying. This fabric is rail-isolated: each -# rail is its own /24 and a QP only comes up when both ends sit on the same -# one. srtslurm writes a single store config for the whole job, so rails can -# only be named by index -- and the index is not stable here, because a down -# card shifts every index above it. Across the nodes inventoried, mlx5_0 is -# 172.16.192 everywhere except b300-012, and b300-003 swaps mlx5_3 with -# mlx5_9. Naming either put two nodes on different subnets under one name, -# which killed one earlier bring-up 36 minutes in. The six named below held on -# every node with a full inventory, and kimik3-b300-nic-topology.sh re-checks -# each one's GID subnet at startup so an uninventoried node fails in seconds -# instead of running to a bad number. +# The fabric is the part that needs saying. b300-dsxe is not RoCE: the compute +# fabric is 16 EFA adapters per node (driver efa, 400 Gb each, two per GPU), +# presenting no netdev and no IPv4, so there are no per-rail subnets to name and +# the mlx5 device names the measured config used do not exist here. The +# inventory was identical on every node checked, so there is also no unstable +# index to pin around -- which was the reason the measured config named rails at +# all. Devices are therefore left to the fabric, and UCX_TLS asks for srd, EFA's +# transport, because EFA does not implement the verbs RC that rc requires. name: kimi-k3-vllm-disagg-b300-1p1d-dcp8-dcp8-dspark4-mooncake-c48-agentic model: @@ -79,7 +75,6 @@ backend: global_segment_size: 281GB local_buffer_size: 4GB protocol: rdma - device_name: mlx5_4,mlx5_8,mlx5_10,mlx5_16,mlx5_20,mlx5_22 mode: embedded enable_offload: false prefill_environment: @@ -115,7 +110,6 @@ backend: UCX_RCACHE_MAX_UNRELEASED: '1024' UCX_TCP_AF_PRIO: inet VLLM_SSM_CONV_STATE_LAYOUT: DS - UCX_NET_DEVICES: mlx5_4:1,mlx5_8:1,mlx5_10:1,mlx5_16:1,mlx5_20:1,mlx5_22:1 DG_JIT_CACHE_DIR: /tmp/dg-cache-kimi-k3-gb300-pd-dspark-mooncake-{job_id} PYTHONHASHSEED: '42' PYTHONNOUSERSITE: '1' @@ -124,8 +118,7 @@ backend: MC_STORE_CLIENT_METRIC: '1' MC_STORE_CLIENT_METRIC_INTERVAL: '5' MC_TE_METRIC: '0' - MC_GID_INDEX: '3' - UCX_TLS: rc,cuda_copy,cuda_ipc,sm,self + UCX_TLS: srd,cuda_copy,cuda_ipc,sm,self decode_environment: VLLM_USE_DIRECT_DCP_A2A: '1' VLLM_USE_DIRECT_DCP_Q_GATHER: '1' @@ -167,8 +160,7 @@ backend: MC_STORE_CLIENT_METRIC: '1' MC_STORE_CLIENT_METRIC_INTERVAL: '5' MC_TE_METRIC: '0' - MC_GID_INDEX: '3' - UCX_TLS: rc,cuda_copy,cuda_ipc,sm,self + UCX_TLS: srd,cuda_copy,cuda_ipc,sm,self vllm_config: prefill: no-enable-flashinfer-autotune: true @@ -216,7 +208,6 @@ sbatch_directives: cpus-per-task: '72' mem: '0' comment: '''{"OccupiedIdleGPUsJobReaper":{"exemptIdleTimeMins":"60","reason":"model_loading","description":"Very large model will take extra loading time."}}''' - exclude: b300-018 srun_options: mem: '0' container-remap-root: '' diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/disagg-b300-1p2d-dcp8-dcp8-dspark4-mooncake-c48-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/disagg-b300-1p2d-dcp8-dcp8-dspark4-mooncake-c48-agentic.yaml index a72e7fecc9..5306b094cd 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/disagg-b300-1p2d-dcp8-dcp8-dspark4-mooncake-c48-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/disagg-b300-1p2d-dcp8-dcp8-dspark4-mooncake-c48-agentic.yaml @@ -8,26 +8,22 @@ # # Only these fields differ from the measured config, each because b300-nv is # not bia: -# device_name, UCX_NET_DEVICES -> the six rails read off this cluster -# MC_GID_INDEX, UCX_TLS -> set; b300-nv is RoCE (see below) +# device_name, UCX_NET_DEVICES -> dropped; see the fabric note below +# MC_GID_INDEX, UCX_TLS -> GID index dropped, UCX_TLS uses srd (below) # HF_HOME -> dropped; benchmark.env sets HF_HUB_CACHE # slurm.time_limit -> 06:00:00 (see the note in the tool) # rejection_sample_method -> block, so the eval verifies (see below) # draft_sample_method -> probabilistic, matching the AL's curve # global_segment_size -> 281GB, matching the B300 arm on main -# sbatch_directives.exclude -> b300-018, which times out UCX/NIXL # -# The rails are the part that needs saying. This fabric is rail-isolated: each -# rail is its own /24 and a QP only comes up when both ends sit on the same -# one. srtslurm writes a single store config for the whole job, so rails can -# only be named by index -- and the index is not stable here, because a down -# card shifts every index above it. Across the nodes inventoried, mlx5_0 is -# 172.16.192 everywhere except b300-012, and b300-003 swaps mlx5_3 with -# mlx5_9. Naming either put two nodes on different subnets under one name, -# which killed one earlier bring-up 36 minutes in. The six named below held on -# every node with a full inventory, and kimik3-b300-nic-topology.sh re-checks -# each one's GID subnet at startup so an uninventoried node fails in seconds -# instead of running to a bad number. +# The fabric is the part that needs saying. b300-dsxe is not RoCE: the compute +# fabric is 16 EFA adapters per node (driver efa, 400 Gb each, two per GPU), +# presenting no netdev and no IPv4, so there are no per-rail subnets to name and +# the mlx5 device names the measured config used do not exist here. The +# inventory was identical on every node checked, so there is also no unstable +# index to pin around -- which was the reason the measured config named rails at +# all. Devices are therefore left to the fabric, and UCX_TLS asks for srd, EFA's +# transport, because EFA does not implement the verbs RC that rc requires. name: kimi-k3-vllm-disagg-b300-1p2d-dcp8-dcp8-dspark4-mooncake-c48-agentic model: @@ -80,7 +76,6 @@ backend: global_segment_size: 281GB local_buffer_size: 4GB protocol: rdma - device_name: mlx5_4,mlx5_8,mlx5_10,mlx5_16,mlx5_20,mlx5_22 mode: embedded enable_offload: false prefill_environment: @@ -116,7 +111,6 @@ backend: UCX_RCACHE_MAX_UNRELEASED: '1024' UCX_TCP_AF_PRIO: inet VLLM_SSM_CONV_STATE_LAYOUT: DS - UCX_NET_DEVICES: mlx5_4:1,mlx5_8:1,mlx5_10:1,mlx5_16:1,mlx5_20:1,mlx5_22:1 DG_JIT_CACHE_DIR: /tmp/dg-cache-kimi-k3-gb300-pd-dspark-mooncake-{job_id} PYTHONHASHSEED: '42' PYTHONNOUSERSITE: '1' @@ -125,8 +119,7 @@ backend: MC_STORE_CLIENT_METRIC: '1' MC_STORE_CLIENT_METRIC_INTERVAL: '5' MC_TE_METRIC: '0' - MC_GID_INDEX: '3' - UCX_TLS: rc,cuda_copy,cuda_ipc,sm,self + UCX_TLS: srd,cuda_copy,cuda_ipc,sm,self decode_environment: VLLM_USE_DIRECT_DCP_A2A: '1' VLLM_USE_DIRECT_DCP_Q_GATHER: '1' @@ -168,8 +161,7 @@ backend: MC_STORE_CLIENT_METRIC: '1' MC_STORE_CLIENT_METRIC_INTERVAL: '5' MC_TE_METRIC: '0' - MC_GID_INDEX: '3' - UCX_TLS: rc,cuda_copy,cuda_ipc,sm,self + UCX_TLS: srd,cuda_copy,cuda_ipc,sm,self vllm_config: prefill: kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"compact_group_io":true,"max_load_batch_keys":2,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' @@ -213,7 +205,6 @@ sbatch_directives: cpus-per-task: '72' mem: '0' comment: '''{"OccupiedIdleGPUsJobReaper":{"exemptIdleTimeMins":"60","reason":"model_loading","description":"Very large model will take extra loading time."}}''' - exclude: b300-018 srun_options: mem: '0' container-remap-root: '' diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/disagg-b300-1p3d-dcp8-dcp8-dspark4-mooncake-c32-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/disagg-b300-1p3d-dcp8-dcp8-dspark4-mooncake-c32-agentic.yaml index a3cc02b379..a19d2d73a4 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/disagg-b300-1p3d-dcp8-dcp8-dspark4-mooncake-c32-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/disagg-b300-1p3d-dcp8-dcp8-dspark4-mooncake-c32-agentic.yaml @@ -8,26 +8,22 @@ # # Only these fields differ from the measured config, each because b300-nv is # not bia: -# device_name, UCX_NET_DEVICES -> the six rails read off this cluster -# MC_GID_INDEX, UCX_TLS -> set; b300-nv is RoCE (see below) +# device_name, UCX_NET_DEVICES -> dropped; see the fabric note below +# MC_GID_INDEX, UCX_TLS -> GID index dropped, UCX_TLS uses srd (below) # HF_HOME -> dropped; benchmark.env sets HF_HUB_CACHE # slurm.time_limit -> 06:00:00 (see the note in the tool) # rejection_sample_method -> block, so the eval verifies (see below) # draft_sample_method -> probabilistic, matching the AL's curve # global_segment_size -> 281GB, matching the B300 arm on main -# sbatch_directives.exclude -> b300-018, which times out UCX/NIXL # -# The rails are the part that needs saying. This fabric is rail-isolated: each -# rail is its own /24 and a QP only comes up when both ends sit on the same -# one. srtslurm writes a single store config for the whole job, so rails can -# only be named by index -- and the index is not stable here, because a down -# card shifts every index above it. Across the nodes inventoried, mlx5_0 is -# 172.16.192 everywhere except b300-012, and b300-003 swaps mlx5_3 with -# mlx5_9. Naming either put two nodes on different subnets under one name, -# which killed one earlier bring-up 36 minutes in. The six named below held on -# every node with a full inventory, and kimik3-b300-nic-topology.sh re-checks -# each one's GID subnet at startup so an uninventoried node fails in seconds -# instead of running to a bad number. +# The fabric is the part that needs saying. b300-dsxe is not RoCE: the compute +# fabric is 16 EFA adapters per node (driver efa, 400 Gb each, two per GPU), +# presenting no netdev and no IPv4, so there are no per-rail subnets to name and +# the mlx5 device names the measured config used do not exist here. The +# inventory was identical on every node checked, so there is also no unstable +# index to pin around -- which was the reason the measured config named rails at +# all. Devices are therefore left to the fabric, and UCX_TLS asks for srd, EFA's +# transport, because EFA does not implement the verbs RC that rc requires. name: kimi-k3-vllm-disagg-b300-1p3d-dcp8-dcp8-dspark4-mooncake-c32-agentic model: @@ -80,7 +76,6 @@ backend: global_segment_size: 281GB local_buffer_size: 4GB protocol: rdma - device_name: mlx5_4,mlx5_8,mlx5_10,mlx5_16,mlx5_20,mlx5_22 mode: embedded enable_offload: false prefill_environment: @@ -116,7 +111,6 @@ backend: UCX_RCACHE_MAX_UNRELEASED: '1024' UCX_TCP_AF_PRIO: inet VLLM_SSM_CONV_STATE_LAYOUT: DS - UCX_NET_DEVICES: mlx5_4:1,mlx5_8:1,mlx5_10:1,mlx5_16:1,mlx5_20:1,mlx5_22:1 DG_JIT_CACHE_DIR: /tmp/dg-cache-kimi-k3-gb300-pd-dspark-mooncake-{job_id} PYTHONHASHSEED: '42' PYTHONNOUSERSITE: '1' @@ -125,8 +119,7 @@ backend: MC_STORE_CLIENT_METRIC: '1' MC_STORE_CLIENT_METRIC_INTERVAL: '5' MC_TE_METRIC: '0' - MC_GID_INDEX: '3' - UCX_TLS: rc,cuda_copy,cuda_ipc,sm,self + UCX_TLS: srd,cuda_copy,cuda_ipc,sm,self decode_environment: VLLM_USE_DIRECT_DCP_A2A: '1' VLLM_USE_DIRECT_DCP_Q_GATHER: '1' @@ -168,8 +161,7 @@ backend: MC_STORE_CLIENT_METRIC: '1' MC_STORE_CLIENT_METRIC_INTERVAL: '5' MC_TE_METRIC: '0' - MC_GID_INDEX: '3' - UCX_TLS: rc,cuda_copy,cuda_ipc,sm,self + UCX_TLS: srd,cuda_copy,cuda_ipc,sm,self vllm_config: prefill: kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"compact_group_io":true,"max_load_batch_keys":2,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' @@ -213,7 +205,6 @@ sbatch_directives: cpus-per-task: '72' mem: '0' comment: '''{"OccupiedIdleGPUsJobReaper":{"exemptIdleTimeMins":"60","reason":"model_loading","description":"Very large model will take extra loading time."}}''' - exclude: b300-018 srun_options: mem: '0' container-remap-root: '' diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index d407e3f4e4..23f0edb82f 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -9799,7 +9799,7 @@ kimik3-fp4-b300-dynamo-vllm-agentic-mooncake-dcp8-agg: image: vllm/vllm-openai:nightly-dev-x86_64-cu13-3696c77 model: moonshotai/Kimi-K3 model-prefix: kimik3 - runner: cluster:b300-nv + runner: cluster:b300-dsxe precision: fp4 framework: dynamo-vllm multinode: true @@ -9828,7 +9828,7 @@ kimik3-fp4-b300-dynamo-vllm-agentic-dspark-mooncake-dcp8-agg: image: vllm/vllm-openai:nightly-dev-x86_64-cu13-3696c77 model: moonshotai/Kimi-K3 model-prefix: kimik3 - runner: cluster:b300-nv + runner: cluster:b300-dsxe precision: fp4 framework: dynamo-vllm multinode: true @@ -9874,7 +9874,7 @@ kimik3-fp4-b300-dynamo-vllm-agentic-dspark-mooncake-dcp8-disagg: image: vllm/vllm-openai:nightly-dev-x86_64-cu13-3696c77 model: moonshotai/Kimi-K3 model-prefix: kimik3 - runner: cluster:b300-nv + runner: cluster:b300-dsxe precision: fp4 framework: dynamo-vllm router: { name: dynamo-router, version: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1" } diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 5b75f9c269..da7de53ee6 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6889,5 +6889,5 @@ - "Add the six measured Kimi-K3 B300 AgentX points: aggregated TP8 x DCP8 without speculative decoding at c70, aggregated DSpark-7 at c4 and c1, and 1P1D, 1P2D and 1P3D DCP8 x DCP8 DSpark-4 at c48, c48 and c32." - "Give Kimi-K3 a model path on b300-dsxe and route the agentic scenario to upstream srt-slurm with the in-repo recipes, without which multi-node Kimi-K3 exits as an unsupported model." - "Apply synthetic acceptance on throughput runs, not only eval-only ones, so the pinned acceptance length reaches speculative-config as it already does on gb300-nv." - - "Verify each node's RDMA rail subnets before the engine starts, because a down card shifts every device index above it and the Mooncake store config is written once for the whole job." + - "Target cluster:b300-dsxe, whose compute fabric is EFA rather than RoCE: drop the pinned transfer devices and RoCE GID index, select the srd transport, and check the adapter inventory before the engine starts." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2814