diff --git a/benchmarks/multi_node/srt-slurm-recipes/configs/kimik3-b300-fabric-check.sh b/benchmarks/multi_node/srt-slurm-recipes/configs/kimik3-b300-fabric-check.sh new file mode 100755 index 0000000000..94026b499b --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/configs/kimik3-b300-fabric-check.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash +# Report this compute node's EFA inventory and GPU affinity before the engine starts. +# +# b300-dsxe's compute fabric is EFA, not RoCE: 16 adapters (driver efa, 400 Gb +# each, two per GPU) that present no netdev and no IPv4. There are no per-rail +# subnets, so there is nothing to pin and nothing to compare subnets against -- +# the recipes leave device selection to the fabric. +# +# What is worth checking is that the node has its full complement. A node +# missing adapters still runs, just slower and asymmetrically against its peers, +# which is the kind of thing that shows up as an unexplained outlier rather than +# an error. Everything here is diagnostic; nothing fails the job. + +set -uo pipefail + +EXPECTED_EFA=16 + +echo "=== [b300-fabric] node: $(hostname) ===" + +mapfile -t EFA_DEVS < <( + for d in /sys/class/infiniband/*; do + [ -e "$d" ] || continue + [ "$(basename "$(readlink -f "$d/device/driver")" 2>/dev/null)" = efa ] || continue + basename "$d" + done | sort +) + +echo "--- EFA adapters: ${#EFA_DEVS[@]} (expected $EXPECTED_EFA) ---" +for dev in "${EFA_DEVS[@]}"; do + state=$(awk '{print $2}' "/sys/class/infiniband/$dev/ports/1/state" 2>/dev/null) + rate=$(awk '{print $1}' "/sys/class/infiniband/$dev/ports/1/rate" 2>/dev/null) + pci=$(basename "$(readlink -f "/sys/class/infiniband/$dev/device")" 2>/dev/null) + printf ' %-12s %-10s %sGb %s\n' "$dev" "${state:-?}" "${rate:-?}" "${pci:-?}" +done + +if [ "${#EFA_DEVS[@]}" -ne "$EXPECTED_EFA" ]; then + echo "[b300-fabric] WARNING: ${#EFA_DEVS[@]} EFA adapters, expected $EXPECTED_EFA." + echo "[b300-fabric] This node will move KV more slowly than its peers." +fi + +down=0 +for dev in "${EFA_DEVS[@]}"; do + state=$(awk '{print $2}' "/sys/class/infiniband/$dev/ports/1/state" 2>/dev/null) + [ "$state" = ACTIVE ] || { echo "[b300-fabric] WARNING: $dev is $state, not ACTIVE."; down=$((down + 1)); } +done +[ "$down" -eq 0 ] && echo "[b300-fabric] all ${#EFA_DEVS[@]} adapters ACTIVE." + +echo "--- GPU / EFA affinity (by PCI bus) ---" +nvidia-smi --query-gpu=index,pci.bus_id --format=csv,noheader 2>/dev/null | while IFS=, read -r idx bus; do + gbus=$(echo "$bus" | tr -d ' ' | cut -d: -f2 | tr 'A-Z' 'a-z') + near="" + for dev in "${EFA_DEVS[@]}"; do + pci=$(basename "$(readlink -f "/sys/class/infiniband/$dev/device")" 2>/dev/null) + nbus=$(echo "$pci" | cut -d: -f2) + d=$(( 16#$gbus - 16#$nbus )); [ "$d" -lt 0 ] && d=$(( -d )) + [ "$d" -le 2 ] && near="$near $dev" + done + printf ' GPU%s (bus %s):%s\n' "$idx" "$gbus" "${near:- none}" +done + +echo "=== [b300-fabric] done ===" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b300-dcp8-dspark7-maxseq2-mooncake-c1-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b300-dcp8-dspark7-maxseq2-mooncake-c1-agentic.yaml new file mode 100644 index 0000000000..766de2864a --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b300-dcp8-dspark7-maxseq2-mooncake-c1-agentic.yaml @@ -0,0 +1,150 @@ +# Kimi-K3 B300 aggregated TP8 x DCP8, agentic coding. +# +# Translated from the bia sweep config that measured this point: +# AGG/kimi-k3-mxfp4-b300-agg-tp8-agentx-062126-c1-weiaggds7-gmu94.yml +# Engine flags, environment, Mooncake settings and vllm_config are that file +# verbatim. The InferenceX plumbing -- identity, benchmark, slurm, the idle-GPU +# reaper comment -- is taken from agg-gb300-dcp8-dspark7-maxseq2-mooncake-agentic.yaml. +# +# Only these fields differ from the measured config, each because b300-nv is +# not bia: +# device_name, UCX_NET_DEVICES -> dropped; see the fabric note below +# MC_GID_INDEX, UCX_TLS -> GID index dropped, UCX_TLS uses srd (below) +# HF_HOME -> dropped; benchmark.env sets HF_HUB_CACHE +# slurm.time_limit -> 06:00:00 (see the note in the tool) +# rejection_sample_method -> block, so the eval verifies (see below) +# draft_sample_method -> probabilistic, matching the AL's curve +# global_segment_size -> 281GB, matching the B300 arm on main +# +# The fabric is the part that needs saying. b300-dsxe is not RoCE: the compute +# fabric is 16 EFA adapters per node (driver efa, 400 Gb each, two per GPU), +# presenting no netdev and no IPv4, so there are no per-rail subnets to name and +# the mlx5 device names the measured config used do not exist here. The +# inventory was identical on every node checked, so there is also no unstable +# index to pin around -- which was the reason the measured config named rails at +# all. Devices are therefore left to the fabric, and UCX_TLS asks for srd, EFA's +# transport, because EFA does not implement the verbs RC that rc requires. + +name: kimi-k3-vllm-agg-b300-dcp8-dspark7-maxseq2-mooncake-c1-agentic +model: + path: moonshotai/Kimi-K3 + container: vllm/vllm-openai:nightly-dev-x86_64-cu13-3696c77 + precision: fp4 +resources: + gpu_type: b300 + gpus_per_node: 8 + agg_nodes: 1 + agg_workers: 1 + gpus_per_agg: 8 +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 +frontend: + type: dynamo + enable_multiple_frontends: false + args: + router-mode: least-loaded + router-session-affinity-ttl-secs: 900 + env: + DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: '3600' + DYN_TOKENIZER_CACHE_BYTES: '8589934592' + PYTHONPYCACHEPREFIX: /tmp/vllm-pycache-kimi-k3 +backend: + type: vllm + connector: null + mooncake_kv_store: + master_extra_args: + - --default_kv_lease_ttl=60000 + - --eviction_high_watermark_ratio=0.95 + - --eviction_ratio=0.10 + store_config: + metadata_server: P2PHANDSHAKE + global_segment_size: 281GB + local_buffer_size: 4GB + protocol: rdma + mode: embedded + enable_offload: false + aggregated_environment: + VLLM_ALLREDUCE_USE_FLASHINFER: '1' + VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: '1' + VLLM_USE_DIRECT_DCP_A2A: '1' + VLLM_USE_DIRECT_DCP_Q_GATHER: '1' + VLLM_USE_DIRECT_DCP_KV_GATHER: '1' + ETCD_LEASE_TTL: '120' + VLLM_ENGINE_READY_TIMEOUT_S: '3600' + VLLM_RPC_TIMEOUT: '600000' + VLLM_USE_V2_MODEL_RUNNER: '1' + VLLM_USE_RUST_FRONTEND: '1' + VLLM_MOONCAKE_LOAD_RECV_THREADS: '4' + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: '0' + VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: '0' + MC_SLICE_SIZE: '1048576' + WITH_NVIDIA_PEERMEM: '0' + VLLM_LOG_STATS_INTERVAL: '1' + PYTHONHASHSEED: '42' + PYTHONNOUSERSITE: '1' + PYTHONUNBUFFERED: '1' + TORCH_CUDA_ARCH_LIST: '10.0' + UCX_TLS: srd,cuda_copy,cuda_ipc,sm,self + vllm_config: + aggregated: + kv-transfer-config: '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"compact_group_io":true,"max_load_batch_keys":2,"enable_offload":false}}' + served-model-name: moonshotai/Kimi-K3 + load-format: fastsafetensors + kv-cache-dtype: fp8 + gpu-memory-utilization: 0.94 + tensor-parallel-size: 8 + decode-context-parallel-size: 8 + dcp-comm-backend: a2a + max-num-seqs: 2 + max-num-batched-tokens: 8192 + trust-remote-code: true + max-cudagraph-capture-size: 1024 + stream-interval: 10 + language-model-only: true + moe-backend: auto + no-enable-flashinfer-autotune: true + enable-cumem-allocator: true + attention-backend: TOKENSPEED_MLA + attention-config: '{"mla_prefill_backend":"TRTLLM_RAGGED","use_prefill_query_quantization":true}' + speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","attention_backend":"TOKENSPEED_MLA","method":"dspark","num_speculative_tokens":7,"rejection_sample_method":"block","draft_sample_method":"probabilistic"}' + enable-prefix-caching: true + prefix-match-unit: 128 +health_check: + max_attempts: 720 + interval_seconds: 10 +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: '300' + AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: '0.25' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'vllm:' + AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 +dynamo: + hash: ba83080ecd31c1ce918559e576d3c5bc9e092ff1 + install: true +identity: + model: + repo: moonshotai/Kimi-K3 + container: + image: vllm/vllm-openai:nightly-dev-x86_64-cu13-3696c77 + frameworks: + dynamo: ba83080ecd31c1ce918559e576d3c5bc9e092ff1 +slurm: + time_limit: 06:00:00 +sbatch_directives: + mem: '0' + cpus-per-task: '72' + comment: '''{"OccupiedIdleGPUsJobReaper":{"exemptIdleTimeMins":"60","reason":"model_loading","description":"Very large model will take extra loading time."}}''' +srun_options: + mem: '0' + container-remap-root: '' diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b300-dcp8-dspark7-maxseq2-mooncake-c4-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b300-dcp8-dspark7-maxseq2-mooncake-c4-agentic.yaml new file mode 100644 index 0000000000..0620e47030 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b300-dcp8-dspark7-maxseq2-mooncake-c4-agentic.yaml @@ -0,0 +1,150 @@ +# Kimi-K3 B300 aggregated TP8 x DCP8, agentic coding. +# +# Translated from the bia sweep config that measured this point: +# AGG/kimi-k3-mxfp4-b300-agg-tp8-agentx-062126-c4-weiaggds7-gmu94.yml +# Engine flags, environment, Mooncake settings and vllm_config are that file +# verbatim. The InferenceX plumbing -- identity, benchmark, slurm, the idle-GPU +# reaper comment -- is taken from agg-gb300-dcp8-dspark7-maxseq2-mooncake-agentic.yaml. +# +# Only these fields differ from the measured config, each because b300-nv is +# not bia: +# device_name, UCX_NET_DEVICES -> dropped; see the fabric note below +# MC_GID_INDEX, UCX_TLS -> GID index dropped, UCX_TLS uses srd (below) +# HF_HOME -> dropped; benchmark.env sets HF_HUB_CACHE +# slurm.time_limit -> 06:00:00 (see the note in the tool) +# rejection_sample_method -> block, so the eval verifies (see below) +# draft_sample_method -> probabilistic, matching the AL's curve +# global_segment_size -> 281GB, matching the B300 arm on main +# +# The fabric is the part that needs saying. b300-dsxe is not RoCE: the compute +# fabric is 16 EFA adapters per node (driver efa, 400 Gb each, two per GPU), +# presenting no netdev and no IPv4, so there are no per-rail subnets to name and +# the mlx5 device names the measured config used do not exist here. The +# inventory was identical on every node checked, so there is also no unstable +# index to pin around -- which was the reason the measured config named rails at +# all. Devices are therefore left to the fabric, and UCX_TLS asks for srd, EFA's +# transport, because EFA does not implement the verbs RC that rc requires. + +name: kimi-k3-vllm-agg-b300-dcp8-dspark7-maxseq2-mooncake-c4-agentic +model: + path: moonshotai/Kimi-K3 + container: vllm/vllm-openai:nightly-dev-x86_64-cu13-3696c77 + precision: fp4 +resources: + gpu_type: b300 + gpus_per_node: 8 + agg_nodes: 1 + agg_workers: 1 + gpus_per_agg: 8 +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 +frontend: + type: dynamo + enable_multiple_frontends: false + args: + router-mode: least-loaded + router-session-affinity-ttl-secs: 900 + env: + DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: '3600' + DYN_TOKENIZER_CACHE_BYTES: '8589934592' + PYTHONPYCACHEPREFIX: /tmp/vllm-pycache-kimi-k3 +backend: + type: vllm + connector: null + mooncake_kv_store: + master_extra_args: + - --default_kv_lease_ttl=60000 + - --eviction_high_watermark_ratio=0.95 + - --eviction_ratio=0.10 + store_config: + metadata_server: P2PHANDSHAKE + global_segment_size: 281GB + local_buffer_size: 4GB + protocol: rdma + mode: embedded + enable_offload: false + aggregated_environment: + VLLM_ALLREDUCE_USE_FLASHINFER: '1' + VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: '1' + VLLM_USE_DIRECT_DCP_A2A: '1' + VLLM_USE_DIRECT_DCP_Q_GATHER: '1' + VLLM_USE_DIRECT_DCP_KV_GATHER: '1' + ETCD_LEASE_TTL: '120' + VLLM_ENGINE_READY_TIMEOUT_S: '3600' + VLLM_RPC_TIMEOUT: '600000' + VLLM_USE_V2_MODEL_RUNNER: '1' + VLLM_USE_RUST_FRONTEND: '1' + VLLM_MOONCAKE_LOAD_RECV_THREADS: '4' + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: '0' + VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: '0' + MC_SLICE_SIZE: '1048576' + WITH_NVIDIA_PEERMEM: '0' + VLLM_LOG_STATS_INTERVAL: '1' + PYTHONHASHSEED: '42' + PYTHONNOUSERSITE: '1' + PYTHONUNBUFFERED: '1' + TORCH_CUDA_ARCH_LIST: '10.0' + UCX_TLS: srd,cuda_copy,cuda_ipc,sm,self + vllm_config: + aggregated: + kv-transfer-config: '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"compact_group_io":true,"max_load_batch_keys":2,"enable_offload":false}}' + served-model-name: moonshotai/Kimi-K3 + load-format: fastsafetensors + kv-cache-dtype: fp8 + gpu-memory-utilization: 0.94 + tensor-parallel-size: 8 + decode-context-parallel-size: 8 + dcp-comm-backend: a2a + max-num-seqs: 2 + max-num-batched-tokens: 8192 + trust-remote-code: true + max-cudagraph-capture-size: 1024 + stream-interval: 10 + language-model-only: true + moe-backend: auto + no-enable-flashinfer-autotune: true + enable-cumem-allocator: true + attention-backend: TOKENSPEED_MLA + attention-config: '{"mla_prefill_backend":"TRTLLM_RAGGED","use_prefill_query_quantization":true}' + speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","attention_backend":"TOKENSPEED_MLA","method":"dspark","num_speculative_tokens":7,"rejection_sample_method":"block","draft_sample_method":"probabilistic"}' + enable-prefix-caching: true + prefix-match-unit: 128 +health_check: + max_attempts: 720 + interval_seconds: 10 +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: '300' + AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: '0.25' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'vllm:' + AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 +dynamo: + hash: ba83080ecd31c1ce918559e576d3c5bc9e092ff1 + install: true +identity: + model: + repo: moonshotai/Kimi-K3 + container: + image: vllm/vllm-openai:nightly-dev-x86_64-cu13-3696c77 + frameworks: + dynamo: ba83080ecd31c1ce918559e576d3c5bc9e092ff1 +slurm: + time_limit: 06:00:00 +sbatch_directives: + mem: '0' + cpus-per-task: '72' + comment: '''{"OccupiedIdleGPUsJobReaper":{"exemptIdleTimeMins":"60","reason":"model_loading","description":"Very large model will take extra loading time."}}''' +srun_options: + mem: '0' + container-remap-root: '' diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b300-dcp8-nospec-mooncake-c70-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b300-dcp8-nospec-mooncake-c70-agentic.yaml new file mode 100644 index 0000000000..f98241c39d --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b300-dcp8-nospec-mooncake-c70-agentic.yaml @@ -0,0 +1,147 @@ +# Kimi-K3 B300 aggregated TP8 x DCP8, agentic coding. +# +# Translated from the bia sweep config that measured this point: +# AGG/kimi-k3-mxfp4-b300-agg-tp8-agentx-062126-c70-weiaggns-gmu94.yml +# Engine flags, environment, Mooncake settings and vllm_config are that file +# verbatim. The InferenceX plumbing -- identity, benchmark, slurm, the idle-GPU +# reaper comment -- is taken from agg-gb300-dcp8-nospec-mooncake-agentic.yaml. +# +# Only these fields differ from the measured config, each because b300-nv is +# not bia: +# device_name, UCX_NET_DEVICES -> dropped; see the fabric note below +# MC_GID_INDEX, UCX_TLS -> GID index dropped, UCX_TLS uses srd (below) +# HF_HOME -> dropped; benchmark.env sets HF_HUB_CACHE +# slurm.time_limit -> 06:00:00 (see the note in the tool) +# rejection_sample_method -> block, so the eval verifies (see below) +# draft_sample_method -> probabilistic, matching the AL's curve +# global_segment_size -> 281GB, matching the B300 arm on main +# +# The fabric is the part that needs saying. b300-dsxe is not RoCE: the compute +# fabric is 16 EFA adapters per node (driver efa, 400 Gb each, two per GPU), +# presenting no netdev and no IPv4, so there are no per-rail subnets to name and +# the mlx5 device names the measured config used do not exist here. The +# inventory was identical on every node checked, so there is also no unstable +# index to pin around -- which was the reason the measured config named rails at +# all. Devices are therefore left to the fabric, and UCX_TLS asks for srd, EFA's +# transport, because EFA does not implement the verbs RC that rc requires. + +name: kimi-k3-vllm-agg-b300-dcp8-nospec-mooncake-c70-agentic +model: + path: moonshotai/Kimi-K3 + container: vllm/vllm-openai:nightly-dev-x86_64-cu13-3696c77 + precision: fp4 +resources: + gpu_type: b300 + gpus_per_node: 8 + agg_nodes: 1 + agg_workers: 1 + gpus_per_agg: 8 +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 +frontend: + type: dynamo + enable_multiple_frontends: false + args: + router-mode: least-loaded + router-session-affinity-ttl-secs: 900 + env: + DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: '3600' + DYN_TOKENIZER_CACHE_BYTES: '8589934592' +backend: + type: vllm + connector: null + mooncake_kv_store: + master_extra_args: + - --default_kv_lease_ttl=60000 + - --eviction_high_watermark_ratio=0.95 + - --eviction_ratio=0.10 + store_config: + metadata_server: P2PHANDSHAKE + global_segment_size: 281GB + local_buffer_size: 4GB + protocol: rdma + mode: embedded + enable_offload: false + aggregated_environment: + VLLM_ALLREDUCE_USE_FLASHINFER: '1' + VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: '1' + VLLM_USE_V2_MODEL_RUNNER: '1' + VLLM_USE_DIRECT_DCP_A2A: '1' + VLLM_USE_DIRECT_DCP_Q_GATHER: '1' + VLLM_USE_DIRECT_DCP_KV_GATHER: '1' + VLLM_ENGINE_READY_TIMEOUT_S: '3600' + VLLM_RPC_TIMEOUT: '600000' + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: '0' + VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: '0' + PYTHONNOUSERSITE: '1' + TORCH_CUDA_ARCH_LIST: '10.0' + PYTHONHASHSEED: '42' + VLLM_HTTP_TIMEOUT_KEEP_ALIVE: '900' + MC_STORE_MEMCPY: '1' + MC_ENABLE_DEST_DEVICE_AFFINITY: '1' + MC_SLICE_SIZE: '1048576' + MC_WORKERS_PER_CTX: '4' + WITH_NVIDIA_PEERMEM: '0' + VLLM_MOONCAKE_LOAD_RECV_THREADS: '4' + PYTHONUNBUFFERED: '1' + UCX_TLS: srd,cuda_copy,cuda_ipc,sm,self + vllm_config: + aggregated: + served-model-name: moonshotai/Kimi-K3 + tensor-parallel-size: 8 + decode-context-parallel-size: 8 + dcp-comm-backend: a2a + gpu-memory-utilization: 0.94 + max-num-batched-tokens: 16384 + trust-remote-code: true + language-model-only: true + load-format: fastsafetensors + moe-backend: auto + no-enable-flashinfer-autotune: true + enable-prefix-caching: true + prefix-match-unit: 128 + kv-cache-dtype: fp8 + stream-interval: 10 + attention-backend: TOKENSPEED_MLA + attention-config: '{"mla_prefill_backend":"TRTLLM_RAGGED","use_prefill_query_quantization":true}' + max-cudagraph-capture-size: 1024 + kv-transfer-config: '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"compact_group_io":true,"max_load_batch_keys":2,"enable_offload":false}}' +health_check: + max_attempts: 720 + interval_seconds: 10 +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: '300' + AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: '0.25' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'vllm:' + AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 +dynamo: + hash: ba83080ecd31c1ce918559e576d3c5bc9e092ff1 + install: true +identity: + model: + repo: moonshotai/Kimi-K3 + container: + image: vllm/vllm-openai:nightly-dev-x86_64-cu13-3696c77 + frameworks: + dynamo: ba83080ecd31c1ce918559e576d3c5bc9e092ff1 +slurm: + time_limit: 06:00:00 +sbatch_directives: + mem: '0' + cpus-per-task: '72' + comment: '''{"OccupiedIdleGPUsJobReaper":{"exemptIdleTimeMins":"60","reason":"model_loading","description":"Very large model will take extra loading time."}}''' +srun_options: + mem: '0' + container-remap-root: '' diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/disagg-b300-1p1d-dcp8-dcp8-dspark4-mooncake-c48-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/disagg-b300-1p1d-dcp8-dcp8-dspark4-mooncake-c48-agentic.yaml new file mode 100644 index 0000000000..4989f5ceb5 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/disagg-b300-1p1d-dcp8-dcp8-dspark4-mooncake-c48-agentic.yaml @@ -0,0 +1,234 @@ +# Kimi-K3 B300 disaggregated, agentic coding. +# +# Translated from the bia sweep config that measured this point: +# DISAGG/kimi-k3-mxfp4-b300-disagg-1p1d-dcp8-dcp8-agentx-062126-c48-weigb300-ns4-gmu94.yml +# Engine flags, environment, Mooncake settings and vllm_config are that file +# verbatim. The InferenceX plumbing -- identity, benchmark, slurm, the idle-GPU +# reaper comment -- is taken from disagg-gb300-1p1d-dcp8-dcp8-dspark4-mooncake-agentic.yaml. +# +# Only these fields differ from the measured config, each because b300-nv is +# not bia: +# device_name, UCX_NET_DEVICES -> dropped; see the fabric note below +# MC_GID_INDEX, UCX_TLS -> GID index dropped, UCX_TLS uses srd (below) +# HF_HOME -> dropped; benchmark.env sets HF_HUB_CACHE +# slurm.time_limit -> 06:00:00 (see the note in the tool) +# rejection_sample_method -> block, so the eval verifies (see below) +# draft_sample_method -> probabilistic, matching the AL's curve +# global_segment_size -> 281GB, matching the B300 arm on main +# +# The fabric is the part that needs saying. b300-dsxe is not RoCE: the compute +# fabric is 16 EFA adapters per node (driver efa, 400 Gb each, two per GPU), +# presenting no netdev and no IPv4, so there are no per-rail subnets to name and +# the mlx5 device names the measured config used do not exist here. The +# inventory was identical on every node checked, so there is also no unstable +# index to pin around -- which was the reason the measured config named rails at +# all. Devices are therefore left to the fabric, and UCX_TLS asks for srd, EFA's +# transport, because EFA does not implement the verbs RC that rc requires. + +name: kimi-k3-vllm-disagg-b300-1p1d-dcp8-dcp8-dspark4-mooncake-c48-agentic +model: + path: moonshotai/Kimi-K3 + container: vllm/vllm-openai:nightly-dev-x86_64-cu13-3696c77 + precision: fp4 +dynamo: + hash: ba83080ecd31c1ce918559e576d3c5bc9e092ff1 + install: true +slurm: + time_limit: 06:00:00 +health_check: + max_attempts: 720 + interval_seconds: 10 +resources: + gpu_type: b300 + gpus_per_node: 8 + prefill_nodes: 1 + decode_nodes: 1 + prefill_workers: 1 + decode_workers: 1 + gpus_per_prefill: 8 + gpus_per_decode: 8 +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 +frontend: + type: dynamo + enable_multiple_frontends: false + args: + router-mode: least-loaded + router-session-affinity-ttl-secs: 900 + env: + DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: '3600' + DYN_TOKENIZER_CACHE_BYTES: '8589934592' +backend: + type: vllm + connector: null + dp_launch_mode: per_node + kv_events_config: + prefill: true + mooncake_kv_store: + master_extra_args: + - --default_kv_lease_ttl=60000 + - --eviction_high_watermark_ratio=0.95 + - --eviction_ratio=0.10 + store_config: + metadata_server: P2PHANDSHAKE + global_segment_size: 281GB + local_buffer_size: 4GB + protocol: rdma + mode: embedded + enable_offload: false + prefill_environment: + VLLM_USE_DIRECT_DCP_A2A: '1' + VLLM_USE_DIRECT_DCP_Q_GATHER: '1' + VLLM_USE_DIRECT_DCP_KV_GATHER: '1' + VLLM_ALLREDUCE_USE_FLASHINFER: '1' + VLLM_KIMI_K3_SHARD_SP_SHARED_EXPERT: '0' + DYN_REQUEST_PLANE: tcp + ETCD_LEASE_TTL: '600' + VLLM_ENGINE_READY_TIMEOUT_S: '3600' + VLLM_RPC_TIMEOUT: '600000' + TILELANG_CLEANUP_TEMP_FILES: '1' + VLLM_USE_NCCL_SYMM_MEM: '0' + NCCL_CUMEM_ENABLE: '1' + NCCL_MNNVL_ENABLE: '0' + NCCL_NVLS_ENABLE: '1' + VLLM_SERVER_DEV_MODE: '1' + VLLM_USE_V2_MODEL_RUNNER: '1' + VLLM_USE_RUST_FRONTEND: '1' + VLLM_MOONCAKE_LOAD_RECV_THREADS: '4' + MC_SLICE_SIZE: '1048576' + VLLM_CONNECTOR_PREFETCH_DEPTH: '8' + VLLM_CONNECTOR_PREFETCH_KV_CAP: '0.65' + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: '1800' + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: '0' + NCCL_P2P_LEVEL: NVL + MC_ENABLE_DEST_DEVICE_AFFINITY: '1' + WITH_NVIDIA_PEERMEM: '0' + NCCL_NET_PLUGIN: none + UCX_MEMTYPE_CACHE: n + UCX_MEMTYPE_REG_WHOLE: n + UCX_RCACHE_MAX_UNRELEASED: '1024' + UCX_TCP_AF_PRIO: inet + VLLM_SSM_CONV_STATE_LAYOUT: DS + DG_JIT_CACHE_DIR: /tmp/dg-cache-kimi-k3-gb300-pd-dspark-mooncake-{job_id} + PYTHONHASHSEED: '42' + PYTHONNOUSERSITE: '1' + PYTHONUNBUFFERED: '1' + TORCH_CUDA_ARCH_LIST: '10.0' + MC_STORE_CLIENT_METRIC: '1' + MC_STORE_CLIENT_METRIC_INTERVAL: '5' + MC_TE_METRIC: '0' + UCX_TLS: srd,cuda_copy,cuda_ipc,sm,self + decode_environment: + VLLM_USE_DIRECT_DCP_A2A: '1' + VLLM_USE_DIRECT_DCP_Q_GATHER: '1' + VLLM_USE_DIRECT_DCP_KV_GATHER: '1' + VLLM_ALLREDUCE_USE_FLASHINFER: '1' + VLLM_KIMI_K3_SHARD_SP_SHARED_EXPERT: '0' + DYN_REQUEST_PLANE: tcp + ETCD_LEASE_TTL: '600' + VLLM_ENGINE_READY_TIMEOUT_S: '3600' + VLLM_RPC_TIMEOUT: '600000' + TILELANG_CLEANUP_TEMP_FILES: '1' + VLLM_USE_NCCL_SYMM_MEM: '0' + NCCL_CUMEM_ENABLE: '1' + NCCL_MNNVL_ENABLE: '0' + NCCL_NVLS_ENABLE: '1' + VLLM_SERVER_DEV_MODE: '1' + VLLM_USE_V2_MODEL_RUNNER: '1' + VLLM_USE_RUST_FRONTEND: '1' + VLLM_MOONCAKE_LOAD_RECV_THREADS: '4' + MC_SLICE_SIZE: '1048576' + VLLM_CONNECTOR_PREFETCH_DEPTH: '8' + VLLM_CONNECTOR_PREFETCH_KV_CAP: '0.65' + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: '1800' + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: '0' + NCCL_P2P_LEVEL: NVL + MC_ENABLE_DEST_DEVICE_AFFINITY: '1' + WITH_NVIDIA_PEERMEM: '0' + NCCL_NET_PLUGIN: none + UCX_MEMTYPE_CACHE: n + UCX_MEMTYPE_REG_WHOLE: n + UCX_RCACHE_MAX_UNRELEASED: '1024' + UCX_TCP_AF_PRIO: inet + VLLM_SSM_CONV_STATE_LAYOUT: DS + DG_JIT_CACHE_DIR: /tmp/dg-cache-kimi-k3-gb300-pd-dspark-mooncake-{job_id} + PYTHONHASHSEED: '42' + PYTHONNOUSERSITE: '1' + PYTHONUNBUFFERED: '1' + TORCH_CUDA_ARCH_LIST: '10.0' + MC_STORE_CLIENT_METRIC: '1' + MC_STORE_CLIENT_METRIC_INTERVAL: '5' + MC_TE_METRIC: '0' + UCX_TLS: srd,cuda_copy,cuda_ipc,sm,self + vllm_config: + prefill: + no-enable-flashinfer-autotune: true + enable-logging-iteration-details: true + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"compact_group_io":true,"max_load_batch_keys":2,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + served-model-name: moonshotai/Kimi-K3 + prefix-match-unit: 128 + load-format: fastsafetensors + kv-cache-dtype: fp8 + gpu-memory-utilization: 0.94 + tensor-parallel-size: 8 + decode-context-parallel-size: 8 + dcp-comm-backend: a2a + enable-cumem-allocator: true + trust-remote-code: true + max-cudagraph-capture-size: 512 + stream-interval: 10 + language-model-only: true + attention-backend: TOKENSPEED_MLA + attention-config: '{"mla_prefill_backend": "TRTLLM_RAGGED", "use_prefill_query_quantization": true}' + speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","attention_backend":"TOKENSPEED_MLA","method":"dspark","num_speculative_tokens":4,"rejection_sample_method":"block","draft_sample_method":"probabilistic"}' + enable-prefix-caching: true + decode: + no-enable-flashinfer-autotune: true + enable-logging-iteration-details: true + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"compact_group_io":true,"max_load_batch_keys":2,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + served-model-name: moonshotai/Kimi-K3 + prefix-match-unit: 128 + load-format: fastsafetensors + kv-cache-dtype: fp8 + gpu-memory-utilization: 0.94 + tensor-parallel-size: 8 + decode-context-parallel-size: 8 + dcp-comm-backend: a2a + enable-cumem-allocator: true + trust-remote-code: true + max-cudagraph-capture-size: 512 + stream-interval: 10 + language-model-only: true + attention-backend: TOKENSPEED_MLA + attention-config: '{"mla_prefill_backend": "TRTLLM_RAGGED", "use_prefill_query_quantization": true}' + speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","attention_backend":"TOKENSPEED_MLA","method":"dspark","num_speculative_tokens":4,"rejection_sample_method":"block","draft_sample_method":"probabilistic"}' + enable-prefix-caching: true +sbatch_directives: + cpus-per-task: '72' + mem: '0' + comment: '''{"OccupiedIdleGPUsJobReaper":{"exemptIdleTimeMins":"60","reason":"model_loading","description":"Very large model will take extra loading time."}}''' +srun_options: + mem: '0' + container-remap-root: '' +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: '14400' + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 +identity: + model: + repo: moonshotai/Kimi-K3 + container: + image: vllm/vllm-openai:nightly-dev-x86_64-cu13-3696c77 + frameworks: + dynamo: ba83080ecd31c1ce918559e576d3c5bc9e092ff1 diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/disagg-b300-1p2d-dcp8-dcp8-dspark4-mooncake-c48-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/disagg-b300-1p2d-dcp8-dcp8-dspark4-mooncake-c48-agentic.yaml new file mode 100644 index 0000000000..5306b094cd --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/disagg-b300-1p2d-dcp8-dcp8-dspark4-mooncake-c48-agentic.yaml @@ -0,0 +1,231 @@ +# Kimi-K3 B300 disaggregated, agentic coding. +# +# Translated from the bia sweep config that measured this point: +# DISAGG/kimi-k3-mxfp4-b300-disagg-1p2d-dcp8-dcp8-agentx-062126-c48-wei1p2d-ns4-gmu94.yml +# Engine flags, environment, Mooncake settings and vllm_config are that file +# verbatim. The InferenceX plumbing -- identity, benchmark, slurm, the idle-GPU +# reaper comment -- is taken from disagg-gb300-1p2d-dcp8-dcp8-dspark4-mooncake-agentic.yaml. +# +# Only these fields differ from the measured config, each because b300-nv is +# not bia: +# device_name, UCX_NET_DEVICES -> dropped; see the fabric note below +# MC_GID_INDEX, UCX_TLS -> GID index dropped, UCX_TLS uses srd (below) +# HF_HOME -> dropped; benchmark.env sets HF_HUB_CACHE +# slurm.time_limit -> 06:00:00 (see the note in the tool) +# rejection_sample_method -> block, so the eval verifies (see below) +# draft_sample_method -> probabilistic, matching the AL's curve +# global_segment_size -> 281GB, matching the B300 arm on main +# +# The fabric is the part that needs saying. b300-dsxe is not RoCE: the compute +# fabric is 16 EFA adapters per node (driver efa, 400 Gb each, two per GPU), +# presenting no netdev and no IPv4, so there are no per-rail subnets to name and +# the mlx5 device names the measured config used do not exist here. The +# inventory was identical on every node checked, so there is also no unstable +# index to pin around -- which was the reason the measured config named rails at +# all. Devices are therefore left to the fabric, and UCX_TLS asks for srd, EFA's +# transport, because EFA does not implement the verbs RC that rc requires. + +name: kimi-k3-vllm-disagg-b300-1p2d-dcp8-dcp8-dspark4-mooncake-c48-agentic +model: + path: moonshotai/Kimi-K3 + container: vllm/vllm-openai:nightly-dev-x86_64-cu13-3696c77 + precision: fp4 +dynamo: + hash: ba83080ecd31c1ce918559e576d3c5bc9e092ff1 + install: true +slurm: + time_limit: 06:00:00 +health_check: + max_attempts: 720 + interval_seconds: 10 +resources: + gpu_type: b300 + gpus_per_node: 8 + prefill_nodes: 1 + decode_nodes: 2 + prefill_workers: 1 + decode_workers: 2 + gpus_per_prefill: 8 + gpus_per_decode: 8 +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 +frontend: + type: dynamo + enable_multiple_frontends: false + args: + router-mode: least-loaded + router-session-affinity-ttl-secs: 900 + env: + DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: '3600' + DYN_TOKENIZER_CACHE_BYTES: '8589934592' + PYTHONPYCACHEPREFIX: /tmp/vllm-pycache-kimi-k3 +backend: + type: vllm + connector: null + dp_launch_mode: per_node + kv_events_config: + prefill: true + mooncake_kv_store: + master_extra_args: + - --default_kv_lease_ttl=60000 + - --eviction_high_watermark_ratio=0.95 + - --eviction_ratio=0.10 + store_config: + metadata_server: P2PHANDSHAKE + global_segment_size: 281GB + local_buffer_size: 4GB + protocol: rdma + mode: embedded + enable_offload: false + prefill_environment: + VLLM_USE_DIRECT_DCP_A2A: '1' + VLLM_USE_DIRECT_DCP_Q_GATHER: '1' + VLLM_USE_DIRECT_DCP_KV_GATHER: '1' + VLLM_ALLREDUCE_USE_FLASHINFER: '1' + VLLM_KIMI_K3_SHARD_SP_SHARED_EXPERT: '0' + DYN_REQUEST_PLANE: tcp + ETCD_LEASE_TTL: '600' + VLLM_ENGINE_READY_TIMEOUT_S: '3600' + VLLM_RPC_TIMEOUT: '600000' + TILELANG_CLEANUP_TEMP_FILES: '1' + VLLM_USE_NCCL_SYMM_MEM: '0' + NCCL_CUMEM_ENABLE: '1' + NCCL_MNNVL_ENABLE: '0' + NCCL_NVLS_ENABLE: '1' + VLLM_SERVER_DEV_MODE: '1' + VLLM_USE_V2_MODEL_RUNNER: '1' + VLLM_USE_RUST_FRONTEND: '1' + VLLM_MOONCAKE_LOAD_RECV_THREADS: '4' + MC_SLICE_SIZE: '1048576' + VLLM_CONNECTOR_PREFETCH_DEPTH: '8' + VLLM_CONNECTOR_PREFETCH_KV_CAP: '0.65' + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: '1800' + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: '0' + NCCL_P2P_LEVEL: NVL + MC_ENABLE_DEST_DEVICE_AFFINITY: '1' + WITH_NVIDIA_PEERMEM: '0' + NCCL_NET_PLUGIN: none + UCX_MEMTYPE_CACHE: n + UCX_MEMTYPE_REG_WHOLE: n + UCX_RCACHE_MAX_UNRELEASED: '1024' + UCX_TCP_AF_PRIO: inet + VLLM_SSM_CONV_STATE_LAYOUT: DS + DG_JIT_CACHE_DIR: /tmp/dg-cache-kimi-k3-gb300-pd-dspark-mooncake-{job_id} + PYTHONHASHSEED: '42' + PYTHONNOUSERSITE: '1' + PYTHONUNBUFFERED: '1' + TORCH_CUDA_ARCH_LIST: '10.0' + MC_STORE_CLIENT_METRIC: '1' + MC_STORE_CLIENT_METRIC_INTERVAL: '5' + MC_TE_METRIC: '0' + UCX_TLS: srd,cuda_copy,cuda_ipc,sm,self + decode_environment: + VLLM_USE_DIRECT_DCP_A2A: '1' + VLLM_USE_DIRECT_DCP_Q_GATHER: '1' + VLLM_USE_DIRECT_DCP_KV_GATHER: '1' + VLLM_ALLREDUCE_USE_FLASHINFER: '1' + VLLM_KIMI_K3_SHARD_SP_SHARED_EXPERT: '0' + DYN_REQUEST_PLANE: tcp + ETCD_LEASE_TTL: '600' + VLLM_ENGINE_READY_TIMEOUT_S: '3600' + VLLM_RPC_TIMEOUT: '600000' + TILELANG_CLEANUP_TEMP_FILES: '1' + VLLM_USE_NCCL_SYMM_MEM: '0' + NCCL_CUMEM_ENABLE: '1' + NCCL_MNNVL_ENABLE: '0' + NCCL_NVLS_ENABLE: '1' + VLLM_SERVER_DEV_MODE: '1' + VLLM_USE_V2_MODEL_RUNNER: '1' + VLLM_USE_RUST_FRONTEND: '1' + VLLM_MOONCAKE_LOAD_RECV_THREADS: '4' + MC_SLICE_SIZE: '1048576' + VLLM_CONNECTOR_PREFETCH_DEPTH: '8' + VLLM_CONNECTOR_PREFETCH_KV_CAP: '0.65' + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: '1800' + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: '0' + NCCL_P2P_LEVEL: NVL + MC_ENABLE_DEST_DEVICE_AFFINITY: '1' + WITH_NVIDIA_PEERMEM: '0' + NCCL_NET_PLUGIN: none + UCX_MEMTYPE_CACHE: n + UCX_MEMTYPE_REG_WHOLE: n + UCX_RCACHE_MAX_UNRELEASED: '1024' + UCX_TCP_AF_PRIO: inet + VLLM_SSM_CONV_STATE_LAYOUT: DS + DG_JIT_CACHE_DIR: /tmp/dg-cache-kimi-k3-gb300-pd-dspark-mooncake-{job_id} + PYTHONHASHSEED: '42' + PYTHONNOUSERSITE: '1' + PYTHONUNBUFFERED: '1' + TORCH_CUDA_ARCH_LIST: '10.0' + MC_STORE_CLIENT_METRIC: '1' + MC_STORE_CLIENT_METRIC_INTERVAL: '5' + MC_TE_METRIC: '0' + UCX_TLS: srd,cuda_copy,cuda_ipc,sm,self + vllm_config: + prefill: + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"compact_group_io":true,"max_load_batch_keys":2,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + served-model-name: moonshotai/Kimi-K3 + prefix-match-unit: 128 + load-format: fastsafetensors + kv-cache-dtype: fp8 + gpu-memory-utilization: 0.94 + tensor-parallel-size: 8 + decode-context-parallel-size: 8 + dcp-comm-backend: a2a + enable-cumem-allocator: true + trust-remote-code: true + max-cudagraph-capture-size: 512 + stream-interval: 10 + language-model-only: true + attention-backend: TOKENSPEED_MLA + attention-config: '{"mla_prefill_backend": "TRTLLM_RAGGED", "use_prefill_query_quantization": true}' + speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","attention_backend":"TOKENSPEED_MLA","method":"dspark","num_speculative_tokens":4,"rejection_sample_method":"block","draft_sample_method":"probabilistic"}' + enable-prefix-caching: true + decode: + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"compact_group_io":true,"max_load_batch_keys":2,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + served-model-name: moonshotai/Kimi-K3 + prefix-match-unit: 128 + load-format: fastsafetensors + kv-cache-dtype: fp8 + gpu-memory-utilization: 0.94 + tensor-parallel-size: 8 + decode-context-parallel-size: 8 + dcp-comm-backend: a2a + enable-cumem-allocator: true + trust-remote-code: true + max-cudagraph-capture-size: 512 + stream-interval: 10 + language-model-only: true + attention-backend: TOKENSPEED_MLA + attention-config: '{"mla_prefill_backend": "TRTLLM_RAGGED", "use_prefill_query_quantization": true}' + speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","attention_backend":"TOKENSPEED_MLA","method":"dspark","num_speculative_tokens":4,"rejection_sample_method":"block","draft_sample_method":"probabilistic"}' + enable-prefix-caching: true +sbatch_directives: + cpus-per-task: '72' + mem: '0' + comment: '''{"OccupiedIdleGPUsJobReaper":{"exemptIdleTimeMins":"60","reason":"model_loading","description":"Very large model will take extra loading time."}}''' +srun_options: + mem: '0' + container-remap-root: '' +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: '14400' + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 +identity: + model: + repo: moonshotai/Kimi-K3 + container: + image: vllm/vllm-openai:nightly-dev-x86_64-cu13-3696c77 + frameworks: + dynamo: ba83080ecd31c1ce918559e576d3c5bc9e092ff1 diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/disagg-b300-1p3d-dcp8-dcp8-dspark4-mooncake-c32-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/disagg-b300-1p3d-dcp8-dcp8-dspark4-mooncake-c32-agentic.yaml new file mode 100644 index 0000000000..a19d2d73a4 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/disagg-b300-1p3d-dcp8-dcp8-dspark4-mooncake-c32-agentic.yaml @@ -0,0 +1,231 @@ +# Kimi-K3 B300 disaggregated, agentic coding. +# +# Translated from the bia sweep config that measured this point: +# DISAGG/kimi-k3-mxfp4-b300-disagg-1p3d-dcp8-dcp8-agentx-062126-c32-wei1p3d-ns4-gmu94.yml +# Engine flags, environment, Mooncake settings and vllm_config are that file +# verbatim. The InferenceX plumbing -- identity, benchmark, slurm, the idle-GPU +# reaper comment -- is taken from disagg-gb300-1p2d-dcp8-dcp8-dspark4-mooncake-agentic.yaml. +# +# Only these fields differ from the measured config, each because b300-nv is +# not bia: +# device_name, UCX_NET_DEVICES -> dropped; see the fabric note below +# MC_GID_INDEX, UCX_TLS -> GID index dropped, UCX_TLS uses srd (below) +# HF_HOME -> dropped; benchmark.env sets HF_HUB_CACHE +# slurm.time_limit -> 06:00:00 (see the note in the tool) +# rejection_sample_method -> block, so the eval verifies (see below) +# draft_sample_method -> probabilistic, matching the AL's curve +# global_segment_size -> 281GB, matching the B300 arm on main +# +# The fabric is the part that needs saying. b300-dsxe is not RoCE: the compute +# fabric is 16 EFA adapters per node (driver efa, 400 Gb each, two per GPU), +# presenting no netdev and no IPv4, so there are no per-rail subnets to name and +# the mlx5 device names the measured config used do not exist here. The +# inventory was identical on every node checked, so there is also no unstable +# index to pin around -- which was the reason the measured config named rails at +# all. Devices are therefore left to the fabric, and UCX_TLS asks for srd, EFA's +# transport, because EFA does not implement the verbs RC that rc requires. + +name: kimi-k3-vllm-disagg-b300-1p3d-dcp8-dcp8-dspark4-mooncake-c32-agentic +model: + path: moonshotai/Kimi-K3 + container: vllm/vllm-openai:nightly-dev-x86_64-cu13-3696c77 + precision: fp4 +dynamo: + hash: ba83080ecd31c1ce918559e576d3c5bc9e092ff1 + install: true +slurm: + time_limit: 06:00:00 +health_check: + max_attempts: 720 + interval_seconds: 10 +resources: + gpu_type: b300 + gpus_per_node: 8 + prefill_nodes: 1 + decode_nodes: 3 + prefill_workers: 1 + decode_workers: 3 + gpus_per_prefill: 8 + gpus_per_decode: 8 +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 +frontend: + type: dynamo + enable_multiple_frontends: false + args: + router-mode: least-loaded + router-session-affinity-ttl-secs: 900 + env: + DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: '3600' + DYN_TOKENIZER_CACHE_BYTES: '8589934592' + PYTHONPYCACHEPREFIX: /tmp/vllm-pycache-kimi-k3 +backend: + type: vllm + connector: null + dp_launch_mode: per_node + kv_events_config: + prefill: true + mooncake_kv_store: + master_extra_args: + - --default_kv_lease_ttl=60000 + - --eviction_high_watermark_ratio=0.95 + - --eviction_ratio=0.10 + store_config: + metadata_server: P2PHANDSHAKE + global_segment_size: 281GB + local_buffer_size: 4GB + protocol: rdma + mode: embedded + enable_offload: false + prefill_environment: + VLLM_USE_DIRECT_DCP_A2A: '1' + VLLM_USE_DIRECT_DCP_Q_GATHER: '1' + VLLM_USE_DIRECT_DCP_KV_GATHER: '1' + VLLM_ALLREDUCE_USE_FLASHINFER: '1' + VLLM_KIMI_K3_SHARD_SP_SHARED_EXPERT: '0' + DYN_REQUEST_PLANE: tcp + ETCD_LEASE_TTL: '600' + VLLM_ENGINE_READY_TIMEOUT_S: '3600' + VLLM_RPC_TIMEOUT: '600000' + TILELANG_CLEANUP_TEMP_FILES: '1' + VLLM_USE_NCCL_SYMM_MEM: '0' + NCCL_CUMEM_ENABLE: '1' + NCCL_MNNVL_ENABLE: '0' + NCCL_NVLS_ENABLE: '1' + VLLM_SERVER_DEV_MODE: '1' + VLLM_USE_V2_MODEL_RUNNER: '1' + VLLM_USE_RUST_FRONTEND: '1' + VLLM_MOONCAKE_LOAD_RECV_THREADS: '4' + MC_SLICE_SIZE: '1048576' + VLLM_CONNECTOR_PREFETCH_DEPTH: '8' + VLLM_CONNECTOR_PREFETCH_KV_CAP: '0.65' + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: '1800' + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: '0' + NCCL_P2P_LEVEL: NVL + MC_ENABLE_DEST_DEVICE_AFFINITY: '1' + WITH_NVIDIA_PEERMEM: '0' + NCCL_NET_PLUGIN: none + UCX_MEMTYPE_CACHE: n + UCX_MEMTYPE_REG_WHOLE: n + UCX_RCACHE_MAX_UNRELEASED: '1024' + UCX_TCP_AF_PRIO: inet + VLLM_SSM_CONV_STATE_LAYOUT: DS + DG_JIT_CACHE_DIR: /tmp/dg-cache-kimi-k3-gb300-pd-dspark-mooncake-{job_id} + PYTHONHASHSEED: '42' + PYTHONNOUSERSITE: '1' + PYTHONUNBUFFERED: '1' + TORCH_CUDA_ARCH_LIST: '10.0' + MC_STORE_CLIENT_METRIC: '1' + MC_STORE_CLIENT_METRIC_INTERVAL: '5' + MC_TE_METRIC: '0' + UCX_TLS: srd,cuda_copy,cuda_ipc,sm,self + decode_environment: + VLLM_USE_DIRECT_DCP_A2A: '1' + VLLM_USE_DIRECT_DCP_Q_GATHER: '1' + VLLM_USE_DIRECT_DCP_KV_GATHER: '1' + VLLM_ALLREDUCE_USE_FLASHINFER: '1' + VLLM_KIMI_K3_SHARD_SP_SHARED_EXPERT: '0' + DYN_REQUEST_PLANE: tcp + ETCD_LEASE_TTL: '600' + VLLM_ENGINE_READY_TIMEOUT_S: '3600' + VLLM_RPC_TIMEOUT: '600000' + TILELANG_CLEANUP_TEMP_FILES: '1' + VLLM_USE_NCCL_SYMM_MEM: '0' + NCCL_CUMEM_ENABLE: '1' + NCCL_MNNVL_ENABLE: '0' + NCCL_NVLS_ENABLE: '1' + VLLM_SERVER_DEV_MODE: '1' + VLLM_USE_V2_MODEL_RUNNER: '1' + VLLM_USE_RUST_FRONTEND: '1' + VLLM_MOONCAKE_LOAD_RECV_THREADS: '4' + MC_SLICE_SIZE: '1048576' + VLLM_CONNECTOR_PREFETCH_DEPTH: '8' + VLLM_CONNECTOR_PREFETCH_KV_CAP: '0.65' + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: '1800' + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: '0' + NCCL_P2P_LEVEL: NVL + MC_ENABLE_DEST_DEVICE_AFFINITY: '1' + WITH_NVIDIA_PEERMEM: '0' + NCCL_NET_PLUGIN: none + UCX_MEMTYPE_CACHE: n + UCX_MEMTYPE_REG_WHOLE: n + UCX_RCACHE_MAX_UNRELEASED: '1024' + UCX_TCP_AF_PRIO: inet + VLLM_SSM_CONV_STATE_LAYOUT: DS + DG_JIT_CACHE_DIR: /tmp/dg-cache-kimi-k3-gb300-pd-dspark-mooncake-{job_id} + PYTHONHASHSEED: '42' + PYTHONNOUSERSITE: '1' + PYTHONUNBUFFERED: '1' + TORCH_CUDA_ARCH_LIST: '10.0' + MC_STORE_CLIENT_METRIC: '1' + MC_STORE_CLIENT_METRIC_INTERVAL: '5' + MC_TE_METRIC: '0' + UCX_TLS: srd,cuda_copy,cuda_ipc,sm,self + vllm_config: + prefill: + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"compact_group_io":true,"max_load_batch_keys":2,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + served-model-name: moonshotai/Kimi-K3 + prefix-match-unit: 128 + load-format: fastsafetensors + kv-cache-dtype: fp8 + gpu-memory-utilization: 0.94 + tensor-parallel-size: 8 + decode-context-parallel-size: 8 + dcp-comm-backend: a2a + enable-cumem-allocator: true + trust-remote-code: true + max-cudagraph-capture-size: 512 + stream-interval: 10 + language-model-only: true + attention-backend: TOKENSPEED_MLA + attention-config: '{"mla_prefill_backend": "TRTLLM_RAGGED", "use_prefill_query_quantization": true}' + speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","attention_backend":"TOKENSPEED_MLA","method":"dspark","num_speculative_tokens":4,"rejection_sample_method":"block","draft_sample_method":"probabilistic"}' + enable-prefix-caching: true + decode: + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"compact_group_io":true,"max_load_batch_keys":2,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + served-model-name: moonshotai/Kimi-K3 + prefix-match-unit: 128 + load-format: fastsafetensors + kv-cache-dtype: fp8 + gpu-memory-utilization: 0.94 + tensor-parallel-size: 8 + decode-context-parallel-size: 8 + dcp-comm-backend: a2a + enable-cumem-allocator: true + trust-remote-code: true + max-cudagraph-capture-size: 512 + stream-interval: 10 + language-model-only: true + attention-backend: TOKENSPEED_MLA + attention-config: '{"mla_prefill_backend": "TRTLLM_RAGGED", "use_prefill_query_quantization": true}' + speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","attention_backend":"TOKENSPEED_MLA","method":"dspark","num_speculative_tokens":4,"rejection_sample_method":"block","draft_sample_method":"probabilistic"}' + enable-prefix-caching: true +sbatch_directives: + cpus-per-task: '72' + mem: '0' + comment: '''{"OccupiedIdleGPUsJobReaper":{"exemptIdleTimeMins":"60","reason":"model_loading","description":"Very large model will take extra loading time."}}''' +srun_options: + mem: '0' + container-remap-root: '' +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: '14400' + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 +identity: + model: + repo: moonshotai/Kimi-K3 + container: + image: vllm/vllm-openai:nightly-dev-x86_64-cu13-3696c77 + frameworks: + dynamo: ba83080ecd31c1ce918559e576d3c5bc9e092ff1 diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 0a6c86df6d..23f0edb82f 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -9773,6 +9773,164 @@ kimik3-fp4-gb300-dynamo-vllm-agentic-mooncake-dcp8-agg: - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-gb300-dcp8-nospec-mooncake-agentic.yaml" + +# Kimi-K3 MXFP4 B300 agentic coding, six points. All three keys below run the +# same TP8 x DCP8 worker; what changes is how many of them there are and +# whether they draft. +# +# The topology is why B300 is a separate entry from GB300. A +# DCP8 worker spans two 4-GPU nodes on GB300 but fits inside one 8-GPU node +# here, so a worker never leaves its NVLink domain and only the +# prefill-to-decode hop crosses the fabric. +# +# 0.75 is the value the existing B300 Kimi-K3 arm uses on this cluster. In the +# single-node path it resolves to 2249 GB and the script divides by TP8; the +# recipes below name the resulting per-rank segment directly, since srt-slurm +# takes global_segment_size as a literal and does no division. +# +# One recipe per concurrency, because the measured configs differ by more than +# concurrency: the drafting arms pin max-num-seqs 2 and capture at 1024, the +# no-spec arm runs max-num-batched-tokens 16384 with moe-backend auto, and the +# disaggregated arms carry the NIXL connector the aggregated ones have no use +# for. + +# One node, no speculative decoding. +kimik3-fp4-b300-dynamo-vllm-agentic-mooncake-dcp8-agg: + image: vllm/vllm-openai:nightly-dev-x86_64-cu13-3696c77 + model: moonshotai/Kimi-K3 + model-prefix: kimik3 + runner: cluster:b300-dsxe + precision: fp4 + framework: dynamo-vllm + multinode: true + disagg: false + scenarios: + agentic-coding: + - dram-utilization: 0.75 + search-space: + - kv-offloading: dram + kv-offload-backend: { name: mooncake, version: "0.3.12.post1" } + conc-list: [70] + num-nodes: 1 + worker: + num-worker: 1 + tp: 8 + dcp-size: 8 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-b300-dcp8-nospec-mooncake-c70-agentic.yaml" + +# One node drafting at DSpark K=7 with max-num-seqs 2. 3.84 is the golden AL +# at K=7 on the probabilistic/block curve, the same figure the B300 +# single-node arm and the GB300 DSpark-7 arm pin. +kimik3-fp4-b300-dynamo-vllm-agentic-dspark-mooncake-dcp8-agg: + image: vllm/vllm-openai:nightly-dev-x86_64-cu13-3696c77 + model: moonshotai/Kimi-K3 + model-prefix: kimik3 + runner: cluster:b300-dsxe + precision: fp4 + framework: dynamo-vllm + multinode: true + disagg: false + scenarios: + agentic-coding: + - dram-utilization: 0.75 + search-space: + - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: { name: mooncake, version: "0.3.12.post1" } + conc-list: [4] + num-nodes: 1 + worker: + num-worker: 1 + tp: 8 + dcp-size: 8 + ep: 1 + dp-attn: false + additional-settings: + - "SYNTHETIC_ACCEPTANCE=true" + - "SYNTHETIC_ACCEPTANCE_LENGTH=3.84" + - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-b300-dcp8-dspark7-maxseq2-mooncake-c4-agentic.yaml" + - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: { name: mooncake, version: "0.3.12.post1" } + conc-list: [1] + num-nodes: 1 + worker: + num-worker: 1 + tp: 8 + dcp-size: 8 + ep: 1 + dp-attn: false + additional-settings: + - "SYNTHETIC_ACCEPTANCE=true" + - "SYNTHETIC_ACCEPTANCE_LENGTH=3.84" + - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-b300-dcp8-dspark7-maxseq2-mooncake-c1-agentic.yaml" + +# One prefill worker feeding one, two or three decode workers, each a whole +# 8-GPU node, drafting at DSpark K=4. +kimik3-fp4-b300-dynamo-vllm-agentic-dspark-mooncake-dcp8-disagg: + image: vllm/vllm-openai:nightly-dev-x86_64-cu13-3696c77 + model: moonshotai/Kimi-K3 + model-prefix: kimik3 + runner: cluster:b300-dsxe + precision: fp4 + framework: dynamo-vllm + router: { name: dynamo-router, version: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1" } + kv-p2p-transfer: nixl + multinode: true + disagg: true + scenarios: + agentic-coding: + - dram-utilization: 0.75 + search-space: + - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: { name: mooncake, version: "0.3.11.post1" } + conc-list: [48] + prefill: + num-worker: 1 + tp: 8 + dcp-size: 8 + ep: 1 + dp-attn: false + additional-settings: + - "SYNTHETIC_ACCEPTANCE=true" + - "SYNTHETIC_ACCEPTANCE_LENGTH=3.36" + - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/disagg-b300-1p1d-dcp8-dcp8-dspark4-mooncake-c48-agentic.yaml" + decode: { num-worker: 1, tp: 8, dcp-size: 8, ep: 1, dp-attn: false } + - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: { name: mooncake, version: "0.3.11.post1" } + conc-list: [48] + prefill: + num-worker: 1 + tp: 8 + dcp-size: 8 + ep: 1 + dp-attn: false + additional-settings: + - "SYNTHETIC_ACCEPTANCE=true" + - "SYNTHETIC_ACCEPTANCE_LENGTH=3.36" + - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/disagg-b300-1p2d-dcp8-dcp8-dspark4-mooncake-c48-agentic.yaml" + decode: { num-worker: 2, tp: 8, dcp-size: 8, ep: 1, dp-attn: false } + - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: { name: mooncake, version: "0.3.11.post1" } + conc-list: [32] + prefill: + num-worker: 1 + tp: 8 + dcp-size: 8 + ep: 1 + dp-attn: false + additional-settings: + - "SYNTHETIC_ACCEPTANCE=true" + - "SYNTHETIC_ACCEPTANCE_LENGTH=3.36" + - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/disagg-b300-1p3d-dcp8-dcp8-dspark4-mooncake-c32-agentic.yaml" + decode: { num-worker: 3, tp: 8, dcp-size: 8, ep: 1, dp-attn: false } + # Kimi-K3 MXFP4 B200 aggregated vLLM (TP8 x PP2, 2 nodes / 16 GPUs), agentic # coding. The native MXFP4 checkpoint (2.8T total params, ~1.4TB weights) does # not fit one 8xB200 node, so TP8 shards attention/dense and PP2 splits layers. diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 14434c76ad..da7de53ee6 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6878,3 +6878,16 @@ description: - "Refresh the Kimi-K3 GB300 AgentX configurations with updated configs." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2811 + +- config-keys: + - kimik3-fp4-b300-dynamo-vllm-agentic-mooncake-dcp8-agg + - kimik3-fp4-b300-dynamo-vllm-agentic-dspark-mooncake-dcp8-agg + - kimik3-fp4-b300-dynamo-vllm-agentic-dspark-mooncake-dcp8-disagg + scenario-type: + - agentic-coding + description: + - "Add the six measured Kimi-K3 B300 AgentX points: aggregated TP8 x DCP8 without speculative decoding at c70, aggregated DSpark-7 at c4 and c1, and 1P1D, 1P2D and 1P3D DCP8 x DCP8 DSpark-4 at c48, c48 and c32." + - "Give Kimi-K3 a model path on b300-dsxe and route the agentic scenario to upstream srt-slurm with the in-repo recipes, without which multi-node Kimi-K3 exits as an unsupported model." + - "Apply synthetic acceptance on throughput runs, not only eval-only ones, so the pinned acceptance length reaches speculative-config as it already does on gb300-nv." + - "Target cluster:b300-dsxe, whose compute fabric is EFA rather than RoCE: drop the pinned transfer devices and RoCE GID index, select the srd transport, and check the adapter inventory before the engine starts." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2814 diff --git a/runners/launch_b300-dsxe.sh b/runners/launch_b300-dsxe.sh index c229146bfa..a85090e9a3 100755 --- a/runners/launch_b300-dsxe.sh +++ b/runners/launch_b300-dsxe.sh @@ -277,10 +277,13 @@ fi # Override the job name in the recipe with the runner name. sed -i "s/^name:.*/name: \"${RUNNER_NAME}\"/" "$CONFIG_PATH" -if [[ "${EVAL_ONLY:-false}" == "true" ]]; then - python3 "$GITHUB_WORKSPACE/runners/inject_synthetic_acceptance.py" \ - "$CONFIG_PATH" "$FRAMEWORK" || exit 1 -fi +# Call this unconditionally, as launch_gb300-nv.sh does. The helper returns +# immediately when neither EVAL_ONLY nor SYNTHETIC_ACCEPTANCE is set. Gating it +# on EVAL_ONLY meant throughput runs never got SYNTHETIC_ACCEPTANCE_LENGTH into +# speculative-config, so those arms drafted for real and their numbers were not +# comparable to the gb300-nv runs they exist to be compared against. +python3 "$GITHUB_WORKSPACE/runners/inject_synthetic_acceptance.py" \ + "$CONFIG_PATH" "$FRAMEWORK" || exit 1 # Weights live on node-local MODEL_ROOT, which this login host cannot stat, so # srtctl's preflight model.path check is always skipped. Runtime loading still