Update Kimi K3 B300 AgentX / 更新 Kimi K3 B300 AgentX

benchmarks/multi_node/srt-slurm-recipes/configs/kimik3-b300-fabric-check.sh

@@ -0,0 +1,61 @@
+#!/usr/bin/env bash
+# Report this compute node's EFA inventory and GPU affinity before the engine starts.
+#
+# b300-dsxe's compute fabric is EFA, not RoCE: 16 adapters (driver efa, 400 Gb
+# each, two per GPU) that present no netdev and no IPv4. There are no per-rail
+# subnets, so there is nothing to pin and nothing to compare subnets against --
+# the recipes leave device selection to the fabric.
+#
+# What is worth checking is that the node has its full complement. A node
+# missing adapters still runs, just slower and asymmetrically against its peers,
+# which is the kind of thing that shows up as an unexplained outlier rather than
+# an error. Everything here is diagnostic; nothing fails the job.
+
+set -uo pipefail
+
+EXPECTED_EFA=16
+
+echo "=== [b300-fabric] node: $(hostname) ==="
+
+mapfile -t EFA_DEVS < <(
+    for d in /sys/class/infiniband/*; do
+        [ -e "$d" ] || continue
+        [ "$(basename "$(readlink -f "$d/device/driver")" 2>/dev/null)" = efa ] || continue
+        basename "$d"
+    done | sort
+)
+
+echo "--- EFA adapters: ${#EFA_DEVS[@]} (expected $EXPECTED_EFA) ---"
+for dev in "${EFA_DEVS[@]}"; do
+    state=$(awk '{print $2}' "/sys/class/infiniband/$dev/ports/1/state" 2>/dev/null)
+    rate=$(awk '{print $1}' "/sys/class/infiniband/$dev/ports/1/rate" 2>/dev/null)
+    pci=$(basename "$(readlink -f "/sys/class/infiniband/$dev/device")" 2>/dev/null)
+    printf '  %-12s %-10s %sGb  %s\n' "$dev" "${state:-?}" "${rate:-?}" "${pci:-?}"
+done
+
+if [ "${#EFA_DEVS[@]}" -ne "$EXPECTED_EFA" ]; then
+    echo "[b300-fabric] WARNING: ${#EFA_DEVS[@]} EFA adapters, expected $EXPECTED_EFA."
+    echo "[b300-fabric] This node will move KV more slowly than its peers."
+fi
+
+down=0
+for dev in "${EFA_DEVS[@]}"; do
+    state=$(awk '{print $2}' "/sys/class/infiniband/$dev/ports/1/state" 2>/dev/null)
+    [ "$state" = ACTIVE ] || { echo "[b300-fabric] WARNING: $dev is $state, not ACTIVE."; down=$((down + 1)); }
+done
+[ "$down" -eq 0 ] && echo "[b300-fabric] all ${#EFA_DEVS[@]} adapters ACTIVE."
+
+echo "--- GPU / EFA affinity (by PCI bus) ---"
+nvidia-smi --query-gpu=index,pci.bus_id --format=csv,noheader 2>/dev/null | while IFS=, read -r idx bus; do
+    gbus=$(echo "$bus" | tr -d ' ' | cut -d: -f2 | tr 'A-Z' 'a-z')
+    near=""
+    for dev in "${EFA_DEVS[@]}"; do
+        pci=$(basename "$(readlink -f "/sys/class/infiniband/$dev/device")" 2>/dev/null)
+        nbus=$(echo "$pci" | cut -d: -f2)
+        d=$(( 16#$gbus - 16#$nbus )); [ "$d" -lt 0 ] && d=$(( -d ))
+        [ "$d" -le 2 ] && near="$near $dev"
+    done
+    printf '  GPU%s (bus %s):%s\n' "$idx" "$gbus" "${near:- none}"
+done
+
+echo "=== [b300-fabric] done ==="

benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b300-dcp8-dspark7-maxseq2-mooncake-c1-agentic.yaml

@@ -0,0 +1,150 @@
+# Kimi-K3 B300 aggregated TP8 x DCP8, agentic coding.
+#
+# Translated from the bia sweep config that measured this point:
+#   AGG/kimi-k3-mxfp4-b300-agg-tp8-agentx-062126-c1-weiaggds7-gmu94.yml
+# Engine flags, environment, Mooncake settings and vllm_config are that file
+# verbatim. The InferenceX plumbing -- identity, benchmark, slurm, the idle-GPU
+# reaper comment -- is taken from agg-gb300-dcp8-dspark7-maxseq2-mooncake-agentic.yaml.
+#
+# Only these fields differ from the measured config, each because b300-nv is
+# not bia:
+#   device_name, UCX_NET_DEVICES -> dropped; see the fabric note below
+#   MC_GID_INDEX, UCX_TLS        -> GID index dropped, UCX_TLS uses srd (below)
+#   HF_HOME                      -> dropped; benchmark.env sets HF_HUB_CACHE
+#   slurm.time_limit             -> 06:00:00 (see the note in the tool)
+#   rejection_sample_method      -> block, so the eval verifies (see below)
+#   draft_sample_method          -> probabilistic, matching the AL's curve
+#   global_segment_size          -> 281GB, matching the B300 arm on main
+#
+# The fabric is the part that needs saying. b300-dsxe is not RoCE: the compute
+# fabric is 16 EFA adapters per node (driver efa, 400 Gb each, two per GPU),
+# presenting no netdev and no IPv4, so there are no per-rail subnets to name and
+# the mlx5 device names the measured config used do not exist here. The
+# inventory was identical on every node checked, so there is also no unstable
+# index to pin around -- which was the reason the measured config named rails at
+# all. Devices are therefore left to the fabric, and UCX_TLS asks for srd, EFA's
+# transport, because EFA does not implement the verbs RC that rc requires.
+
+name: kimi-k3-vllm-agg-b300-dcp8-dspark7-maxseq2-mooncake-c1-agentic
+model:
+  path: moonshotai/Kimi-K3
+  container: vllm/vllm-openai:nightly-dev-x86_64-cu13-3696c77
+  precision: fp4
+resources:
+  gpu_type: b300
+  gpus_per_node: 8
+  agg_nodes: 1
+  agg_workers: 1
+  gpus_per_agg: 8
+infra:
+  etcd_nats_dedicated_node: false
+  nats_max_payload_mb: 32
+frontend:
+  type: dynamo
+  enable_multiple_frontends: false
+  args:
+    router-mode: least-loaded
+    router-session-affinity-ttl-secs: 900
+  env:
+    DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: '3600'
+    DYN_TOKENIZER_CACHE_BYTES: '8589934592'
+    PYTHONPYCACHEPREFIX: /tmp/vllm-pycache-kimi-k3
+backend:
+  type: vllm
+  connector: null
+  mooncake_kv_store:
+    master_extra_args:
+    - --default_kv_lease_ttl=60000
+    - --eviction_high_watermark_ratio=0.95
+    - --eviction_ratio=0.10
+    store_config:
+      metadata_server: P2PHANDSHAKE
+      global_segment_size: 281GB
+      local_buffer_size: 4GB
+      protocol: rdma
+      mode: embedded
+      enable_offload: false
+  aggregated_environment:
+    VLLM_ALLREDUCE_USE_FLASHINFER: '1'
+    VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: '1'
+    VLLM_USE_DIRECT_DCP_A2A: '1'
+    VLLM_USE_DIRECT_DCP_Q_GATHER: '1'
+    VLLM_USE_DIRECT_DCP_KV_GATHER: '1'
+    ETCD_LEASE_TTL: '120'
+    VLLM_ENGINE_READY_TIMEOUT_S: '3600'
+    VLLM_RPC_TIMEOUT: '600000'
+    VLLM_USE_V2_MODEL_RUNNER: '1'
+    VLLM_USE_RUST_FRONTEND: '1'
+    VLLM_MOONCAKE_LOAD_RECV_THREADS: '4'
+    VLLM_PREFIX_CACHE_RETENTION_INTERVAL: '0'
+    VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: '0'
+    MC_SLICE_SIZE: '1048576'
+    WITH_NVIDIA_PEERMEM: '0'
+    VLLM_LOG_STATS_INTERVAL: '1'
+    PYTHONHASHSEED: '42'
+    PYTHONNOUSERSITE: '1'
+    PYTHONUNBUFFERED: '1'
+    TORCH_CUDA_ARCH_LIST: '10.0'
+    UCX_TLS: srd,cuda_copy,cuda_ipc,sm,self
+  vllm_config:
+    aggregated:
+      kv-transfer-config: '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"compact_group_io":true,"max_load_batch_keys":2,"enable_offload":false}}'
+      served-model-name: moonshotai/Kimi-K3
+      load-format: fastsafetensors
+      kv-cache-dtype: fp8
+      gpu-memory-utilization: 0.94
+      tensor-parallel-size: 8
+      decode-context-parallel-size: 8
+      dcp-comm-backend: a2a
+      max-num-seqs: 2
+      max-num-batched-tokens: 8192
+      trust-remote-code: true
+      max-cudagraph-capture-size: 1024
+      stream-interval: 10
+      language-model-only: true
+      moe-backend: auto
+      no-enable-flashinfer-autotune: true
+      enable-cumem-allocator: true
+      attention-backend: TOKENSPEED_MLA
+      attention-config: '{"mla_prefill_backend":"TRTLLM_RAGGED","use_prefill_query_quantization":true}'
+      speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","attention_backend":"TOKENSPEED_MLA","method":"dspark","num_speculative_tokens":7,"rejection_sample_method":"block","draft_sample_method":"probabilistic"}'
+      enable-prefix-caching: true
+      prefix-match-unit: 128
+health_check:
+  max_attempts: 720
+  interval_seconds: 10
+benchmark:
+  type: custom
+  command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh
+  env:
+    INFMAX_CONTAINER_WORKSPACE: /infmax-workspace
+    AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: '300'
+    AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: '0.25'
+    AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'vllm:'
+    AIPERF_HTTP_TCP_USER_TIMEOUT: '900000'
+    RESULT_DIR: /logs/agentic
+    PORT: '8000'
+    IS_MULTINODE: 'true'
+    AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0'
+    AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache
+    HF_HUB_CACHE: /hf_hub_cache
+    WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126
+dynamo:
+  hash: ba83080ecd31c1ce918559e576d3c5bc9e092ff1
+  install: true
+identity:
+  model:
+    repo: moonshotai/Kimi-K3
+  container:
+    image: vllm/vllm-openai:nightly-dev-x86_64-cu13-3696c77
+  frameworks:
+    dynamo: ba83080ecd31c1ce918559e576d3c5bc9e092ff1
+slurm:
+  time_limit: 06:00:00
+sbatch_directives:
+  mem: '0'
+  cpus-per-task: '72'
+  comment: '''{"OccupiedIdleGPUsJobReaper":{"exemptIdleTimeMins":"60","reason":"model_loading","description":"Very large model will take extra loading time."}}'''
+srun_options:
+  mem: '0'
+  container-remap-root: ''

benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b300-dcp8-dspark7-maxseq2-mooncake-c4-agentic.yaml

@@ -0,0 +1,150 @@
+# Kimi-K3 B300 aggregated TP8 x DCP8, agentic coding.
+#
+# Translated from the bia sweep config that measured this point:
+#   AGG/kimi-k3-mxfp4-b300-agg-tp8-agentx-062126-c4-weiaggds7-gmu94.yml
+# Engine flags, environment, Mooncake settings and vllm_config are that file
+# verbatim. The InferenceX plumbing -- identity, benchmark, slurm, the idle-GPU
+# reaper comment -- is taken from agg-gb300-dcp8-dspark7-maxseq2-mooncake-agentic.yaml.
+#
+# Only these fields differ from the measured config, each because b300-nv is
+# not bia:
+#   device_name, UCX_NET_DEVICES -> dropped; see the fabric note below
+#   MC_GID_INDEX, UCX_TLS        -> GID index dropped, UCX_TLS uses srd (below)
+#   HF_HOME                      -> dropped; benchmark.env sets HF_HUB_CACHE
+#   slurm.time_limit             -> 06:00:00 (see the note in the tool)
+#   rejection_sample_method      -> block, so the eval verifies (see below)
+#   draft_sample_method          -> probabilistic, matching the AL's curve
+#   global_segment_size          -> 281GB, matching the B300 arm on main
+#
+# The fabric is the part that needs saying. b300-dsxe is not RoCE: the compute
+# fabric is 16 EFA adapters per node (driver efa, 400 Gb each, two per GPU),
+# presenting no netdev and no IPv4, so there are no per-rail subnets to name and
+# the mlx5 device names the measured config used do not exist here. The
+# inventory was identical on every node checked, so there is also no unstable
+# index to pin around -- which was the reason the measured config named rails at
+# all. Devices are therefore left to the fabric, and UCX_TLS asks for srd, EFA's
+# transport, because EFA does not implement the verbs RC that rc requires.
+
+name: kimi-k3-vllm-agg-b300-dcp8-dspark7-maxseq2-mooncake-c4-agentic
+model:
+  path: moonshotai/Kimi-K3
+  container: vllm/vllm-openai:nightly-dev-x86_64-cu13-3696c77
+  precision: fp4
+resources:
+  gpu_type: b300
+  gpus_per_node: 8
+  agg_nodes: 1
+  agg_workers: 1
+  gpus_per_agg: 8
+infra:
+  etcd_nats_dedicated_node: false
+  nats_max_payload_mb: 32
+frontend:
+  type: dynamo
+  enable_multiple_frontends: false
+  args:
+    router-mode: least-loaded
+    router-session-affinity-ttl-secs: 900
+  env:
+    DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: '3600'
+    DYN_TOKENIZER_CACHE_BYTES: '8589934592'
+    PYTHONPYCACHEPREFIX: /tmp/vllm-pycache-kimi-k3
+backend:
+  type: vllm
+  connector: null
+  mooncake_kv_store:
+    master_extra_args:
+    - --default_kv_lease_ttl=60000
+    - --eviction_high_watermark_ratio=0.95
+    - --eviction_ratio=0.10
+    store_config:
+      metadata_server: P2PHANDSHAKE
+      global_segment_size: 281GB
+      local_buffer_size: 4GB
+      protocol: rdma
+      mode: embedded
+      enable_offload: false
+  aggregated_environment:
+    VLLM_ALLREDUCE_USE_FLASHINFER: '1'
+    VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: '1'
+    VLLM_USE_DIRECT_DCP_A2A: '1'
+    VLLM_USE_DIRECT_DCP_Q_GATHER: '1'
+    VLLM_USE_DIRECT_DCP_KV_GATHER: '1'
+    ETCD_LEASE_TTL: '120'
+    VLLM_ENGINE_READY_TIMEOUT_S: '3600'
+    VLLM_RPC_TIMEOUT: '600000'
+    VLLM_USE_V2_MODEL_RUNNER: '1'
+    VLLM_USE_RUST_FRONTEND: '1'
+    VLLM_MOONCAKE_LOAD_RECV_THREADS: '4'
+    VLLM_PREFIX_CACHE_RETENTION_INTERVAL: '0'
+    VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: '0'
+    MC_SLICE_SIZE: '1048576'
+    WITH_NVIDIA_PEERMEM: '0'
+    VLLM_LOG_STATS_INTERVAL: '1'
+    PYTHONHASHSEED: '42'
+    PYTHONNOUSERSITE: '1'
+    PYTHONUNBUFFERED: '1'
+    TORCH_CUDA_ARCH_LIST: '10.0'
+    UCX_TLS: srd,cuda_copy,cuda_ipc,sm,self
+  vllm_config:
+    aggregated:
+      kv-transfer-config: '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"compact_group_io":true,"max_load_batch_keys":2,"enable_offload":false}}'
+      served-model-name: moonshotai/Kimi-K3
+      load-format: fastsafetensors
+      kv-cache-dtype: fp8
+      gpu-memory-utilization: 0.94
+      tensor-parallel-size: 8
+      decode-context-parallel-size: 8
+      dcp-comm-backend: a2a
+      max-num-seqs: 2
+      max-num-batched-tokens: 8192
+      trust-remote-code: true
+      max-cudagraph-capture-size: 1024
+      stream-interval: 10
+      language-model-only: true
+      moe-backend: auto
+      no-enable-flashinfer-autotune: true
+      enable-cumem-allocator: true
+      attention-backend: TOKENSPEED_MLA
+      attention-config: '{"mla_prefill_backend":"TRTLLM_RAGGED","use_prefill_query_quantization":true}'
+      speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","attention_backend":"TOKENSPEED_MLA","method":"dspark","num_speculative_tokens":7,"rejection_sample_method":"block","draft_sample_method":"probabilistic"}'
+      enable-prefix-caching: true
+      prefix-match-unit: 128
+health_check:
+  max_attempts: 720
+  interval_seconds: 10
+benchmark:
+  type: custom
+  command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh
+  env:
+    INFMAX_CONTAINER_WORKSPACE: /infmax-workspace
+    AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: '300'
+    AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: '0.25'
+    AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'vllm:'
+    AIPERF_HTTP_TCP_USER_TIMEOUT: '900000'
+    RESULT_DIR: /logs/agentic
+    PORT: '8000'
+    IS_MULTINODE: 'true'
+    AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0'
+    AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache
+    HF_HUB_CACHE: /hf_hub_cache
+    WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126
+dynamo:
+  hash: ba83080ecd31c1ce918559e576d3c5bc9e092ff1
+  install: true
+identity:
+  model:
+    repo: moonshotai/Kimi-K3
+  container:
+    image: vllm/vllm-openai:nightly-dev-x86_64-cu13-3696c77
+  frameworks:
+    dynamo: ba83080ecd31c1ce918559e576d3c5bc9e092ff1
+slurm:
+  time_limit: 06:00:00
+sbatch_directives:
+  mem: '0'
+  cpus-per-task: '72'
+  comment: '''{"OccupiedIdleGPUsJobReaper":{"exemptIdleTimeMins":"60","reason":"model_loading","description":"Very large model will take extra loading time."}}'''
+srun_options:
+  mem: '0'
+  container-remap-root: ''