diff --git a/.github/workflows/run-sweep.yml b/.github/workflows/run-sweep.yml index c2d1445b24..9be5c651b6 100644 --- a/.github/workflows/run-sweep.yml +++ b/.github/workflows/run-sweep.yml @@ -109,7 +109,7 @@ jobs: ALL_EVALS: ${{ contains(github.event.pull_request.labels.*.name, 'all-evals') }} EVALS_ONLY: ${{ contains(github.event.pull_request.labels.*.name, 'evals-only') }} run: | - pip install pydantic pyyaml + pip install "pydantic>=2" pyyaml CMD=( python3 utils/validate_perf_changelog.py --changelog-file perf-changelog.yaml diff --git a/.github/workflows/test-changelog-gate.yml b/.github/workflows/test-changelog-gate.yml index c1f11f34f8..65bd1eebc5 100644 --- a/.github/workflows/test-changelog-gate.yml +++ b/.github/workflows/test-changelog-gate.yml @@ -68,7 +68,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - python -m pip install pytest pydantic pyyaml tabulate + python -m pip install pytest "pydantic>=2" pyyaml tabulate - name: Run validator, reuse, and gating tests run: | diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/mi355x-fp4/8k1k/disagg-1p1d-tp4.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/mi355x-fp4/8k1k/disagg-1p1d-tp4.yaml new file mode 100644 index 0000000000..4055a2961c --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/mi355x-fp4/8k1k/disagg-1p1d-tp4.yaml @@ -0,0 +1,147 @@ +# Compatibility validation of the archived MI355X MiniMax-M3 FP4 recipe. +# Preserve the original TP4/TP4 serving and benchmark settings. srt-slurm +# owns allocation, native vLLM Router discovery, health, logs, and teardown. +name: mi355x-vllm-minimaxm3-disagg-1p1d-tp4-8k1k + +model: + # Same provisioned checkpoint selected by the retired AMD launcher. + path: /it-share/data/MiniMax-M3-MXFP4 + container: vllm/vllm-openai-rocm:v0.28.0 + precision: fp4 + +identity: + model: + repo: amd/MiniMax-M3-MXFP4 + container: + image: vllm/vllm-openai-rocm:v0.28.0 + frameworks: + vllm-router: 1d10e71fb7bb4c0adc9f2c16ec77bf5dd4aa1586 + +slurm: + time_limit: "08:00:00" + +resources: + gpu_type: mi355x + gpus_per_node: 8 + prefill_nodes: 1 + decode_nodes: 1 + prefill_workers: 1 + decode_workers: 1 + gpus_per_prefill: 4 + gpus_per_decode: 4 + +frontend: + type: vllm-router + container_image: vllm/vllm-router:nightly-20260903-1d10e71 + enable_multiple_frontends: false + args: + policy: consistent_hash + prefill-policy: consistent_hash + decode-policy: consistent_hash + log-level: info + +environment: &runtime_environment + HF_HOME: /hf_hub_cache + HF_HUB_CACHE: /hf_hub_cache + HUGGINGFACE_HUB_CACHE: /hf_hub_cache + PYTHONDONTWRITEBYTECODE: "1" + PYTHONUNBUFFERED: "1" + GLOO_SOCKET_IFNAME: eno0 + NCCL_SOCKET_IFNAME: eno0 + NCCL_IB_HCA: rdma0,rdma1,rdma2,rdma3,rdma4,rdma5,rdma6,rdma7 + MORI_RDMA_TC: "104" + MORI_IO_SQ_BACKOFF_TIMEOUT_US: "50000" + MORI_IO_QP_MAX_SEND_WR: "16384" + MORI_IO_QP_MAX_CQE: "32768" + MORI_IO_QP_MAX_SGE: "2" + MORI_IO_TC_DISABLE: "0" + VLLM_USE_V1: "1" + VLLM_SERVER_DEV_MODE: "0" + VLLM_DISABLE_REQUEST_ID_RANDOMIZATION: "1" + VLLM_ROCM_USE_AITER: "1" + VLLM_ROCM_USE_AITER_MOE: "1" + VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS: "1" + VLLM_USE_BREAKABLE_CUDAGRAPH: "0" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + +backend: + type: vllm + connector: moriio + prefill_environment: + VLLM_ROCM_QUICK_REDUCE_QUANTIZATION: INT4 + VLLM_ROCM_QUICK_REDUCE_MAX_SIZE_BYTES_MB: "2048" + vllm_config: + prefill: &server_options + tensor-parallel-size: 4 + served-model-name: MiniMax-M3-MXFP4 + max-num-batched-tokens: 32768 + max-num-seqs: 512 + block-size: 128 + language-model-only: true + attention-backend: TRITON_ATTN + moe-backend: aiter + no-enable-prefix-caching: true + gpu-memory-utilization: 0.90 + tool-call-parser: minimax_m3 + reasoning-parser: minimax_m3 + enable-auto-tool-choice: true + trust-remote-code: true + decode: *server_options + +# Preserve the original exclusive two-node allocation, although each worker +# uses only four GPUs. Do not pack the roles onto one node. +sbatch_directives: + cpus-per-task: "128" + mem: "0" + exclusive: "" + +srun_options: + container-writable: "" + container-remap-root: "" + mem: "0" + +health_check: + max_attempts: 720 + interval_seconds: 5 + +benchmark: + type: custom + command: | + set -euo pipefail + result_root="/results/${SLURM_JOB_ID}" + mkdir -p "${result_root}/fixed-seq" + archive_runtime_logs() { + tar -C /logs -czf "${result_root}/runtime-logs.tar.gz" . 2>/dev/null || true + } + trap archive_runtime_logs EXIT + for concurrency in ${CONC_LIST}; do + num_prompts=$((concurrency * 10)) + if (( num_prompts < 16 )); then num_prompts=16; fi + python3 /infmax-workspace/utils/bench_serving/benchmark_serving.py \ + --backend openai \ + --base-url "http://${SRT_FRONTEND_HOST}:${SRT_FRONTEND_PORT}" \ + --endpoint /v1/completions \ + --model MiniMax-M3-MXFP4 \ + --tokenizer /model \ + --dataset-name random \ + --random-input-len 8192 \ + --random-output-len 1024 \ + --random-range-ratio "${RANDOM_RANGE_RATIO:-0.8}" \ + --num-warmups "$((concurrency * 2))" \ + --num-prompts "${num_prompts}" \ + --max-concurrency "${concurrency}" \ + --request-rate inf \ + --ignore-eos \ + --trust-remote-code \ + --percentile-metrics ttft,tpot,itl,e2el \ + --save-result \ + --result-dir "${result_root}/fixed-seq" \ + --result-filename "minimaxm3-vllm-mi355x-disagg-1p1d-tp4-isl8192-osl1024-c${concurrency}.json" + # The original vLLM benchmark allowed the idle KV reaper ten seconds. + sleep 10 + done + env: + <<: *runtime_environment + MODEL_NAME: MiniMax-M3-MXFP4 + TRANSFORMERS_VERBOSITY: error + TOKENIZERS_PARALLELISM: "false" diff --git a/benchmarks/single_node/agentic/kimik3_fp4_mi355x_mtp.sh b/benchmarks/single_node/agentic/kimik3_fp4_mi355x_mtp.sh index a8f2c42c1c..f0416e4af0 100644 --- a/benchmarks/single_node/agentic/kimik3_fp4_mi355x_mtp.sh +++ b/benchmarks/single_node/agentic/kimik3_fp4_mi355x_mtp.sh @@ -35,10 +35,10 @@ set -x # GPU_MEM_UTIL 0.95 (reference) # MAX_NUM_BATCHED_TOKENS 8192 (default) # AITER_A8W4 1 (reference; 0 = aiter a16w4 MoE path) -# LANGUAGE_MODEL_ONLY true +# LANGUAGE_MODEL_ONLY true # KV_CACHE_DTYPE fp8 (default for every arm; =auto for a bf16 A/B) # KV_BLOCK_SIZE unset (unset -> vLLM sizes the page; 128 under fp8) -# MAX_MODEL_LEN 1M +# MAX_MODEL_LEN 1M # SPEC_DECODE true (this is the _mtp DSpark recipe; =false for a no-spec A/B) # SPEC_NUM_TOKENS 2 (DSpark draft length; validated by the _mtp config) @@ -149,10 +149,9 @@ case "${KV_OFFLOAD_BACKEND:-}" in lmcache) require_agentic_kv_offload_backend "$KV_OFFLOAD_BACKEND" - # Keep the image's tested torch/ROCm stack and install only LMCache's - # missing runtime dependencies, same as the MiniMax-M3 lmcache arm. - LMCACHE_VERSION="0.5.5.dev60+rocm7.2" + LMCACHE_VERSION=0.5.5.dev89+rocm7.2 LMCACHE_ROCM_INDEX="https://github.com/LMCache/LMCache/releases/expanded_assets/nightly-rocm" + agentic_pip_install --quiet --no-cache-dir --no-deps \ "sortedcontainers==2.4.0" \ "opentelemetry-exporter-prometheus==0.61b0" \ @@ -197,6 +196,18 @@ case "${KV_OFFLOAD_BACKEND:-}" in LMCACHE_L1_SIZE_GB="$TOTAL_CPU_DRAM_GB" + # DCP shards decode KV across the TP ranks, so the LMCache GPU transfer + # pool needs one worker per rank; a non-DCP arm only needs a single worker. + # The DCP KV interleave also needs the larger 12288 chunk; a non-DCP arm + # uses the 3072 minimum (one KDA state group). + if [ "${DCP_SIZE:-1}" -gt 1 ]; then + LMCACHE_MAX_GPU_WORKERS=8 + LMCACHE_CHUNK_SIZE=12288 + else + LMCACHE_MAX_GPU_WORKERS=1 + LMCACHE_CHUNK_SIZE=3072 + fi + LMCACHE_CMD=( lmcache server --host 127.0.0.1 @@ -205,12 +216,12 @@ case "${KV_OFFLOAD_BACKEND:-}" in --http-port "$LMCACHE_HTTP_PORT" --l1-size-gb "$LMCACHE_L1_SIZE_GB" --l1-init-size-gb 10 - --chunk-size 3072 + --chunk-size "$LMCACHE_CHUNK_SIZE" --separate-object-groups --enable-extra-logging --extra-logging-interval 30 --max-cpu-workers 8 - --max-gpu-workers 1 + --max-gpu-workers "$LMCACHE_MAX_GPU_WORKERS" --eviction-policy LRU --supported-transfer-mode lmcache_driven --shm-name "" @@ -256,16 +267,21 @@ case "$CONC" in GPU_MEM_UTIL=0.9 MAX_NUM_BATCHED_TOKENS=16384 ;; - 2|4|8|10|12|14) + 4|8|10|12|14) SYNTHETIC_ACCEPT_LEN=3.00 SPEC_NUM_TOKENS=3 GPU_MEM_UTIL=0.9 MAX_NUM_BATCHED_TOKENS=8192 ;; + 44|48|52) + SPEC_NUM_TOKENS=0 + GPU_MEM_UTIL=0.9 + MAX_NUM_BATCHED_TOKENS=8192 + ;; *) SPEC_NUM_TOKENS=0 - GPU_MEM_UTIL=0.85 - MAX_NUM_BATCHED_TOKENS=4096 + GPU_MEM_UTIL=0.9 + MAX_NUM_BATCHED_TOKENS=8192 ;; esac @@ -306,7 +322,7 @@ CP_ARGS=() ATTN_BE_ARGS=() if [ "$DCP_SIZE" -gt 1 ]; then CP_ARGS+=(--decode-context-parallel-size "$DCP_SIZE" --dcp-comm-backend a2a) - ATTN_BE_ARGS+=(--attention-backend TRITON_MLA) + ATTN_BE_ARGS+=(--attention-backend ROCM_AITER_MLA) fi export VLLM_USE_DIRECT_DCP_A2A=0 export VLLM_USE_DIRECT_DCP_Q_GATHER=0 diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 0d5dbbec00..db323f0c09 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -395,7 +395,7 @@ dsr1-fp8-mi355x-atom-mtp: search-space: - { tp: 8, conc-start: 4, conc-end: 256, spec-decoding: mtp } kimik3-fp4-mi355x-vllm-agentic-mtp: - image: vllm/vllm-openai-rocm:nightly-1dc464d42681d22f38caf1fdc1eb632dc4421c45 + image: vllm/vllm-openai-rocm:nightly-7c5dc571cbd1064ecc8a9b1045637ff647aa22cb model: moonshotai/Kimi-K3 model-prefix: kimik3 runner: cluster:mi355x-amds @@ -406,8 +406,9 @@ kimik3-fp4-mi355x-vllm-agentic-mtp: agentic-coding: - dram-utilization: 0.60 search-space: - - { tp: 8, kv-offloading: none, conc-list: [1] , spec-decoding: mtp} - - { tp: 8, ep: 1, kv-offloading: dram, kv-offload-backend: { name: lmcache, version: "0.5.5.dev60+rocm7.2" }, conc-list: [4, 8, 10, 12, 14], spec-decoding: mtp } + - { tp: 8, ep: 1, dcp-size: 1, kv-offloading: none, conc-list: [1] , spec-decoding: mtp} + - { tp: 8, ep: 1, dcp-size: 1, kv-offloading: dram, kv-offload-backend: { name: lmcache, version: "0.5.5.dev89+rocm7.2" }, conc-list: [4, 8, 10, 12, 14], spec-decoding: mtp } + - { tp: 8, ep: 1, dcp-size: 8, kv-offloading: dram, kv-offload-backend: { name: lmcache, version: "0.5.5.dev89+rocm7.2" }, conc-list: [44, 48, 52], spec-decoding: mtp } # Kimi-K3 MXFP4 agentic-coding benchmark on MI355X via ATOM with DSpark # speculative decoding (2 draft tokens -> golden AL 2.51, diff --git a/configs/deprecated/amd-minimaxm3-8k1k-master.yaml b/configs/deprecated/amd-minimaxm3-8k1k-master.yaml index c8f5386738..8ffc4aff28 100644 --- a/configs/deprecated/amd-minimaxm3-8k1k-master.yaml +++ b/configs/deprecated/amd-minimaxm3-8k1k-master.yaml @@ -42,6 +42,43 @@ minimaxm3-fp8-mi355x-vllm-mtp: osl: 1024 search-space: - { tp: 4, conc-start: 1, conc-end: 512, spec-decoding: mtp } +# Archived compatibility lane; opt in through e2e-tests.yml, not active sweeps. +# The old AMD launch infrastructure is removed by the shared base PR. +minimaxm3-fp4-mi355x-vllm-disagg: + image: vllm/vllm-openai-rocm:v0.28.0 + model: amd/MiniMax-M3-MXFP4 + model-prefix: minimaxm3 + runner: cluster:mi355x-amds + precision: fp4 + framework: vllm-disagg + router: { name: vllm-router, version: "1d10e71fb7bb4c0adc9f2c16ec77bf5dd4aa1586" } + kv-p2p-transfer: moriio + multinode: true + disagg: true + scenarios: + fixed-seq-len: + - isl: 8192 + osl: 1024 + search-space: + - spec-decoding: "none" + conc-list: [ 1, 2, 4, 8, 16, 32, 64, 128, 256 ] + prefill: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/vllm/minimax-m3/mi355x-fp4/8k1k/disagg-1p1d-tp4.yaml" + - "SRT_SLURM_COMMIT=33a3d811b7956973e4ef78b1d0cf4a5d56dad6a1" + decode: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/vllm/minimax-m3/mi355x-fp4/8k1k/disagg-1p1d-tp4.yaml" + - "SRT_SLURM_COMMIT=33a3d811b7956973e4ef78b1d0cf4a5d56dad6a1" + minimaxm3-fp4-mi355x-vllm: image: vllm/vllm-openai-rocm:nightly-69715823df89b11ee684b84066390cbb9092d5c1 model: amd/MiniMax-M3-MXFP4 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index a0f56d8a79..c54a65fa9e 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6832,6 +6832,16 @@ - "Isolate SGLang runtime caches per Slurm allocation to prevent concurrent sweep cells from sharing per-rank cache files." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2808 +- config-keys: + - kimik3-fp4-mi355x-vllm-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Bump the MI355X K3 vLLM recipe to vLLM ROCm nightly 7c5dc571 and LMCache 0.5.5rc3." + - "Add a TP8/DCP8 spec-mtp DRAM-offload arm at concurrency 44/48/52." + - "Retune the LMCache tier (12288-token chunk, 8 GPU workers) and the no-spec decode profile (gpu-memory-utilization 0.9, 8192 batched tokens); switch the DCP attention backend to ROCM_AITER_MLA." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2810 + - config-keys: - glm5.2-fp4-gb300-dynamo-trt-agentic-disagg-mtp scenario-type: @@ -6885,3 +6895,22 @@ - "Remove the mandatory host-repair hook, RDMA/GPU-drain helper scripts, and separate image/model staging allocation; use provisioned caches and native Pyxis container lifecycle." - "Keep InferenceX-specific benchmark/eval metadata and artifact collection in a portable adapter while preserving the existing serving and workload settings." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2542 + +- config-keys: + - minimaxm3-fp4-mi355x-vllm-disagg + scenario-type: + - fixed-seq-len + description: + - "Port the archived MiniMax-M3 FP4 MI355X 1P1D TP4/TP4 MoRI-IO configuration to native srt-slurm and vLLM Router, retaining all nine 8k/1k concurrency points and explicit engine options." + - "Use official vLLM ROCm v0.28.0 and vLLM Router nightly-20260903-1d10e71 because both original image tags are no longer published. No engine patches or setup hooks." + - "Keep this deprecated configuration opt-in; validate through the upstream end-to-end workflow without restoring it to active production sweeps." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2824 + +- config-keys: + - minimaxm3-fp4-mi355x-vllm-disagg + scenario-type: + - fixed-seq-len + description: + - "Pin srt-slurm 33a3d811b795 for this archived compatibility lane: preserve upstream ephemeral-port allocation for MoRI TP child processes instead of injecting one shared VLLM_PORT scan base." + - "Keep the official engine/router images, model options, workload, and all previously validated recipes and runtime pins unchanged." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2824