diff --git a/benchmarks/single_node/agentic/apply_atom_pr2106_patch.sh b/benchmarks/single_node/agentic/apply_atom_pr2106_patch.sh new file mode 100755 index 0000000000..b044b0e15b --- /dev/null +++ b/benchmarks/single_node/agentic/apply_atom_pr2106_patch.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Apply ROCm/ATOM PR #2106 to the pinned image before serving. +# The patch fixes the MiniMax-M3 EAGLE3 draft KV layout and block-table +# freshness issues that can cause prefix-cache warmup OOMs or stale-KV reads. + +ATOM_ROOT="${ATOM_ROOT:-/app/ATOM}" +# Pin the PR's current base/head range so a later PR update cannot silently +# change an already reviewed benchmark run. +PATCH_URL="https://github.com/ROCm/ATOM/compare/86476f716e9887a7cb2e423deb88712737f702f5...c63ab67e58d39f1c887d2b6af13ac0b9034a1a1a.diff" +PATCH_FILE="$(mktemp /tmp/atom-pr2106.XXXXXX.patch)" +trap 'rm -f "$PATCH_FILE"' EXIT +PATCH_EXCLUDES=( + --exclude=atom/model_ops/attentions/backends.py + --exclude=atom/spec_decode/eagle_proposer.py +) + +if [[ ! -d "$ATOM_ROOT" ]]; then + echo "ERROR: ATOM source tree not found at $ATOM_ROOT" >&2 + exit 1 +fi + +curl -fsSL "$PATCH_URL" -o "$PATCH_FILE" + +if git -C "$ATOM_ROOT" rev-parse --is-inside-work-tree >/dev/null 2>&1; then + if git -C "$ATOM_ROOT" apply --reverse --check "$PATCH_FILE" >/dev/null 2>&1; then + echo "ATOM PR #2106 already applied" + exit 0 + fi + + # These files were structurally rewritten after the pinned image's ATOM + # commit. Their freshness guard is orthogonal to the draft-KV OOM fix and + # is intentionally omitted until it can be ported against that revision. + git -C "$ATOM_ROOT" apply --check "${PATCH_EXCLUDES[@]}" "$PATCH_FILE" + git -C "$ATOM_ROOT" apply "${PATCH_EXCLUDES[@]}" "$PATCH_FILE" +else + if [[ -f "$ATOM_ROOT/atom/spec_decode/draft_kv_layout.py" ]]; then + echo "ATOM PR #2106 already applied" + exit 0 + fi + if patch --dry-run -p1 -d "$ATOM_ROOT" < "$PATCH_FILE" >/dev/null 2>&1; then + patch -p1 -d "$ATOM_ROOT" < "$PATCH_FILE" + else + echo "ERROR: ATOM PR #2106 does not apply cleanly to $ATOM_ROOT" >&2 + exit 1 + fi +fi +echo "Applied ROCm/ATOM PR #2106 to $ATOM_ROOT" diff --git a/benchmarks/single_node/agentic/apply_atom_pr2147_patch.sh b/benchmarks/single_node/agentic/apply_atom_pr2147_patch.sh new file mode 100644 index 0000000000..cedf2e3df8 --- /dev/null +++ b/benchmarks/single_node/agentic/apply_atom_pr2147_patch.sh @@ -0,0 +1,75 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Apply ROCm/ATOM PR #2147 on top of the pinned MiniMax image checkout. +# +# PR #2147 supersedes PR #2106 for the MiniMax-M3 EAGLE3 draft-KV layout work: +# both fix the draft/target KV pool disagreement that can OOM during prefix-cache +# warmup. The pinned image predates #2147's merge base, so we fast-forward the +# in-image tree to the current PR head rather than applying the PR diff alone. +# +# A small MiniMax EAGLE3 follow-up fix is applied after the forward diff because +# the upstream check compared kernel block sizes (16 vs 128) instead of the +# scheduler block size both sides actually index (128). + +ATOM_ROOT="${ATOM_ROOT:-/app/ATOM}" +PINNED_ATOM_SHA="00760297ef69af7ab5d345af9c8fc6da00f5314d" +PR2147_HEAD_SHA="71ad8c69c5377738657907e24c5aff8672c0f004" +PATCH_URL="https://github.com/ROCm/ATOM/compare/${PINNED_ATOM_SHA}...${PR2147_HEAD_SHA}.diff" +PATCH_FILE="$(mktemp /tmp/atom-pr2147.XXXXXX.patch)" +MARKER_FILE="$ATOM_ROOT/.inferencex-pr2147-applied" +DRAFT_KV_FILE="$ATOM_ROOT/atom/spec_decode/draft_kv.py" + +if [[ ! -d "$ATOM_ROOT" ]]; then + echo "ERROR: ATOM source tree not found at $ATOM_ROOT" >&2 + exit 1 +fi + +trap 'rm -f "$PATCH_FILE"' EXIT + +if [[ -f "$MARKER_FILE" ]]; then + echo "ATOM PR #2147 already applied ($(cat "$MARKER_FILE"))" + exit 0 +fi + +curl -fsSL "$PATCH_URL" -o "$PATCH_FILE" +git -C "$ATOM_ROOT" apply --check "$PATCH_FILE" +git -C "$ATOM_ROOT" apply "$PATCH_FILE" + +python3 - "$DRAFT_KV_FILE" <<'PY' +from pathlib import Path +import sys + +path = Path(sys.argv[1]) +text = path.read_text() +old = """ if pool.block_size != target.block_size: + raise ValueError( + f"the draft's blocks are {pool.block_size} tokens and the " + f"target's {target.block_size}; a draft is indexed by the " + "target's block tables, so it cannot yet block at anything else" + )""" +new = """ pool_scheduler_block = pool.block_size * ( + runner.block_size // pool.block_size + ) + target_scheduler_block = target.block_size * target.block_ratio + if pool_scheduler_block != target_scheduler_block: + raise ValueError( + f"the draft indexes {pool_scheduler_block}-token scheduler " + f"blocks but the target indexes {target_scheduler_block}-token " + "scheduler blocks; a draft is indexed by the target's block " + "tables, so the two must agree at scheduler granularity" + )""" +if old not in text: + if "pool_scheduler_block" in text: + print("MiniMax EAGLE3 scheduler-block fix already present") + else: + raise SystemExit( + "ERROR: expected draft_kv.py block-size guard not found after PR #2147 apply" + ) +else: + path.write_text(text.replace(old, new, 1)) + print("Applied MiniMax EAGLE3 scheduler-block alignment fix to draft_kv.py") +PY + +printf '%s\n' "$PR2147_HEAD_SHA" > "$MARKER_FILE" +echo "Applied ROCm/ATOM PR #2147 (${PINNED_ATOM_SHA} -> ${PR2147_HEAD_SHA}) to $ATOM_ROOT" diff --git a/benchmarks/single_node/agentic/minimaxm3_fp4_mi355x_atom_mtp.sh b/benchmarks/single_node/agentic/minimaxm3_fp4_mi355x_atom_mtp.sh new file mode 100644 index 0000000000..fe194e73b1 --- /dev/null +++ b/benchmarks/single_node/agentic/minimaxm3_fp4_mi355x_atom_mtp.sh @@ -0,0 +1,345 @@ +#!/usr/bin/env bash +set -eo pipefail +set -x + +# Agentic trace replay benchmark for MiniMax-M3 MXFP4 on MI355X / MI350X (gfx950) +# using the official ATOM MiniMax-M3 launch settings. +# +# Companion to minimaxm3_fp4_mi355x_mtp.sh, which runs the same checkpoint under +# vLLM, so the two arms are directly comparable. +# +# TP2 and TP4 follow the official ATOM MiniMax-M3 MXFP4 recipe. TP8 is also +# accepted for larger-memory variants and manual smoke tests. +# +# The ATOM image is launched with the official MiniMax attention/index-cache +# settings; Kimi-specific vLLM patches are not sourced here. +# +# Required env vars: +# MODEL, MODEL_PATH, TP, DCP_SIZE, CONC, KV_OFFLOADING, KV_OFFLOAD_BACKEND, +# TOTAL_CPU_DRAM_GB, RESULT_DIR, DURATION, EP_SIZE, DP_ATTENTION + +source "$(dirname "$0")/../../benchmark_lib.sh" + +check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION EP_SIZE DP_ATTENTION + +export RESULT_FILENAME="${RESULT_FILENAME:-minimaxm3_agentic}" + +echo "MODEL=$MODEL TP=$TP DCP_SIZE=${DCP_SIZE:-1} CONC=$CONC KV_OFFLOADING=$KV_OFFLOADING TOTAL_CPU_DRAM_GB=$TOTAL_CPU_DRAM_GB RESULT_DIR=$RESULT_DIR DURATION=$DURATION EP_SIZE=$EP_SIZE DP_ATTENTION=$DP_ATTENTION" + +if [[ -v SLURM_JOB_ID ]]; then + echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" +fi + +if [ "$TP" -ne 2 ] && [ "$TP" -ne 4 ] && [ "$TP" -ne 8 ]; then + echo "Error: MiniMax-M3 MXFP4 supports TP2, TP4, or TP8 on 288 GB gfx950 parts." >&2 + exit 1 +fi + +if [[ -v ROCR_VISIBLE_DEVICES ]]; then + export HIP_VISIBLE_DEVICES="$ROCR_VISIBLE_DEVICES" +fi + +if [[ -n "$MODEL_PATH" ]]; then + if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then + hf download "$MODEL" --local-dir "$MODEL_PATH" + fi +else + hf download "$MODEL" + export MODEL_PATH="$MODEL" +fi + +DRAFT_MODEL="Inferact/MiniMax-M3-EAGLE3-GQA" +hf download "$DRAFT_MODEL" + +wait_for_amd_gpu_clean + +rocm-smi || true +amd-smi || true + +resolve_trace_source +install_agentic_deps +# ATOM's server runs from the image's system venv rather than the temporary +# AIPerf venv created by install_agentic_deps. MiniMax's tokenizer fallback +# requires these packages in that server environment. +ATOM_RUNTIME_DEPS=/tmp/inferencex-atom-runtime-deps +/opt/venv/bin/python -m pip install --quiet --target "$ATOM_RUNTIME_DEPS" --no-deps sentencepiece tiktoken + +# The pinned ATOM image predates upstream KV-pool fixes. Apply #2147 when set, +# otherwise keep the reviewed #2106 patch for the current recipe image. +case "${ATOM_PATCH:-2147}" in + 2147) + bash "$(dirname "$0")/apply_atom_pr2147_patch.sh" + ;; + 2106) + bash "$(dirname "$0")/apply_atom_pr2106_patch.sh" + ;; + *) + echo "Unsupported ATOM_PATCH=${ATOM_PATCH} (expected 2106 or 2147)" >&2 + exit 1 + ;; +esac + +# Require the ATOM Prometheus stream in every official result. AIPerf +# deduplicates this endpoint against its automatic localhost discovery. +export AIPERF_SERVER_METRICS_URLS="http://localhost:${PORT}/metrics" +export AIPERF_REQUIRED_SERVER_METRIC_PREFIX="atom:" + +# Long agentic turns against a 1M context: keep the client from timing out +# mid-request while the server is prefill-bound. Matches the vLLM K3 arm. +export AIPERF_HTTP_TCP_USER_TIMEOUT=900000 + +# VRAM space check +wait_for_amd_gpu_clean + +# ---- Server config ---------------------------------------------------------- +SERVER_LOG="$RESULT_DIR/server.log" +mkdir -p "$RESULT_DIR" + +SERVER_PID="" +cleanup_agentic_services() { + local exit_code=$? + trap - EXIT INT TERM + set +e + stop_background_process_tree "$SERVER_PID" "ATOM server" 60 + exit "$exit_code" +} +trap cleanup_agentic_services EXIT +trap 'exit 130' INT +trap 'exit 143' TERM + +# ---- Per-concurrency knobs -------------------------------------------------- +# Concurrency 1-4 is the latency floor: everything GPU-resident, no decode +# context parallelism, the deepest draft the golden curve publishes, and an +# 8192-token prefill step. +# Higher concurrency retains ATOM state replay checkpoints; KV offloading is +# selected independently by KV_OFFLOADING/KV_OFFLOAD_BACKEND below. +# +# STATE_OFFLOAD_CPU_GIB is the per-rank slice of the CPU budget reserved for +# the MiniMax-M3 state; 0 leaves the whole budget to the paged KV. +case "$CONC" in + # No KV offload; the working set fits in HBM. + 1|2|4|5) + MAX_NUM_SEQS=32 + MAX_NUM_BATCHED_TOKENS=8192 + GPU_MEM_UTIL=0.88 + ATOM_ENABLE_REPLAYSSM=0 + STATE_CHECKPOINT_SLOTS="" + NUM_SPEC_TOKENS=3 + SPEC_DECODE_AL=2.78 + STATE_OFFLOAD_CPU_GIB=0 + ;; + # Higher concurrency settings retain ATOM's state replay checkpoints. KV + # offloading itself is selected by KV_OFFLOADING/KV_OFFLOAD_BACKEND. + 8|10|12|14|15|20|24|28) + MAX_NUM_SEQS=32 + MAX_NUM_BATCHED_TOKENS=4096 + GPU_MEM_UTIL=0.88 + ATOM_ENABLE_REPLAYSSM=1 + STATE_CHECKPOINT_SLOTS=96 + NUM_SPEC_TOKENS=3 + SPEC_DECODE_AL=2.78 + STATE_OFFLOAD_CPU_GIB=0 + ;; + # LMCache paged-KV tier plus ATOM's CPU state tier, 32 GB/rank carved out + # for the attention state. + 16) + MAX_NUM_SEQS=32 + MAX_NUM_BATCHED_TOKENS=8192 + GPU_MEM_UTIL=0.86 + ATOM_ENABLE_REPLAYSSM=0 + STATE_CHECKPOINT_SLOTS="" + NUM_SPEC_TOKENS=3 + SPEC_DECODE_AL=2.78 + STATE_OFFLOAD_CPU_GIB=32 + ;; + 32) + MAX_NUM_SEQS=64 + MAX_NUM_BATCHED_TOKENS=8192 + GPU_MEM_UTIL=0.86 + ATOM_ENABLE_REPLAYSSM=0 + STATE_CHECKPOINT_SLOTS="" + NUM_SPEC_TOKENS=3 + SPEC_DECODE_AL=2.78 + STATE_OFFLOAD_CPU_GIB=32 + ;; + 40) + MAX_NUM_SEQS=80 + MAX_NUM_BATCHED_TOKENS=8192 + GPU_MEM_UTIL=0.86 + ATOM_ENABLE_REPLAYSSM=0 + STATE_CHECKPOINT_SLOTS="" + NUM_SPEC_TOKENS=0 + SPEC_DECODE_AL=0 + STATE_OFFLOAD_CPU_GIB=32 + ;; + 56) + MAX_NUM_SEQS=72 + MAX_NUM_BATCHED_TOKENS=4096 + GPU_MEM_UTIL=0.88 + ATOM_ENABLE_REPLAYSSM=0 + STATE_CHECKPOINT_SLOTS="" + NUM_SPEC_TOKENS=0 + SPEC_DECODE_AL=0 + STATE_OFFLOAD_CPU_GIB=32 + ;; + *) + echo "Unsupported CONC=$CONC" >&2 + exit 2 + ;; +esac +# Official MiniMax-M3 ATOM launch settings. Concurrency remains a benchmark +# input, while the server capacity knobs follow the validated reference. +MAX_NUM_SEQS=$((2 * CONC)) +MAX_NUM_BATCHED_TOKENS=32768 +GPU_MEM_UTIL=0.9 +export ATOM_ENABLE_REPLAYSSM + +# Extra in-GPU state checkpoint slots beyond the in-flight floor. Checkpoints +# and live requests share one pool, so without this the room to retain a +# checkpoint is whatever max-num-seqs happens to leave. +STATE_CKPT_ARGS=() +if [ -n "$STATE_CHECKPOINT_SLOTS" ]; then + STATE_CKPT_ARGS=(--state-checkpoint-slots "$STATE_CHECKPOINT_SLOTS") +fi + +# ---- KV offload ------------------------------------------------------------- +# K3 is a hybrid: MiniMax-M3 attention carries a per-request recurrent state +# alongside the paged KV. The paged KV rides this LMCache tier from +# concurrency 8 up; from concurrency 16 up the CPU state tier is switched on +# alongside it, because the state tier is what makes a resumed agentic turn +# cheap and the paged KV tier alone cannot restore one. +OFFLOAD_ARGS=() + +case "$KV_OFFLOAD_BACKEND" in + "") + require_agentic_kv_offload_none + ;; + lmcache) + require_agentic_kv_offload_backend lmcache + + # TOTAL_CPU_DRAM_GB is the AGGREGATE budget from the matrix generator. + # LMCACHE_MAX_LOCAL_CPU_SIZE and OFFLOAD_STATE_CPU_SIZE are per rank and + # every rank allocates its own, so the aggregate is divided by TP as the + # agentic README requires. Handing a rank the whole aggregate does not + # just overcommit -- it never finishes pinning and hangs the launch + # partway through. + PER_RANK_CPU_GB="$((TOTAL_CPU_DRAM_GB / TP))" + LMCACHE_CPU_GB="$((PER_RANK_CPU_GB - STATE_OFFLOAD_CPU_GIB))" + + export PYTHONHASHSEED=0 + export LMCACHE_LOCAL_CPU=True + export LMCACHE_MAX_LOCAL_CPU_SIZE="$LMCACHE_CPU_GB" + # DCP-locked: the offload hash block is block-size(128) x dcp(8) = 1024, + # so the KV grid and the state-checkpoint grid coincide and the joint + # load aims both legs at one boundary. 512 or 2048 misaligns it. + export LMCACHE_CHUNK_SIZE=1024 + export OFFLOAD_KV_FOR_HYBRID=1 + # Statistics only -- per-step offload counters in the connector. Kept on + # because the submitted numbers were measured with it on. + export OFFLOAD_PROFILE=1 + + if [ "$STATE_OFFLOAD_CPU_GIB" -gt 0 ]; then + # CPU state-offload tier for the attention state. + export OFFLOAD_STATE=1 + export OFFLOAD_STATE_CPU_SIZE="$STATE_OFFLOAD_CPU_GIB" + export OFFLOAD_STATE_STAGING_GROUPS=8 + export OFFLOAD_STATE_MIN_LOAD_TOKENS=0 + # Must be set: the staging buffer defaults to 2 chunks (8 MiB), one + # K3 state entry is 54.78 MiB, and a buffer too small to hold one + # entry makes the tier decline to build -- one log line, then + # nothing offloads, which reads exactly like a tier that is on and + # idle. + export OFFLOAD_GPU_STAGING_CHUNKS=32 + fi + + OFFLOAD_ARGS=( + --kv-transfer-config + "{\"kv_connector\":\"lmcache_offload\",\"kv_role\":\"offload\"}" + ) + ;; + *) + echo "Unsupported KV_OFFLOAD_BACKEND: $KV_OFFLOAD_BACKEND (expected empty or lmcache)" >&2 + exit 1 + ;; +esac + +# ---- ATOM env --------------------------------------------------------------- +echo "Starting atom server..." +export PYTHONNOUSERSITE=1 + +# Required by ATOM: without it the aiter kernel logs flood the server log for +# the whole 3600 s replay. +export AITER_LOG_LEVEL="${AITER_LOG_LEVEL:-WARNING}" +export AITER_SITUV2_A4W4=1 +export AITER_QUICK_REDUCE_QUANTIZATION=INT4 +export AITER_FLYDSL_STAGE2_FP8=1 +export ATOM_FORCE_ATTN_TRITON=1 +# Anchor-only state checkpointing: the demand rung is 47% of checkpoint writes +# but reads back 2.8% of the time, against 85.2% for a prompt-end anchor, so it +# costs more in evictions than its reuse is worth on these traces. + +# ---- Speculative ------------------------------------------------------------ +# golden_al_distribution/minimaxm3_eagle3_gqa.yaml: +# minimax-m3.thinking_on[3] -> AL 2.78 +# Match the MI355X vLLM arm: EAGLE3-GQA draft, synthetic acceptance length on +# throughput runs, and real target verification on eval-only runs. +SPEC_ARGS=() +if [ "$NUM_SPEC_TOKENS" -gt 0 ]; then + SPEC_ARGS=( + --method eagle3 + --draft-model "$DRAFT_MODEL" + --num-speculative-tokens "$NUM_SPEC_TOKENS" + ) + if [ "${EVAL_ONLY}" != "true" ]; then + SPEC_ARGS+=(--spec-decode-acceptance-length "$SPEC_DECODE_AL") + fi +fi +echo "SPEC_DECODE_AL=$SPEC_DECODE_AL NUM_SPEC_TOKENS=$NUM_SPEC_TOKENS" + +# ---- LLM server ------------------------------------------------------------- +ATOM_CMD=( + python -m atom.entrypoints.openai_server + --model "$MODEL_PATH" + --served-model-name "$MODEL" + --host 0.0.0.0 + --server-port "$PORT" + --trust-remote-code + --tensor-parallel-size "$TP" + --kv_cache_dtype fp8 + --block-size 128 + --max-num-seqs "$MAX_NUM_SEQS" + --max-num-batched-tokens "$MAX_NUM_BATCHED_TOKENS" + --gpu-memory-utilization "$GPU_MEM_UTIL" + --index-cache-dtype fp8 + --hf-overrides '{"use_index_cache": true, "index_topk_freq": 4}' + --online_quant_config '{"global_quant_config":"ptpc_fp8","exclude_layer":["lm_head","model.embed_tokens","vision_tower","multi_modal_projector","patch_merge_mlp","*block_sparse_moe"]}' + --default-chat-template-kwargs '{"thinking_mode":"enabled"}' + "${SPEC_ARGS[@]}" + "${OFFLOAD_ARGS[@]}" +) +if [[ "${ENABLE_PREFIX_CACHING:-true}" != "true" ]]; then + ATOM_CMD+=(--no-enable_prefix_caching) +fi +if declare -F write_command >/dev/null 2>&1; then + write_command "$RESULT_DIR/server_command.txt" "${ATOM_CMD[@]}" +else + printf '%q ' "${ATOM_CMD[@]}" > "$RESULT_DIR/server_command.txt" + printf '\n' >> "$RESULT_DIR/server_command.txt" +fi +PYTHONPATH="$ATOM_RUNTIME_DEPS${PYTHONPATH:+:$PYTHONPATH}" \ + "${ATOM_CMD[@]}" > "$SERVER_LOG" 2>&1 & +SERVER_PID=$! +echo "Server PID: $SERVER_PID" + +wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" + +# ---- Run benchmark ---------------------------------------------------------- +if [ "${EVAL_ONLY}" = "true" ]; then + export EVAL_FRAMEWORK="${EVAL_FRAMEWORK:-lm-eval}" + export EVAL_TASKS_DIR="${EVAL_TASKS_DIR:-utils/evals/gsm8k.yaml}" + run_eval --port "$PORT" +else + build_replay_cmd "$RESULT_DIR" + REPLAY_CMD+=" --apply-chat-template" + run_agentic_replay_and_write_outputs "$RESULT_DIR" +fi diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 62daacca6f..452f929f79 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -685,6 +685,21 @@ kimik3-fp4-mi355x-atom-agentic-mtp: search-space: - { tp: 8, dcp-size: 8, kv-offloading: dram, kv-offload-backend: { name: lmcache, version: "0.4.5" }, conc-list: [32, 40, 56], spec-decoding: mtp } +minimaxm3-fp4-mi355x-atom-agentic-mtp: + image: rocm/atom-dev@sha256:1cd6abf29958cf0c332041b945249f7303ac86a36352f976b26d844ad72b4f5c + model: amd/MiniMax-M3-MXFP4 + model-prefix: minimaxm3 + runner: cluster:mi355x-amds + precision: fp4 + framework: atom + multinode: false + scenarios: + agentic-coding: + - dram-utilization: 0.20 + search-space: + - { tp: 4, kv-offloading: none, conc-list: [1, 2, 4, 5, 8, 10, 12, 15, 20, 24, 28, 32], spec-decoding: mtp } + - { tp: 2, kv-offloading: none, conc-list: [1, 2, 5], spec-decoding: mtp } + dsr1-fp4-mi355x-sglang-disagg: image: lmsysorg/sglang-rocm:v0.5.12-rocm720-mi35x-20260519 model: amd/DeepSeek-R1-0528-MXFP4-v2 diff --git a/docs/waiver/2106.md b/docs/waiver/2106.md new file mode 100644 index 0000000000..0107a81df7 --- /dev/null +++ b/docs/waiver/2106.md @@ -0,0 +1,62 @@ +# Inference-engine patch waiver — PR #2106 + +Filed per [`docs/PR_REVIEW_CHECKLIST.md`](../PR_REVIEW_CHECKLIST.md) (image-provenance / +patch item): the MiniMax-M3 ATOM benchmark applies an upstream serving-stack patch to +the pinned image before serving. + +## Config covered + +- **Master config entry:** `minimaxm3-fp4-mi355x-atom-agentic-mtp` in + [`configs/amd-master.yaml`](../../configs/amd-master.yaml) +- **Pinned image:** `rocm/atom-dev@sha256:1cd6abf29958cf0c332041b945249f7303ac86a36352f976b26d844ad72b4f5c` +- **Patch entrypoint:** `ATOM_PATCH` selects the patch script from + [`benchmarks/single_node/agentic/minimaxm3_fp4_mi355x_atom_mtp.sh`](../../benchmarks/single_node/agentic/minimaxm3_fp4_mi355x_atom_mtp.sh) + (`2147` default, `2106` retained for comparison) +- **Patch scripts:** + [`apply_atom_pr2147_patch.sh`](../../benchmarks/single_node/agentic/apply_atom_pr2147_patch.sh) + (default) and + [`apply_atom_pr2106_patch.sh`](../../benchmarks/single_node/agentic/apply_atom_pr2106_patch.sh) + +## What is patched + +### Default: PR #2147 + +The default patch script fast-forwards the pinned image checkout to +[ROCm/ATOM #2147](https://github.com/ROCm/ATOM/pull/2147) +(`00760297ef69af7ab5d345af9c8fc6da00f5314d` → +`71ad8c69c5377738657907e24c5aff8672c0f004`) and applies a MiniMax EAGLE3 +scheduler-block alignment follow-up in `draft_kv.py`. GSM8K and infra smoke +validated this path on MI355X. + +### Legacy: PR #2106 + +The legacy patch script downloads the pinned base-to-head patch for +[ROCm/ATOM #2106](https://github.com/ROCm/ATOM/pull/2106) +(`86476f716e9887a7cb2e423deb88712737f702f5` → +`c63ab67e58d39f1c887d2b6af13ac0b9034a1a1a`), verifies it applies cleanly, and +applies it idempotently to the ATOM source tree in `/app/ATOM`. +The upstream change allocates the MiniMax-M3 EAGLE3 MHA draft KV pool in flash +layout and validates block-table freshness, avoiding the full-pool V-layout +conversion that can OOM during prefix-cache warmup. + +The pinned image contains a later, structurally divergent ATOM checkout. The +flash-layout portion applies cleanly; the two freshness-guard files +(`attentions/backends.py` and `spec_decode/eagle_proposer.py`) are excluded +until those guards are ported to the image revision. The excluded guard is +correctness hardening, not part of the warmup OOM fix. + +## Why the unmodified upstream image cannot run this benchmark + +The pinned image predates PR #2106. With MiniMax-M3 MXFP4, EAGLE3, prefix +caching, FP8 KV/index cache, and long AgentX traces, the unmodified image can +attempt a multi-gigabyte draft KV conversion during warmup. The observed +`conc=24` run stopped with requests in flight and no progress, consistent with +the OOM condition documented by the upstream PR. The patch is required for this +benchmark path to run reliably. + +## Removal plan + +Remove `apply_atom_pr2106_patch.sh`, its invocation, and this waiver once a +published ATOM image contains PR #2106. At that point update the pinned image +digest in `configs/amd-master.yaml` and verify the MiniMax AgentX smoke, +full-sweep, and GSM8K evaluation paths without the patch. diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 2f58d1b442..e120870aea 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6730,3 +6730,131 @@ - "Configure model-specific tool and reasoning parsers for the Kimi-K3 and MiniMax-M3 serving paths so opt-in verifier requests produce structured OpenAI tool calls." - "Limit shared AgentX readiness and synthetic-acceptance rewrites to eval-only runs so default throughput recipe rendering remains unchanged." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2634 + +- config-keys: + - minimaxm3-fp4-mi355x-atom-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Add MiniMax-M3 MXFP4 on MI355X with ATOM agentic speculative decoding and LMCache DRAM offload." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX + +- config-keys: + - minimaxm3-fp4-mi355x-atom-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Align MiniMax-M3 ATOM agentic MTP concurrency and LMCache DRAM tiers with the validated Kimi-K3 MI355X recipe." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX + +- config-keys: + - minimaxm3-fp4-mi355x-atom-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Use the official ATOM MiniMax-M3 MXFP4 FP8 index-cache and online quantization settings for the TP8 agentic path." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX + +- config-keys: + - minimaxm3-fp4-mi355x-atom-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Remove the 32K serving-only context cap from the MiniMax-M3 ATOM AgentX path so long agentic traces use the model context limit." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX + +- config-keys: + - minimaxm3-fp4-mi355x-atom-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Switch the MiniMax-M3 ATOM AgentX recipe to TP4 and enable prefix caching for the MI355X run." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX + +- config-keys: + - minimaxm3-fp4-mi355x-atom-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Expand the TP4 no-offload MiniMax-M3 ATOM AgentX sweep to the requested concurrency points from 1 through 32." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX + +- config-keys: + - minimaxm3-fp4-mi355x-atom-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Pin the MiniMax-M3 ATOM AgentX recipe to the latest image validated by the TP4 prefix-cache smoke." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX + +- config-keys: + - minimaxm3-fp4-mi355x-atom-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Replace the floating ATOM latest tag with the exact image digest validated on MI355X." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX + +- config-keys: + - minimaxm3-fp4-mi355x-atom-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Enable prefix caching by default in the MiniMax-M3 ATOM AgentX entrypoint for generated Actions runs." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX + +- config-keys: + - minimaxm3-fp4-mi355x-atom-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Scale ATOM max-num-seqs with concurrency using two scheduler slots per AgentX lane." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX + +- config-keys: + - minimaxm3-fp4-mi355x-atom-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Apply ROCm/ATOM PR #2106 before serving to fix MiniMax-M3 EAGLE3 draft KV warmup OOM and stale block-table reads." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX + +- config-keys: + - minimaxm3-fp4-mi355x-atom-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Add the TP2 no-offload MiniMax-M3 ATOM AgentX MTP search space at concurrencies 1, 2, and 5." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX + +- config-keys: + - minimaxm3-fp4-mi355x-atom-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Allow the MiniMax-M3 ATOM AgentX launcher to accept TP2 search-space configurations." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX + +- config-keys: + - minimaxm3-fp4-mi355x-atom-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Raise the MiniMax-M3 ATOM AgentX GPU memory utilization target to 0.9." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX + +- config-keys: + - minimaxm3-fp4-mi355x-atom-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Align MiniMax-M3 ATOM AgentX with the MI355X vLLM arm: EAGLE3-GQA draft, synthetic acceptance length 2.78, and thinking_mode enabled." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2816 + +- config-keys: + - minimaxm3-fp4-mi355x-atom-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Default to ROCm/ATOM PR #2147 KV-pool forward diff plus MiniMax EAGLE3 scheduler-block alignment; keep PR #2106 selectable via ATOM_PATCH and default EVAL_ONLY to lm-eval gsm8k." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2816 diff --git a/utils/matrix_logic/test_generate_sweep_configs.py b/utils/matrix_logic/test_generate_sweep_configs.py index 520e4105e0..df47dc376e 100644 --- a/utils/matrix_logic/test_generate_sweep_configs.py +++ b/utils/matrix_logic/test_generate_sweep_configs.py @@ -2359,9 +2359,9 @@ def test_kimi_minimax_trimmed_eval_matrix_covers_current_configs( json.dumps(manifest, separators=(',', ':')).encode() ).hexdigest() - assert len(manifest) == 63 + assert len(manifest) == 65 assert manifest_digest == ( - '1630cdd6fbf77302ee3286b118710576aa090e1502b3e5ec482b7f537a3f1132' + '65c954899fe920db1ca592bc19bd464018d99d85d48d85cccdf712e9f9696913' ), manifest_digest for row in rows: if isinstance(row['conc'], list):