diff --git a/benchmarks/multi_node/amd_utils/env.sh b/benchmarks/multi_node/amd_utils/env.sh index 0cf3978bbf..88b14712ee 100755 --- a/benchmarks/multi_node/amd_utils/env.sh +++ b/benchmarks/multi_node/amd_utils/env.sh @@ -334,6 +334,8 @@ else unset MORI_MOE_MAX_INPUT_TOKENS_PREFILL unset MORI_MOE_MAX_INPUT_TOKENS_DECODE + export SGLANG_MORI_RECV_BOUND=1 + # PER_RANK dispatch tokens pinned independently (16384 prefill / 128 # decode); server_sglang.sh prefers these over the MORI_MAX_DISPATCH_* # coupling when set. @@ -346,32 +348,49 @@ else # Overlap plan stream on for DSv4 (global default is 0) # export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=0 - # DSv4 model kernel routing (mirrors the single-node / manual PD recipe) + # DSv4 model kernel routing export SGLANG_DEFAULT_THINKING=1 export SGLANG_DSV4_REASONING_EFFORT=high - export SGLANG_OPT_DEEPGEMM_HC_PRENORM=false - export SGLANG_USE_AITER=1 + # export SGLANG_OPT_DEEPGEMM_HC_PRENORM=false + # export SGLANG_USE_AITER=1 export SGLANG_USE_ROCM700A=0 - export SGLANG_OPT_USE_FUSED_COMPRESS=true + # export SGLANG_OPT_USE_FUSED_COMPRESS=true export SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton - export SGLANG_OPT_FP8_WO_A_GEMM=false - export SGLANG_OPT_USE_JIT_INDEXER_METADATA=false - export SGLANG_OPT_USE_TOPK_V2=false - export SGLANG_OPT_USE_AITER_INDEXER=${SGLANG_OPT_USE_AITER_INDEXER:-true} - export SGLANG_OPT_USE_TILELANG_INDEXER=false - export SGLANG_OPT_USE_TILELANG_MHC_PRE=false - export SGLANG_OPT_USE_TILELANG_MHC_POST=false - export SGLANG_FP8_PAGED_MQA_LOGITS_TORCH=1 - export SGLANG_OPT_USE_FUSED_COMPRESS_TRITON=true - export SGLANG_OPT_USE_MULTI_STREAM_OVERLAP=false - export SGLANG_ROCM_USE_MULTI_STREAM=false + # export SGLANG_OPT_FP8_WO_A_GEMM=false + # export SGLANG_OPT_USE_JIT_INDEXER_METADATA=false + # export SGLANG_OPT_USE_TOPK_V2=false + # export SGLANG_OPT_USE_AITER_INDEXER=${SGLANG_OPT_USE_AITER_INDEXER:-true} + # export SGLANG_OPT_USE_TILELANG_INDEXER=false + # export SGLANG_OPT_USE_TILELANG_MHC_PRE=false + # export SGLANG_OPT_USE_TILELANG_MHC_POST=false + # export SGLANG_FP8_PAGED_MQA_LOGITS_TORCH=1 + # export SGLANG_OPT_USE_FUSED_COMPRESS_TRITON=true + # export SGLANG_OPT_USE_MULTI_STREAM_OVERLAP=false + # export SGLANG_ROCM_USE_MULTI_STREAM=false export AITER_BF16_FP8_MOE_BOUND=0 - export SGLANG_EAGER_INPUT_NO_COPY=true - export SGLANG_SHARED_EXPERT_TP1=1 - export SGLANG_DP_SHARED_EXPERT_LOCAL=1 - export SGLANG_DP_USE_GATHERV=1 - export SGLANG_DP_USE_REDUCE_SCATTER=1 - export GPU_MAX_HW_QUEUES=5 + # export SGLANG_EAGER_INPUT_NO_COPY=true + export TORCH_BLAS_PREFER_HIPBLASLT=1 + # aiter batched GEMM for the absorbed MLA projections, carried by the v0.5.18 + # image and off by default in environ.py. + export SGLANG_OPT_USE_AITER_BATCHED_GEMM=1 + # DP-attention-only SGLang internal knobs (shared-expert TP1 placement, + # gatherv/reduce-scatter collectives) plus the wider HW-queue count DP + # ranks need to overlap MoRI dispatch with compute. Mirrors the + # single-node dsv4-fp4-mi355x-sglang-agentic-mtp recipe's + # `if [ "$DP_ATTENTION" = "true" ]` block (InferenceX PR #2800): + # these must NOT apply to the TP-only (no-DP) arms. Multi-node splits + # DP-attention per role (PREFILL_ENABLE_DP / DECODE_ENABLE_DP); every + # current search-space arm sets both the same way, so either one + # flags the whole node. + if [[ "$PREFILL_ENABLE_DP" == "true" || "$DECODE_ENABLE_DP" == "true" ]]; then + export SGLANG_SHARED_EXPERT_TP1=1 + export SGLANG_DP_SHARED_EXPERT_LOCAL=1 + export SGLANG_DP_USE_GATHERV=1 + export SGLANG_DP_USE_REDUCE_SCATTER=1 + export GPU_MAX_HW_QUEUES="${GPU_MAX_HW_QUEUES_DP:-5}" + else + export GPU_MAX_HW_QUEUES=2 + fi fi fi \ No newline at end of file diff --git a/benchmarks/multi_node/amd_utils/models.yaml b/benchmarks/multi_node/amd_utils/models.yaml index 88a61ccc10..edebb9ee3b 100644 --- a/benchmarks/multi_node/amd_utils/models.yaml +++ b/benchmarks/multi_node/amd_utils/models.yaml @@ -10,6 +10,11 @@ # base_flags: str # Common flags for both prefill and decode # mtp_flags: str # Appended to decode when DECODE_MTP_SIZE > 0 # dp_flags: str # Appended when DP attention is enabled (prefill or decode) +# no_dp_flags: str # Appended when DP attention is disabled. Mutually exclusive +# # with dp_flags -- use this (not base_flags) for any flag +# # that conflicts with one in dp_flags (e.g. shared-experts +# # fusion), so the two conflicting flags are never both +# # emitted onto the same command line. # ep_flags: str # Appended when EP is enabled. EP-specific MoE knobs only # # (a2a backend, deepep mode, ep-dispatch algorithm). With # # ep=1 these are dropped so the MoE runs tensor-parallel (TP). @@ -364,36 +369,49 @@ DeepSeek-R1-0528-MXFP4-v2: cuda_graph_bs_range: "1-128" DeepSeek-V4-Pro-AgentX: - base_flags: "--watchdog-timeout 3600 --load-balance-method round_robin --kv-cache-dtype fp8_e4m3 --attention-backend dsv4 --page-size 256 --swa-full-tokens-ratio 0.1 --disable-shared-experts-fusion --tool-call-parser deepseekv4 --reasoning-parser deepseek-v4 --disaggregation-transfer-backend mori --log-level info --log-level-http error" - dp_flags: "--enable-dp-attention --enable-prefill-delayer" + base_flags: "--enable-deepseek-v4-fp4-indexer --watchdog-timeout 3600 --load-balance-method round_robin --kv-cache-dtype fp8_e4m3 --attention-backend dsv4 --page-size 256 --swa-full-tokens-ratio 0.1 --tool-call-parser deepseekv4 --reasoning-parser deepseek-v4 --disaggregation-transfer-backend mori --log-level info --log-level-http error" + # --enforce-shared-experts-fusion and dp_flags' --disable-shared-experts-fusion + # are documented by sglang as mutually exclusive (server_args.py), so each lives + # in its own bucket (no_dp_flags / dp_flags) rather than both riding on the + # command line at once relying on DeepseekV4ForCausalLM's internal check order. + no_dp_flags: "--enforce-shared-experts-fusion" + # DP-attention overrides mirror the single-node dsv4-fp4-mi355x-sglang-agentic-mtp + # recipe's `if [ "$DP_ATTENTION" = "true" ]` block (InferenceX PR #2800): + # --swa-full-tokens-ratio 0.15 widens the SWA ring for the larger DP batch, and + # --tokenizer-worker-num matches the only DP arm's TP (8) today -- bump this if + # a different-TP DP arm is added. + dp_flags: "--enable-dp-attention --enable-prefill-delayer --disable-shared-experts-fusion --swa-full-tokens-ratio 0.15 --enable-dp-attention-local-control-broadcast --tokenizer-worker-num 8 --stream-interval 20 --prefill-decode-interval 10" ep_flags: "--ep-dispatch-algorithm fake --moe-a2a-backend mori --deepep-mode normal" mtp_flags: "--speculative-algorithm EAGLE --speculative-eagle-topk 1" prefill: - mem_fraction_static: 0.85 + mem_fraction_static: 0.86 disable_radix_cache: false disable_cuda_graph: true dp: max_running_requests: 1024 - chunked_prefill_size: "MORI_MAX_DISPATCH_TOKENS_PREFILL * PREFILL_TP_SIZE" # dsv4 compressor kernel uint16 token cap (255*256) + chunked_prefill_size: "16384 * PREFILL_TP_SIZE if PREFILL_TP_SIZE == 8 else 8192 * PREFILL_TP_SIZE" # dsv4 compressor kernel uint16 token cap (255*256) context_length: 1048576 - # max_total_tokens: 1048576 + # Mirrors the single-node recipe's --enable-two-batch-overlap inside the + # DP-attention block; prefill-only, matching that recipe's PARALLEL_ARGS + # scope. Also flips server_sglang.sh's PREFILL_SDMA_ENV to + # MORI_ENABLE_SDMA=true, required alongside two-batch-overlap. + enable_two_batch_overlap: true no_dp: max_running_requests: 64 - # Small prefill chunks interleave long-context agentic prefills across - # requests instead of letting one ~100K-token prefill monopolize the - # engine (the conc>=16 queue-saturation / decode-stall failure mode). - # Mirrors the single-node DSv4 agentic recipe (dsv4_fp4_mi355x.sh=8192). - # Was 65280 (255*256, the dsv4 compressor kernel uint16 token cap); 8192 - # (32*256) stays a page-size multiple well under that cap. - chunked_prefill_size: 8192 + # TP8 widens to 16384 to fix the conc>=16 queue-saturation / decode-stall + # failure mode; TP4 stays at 8192 = 32*256, a page-size multiple well + # under the dsv4 compressor kernel's uint16 token cap. Mirrors the + # single-node dsv4-fp4-mi355x-sglang-agentic-mtp recipe's TP-conditional + # fix (InferenceX PR #2784). + chunked_prefill_size: "16384 if PREFILL_TP_SIZE == 8 else 8192" context_length: 1048576 decode: - mem_fraction_static: 0.85 + mem_fraction_static: 0.86 prefill_round_robin_balance: true disagg_decode_enable_radix_cache: false dp: max_running_requests: 1024 - cuda_graph_bs_range: "1-128" + cuda_graph_bs_range: "1-256" no_dp: max_running_requests: 128 cuda_graph_bs_range: "1-128" diff --git a/benchmarks/multi_node/amd_utils/server_sglang.sh b/benchmarks/multi_node/amd_utils/server_sglang.sh index 7815e5a911..bf236fcd89 100755 --- a/benchmarks/multi_node/amd_utils/server_sglang.sh +++ b/benchmarks/multi_node/amd_utils/server_sglang.sh @@ -129,6 +129,7 @@ def parse_range(cuda_range, default_start, default_end): print(f'MODEL_BASE_FLAGS=\"{m.get(\"base_flags\", \"\")}\"') print(f'MODEL_MTP_FLAGS=\"{m.get(\"mtp_flags\", \"\")}\"') print(f'MODEL_DP_FLAGS=\"{m.get(\"dp_flags\", \"\")}\"') +print(f'MODEL_NO_DP_FLAGS=\"{m.get(\"no_dp_flags\", \"\")}\"') print(f'MODEL_EP_FLAGS=\"{m.get(\"ep_flags\", \"\")}\"') prefill = m.get('prefill', {}) @@ -395,6 +396,7 @@ build_server_config() { base_config="$(apply_kv_p2p_transfer_override "$MODEL_BASE_FLAGS")" local mtp_config="" local dp_config="" + local no_dp_config="" local ep_config="" local specific_config="" @@ -403,9 +405,16 @@ build_server_config() { mtp_config="${MODEL_MTP_FLAGS} --speculative-num-steps ${decode_mtp_size} --speculative-num-draft-tokens $((decode_mtp_size + 1))" fi - # DP config (only if DP is enabled) + # DP config (only if DP is enabled). no_dp_config is DP's mirror image: flags + # that only make sense (or only avoid conflicting with a DP-only flag) when + # DP attention is OFF, e.g. --enforce-shared-experts-fusion vs dp_flags' + # --disable-shared-experts-fusion. Keeping them in separate, mutually + # exclusive buckets means the two conflicting flags are never both emitted + # onto the same command line -- unlike base_flags, which always applies. if [[ "$enable_dp" == "true" ]]; then dp_config="$MODEL_DP_FLAGS" + else + no_dp_config="$MODEL_NO_DP_FLAGS" fi # EP config (only if EP is enabled): a2a backend, deepep mode, ep-dispatch algo. @@ -443,6 +452,9 @@ build_server_config() { if [[ -n "$dp_config" ]]; then full_config="$full_config $dp_config" fi + if [[ -n "$no_dp_config" ]]; then + full_config="$full_config $no_dp_config" + fi if [[ -n "$specific_config" ]]; then full_config="$full_config $specific_config" fi diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 85745ccdc3..d03362a4f5 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -1580,7 +1580,7 @@ minimaxm3-fp8-mi325x-vllm-agentic-mtp: - { tp: 8, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 2, 4, 8, 10, 12, 14, 16, 18] } dsv4-fp4-mi355x-sglang-disagg-agentic-hicache-mtp: - image: lmsysorg/sglang-rocm:v0.5.17-rocm720-mi35x-20260817 + image: lmsysorg/sglang-rocm:v0.5.18-rocm720-mi35x-20260903 model: deepseek-ai/DeepSeek-V4-Pro model-prefix: dsv4 runner: cluster:mi355x-amds @@ -1594,7 +1594,45 @@ dsv4-fp4-mi355x-sglang-disagg-agentic-hicache-mtp: - dram-utilization: 0.80 search-space: - spec-decoding: "mtp" - conc-list: [ 1,2,4,8,16,32,48,64 ] + conc-list: [ 1,4,8,10] + kv-offloading: none + prefill: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + additional-settings: + - "PREFILL_NODES=1" + - "CLIENT_IMAGE=lmsysorg/sglang-rocm:v0.5.18-rocm720-mi35x-20260903" + decode: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + additional-settings: + - "DECODE_NODES=1" + - "DECODE_MTP_SIZE=3" + - spec-decoding: "mtp" + conc-list: [ 16] + kv-offloading: none + prefill: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: false + additional-settings: + - "PREFILL_NODES=1" + - "CLIENT_IMAGE=lmsysorg/sglang-rocm:v0.5.18-rocm720-mi35x-20260903" + decode: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: false + additional-settings: + - "DECODE_NODES=1" + - "DECODE_MTP_SIZE=3" + - spec-decoding: "mtp" + conc-list: [ 32, 48, 64 ] kv-offloading: dram kv-offload-backend: { name: hicache } prefill: @@ -1604,7 +1642,7 @@ dsv4-fp4-mi355x-sglang-disagg-agentic-hicache-mtp: dp-attn: false additional-settings: - "PREFILL_NODES=1" - - "CLIENT_IMAGE=lmsysorg/sglang-rocm:v0.5.15.post1-rocm720-mi35x-20260719" + - "CLIENT_IMAGE=lmsysorg/sglang-rocm:v0.5.18-rocm720-mi35x-20260903" decode: num-worker: 1 tp: 8 @@ -1614,21 +1652,21 @@ dsv4-fp4-mi355x-sglang-disagg-agentic-hicache-mtp: - "DECODE_NODES=1" - "DECODE_MTP_SIZE=3" - spec-decoding: "mtp" - conc-list: [ 64, 96, 128, 192 ] + conc-list: [ 64, 96, 128, 192, 256 ] kv-offloading: dram kv-offload-backend: { name: hicache } prefill: num-worker: 1 tp: 8 - ep: 8 + ep: 1 dp-attn: true additional-settings: - "PREFILL_NODES=1" - - "CLIENT_IMAGE=lmsysorg/sglang-rocm:v0.5.15.post1-rocm720-mi35x-20260719" + - "CLIENT_IMAGE=lmsysorg/sglang-rocm:v0.5.18-rocm720-mi35x-20260903" decode: num-worker: 1 tp: 8 - ep: 8 + ep: 1 dp-attn: true additional-settings: - "DECODE_NODES=1" diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 58f5e400fe..36e4bd5556 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6685,6 +6685,18 @@ - "Bump image from lmsysorg/sglang-rocm:v0.5.16-rocm720-mi35x-20260726 to lmsysorg/sglang-rocm:v0.5.18-rocm720-mi35x-20260828" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2763 +- config-keys: + - dsv4-fp4-mi355x-sglang-disagg-agentic-hicache-mtp + scenario-type: + - agentic-coding + description: + - "Bump the image to lmsysorg/sglang-rocm:v0.5.18-rocm720-mi35x-20260903, switch the active DP-attention arm's decode side from ep:8 to ep:1 (DP-attention over TP-sharded MoE instead of expert-parallel dispatch), and widen conc-list to [64, 96, 128, 192, 256]. " + - "models.yaml: round out dp_flags with --tokenizer-worker-num 8, --enable-dp-attention-local-control-broadcast, --stream-interval 20, and --prefill-decode-interval 10 to mirror the single-node DP-attention recipe block; add enable_two_batch_overlap under prefill.dp; raise mem_fraction_static 0.85 -> 0.86 for both prefill and decode. add --enable-deepseek-v4-fp4-indexer to DeepSeek-V4-Pro-AgentX base_flags" + - "env.sh: gate the DP-attention-only SGLang knobs (SGLANG_SHARED_EXPERT_TP1, SGLANG_DP_SHARED_EXPERT_LOCAL, SGLANG_DP_USE_GATHERV, SGLANG_DP_USE_REDUCE_SCATTER, wider GPU_MAX_HW_QUEUES) behind PREFILL_ENABLE_DP/DECODE_ENABLE_DP instead of exporting them unconditionally, so TP-only arms keep GPU_MAX_HW_QUEUES=2; add SGLANG_MORI_RECV_BOUND=1 for the disagg MoRI path." + - "Split --enforce-shared-experts-fusion and --disable-shared-experts-fusion into mutually exclusive no_dp_flags/dp_flags buckets (new models.yaml no_dp_flags field plus MODEL_NO_DP_FLAGS in server_sglang.sh's build_server_config()). Both flags previously landed on the DP arm's command line simultaneously and only resolved correctly by relying on the internal check order in DeepseekV4ForCausalLM.determine_num_fused_shared_experts(), not a guaranteed contract." + - "prefill.no_dp.chunked_prefill_size: switch from a fixed 8192 to the TP-conditional formula '16384 if PREFILL_TP_SIZE == 8 else 8192'." + - "Widen decode.dp.cuda_graph_bs_range from 1-128 to 1-256 to cover the widened conc-list." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2823 - config-keys: - minimaxm3-fp4-b300-vllm-agentic-mtp