[AMD][DSV4] Disagg AgentX DP-attention tuning, HiCache ratio investigation, FP4 sparse-attention indexer

benchmarks/multi_node/amd_utils/env.sh

@@ -334,6 +334,8 @@ else
         unset MORI_MOE_MAX_INPUT_TOKENS_PREFILL
         unset MORI_MOE_MAX_INPUT_TOKENS_DECODE
 
+        export SGLANG_MORI_RECV_BOUND=1
+
         # PER_RANK dispatch tokens pinned independently (16384 prefill / 128
         # decode); server_sglang.sh prefers these over the MORI_MAX_DISPATCH_*
         # coupling when set.
@@ -346,32 +348,49 @@ else
         # Overlap plan stream on for DSv4 (global default is 0)
         # export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=0
 
-        # DSv4 model kernel routing (mirrors the single-node / manual PD recipe)
+        # DSv4 model kernel routing
         export SGLANG_DEFAULT_THINKING=1
         export SGLANG_DSV4_REASONING_EFFORT=high
-        export SGLANG_OPT_DEEPGEMM_HC_PRENORM=false
-        export SGLANG_USE_AITER=1
+        # export SGLANG_OPT_DEEPGEMM_HC_PRENORM=false
+        # export SGLANG_USE_AITER=1
         export SGLANG_USE_ROCM700A=0
-        export SGLANG_OPT_USE_FUSED_COMPRESS=true
+        # export SGLANG_OPT_USE_FUSED_COMPRESS=true
         export SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton
-        export SGLANG_OPT_FP8_WO_A_GEMM=false
-        export SGLANG_OPT_USE_JIT_INDEXER_METADATA=false
-        export SGLANG_OPT_USE_TOPK_V2=false
-        export SGLANG_OPT_USE_AITER_INDEXER=${SGLANG_OPT_USE_AITER_INDEXER:-true}
-        export SGLANG_OPT_USE_TILELANG_INDEXER=false
-        export SGLANG_OPT_USE_TILELANG_MHC_PRE=false
-        export SGLANG_OPT_USE_TILELANG_MHC_POST=false
-        export SGLANG_FP8_PAGED_MQA_LOGITS_TORCH=1
-        export SGLANG_OPT_USE_FUSED_COMPRESS_TRITON=true
-        export SGLANG_OPT_USE_MULTI_STREAM_OVERLAP=false
-        export SGLANG_ROCM_USE_MULTI_STREAM=false
+        # export SGLANG_OPT_FP8_WO_A_GEMM=false
+        # export SGLANG_OPT_USE_JIT_INDEXER_METADATA=false
+        # export SGLANG_OPT_USE_TOPK_V2=false
+        # export SGLANG_OPT_USE_AITER_INDEXER=${SGLANG_OPT_USE_AITER_INDEXER:-true}
+        # export SGLANG_OPT_USE_TILELANG_INDEXER=false
+        # export SGLANG_OPT_USE_TILELANG_MHC_PRE=false
+        # export SGLANG_OPT_USE_TILELANG_MHC_POST=false
+        # export SGLANG_FP8_PAGED_MQA_LOGITS_TORCH=1
+        # export SGLANG_OPT_USE_FUSED_COMPRESS_TRITON=true
+        # export SGLANG_OPT_USE_MULTI_STREAM_OVERLAP=false
+        # export SGLANG_ROCM_USE_MULTI_STREAM=false
         export AITER_BF16_FP8_MOE_BOUND=0
-        export SGLANG_EAGER_INPUT_NO_COPY=true
-        export SGLANG_SHARED_EXPERT_TP1=1
-        export SGLANG_DP_SHARED_EXPERT_LOCAL=1
-        export SGLANG_DP_USE_GATHERV=1
-        export SGLANG_DP_USE_REDUCE_SCATTER=1
-        export GPU_MAX_HW_QUEUES=5
+        # export SGLANG_EAGER_INPUT_NO_COPY=true
+        export TORCH_BLAS_PREFER_HIPBLASLT=1
+        # aiter batched GEMM for the absorbed MLA projections, carried by the v0.5.18
+        # image and off by default in environ.py.
+        export SGLANG_OPT_USE_AITER_BATCHED_GEMM=1
+        # DP-attention-only SGLang internal knobs (shared-expert TP1 placement,
+        # gatherv/reduce-scatter collectives) plus the wider HW-queue count DP
+        # ranks need to overlap MoRI dispatch with compute. Mirrors the
+        # single-node dsv4-fp4-mi355x-sglang-agentic-mtp recipe's
+        # `if [ "$DP_ATTENTION" = "true" ]` block (InferenceX PR #2800):
+        # these must NOT apply to the TP-only (no-DP) arms. Multi-node splits
+        # DP-attention per role (PREFILL_ENABLE_DP / DECODE_ENABLE_DP); every
+        # current search-space arm sets both the same way, so either one
+        # flags the whole node.
+        if [[ "$PREFILL_ENABLE_DP" == "true" || "$DECODE_ENABLE_DP" == "true" ]]; then
+            export SGLANG_SHARED_EXPERT_TP1=1
+            export SGLANG_DP_SHARED_EXPERT_LOCAL=1
+            export SGLANG_DP_USE_GATHERV=1
+            export SGLANG_DP_USE_REDUCE_SCATTER=1
+            export GPU_MAX_HW_QUEUES="${GPU_MAX_HW_QUEUES_DP:-5}"
+        else
+            export GPU_MAX_HW_QUEUES=2
+        fi
     fi
 
 fi

benchmarks/multi_node/amd_utils/models.yaml

@@ -10,6 +10,11 @@
 #     base_flags: str          # Common flags for both prefill and decode
 #     mtp_flags: str           # Appended to decode when DECODE_MTP_SIZE > 0
 #     dp_flags: str            # Appended when DP attention is enabled (prefill or decode)
+#     no_dp_flags: str         # Appended when DP attention is disabled. Mutually exclusive
+#                              # with dp_flags -- use this (not base_flags) for any flag
+#                              # that conflicts with one in dp_flags (e.g. shared-experts
+#                              # fusion), so the two conflicting flags are never both
+#                              # emitted onto the same command line.
 #     ep_flags: str            # Appended when EP is enabled. EP-specific MoE knobs only
 #                              # (a2a backend, deepep mode, ep-dispatch algorithm). With
 #                              # ep=1 these are dropped so the MoE runs tensor-parallel (TP).
@@ -364,36 +369,49 @@ DeepSeek-R1-0528-MXFP4-v2:
       cuda_graph_bs_range: "1-128"
 
 DeepSeek-V4-Pro-AgentX:
-  base_flags: "--watchdog-timeout 3600 --load-balance-method round_robin --kv-cache-dtype fp8_e4m3 --attention-backend dsv4 --page-size 256 --swa-full-tokens-ratio 0.1 --disable-shared-experts-fusion --tool-call-parser deepseekv4 --reasoning-parser deepseek-v4 --disaggregation-transfer-backend mori --log-level info --log-level-http error"
-  dp_flags: "--enable-dp-attention --enable-prefill-delayer"
+  base_flags: "--enable-deepseek-v4-fp4-indexer --watchdog-timeout 3600 --load-balance-method round_robin --kv-cache-dtype fp8_e4m3 --attention-backend dsv4 --page-size 256 --swa-full-tokens-ratio 0.1 --tool-call-parser deepseekv4 --reasoning-parser deepseek-v4 --disaggregation-transfer-backend mori --log-level info --log-level-http error"
+  # --enforce-shared-experts-fusion and dp_flags' --disable-shared-experts-fusion
+  # are documented by sglang as mutually exclusive (server_args.py), so each lives
+  # in its own bucket (no_dp_flags / dp_flags) rather than both riding on the
+  # command line at once relying on DeepseekV4ForCausalLM's internal check order.
+  no_dp_flags: "--enforce-shared-experts-fusion"
+  # DP-attention overrides mirror the single-node dsv4-fp4-mi355x-sglang-agentic-mtp
+  # recipe's `if [ "$DP_ATTENTION" = "true" ]` block (InferenceX PR #2800):
+  # --swa-full-tokens-ratio 0.15 widens the SWA ring for the larger DP batch, and
+  # --tokenizer-worker-num matches the only DP arm's TP (8) today -- bump this if
+  # a different-TP DP arm is added.
+  dp_flags: "--enable-dp-attention --enable-prefill-delayer --disable-shared-experts-fusion --swa-full-tokens-ratio 0.15 --enable-dp-attention-local-control-broadcast --tokenizer-worker-num 8 --stream-interval 20 --prefill-decode-interval 10"
   ep_flags: "--ep-dispatch-algorithm fake --moe-a2a-backend mori --deepep-mode normal"
   mtp_flags: "--speculative-algorithm EAGLE --speculative-eagle-topk 1"
   prefill:
-    mem_fraction_static: 0.85
+    mem_fraction_static: 0.86
     disable_radix_cache: false
     disable_cuda_graph: true
     dp:
       max_running_requests: 1024
-      chunked_prefill_size: "MORI_MAX_DISPATCH_TOKENS_PREFILL * PREFILL_TP_SIZE"  # dsv4 compressor kernel uint16 token cap (255*256)
+      chunked_prefill_size: "16384 * PREFILL_TP_SIZE if PREFILL_TP_SIZE == 8 else 8192 * PREFILL_TP_SIZE"  # dsv4 compressor kernel uint16 token cap (255*256)
       context_length: 1048576
-      # max_total_tokens: 1048576
+      # Mirrors the single-node recipe's --enable-two-batch-overlap inside the
+      # DP-attention block; prefill-only, matching that recipe's PARALLEL_ARGS
+      # scope. Also flips server_sglang.sh's PREFILL_SDMA_ENV to
+      # MORI_ENABLE_SDMA=true, required alongside two-batch-overlap.
+      enable_two_batch_overlap: true
     no_dp:
       max_running_requests: 64
-      # Small prefill chunks interleave long-context agentic prefills across
-      # requests instead of letting one ~100K-token prefill monopolize the
-      # engine (the conc>=16 queue-saturation / decode-stall failure mode).
-      # Mirrors the single-node DSv4 agentic recipe (dsv4_fp4_mi355x.sh=8192).
-      # Was 65280 (255*256, the dsv4 compressor kernel uint16 token cap); 8192
-      # (32*256) stays a page-size multiple well under that cap.
-      chunked_prefill_size: 8192
+      # TP8 widens to 16384 to fix the conc>=16 queue-saturation / decode-stall
+      # failure mode; TP4 stays at 8192 = 32*256, a page-size multiple well
+      # under the dsv4 compressor kernel's uint16 token cap. Mirrors the
+      # single-node dsv4-fp4-mi355x-sglang-agentic-mtp recipe's TP-conditional
+      # fix (InferenceX PR #2784).
+      chunked_prefill_size: "16384 if PREFILL_TP_SIZE == 8 else 8192"
       context_length: 1048576
   decode:
-    mem_fraction_static: 0.85
+    mem_fraction_static: 0.86
     prefill_round_robin_balance: true
     disagg_decode_enable_radix_cache: false
     dp:
       max_running_requests: 1024
-      cuda_graph_bs_range: "1-128"
+      cuda_graph_bs_range: "1-256"
     no_dp:
       max_running_requests: 128
       cuda_graph_bs_range: "1-128"

benchmarks/multi_node/amd_utils/server_sglang.sh

@@ -129,6 +129,7 @@ def parse_range(cuda_range, default_start, default_end):
 print(f'MODEL_BASE_FLAGS=\"{m.get(\"base_flags\", \"\")}\"')
 print(f'MODEL_MTP_FLAGS=\"{m.get(\"mtp_flags\", \"\")}\"')
 print(f'MODEL_DP_FLAGS=\"{m.get(\"dp_flags\", \"\")}\"')
+print(f'MODEL_NO_DP_FLAGS=\"{m.get(\"no_dp_flags\", \"\")}\"')
 print(f'MODEL_EP_FLAGS=\"{m.get(\"ep_flags\", \"\")}\"')
 
 prefill = m.get('prefill', {})
@@ -395,6 +396,7 @@ build_server_config() {
     base_config="$(apply_kv_p2p_transfer_override "$MODEL_BASE_FLAGS")"
     local mtp_config=""
     local dp_config=""
+    local no_dp_config=""
     local ep_config=""
     local specific_config=""
 
@@ -403,9 +405,16 @@ build_server_config() {
         mtp_config="${MODEL_MTP_FLAGS} --speculative-num-steps ${decode_mtp_size} --speculative-num-draft-tokens $((decode_mtp_size + 1))"
     fi
 
-    # DP config (only if DP is enabled)
+    # DP config (only if DP is enabled). no_dp_config is DP's mirror image: flags
+    # that only make sense (or only avoid conflicting with a DP-only flag) when
+    # DP attention is OFF, e.g. --enforce-shared-experts-fusion vs dp_flags'
+    # --disable-shared-experts-fusion. Keeping them in separate, mutually
+    # exclusive buckets means the two conflicting flags are never both emitted
+    # onto the same command line -- unlike base_flags, which always applies.
     if [[ "$enable_dp" == "true" ]]; then
         dp_config="$MODEL_DP_FLAGS"
+    else
+        no_dp_config="$MODEL_NO_DP_FLAGS"
     fi
 
     # EP config (only if EP is enabled): a2a backend, deepep mode, ep-dispatch algo.
@@ -443,6 +452,9 @@ build_server_config() {
     if [[ -n "$dp_config" ]]; then
         full_config="$full_config $dp_config"
     fi
+    if [[ -n "$no_dp_config" ]]; then
+        full_config="$full_config $no_dp_config"
+    fi
     if [[ -n "$specific_config" ]]; then
         full_config="$full_config $specific_config"
     fi

configs/amd-master.yaml

@@ -1580,7 +1580,7 @@ minimaxm3-fp8-mi325x-vllm-agentic-mtp:
       - { tp: 8, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 2, 4, 8, 10, 12, 14, 16, 18] }
 
 dsv4-fp4-mi355x-sglang-disagg-agentic-hicache-mtp:
-  image: lmsysorg/sglang-rocm:v0.5.17-rocm720-mi35x-20260817
+  image: lmsysorg/sglang-rocm:v0.5.18-rocm720-mi35x-20260903
   model: deepseek-ai/DeepSeek-V4-Pro
   model-prefix: dsv4
   runner: cluster:mi355x-amds
@@ -1594,7 +1594,45 @@ dsv4-fp4-mi355x-sglang-disagg-agentic-hicache-mtp:
     - dram-utilization: 0.80
       search-space:
       - spec-decoding: "mtp"
-        conc-list: [ 1,2,4,8,16,32,48,64 ]
+        conc-list: [ 1,4,8,10]
+        kv-offloading: none
+        prefill:
+          num-worker: 1
+          tp: 4
+          ep: 1
+          dp-attn: false
+          additional-settings:
+          - "PREFILL_NODES=1"
+          - "CLIENT_IMAGE=lmsysorg/sglang-rocm:v0.5.18-rocm720-mi35x-20260903"
+        decode:
+          num-worker: 1
+          tp: 4
+          ep: 1
+          dp-attn: false
+          additional-settings:
+          - "DECODE_NODES=1"
+          - "DECODE_MTP_SIZE=3"
+      - spec-decoding: "mtp"
+        conc-list: [ 16]
+        kv-offloading: none
+        prefill:
+          num-worker: 1
+          tp: 8
+          ep: 1
+          dp-attn: false
+          additional-settings:
+          - "PREFILL_NODES=1"
+          - "CLIENT_IMAGE=lmsysorg/sglang-rocm:v0.5.18-rocm720-mi35x-20260903"
+        decode:
+          num-worker: 1
+          tp: 8
+          ep: 1
+          dp-attn: false
+          additional-settings:
+          - "DECODE_NODES=1"
+          - "DECODE_MTP_SIZE=3"
+      - spec-decoding: "mtp"
+        conc-list: [ 32, 48, 64 ]
         kv-offloading: dram
         kv-offload-backend: { name: hicache }
         prefill:
@@ -1604,7 +1642,7 @@ dsv4-fp4-mi355x-sglang-disagg-agentic-hicache-mtp:
           dp-attn: false
           additional-settings:
           - "PREFILL_NODES=1"
-          - "CLIENT_IMAGE=lmsysorg/sglang-rocm:v0.5.15.post1-rocm720-mi35x-20260719"
+          - "CLIENT_IMAGE=lmsysorg/sglang-rocm:v0.5.18-rocm720-mi35x-20260903"
         decode:
           num-worker: 1
           tp: 8
@@ -1614,21 +1652,21 @@ dsv4-fp4-mi355x-sglang-disagg-agentic-hicache-mtp:
           - "DECODE_NODES=1"
           - "DECODE_MTP_SIZE=3"
       - spec-decoding: "mtp"
-        conc-list: [ 64, 96, 128, 192 ]
+        conc-list: [ 64, 96, 128, 192, 256 ]
         kv-offloading: dram
         kv-offload-backend: { name: hicache }
         prefill:
           num-worker: 1
           tp: 8
-          ep: 8
+          ep: 1
           dp-attn: true
           additional-settings:
           - "PREFILL_NODES=1"
-          - "CLIENT_IMAGE=lmsysorg/sglang-rocm:v0.5.15.post1-rocm720-mi35x-20260719"
+          - "CLIENT_IMAGE=lmsysorg/sglang-rocm:v0.5.18-rocm720-mi35x-20260903"
         decode:
           num-worker: 1
           tp: 8
-          ep: 8
+          ep: 1
           dp-attn: true
           additional-settings:
           - "DECODE_NODES=1"

perf-changelog.yaml

@@ -6685,6 +6685,18 @@
     - "Bump image from lmsysorg/sglang-rocm:v0.5.16-rocm720-mi35x-20260726 to lmsysorg/sglang-rocm:v0.5.18-rocm720-mi35x-20260828"
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2763
 
+- config-keys:
+    - dsv4-fp4-mi355x-sglang-disagg-agentic-hicache-mtp
+  scenario-type:
+    - agentic-coding
+  description:
+    - "Bump the image to lmsysorg/sglang-rocm:v0.5.18-rocm720-mi35x-20260903, switch the active DP-attention arm's decode side from ep:8 to ep:1 (DP-attention over TP-sharded MoE instead of expert-parallel dispatch), and widen conc-list to [64, 96, 128, 192, 256]. "
+    - "models.yaml: round out dp_flags with --tokenizer-worker-num 8, --enable-dp-attention-local-control-broadcast, --stream-interval 20, and --prefill-decode-interval 10 to mirror the single-node DP-attention recipe block; add enable_two_batch_overlap under prefill.dp; raise mem_fraction_static 0.85 -> 0.86 for both prefill and decode. add --enable-deepseek-v4-fp4-indexer to DeepSeek-V4-Pro-AgentX base_flags"
+    - "env.sh: gate the DP-attention-only SGLang knobs (SGLANG_SHARED_EXPERT_TP1, SGLANG_DP_SHARED_EXPERT_LOCAL, SGLANG_DP_USE_GATHERV, SGLANG_DP_USE_REDUCE_SCATTER, wider GPU_MAX_HW_QUEUES) behind PREFILL_ENABLE_DP/DECODE_ENABLE_DP instead of exporting them unconditionally, so TP-only arms keep GPU_MAX_HW_QUEUES=2; add SGLANG_MORI_RECV_BOUND=1 for the disagg MoRI path."
+    - "Split --enforce-shared-experts-fusion and --disable-shared-experts-fusion into mutually exclusive no_dp_flags/dp_flags buckets (new models.yaml no_dp_flags field plus MODEL_NO_DP_FLAGS in server_sglang.sh's build_server_config()). Both flags previously landed on the DP arm's command line simultaneously and only resolved correctly by relying on the internal check order in DeepseekV4ForCausalLM.determine_num_fused_shared_experts(), not a guaranteed contract."
+    - "prefill.no_dp.chunked_prefill_size: switch from a fixed 8192 to the TP-conditional formula '16384 if PREFILL_TP_SIZE == 8 else 8192'."
+    - "Widen decode.dp.cuda_graph_bs_range from 1-128 to 1-256 to cover the widened conc-list."
+  pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2823
 
 - config-keys:
     - minimaxm3-fp4-b300-vllm-agentic-mtp