From 7fc9b3beb466243b93a37046b448beb120d3908c Mon Sep 17 00:00:00 2001 From: "Artem Y. Polyakov" Date: Thu, 3 Sep 2026 09:43:33 +0200 Subject: [PATCH 1/6] CollectiveX: move to NCCL EP v0.2 and extend platform coverage NCCL EP v0.2 fixes several critical bugs that were limiting the set of supported platforms for LL mode: * Support for multiple handles per group was fixed * The combine pipeline was properly fenced The release also resolves the NCCL dependency issue that was causing HT IMA. This commit retargets the nccl-ep backend to NCCL EP v0.2 and lifts the previous platform and configuration limitations accordingly. Signed-off-by: Artem Y. Polyakov --- experimental/CollectiveX/bench/ep_nccl.py | 22 ++----------------- .../CollectiveX/configs/platform_config.json | 14 ++++++------ experimental/CollectiveX/runtime/common.sh | 2 +- 3 files changed, 10 insertions(+), 28 deletions(-) diff --git a/experimental/CollectiveX/bench/ep_nccl.py b/experimental/CollectiveX/bench/ep_nccl.py index 8d8519321e..c018eedf0a 100644 --- a/experimental/CollectiveX/bench/ep_nccl.py +++ b/experimental/CollectiveX/bench/ep_nccl.py @@ -68,24 +68,8 @@ # sizes the pre-allocated receive (and so the transport footprint), _LL_LADDER_CAP bounds which # token counts are measured. Separating them lets the ladder be clamped around a kernel defect # without moving the footprint and silently re-basing the rungs that remain. -# -# The ladder sits below the buffer because nccl_ep's low_latency.cu is a port of DeepEP's -# PRE-FIX low-latency combine: in the combine recv pipeline the reduction warps read shared -# memory and then mbarrier_arrive(emptyBarriers[stageIdx]) with no fence.proxy.async.shared::cta -# between, so the producer's next TMA load can overwrite a stage while consumer reads are still -# in flight. DeepEP closed exactly this with a one-line fence in PR #642; the fence is absent -# both at our pin and at NVIDIA/nccl master, so it is unfixed upstream. -# -# Observed on gb300 EP8 BF16 at T=256: 1 failure in 5 executions, bimodal -- healthy rows give -# max relative error 0.0039, the failure gave 0.4704, with nothing between, which is a discrete -# corrupted write rather than tolerance noise. -# -# THIS CLAMP IS NOT A SAFETY BOUNDARY. The fence is missing on every combine recv; T=256 is only -# the rung with the most pipeline iterations, and the receive plane is not even full there. Lower -# rungs are LESS LIKELY to hit the race, not immune. Restore _LL_LADDER_CAP to _LL_BUFFER_CAP -# once a fixed wheel ships. _LL_BUFFER_CAP = 256 -_LL_LADDER_CAP = 128 +_LL_LADDER_CAP = 256 class NCCLEPBackend(EPBackend): @@ -145,9 +129,7 @@ def __init__(self, args, rank, world_size, local_rank, device): def buffer_cap(self, args): if self._ll: - # Bounds which token counts are MEASURED. Below _LL_BUFFER_CAP today because the - # combine recv pipeline races (see the constants above); the harness reports every - # dropped rung rather than silently truncating. + # Bounds which token counts are MEASURED. return _LL_LADDER_CAP return None diff --git a/experimental/CollectiveX/configs/platform_config.json b/experimental/CollectiveX/configs/platform_config.json index 4281cbbee6..df33c64a9d 100644 --- a/experimental/CollectiveX/configs/platform_config.json +++ b/experimental/CollectiveX/configs/platform_config.json @@ -10,7 +10,7 @@ "scale_up_domain": 8, "scale_up_transport": "nvlink", "launcher": "single-slurm", - "backends": {"deepep-v2": [8, 16], "uccl-ep": [8], "nccl-ep": [8]}, + "backends": {"deepep-v2": [8, 16], "uccl-ep": [8], "nccl-ep": [8, 16]}, "ll_backends": {"deepep-v2": [8], "uccl-ep": [8], "nccl-ep": [8]}, "fabric": {"nic": "ConnectX-7 2x200GbE", "switch": "Arista 7060DX5-64S (Tomahawk4, 25.6T)"}, "operator": { @@ -34,7 +34,7 @@ "scale_up_domain": 8, "scale_up_transport": "nvlink", "launcher": "single-slurm", - "backends": {"deepep-v2": [8, 16], "uccl-ep": [8], "nccl-ep": [8]}, + "backends": {"deepep-v2": [8, 16], "uccl-ep": [8], "nccl-ep": [8, 16]}, "ll_backends": {"deepep-v2": [8], "uccl-ep": [8], "nccl-ep": [8]}, "fabric": {"nic": "ConnectX-7 400G", "switch": "NVIDIA Quantum-2 QM9790 (25.6T, InfiniBand)"}, "operator": { @@ -54,8 +54,8 @@ "scale_up_domain": 8, "scale_up_transport": "nvlink", "launcher": "single-slurm", - "backends": {"deepep-v2": [8, 16], "uccl-ep": [8, 16], "nccl-ep": [8]}, - "ll_backends": {"deepep-v2": [8, 16], "uccl-ep": [8], "nccl-ep": [8]}, + "backends": {"deepep-v2": [8, 16], "uccl-ep": [8, 16], "nccl-ep": [8, 16]}, + "ll_backends": {"deepep-v2": [8, 16], "uccl-ep": [8], "nccl-ep": [8, 16]}, "fabric": {"nic": "8x ConnectX-7-class 400G InfiniBand rails (bare metal, gdrdrv)", "switch": "InfiniBand (model not surveyed)"}, "operator": { "partition": "batch_1", @@ -76,7 +76,7 @@ "scale_up_domain": 8, "scale_up_transport": "nvlink", "launcher": "single-slurm", - "backends": {"deepep-v2": [8, 16], "nccl-ep": [8]}, + "backends": {"deepep-v2": [8, 16], "nccl-ep": [8, 16]}, "ll_backends": {"deepep-v2": [8], "nccl-ep": [8]}, "fabric": {"nic": "ConnectX-8 2x400GbE", "switch": "NVIDIA Spectrum-X SN5600 (51.2T)"}, "operator": { @@ -104,7 +104,7 @@ "scale_up_transport": "mnnvl", "launcher": "gb-nv", "backends": {"deepep-v2": [8, 16], "nccl-ep": [8, 16], "flashinfer-ep": [8, 16]}, - "ll_backends": {"deepep-v2": [8, 16], "nccl-ep": [8]}, + "ll_backends": {"deepep-v2": [8, 16], "nccl-ep": [8, 16]}, "fabric": {"nic": "MNNVL (scale-out not used)", "switch": "NVLink NVL72"}, "operator": { "partition": "batch", @@ -124,7 +124,7 @@ "scale_up_transport": "mnnvl", "launcher": "gb-nv", "backends": {"deepep-v2": [8, 16], "nccl-ep": [8, 16], "flashinfer-ep": [8, 16]}, - "ll_backends": {"deepep-v2": [8, 16], "nccl-ep": [8]}, + "ll_backends": {"deepep-v2": [8, 16], "nccl-ep": [8, 16]}, "fabric": {"nic": "MNNVL (scale-out not used)", "switch": "NVLink NVL72"}, "operator": { "partition": "batch_1", diff --git a/experimental/CollectiveX/runtime/common.sh b/experimental/CollectiveX/runtime/common.sh index 73388776ca..8caa63b399 100644 --- a/experimental/CollectiveX/runtime/common.sh +++ b/experimental/CollectiveX/runtime/common.sh @@ -30,7 +30,7 @@ COLLX_UCCL_COMMIT="fc1b582031221645ea9fce58aeb57187713145e3" # below are the fallback, deferred until on-metal bring-up shows the wheel is insufficient: # contrib/nccl_ep is absent from the v2.29.x / v2.30.4 release tags, so any such build must use # this post-merge master commit (which contains contrib/nccl_ep), NOT a release tag. -COLLX_NCCL4PY_SPEC="nccl4py[cu13]==0.3.1" +COLLX_NCCL4PY_SPEC="nccl-extensions[cu13]" COLLX_NCCL_EP_REPO="https://github.com/NVIDIA/nccl" COLLX_NCCL_EP_COMMIT="9d22d5dfec8391ee65b56df139d471f8e08e921e" From 293515bcdf5bbca14f8297511f3b88d50f2c7e68 Mon Sep 17 00:00:00 2001 From: "Artem Y. Polyakov" Date: Fri, 4 Sep 2026 08:40:47 +0200 Subject: [PATCH 2/6] CollectiveX: pin the NCCL EP wheel to nccl-extensions==0.1.0 Make sure that expected version of NCCL EP (v0.2) is installed. NCCL EP ships as part of nccl-extensions v0.1.0 wheel. Signed-off-by: Artem Y. Polyakov --- experimental/CollectiveX/runtime/common.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/experimental/CollectiveX/runtime/common.sh b/experimental/CollectiveX/runtime/common.sh index 8caa63b399..35254df0cd 100644 --- a/experimental/CollectiveX/runtime/common.sh +++ b/experimental/CollectiveX/runtime/common.sh @@ -30,7 +30,7 @@ COLLX_UCCL_COMMIT="fc1b582031221645ea9fce58aeb57187713145e3" # below are the fallback, deferred until on-metal bring-up shows the wheel is insufficient: # contrib/nccl_ep is absent from the v2.29.x / v2.30.4 release tags, so any such build must use # this post-merge master commit (which contains contrib/nccl_ep), NOT a release tag. -COLLX_NCCL4PY_SPEC="nccl-extensions[cu13]" +COLLX_NCCL4PY_SPEC="nccl-extensions[cu13]==0.1.0" COLLX_NCCL_EP_REPO="https://github.com/NVIDIA/nccl" COLLX_NCCL_EP_COMMIT="9d22d5dfec8391ee65b56df139d471f8e08e921e" From 0489b1f341c997e6651e88513653b33e65dd2cb3 Mon Sep 17 00:00:00 2001 From: "Artem Y. Polyakov" Date: Fri, 4 Sep 2026 15:53:53 +0200 Subject: [PATCH 3/6] CollectiveX: Update NCCL EP backend sources information Actualize runtime/common.sh: * NCCL EP repository (migrated from nccl to nccl-extensions) * NCCL EP v0.2 commit Signed-off-by: Artem Y. Polyakov --- experimental/CollectiveX/runtime/common.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/experimental/CollectiveX/runtime/common.sh b/experimental/CollectiveX/runtime/common.sh index 35254df0cd..1c3b143ffa 100644 --- a/experimental/CollectiveX/runtime/common.sh +++ b/experimental/CollectiveX/runtime/common.sh @@ -31,8 +31,8 @@ COLLX_UCCL_COMMIT="fc1b582031221645ea9fce58aeb57187713145e3" # contrib/nccl_ep is absent from the v2.29.x / v2.30.4 release tags, so any such build must use # this post-merge master commit (which contains contrib/nccl_ep), NOT a release tag. COLLX_NCCL4PY_SPEC="nccl-extensions[cu13]==0.1.0" -COLLX_NCCL_EP_REPO="https://github.com/NVIDIA/nccl" -COLLX_NCCL_EP_COMMIT="9d22d5dfec8391ee65b56df139d471f8e08e921e" +COLLX_NCCL_EP_REPO="https://github.com/NVIDIA/nccl-extensions" +COLLX_NCCL_EP_COMMIT="e57f0dad43dc1ca5bf96f09bf4075afc2eae6599" # Print bounded command output without maintaining a parallel failure taxonomy. collx_log_tail() { From 87dfcb58e3b937d0813b1a8d5c2b98611dd6eb45 Mon Sep 17 00:00:00 2001 From: "Artem Y. Polyakov" Date: Fri, 4 Sep 2026 16:23:25 +0200 Subject: [PATCH 4/6] CollectiveX: Fix documentation WRT NCCL EP v0.2 Update the documentation in relevant areas to accurately reflect platform coverage, configuration limitations, and packaging nuances. Signed-off-by: Artem Y. Polyakov --- experimental/CollectiveX/README.md | 15 ++++------ experimental/CollectiveX/bench/ep_nccl.py | 27 ++++++++++------- experimental/CollectiveX/docs/methodology.md | 29 +++++++++---------- experimental/CollectiveX/runtime/common.sh | 10 +++---- .../CollectiveX/runtime/prepare_backend.sh | 15 +++++----- experimental/CollectiveX/sweep_matrix.py | 4 +-- 6 files changed, 51 insertions(+), 49 deletions(-) diff --git a/experimental/CollectiveX/README.md b/experimental/CollectiveX/README.md index 34b25f5132..3edde334fe 100644 --- a/experimental/CollectiveX/README.md +++ b/experimental/CollectiveX/README.md @@ -17,9 +17,9 @@ dimension, with a BF16 control plus, on every backend whose FP8 dispatch is supp (DeepEP V2, MoRI, UCCL-EP, FlashInfer EP), an FP8 dispatch, caller-prequantized in `normal` mode (in `low-latency` the DeepEP and UCCL-EP kernels quantize internally from BF16. MoRI stays caller-prequantized, while FlashInfer has no `low-latency` path). That caller-side quantize is charged -inside the measured dispatch, because a production forward pass pays it on the critical path. NCCL EP -is BF16-only this release, so it emits the control alone. Coverage is uniform routing only. Cases run -in one of two modes: +inside the measured dispatch, because a production forward pass pays it on the critical path. NCCL +EP is BF16-only here, so it emits the control alone; wiring the FP8 dispatch support added in v0.2 +is future work. Coverage is uniform routing only. Cases run in one of two modes: - `normal` uses `layout-and-dispatch-v1`, rank-deduplicated token payloads, and activation-only, unweighted rank-sum combine. It runs the full decode and prefill ladders. @@ -38,12 +38,7 @@ in one of two modes: `kNumMaxWarpGroups` is 16, since upstream raised `kNumMaxTopK` 9 -> 16 (uccl#1016, 2026-07-13) and our pin is six days later. The product is 16 for every CU count, so this is a dated regression rather than a hardware limit, and the AMD SKUs keep UCCL-EP normal mode without LL), - and NCCL EP EP8 on all six NVIDIA SKUs, restored once the single-handle fix removed the - [NVIDIA/nccl#2303](https://github.com/NVIDIA/nccl/issues/2303) signal aliasing that had wedged them. - B300 carries the `candidate` NCCL EP as its *only* low-latency row, so it has no production - decode coverage. DeepEP V2 emits no LL row at all on B300 (the IBGDA address-handle wall in the - backend table below), and `_ll_runnable` adds only runnable cells, so that wall is prose here - rather than a classified matrix row). + and NCCL EP at EP8 on all six NVIDIA SKUs and at EP16 on B200 (nscale), GB200 and GB300). Scoped single-node EP8 runs over the intra-node NVLink/XGMI low-latency path (no `/dev/gdrdrv` needed, as validated on H200 with it absent). NVSHMEM/IBGDA on the wire carries payload only on a multi-node scale-out (EP16) run. The legacy Buffer still @@ -130,7 +125,7 @@ scale-up domain. | DeepEP V2 | `production`, with vLLM `--all2all-backend deepep_v2`, SGLang `--moe-a2a-backend deepep` | `normal` mode is PR #605 `ElasticBuffer` plus exact upstream #630 and #640 fixes: LSA for scale-up and GIN for x86 EP16 scale-out. FP8 dispatch via `use_fp8_dispatch` (blockwise e4m3fn) alongside BF16. `low-latency` mode is the legacy `deep_ep.Buffer` IBGDA decode kernels (per-expert padded layout, weighted combine, `use_fp8` e4m3fn), decode only, with EP8 wherever enabled, plus EP16 on GB200/GB300 (inside the MNNVL domain) and on B200's nscale bare-metal pool (IBGDA over native IB rails with `/dev/gdrdrv`, although the prior virtualized b200 pool could never run it). B300 is an unsupported coverage row in `low-latency`: the legacy Buffer self-enables NVSHMEM IBGDA even for a single-node EP8 run, and on B300 address-handle creation fails (`ibgda.cpp:2234 Unable to create ah`), rc255 on all eight ranks. `NVSHMEM_DISABLE_IB=1` does not help. The Buffer re-enables IBGDA regardless, and the run fails identically with it set and unset (measured on b300-002 and b300-011) | | MoRI | `production`, with vLLM `--all2all-backend mori_*`, SGLang `--moe-a2a-backend mori` | `normal` mode uses the direct `IntraNode` kernel for scale-up EP8 on every CDNA SKU. EP16 remains an unsupported coverage row on all three CDNA SKUs. Part of the old ROCm/mori#475 corruption was this harness passing dispatch's returned recv-slot indices to `combine()` instead of the rank's own routing (root-caused upstream, guarded by ROCm/mori#546, kernels unchanged) — with the corrected call, single-shot InterNodeV1 is clean through T=512 on mi355x — but a residual stochastic corruption remains from T~128 up under repeated execution and is near-certain at prefill sizes (run 33045314017/33050026476; unaffected by per-pair drains, so not a buffer-reuse race; upstream cannot reproduce on ionic driver 26.03 vs our 25.11). The tw pairs additionally have no cross-node GPU fabric, and mi355x EP16 has no publishable transport today: uccl-ep's CPU-proxy RDMA is functional on Pollara but ~13x under its documented bandwidth (~6 GB/s vs 82; unchanged by registration mode or traffic class — ionic-driver suspect, same 25.11-vs-26.03 delta as the mori residual). `low-latency` mode selects the `IntraNodeLL` decode kernel (single-call, pure-intranode, same compact layout and unweighted combine as `IntraNode`), decode/EP8 only. FP8 dispatch is caller-prequantized (per-SKU e4m3fnuz on gfx942, e4m3fn on gfx950). Combine stays BF16 (`quant_type=none`) alongside BF16 dispatch | | UCCL-EP | `candidate` (no engine exposes a UCCL-EP selector) | [UCCL](https://github.com/uccl-project/uccl) EP: a drop-in, API-identical DeepEP replacement whose CPU proxies issue GPUDirect RDMA over plain `libibverbs` (no NVSHMEM/IBGDA), with software message ordering, atomics, and flow control. Scale-up is single-node `cudaIpc` over NVLink/XGMI (never MNNVL). `normal` mode is the legacy `Buffer` `dispatch`/`combine` (unweighted rank-sum). `low-latency` reuses the legacy `low_latency_dispatch`/`low_latency_combine` decode kernels (weighted combine), decode/EP8 only. FP8 dispatch is caller-prequantized in `normal` mode (blockwise e4m3fn, per-SKU e4m3fnuz on gfx942). In `low-latency` mode the caller sends BF16 and the decode kernel quantizes to e4m3 internally (`use_fp8`). Combine is BF16. Runs on NVIDIA and AMD (H100/H200/B200 + MI300X/MI325X/MI355X), EP8 scale-up. Cross-node EP16 is functional (the internode RDMA path connects and the light case passes correctness) but its CPU-proxy throughput overruns the standardized per-case wall-clock budget on heavy token counts, so EP16 is an unsupported coverage row for now | -| NCCL EP | `candidate` (NVIDIA's own library, but no engine exposes an NCCL-EP selector) | [NCCL EP](https://github.com/NVIDIA/nccl/tree/master/contrib/nccl_ep): NVIDIA's native MoE dispatch/combine on the NCCL Device API, using LSA (NVLink load/store) intra-node and GIN (GPU-Initiated Networking) inter-node, driven through the `nccl4py` bindings. `normal` mode selects the `HIGH_THROUGHPUT` algorithm (FLAT `[N, hidden]` receive, unweighted rank-sum combine). The `LOW_LATENCY` algorithm carries an EP8 `ll_backends` row on all six NVIDIA SKUs, restored once the single-handle fix removed the NVIDIA/nccl#2303 signal aliasing. That LL decode ladder is clamped to T<=128, below its 256-slot receive: `nccl_ep`'s combine recv pipeline is a port of DeepEP's pre-#642 kernel and is missing the same shared-memory fence before `mbarrier_arrive`, which corrupted T=256 on GB300 in 1 of 5 executions. It was bimodal, with healthy rows at 0.0039 relative error against 0.4704 on the failure. The fence is absent at NVIDIA/nccl master, so it is unfixed upstream. The clamp lowers exposure and is **not** a safety boundary: the fence is missing on every combine recv and T=256 is merely the rung with the most pipeline iterations, so lower rungs are less likely to hit the race rather than immune. Restore when a fixed wheel ships BF16 only: `contrib/nccl_ep/RELEASE.md` says "No FP8 support", so no FP8 case is emitted. That note is worth re-testing rather than trusting, because the C library at our pinned commit does read `inputs->scales` and switch on e4m3/e5m2, the two documented FP8 exclusions are expert-major layouts we do not use, and `NVIDIA/nccl` has not moved since 2026-06-11 while `NVIDIA/nccl-extensions` has replaced that row outright. NVIDIA-only and CUDA 13 only. EP8 scale-up on H100/H200/B200/B300 plus EP8 and EP16 on GB200/GB300, where EP16 stays inside the MNNVL scale-up domain. x86 EP16 scale-out is an unsupported coverage row: the cross-node GIN path faults inside `nccl_ep.cc` identically on RoCE and IB across four SKUs, a GDAKI limit rather than a fabric-selection one | +| NCCL EP | `candidate` (NVIDIA's own library, but no engine exposes an NCCL-EP selector) | [NCCL EP](https://github.com/NVIDIA/nccl-extensions/tree/main/nccl_ep): NVIDIA's native MoE dispatch/combine on the NCCL Device API, using LSA (NVLink load/store) intra-node and GIN (GPU-Initiated Networking) inter-node, driven through the `nccl-extensions` wheel's `nccl.ep` bindings (`nccl.core` comes from its `nccl4py` dependency). `normal` mode selects the `HIGH_THROUGHPUT` algorithm (FLAT `[N, hidden]` receive, unweighted rank-sum combine) and runs EP8 and EP16 on all six NVIDIA SKUs: on GB200/GB300 EP16 stays inside the MNNVL scale-up domain, and on H100/H200/B200/B300 it is a 2x8 RDMA scale-out over the cross-node GIN path. The `LOW_LATENCY` algorithm carries an `ll_backends` row at EP8 on all six NVIDIA SKUs and at EP16 on B200 (nscale), GB200 and GB300. The LL decode ladder runs to its full 256-slot receive, so only the 512 point is dropped. BF16 only; wiring the FP8 dispatch support added in v0.2 is future work. NVIDIA-only and CUDA 13 only. | | FlashInfer EP | `production`, with vLLM `--all2all-backend flashinfer_nvlink_one_sided` | [FlashInfer](https://github.com/flashinfer-ai/flashinfer) `MoeAlltoAll`: TensorRT-LLM's one-sided MNNVL all-to-all, where each rank writes tokens straight into its peers' workspace windows and combine reads them back, with no send/recv pairing and no NVSHMEM. `normal` mode only (there is one kernel family and no separate decode path), and GB200/GB300 only, since the transport is MNNVL. FP8 dispatch is caller-prequantized blockwise e4m3fn, carried as a fourth dispatch payload alongside its per-128-block FP32 scales, with the combine plane forced to BF16. The C++ `toNvDataType` accepts only fp16/bf16/fp32 for combine, so an FP8 combine buffer would raise rather than corrupt. EP8 and EP16, both inside the scale-up domain. Unlike every other backend here, its combine accumulates in the PAYLOAD dtype rather than FP32: wheels before 0.6.16 reduce the top-k contributions with a pairwise BF16 tree that rounds at every level, so the oracle models that reduction directly (`combine_reduction = "topk-slot-tree"`) instead of widening the tolerance. 0.6.16 moved the accumulator to FP32, and the adapter switches models on the installed version | DeepEP V2 means the `ElasticBuffer` implementation introduced by diff --git a/experimental/CollectiveX/bench/ep_nccl.py b/experimental/CollectiveX/bench/ep_nccl.py index c018eedf0a..e93deb6f49 100644 --- a/experimental/CollectiveX/bench/ep_nccl.py +++ b/experimental/CollectiveX/bench/ep_nccl.py @@ -1,9 +1,9 @@ #!/usr/bin/env python3 """NCCL EP adapter: NVIDIA's native MoE dispatch/combine on the NCCL Device API. -NCCL EP (github.com/NVIDIA/nccl/contrib/nccl_ep, arXiv 2603.13606) is a ground-up MoE -communication library built on NCCL's Device API — LSA (NVLink load/store) intra-node and -GIN (GPU-Initiated Networking) inter-node — with two algorithms selected per case: +NCCL EP (https://github.com/NVIDIA/nccl-extensions/tree/main/nccl_ep, arXiv 2603.13606) +is a ground-up MoE communication library built on NCCL's Device API — LSA (NVLink load/store) +intra-node and GIN (GPU-Initiated Networking) inter-node — with two algorithms selected per case: normal -> HIGH_THROUGHPUT (HT), the Hybrid-EP-derived prefill/train path. FLAT recv layout ``[N, hidden]`` (one row per received token, no expert structure) with an unweighted rank-sum combine — identical semantics to deepep-v2 normal. @@ -13,19 +13,24 @@ Because both map exactly onto the two combine contracts CollectiveX already models, the scale_up_domain two-level combine oracle in ep_harness applies unchanged. -BF16 only: `contrib/nccl_ep/RELEASE.md` says "No FP8 support", so this adapter does not -override the FP8 encode hooks (SUPPORTED_PRECISIONS=("bf16",)). Re-test before trusting that -note — the C library at our pinned commit reads `inputs->scales` and switches on e4m3/e5m2, -and the two documented FP8 exclusions are expert-major layouts we do not use. +BF16 only, by adapter scope rather than library limit; NCCL EP v0.2 added dispatch-path +quantization (`nccl_ep/docs/release/RELEASE_NOTES_v0.2.md`) carrying both recipes this suite +would need: `QUANT_FWD` forwards caller-supplied scales, matching the `normal`-mode +caller-prequantized FP8 the other backends run, and `DS_FP8E3M4` quantizes BF16 to E4M3 with +per-128 FP32 scales inside the LL dispatch, matching their `low-latency` in-kernel path. +Neither is wired here yet, so this adapter keeps SUPPORTED_PRECISIONS=("bf16",) and emits no +FP8 case; enabling FP8 support is future work. +Combine stays BF16 either way: the non-experimental combine path accepts only bf16/fp16/fp32. Communicator bootstrap: NCCL EP forms its OWN NCCL communicator (separate from PyTorch's process group) via ``Communicator.init(nranks, rank, unique_id)``. Upstream broadcasts the unique id with MPI; CollectiveX has no MPI, so rank 0 generates the id and we broadcast its bytes over the already-initialized torch process group (see ``_bootstrap_comm``). -Python bindings are ``nccl4py`` (``import nccl.core`` + ``nccl.ep``); the API surface used -here is verified against upstream ``bindings/nccl4py/nccl/ep`` and driven exactly as -``contrib/nccl_ep/ep_test.py`` drives it. +Python bindings: ``nccl.ep`` ships in the ``nccl-extensions`` wheel and ``nccl.core`` in its +``nccl4py`` dependency, which is why COLLX_NCCL4PY_SPEC keeps that name. The API surface used +here is verified against upstream ``python/nccl/ep`` and driven exactly as +``nccl_ep/ep_test.py`` drives it. """ from __future__ import annotations @@ -262,6 +267,8 @@ def _ensure_handle(self, p): does not hang outright, which makes every latency drawn from such a run suspect. Filed upstream as NVIDIA/nccl#2303; reproduced on a stock wheel by ladder [1] (one handle, clean) vs ladder [1, 2] (two handles, 64 dispatch + 6 combine receive timeouts -> 719). + UPDATE: the support for multiple handles per group was added in NCCL EP v0.2, this + backend should support it now. Wiring it is future work. `ncclEpInitHandle` takes no token count and `ncclEpUpdateHandle` is documented as a "per-step collective: prepare the handle for the given top-k routing decisions", so diff --git a/experimental/CollectiveX/docs/methodology.md b/experimental/CollectiveX/docs/methodology.md index e5798e1eeb..d6fa52912e 100644 --- a/experimental/CollectiveX/docs/methodology.md +++ b/experimental/CollectiveX/docs/methodology.md @@ -81,7 +81,8 @@ inside it. Compiling the quantize reduces this rather than causing it. An eager quantize op into an already-busy stream, so the small-T end of an FP8 `normal` row is the least production-representative number the suite emits. Compare FP8 and BF16 at the top of the ladder. `low-latency` rows are unaffected: those kernels quantize internally or take pre-quantized input by -API contract. NCCL EP is BF16-only this release, so its cells carry the control alone. The +API contract. NCCL EP is BF16-only here, so its cells carry the control alone; wiring the FP8 +dispatch support added in v0.2 is future work. The per-backend precision set lives in `sweep_matrix.py`'s `BACKEND_PRECISIONS` and a backend never emits a case for a precision it does not support. `normal`-mode cases use the `layout-and-dispatch-v1` semantics. `low-latency` cases use each backend's decode-kernel semantics @@ -182,13 +183,13 @@ than by precedent. UCCL-EP is a drop-in, API-identical DeepEP replacement that k `libibverbs`, with no NVSHMEM/IBGDA and with software message ordering, atomics, and flow control. Its scale-up is single-node `cudaIpc` over NVLink/XGMI (so the scale-up domain is one physical node, never MNNVL) and its EP16 scale-out uses the same per-SKU RDMA rails as the other backends. NCCL EP -is NVIDIA's native MoE dispatch/combine on the NCCL Device API, driven through the `nccl4py` -bindings. `normal` mode selects its `HIGH_THROUGHPUT` algorithm, whose FLAT `[N, hidden]` receive and -unweighted rank-sum combine match `layout-and-dispatch-v1` exactly, so the same oracle applies. It is -NVIDIA-only and CUDA 13 only, and runs EP8 scale-up on H100/H200/B200/B300 plus EP8 and EP16 on -GB200/GB300, where EP16 stays inside the MNNVL scale-up domain. X86 EP16 scale-out is an unsupported -coverage row, its cross-node GIN path faulting inside `nccl_ep.cc` identically on RoCE and IB across -four SKUs. This is a GDAKI limit, not a fabric-selection one. FlashInfer EP is TensorRT-LLM's one-sided MNNVL `MoeAlltoAll`, in which each rank writes tokens directly into its peers' workspace windows and combine reads them back, so there is no send/recv pairing and no NVSHMEM. It is GB200/GB300-only for that reason, and runs EP8 and EP16 inside the MNNVL scale-up domain. Its combine is the one place a backend's accumulator precision changes the expectation rather than the tolerance: through 0.6.15 the kernel holds its top-k accumulators in the payload dtype and reduces them with a hand-unrolled pairwise tree, so every level rounds to BF16, and the oracle reproduces that tree exactly rather than loosening the gate to absorb it (0.6.16 rewrote the accumulator to FP32. The adapter reads the installed version and picks the matching model). Those throughput kernels run across the full token ladder in the `normal` mode. Its FP8 dispatch is the one (backend, precision) pair here that is realizable but off every deployed path. vLLM accepts only nvfp4/mxfp8/bf16 on this transport, so `sweep_matrix.py`'s `OFF_PATH_PRECISIONS` keeps it out of the default matrix and a production sweep measures only configurations an engine can select. Naming the precision explicitly (`--precisions fp8`) opts it back in for transport comparison against DeepEP V2/UCCL-EP at matching bytes and block size: the one place a precision filter ADDS rows rather than only removing them. +is NVIDIA's native MoE dispatch/combine on the NCCL Device API, driven through the +`nccl-extensions` wheel's `nccl.ep` bindings (`nccl.core` comes from its `nccl4py` dependency). +`normal` mode selects its `HIGH_THROUGHPUT` algorithm, whose FLAT `[N, hidden]` receive and +unweighted rank-sum combine match `layout-and-dispatch-v1` exactly, so the same oracle applies, and +runs EP8 and EP16 on all six NVIDIA SKUs: EP16 stays inside the MNNVL scale-up domain on GB200/GB300 +and is a 2x8 RDMA scale-out over the cross-node GIN path on H100/H200/B200/B300. It is NVIDIA-only +and CUDA 13 only. FlashInfer EP is TensorRT-LLM's one-sided MNNVL `MoeAlltoAll`, in which each rank writes tokens directly into its peers' workspace windows and combine reads them back, so there is no send/recv pairing and no NVSHMEM. It is GB200/GB300-only for that reason, and runs EP8 and EP16 inside the MNNVL scale-up domain. Its combine is the one place a backend's accumulator precision changes the expectation rather than the tolerance: through 0.6.15 the kernel holds its top-k accumulators in the payload dtype and reduces them with a hand-unrolled pairwise tree, so every level rounds to BF16, and the oracle reproduces that tree exactly rather than loosening the gate to absorb it (0.6.16 rewrote the accumulator to FP32. The adapter reads the installed version and picks the matching model). Those throughput kernels run across the full token ladder in the `normal` mode. Its FP8 dispatch is the one (backend, precision) pair here that is realizable but off every deployed path. vLLM accepts only nvfp4/mxfp8/bf16 on this transport, so `sweep_matrix.py`'s `OFF_PATH_PRECISIONS` keeps it out of the default matrix and a production sweep measures only configurations an engine can select. Naming the precision explicitly (`--precisions fp8`) opts it back in for transport comparison against DeepEP V2/UCCL-EP at matching bytes and block size: the one place a precision filter ADDS rows rather than only removing them. A second `low-latency` mode adds each backend's decode-optimized kernel family. On DeepEP it drives the legacy `deep_ep.Buffer` low-latency decode kernels (`low_latency_dispatch`/`low_latency_combine`), @@ -210,13 +211,11 @@ EP8 on MI300X/MI325X/MI355X, and UCCL-EP EP8 on H100/H200/B200 only (the legacy `Buffer` low-latency kernels, which at EP8 run `cudaIpc` over NVLink, not the CPU-proxy RDMA path, because the adapter passes `is_intranode` and UCCL then never starts its proxies. The AMD SKUs drop LL: upstream raised `kNumMaxTopK` 9 -> 16 six days before our pin, and the resulting host assert -cannot hold on AMD's 16 warp groups), and NCCL EP EP8 on all six NVIDIA SKUs. Its -`LOW_LATENCY` algorithm is the DeepEP-derived decode path, EXPERT_MAJOR receive with a source-side -weighted-kernel-sum combine. Those rows were dropped while every LL leg wedged on stale peer signals -([NVIDIA/nccl#2303](https://github.com/NVIDIA/nccl/issues/2303)) and restored once the single-handle -adapter removed the aliasing that caused it. B300 carries NCCL EP as its only -low-latency row, and it is a `candidate` transport, so that SKU publishes no production decode -coverage. Whether a given SKU/backend/EP/mode cell is attempted is a capability +cannot hold on AMD's 16 warp groups), and NCCL EP at EP8 on all six NVIDIA SKUs and at EP16 on +B200 (nscale), GB200 and GB300. Its `LOW_LATENCY` algorithm is the DeepEP-derived decode path, +EXPERT_MAJOR receive with a source-side weighted-kernel-sum combine, and its decode ladder runs to +the full 256-slot receive, so only the 512 point is dropped. +Whether a given SKU/backend/EP/mode cell is attempted is a capability fact. Whether it succeeded is decided only by the emitted artifact. ## Workload Identity diff --git a/experimental/CollectiveX/runtime/common.sh b/experimental/CollectiveX/runtime/common.sh index 1c3b143ffa..82bdfda587 100644 --- a/experimental/CollectiveX/runtime/common.sh +++ b/experimental/CollectiveX/runtime/common.sh @@ -25,11 +25,11 @@ COLLX_UCCL_REPO="https://github.com/uccl-project/uccl" COLLX_UCCL_COMMIT="fc1b582031221645ea9fce58aeb57187713145e3" # NCCL EP (NVIDIA's native MoE dispatch/combine on the NCCL Device API). Primary path is the -# published nccl4py wheel — it bundles libnccl_ep.so's JIT runtime and pulls the matching -# nvidia-nccl-cu13 (>= 2.30, carrying the Device API + GIN nccl.ep needs). The from-source pins -# below are the fallback, deferred until on-metal bring-up shows the wheel is insufficient: -# contrib/nccl_ep is absent from the v2.29.x / v2.30.4 release tags, so any such build must use -# this post-merge master commit (which contains contrib/nccl_ep), NOT a release tag. +# published nccl-extensions wheel — it bundles libnccl_ep.so's JIT runtime and pulls the matching +# nvidia-nccl-cu13 (2.30.7, pinned by the [cu13] extra, carrying the Device API + GIN nccl.ep +# needs). The from-source pins below are the fallback, deferred until on-metal bring-up shows the +# wheel is insufficient; the commit is the head of the release the wheel is built from, so a +# from-source build and the wheel resolve to the same nccl_ep tree. COLLX_NCCL4PY_SPEC="nccl-extensions[cu13]==0.1.0" COLLX_NCCL_EP_REPO="https://github.com/NVIDIA/nccl-extensions" COLLX_NCCL_EP_COMMIT="e57f0dad43dc1ca5bf96f09bf4075afc2eae6599" diff --git a/experimental/CollectiveX/runtime/prepare_backend.sh b/experimental/CollectiveX/runtime/prepare_backend.sh index fed9b6ec03..2531906d07 100644 --- a/experimental/CollectiveX/runtime/prepare_backend.sh +++ b/experimental/CollectiveX/runtime/prepare_backend.sh @@ -429,8 +429,9 @@ nccl_ep_cache_root() { # Put the installed wheel ($root/site) on PYTHONPATH for the probe and rank tasks, and the # wheel-bundled NCCL runtime lib dir ahead of the image torch's older NCCL on the loader path -# (nccl.ep needs NCCL >= 2.29.3's Device API + GIN; the image torch bundles an older NCCL). Both -# PYTHONPATH and LD_LIBRARY_PATH are already carried to the ranks by write_rank_env. +# (nccl.ep needs a Device API + GIN capable NCCL — the nccl-extensions[cu13] extra pins 2.30.7 — +# while the image torch bundles an older one). Both PYTHONPATH and LD_LIBRARY_PATH are already +# carried to the ranks by write_rank_env. nccl_ep_activate() { local root="$1" site="$1/site" nccl_lib [ -d "$site" ] || { collx_log "ERROR: NCCL EP cache site is unavailable"; return 1; } @@ -459,10 +460,10 @@ import nccl.ep # noqa: F401 PY } -# Primary install: the published nccl4py[cu13] wheel + deps into $root/site via pip --target -# (self-contained; the runtime imports it through PYTHONPATH, so cache-hit and cache-miss paths -# import identically — mirrors uccl_install's copy-to-cache scheme). The from-source fallback -# (OpenMPI + build NCCL + contrib/nccl_ep from COLLX_NCCL_EP_COMMIT, with a matching launcher +# Primary install: the published nccl-extensions[cu13] wheel + deps into $root/site via pip +# --target (self-contained; the runtime imports it through PYTHONPATH, so cache-hit and cache-miss +# paths import identically — mirrors uccl_install's copy-to-cache scheme). The from-source fallback +# (OpenMPI + build NCCL + nccl_ep from COLLX_NCCL_EP_COMMIT, with a matching launcher # source-staging arm) is deferred until bring-up shows the wheel does not ship libnccl_ep.so. nccl_ep_install() { local root="$1" site="$1/site" @@ -476,7 +477,7 @@ nccl_ep_install() { # not apply; torch is imported from the image at runtime (nccl.ep's torch interop resolver). python3 -m pip install -q --disable-pip-version-check --no-input \ --target "$site" "$COLLX_NCCL4PY_SPEC" >&2 2>&1 \ - || { collx_log "ERROR: NCCL EP nccl4py install failed"; return 1; } + || { collx_log "ERROR: NCCL EP wheel install failed"; return 1; } nccl_ep_activate "$root" \ || { collx_log "ERROR: NCCL EP environment activation failed"; return 1; } nccl_ep_probe || { collx_log "ERROR: NCCL EP import probe failed"; return 1; } diff --git a/experimental/CollectiveX/sweep_matrix.py b/experimental/CollectiveX/sweep_matrix.py index a99613d689..919e88283f 100644 --- a/experimental/CollectiveX/sweep_matrix.py +++ b/experimental/CollectiveX/sweep_matrix.py @@ -38,8 +38,8 @@ def _load_config(name: str) -> dict[str, Any]: "deepep-v2": ("bf16", "fp8"), "mori": ("bf16", "fp8"), "uccl-ep": ("bf16", "fp8"), - # NCCL EP is BF16-only on the strength of RELEASE.md's "No FP8 support" row, which is - # worth re-testing — see the note in bench/ep_nccl.py. + # NCCL EP is BF16-only now; NCCL EP v0.2 supports FP8 dispatch, but the integration is + # pending (see bench/ep_nccl.py). "nccl-ep": ("bf16",), # FlashInfer FP8 is dispatch-side only (scales as a fourth payload, combine stays BF16), # and uses the same per-128-block e4m3 recipe as deepep-v2/uccl-ep so the axis is From eca48f79addc281c8f2a0ed27e64234085f150b2 Mon Sep 17 00:00:00 2001 From: "Artem Y. Polyakov" Date: Fri, 4 Sep 2026 17:57:35 +0200 Subject: [PATCH 5/6] CollectiveX: Minor fix in NCCL EP backend variable naming Rename COLLX_NCCL4PY_SPEC -> COLLX_NCCL_EP_SPEC to reflect that starting from NCCL EP v0.2 it is destributed via nccl-extensions wheel (not nccl4py). Signed-off-by: Artem Y. Polyakov --- experimental/CollectiveX/bench/ep_nccl.py | 5 +++-- experimental/CollectiveX/runtime/common.sh | 5 ++++- experimental/CollectiveX/runtime/prepare_backend.sh | 12 ++++++------ 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/experimental/CollectiveX/bench/ep_nccl.py b/experimental/CollectiveX/bench/ep_nccl.py index e93deb6f49..80aea627ac 100644 --- a/experimental/CollectiveX/bench/ep_nccl.py +++ b/experimental/CollectiveX/bench/ep_nccl.py @@ -28,8 +28,9 @@ bytes over the already-initialized torch process group (see ``_bootstrap_comm``). Python bindings: ``nccl.ep`` ships in the ``nccl-extensions`` wheel and ``nccl.core`` in its -``nccl4py`` dependency, which is why COLLX_NCCL4PY_SPEC keeps that name. The API surface used -here is verified against upstream ``python/nccl/ep`` and driven exactly as +``nccl4py`` dependency, so the installed spec (COLLX_NCCL_EP_SPEC) names nccl-extensions and +pulls nccl4py transitively. The API surface used here is verified against upstream +``python/nccl/ep`` and driven exactly as ``nccl_ep/ep_test.py`` drives it. """ from __future__ import annotations diff --git a/experimental/CollectiveX/runtime/common.sh b/experimental/CollectiveX/runtime/common.sh index 82bdfda587..6eff113845 100644 --- a/experimental/CollectiveX/runtime/common.sh +++ b/experimental/CollectiveX/runtime/common.sh @@ -30,7 +30,10 @@ COLLX_UCCL_COMMIT="fc1b582031221645ea9fce58aeb57187713145e3" # needs). The from-source pins below are the fallback, deferred until on-metal bring-up shows the # wheel is insufficient; the commit is the head of the release the wheel is built from, so a # from-source build and the wheel resolve to the same nccl_ep tree. -COLLX_NCCL4PY_SPEC="nccl-extensions[cu13]==0.1.0" +# The package version and the library version differ on purpose: nccl-extensions 0.1.0 ships +# NCCL EP v0.2 (NCCL_EP_MAJOR 0 / MINOR 2 in nccl_ep/include/nccl_ep.h). Do not "align" one to +# the other — the docs refer to the library version, this spec to the package version. +COLLX_NCCL_EP_SPEC="nccl-extensions[cu13]==0.1.0" COLLX_NCCL_EP_REPO="https://github.com/NVIDIA/nccl-extensions" COLLX_NCCL_EP_COMMIT="e57f0dad43dc1ca5bf96f09bf4075afc2eae6599" diff --git a/experimental/CollectiveX/runtime/prepare_backend.sh b/experimental/CollectiveX/runtime/prepare_backend.sh index 2531906d07..d32d9d1e7b 100644 --- a/experimental/CollectiveX/runtime/prepare_backend.sh +++ b/experimental/CollectiveX/runtime/prepare_backend.sh @@ -408,7 +408,7 @@ uccl_prepare() { # Slug of the pinned pip spec, safe as a cache-dir path component. nccl_ep_spec_slug() { - printf '%s' "$COLLX_NCCL4PY_SPEC" | tr -cs 'A-Za-z0-9_.-' '-' + printf '%s' "$COLLX_NCCL_EP_SPEC" | tr -cs 'A-Za-z0-9_.-' '-' } # Cache root keyed by cpu + build arch + image + pinned wheel spec, under the shared /cx-cache @@ -472,11 +472,11 @@ nccl_ep_install() { fi mkdir -m 700 "$root" || { collx_log "ERROR: NCCL EP cache-create failed"; return 1; } mkdir -p "$site" || { collx_log "ERROR: NCCL EP cache-site-create failed"; return 1; } - collx_log "NCCL EP: installing $COLLX_NCCL4PY_SPEC (pip --target)" + collx_log "NCCL EP: installing $COLLX_NCCL_EP_SPEC (pip --target)" # --target installs into an isolated tree and does not touch the system env, so PEP 668 does # not apply; torch is imported from the image at runtime (nccl.ep's torch interop resolver). python3 -m pip install -q --disable-pip-version-check --no-input \ - --target "$site" "$COLLX_NCCL4PY_SPEC" >&2 2>&1 \ + --target "$site" "$COLLX_NCCL_EP_SPEC" >&2 2>&1 \ || { collx_log "ERROR: NCCL EP wheel install failed"; return 1; } nccl_ep_activate "$root" \ || { collx_log "ERROR: NCCL EP environment activation failed"; return 1; } @@ -496,7 +496,7 @@ nccl_ep_prepare() { command -v flock >/dev/null \ || { collx_log "ERROR: flock is required for NCCL EP caching"; return 1; } mkdir -p "${root%/*}" || return 1 - collx_log "NCCL EP: preparing $COLLX_NCCL4PY_SPEC (shared cache $root)" + collx_log "NCCL EP: preparing $COLLX_NCCL_EP_SPEC (shared cache $root)" if ! ( [ ! -L "$lock_path" ] || { collx_log "ERROR: NCCL EP cache lock is unsafe"; exit 1; } (umask 077; : >> "$lock_path") && chmod 600 "$lock_path" \ @@ -511,14 +511,14 @@ nccl_ep_prepare() { fi else root="/tmp/collectivex-nccl-ep-cache-$(nccl_ep_spec_slug)" - collx_log "NCCL EP: preparing $COLLX_NCCL4PY_SPEC (node-local $root; no shared cache mounted)" + collx_log "NCCL EP: preparing $COLLX_NCCL_EP_SPEC (node-local $root; no shared cache mounted)" if [ ! -f "$root/.ready" ] || [ ! -d "$root/site" ]; then nccl_ep_install "$root" || return 1 fi fi nccl_ep_activate "$root" || return 1 nccl_ep_probe || { collx_log "ERROR: NCCL EP import probe failed"; return 1; } - collx_log "NCCL EP ready ($COLLX_NCCL4PY_SPEC; libnccl_ep.so JIT runtime, NCCL Device API LSA/GIN)" + collx_log "NCCL EP ready ($COLLX_NCCL_EP_SPEC; libnccl_ep.so JIT runtime, NCCL Device API LSA/GIN)" } # ---- container boundary ---------------------------------------------------- From e40eac1a914d8f8095d31c895623cc76e37b8732 Mon Sep 17 00:00:00 2001 From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com> Date: Thu, 10 Sep 2026 11:31:53 +0800 Subject: [PATCH 6/6] CollectiveX: document x86 EP16 NCCL EP scale-out as validated under v0.2 Run 34432070017 measured the HIGH_THROUGHPUT EP16 rows on h100-dgxc and h200-dgxc as real 2x8 RDMA scale-out with every decode and prefill rung correct, so the coverage prose no longer calls those rows unsupported. --- experimental/CollectiveX/README.md | 2 +- experimental/CollectiveX/docs/methodology.md | 11 +++++------ 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/experimental/CollectiveX/README.md b/experimental/CollectiveX/README.md index 89539a6a66..acd638f6ba 100644 --- a/experimental/CollectiveX/README.md +++ b/experimental/CollectiveX/README.md @@ -134,7 +134,7 @@ scale-up domain. | DeepEP V2 | `production`, with vLLM `--all2all-backend deepep_v2`, SGLang `--moe-a2a-backend deepep` | `normal` mode is PR #605 `ElasticBuffer` plus exact upstream #630 and #640 fixes: LSA for scale-up and GIN for x86 EP16 scale-out. FP8 dispatch via `use_fp8_dispatch` (blockwise e4m3fn) alongside BF16. `low-latency` mode is the legacy `deep_ep.Buffer` IBGDA decode kernels (per-expert padded layout, weighted combine, `use_fp8` e4m3fn), decode only, with EP8 wherever enabled, plus EP16 on GB200/GB300 (inside the MNNVL domain) and on B200's nscale bare-metal pool (IBGDA over native IB rails with `/dev/gdrdrv`, although the prior virtualized b200 pool could never run it). B300 is an unsupported coverage row in `low-latency`: the legacy Buffer self-enables NVSHMEM IBGDA even for a single-node EP8 run, and on B300 address-handle creation fails (`ibgda.cpp:2234 Unable to create ah`), rc255 on all eight ranks. `NVSHMEM_DISABLE_IB=1` does not help. The Buffer re-enables IBGDA regardless, and the run fails identically with it set and unset (measured on b300-002 and b300-011) | | MoRI | `production`, with vLLM `--all2all-backend mori_*`, SGLang `--moe-a2a-backend mori` | `normal` mode uses the direct `IntraNode` kernel for scale-up EP8 on every CDNA SKU. EP16 remains an unsupported coverage row on all three CDNA SKUs. Part of the old ROCm/mori#475 corruption was this harness passing dispatch's returned recv-slot indices to `combine()` instead of the rank's own routing (root-caused upstream, guarded by ROCm/mori#546, kernels unchanged) — with the corrected call, single-shot InterNodeV1 is clean through T=512 on mi355x — but a residual stochastic corruption remains from T~128 up under repeated execution and is near-certain at prefill sizes (run 33045314017/33050026476; unaffected by per-pair drains, so not a buffer-reuse race; upstream cannot reproduce on ionic driver 26.03 vs our 25.11). The tw pairs additionally have no cross-node GPU fabric, and mi355x EP16 has no publishable transport today: uccl-ep's CPU-proxy RDMA is functional on Pollara but ~13x under its documented bandwidth (~6 GB/s vs 82; unchanged by registration mode or traffic class — ionic-driver suspect, same 25.11-vs-26.03 delta as the mori residual). `low-latency` mode selects the `AsyncLL` decode kernel driven split-phase (dispatch send+recv, combine send+recv — the kernel SGLang deploys; same compact layout and unweighted combine as `IntraNode`; earlier rows measured `IntraNodeLL`, discriminated by `kernel_generation`), decode/EP8 only. FP8 dispatch is caller-prequantized (per-SKU e4m3fnuz on gfx942, e4m3fn on gfx950). Combine stays BF16 (`quant_type=none`) alongside BF16 dispatch | | UCCL-EP | `candidate` (no engine exposes a UCCL-EP selector) | [UCCL](https://github.com/uccl-project/uccl) EP: a drop-in, API-identical DeepEP replacement whose CPU proxies issue GPUDirect RDMA over plain `libibverbs` (no NVSHMEM/IBGDA), with software message ordering, atomics, and flow control. Scale-up is single-node `cudaIpc` over NVLink/XGMI (never MNNVL). `normal` mode is the legacy `Buffer` `dispatch`/`combine` (unweighted rank-sum). `low-latency` reuses the legacy `low_latency_dispatch`/`low_latency_combine` decode kernels (weighted combine), decode/EP8 only. FP8 dispatch is caller-prequantized in `normal` mode (blockwise e4m3fn, per-SKU e4m3fnuz on gfx942). In `low-latency` mode the caller sends BF16 and the decode kernel quantizes to e4m3 internally (`use_fp8`). Combine is BF16. Runs on NVIDIA and AMD (H100/H200/B200 + MI300X/MI325X/MI355X), EP8 scale-up. Cross-node EP16 is functional (the internode RDMA path connects and the light case passes correctness) but its CPU-proxy throughput overruns the standardized per-case wall-clock budget on heavy token counts, so EP16 is an unsupported coverage row for now | -| NCCL EP | `candidate` (NVIDIA's own library, but no engine exposes an NCCL-EP selector) | [NCCL EP v0.2](https://github.com/NVIDIA/nccl-extensions): NVIDIA's native MoE dispatch/combine on the NCCL Device API, using LSA (NVLink load/store) intra-node and GIN (GPU-Initiated Networking) inter-node. Since v0.2 it ships as the `nccl-extensions` PyPI wheel (which owns `nccl.ep`; `nccl4py` — pinned alongside — provides only `nccl.core`), replacing the `contrib/nccl_ep` tree in `NVIDIA/nccl`, which has not moved since 2026-06-11. `normal` mode selects the `HIGH_THROUGHPUT` algorithm (FLAT `[N, hidden]` receive, unweighted rank-sum combine). The `LOW_LATENCY` algorithm carries `ll_backends` rows at EP8 on H100/H200/B300 and EP8+EP16 on B200/GB200/GB300. Its decode ladder runs unclamped to the full 256-slot receive: v0.1's combine recv pipeline was a port of DeepEP's pre-#642 kernel, missing the shared-memory fence before `mbarrier_arrive` (bimodal T=256 corruption on GB300, 1 of 5 executions at 0.4704 relative error against healthy 0.0039), and the v0.2 wheel ships that fence (`fence_view_async_shared` before the `emptyBarriers` arrive in the shipped `ll_ep.cuh`), the documented restore condition for the T<=128 clamp it forced. BF16 only for now: v0.2 exposes FP8 dispatch (`DS_FP8E3M4`) and an experimental NVFP4 combine recipe, but wiring those into the fp8_consume model is separate bring-up, so no FP8 case is emitted yet. NVIDIA-only and CUDA 13 only. EP8 scale-up on H100/H200/B300, EP8 and EP16 on B200 (v0.2 fixes the RDMA EP16 GIN fault this pool hit under v0.1) and on GB200/GB300, where EP16 stays inside the MNNVL scale-up domain. x86 EP16 scale-out elsewhere (H100/H200/B300) remains an unsupported coverage row pending revalidation: under v0.1 the cross-node GIN path faulted inside `nccl_ep.cc` identically on RoCE and IB, and only the B200 configuration is claimed fixed | +| NCCL EP | `candidate` (NVIDIA's own library, but no engine exposes an NCCL-EP selector) | [NCCL EP v0.2](https://github.com/NVIDIA/nccl-extensions): NVIDIA's native MoE dispatch/combine on the NCCL Device API, using LSA (NVLink load/store) intra-node and GIN (GPU-Initiated Networking) inter-node. Since v0.2 it ships as the `nccl-extensions` PyPI wheel (which owns `nccl.ep`; `nccl4py` — pinned alongside — provides only `nccl.core`), replacing the `contrib/nccl_ep` tree in `NVIDIA/nccl`, which has not moved since 2026-06-11. `normal` mode selects the `HIGH_THROUGHPUT` algorithm (FLAT `[N, hidden]` receive, unweighted rank-sum combine). The `LOW_LATENCY` algorithm carries `ll_backends` rows at EP8 on H100/H200/B300 and EP8+EP16 on B200/GB200/GB300. Its decode ladder runs unclamped to the full 256-slot receive: v0.1's combine recv pipeline was a port of DeepEP's pre-#642 kernel, missing the shared-memory fence before `mbarrier_arrive` (bimodal T=256 corruption on GB300, 1 of 5 executions at 0.4704 relative error against healthy 0.0039), and the v0.2 wheel ships that fence (`fence_view_async_shared` before the `emptyBarriers` arrive in the shipped `ll_ep.cuh`), the documented restore condition for the T<=128 clamp it forced. BF16 only for now: v0.2 exposes FP8 dispatch (`DS_FP8E3M4`) and an experimental NVFP4 combine recipe, but wiring those into the fp8_consume model is separate bring-up, so no FP8 case is emitted yet. NVIDIA-only and CUDA 13 only. EP8 and EP16 on all six NVIDIA SKUs: EP16 stays inside the MNNVL scale-up domain on GB200/GB300 and is a 2x8 RDMA scale-out over the cross-node GIN path on H100/H200/B200/B300. Under v0.1 that GIN path faulted inside `nccl_ep.cc` identically on RoCE and IB on every x86 pool; v0.2 clears it, revalidated on-metal on B200, H100 and H200, with B300 enabled on the same fix pending its pool's revalidation | | FlashInfer EP | `production`, with vLLM `--all2all-backend flashinfer_nvlink_one_sided` | [FlashInfer](https://github.com/flashinfer-ai/flashinfer) `MoeAlltoAll`: TensorRT-LLM's one-sided MNNVL all-to-all, where each rank writes tokens straight into its peers' workspace windows and combine reads them back, with no send/recv pairing and no NVSHMEM. `normal` mode only (there is one kernel family and no separate decode path), and GB200/GB300 only, since the transport is MNNVL. FP8 dispatch is caller-prequantized blockwise e4m3fn, carried as a fourth dispatch payload alongside its per-128-block FP32 scales, with the combine plane forced to BF16. The C++ `toNvDataType` accepts only fp16/bf16/fp32 for combine, so an FP8 combine buffer would raise rather than corrupt. EP8 and EP16, both inside the scale-up domain. Unlike every other backend here, its combine accumulates in the PAYLOAD dtype rather than FP32: wheels before 0.6.16 reduce the top-k contributions with a pairwise BF16 tree that rounds at every level, so the oracle models that reduction directly (`combine_reduction = "topk-slot-tree"`) instead of widening the tolerance. 0.6.16 moved the accumulator to FP32, and the adapter switches models on the installed version | DeepEP V2 means the `ElasticBuffer` implementation introduced by diff --git a/experimental/CollectiveX/docs/methodology.md b/experimental/CollectiveX/docs/methodology.md index e7ecbe844d..872b3ef9ad 100644 --- a/experimental/CollectiveX/docs/methodology.md +++ b/experimental/CollectiveX/docs/methodology.md @@ -187,12 +187,11 @@ is NVIDIA's native MoE dispatch/combine on the NCCL Device API, shipped since v0 `nccl-extensions` wheel (which owns `nccl.ep`; `nccl4py`, pinned alongside, provides `nccl.core`). `normal` mode selects its `HIGH_THROUGHPUT` algorithm, whose FLAT `[N, hidden]` receive and unweighted rank-sum combine match `layout-and-dispatch-v1` exactly, so the same oracle applies. It is -NVIDIA-only and CUDA 13 only, and runs EP8 scale-up on H100/H200/B300, EP8 and EP16 on B200 (v0.2 -fixes the RDMA EP16 GIN fault that pool hit under v0.1), and EP8 and EP16 on GB200/GB300, where EP16 -stays inside the MNNVL scale-up domain. X86 EP16 scale-out elsewhere (H100/H200/B300) remains an -unsupported coverage row pending revalidation: under v0.1 the cross-node GIN path faulted inside -`nccl_ep.cc` identically on RoCE and IB — a GDAKI limit, not a fabric-selection one — and only the -B200 configuration is claimed fixed in v0.2. FlashInfer EP is TensorRT-LLM's one-sided MNNVL `MoeAlltoAll`, in which each rank writes tokens directly into its peers' workspace windows and combine reads them back, so there is no send/recv pairing and no NVSHMEM. It is GB200/GB300-only for that reason, and runs EP8 and EP16 inside the MNNVL scale-up domain. Its combine is the one place a backend's accumulator precision changes the expectation rather than the tolerance: through 0.6.15 the kernel holds its top-k accumulators in the payload dtype and reduces them with a hand-unrolled pairwise tree, so every level rounds to BF16, and the oracle reproduces that tree exactly rather than loosening the gate to absorb it (0.6.16 rewrote the accumulator to FP32. The adapter reads the installed version and picks the matching model). Those throughput kernels run across the full token ladder in the `normal` mode. Its FP8 dispatch is the one (backend, precision) pair here that is realizable but off every deployed path. vLLM accepts only nvfp4/mxfp8/bf16 on this transport, so `sweep_matrix.py`'s `OFF_PATH_PRECISIONS` keeps it out of the default matrix and a production sweep measures only configurations an engine can select. Naming the precision explicitly (`--precisions fp8`) opts it back in for transport comparison against DeepEP V2/UCCL-EP at matching bytes and block size: the one place a precision filter ADDS rows rather than only removing them. +NVIDIA-only and CUDA 13 only, and runs EP8 and EP16 on all six NVIDIA SKUs: EP16 stays inside the +MNNVL scale-up domain on GB200/GB300 and is a 2x8 RDMA scale-out over the cross-node GIN path on +H100/H200/B200/B300. Under v0.1 that GIN path faulted inside `nccl_ep.cc` identically on RoCE and IB +on every x86 pool; v0.2 clears it, revalidated on-metal on B200, H100 and H200 (full decode and +prefill ladders correct), with B300 enabled on the same fix pending its pool's revalidation. FlashInfer EP is TensorRT-LLM's one-sided MNNVL `MoeAlltoAll`, in which each rank writes tokens directly into its peers' workspace windows and combine reads them back, so there is no send/recv pairing and no NVSHMEM. It is GB200/GB300-only for that reason, and runs EP8 and EP16 inside the MNNVL scale-up domain. Its combine is the one place a backend's accumulator precision changes the expectation rather than the tolerance: through 0.6.15 the kernel holds its top-k accumulators in the payload dtype and reduces them with a hand-unrolled pairwise tree, so every level rounds to BF16, and the oracle reproduces that tree exactly rather than loosening the gate to absorb it (0.6.16 rewrote the accumulator to FP32. The adapter reads the installed version and picks the matching model). Those throughput kernels run across the full token ladder in the `normal` mode. Its FP8 dispatch is the one (backend, precision) pair here that is realizable but off every deployed path. vLLM accepts only nvfp4/mxfp8/bf16 on this transport, so `sweep_matrix.py`'s `OFF_PATH_PRECISIONS` keeps it out of the default matrix and a production sweep measures only configurations an engine can select. Naming the precision explicitly (`--precisions fp8`) opts it back in for transport comparison against DeepEP V2/UCCL-EP at matching bytes and block size: the one place a precision filter ADDS rows rather than only removing them. A second `low-latency` mode adds each backend's decode-optimized kernel family. On DeepEP it drives the legacy `deep_ep.Buffer` low-latency decode kernels (`low_latency_dispatch`/`low_latency_combine`),