From 762531b500dac23c25940f83c4c2a22053d06f1b Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Tue, 1 Sep 2026 10:04:46 -1000 Subject: [PATCH 1/5] Enable ARM (aarch64) support for cuvs-java and cuvs-lucene MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The C++ core already builds and is tested on both amd64 and arm64; cuvs-java had a hardcoded amd64-only gate that cuvs-lucene inherited. Java bytecode is portable, but cuvs-java isn't pure Java — it uses the Panama FFM API to call into native libcuvs_c.so, and those bindings are generated by jextract, which bakes struct/function ABI layouts (computed by parsing the C headers with clang for the host's target triple) into the generated .class files at build time. Reusing an amd64-generated binding on aarch64 should work, since no arch-conditional logic in the jextract-visible header surface changes any type layout, struct offset, or function signature between x86_64 and aarch64 Linux. We therefore expect the plain (no-native-bundled) jar to be arch-portable in practice. To verify that, the aarch64 cuvs-lucene CI job installs the amd64-built jar and runs cuvs-lucene's full test suite against it on aarch64 hardware with an aarch64-native libcuvs_c.so. Closes #1236. --- .github/workflows/build.yaml | 18 +- .github/workflows/pr.yaml | 17 +- .github/workflows/test.yaml | 17 +- java/cuvs-java/pom.xml | 178 ++++++++++++++++++ .../nvidia/cuvs/spi/CuVSServiceProvider.java | 5 +- .../nvidia/cuvs/BruteForceAndSearchIT.java | 2 +- .../nvidia/cuvs/BruteForceRandomizedIT.java | 4 +- .../nvidia/cuvs/CagraAceBuildAndSearchIT.java | 2 +- .../nvidia/cuvs/CagraBuildAndSearchIT.java | 2 +- .../cuvs/CagraMultiThreadStabilityIT.java | 2 +- .../com/nvidia/cuvs/CagraRandomizedIT.java | 2 +- .../java/com/nvidia/cuvs/CuVSMatrixIT.java | 4 +- .../java/com/nvidia/cuvs/CuVSResourcesIT.java | 2 +- .../java/com/nvidia/cuvs/CuVSTestCase.java | 5 +- .../com/nvidia/cuvs/FilterBitsetHandleIT.java | 2 +- .../test/java/com/nvidia/cuvs/GPUInfoIT.java | 4 +- .../nvidia/cuvs/HnswAceBuildAndSearchIT.java | 2 +- .../com/nvidia/cuvs/HnswBuildAndSearchIT.java | 2 +- .../com/nvidia/cuvs/HnswRandomizedIT.java | 2 +- .../cuvs/MemoryTrackingResourcesIT.java | 2 +- .../cuvs/MultiPartitionCagraSearchIT.java | 2 +- .../java/com/nvidia/cuvs/TieredIndexIT.java | 4 +- .../nvidia/cuvs/internal/common/UtilIT.java | 4 +- .../com/nvidia/cuvs/spi/CuVSProviderIT.java | 4 +- java/panama-bindings/generate-bindings.sh | 21 ++- 25 files changed, 261 insertions(+), 48 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index bdd46b298f..1352b32fd2 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -163,7 +163,6 @@ jobs: with: build_type: pull-request matrix_name: conda-cpp-build - matrix_filter: map(select(.ARCH == "amd64")) java-build: needs: java-build-matrix permissions: @@ -182,15 +181,17 @@ jobs: with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} - arch: "amd64" + arch: "${{ matrix.ARCH }}" date: ${{ inputs.date }} container_image: "rapidsai/ci-conda:26.10-cuda${{ matrix.CUDA_VER }}-${{ matrix.LINUX_VER }}-py${{ matrix.PY_VER }}" script: "ci/build_java.sh" - artifact-name: "cuvs-java-cuda${{ matrix.CUDA_VER }}" + # amd64 is the official published artifact; keep its name unqualified and only suffix other arches so they don't collide with it. + artifact-name: "cuvs-java-cuda${{ matrix.CUDA_VER }}${{ matrix.ARCH != 'amd64' && format('-{0}', matrix.ARCH) || '' }}" file_to_upload: "java/cuvs-java/target/" sha: ${{ inputs.sha }} lucene-build: - # Depends on the Java job for the cuvs-java artifact that cuvs-lucene builds against. + # Depends on the Java job for the cuvs-java artifact that cuvs-lucene builds against, and + # reuses its matrix so the two always agree on the set of CUDA versions and arches to test on. needs: [java-build-matrix, java-build] permissions: actions: read @@ -208,11 +209,16 @@ jobs: with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} - arch: "amd64" + arch: "${{ matrix.ARCH }}" date: ${{ inputs.date }} container_image: "rapidsai/ci-conda:26.10-cuda${{ matrix.CUDA_VER }}-${{ matrix.LINUX_VER }}-py${{ matrix.PY_VER }}" + # Always installs the amd64-built (official) cuvs-java jar, even when this job itself runs on + # a non-amd64 runner. cuvs-lucene depends on the plain jar, which bundles no native code, so + # running its tests on arm64 against the amd64-built jar doubles as a cross-arch check that the + # jextract-generated Panama bindings baked into that jar work unmodified on arm64 hardware. script: "ci/build_lucene.sh cuvs-java-cuda${{ matrix.CUDA_VER }}" - artifact-name: "cuvs-lucene-cuda${{ matrix.CUDA_VER }}" + # amd64 is the official published artifact; keep its name unqualified and only suffix other arches so they don't collide with it. + artifact-name: "cuvs-lucene-cuda${{ matrix.CUDA_VER }}${{ matrix.ARCH != 'amd64' && format('-{0}', matrix.ARCH) || '' }}" file_to_upload: "java/cuvs-lucene/target/" sha: ${{ inputs.sha }} python-build: diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index d6f7cd7d7e..4b8d199aa4 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -589,7 +589,6 @@ jobs: with: build_type: pull-request matrix_name: conda-cpp-build - matrix_filter: map(select(.ARCH == "amd64")) conda-java-build-and-tests: needs: conda-java-build-and-tests-matrix permissions: @@ -608,14 +607,15 @@ jobs: with: build_type: pull-request node_type: "gpu-l4-latest-1" - arch: "amd64" + arch: "${{ matrix.ARCH }}" container_image: "rapidsai/ci-conda:26.10-cuda${{ matrix.CUDA_VER }}-${{ matrix.LINUX_VER }}-py${{ matrix.PY_VER }}" script: "ci/test_java.sh" - artifact-name: "cuvs-java-cuda${{ matrix.CUDA_VER }}" + # amd64 is the official published artifact; keep its name unqualified and only suffix other arches so they don't collide with it. + artifact-name: "cuvs-java-cuda${{ matrix.CUDA_VER }}${{ matrix.ARCH != 'amd64' && format('-{0}', matrix.ARCH) || '' }}" file_to_upload: "java/cuvs-java/target/" conda-lucene-build-and-tests: # Depends on the Java job for the cuvs-java artifact that cuvs-lucene builds against, and - # reuses its matrix so the two always agree on the set of CUDA versions. + # reuses its matrix so the two always agree on the set of CUDA versions and arches to test on. needs: [conda-java-build-and-tests-matrix, conda-java-build-and-tests] permissions: actions: read @@ -633,10 +633,15 @@ jobs: with: build_type: pull-request node_type: "gpu-l4-latest-1" - arch: "amd64" + arch: "${{ matrix.ARCH }}" container_image: "rapidsai/ci-conda:26.10-cuda${{ matrix.CUDA_VER }}-${{ matrix.LINUX_VER }}-py${{ matrix.PY_VER }}" + # Always installs the amd64-built (official) cuvs-java jar, even when this job itself runs on + # a non-amd64 runner. cuvs-lucene depends on the plain jar, which bundles no native code, so + # running its tests on arm64 against the amd64-built jar doubles as a cross-arch check that the + # jextract-generated Panama bindings baked into that jar work unmodified on arm64 hardware. script: "ci/test_lucene.sh cuvs-java-cuda${{ matrix.CUDA_VER }}" - artifact-name: "cuvs-lucene-cuda${{ matrix.CUDA_VER }}" + # amd64 is the official published artifact; keep its name unqualified and only suffix other arches so they don't collide with it. + artifact-name: "cuvs-lucene-cuda${{ matrix.CUDA_VER }}${{ matrix.ARCH != 'amd64' && format('-{0}', matrix.ARCH) || '' }}" file_to_upload: "java/cuvs-lucene/target/" rust-build-matrix: needs: [conda-cpp-build, changed-files] diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index c2ff71b3a2..fc8359277a 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -77,7 +77,6 @@ jobs: with: build_type: ${{ inputs.build_type }} matrix_name: conda-cpp-build - matrix_filter: map(select(.ARCH == "amd64")) conda-java-build-and-tests: needs: conda-java-build-and-tests-matrix permissions: @@ -99,14 +98,15 @@ jobs: date: ${{ inputs.date }} sha: ${{ inputs.sha }} node_type: "gpu-l4-latest-1" - arch: "amd64" + arch: "${{ matrix.ARCH }}" container_image: "rapidsai/ci-conda:26.10-cuda${{ matrix.CUDA_VER }}-${{ matrix.LINUX_VER }}-py${{ matrix.PY_VER }}" script: "ci/test_java.sh" - artifact-name: "cuvs-java-cuda${{ matrix.CUDA_VER }}" + # amd64 is the official published artifact; keep its name unqualified and only suffix other arches so they don't collide with it. + artifact-name: "cuvs-java-cuda${{ matrix.CUDA_VER }}${{ matrix.ARCH != 'amd64' && format('-{0}', matrix.ARCH) || '' }}" file_to_upload: "java/cuvs-java/target/" conda-lucene-build-and-tests: # Depends on the Java job for the cuvs-java artifact that cuvs-lucene builds against, and - # reuses its matrix so the two always agree on the set of CUDA versions. + # reuses its matrix so the two always agree on the set of CUDA versions and arches to test on. needs: [conda-java-build-and-tests-matrix, conda-java-build-and-tests] permissions: actions: read @@ -127,10 +127,15 @@ jobs: date: ${{ inputs.date }} sha: ${{ inputs.sha }} node_type: "gpu-l4-latest-1" - arch: "amd64" + arch: "${{ matrix.ARCH }}" container_image: "rapidsai/ci-conda:26.10-cuda${{ matrix.CUDA_VER }}-${{ matrix.LINUX_VER }}-py${{ matrix.PY_VER }}" + # Always installs the amd64-built (official) cuvs-java jar, even when this job itself runs on + # a non-amd64 runner. cuvs-lucene depends on the plain jar, which bundles no native code, so + # running its tests on arm64 against the amd64-built jar doubles as a cross-arch check that the + # jextract-generated Panama bindings baked into that jar work unmodified on arm64 hardware. script: "ci/test_lucene.sh cuvs-java-cuda${{ matrix.CUDA_VER }}" - artifact-name: "cuvs-lucene-cuda${{ matrix.CUDA_VER }}" + # amd64 is the official published artifact; keep its name unqualified and only suffix other arches so they don't collide with it. + artifact-name: "cuvs-lucene-cuda${{ matrix.CUDA_VER }}${{ matrix.ARCH != 'amd64' && format('-{0}', matrix.ARCH) || '' }}" file_to_upload: "java/cuvs-lucene/target/" wheel-tests-cuvs: permissions: diff --git a/java/cuvs-java/pom.xml b/java/cuvs-java/pom.xml index d4d7708a80..4b7f551139 100644 --- a/java/cuvs-java/pom.xml +++ b/java/cuvs-java/pom.xml @@ -285,6 +285,9 @@ SPDX-License-Identifier: Apache-2.0 x86_64-cuda12 + + amd64 + cuda.version 12 @@ -368,6 +371,9 @@ SPDX-License-Identifier: Apache-2.0 x86_64-cuda13 + + amd64 + cuda.version 13 @@ -447,5 +453,177 @@ SPDX-License-Identifier: Apache-2.0 + + + aarch64-cuda12 + + + aarch64 + + + cuda.version + 12 + + + + aarch64-cuda12 + ${native.build.path}/cuda12 + + + + + org.apache.maven.plugins + maven-resources-plugin + 3.3.1 + + + copy-native-libs + prepare-package + + copy-resources + + + true + ${project.build.directory}/native-libs/${os.arch}/${os.name} + + + ${native.lib.path} + + libcuvs.so + libcuvs_c.so + + + + ${native.lib.path}/_deps/rmm-build + + librmm.so + + + + ${native.lib.path}/_deps/rapids_logger-build + + librapids_logger.so + + + + + + + + + org.apache.maven.plugins + maven-assembly-plugin + 3.4.2 + + + src/assembly/native-with-deps.xml + + + + true + true + 12 + + + + + + assemble-native + package + + single + + + + + + + + + + aarch64-cuda13 + + + aarch64 + + + cuda.version + 13 + + + + aarch64-cuda13 + ${native.build.path}/cuda13 + + + + + org.apache.maven.plugins + maven-resources-plugin + 3.3.1 + + + copy-native-libs + prepare-package + + copy-resources + + + true + ${project.build.directory}/native-libs/${os.arch}/${os.name} + + + ${native.lib.path} + + libcuvs.so + libcuvs_c.so + + + + ${native.lib.path}/_deps/rmm-build + + librmm.so + + + + ${native.lib.path}/_deps/rapids_logger-build + + librapids_logger.so + + + + + + + + + org.apache.maven.plugins + maven-assembly-plugin + 3.4.2 + + + src/assembly/native-with-deps.xml + + + + true + true + 13 + + + + + + assemble-native + package + + single + + + + + + + diff --git a/java/cuvs-java/src/main/java/com/nvidia/cuvs/spi/CuVSServiceProvider.java b/java/cuvs-java/src/main/java/com/nvidia/cuvs/spi/CuVSServiceProvider.java index 874f35ee20..b4f16a084c 100644 --- a/java/cuvs-java/src/main/java/com/nvidia/cuvs/spi/CuVSServiceProvider.java +++ b/java/cuvs-java/src/main/java/com/nvidia/cuvs/spi/CuVSServiceProvider.java @@ -39,7 +39,7 @@ static CuVSProvider builtinProvider() { var osArch = System.getProperty("os.arch"); var supportedJavaRuntime = javaRuntimeVersion > 21; var supportedOs = osName.startsWith("Linux"); - var supportedArchitecture = osArch.equals("amd64"); + var supportedArchitecture = osArch.equals("amd64") || osArch.equals("aarch64"); if (supportedJavaRuntime && supportedOs && supportedArchitecture) { try { var cls = Class.forName("com.nvidia.cuvs.spi.JDKProvider"); @@ -63,7 +63,8 @@ static CuVSProvider builtinProvider() { unsupportedReasons.add("cuvs-java supports only Linux, but os.name is " + osName); } if (!supportedArchitecture) { - unsupportedReasons.add("cuvs-java supports only x86-64 (amd64), but os.arch is " + osArch); + unsupportedReasons.add( + "cuvs-java supports only x86-64 (amd64) and arm64 (aarch64), but os.arch is " + osArch); } return new UnsupportedProvider(String.join("; ", unsupportedReasons)); diff --git a/java/cuvs-java/src/test/java/com/nvidia/cuvs/BruteForceAndSearchIT.java b/java/cuvs-java/src/test/java/com/nvidia/cuvs/BruteForceAndSearchIT.java index ab59395dd2..1f28c45950 100644 --- a/java/cuvs-java/src/test/java/com/nvidia/cuvs/BruteForceAndSearchIT.java +++ b/java/cuvs-java/src/test/java/com/nvidia/cuvs/BruteForceAndSearchIT.java @@ -23,7 +23,7 @@ public class BruteForceAndSearchIT extends CuVSTestCase { @Before public void setup() { - assumeTrue("not supported on " + System.getProperty("os.name"), isLinuxAmd64()); + assumeTrue("not supported on " + System.getProperty("os.name"), isLinuxSupportedArch()); CuVSProvider.provider().enableRMMPooledMemory(10, 60); } diff --git a/java/cuvs-java/src/test/java/com/nvidia/cuvs/BruteForceRandomizedIT.java b/java/cuvs-java/src/test/java/com/nvidia/cuvs/BruteForceRandomizedIT.java index 22af6fb4c3..96c10a0936 100644 --- a/java/cuvs-java/src/test/java/com/nvidia/cuvs/BruteForceRandomizedIT.java +++ b/java/cuvs-java/src/test/java/com/nvidia/cuvs/BruteForceRandomizedIT.java @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ package com.nvidia.cuvs; @@ -25,7 +25,7 @@ public class BruteForceRandomizedIT extends CuVSTestCase { @Before public void setup() { - assumeTrue(isLinuxAmd64()); + assumeTrue(isLinuxSupportedArch()); initializeRandom(); log.trace("Random context initialized for test."); CuVSProvider.provider().enableRMMPooledMemory(10, 60); diff --git a/java/cuvs-java/src/test/java/com/nvidia/cuvs/CagraAceBuildAndSearchIT.java b/java/cuvs-java/src/test/java/com/nvidia/cuvs/CagraAceBuildAndSearchIT.java index f762334619..423c58db35 100644 --- a/java/cuvs-java/src/test/java/com/nvidia/cuvs/CagraAceBuildAndSearchIT.java +++ b/java/cuvs-java/src/test/java/com/nvidia/cuvs/CagraAceBuildAndSearchIT.java @@ -36,7 +36,7 @@ public class CagraAceBuildAndSearchIT extends CuVSTestCase { @Before public void setup() { - assumeTrue("not supported on " + System.getProperty("os.name"), isLinuxAmd64()); + assumeTrue("not supported on " + System.getProperty("os.name"), isLinuxSupportedArch()); initializeRandom(); log.trace("Random context initialized for test."); } diff --git a/java/cuvs-java/src/test/java/com/nvidia/cuvs/CagraBuildAndSearchIT.java b/java/cuvs-java/src/test/java/com/nvidia/cuvs/CagraBuildAndSearchIT.java index 500ef451f3..5bcd7a00d0 100644 --- a/java/cuvs-java/src/test/java/com/nvidia/cuvs/CagraBuildAndSearchIT.java +++ b/java/cuvs-java/src/test/java/com/nvidia/cuvs/CagraBuildAndSearchIT.java @@ -47,7 +47,7 @@ public class CagraBuildAndSearchIT extends CuVSTestCase { @Before public void setup() { - assumeTrue("not supported on " + System.getProperty("os.name"), isLinuxAmd64()); + assumeTrue("not supported on " + System.getProperty("os.name"), isLinuxSupportedArch()); initializeRandom(); log.trace("Random context initialized for test."); } diff --git a/java/cuvs-java/src/test/java/com/nvidia/cuvs/CagraMultiThreadStabilityIT.java b/java/cuvs-java/src/test/java/com/nvidia/cuvs/CagraMultiThreadStabilityIT.java index 371e842973..17790eb2fc 100644 --- a/java/cuvs-java/src/test/java/com/nvidia/cuvs/CagraMultiThreadStabilityIT.java +++ b/java/cuvs-java/src/test/java/com/nvidia/cuvs/CagraMultiThreadStabilityIT.java @@ -44,7 +44,7 @@ private interface ResourcesSupplier { @Before public void setup() { - assumeTrue("not supported on " + System.getProperty("os.name"), isLinuxAmd64()); + assumeTrue("not supported on " + System.getProperty("os.name"), isLinuxSupportedArch()); initializeRandom(); log.trace("Multi-threaded stability test initialized"); } diff --git a/java/cuvs-java/src/test/java/com/nvidia/cuvs/CagraRandomizedIT.java b/java/cuvs-java/src/test/java/com/nvidia/cuvs/CagraRandomizedIT.java index 2dcb8b1cd0..9110f6ddc2 100644 --- a/java/cuvs-java/src/test/java/com/nvidia/cuvs/CagraRandomizedIT.java +++ b/java/cuvs-java/src/test/java/com/nvidia/cuvs/CagraRandomizedIT.java @@ -24,7 +24,7 @@ public class CagraRandomizedIT extends CuVSTestCase { @Before public void setup() { - assumeTrue("not supported on " + System.getProperty("os.name"), isLinuxAmd64()); + assumeTrue("not supported on " + System.getProperty("os.name"), isLinuxSupportedArch()); initializeRandom(); log.trace("Random context initialized for test."); } diff --git a/java/cuvs-java/src/test/java/com/nvidia/cuvs/CuVSMatrixIT.java b/java/cuvs-java/src/test/java/com/nvidia/cuvs/CuVSMatrixIT.java index cd3cf43e55..954a7d1e4f 100644 --- a/java/cuvs-java/src/test/java/com/nvidia/cuvs/CuVSMatrixIT.java +++ b/java/cuvs-java/src/test/java/com/nvidia/cuvs/CuVSMatrixIT.java @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ package com.nvidia.cuvs; @@ -25,7 +25,7 @@ public class CuVSMatrixIT extends CuVSTestCase { @Before public void setup() { - assumeTrue("not supported on " + System.getProperty("os.name"), isLinuxAmd64()); + assumeTrue("not supported on " + System.getProperty("os.name"), isLinuxSupportedArch()); initializeRandom(); } diff --git a/java/cuvs-java/src/test/java/com/nvidia/cuvs/CuVSResourcesIT.java b/java/cuvs-java/src/test/java/com/nvidia/cuvs/CuVSResourcesIT.java index 00753b42c4..632eefc6a1 100644 --- a/java/cuvs-java/src/test/java/com/nvidia/cuvs/CuVSResourcesIT.java +++ b/java/cuvs-java/src/test/java/com/nvidia/cuvs/CuVSResourcesIT.java @@ -20,7 +20,7 @@ public class CuVSResourcesIT extends CuVSTestCase { @Before public void setup() { - assumeTrue("not supported on " + System.getProperty("os.name"), isLinuxAmd64()); + assumeTrue("not supported on " + System.getProperty("os.name"), isLinuxSupportedArch()); } @Test diff --git a/java/cuvs-java/src/test/java/com/nvidia/cuvs/CuVSTestCase.java b/java/cuvs-java/src/test/java/com/nvidia/cuvs/CuVSTestCase.java index 62e636eb82..53cbe47392 100644 --- a/java/cuvs-java/src/test/java/com/nvidia/cuvs/CuVSTestCase.java +++ b/java/cuvs-java/src/test/java/com/nvidia/cuvs/CuVSTestCase.java @@ -155,9 +155,10 @@ public boolean equals(Object o) { assertEquals(sortedExpected, sortedActual); } - protected static boolean isLinuxAmd64() { + protected static boolean isLinuxSupportedArch() { String name = System.getProperty("os.name"); - return (name.startsWith("Linux")) && System.getProperty("os.arch").equals("amd64"); + String arch = System.getProperty("os.arch"); + return (name.startsWith("Linux")) && (arch.equals("amd64") || arch.equals("aarch64")); } protected static int[][] createIntMatrix() { diff --git a/java/cuvs-java/src/test/java/com/nvidia/cuvs/FilterBitsetHandleIT.java b/java/cuvs-java/src/test/java/com/nvidia/cuvs/FilterBitsetHandleIT.java index 2abb153207..d7464ca97e 100644 --- a/java/cuvs-java/src/test/java/com/nvidia/cuvs/FilterBitsetHandleIT.java +++ b/java/cuvs-java/src/test/java/com/nvidia/cuvs/FilterBitsetHandleIT.java @@ -52,7 +52,7 @@ public class FilterBitsetHandleIT extends CuVSTestCase { @Before public void setup() { - assumeTrue("not supported on " + System.getProperty("os.name"), isLinuxAmd64()); + assumeTrue("not supported on " + System.getProperty("os.name"), isLinuxSupportedArch()); initializeRandom(); } diff --git a/java/cuvs-java/src/test/java/com/nvidia/cuvs/GPUInfoIT.java b/java/cuvs-java/src/test/java/com/nvidia/cuvs/GPUInfoIT.java index fbfab24518..4b9fc045a3 100644 --- a/java/cuvs-java/src/test/java/com/nvidia/cuvs/GPUInfoIT.java +++ b/java/cuvs-java/src/test/java/com/nvidia/cuvs/GPUInfoIT.java @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ package com.nvidia.cuvs; @@ -19,7 +19,7 @@ public class GPUInfoIT extends CuVSTestCase { @Before public void setup() { - assumeTrue("not supported on " + System.getProperty("os.name"), isLinuxAmd64()); + assumeTrue("not supported on " + System.getProperty("os.name"), isLinuxSupportedArch()); } @Test diff --git a/java/cuvs-java/src/test/java/com/nvidia/cuvs/HnswAceBuildAndSearchIT.java b/java/cuvs-java/src/test/java/com/nvidia/cuvs/HnswAceBuildAndSearchIT.java index 835fcd6e23..351873ce9a 100644 --- a/java/cuvs-java/src/test/java/com/nvidia/cuvs/HnswAceBuildAndSearchIT.java +++ b/java/cuvs-java/src/test/java/com/nvidia/cuvs/HnswAceBuildAndSearchIT.java @@ -40,7 +40,7 @@ public class HnswAceBuildAndSearchIT extends CuVSTestCase { @Before public void setup() { - assumeTrue("not supported on " + System.getProperty("os.name"), isLinuxAmd64()); + assumeTrue("not supported on " + System.getProperty("os.name"), isLinuxSupportedArch()); initializeRandom(); log.trace("Random context initialized for test."); } diff --git a/java/cuvs-java/src/test/java/com/nvidia/cuvs/HnswBuildAndSearchIT.java b/java/cuvs-java/src/test/java/com/nvidia/cuvs/HnswBuildAndSearchIT.java index 7cb4f0c3fa..78f86bd964 100644 --- a/java/cuvs-java/src/test/java/com/nvidia/cuvs/HnswBuildAndSearchIT.java +++ b/java/cuvs-java/src/test/java/com/nvidia/cuvs/HnswBuildAndSearchIT.java @@ -28,7 +28,7 @@ public class HnswBuildAndSearchIT extends CuVSTestCase { @Before public void setup() { - assumeTrue("not supported on " + System.getProperty("os.name"), isLinuxAmd64()); + assumeTrue("not supported on " + System.getProperty("os.name"), isLinuxSupportedArch()); } // Sample data and query diff --git a/java/cuvs-java/src/test/java/com/nvidia/cuvs/HnswRandomizedIT.java b/java/cuvs-java/src/test/java/com/nvidia/cuvs/HnswRandomizedIT.java index fe5ca88073..4ace8fc92e 100644 --- a/java/cuvs-java/src/test/java/com/nvidia/cuvs/HnswRandomizedIT.java +++ b/java/cuvs-java/src/test/java/com/nvidia/cuvs/HnswRandomizedIT.java @@ -28,7 +28,7 @@ public class HnswRandomizedIT extends CuVSTestCase { @Before public void setup() { - assumeTrue("not supported on " + System.getProperty("os.name"), isLinuxAmd64()); + assumeTrue("not supported on " + System.getProperty("os.name"), isLinuxSupportedArch()); initializeRandom(); log.trace("Random context initialized for test."); } diff --git a/java/cuvs-java/src/test/java/com/nvidia/cuvs/MemoryTrackingResourcesIT.java b/java/cuvs-java/src/test/java/com/nvidia/cuvs/MemoryTrackingResourcesIT.java index b80a789025..1d9d40ab27 100644 --- a/java/cuvs-java/src/test/java/com/nvidia/cuvs/MemoryTrackingResourcesIT.java +++ b/java/cuvs-java/src/test/java/com/nvidia/cuvs/MemoryTrackingResourcesIT.java @@ -18,7 +18,7 @@ public class MemoryTrackingResourcesIT extends CuVSTestCase { @Before public void setup() { - assumeTrue("not supported on " + System.getProperty("os.name"), isLinuxAmd64()); + assumeTrue("not supported on " + System.getProperty("os.name"), isLinuxSupportedArch()); } @Test diff --git a/java/cuvs-java/src/test/java/com/nvidia/cuvs/MultiPartitionCagraSearchIT.java b/java/cuvs-java/src/test/java/com/nvidia/cuvs/MultiPartitionCagraSearchIT.java index 62e13b733e..2d1e33484a 100644 --- a/java/cuvs-java/src/test/java/com/nvidia/cuvs/MultiPartitionCagraSearchIT.java +++ b/java/cuvs-java/src/test/java/com/nvidia/cuvs/MultiPartitionCagraSearchIT.java @@ -40,7 +40,7 @@ public class MultiPartitionCagraSearchIT extends CuVSTestCase { @Before public void setup() { - assumeTrue("not supported on " + System.getProperty("os.name"), isLinuxAmd64()); + assumeTrue("not supported on " + System.getProperty("os.name"), isLinuxSupportedArch()); initializeRandom(); log.trace("Random context initialized for test."); } diff --git a/java/cuvs-java/src/test/java/com/nvidia/cuvs/TieredIndexIT.java b/java/cuvs-java/src/test/java/com/nvidia/cuvs/TieredIndexIT.java index 7965ca34b6..6a47e506ca 100644 --- a/java/cuvs-java/src/test/java/com/nvidia/cuvs/TieredIndexIT.java +++ b/java/cuvs-java/src/test/java/com/nvidia/cuvs/TieredIndexIT.java @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ package com.nvidia.cuvs; @@ -31,7 +31,7 @@ public class TieredIndexIT extends CuVSTestCase { @Before public void setup() { - assumeTrue("not supported on " + System.getProperty("os.name"), isLinuxAmd64()); + assumeTrue("not supported on " + System.getProperty("os.name"), isLinuxSupportedArch()); initializeRandom(); CuVSProvider.provider().enableRMMPooledMemory(10, 60); log.debug("Random context initialized for test"); diff --git a/java/cuvs-java/src/test/java/com/nvidia/cuvs/internal/common/UtilIT.java b/java/cuvs-java/src/test/java/com/nvidia/cuvs/internal/common/UtilIT.java index b6a5664831..3bd8d81464 100644 --- a/java/cuvs-java/src/test/java/com/nvidia/cuvs/internal/common/UtilIT.java +++ b/java/cuvs-java/src/test/java/com/nvidia/cuvs/internal/common/UtilIT.java @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ package com.nvidia.cuvs.internal.common; @@ -22,7 +22,7 @@ public class UtilIT extends CuVSTestCase { @Before public void setup() { - assumeTrue("not supported on " + System.getProperty("os.name"), isLinuxAmd64()); + assumeTrue("not supported on " + System.getProperty("os.name"), isLinuxSupportedArch()); } @Test diff --git a/java/cuvs-java/src/test/java/com/nvidia/cuvs/spi/CuVSProviderIT.java b/java/cuvs-java/src/test/java/com/nvidia/cuvs/spi/CuVSProviderIT.java index c40c42204e..076db40553 100644 --- a/java/cuvs-java/src/test/java/com/nvidia/cuvs/spi/CuVSProviderIT.java +++ b/java/cuvs-java/src/test/java/com/nvidia/cuvs/spi/CuVSProviderIT.java @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ package com.nvidia.cuvs.spi; @@ -18,7 +18,7 @@ public class CuVSProviderIT extends CuVSTestCase { @Before public void setup() { - assumeTrue("not supported on " + System.getProperty("os.name"), isLinuxAmd64()); + assumeTrue("not supported on " + System.getProperty("os.name"), isLinuxSupportedArch()); // Clear sysprop from previous runs/command line System.clearProperty("cuvs.max_version"); } diff --git a/java/panama-bindings/generate-bindings.sh b/java/panama-bindings/generate-bindings.sh index cdefd1ae1c..315b58f89f 100755 --- a/java/panama-bindings/generate-bindings.sh +++ b/java/panama-bindings/generate-bindings.sh @@ -10,7 +10,16 @@ REPODIR=$(cd "$(dirname "$0")"; cd ../../ ; pwd) CURDIR=$(cd "$(dirname "$0")"; pwd) TARGET_PACKAGE="com.nvidia.cuvs.internal.panama" -TARGET_DIR="targets/x86_64-linux/include" +ARCH=$(uname -m) +case "${ARCH}" in + x86_64) CUDA_TARGET_ARCH="x86_64-linux" ;; + aarch64) CUDA_TARGET_ARCH="sbsa-linux" ;; + *) + echo "Unsupported architecture for CUDA include directory lookup: ${ARCH}" + exit 1 + ;; +esac +TARGET_DIR="targets/${CUDA_TARGET_ARCH}/include" if [ -n "${CONDA_PREFIX:-}" ] && [ -d "${CONDA_PREFIX}/${TARGET_DIR}" ]; then CUDA_INCLUDE_DIR="${CONDA_PREFIX}/${TARGET_DIR}" elif [ -d "/usr/local/cuda/${TARGET_DIR}" ]; then @@ -25,7 +34,15 @@ export PATH if [[ $(command -v jextract) == "" ]]; then - JEXTRACT_FILENAME="openjdk-22-jextract+6-47_linux-x64_bin.tar.gz" + case "${ARCH}" in + x86_64) JEXTRACT_ARCH="linux-x64" ;; + aarch64) JEXTRACT_ARCH="linux-aarch64" ;; + *) + echo "Unsupported architecture for jextract download: ${ARCH}" + exit 1 + ;; + esac + JEXTRACT_FILENAME="openjdk-22-jextract+6-47_${JEXTRACT_ARCH}_bin.tar.gz" JEXTRACT_DOWNLOAD_URL="https://download.java.net/java/early_access/jextract/22/6/${JEXTRACT_FILENAME}" echo "jextract doesn't exist. Downloading it from $JEXTRACT_DOWNLOAD_URL."; wget -c $JEXTRACT_DOWNLOAD_URL From 7ed00a6e76d759f00e67bcf28870c230870b2d23 Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Wed, 2 Sep 2026 11:56:27 -1000 Subject: [PATCH 2/5] Don't build cuvs-java and cuvs-lucene on arm64. Test jars built on amd64 to run tests instead. --- .github/workflows/build.yaml | 17 ++---- .github/workflows/pr.yaml | 76 ++++++++++++++++++++---- .github/workflows/test.yaml | 78 ++++++++++++++++++++---- ci/test_java_prebuilt.sh | 83 ++++++++++++++++++++++++++ ci/test_lucene_prebuilt.sh | 112 +++++++++++++++++++++++++++++++++++ 5 files changed, 333 insertions(+), 33 deletions(-) create mode 100755 ci/test_java_prebuilt.sh create mode 100755 ci/test_lucene_prebuilt.sh diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 1352b32fd2..325b88dd15 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -163,6 +163,7 @@ jobs: with: build_type: pull-request matrix_name: conda-cpp-build + matrix_filter: map(select(.ARCH == "amd64")) java-build: needs: java-build-matrix permissions: @@ -181,17 +182,16 @@ jobs: with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} - arch: "${{ matrix.ARCH }}" + arch: "amd64" date: ${{ inputs.date }} container_image: "rapidsai/ci-conda:26.10-cuda${{ matrix.CUDA_VER }}-${{ matrix.LINUX_VER }}-py${{ matrix.PY_VER }}" script: "ci/build_java.sh" - # amd64 is the official published artifact; keep its name unqualified and only suffix other arches so they don't collide with it. - artifact-name: "cuvs-java-cuda${{ matrix.CUDA_VER }}${{ matrix.ARCH != 'amd64' && format('-{0}', matrix.ARCH) || '' }}" + artifact-name: "cuvs-java-cuda${{ matrix.CUDA_VER }}" file_to_upload: "java/cuvs-java/target/" sha: ${{ inputs.sha }} lucene-build: # Depends on the Java job for the cuvs-java artifact that cuvs-lucene builds against, and - # reuses its matrix so the two always agree on the set of CUDA versions and arches to test on. + # reuses its matrix so the two always agree on the set of CUDA versions. needs: [java-build-matrix, java-build] permissions: actions: read @@ -209,16 +209,11 @@ jobs: with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} - arch: "${{ matrix.ARCH }}" + arch: "amd64" date: ${{ inputs.date }} container_image: "rapidsai/ci-conda:26.10-cuda${{ matrix.CUDA_VER }}-${{ matrix.LINUX_VER }}-py${{ matrix.PY_VER }}" - # Always installs the amd64-built (official) cuvs-java jar, even when this job itself runs on - # a non-amd64 runner. cuvs-lucene depends on the plain jar, which bundles no native code, so - # running its tests on arm64 against the amd64-built jar doubles as a cross-arch check that the - # jextract-generated Panama bindings baked into that jar work unmodified on arm64 hardware. script: "ci/build_lucene.sh cuvs-java-cuda${{ matrix.CUDA_VER }}" - # amd64 is the official published artifact; keep its name unqualified and only suffix other arches so they don't collide with it. - artifact-name: "cuvs-lucene-cuda${{ matrix.CUDA_VER }}${{ matrix.ARCH != 'amd64' && format('-{0}', matrix.ARCH) || '' }}" + artifact-name: "cuvs-lucene-cuda${{ matrix.CUDA_VER }}" file_to_upload: "java/cuvs-lucene/target/" sha: ${{ inputs.sha }} python-build: diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 4b8d199aa4..dc2b4606d4 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -26,7 +26,10 @@ jobs: - rocky8-clib-tests - conda-java-build-and-tests-matrix - conda-java-build-and-tests + - conda-java-tests-other-arch-matrix + - conda-java-tests-other-arch - conda-lucene-build-and-tests + - conda-lucene-tests-other-arch - rust-build-matrix - rust-build - go-build-matrix @@ -589,6 +592,7 @@ jobs: with: build_type: pull-request matrix_name: conda-cpp-build + matrix_filter: map(select(.ARCH == "amd64")) conda-java-build-and-tests: needs: conda-java-build-and-tests-matrix permissions: @@ -607,15 +611,47 @@ jobs: with: build_type: pull-request node_type: "gpu-l4-latest-1" - arch: "${{ matrix.ARCH }}" + arch: "amd64" container_image: "rapidsai/ci-conda:26.10-cuda${{ matrix.CUDA_VER }}-${{ matrix.LINUX_VER }}-py${{ matrix.PY_VER }}" script: "ci/test_java.sh" - # amd64 is the official published artifact; keep its name unqualified and only suffix other arches so they don't collide with it. - artifact-name: "cuvs-java-cuda${{ matrix.CUDA_VER }}${{ matrix.ARCH != 'amd64' && format('-{0}', matrix.ARCH) || '' }}" + artifact-name: "cuvs-java-cuda${{ matrix.CUDA_VER }}" file_to_upload: "java/cuvs-java/target/" + conda-java-tests-other-arch-matrix: + # All architectures the C++ core builds for, except amd64. + needs: [conda-cpp-build, changed-files] + permissions: + contents: read + uses: rapidsai/shared-workflows/.github/workflows/compute-matrix.yaml@main + if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_java || fromJSON(needs.changed-files.outputs.changed_file_groups).test_cpp + with: + build_type: pull-request + matrix_name: conda-cpp-build + matrix_filter: map(select(.ARCH != "amd64")) + conda-java-tests-other-arch: + # Verifies that the amd64-built cuvs-java jar (jextract-generated Panama bindings and + # all) also works correctly on every non-amd64 architecture, by running its own IT test + # suite against the amd64-built classes without recompiling -- see ci/test_java_prebuilt.sh. + needs: [conda-java-tests-other-arch-matrix, conda-java-build-and-tests] + permissions: + actions: read + contents: read + id-token: write + packages: read + pull-requests: read + secrets: inherit # zizmor: ignore[secrets-inherit] + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@main + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.conda-java-tests-other-arch-matrix.outputs.matrix) }} + with: + build_type: pull-request + node_type: "gpu-l4-latest-1" + arch: "${{ matrix.ARCH }}" + container_image: "rapidsai/ci-conda:26.10-cuda${{ matrix.CUDA_VER }}-${{ matrix.LINUX_VER }}-py${{ matrix.PY_VER }}" + script: "ci/test_java_prebuilt.sh ${{ matrix.ARCH }} cuvs-java-cuda${{ matrix.CUDA_VER }}" conda-lucene-build-and-tests: # Depends on the Java job for the cuvs-java artifact that cuvs-lucene builds against, and - # reuses its matrix so the two always agree on the set of CUDA versions and arches to test on. + # reuses its matrix so the two always agree on the set of CUDA versions. needs: [conda-java-build-and-tests-matrix, conda-java-build-and-tests] permissions: actions: read @@ -633,16 +669,34 @@ jobs: with: build_type: pull-request node_type: "gpu-l4-latest-1" - arch: "${{ matrix.ARCH }}" + arch: "amd64" container_image: "rapidsai/ci-conda:26.10-cuda${{ matrix.CUDA_VER }}-${{ matrix.LINUX_VER }}-py${{ matrix.PY_VER }}" - # Always installs the amd64-built (official) cuvs-java jar, even when this job itself runs on - # a non-amd64 runner. cuvs-lucene depends on the plain jar, which bundles no native code, so - # running its tests on arm64 against the amd64-built jar doubles as a cross-arch check that the - # jextract-generated Panama bindings baked into that jar work unmodified on arm64 hardware. script: "ci/test_lucene.sh cuvs-java-cuda${{ matrix.CUDA_VER }}" - # amd64 is the official published artifact; keep its name unqualified and only suffix other arches so they don't collide with it. - artifact-name: "cuvs-lucene-cuda${{ matrix.CUDA_VER }}${{ matrix.ARCH != 'amd64' && format('-{0}', matrix.ARCH) || '' }}" + artifact-name: "cuvs-lucene-cuda${{ matrix.CUDA_VER }}" file_to_upload: "java/cuvs-lucene/target/" + conda-lucene-tests-other-arch: + # Verifies that cuvs-lucene, built against the amd64-built cuvs-java jar, also works + # correctly on every non-amd64 architecture, by running its test suite against the + # amd64-built classes without recompiling -- see ci/test_lucene_prebuilt.sh. Reuses + # conda-java-tests-other-arch-matrix so the two always agree on which architectures to test. + needs: [conda-java-tests-other-arch-matrix, conda-java-build-and-tests, conda-lucene-build-and-tests] + permissions: + actions: read + contents: read + id-token: write + packages: read + pull-requests: read + secrets: inherit # zizmor: ignore[secrets-inherit] + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@main + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.conda-java-tests-other-arch-matrix.outputs.matrix) }} + with: + build_type: pull-request + node_type: "gpu-l4-latest-1" + arch: "${{ matrix.ARCH }}" + container_image: "rapidsai/ci-conda:26.10-cuda${{ matrix.CUDA_VER }}-${{ matrix.LINUX_VER }}-py${{ matrix.PY_VER }}" + script: "ci/test_lucene_prebuilt.sh ${{ matrix.ARCH }} cuvs-java-cuda${{ matrix.CUDA_VER }} cuvs-lucene-cuda${{ matrix.CUDA_VER }}" rust-build-matrix: needs: [conda-cpp-build, changed-files] permissions: diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index fc8359277a..bf2f2724af 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -77,6 +77,7 @@ jobs: with: build_type: ${{ inputs.build_type }} matrix_name: conda-cpp-build + matrix_filter: map(select(.ARCH == "amd64")) conda-java-build-and-tests: needs: conda-java-build-and-tests-matrix permissions: @@ -98,15 +99,49 @@ jobs: date: ${{ inputs.date }} sha: ${{ inputs.sha }} node_type: "gpu-l4-latest-1" - arch: "${{ matrix.ARCH }}" + arch: "amd64" container_image: "rapidsai/ci-conda:26.10-cuda${{ matrix.CUDA_VER }}-${{ matrix.LINUX_VER }}-py${{ matrix.PY_VER }}" script: "ci/test_java.sh" - # amd64 is the official published artifact; keep its name unqualified and only suffix other arches so they don't collide with it. - artifact-name: "cuvs-java-cuda${{ matrix.CUDA_VER }}${{ matrix.ARCH != 'amd64' && format('-{0}', matrix.ARCH) || '' }}" + artifact-name: "cuvs-java-cuda${{ matrix.CUDA_VER }}" file_to_upload: "java/cuvs-java/target/" + conda-java-tests-other-arch-matrix: + # All architectures the C++ core builds for, except amd64 -- so a newly added + # architecture is automatically picked up here without a workflow change. + permissions: + contents: read + uses: rapidsai/shared-workflows/.github/workflows/compute-matrix.yaml@main + with: + build_type: ${{ inputs.build_type }} + matrix_name: conda-cpp-build + matrix_filter: map(select(.ARCH != "amd64")) + conda-java-tests-other-arch: + # Verifies that the amd64-built cuvs-java jar (jextract-generated Panama bindings and + # all) also works correctly on every non-amd64 architecture, by running its own IT test + # suite against the amd64-built classes without recompiling -- see ci/test_java_prebuilt.sh. + needs: [conda-java-tests-other-arch-matrix, conda-java-build-and-tests] + permissions: + actions: read + contents: read + id-token: write + packages: read + pull-requests: read + secrets: inherit # zizmor: ignore[secrets-inherit] + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@main + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.conda-java-tests-other-arch-matrix.outputs.matrix) }} + with: + build_type: ${{ inputs.build_type }} + branch: ${{ inputs.branch }} + date: ${{ inputs.date }} + sha: ${{ inputs.sha }} + node_type: "gpu-l4-latest-1" + arch: "${{ matrix.ARCH }}" + container_image: "rapidsai/ci-conda:26.10-cuda${{ matrix.CUDA_VER }}-${{ matrix.LINUX_VER }}-py${{ matrix.PY_VER }}" + script: "ci/test_java_prebuilt.sh ${{ matrix.ARCH }} cuvs-java-cuda${{ matrix.CUDA_VER }}" conda-lucene-build-and-tests: # Depends on the Java job for the cuvs-java artifact that cuvs-lucene builds against, and - # reuses its matrix so the two always agree on the set of CUDA versions and arches to test on. + # reuses its matrix so the two always agree on the set of CUDA versions. needs: [conda-java-build-and-tests-matrix, conda-java-build-and-tests] permissions: actions: read @@ -127,16 +162,37 @@ jobs: date: ${{ inputs.date }} sha: ${{ inputs.sha }} node_type: "gpu-l4-latest-1" - arch: "${{ matrix.ARCH }}" + arch: "amd64" container_image: "rapidsai/ci-conda:26.10-cuda${{ matrix.CUDA_VER }}-${{ matrix.LINUX_VER }}-py${{ matrix.PY_VER }}" - # Always installs the amd64-built (official) cuvs-java jar, even when this job itself runs on - # a non-amd64 runner. cuvs-lucene depends on the plain jar, which bundles no native code, so - # running its tests on arm64 against the amd64-built jar doubles as a cross-arch check that the - # jextract-generated Panama bindings baked into that jar work unmodified on arm64 hardware. script: "ci/test_lucene.sh cuvs-java-cuda${{ matrix.CUDA_VER }}" - # amd64 is the official published artifact; keep its name unqualified and only suffix other arches so they don't collide with it. - artifact-name: "cuvs-lucene-cuda${{ matrix.CUDA_VER }}${{ matrix.ARCH != 'amd64' && format('-{0}', matrix.ARCH) || '' }}" + artifact-name: "cuvs-lucene-cuda${{ matrix.CUDA_VER }}" file_to_upload: "java/cuvs-lucene/target/" + conda-lucene-tests-other-arch: + # Verifies that cuvs-lucene, built against the amd64-built cuvs-java jar, also works + # correctly on every non-amd64 architecture, by running its test suite against the + # amd64-built classes without recompiling -- see ci/test_lucene_prebuilt.sh. Reuses + # conda-java-tests-other-arch-matrix so the two always agree on which architectures to test. + needs: [conda-java-tests-other-arch-matrix, conda-java-build-and-tests, conda-lucene-build-and-tests] + permissions: + actions: read + contents: read + id-token: write + packages: read + pull-requests: read + secrets: inherit # zizmor: ignore[secrets-inherit] + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@main + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.conda-java-tests-other-arch-matrix.outputs.matrix) }} + with: + build_type: ${{ inputs.build_type }} + branch: ${{ inputs.branch }} + date: ${{ inputs.date }} + sha: ${{ inputs.sha }} + node_type: "gpu-l4-latest-1" + arch: "${{ matrix.ARCH }}" + container_image: "rapidsai/ci-conda:26.10-cuda${{ matrix.CUDA_VER }}-${{ matrix.LINUX_VER }}-py${{ matrix.PY_VER }}" + script: "ci/test_lucene_prebuilt.sh ${{ matrix.ARCH }} cuvs-java-cuda${{ matrix.CUDA_VER }} cuvs-lucene-cuda${{ matrix.CUDA_VER }}" wheel-tests-cuvs: permissions: actions: read diff --git a/ci/test_java_prebuilt.sh b/ci/test_java_prebuilt.sh new file mode 100755 index 0000000000..80ca216c8a --- /dev/null +++ b/ci/test_java_prebuilt.sh @@ -0,0 +1,83 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +# Runs cuvs-java's own IT test suite against the classes from a prior amd64 build job, +# without recompiling (or running jextract) on this host. cuvs-java is always built on +# amd64; this verifies that the resulting jextract-generated Panama bindings also work +# correctly against a native libcuvs_c.so on the given (non-amd64) architecture. +# +# Takes the target architecture (as used by the GitHub Actions job matrix, e.g. "arm64") +# and the name of the cuvs-java artifact uploaded by the amd64 Java job. + +ARCH="${1:?Usage: $0 }" +CUVS_JAVA_ARTIFACT="${2:?Usage: $0 }" + +rapids-logger "Testing the amd64-built cuvs-java artifact on ${ARCH}" + +case "${ARCH}" in + amd64) CONDA_ARCH="x86_64" ;; + arm64) CONDA_ARCH="aarch64" ;; + *) CONDA_ARCH="${ARCH}" ;; +esac + +if [ -e "/opt/conda/etc/profile.d/conda.sh" ]; then + . /opt/conda/etc/profile.d/conda.sh +fi + +rapids-logger "Check GPU usage" +nvidia-smi + +rapids-logger "Configuring conda strict channel priority" +conda config --set channel_priority strict + +rapids-logger "Downloading artifacts from previous jobs" +CPP_CHANNEL=$(rapids-download-from-github "$(rapids-artifact-name conda_cpp libcuvs cuvs --cuda "$RAPIDS_CUDA_VERSION")") +CUVS_JAVA_DIR=$(rapids-download-from-github "${CUVS_JAVA_ARTIFACT}") + +rapids-logger "Generate Java testing dependencies" + +ENV_YAML_DIR="$(mktemp -d)" + +rapids-dependency-file-generator \ + --output conda \ + --file-key java \ + --prepend-channel "${CPP_CHANNEL}" \ + --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=${CONDA_ARCH}" | tee "${ENV_YAML_DIR}/env.yaml" + +rapids-mamba-retry env create --yes -f "${ENV_YAML_DIR}/env.yaml" -n java + +# Temporarily allow unbound variables for conda activation. +set +u +conda activate java +set -u + +rapids-print-env + +# libcuvs comes from the conda environment here, matching the architecture this script is +# running on. +export LD_LIBRARY_PATH="${CONDA_PREFIX}/lib${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}" + +rapids-logger "Restore the amd64-built target/ directory so Maven can run the tests without recompiling" + +rm -rf java/cuvs-java/target +mkdir -p java/cuvs-java/target +cp -a "${CUVS_JAVA_DIR}/." java/cuvs-java/target/ + +EXITCODE=0 +trap "EXITCODE=1" ERR +set +e + +rapids-logger "Run cuvs-java IT tests against the amd64-built classes" + +# Invoking the failsafe plugin's goals directly (instead of through the "integration-test" +# lifecycle phase) runs only those goals -- it does not trigger compile/test-compile first, +# so this executes the already-compiled amd64 test classes as-is. +pushd java/cuvs-java +mvn --batch-mode failsafe:integration-test failsafe:verify +popd + +rapids-logger "Test script exiting with value: $EXITCODE" +exit ${EXITCODE} diff --git a/ci/test_lucene_prebuilt.sh b/ci/test_lucene_prebuilt.sh new file mode 100755 index 0000000000..35f81a14ce --- /dev/null +++ b/ci/test_lucene_prebuilt.sh @@ -0,0 +1,112 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +# Runs cuvs-lucene's test suite against the classes from a prior amd64 build job, without +# recompiling on this host. cuvs-lucene is always built on amd64, against the amd64-built +# cuvs-java jar; this verifies that pairing also works correctly on the given (non-amd64) +# architecture. Since cuvs-lucene depends on the plain (no bundled natives) cuvs-java jar, +# this doubles as a cross-arch check that the jextract-generated Panama bindings baked +# into the amd64 jar work unmodified against a native libcuvs_c.so on that architecture. +# +# Takes the target architecture (as used by the GitHub Actions job matrix, e.g. "arm64") +# and the names of the cuvs-java and cuvs-lucene artifacts uploaded by the amd64 jobs. + +ARCH="${1:?Usage: $0 }" +CUVS_JAVA_ARTIFACT="${2:?Usage: $0 }" +CUVS_LUCENE_ARTIFACT="${3:?Usage: $0 }" + +rapids-logger "Testing the amd64-built cuvs-java/cuvs-lucene artifacts on ${ARCH}" + +case "${ARCH}" in + amd64) CONDA_ARCH="x86_64" ;; + arm64) CONDA_ARCH="aarch64" ;; + *) CONDA_ARCH="${ARCH}" ;; +esac + +if [ -e "/opt/conda/etc/profile.d/conda.sh" ]; then + . /opt/conda/etc/profile.d/conda.sh +fi + +rapids-logger "Check GPU usage" +nvidia-smi + +rapids-logger "Configuring conda strict channel priority" +conda config --set channel_priority strict + +rapids-logger "Downloading artifacts from previous jobs" +CPP_CHANNEL=$(rapids-download-from-github "$(rapids-artifact-name conda_cpp libcuvs cuvs --cuda "$RAPIDS_CUDA_VERSION")") +CUVS_JAVA_DIR=$(rapids-download-from-github "${CUVS_JAVA_ARTIFACT}") +CUVS_LUCENE_DIR=$(rapids-download-from-github "${CUVS_LUCENE_ARTIFACT}") + +rapids-logger "Generate Java testing dependencies" + +ENV_YAML_DIR="$(mktemp -d)" + +rapids-dependency-file-generator \ + --output conda \ + --file-key java \ + --prepend-channel "${CPP_CHANNEL}" \ + --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=${CONDA_ARCH}" | tee "${ENV_YAML_DIR}/env.yaml" + +rapids-mamba-retry env create --yes -f "${ENV_YAML_DIR}/env.yaml" -n java + +# Temporarily allow unbound variables for conda activation. +set +u +conda activate java +set -u + +rapids-print-env + +# libcuvs comes from the conda environment here, matching the architecture this script is +# running on. cuvs-lucene depends on the plain cuvs-java jar, which bundles no native +# libraries, so the JVM resolves libcuvs_c.so through the dynamic loader. +export LD_LIBRARY_PATH="${CONDA_PREFIX}/lib${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}" + +rapids-logger "Install the amd64-built cuvs-java artifact into the local Maven repository" + +# cuvs-lucene resolves cuvs-java from the local Maven repository. Its pom.xml travels with +# the artifact and supplies the coordinates, so no version needs to be hardcoded. +CUVS_JAVA_POM="${CUVS_JAVA_DIR}/pom.xml" +if [ ! -f "${CUVS_JAVA_POM}" ]; then + echo "Could not find pom.xml in the cuvs-java artifact at ${CUVS_JAVA_DIR}" >&2 + exit 1 +fi + +# The artifact also carries the per-architecture native jar and the sources/javadoc/test jars; +# cuvs-lucene depends on the plain one. +mapfile -t CUVS_JAVA_JARS < <(find "${CUVS_JAVA_DIR}" -maxdepth 1 -name 'cuvs-java-*.jar' \ + ! -name '*-sources.jar' ! -name '*-javadoc.jar' ! -name '*-tests.jar' ! -name '*-cuda*.jar') +if [ "${#CUVS_JAVA_JARS[@]}" -ne 1 ]; then + echo "Expected exactly one cuvs-java jar in ${CUVS_JAVA_DIR}, found: ${CUVS_JAVA_JARS[*]:-none}" >&2 + exit 1 +fi + +# Install cuvs-java jar into .m2, cd is needed to pick up pom.xml in order to avoid rate limit of main maven repo +pushd java/cuvs-lucene +mvn --batch-mode install:install-file -Dfile="${CUVS_JAVA_JARS[0]}" -DpomFile="${CUVS_JAVA_POM}" +popd + +rapids-logger "Restore the amd64-built cuvs-lucene target/ directory so Maven can run the tests without recompiling" + +rm -rf java/cuvs-lucene/target +mkdir -p java/cuvs-lucene/target +cp -a "${CUVS_LUCENE_DIR}/." java/cuvs-lucene/target/ + +EXITCODE=0 +trap "EXITCODE=1" ERR +set +e + +rapids-logger "Run cuvs-lucene tests against the amd64-built classes" + +# Invoking the surefire plugin's goal directly (instead of through the "test" lifecycle +# phase) runs only that goal -- it does not trigger compile/test-compile first, so this +# executes the already-compiled amd64 test classes as-is. +pushd java/cuvs-lucene +mvn --batch-mode surefire:test +popd + +rapids-logger "Test script exiting with value: $EXITCODE" +exit ${EXITCODE} From 04956412dd041307c7eb56450e29d771f607da35 Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Wed, 2 Sep 2026 13:43:12 -1000 Subject: [PATCH 3/5] Fix testing issue on arm64 --- ci/test_java_prebuilt.sh | 17 +++++++++++++---- ci/test_lucene_prebuilt.sh | 11 +++++++---- 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/ci/test_java_prebuilt.sh b/ci/test_java_prebuilt.sh index 80ca216c8a..8c3bebd0a5 100755 --- a/ci/test_java_prebuilt.sh +++ b/ci/test_java_prebuilt.sh @@ -66,17 +66,26 @@ rm -rf java/cuvs-java/target mkdir -p java/cuvs-java/target cp -a "${CUVS_JAVA_DIR}/." java/cuvs-java/target/ +# Guarantee the compiler plugin's staleness check always sees these as up to date, +# regardless of how the checkout and artifact-download timestamps happen to compare -- +# otherwise a "source newer than class" mismatch triggers a real recompile, which would +# fail here since the jextract-generated Panama binding sources are gitignored and were +# never generated on this host. +find java/cuvs-java/target/classes java/cuvs-java/target/test-classes -type f -exec touch {} + + EXITCODE=0 trap "EXITCODE=1" ERR set +e rapids-logger "Run cuvs-java IT tests against the amd64-built classes" -# Invoking the failsafe plugin's goals directly (instead of through the "integration-test" -# lifecycle phase) runs only those goals -- it does not trigger compile/test-compile first, -# so this executes the already-compiled amd64 test classes as-is. +# cuvs-java is a multi-release JAR (JDKProvider and the jextract-generated bindings live +# under target/classes/META-INF/versions/22), and that layout is only resolvable on the +# module path through an actual packaged JAR, not a bare target/classes directory. The +# "verify" phase repackages the JAR (fast -- no recompilation, since the classes are +# already up to date) before running the IT tests, so it sees the right layout. pushd java/cuvs-java -mvn --batch-mode failsafe:integration-test failsafe:verify +mvn --batch-mode verify popd rapids-logger "Test script exiting with value: $EXITCODE" diff --git a/ci/test_lucene_prebuilt.sh b/ci/test_lucene_prebuilt.sh index 35f81a14ce..a1561e76c1 100755 --- a/ci/test_lucene_prebuilt.sh +++ b/ci/test_lucene_prebuilt.sh @@ -95,17 +95,20 @@ rm -rf java/cuvs-lucene/target mkdir -p java/cuvs-lucene/target cp -a "${CUVS_LUCENE_DIR}/." java/cuvs-lucene/target/ +# Guarantee the compiler plugin's staleness check always sees these as up to date, +# regardless of how the checkout and artifact-download timestamps happen to compare. +find java/cuvs-lucene/target/classes java/cuvs-lucene/target/test-classes -type f -exec touch {} + + EXITCODE=0 trap "EXITCODE=1" ERR set +e rapids-logger "Run cuvs-lucene tests against the amd64-built classes" -# Invoking the surefire plugin's goal directly (instead of through the "test" lifecycle -# phase) runs only that goal -- it does not trigger compile/test-compile first, so this -# executes the already-compiled amd64 test classes as-is. +# The "test" phase is cheap here since the compiler plugin skips recompilation once the +# touch above marks target/ as current. pushd java/cuvs-lucene -mvn --batch-mode surefire:test +mvn --batch-mode test popd rapids-logger "Test script exiting with value: $EXITCODE" From b5e4d5a458ec27a78908e987783412d1175be678 Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Wed, 2 Sep 2026 16:46:00 -1000 Subject: [PATCH 4/5] Let's try it again --- ci/test_java_prebuilt.sh | 17 ++++------------- ci/test_lucene_prebuilt.sh | 11 ++++------- java/cuvs-java/pom.xml | 34 ++++++++++++++++++++++++++++++++++ java/cuvs-lucene/pom.xml | 33 +++++++++++++++++++++++++++++++++ 4 files changed, 75 insertions(+), 20 deletions(-) diff --git a/ci/test_java_prebuilt.sh b/ci/test_java_prebuilt.sh index 8c3bebd0a5..dbab0ecd72 100755 --- a/ci/test_java_prebuilt.sh +++ b/ci/test_java_prebuilt.sh @@ -66,26 +66,17 @@ rm -rf java/cuvs-java/target mkdir -p java/cuvs-java/target cp -a "${CUVS_JAVA_DIR}/." java/cuvs-java/target/ -# Guarantee the compiler plugin's staleness check always sees these as up to date, -# regardless of how the checkout and artifact-download timestamps happen to compare -- -# otherwise a "source newer than class" mismatch triggers a real recompile, which would -# fail here since the jextract-generated Panama binding sources are gitignored and were -# never generated on this host. -find java/cuvs-java/target/classes java/cuvs-java/target/test-classes -type f -exec touch {} + - EXITCODE=0 trap "EXITCODE=1" ERR set +e rapids-logger "Run cuvs-java IT tests against the amd64-built classes" -# cuvs-java is a multi-release JAR (JDKProvider and the jextract-generated bindings live -# under target/classes/META-INF/versions/22), and that layout is only resolvable on the -# module path through an actual packaged JAR, not a bare target/classes directory. The -# "verify" phase repackages the JAR (fast -- no recompilation, since the classes are -# already up to date) before running the IT tests, so it sees the right layout. +# -Dskip.compile activates the pom's "skip-compile" profile, which disables all +# compilation for this run, forcing test to use amd64-compiled jar instead of +# local code. pushd java/cuvs-java -mvn --batch-mode verify +mvn --batch-mode verify -Dskip.compile=true popd rapids-logger "Test script exiting with value: $EXITCODE" diff --git a/ci/test_lucene_prebuilt.sh b/ci/test_lucene_prebuilt.sh index a1561e76c1..85516ca68f 100755 --- a/ci/test_lucene_prebuilt.sh +++ b/ci/test_lucene_prebuilt.sh @@ -95,20 +95,17 @@ rm -rf java/cuvs-lucene/target mkdir -p java/cuvs-lucene/target cp -a "${CUVS_LUCENE_DIR}/." java/cuvs-lucene/target/ -# Guarantee the compiler plugin's staleness check always sees these as up to date, -# regardless of how the checkout and artifact-download timestamps happen to compare. -find java/cuvs-lucene/target/classes java/cuvs-lucene/target/test-classes -type f -exec touch {} + - EXITCODE=0 trap "EXITCODE=1" ERR set +e rapids-logger "Run cuvs-lucene tests against the amd64-built classes" -# The "test" phase is cheap here since the compiler plugin skips recompilation once the -# touch above marks target/ as current. +# -Dskip.compile activates the pom's "skip-compile" profile, which disables all +# compilation for this run, forcing test to use amd64-compiled jar instead of +# local code. pushd java/cuvs-lucene -mvn --batch-mode test +mvn --batch-mode test -Dskip.compile=true popd rapids-logger "Test script exiting with value: $EXITCODE" diff --git a/java/cuvs-java/pom.xml b/java/cuvs-java/pom.xml index 4b7f551139..7b4fff9484 100644 --- a/java/cuvs-java/pom.xml +++ b/java/cuvs-java/pom.xml @@ -282,6 +282,40 @@ SPDX-License-Identifier: Apache-2.0 + + + skip-compile + + + skip.compile + + + + + + org.apache.maven.plugins + maven-compiler-plugin + + + default-compile + none + + + compile-java-22 + none + + + default-testCompile + none + + + + + + x86_64-cuda12 diff --git a/java/cuvs-lucene/pom.xml b/java/cuvs-lucene/pom.xml index 48dc6d7889..afa593cdfd 100644 --- a/java/cuvs-lucene/pom.xml +++ b/java/cuvs-lucene/pom.xml @@ -248,4 +248,37 @@ SPDX-License-Identifier: Apache-2.0 + + + + + skip-compile + + + skip.compile + + + + + + org.apache.maven.plugins + maven-compiler-plugin + + + default-compile + none + + + default-testCompile + none + + + + + + + From 292ed0fa04e2fd20a1a83ba166c79708c5126be6 Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Wed, 2 Sep 2026 16:49:35 -1000 Subject: [PATCH 5/5] Fix spaces --- ci/test_java_prebuilt.sh | 2 +- ci/test_lucene_prebuilt.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/test_java_prebuilt.sh b/ci/test_java_prebuilt.sh index dbab0ecd72..5f811aa9b5 100755 --- a/ci/test_java_prebuilt.sh +++ b/ci/test_java_prebuilt.sh @@ -73,7 +73,7 @@ set +e rapids-logger "Run cuvs-java IT tests against the amd64-built classes" # -Dskip.compile activates the pom's "skip-compile" profile, which disables all -# compilation for this run, forcing test to use amd64-compiled jar instead of +# compilation for this run, forcing test to use amd64-compiled jar instead of # local code. pushd java/cuvs-java mvn --batch-mode verify -Dskip.compile=true diff --git a/ci/test_lucene_prebuilt.sh b/ci/test_lucene_prebuilt.sh index 85516ca68f..477ff75ad3 100755 --- a/ci/test_lucene_prebuilt.sh +++ b/ci/test_lucene_prebuilt.sh @@ -102,7 +102,7 @@ set +e rapids-logger "Run cuvs-lucene tests against the amd64-built classes" # -Dskip.compile activates the pom's "skip-compile" profile, which disables all -# compilation for this run, forcing test to use amd64-compiled jar instead of +# compilation for this run, forcing test to use amd64-compiled jar instead of # local code. pushd java/cuvs-lucene mvn --batch-mode test -Dskip.compile=true