diff --git a/c/include/cuvs/neighbors/hnsw.h b/c/include/cuvs/neighbors/hnsw.h index 15eb1b0569..823656410c 100644 --- a/c/include/cuvs/neighbors/hnsw.h +++ b/c/include/cuvs/neighbors/hnsw.h @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -109,7 +109,7 @@ CUVS_EXPORT cuvsError_t cuvsHnswAceParamsDestroy(cuvsHnswAceParams_t params); struct cuvsHnswIndexParams { /* hierarchy of the hnsw index */ enum cuvsHnswHierarchy hierarchy; - /** Size of the candidate list during hierarchy construction when hierarchy is `CPU`*/ + /** Maximum candidate list size used during index construction. */ int ef_construction; /** Number of host threads to use to construct hierarchy when hierarchy is `CPU` or `GPU`. When the value is 0, the number of threads is automatically determined to the @@ -119,15 +119,15 @@ struct cuvsHnswIndexParams { is parallelized with the help of CPU threads. */ int num_threads; - /** HNSW M parameter: number of bi-directional links per node (used when building with ACE). - * graph_degree = m * 2, intermediate_graph_degree = m * 3. + /** HNSW M parameter: number of bi-directional links per node. When the graph is built on the GPU, + * this parameter is used to derive the internal CAGRA graph build parameters. */ size_t M; /** Distance type for the index. */ cuvsDistanceType metric; /** - * Optional: specify ACE parameters for building HNSW index using ACE algorithm. - * Set to nullptr for default behavior (from_cagra conversion). + * Optional ACE parameters for out-of-core graph construction. + * Set to nullptr to select the graph build algorithm automatically. */ cuvsHnswAceParams_t ace_params; }; @@ -285,22 +285,20 @@ CUVS_EXPORT cuvsError_t cuvsHnswFromCagraWithDataset(cuvsResources_t res, */ /** - * @defgroup hnsw_c_index_build Build HNSW index using ACE algorithm + * @defgroup hnsw_c_index_build Build an HNSW index * @{ */ /** - * @brief Build an HNSW index using ACE (Augmented Core Extraction) algorithm. + * @brief Build an HNSW index from HNSW parameters. * - * ACE enables building HNSW indexes for datasets too large to fit in GPU memory by: - * 1. Partitioning the dataset using balanced k-means into core and augmented partitions - * 2. Building sub-indexes for each partition independently - * 3. Concatenating sub-graphs into a final unified index + * The graph is built on the GPU and converted to an HNSW index that can be searched on the CPU. + * The graph build algorithm is selected automatically unless explicit ACE parameters are provided. * * NOTE: This function requires CUDA to be available at runtime. * * @param[in] res cuvsResources_t opaque C handle - * @param[in] params cuvsHnswIndexParams_t with ACE parameters configured + * @param[in] params cuvsHnswIndexParams_t with HNSW build parameters * @param[in] dataset DLManagedTensor* host dataset to build index from * @param[out] index cuvsHnswIndex_t to return the built HNSW index * @@ -314,18 +312,10 @@ CUVS_EXPORT cuvsError_t cuvsHnswFromCagraWithDataset(cuvsResources_t res, * cuvsResources_t res; * cuvsResourcesCreate(&res); * - * // Create ACE parameters - * cuvsHnswAceParams_t ace_params; - * cuvsHnswAceParamsCreate(&ace_params); - * ace_params->npartitions = 4; - * ace_params->use_disk = true; - * ace_params->build_dir = "/tmp/hnsw_ace_build"; - * * // Create index parameters * cuvsHnswIndexParams_t params; * cuvsHnswIndexParamsCreate(¶ms); * params->hierarchy = GPU; - * params->ace_params = ace_params; * params->M = 32; * params->ef_construction = 120; * @@ -340,7 +330,6 @@ CUVS_EXPORT cuvsError_t cuvsHnswFromCagraWithDataset(cuvsResources_t res, * cuvsHnswBuild(res, params, &dataset, hnsw_index); * * // Clean up - * cuvsHnswAceParamsDestroy(ace_params); * cuvsHnswIndexParamsDestroy(params); * cuvsHnswIndexDestroy(hnsw_index); * cuvsResourcesDestroy(res); diff --git a/c/src/neighbors/hnsw.cpp b/c/src/neighbors/hnsw.cpp index c69eda0ca0..edbd9a89e3 100644 --- a/c/src/neighbors/hnsw.cpp +++ b/c/src/neighbors/hnsw.cpp @@ -1,6 +1,6 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -37,15 +37,17 @@ void _build(cuvsResources_t res, cpp_params.M = params->M; cpp_params.metric = static_cast(params->metric); - // Configure ACE parameters - RAFT_EXPECTS(params->ace_params != nullptr, "ACE parameters must be set for hnsw::build"); - auto ace_params = cuvs::neighbors::hnsw::graph_build_params::ace_params(); - ace_params.npartitions = params->ace_params->npartitions; - ace_params.build_dir = params->ace_params->build_dir ? params->ace_params->build_dir : "/tmp/hnsw_ace_build"; - ace_params.use_disk = params->ace_params->use_disk; - ace_params.max_host_memory_gb = params->ace_params->max_host_memory_gb; - ace_params.max_gpu_memory_gb = params->ace_params->max_gpu_memory_gb; - cpp_params.graph_build_params = ace_params; + if (params->ace_params != nullptr) { + auto ace_params = cuvs::neighbors::hnsw::graph_build_params::ace_params(); + ace_params.npartitions = params->ace_params->npartitions; + ace_params.build_dir = params->ace_params->build_dir + ? params->ace_params->build_dir + : "/tmp/hnsw_ace_build"; + ace_params.use_disk = params->ace_params->use_disk; + ace_params.max_host_memory_gb = params->ace_params->max_host_memory_gb; + ace_params.max_gpu_memory_gb = params->ace_params->max_gpu_memory_gb; + cpp_params.graph_build_params = ace_params; + } using dataset_mdspan_type = raft::host_matrix_view; auto dataset_mds = cuvs::core::from_dlpack(dataset_tensor); diff --git a/c/tests/neighbors/ann_hnsw_c.cu b/c/tests/neighbors/ann_hnsw_c.cu index e97d0edfb1..e7a9604788 100644 --- a/c/tests/neighbors/ann_hnsw_c.cu +++ b/c/tests/neighbors/ann_hnsw_c.cu @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2023-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2023-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -123,6 +123,37 @@ TEST(CagraHnswC, BuildSearch) cuvsResourcesDestroy(res); } +TEST(HnswC, BuildWithoutAce) +{ + cuvsResources_t res; + cuvsResourcesCreate(&res); + + DLManagedTensor dataset_tensor; + dataset_tensor.dl_tensor.data = dataset; + dataset_tensor.dl_tensor.device.device_type = kDLCPU; + dataset_tensor.dl_tensor.ndim = 2; + dataset_tensor.dl_tensor.dtype.code = kDLFloat; + dataset_tensor.dl_tensor.dtype.bits = 32; + dataset_tensor.dl_tensor.dtype.lanes = 1; + int64_t dataset_shape[2] = {4, 2}; + dataset_tensor.dl_tensor.shape = dataset_shape; + dataset_tensor.dl_tensor.strides = nullptr; + + cuvsHnswIndexParams_t hnsw_params; + cuvsHnswIndexParamsCreate(&hnsw_params); + hnsw_params->M = 2; + hnsw_params->ef_construction = 100; + + cuvsHnswIndex_t hnsw_index; + cuvsHnswIndexCreate(&hnsw_index); + + ASSERT_EQ(cuvsHnswBuild(res, hnsw_params, &dataset_tensor, hnsw_index), CUVS_SUCCESS); + + cuvsHnswIndexParamsDestroy(hnsw_params); + cuvsHnswIndexDestroy(hnsw_index); + cuvsResourcesDestroy(res); +} + TEST(HnswAceC, BuildSearch) { // create cuvsResources_t diff --git a/fern/pages/c_api/c-api-neighbors-hnsw.md b/fern/pages/c_api/c-api-neighbors-hnsw.md index 317c9aa4f0..497fe89584 100644 --- a/fern/pages/c_api/c-api-neighbors-hnsw.md +++ b/fern/pages/c_api/c-api-neighbors-hnsw.md @@ -282,12 +282,12 @@ cuvsHnswIndex_t hnsw_index); [`cuvsError_t`](/api-reference/c-api-core-c-api#cuvserror-t) -## Build HNSW index using ACE algorithm +## Build an HNSW index ### cuvsHnswBuild -Build an HNSW index using ACE (Augmented Core Extraction) algorithm. +Build an HNSW index from HNSW parameters. ```c cuvsError_t cuvsHnswBuild(cuvsResources_t res, @@ -296,11 +296,8 @@ DLManagedTensor* dataset, cuvsHnswIndex_t index); ``` -ACE enables building HNSW indexes for datasets too large to fit in GPU memory by: - -1. Partitioning the dataset using balanced k-means into core and augmented partitions -2. Building sub-indexes for each partition independently -3. Concatenating sub-graphs into a final unified index +The graph is built on the GPU and converted to an HNSW index that can be searched on the CPU. +The graph build algorithm is selected automatically unless explicit ACE parameters are provided. NOTE: This function requires CUDA to be available at runtime. @@ -309,7 +306,7 @@ NOTE: This function requires CUDA to be available at runtime. | Name | Direction | Type | Description | | --- | --- | --- | --- | | `res` | in | [`cuvsResources_t`](/api-reference/c-api-core-c-api#cuvsresources-t) | cuvsResources_t opaque C handle | -| `params` | in | `cuvsHnswIndexParams_t` | cuvsHnswIndexParams_t with ACE parameters configured | +| `params` | in | `cuvsHnswIndexParams_t` | cuvsHnswIndexParams_t with HNSW build parameters | | `dataset` | in | `DLManagedTensor*` | DLManagedTensor* host dataset to build index from | | `index` | out | [`cuvsHnswIndex_t`](/api-reference/c-api-neighbors-hnsw#cuvshnswindex) | cuvsHnswIndex_t to return the built HNSW index | diff --git a/fern/pages/java_api/java-api-com-nvidia-cuvs-hnswindex.md b/fern/pages/java_api/java-api-com-nvidia-cuvs-hnswindex.md index df40dd733f..36d2623a2e 100644 --- a/fern/pages/java_api/java-api-com-nvidia-cuvs-hnswindex.md +++ b/fern/pages/java_api/java-api-com-nvidia-cuvs-hnswindex.md @@ -101,21 +101,16 @@ _Source: `java/cuvs-java/src/main/java/com/nvidia/cuvs/HnswIndex.java:53`_ static HnswIndex build(CuVSResources resources, HnswIndexParams hnswParams, CuVSMatrix dataset) throws Throwable ``` -Builds an HNSW index using the ACE (Augmented Core Extraction) algorithm. - -ACE enables building HNSW indexes for datasets too large to fit in GPU -memory by partitioning the dataset and building sub-indexes for each -partition independently. - -NOTE: This method requires `hnswParams.getAceParams()` to be set with -an instance of HnswAceParams. +Builds an HNSW index from HNSW parameters. The graph is built on the GPU and converted to an +HNSW index that can be searched on the CPU. The graph build algorithm is selected automatically +unless explicit ACE parameters are provided. **Parameters** | Name | Description | | --- | --- | | `resources` | The CuVS resources | -| `hnswParams` | Parameters for the HNSW index with ACE configuration | +| `hnswParams` | Parameters for the HNSW index | | `dataset` | The dataset to build the index from | **Returns** diff --git a/fern/pages/java_api/java-api-com-nvidia-cuvs-hnswindexparams.md b/fern/pages/java_api/java-api-com-nvidia-cuvs-hnswindexparams.md index 518043dc8d..7b1951fc08 100644 --- a/fern/pages/java_api/java-api-com-nvidia-cuvs-hnswindexparams.md +++ b/fern/pages/java_api/java-api-com-nvidia-cuvs-hnswindexparams.md @@ -83,8 +83,7 @@ public long getM() ``` Gets the HNSW M parameter: number of bi-directional links per node -(used when building with ACE). graph_degree = m * 2, -intermediate_graph_degree = m * 3. +used to derive the internal graph build parameters for GPU construction. **Returns** @@ -112,7 +111,8 @@ _Source: `java/cuvs-java/src/main/java/com/nvidia/cuvs/HnswIndexParams.java:142` public HnswAceParams getAceParams() ``` -Gets the ACE parameters for building HNSW index using ACE algorithm. +Gets the optional ACE parameters for explicit out-of-core graph construction. When not set, the +graph build algorithm is selected automatically. **Returns** @@ -159,14 +159,13 @@ _Source: `java/cuvs-java/src/main/java/com/nvidia/cuvs/HnswIndexParams.java:202` public Builder withEfConstruction(int efConstruction) ``` -Sets the size of the candidate list during hierarchy construction when -hierarchy is `CPU`. +Sets the maximum candidate list size used during index construction. **Parameters** | Name | Description | | --- | --- | -| `efConstruction` | the size of the candidate list during hierarchy construction when hierarchy is `CPU` | +| `efConstruction` | the maximum candidate list size used during construction | **Returns** @@ -222,8 +221,7 @@ public Builder withM(long m) ``` Sets the HNSW M parameter: number of bi-directional links per node -(used when building with ACE). graph_degree = m * 2, -intermediate_graph_degree = m * 3. +used to derive the internal graph build parameters for GPU construction. **Parameters** @@ -263,7 +261,8 @@ _Source: `java/cuvs-java/src/main/java/com/nvidia/cuvs/HnswIndexParams.java:262` public Builder withAceParams(HnswAceParams aceParams) ``` -Sets the ACE parameters for building HNSW index using ACE algorithm. +Sets optional ACE parameters for explicit out-of-core graph construction. When not set, the graph +build algorithm is selected automatically. **Parameters** diff --git a/fern/pages/java_api/java-api-com-nvidia-cuvs-spi-cuvsprovider.md b/fern/pages/java_api/java-api-com-nvidia-cuvs-spi-cuvsprovider.md index 17557e5e57..5cc6c3eeb0 100644 --- a/fern/pages/java_api/java-api-com-nvidia-cuvs-spi-cuvsprovider.md +++ b/fern/pages/java_api/java-api-com-nvidia-cuvs-spi-cuvsprovider.md @@ -229,14 +229,14 @@ _Source: `java/cuvs-java/src/main/java/com/nvidia/cuvs/spi/CuVSProvider.java:123 HnswIndex hnswIndexBuild(CuVSResources resources, HnswIndexParams hnswParams, CuVSMatrix dataset) throws Throwable ``` -Builds an HNSW index using the ACE (Augmented Core Extraction) algorithm. +Builds an HNSW index from HNSW parameters using GPU graph construction. **Parameters** | Name | Description | | --- | --- | | `resources` | The CuVS resources | -| `hnswParams` | Parameters for the HNSW index with ACE configuration | +| `hnswParams` | Parameters for the HNSW index | | `dataset` | The dataset to build the index from | **Returns** diff --git a/fern/pages/python_api/python-api-neighbors-hnsw.md b/fern/pages/python_api/python-api-neighbors-hnsw.md index a27d58918e..d4578b81ad 100644 --- a/fern/pages/python_api/python-api-neighbors-hnsw.md +++ b/fern/pages/python_api/python-api-neighbors-hnsw.md @@ -87,11 +87,11 @@ Parameters to build index for HNSW nearest neighbor search | Name | Type | Description | | --- | --- | --- | | `hierarchy` | `string, default = "gpu" (optional)` | The hierarchy of the HNSW index.
Valid values are ["none", "cpu", "gpu"].
- "none": No hierarchy is built.
- "cpu": Hierarchy is built using CPU.
- "gpu": Hierarchy is built using GPU. | -| `ef_construction` | `int, default = 200 (optional)` | Maximum number of candidate list size used during construction when hierarchy is `cpu`. | +| `ef_construction` | `int, default = 200 (optional)` | Maximum candidate list size used during index construction. | | `num_threads` | `int, default = 0 (optional)` | Number of CPU threads used to increase construction parallelism when hierarchy is `cpu` or `gpu`. When the value is 0, the number of threads is automatically determined to the maximum number of threads available.
NOTE: When hierarchy is `gpu`, while the majority of the work is done on the GPU, initialization of the HNSW index itself and some other work is parallelized with the help of CPU threads. | -| `M` | `int, default = 32 (optional)` | HNSW M parameter: number of bi-directional links per node (used when building with ACE). graph_degree = m * 2, intermediate_graph_degree = m * 3. | +| `M` | `int, default = 32 (optional)` | HNSW M parameter: number of bi-directional links per node. When the graph is built on the GPU, this parameter is used to derive the internal CAGRA graph build parameters. | | `metric` | `string, default = "sqeuclidean" (optional)` | Distance metric to use.
Valid values: ["sqeuclidean", "inner_product"] | -| `ace_params` | `AceParams, default = None (optional)` | ACE parameters for building HNSW index using ACE algorithm. If set, enables the build() function to use ACE for index construction. | +| `ace_params` | `AceParams, default = None (optional)` | Explicit ACE parameters for out-of-core graph construction. When not set, the graph build algorithm is selected automatically. | **Constructor** @@ -200,20 +200,17 @@ def num_threads(self) def build(IndexParams index_params, dataset, resources=None) ``` -Build an HNSW index using the ACE (Augmented Core Extraction) algorithm. +Build an HNSW index from HNSW parameters. -ACE enables building HNSW indices for datasets too large to fit in GPU -memory by partitioning the dataset and building sub-indices for each -partition independently. - -NOTE: This function requires `index_params.ace_params` to be set with -an instance of AceParams. +The graph is built on the GPU and converted to an HNSW index that can be +searched on the CPU. The graph build algorithm is selected automatically +unless explicit ACE parameters are provided. **Parameters** | Name | Type | Description | | --- | --- | --- | -| `index_params` | `IndexParams` | Parameters for the HNSW index with ACE configuration. Must have `ace_params` set. | +| `index_params` | `IndexParams` | Parameters for the HNSW index. | | `dataset` | `Host array interface compliant matrix shape (n_samples, dim)` | Supported dtype [float32, float16, int8, uint8] | | `resources` | `cuvs.common.Resources, optional` | | @@ -234,17 +231,9 @@ an instance of AceParams. >>> dataset = np.random.random_sample((n_samples, n_features), ... dtype=np.float32) >>> ->>> # Create ACE parameters ->>> ace_params = hnsw.AceParams( -... npartitions=4, -... use_disk=True, -... build_dir="/tmp/hnsw_ace_build" -... ) ->>> ->>> # Create index parameters with ACE +>>> # Create HNSW index parameters >>> index_params = hnsw.IndexParams( ... hierarchy="gpu", -... ace_params=ace_params, ... ef_construction=120, ... M=32, ... metric="sqeuclidean" diff --git a/java/cuvs-java/src/main/java/com/nvidia/cuvs/HnswIndex.java b/java/cuvs-java/src/main/java/com/nvidia/cuvs/HnswIndex.java index 3eef491b62..50216729a8 100644 --- a/java/cuvs-java/src/main/java/com/nvidia/cuvs/HnswIndex.java +++ b/java/cuvs-java/src/main/java/com/nvidia/cuvs/HnswIndex.java @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ package com.nvidia.cuvs; @@ -58,17 +58,12 @@ static HnswIndex fromCagra(HnswIndexParams hnswParams, CagraIndex cagraIndex) th } /** - * Builds an HNSW index using the ACE (Augmented Core Extraction) algorithm. - * - * ACE enables building HNSW indexes for datasets too large to fit in GPU - * memory by partitioning the dataset and building sub-indexes for each - * partition independently. - * - * NOTE: This method requires `hnswParams.getAceParams()` to be set with - * an instance of HnswAceParams. + * Builds an HNSW index from HNSW parameters. The graph is built on the GPU and converted to an + * HNSW index that can be searched on the CPU. The graph build algorithm is selected automatically + * unless explicit ACE parameters are provided. * * @param resources The CuVS resources - * @param hnswParams Parameters for the HNSW index with ACE configuration + * @param hnswParams Parameters for the HNSW index * @param dataset The dataset to build the index from * @return A new HNSW index ready for search * @throws Throwable if an error occurs during building @@ -78,7 +73,6 @@ static HnswIndex build(CuVSResources resources, HnswIndexParams hnswParams, CuVS Objects.requireNonNull(resources); Objects.requireNonNull(hnswParams); Objects.requireNonNull(dataset); - Objects.requireNonNull(hnswParams.getAceParams(), "ACE parameters must be set for build()"); return CuVSProvider.provider().hnswIndexBuild(resources, hnswParams, dataset); } diff --git a/java/cuvs-java/src/main/java/com/nvidia/cuvs/HnswIndexParams.java b/java/cuvs-java/src/main/java/com/nvidia/cuvs/HnswIndexParams.java index 070cbedae1..5956b6cdc6 100644 --- a/java/cuvs-java/src/main/java/com/nvidia/cuvs/HnswIndexParams.java +++ b/java/cuvs-java/src/main/java/com/nvidia/cuvs/HnswIndexParams.java @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ package com.nvidia.cuvs; @@ -126,8 +126,7 @@ public int getVectorDimension() { /** * Gets the HNSW M parameter: number of bi-directional links per node - * (used when building with ACE). graph_degree = m * 2, - * intermediate_graph_degree = m * 3. + * used to derive the internal graph build parameters for GPU construction. * * @return the M parameter */ @@ -145,7 +144,8 @@ public CuvsDistanceType getMetric() { } /** - * Gets the ACE parameters for building HNSW index using ACE algorithm. + * Gets the optional ACE parameters for explicit out-of-core graph construction. When not set, the + * graph build algorithm is selected automatically. * * @return the ACE parameters, or null if not set */ @@ -206,11 +206,9 @@ public Builder withHierarchy(CuvsHnswHierarchy hierarchy) { } /** - * Sets the size of the candidate list during hierarchy construction when - * hierarchy is `CPU`. + * Sets the maximum candidate list size used during index construction. * - * @param efConstruction the size of the candidate list during hierarchy - * construction when hierarchy is `CPU` + * @param efConstruction the maximum candidate list size used during construction * @return an instance of Builder */ public Builder withEfConstruction(int efConstruction) { @@ -242,9 +240,8 @@ public Builder withVectorDimension(int vectorDimension) { } /** - * Sets the HNSW M parameter: number of bi-directional links per node - * (used when building with ACE). graph_degree = m * 2, - * intermediate_graph_degree = m * 3. + * Sets the HNSW M parameter: number of bi-directional links per node used to derive the internal + * graph build parameters for GPU construction. * * @param m the M parameter * @return an instance of Builder @@ -266,7 +263,8 @@ public Builder withMetric(CuvsDistanceType metric) { } /** - * Sets the ACE parameters for building HNSW index using ACE algorithm. + * Sets optional ACE parameters for explicit out-of-core graph construction. When not set, the + * graph build algorithm is selected automatically. * * @param aceParams the ACE parameters * @return an instance of Builder diff --git a/java/cuvs-java/src/main/java/com/nvidia/cuvs/spi/CuVSProvider.java b/java/cuvs-java/src/main/java/com/nvidia/cuvs/spi/CuVSProvider.java index 60a2928aec..c18074ea58 100644 --- a/java/cuvs-java/src/main/java/com/nvidia/cuvs/spi/CuVSProvider.java +++ b/java/cuvs-java/src/main/java/com/nvidia/cuvs/spi/CuVSProvider.java @@ -150,10 +150,10 @@ HnswIndex.Builder newHnswIndexBuilder(CuVSResources cuVSResources) HnswIndex hnswIndexFromCagra(HnswIndexParams hnswParams, CagraIndex cagraIndex) throws Throwable; /** - * Builds an HNSW index using the ACE (Augmented Core Extraction) algorithm. + * Builds an HNSW index from HNSW parameters using GPU graph construction. * * @param resources The CuVS resources - * @param hnswParams Parameters for the HNSW index with ACE configuration + * @param hnswParams Parameters for the HNSW index * @param dataset The dataset to build the index from * @return A new HNSW index ready for search * @throws Throwable if an error occurs during building diff --git a/java/cuvs-java/src/main/java22/com/nvidia/cuvs/internal/HnswIndexImpl.java b/java/cuvs-java/src/main/java22/com/nvidia/cuvs/internal/HnswIndexImpl.java index ca528ac010..e9d4d29a85 100644 --- a/java/cuvs-java/src/main/java22/com/nvidia/cuvs/internal/HnswIndexImpl.java +++ b/java/cuvs-java/src/main/java22/com/nvidia/cuvs/internal/HnswIndexImpl.java @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ package com.nvidia.cuvs.internal; @@ -243,10 +243,10 @@ public static HnswIndex.Builder newBuilder(CuVSResources cuvsResources) { } /** - * Builds an HNSW index using the ACE algorithm. + * Builds an HNSW index from HNSW parameters using GPU graph construction. * * @param resources The CuVS resources - * @param hnswParams Parameters for the HNSW index with ACE configuration + * @param hnswParams Parameters for the HNSW index * @param dataset The dataset to build the index from * @return A new HNSW index ready for search * @throws Throwable if an error occurs during building @@ -256,7 +256,6 @@ public static HnswIndex build(CuVSResources resources, HnswIndexParams hnswParam Objects.requireNonNull(resources); Objects.requireNonNull(hnswParams); Objects.requireNonNull(dataset); - Objects.requireNonNull(hnswParams.getAceParams(), "ACE parameters must be set for build()"); // Create HNSW index MemorySegment hnswIndex = createHnswIndexHandle(); @@ -268,7 +267,7 @@ public static HnswIndex build(CuVSResources resources, HnswIndexParams hnswParam MemorySegment hnswParamsMemorySegment = hnswParamsHandle.handle(); - // Link ACE params to HNSW index params + // Link optional ACE params to HNSW index params cuvsHnswIndexParams.ace_params(hnswParamsMemorySegment, aceParamsHandle.handle()); // Prepare dataset tensor @@ -302,6 +301,10 @@ private static CloseableHandle createHnswIndexParamsForBuild(Arena arena, HnswIn } private static CloseableHandle createHnswAceParams(Arena arena, HnswAceParams aceParams) { + if (aceParams == null) { + return CloseableHandle.NULL; + } + var params = createHnswAceParamsNative(); MemorySegment seg = params.handle(); diff --git a/java/cuvs-java/src/test/java/com/nvidia/cuvs/HnswBuildAndSearchIT.java b/java/cuvs-java/src/test/java/com/nvidia/cuvs/HnswBuildAndSearchIT.java index c963efa374..b5b5a91c13 100644 --- a/java/cuvs-java/src/test/java/com/nvidia/cuvs/HnswBuildAndSearchIT.java +++ b/java/cuvs-java/src/test/java/com/nvidia/cuvs/HnswBuildAndSearchIT.java @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ package com.nvidia.cuvs; @@ -123,6 +123,38 @@ public void testIndexingAndSearchingFlow() throws Throwable { } } + @Test + public void testBuildWithoutAce() throws Throwable { + List> expectedResults = + Arrays.asList( + Map.of(3, 0.038782578f), + Map.of(0, 0.12472608f), + Map.of(3, 0.047766715f), + Map.of(1, 0.15224178f)); + + HnswIndexParams indexParams = + new HnswIndexParams.Builder() + .withHierarchy(HnswHierarchy.GPU) + .withM(2) + .withEfConstruction(100) + .withMetric(CuvsDistanceType.L2Expanded) + .withVectorDimension(2) + .build(); + + try (CuVSResources resources = CheckedCuVSResources.create(); + CuVSMatrix datasetMatrix = CuVSMatrix.ofArray(dataset); + HnswIndex index = HnswIndex.build(resources, indexParams, datasetMatrix)) { + HnswQuery query = + new HnswQuery.Builder(resources) + .withQueryVectors(queries) + .withSearchParams(new HnswSearchParams.Builder().withEF(100).build()) + .withTopK(1) + .build(); + + checkResults(expectedResults, index.search(query).getResults()); + } + } + @Test public void testIndexingAndSearchingWithFunctionMapping() throws Throwable { // Expected search results diff --git a/python/cuvs/cuvs/neighbors/hnsw/hnsw.pyx b/python/cuvs/cuvs/neighbors/hnsw/hnsw.pyx index 6757695de3..91256a0e18 100644 --- a/python/cuvs/cuvs/neighbors/hnsw/hnsw.pyx +++ b/python/cuvs/cuvs/neighbors/hnsw/hnsw.pyx @@ -1,5 +1,5 @@ # -# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # cython: language_level=3 @@ -129,8 +129,7 @@ cdef class IndexParams: - "cpu": Hierarchy is built using CPU. - "gpu": Hierarchy is built using GPU. ef_construction : int, default = 200 (optional) - Maximum number of candidate list size used during construction - when hierarchy is `cpu`. + Maximum candidate list size used during index construction. num_threads : int, default = 0 (optional) Number of CPU threads used to increase construction parallelism when hierarchy is `cpu` or `gpu`. When the value is 0, the number of @@ -140,14 +139,14 @@ cdef class IndexParams: on the GPU, initialization of the HNSW index itself and some other work is parallelized with the help of CPU threads. M : int, default = 32 (optional) - HNSW M parameter: number of bi-directional links per node - (used when building with ACE). graph_degree = m * 2, - intermediate_graph_degree = m * 3. + HNSW M parameter: number of bi-directional links per node. When the + graph is built on the GPU, this parameter is used to derive the + internal CAGRA graph build parameters. metric : string, default = "sqeuclidean" (optional) Distance metric to use. Valid values: ["sqeuclidean", "inner_product"] ace_params : AceParams, default = None (optional) - ACE parameters for building HNSW index using ACE algorithm. If set, - enables the build() function to use ACE for index construction. + Explicit ACE parameters for out-of-core graph construction. When not + set, the graph build algorithm is selected automatically. """ cdef cuvsHnswIndexParams* params @@ -471,20 +470,16 @@ def from_cagra(IndexParams index_params, cagra.Index cagra_index, @auto_sync_resources def build(IndexParams index_params, dataset, resources=None): """ - Build an HNSW index using the ACE (Augmented Core Extraction) algorithm. + Build an HNSW index from HNSW parameters. - ACE enables building HNSW indices for datasets too large to fit in GPU - memory by partitioning the dataset and building sub-indices for each - partition independently. - - NOTE: This function requires `index_params.ace_params` to be set with - an instance of AceParams. + The graph is built on the GPU and converted to an HNSW index that can be + searched on the CPU. The graph build algorithm is selected automatically + unless explicit ACE parameters are provided. Parameters ---------- index_params : IndexParams - Parameters for the HNSW index with ACE configuration. - Must have `ace_params` set. + Parameters for the HNSW index. dataset : Host array interface compliant matrix shape (n_samples, dim) Supported dtype [float32, float16, int8, uint8] {resources_docstring} @@ -504,17 +499,9 @@ def build(IndexParams index_params, dataset, resources=None): >>> dataset = np.random.random_sample((n_samples, n_features), ... dtype=np.float32) >>> - >>> # Create ACE parameters - >>> ace_params = hnsw.AceParams( - ... npartitions=4, - ... use_disk=True, - ... build_dir="/tmp/hnsw_ace_build" - ... ) - >>> - >>> # Create index parameters with ACE + >>> # Create HNSW index parameters >>> index_params = hnsw.IndexParams( ... hierarchy="gpu", - ... ace_params=ace_params, ... ef_construction=120, ... M=32, ... metric="sqeuclidean" @@ -532,10 +519,6 @@ def build(IndexParams index_params, dataset, resources=None): ... k=10 ... ) """ - if index_params.ace_params is None: - raise ValueError("index_params.ace_params must be set for hnsw.build(). " - "Use AceParams to configure ACE algorithm parameters.") - dataset_ai = wrap_array(dataset) _check_input_array(dataset_ai, [np.dtype('float32'), np.dtype('float16'), diff --git a/python/cuvs/cuvs/tests/test_hnsw.py b/python/cuvs/cuvs/tests/test_hnsw.py index 3eef920264..86e12e96f5 100644 --- a/python/cuvs/cuvs/tests/test_hnsw.py +++ b/python/cuvs/cuvs/tests/test_hnsw.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # @@ -91,6 +91,29 @@ def test_hnsw(dtype, k, ef, num_threads, metric, build_algo, hierarchy): ) +def test_hnsw_build_hnsw_first_api(): + dataset = generate_data((2000, 16), np.float32) + queries = generate_data((100, 16), np.float32) + k = 10 + + index_params = hnsw.IndexParams( + hierarchy="gpu", + M=16, + ef_construction=120, + metric="sqeuclidean", + ) + index = hnsw.build(index_params, dataset) + + _, neighbors = hnsw.search(hnsw.SearchParams(ef=100), index, queries, k) + + reference = NearestNeighbors( + n_neighbors=k, algorithm="brute", metric="sqeuclidean" + ).fit(dataset) + _, reference_neighbors = reference.kneighbors(queries) + + assert calc_recall(neighbors, reference_neighbors) >= 0.9 + + def run_hnsw_extend_test( n_rows=10000, add_rows=2000, diff --git a/rust/cuvs-sys/src/bindings.rs b/rust/cuvs-sys/src/bindings.rs index 171af6f422..f36c60aea2 100644 --- a/rust/cuvs-sys/src/bindings.rs +++ b/rust/cuvs-sys/src/bindings.rs @@ -2084,15 +2084,15 @@ unsafe extern "C" { #[derive(Debug, Copy, Clone)] pub struct cuvsHnswIndexParams { pub hierarchy: cuvsHnswHierarchy, - #[doc = " Size of the candidate list during hierarchy construction when hierarchy is `CPU`"] + #[doc = " Maximum candidate list size used during index construction."] pub ef_construction: ::std::os::raw::c_int, #[doc = " Number of host threads to use to construct hierarchy when hierarchy is `CPU` or `GPU`.\nWhen the value is 0, the number of threads is automatically determined to the\nmaximum number of threads available.\nNOTE: When hierarchy is `GPU`, while the majority of the work is done on the GPU,\ninitialization of the HNSW index itself and some other work\nis parallelized with the help of CPU threads."] pub num_threads: ::std::os::raw::c_int, - #[doc = " HNSW M parameter: number of bi-directional links per node (used when building with ACE).\n graph_degree = m * 2, intermediate_graph_degree = m * 3."] + #[doc = " HNSW M parameter: number of bi-directional links per node. When the graph is built on the GPU,\n this parameter is used to derive the internal CAGRA graph build parameters."] pub M: usize, #[doc = " Distance type for the index."] pub metric: cuvsDistanceType, - #[doc = " Optional: specify ACE parameters for building HNSW index using ACE algorithm.\n Set to nullptr for default behavior (from_cagra conversion)."] + #[doc = " Optional ACE parameters for out-of-core graph construction.\n Set to nullptr to select the graph build algorithm automatically."] pub ace_params: cuvsHnswAceParams_t, } #[allow(clippy::unnecessary_operation, clippy::identity_op)] @@ -2196,7 +2196,7 @@ unsafe extern "C" { } unsafe extern "C" { #[must_use] - #[doc = " @brief Build an HNSW index using ACE (Augmented Core Extraction) algorithm.\n\n ACE enables building HNSW indexes for datasets too large to fit in GPU memory by:\n 1. Partitioning the dataset using balanced k-means into core and augmented partitions\n 2. Building sub-indexes for each partition independently\n 3. Concatenating sub-graphs into a final unified index\n\n NOTE: This function requires CUDA to be available at runtime.\n\n @param[in] res cuvsResources_t opaque C handle\n @param[in] params cuvsHnswIndexParams_t with ACE parameters configured\n @param[in] dataset DLManagedTensor* host dataset to build index from\n @param[out] index cuvsHnswIndex_t to return the built HNSW index\n\n @return cuvsError_t\n\n @code{.c}\n #include \n #include \n\n // Create cuvsResources_t\n cuvsResources_t res;\n cuvsResourcesCreate(&res);\n\n // Create ACE parameters\n cuvsHnswAceParams_t ace_params;\n cuvsHnswAceParamsCreate(&ace_params);\n ace_params->npartitions = 4;\n ace_params->use_disk = true;\n ace_params->build_dir = \"/tmp/hnsw_ace_build\";\n\n // Create index parameters\n cuvsHnswIndexParams_t params;\n cuvsHnswIndexParamsCreate(¶ms);\n params->hierarchy = GPU;\n params->ace_params = ace_params;\n params->M = 32;\n params->ef_construction = 120;\n\n // Create HNSW index\n cuvsHnswIndex_t hnsw_index;\n cuvsHnswIndexCreate(&hnsw_index);\n\n // Assume dataset is a populated DLManagedTensor with host data\n DLManagedTensor dataset;\n\n // Build the index\n cuvsHnswBuild(res, params, &dataset, hnsw_index);\n\n // Clean up\n cuvsHnswAceParamsDestroy(ace_params);\n cuvsHnswIndexParamsDestroy(params);\n cuvsHnswIndexDestroy(hnsw_index);\n cuvsResourcesDestroy(res);\n @endcode"] + #[doc = " @brief Build an HNSW index from HNSW parameters.\n\n The graph is built on the GPU and converted to an HNSW index that can be searched on the CPU.\n The graph build algorithm is selected automatically unless explicit ACE parameters are provided.\n\n NOTE: This function requires CUDA to be available at runtime.\n\n @param[in] res cuvsResources_t opaque C handle\n @param[in] params cuvsHnswIndexParams_t with HNSW build parameters\n @param[in] dataset DLManagedTensor* host dataset to build index from\n @param[out] index cuvsHnswIndex_t to return the built HNSW index\n\n @return cuvsError_t\n\n @code{.c}\n #include \n #include \n\n // Create cuvsResources_t\n cuvsResources_t res;\n cuvsResourcesCreate(&res);\n\n // Create index parameters\n cuvsHnswIndexParams_t params;\n cuvsHnswIndexParamsCreate(¶ms);\n params->hierarchy = GPU;\n params->M = 32;\n params->ef_construction = 120;\n\n // Create HNSW index\n cuvsHnswIndex_t hnsw_index;\n cuvsHnswIndexCreate(&hnsw_index);\n\n // Assume dataset is a populated DLManagedTensor with host data\n DLManagedTensor dataset;\n\n // Build the index\n cuvsHnswBuild(res, params, &dataset, hnsw_index);\n\n // Clean up\n cuvsHnswIndexParamsDestroy(params);\n cuvsHnswIndexDestroy(hnsw_index);\n cuvsResourcesDestroy(res);\n @endcode"] pub fn cuvsHnswBuild( res: cuvsResources_t, params: cuvsHnswIndexParams_t,