feat(perf): add cache prefetch auto tuning to the graph

Author: Neiko2002

cpp/API.md

@@ -141,6 +141,12 @@ public:
     bool isMutable();                                      ///< Check if graph allows modifications
     InternalGraph& internal();                             ///< Access underlying internal graph
 
+   // --- Traversal Tuning ---
+
+   /// Auto-tune traversal prefetch parameters (po, pl, nl) by timing traversal over sampled vertices.
+   /// The graph self-samples its own stored features as queries; call once after construction.
+   void optimize(size_t sample_count = 50, uint32_t k = 100, uint32_t ef = 200, uint32_t seed = 7);
+
     // --- Conversions & Persistence ---
 
     /// Convert to a compact, read-only graph (ReadOnlyGraph) optimized for search
@@ -333,6 +339,34 @@ vector<ResultSet> rerank(
     size_t num_threads = 0
 );
 
+/// Stateful reranker holding base vectors for repeated exact-distance candidate reranking
+template <typename DataT>
+class Reranker {
+public:
+    Reranker(FloatSpace space, const DataT* base_vectors, size_t num_base_vectors);
+
+    FloatSpace getSpace() const;                    ///< Distance metric and dimensionality
+    const DataT* getBaseVectors() const;            ///< Held base vectors
+    size_t getNumBaseVectors() const;               ///< Number of held base vectors
+    int32_t getPo() const;                          ///< Rerank feature prefetch offset (0 = prefetch disabled)
+    int32_t getPl() const;                          ///< Cache lines prefetched per feature vector (0 = whole vector)
+    void setPo(int32_t po);                         ///< Set the prefetch offset; values <= 0 disable prefetching
+    void setPl(int32_t pl);                         ///< Set the prefetch length; values <= 0 prefetch the whole vector
+    void setPrefetch(int32_t po, int32_t pl);       ///< Set both prefetch parameters at once
+
+    /// Rerank candidate indices for one query, returning the top-k as a ResultSet (max-heap)
+    ResultSet rerank(span<const byte> query, span<const uint32_t> candidate_indices, uint32_t k) const;
+
+    /// Multi-threaded batch reranking of candidate indices for many queries
+    vector<ResultSet> rerank(
+        const void* queries, size_t num_queries, const uint32_t* base_candidates,
+        size_t candidates_per_query, size_t k_top = 0, size_t num_threads = 0
+    ) const;
+
+    /// Auto-tune rerank prefetch parameters (po, pl) by timing reranking over sampled base vectors
+    void optimize(size_t sample_count = 50, size_t num_candidates = 100, uint32_t k = 10);
+};
+
 } // namespace deglib::search
 ```
 
@@ -512,6 +546,14 @@ public:
 
     ResultSet search(span<float> query, uint32_t k, float eps = 0.0f, Filter* filter = nullptr, uint32_t max_dc = 0);
     ResultSet explore(uint32_t entry_index, uint32_t k, uint32_t max_dc = 0, float eps = 0.0f, bool include_entry = true, Filter* filter = nullptr);
+
+    // --- Traversal Prefetch Tuning ---
+    int32_t getPo() const;                               ///< Feature prefetch offset (candidates ahead)
+    int32_t getPl() const;                               ///< Feature prefetch look-ahead (cache lines)
+    int32_t getNl() const;                               ///< Neighbor prefetch look-ahead
+    void setPrefetch(int32_t po, int32_t pl, int32_t nl = 3);
+    /// Auto-tune (po, pl, nl) by timing traversal over sampled vertices (self-sampled features).
+    void optimize(size_t sample_count = 50, uint32_t k = 100, uint32_t ef = 200, uint32_t seed = 7);
 };
 
 /// Abstract base interface for mutable graphs supporting vertex & edge updates

cpp/deglib/include/deglib/graph.h

@@ -218,6 +218,20 @@ class DynamicExplorationGraph {
     void setNl(int32_t nl) { internal_graph_->setNl(nl); }
     void setPrefetch(int32_t po, int32_t pl, int32_t nl = 3) { internal_graph_->setPrefetch(po, pl, nl); }
 
+    /**
+     * Auto-tune the graph's traversal prefetch parameters (po, pl, nl) by empirically timing
+     * traversal over sampled vertices. The graph self-samples its own stored features as queries,
+     * so no external query buffer is needed. Forwards to the internal graph's optimize().
+     *
+     * @param sample_count  Number of vertices sampled as queries (capped at the graph size).
+     * @param k             Result count per traversal (the expected query operating point).
+     * @param ef            Beam width per traversal; clamped to at least k.
+     * @param seed          Random seed for query sampling.
+     */
+    void optimize(size_t sample_count = 50, uint32_t k = 100, uint32_t ef = 200, uint32_t seed = 7) const {
+        internal_graph_->optimize(sample_count, k, ef, seed);
+    }
+
     bool saveGraph(const std::string& path) const {
         const auto* mutable_graph = dynamic_cast<const deglib::graph::MutableGraph*>(internal_graph_);
         if (mutable_graph == nullptr) {

cpp/deglib/include/deglib/graph/internal_graph.h

@@ -7,6 +7,7 @@
 #include "deglib/search/result_list.h"
 
 #include <algorithm>
+#include <chrono>
 #include <array>
 #include <cstddef>
 #include <cstdint>
@@ -119,9 +120,11 @@ class InternalGraph {
 
   protected:
     std::vector<uint32_t> entry_vertex_indices_{0};
-    int32_t po_ = 8;
-    int32_t pl_ = 3;
-    int32_t nl_ = 3;
+    // Prefetch tuning knobs: performance cache state, not logical identity, hence mutable so
+    // optimize()/setPrefetch() can run through a const InternalGraph reference.
+    mutable int32_t po_ = 8;
+    mutable int32_t pl_ = 3;
+    mutable int32_t nl_ = 3;
 
   public:
     const std::vector<uint32_t>& getEntryVertexIndices() const { return entry_vertex_indices_; }
@@ -133,20 +136,83 @@ class InternalGraph {
     int32_t getPo() const noexcept { return po_; }
     int32_t getPl() const noexcept { return pl_; }
     int32_t getNl() const noexcept { return nl_; }
-    void setPo(int32_t po) noexcept {
+    void setPo(int32_t po) const noexcept {
         if (po > 0) po_ = po;
     }
-    void setPl(int32_t pl) noexcept {
+    void setPl(int32_t pl) const noexcept {
         if (pl > 0) pl_ = pl;
     }
-    void setNl(int32_t nl) noexcept {
+    void setNl(int32_t nl) const noexcept {
         if (nl > 0) nl_ = nl;
     }
-    void setPrefetch(int32_t po, int32_t pl, int32_t nl = 3) noexcept {
+    void setPrefetch(int32_t po, int32_t pl, int32_t nl = 3) const noexcept {
         if (po > 0) po_ = po;
         if (pl > 0) pl_ = pl;
         if (nl > 0) nl_ = nl;
     }
+
+    /**
+     * Auto-tunes the traversal prefetch parameters (po, pl, nl) by empirically timing graph
+     * traversal over sampled vertices. Mirrors Reranker::optimize: the graph self-samples its own
+     * stored features as queries, so no external query buffer or quantization round-trip is needed
+     * and the timing exercises the real traversal path (search_ef_intern).
+     *
+     * @param sample_count  Number of vertices sampled as queries (capped at the graph size).
+     * @param k             Result count per traversal (the expected query operating point).
+     * @param ef            Beam width per traversal; clamped to at least k.
+     * @param seed          Random seed for query sampling.
+     */
+    void optimize(size_t sample_count = 50, uint32_t k = 100, uint32_t ef = 200, uint32_t seed = 7) const {
+        const uint32_t n_vertices = size();
+        const size_t n_queries = std::min(sample_count, static_cast<size_t>(n_vertices));
+        // Below one full neighborhood the traversal does no real work; the timing would be meaningless.
+        if (n_queries == 0 || k == 0 || n_queries < getEdgesPerVertex()) return;
+        ef = std::max(ef, k);
+
+        std::vector<uint32_t> sample_idx(n_queries);
+        uint32_t rng = seed * 2654435761u + 1u;
+        for (auto& idx : sample_idx) {
+            rng ^= rng << 13;
+            rng ^= rng >> 17;
+            rng ^= rng << 5;
+            idx = rng % n_vertices;
+        }
+
+        const std::vector<int32_t> try_pos = {4, 8, 12, 16};
+        const std::vector<int32_t> try_pls = {2, 3, 4};
+        const std::vector<int32_t> try_nls = {2, 3, 4};
+
+        const std::vector<uint32_t>& entries = getEntryVertexIndices();
+        int32_t best_po = po_;
+        int32_t best_pl = pl_;
+        int32_t best_nl = nl_;
+        double best_time = std::numeric_limits<double>::max();
+
+        for (int32_t po : try_pos) {
+            for (int32_t pl : try_pls) {
+                for (int32_t nl : try_nls) {
+                    setPrefetch(po, pl, nl);
+
+                    for (size_t i = 0; i < std::min<size_t>(3, n_queries); ++i) {
+                        search_ef_intern(entries, getFeatureVector(sample_idx[i]), k, ef);
+                    }
+                    const auto t_start = std::chrono::high_resolution_clock::now();
+                    for (size_t i = 0; i < n_queries; ++i) {
+                        search_ef_intern(entries, getFeatureVector(sample_idx[i]), k, ef);
+                    }
+                    const double dur = std::chrono::duration<double, std::micro>(std::chrono::high_resolution_clock::now() - t_start).count();
+
+                    if (dur < best_time) {
+                        best_time = dur;
+                        best_po = po;
+                        best_pl = pl;
+                        best_nl = nl;
+                    }
+                }
+            }
+        }
+        setPrefetch(best_po, best_pl, best_nl);
+    }
     /**
      * Perform a search but stops when the to_vertex was found.
      */

cpp/deglib/include/deglib/search.h

@@ -34,7 +34,7 @@ class Reranker {
     deglib::distances::FloatSpace space_;
     const DataT* base_vectors_ = nullptr;
     size_t num_base_vectors_ = 0;
-    int32_t po_ = 0;  // Prefetch lookahead offset for candidate feature vectors (0 disables prefetch).
+    int32_t po_ = 8;  // Prefetch lookahead offset for candidate feature vectors (0 disables prefetch).
     int32_t pl_ = 0;  // Cache lines to prefetch per feature vector (0 = prefetch the full vector).
 
     /**
@@ -169,11 +169,17 @@ class Reranker {
     size_t getNumBaseVectors() const noexcept { return num_base_vectors_; }
     int32_t getPo() const noexcept { return po_; }
     int32_t getPl() const noexcept { return pl_; }
+    void setPo(int32_t po) noexcept { po_ = (po > 0) ? po : 0; }
+    void setPl(int32_t pl) noexcept { pl_ = (pl > 0) ? pl : 0; }
+    void setPrefetch(int32_t po, int32_t pl) noexcept {
+        po_ = (po > 0) ? po : 0;
+        pl_ = (pl > 0) ? pl : 0;
+    }
 
     /**
      * Auto-tunes the rerank prefetch parameters (po, pl) using empirical timing on sample queries.
      * The tuning reflects the real operating point: each query scores `num_candidates` candidates and
-     * reduces them to the top `k`. po_ = 0 keeps the non-prefetch path.
+     * reduces them to the top `k`. Prefetch is on by default (po = 8, pl = 0); tuning only replaces one good setting with another.
      *
      * @param sample_queries  Pointer to contiguous sample queries of dimension dim.
      * @param n_queries       Number of sample queries.

cpp/deglib/include/deglib/search/searcher.h

@@ -467,64 +467,10 @@ class SearcherImpl : public SearcherBase {
             refiner_.optimize(50, 100, k);
         }
 
-        // 3. Prefetch auto-tuning for graph traversal. Needs a graph at least one neighborhood
-        //    wide; below that the traversal does no real work and the timing is meaningless.
-        if (graph_->size() < graph_->getEdgesPerVertex()) return;
-
-        const size_t test_q_count = std::min(size_t(50), size_t(graph_->size()));
-        const uint32_t dim = graph_->getFeatureSpace().dim();
-
-        // Synthetic float queries in the searcher's input space. The graph stores features in its
-        // own (possibly quantized) feature space, so reinterpreting getFeatureVector() as float
-        // would read out of bounds and produce meaningless values; the searcher quantizes these
-        // queries before traversal, which is exactly the path being timed.
-        std::vector<float> sample_queries(test_q_count * size_t(dim));
-        uint32_t rng = seed * 2654435761u + 1u;
-        for (auto& v : sample_queries) {
-            rng ^= rng << 13;
-            rng ^= rng >> 17;
-            rng ^= rng << 5;
-            v = (static_cast<float>(rng & 0xFFFFFF) / 8388608.0f) - 1.0f;  // [-1, 1)
-        }
-
-        const std::vector<int32_t> try_pos = {4, 8, 12, 16};
-        const std::vector<int32_t> try_pls = {2, 3, 4};
-        const std::vector<int32_t> try_nls = {2, 3, 4};
-
-        int32_t best_po = graph_->getPo();
-        int32_t best_pl = graph_->getPl();
-        int32_t best_nl = graph_->getNl();
-        double best_time = std::numeric_limits<double>::max();
-        std::vector<uint32_t> dummy_out(k);
-        const uint32_t ef = std::max<uint32_t>(200, k);
-
-        for (int32_t po : try_pos) {
-            for (int32_t pl : try_pls) {
-                for (int32_t nl : try_nls) {
-                    const_cast<deglib::graph::InternalGraph*>(graph_)->setPrefetch(po, pl, nl);
-
-                    for (size_t i = 0; i < std::min(size_t(3), test_q_count); ++i) {
-                        search_single_ef_typed(sample_queries.data() + i * size_t(dim), k, ef, 1.0f, dummy_out.data(), nullptr, true);
-                    }
-
-                    auto t_start = std::chrono::high_resolution_clock::now();
-                    for (size_t i = 0; i < test_q_count; ++i) {
-                        search_single_ef_typed(sample_queries.data() + i * size_t(dim), k, ef, 1.0f, dummy_out.data(), nullptr, true);
-                    }
-                    auto t_end = std::chrono::high_resolution_clock::now();
-                    double dur = std::chrono::duration<double, std::micro>(t_end - t_start).count();
-
-                    if (dur < best_time) {
-                        best_time = dur;
-                        best_po = po;
-                        best_pl = pl;
-                        best_nl = nl;
-                    }
-                }
-            }
-        }
-
-        const_cast<deglib::graph::InternalGraph*>(graph_)->setPrefetch(best_po, best_pl, best_nl);
+        // 3. Auto-tune the graph's traversal prefetch parameters. The graph self-samples its own
+        //    stored features as queries, so no external query buffer or quantization round-trip is
+        //    needed; it guards the too-small-graph case internally.
+        graph_->optimize(50, k, std::max<uint32_t>(200, k), seed);
     }
 
     template <typename QueryT>