[MOD-17527] Widen and unsign the plain uint8 integer accumulators

src/VecSim/spaces/IP/IP.cpp

@@ -238,11 +238,15 @@ float FP16_InnerProduct(const void *pVect1, const void *pVect2, size_t dimension
 }
 
 // Return type for the inner product functions.
-// The type should be able to hold `dimension * MAX_VAL(int_elem_t) * MAX_VAL(int_elem_t)`.
-// To support dimension up to 2^16, we need the difference between the type and int_elem_t to be at
-// least 2 bytes. We assert that in the implementation.
+// The type must hold `dimension * MAX_VAL(int_elem_t) * MAX_VAL(int_elem_t)`. For uint8 that is
+// 65025 * dimension, which overflows a 32-bit int from dimension 33,026 -- the alias was previously
+// `int` for any 1-byte element, so UINT8_InnerProduct executed signed-overflow UB there.
+//
+// Signedness follows the element type, which matters for the wrappers below: UINT8_InnerProduct
+// converts to float before subtracting from 1, so an unsigned accumulator is fine there, while
+// INT8_InnerProduct computes `1 - ip` in integer arithmetic and needs a signed one.
 template <typename int_elem_t>
-using ret_t = std::conditional_t<sizeof(int_elem_t) == 1, int, long long>;
+using ret_t = std::conditional_t<std::is_unsigned_v<int_elem_t>, uint64_t, int64_t>;
 
 template <typename int_elem_t>
 static inline ret_t<int_elem_t>
@@ -273,7 +277,11 @@ float INT8_Cosine(const void *pVect1v, const void *pVect2v, size_t dimension) {
 float UINT8_InnerProduct(const void *pVect1v, const void *pVect2v, size_t dimension) {
     const auto *pVect1 = static_cast<const uint8_t *>(pVect1v);
     const auto *pVect2 = static_cast<const uint8_t *>(pVect2v);
-    return 1 - INTEGER_InnerProductImp(pVect1, pVect2, dimension);
+    // Integer subtract then a single conversion, matching INT8_InnerProduct above. The cast to a
+    // signed type is required because ret_t is unsigned for uint8: 1 - an unsigned total would
+    // wrap. The SIMD kernels do the same, so their results stay bit-identical to this one.
+    const auto ip = static_cast<int64_t>(INTEGER_InnerProductImp(pVect1, pVect2, dimension));
+    return static_cast<float>(1 - ip);
 }
 
 float UINT8_Cosine(const void *pVect1v, const void *pVect2v, size_t dimension) {

src/VecSim/spaces/IP/IP_AVX512F_BW_VL_VNNI_SQ8_SQ8.h

@@ -40,7 +40,15 @@ template <unsigned char residual> // 0..63
 float SQ8_SQ8_InnerProductImp(const void *pVec1v, const void *pVec2v, size_t dimension) {
     // Compute raw dot product using efficient UINT8 AVX512 VNNI implementation
     // UINT8_InnerProductImp uses _mm512_dpwssd_epi32 for native integer dot product
-    int dot_product = UINT8_InnerProductImp<residual>(pVec1v, pVec2v, dimension);
+    // uint32_t, matching what the helper returns. This kernel is reachable at any dimension: unlike
+    // the plain uint8 choosers, the SQ8_SQ8 choosers have no dimension guard, so the previous int
+    // narrowed and wrapped past 33,025 and the float ones lost exactness past 258.
+    //
+    // Note this calls the helper directly rather than through a uint8 chooser, so it does not get
+    // the chunked accumulation those choosers select past spaces::UINT8_CHUNK_ELEMENTS: the total
+    // here is still a single 32-bit reduce. SQ8 is capped well below that by its uint32
+    // q_sum_squares metadata slot, so the fence belongs with SQ8 index creation (#1007), not here.
+    const uint32_t dot_product = UINT8_InnerProductImp<residual>(pVec1v, pVec2v, dimension);
 
     // Get dequantization parameters and precomputed values from the end of vectors
     // Layout: [data (dim)] [min (float)] [delta (float)] [sum (float)]

src/VecSim/spaces/IP/IP_AVX512F_BW_VL_VNNI_UINT8.h

@@ -8,6 +8,18 @@
  */
 #pragma once
 #include "VecSim/spaces/space_includes.h"
+#include "VecSim/spaces/spaces.h" // spaces::UINT8_CHUNK_ELEMENTS
+
+// uint8 inner product: Imp returns the raw integer total and the wrappers convert it. The chooser
+// picks plain up to spaces::UINT8_CHUNK_ELEMENTS and chunked above it, once per index; spaces.h
+// carries the chunk-size argument.
+//
+// Imp is static and always_inline so the plain wrapper's codegen is unchanged now that Imp has
+// several callers.
+// The chunked wrapper's first chunk absorbs the residual and its length is a runtime min against
+// the dimension, because a compile-time trip count cost 8-9.5% in accumulator copies; later chunks
+// share one out-of-line copy of the kernel.
+// The inner product subtracts in integer and converts once, signed because the total is not.
 
 static inline void InnerProductStep(uint8_t *&pVect1, uint8_t *&pVect2, __m512i &sum) {
     __m512i va = _mm512_loadu_epi8(pVect1); // AVX512BW
@@ -30,9 +42,12 @@ static inline void InnerProductStep(uint8_t *&pVect1, uint8_t *&pVect2, __m512i
     // with the corresponding 32-bit integer in src, and store the packed 32-bit results in dst.
 }
 
+// always_inline, not merely inline: the chunked wrapper below calls this twice, and without the
+// attribute GCC outlines it once it has several callers, which also costs the plain wrapper its
+// inlining. Measured: the plain residual-0 wrapper went from 33 instructions to 9 plus a call.
 template <unsigned char residual> // 0..63
-static inline int UINT8_InnerProductImp(const void *pVect1v, const void *pVect2v,
-                                        size_t dimension) {
+__attribute__((always_inline)) static inline uint32_t
+UINT8_InnerProductImp(const void *pVect1v, const void *pVect2v, size_t dimension) {
     uint8_t *pVect1 = (uint8_t *)pVect1v;
     uint8_t *pVect2 = (uint8_t *)pVect2v;
 
@@ -87,19 +102,70 @@ static inline int UINT8_InnerProductImp(const void *pVect1v, const void *pVect2v
         } while (pVect1 < pEnd1);
     }
 
-    return _mm512_reduce_add_epi32(sum);
+    // Unsigned, and exact for up to spaces::UINT8_CHUNK_ELEMENTS elements.
+    return static_cast<uint32_t>(_mm512_reduce_add_epi32(sum));
 }
 
 template <unsigned char residual> // 0..63
 float UINT8_InnerProductSIMD64_AVX512F_BW_VL_VNNI(const void *pVect1v, const void *pVect2v,
                                                   size_t dimension) {
 
-    return 1 - UINT8_InnerProductImp<residual>(pVect1v, pVect2v, dimension);
+    const auto ip =
+        static_cast<int64_t>(UINT8_InnerProductImp<residual>(pVect1v, pVect2v, dimension));
+    return static_cast<float>(1 - ip);
 }
 template <unsigned char residual> // 0..63
 float UINT8_CosineSIMD64_AVX512F_BW_VL_VNNI(const void *pVect1v, const void *pVect2v,
                                             size_t dimension) {
-    float ip = UINT8_InnerProductImp<residual>(pVect1v, pVect2v, dimension);
+    float ip = static_cast<float>(UINT8_InnerProductImp<residual>(pVect1v, pVect2v, dimension));
+    const float norm_v1 = load_unaligned<float>(static_cast<const uint8_t *>(pVect1v) + dimension);
+    const float norm_v2 = load_unaligned<float>(static_cast<const uint8_t *>(pVect2v) + dimension);
+    return 1.0f - ip / (norm_v1 * norm_v2);
+}
+
+__attribute__((noinline)) static uint32_t
+UINT8_InnerProductFullChunk_AVX512F_BW_VL_VNNI(const uint8_t *pVect1, const uint8_t *pVect2,
+                                               size_t dimension) {
+    return UINT8_InnerProductImp<0>(pVect1, pVect2, dimension);
+}
+
+template <unsigned char residual> // 0..63
+static inline uint64_t UINT8_InnerProductChunkedImp(const void *pVect1v, const void *pVect2v,
+                                                    size_t dimension) {
+    const auto *pVect1 = static_cast<const uint8_t *>(pVect1v);
+    const auto *pVect2 = static_cast<const uint8_t *>(pVect2v);
+
+    constexpr size_t chunk = spaces::UINT8_CHUNK_ELEMENTS;
+    constexpr size_t first_chunk = residual + (chunk - residual) / 64 * 64;
+    const size_t first = dimension < first_chunk ? dimension : first_chunk;
+    uint64_t total = UINT8_InnerProductImp<residual>(pVect1, pVect2, first);
+    pVect1 += first;
+    pVect2 += first;
+    size_t remaining = dimension - first;
+
+    while (remaining) {
+        const size_t step = remaining < chunk ? remaining : chunk;
+        total += UINT8_InnerProductFullChunk_AVX512F_BW_VL_VNNI(pVect1, pVect2, step);
+        pVect1 += step;
+        pVect2 += step;
+        remaining -= step;
+    }
+    return total;
+}
+
+template <unsigned char residual> // 0..63
+float UINT8_InnerProductSIMD64_AVX512F_BW_VL_VNNI_Chunked(const void *pVect1v, const void *pVect2v,
+                                                          size_t dimension) {
+    const auto ip =
+        static_cast<int64_t>(UINT8_InnerProductChunkedImp<residual>(pVect1v, pVect2v, dimension));
+    return static_cast<float>(1 - ip);
+}
+
+template <unsigned char residual> // 0..63
+float UINT8_CosineSIMD64_AVX512F_BW_VL_VNNI_Chunked(const void *pVect1v, const void *pVect2v,
+                                                    size_t dimension) {
+    const float ip =
+        static_cast<float>(UINT8_InnerProductChunkedImp<residual>(pVect1v, pVect2v, dimension));
     const float norm_v1 = load_unaligned<float>(static_cast<const uint8_t *>(pVect1v) + dimension);
     const float norm_v2 = load_unaligned<float>(static_cast<const uint8_t *>(pVect2v) + dimension);
     return 1.0f - ip / (norm_v1 * norm_v2);

src/VecSim/spaces/IP/IP_NEON_DOTPROD_SQ8_SQ8.h

@@ -41,7 +41,15 @@ float SQ8_SQ8_InnerProductSIMD64_NEON_DOTPROD_IMP(const void *pVec1v, const void
                                                   size_t dimension) {
     // Compute raw dot product using efficient UINT8 DOTPROD implementation
     // UINT8_InnerProductImp uses vdotq_u32 for native uint8 dot product
-    float dot_product = UINT8_InnerProductImp<residual>(pVec1v, pVec2v, dimension);
+    // uint32_t, matching what the helper returns. This kernel is reachable at any dimension: unlike
+    // the plain uint8 choosers, the SQ8_SQ8 choosers have no dimension guard, so the previous int
+    // narrowed and wrapped past 33,025 and the float ones lost exactness past 258.
+    //
+    // Note this calls the helper directly rather than through a uint8 chooser, so it does not get
+    // the chunked accumulation those choosers select past spaces::UINT8_CHUNK_ELEMENTS: the total
+    // here is still a single 32-bit reduce. SQ8 is capped well below that by its uint32
+    // q_sum_squares metadata slot, so the fence belongs with SQ8 index creation (#1007), not here.
+    const uint32_t dot_product = UINT8_InnerProductImp<residual>(pVec1v, pVec2v, dimension);
 
     // Get dequantization parameters and precomputed values from the end of vectors
     // Layout: [data (dim)] [min (float)] [delta (float)] [sum (float)]

src/VecSim/spaces/IP/IP_NEON_DOTPROD_UINT8.h

@@ -8,8 +8,21 @@
  */
 #pragma once
 #include "VecSim/spaces/space_includes.h"
+#include "VecSim/spaces/spaces.h" // spaces::UINT8_CHUNK_ELEMENTS
 #include <arm_neon.h>
 
+// uint8 inner product: Imp returns the raw integer total and the wrappers convert it. The chooser
+// picks plain up to spaces::UINT8_CHUNK_ELEMENTS and chunked above it, once per index; spaces.h
+// carries the chunk-size argument.
+//
+// Imp is static because IP_NEON_UINT8.h defines the same name with a different body and
+// aarch64 gcc 12.3 outlines it, so shared linkage lets a NEON call site execute udot and fault
+// where asimddp is absent. always_inline keeps the plain wrapper's codegen unchanged.
+// The chunked wrapper's first chunk absorbs the residual and its length is a runtime min against
+// the dimension, because a compile-time trip count cost 8-9.5% in accumulator copies; later chunks
+// share one out-of-line copy of the kernel.
+// The inner product subtracts in integer and converts once, signed because the total is not.
+
 __attribute__((always_inline)) static inline void InnerProductOp(uint8x16_t &v1, uint8x16_t &v2,
                                                                  uint32x4_t &sum) {
     sum = vdotq_u32(sum, v1, v2);
@@ -27,7 +40,8 @@ InnerProductStep(uint8_t *&pVect1, uint8_t *&pVect2, uint32x4_t &sum) {
 }
 
 template <unsigned char residual> // 0..63
-float UINT8_InnerProductImp(const void *pVect1v, const void *pVect2v, size_t dimension) {
+__attribute__((always_inline)) static inline uint32_t
+UINT8_InnerProductImp(const void *pVect1v, const void *pVect2v, size_t dimension) {
     uint8_t *pVect1 = (uint8_t *)pVect1v;
     uint8_t *pVect2 = (uint8_t *)pVect2v;
 
@@ -97,20 +111,69 @@ float UINT8_InnerProductImp(const void *pVect1v, const void *pVect2v, size_t dim
 
     uint32x4_t total_sum = vaddq_u32(sum0, sum1);
 
-    int32_t result = vaddvq_u32(total_sum);
-
-    return static_cast<float>(result);
+    // ADDV, unsigned, and exact for up to spaces::UINT8_CHUNK_ELEMENTS elements.
+    return vaddvq_u32(total_sum);
 }
 
 template <unsigned char residual> // 0..63
 float UINT8_InnerProductSIMD16_NEON_DOTPROD(const void *pVect1v, const void *pVect2v,
                                             size_t dimension) {
-    return 1.0f - UINT8_InnerProductImp<residual>(pVect1v, pVect2v, dimension);
+    const auto ip =
+        static_cast<int64_t>(UINT8_InnerProductImp<residual>(pVect1v, pVect2v, dimension));
+    return static_cast<float>(1 - ip);
 }
 
 template <unsigned char residual> // 0..63
 float UINT8_CosineSIMD_NEON_DOTPROD(const void *pVect1v, const void *pVect2v, size_t dimension) {
-    float ip = UINT8_InnerProductImp<residual>(pVect1v, pVect2v, dimension);
+    float ip = static_cast<float>(UINT8_InnerProductImp<residual>(pVect1v, pVect2v, dimension));
+    const float norm_v1 = load_unaligned<float>(static_cast<const uint8_t *>(pVect1v) + dimension);
+    const float norm_v2 = load_unaligned<float>(static_cast<const uint8_t *>(pVect2v) + dimension);
+    return 1.0f - ip / (norm_v1 * norm_v2);
+}
+
+__attribute__((noinline)) static uint32_t
+UINT8_InnerProductFullChunk_NEON_DOTPROD(const uint8_t *pVect1, const uint8_t *pVect2,
+                                         size_t dimension) {
+    return UINT8_InnerProductImp<0>(pVect1, pVect2, dimension);
+}
+
+template <unsigned char residual> // 0..63
+static inline uint64_t UINT8_InnerProductChunkedImp(const void *pVect1v, const void *pVect2v,
+                                                    size_t dimension) {
+    const auto *pVect1 = static_cast<const uint8_t *>(pVect1v);
+    const auto *pVect2 = static_cast<const uint8_t *>(pVect2v);
+
+    constexpr size_t chunk = spaces::UINT8_CHUNK_ELEMENTS;
+    constexpr size_t first_chunk = residual + (chunk - residual) / 64 * 64;
+    const size_t first = dimension < first_chunk ? dimension : first_chunk;
+    uint64_t total = UINT8_InnerProductImp<residual>(pVect1, pVect2, first);
+    pVect1 += first;
+    pVect2 += first;
+    size_t remaining = dimension - first;
+
+    while (remaining) {
+        const size_t step = remaining < chunk ? remaining : chunk;
+        total += UINT8_InnerProductFullChunk_NEON_DOTPROD(pVect1, pVect2, step);
+        pVect1 += step;
+        pVect2 += step;
+        remaining -= step;
+    }
+    return total;
+}
+
+template <unsigned char residual> // 0..63
+float UINT8_InnerProductSIMD16_NEON_DOTPROD_Chunked(const void *pVect1v, const void *pVect2v,
+                                                    size_t dimension) {
+    const auto ip =
+        static_cast<int64_t>(UINT8_InnerProductChunkedImp<residual>(pVect1v, pVect2v, dimension));
+    return static_cast<float>(1 - ip);
+}
+
+template <unsigned char residual> // 0..63
+float UINT8_CosineSIMD_NEON_DOTPROD_Chunked(const void *pVect1v, const void *pVect2v,
+                                            size_t dimension) {
+    float ip =
+        static_cast<float>(UINT8_InnerProductChunkedImp<residual>(pVect1v, pVect2v, dimension));
     const float norm_v1 = load_unaligned<float>(static_cast<const uint8_t *>(pVect1v) + dimension);
     const float norm_v2 = load_unaligned<float>(static_cast<const uint8_t *>(pVect2v) + dimension);
     return 1.0f - ip / (norm_v1 * norm_v2);

src/VecSim/spaces/IP/IP_NEON_SQ8_SQ8.h

@@ -41,7 +41,15 @@ float SQ8_SQ8_InnerProductSIMD64_NEON_IMP(const void *pVec1v, const void *pVec2v
                                           size_t dimension) {
     // Compute raw dot product using efficient UINT8 implementation
     // UINT8_InnerProductImp processes 16 elements at a time using native uint8 instructions
-    float dot_product = UINT8_InnerProductImp<residual>(pVec1v, pVec2v, dimension);
+    // uint32_t, matching what the helper returns. This kernel is reachable at any dimension: unlike
+    // the plain uint8 choosers, the SQ8_SQ8 choosers have no dimension guard, so the previous int
+    // narrowed and wrapped past 33,025 and the float ones lost exactness past 258.
+    //
+    // Note this calls the helper directly rather than through a uint8 chooser, so it does not get
+    // the chunked accumulation those choosers select past spaces::UINT8_CHUNK_ELEMENTS: the total
+    // here is still a single 32-bit reduce. SQ8 is capped well below that by its uint32
+    // q_sum_squares metadata slot, so the fence belongs with SQ8 index creation (#1007), not here.
+    const uint32_t dot_product = UINT8_InnerProductImp<residual>(pVec1v, pVec2v, dimension);
 
     // Get dequantization parameters and precomputed values from the end of pVec1
     // Layout: [data (dim)] [min (float)] [delta (float)] [sum (float)]