diff --git a/setutil.go b/setutil.go index f290d460..2e16608f 100644 --- a/setutil.go +++ b/setutil.go @@ -1,6 +1,6 @@ package roaring -func difference(set1 []uint16, set2 []uint16, buffer []uint16) int { +func localdifference(set1 []uint16, set2 []uint16, buffer []uint16) int { if len(set2) == 0 { buffer = buffer[:len(set1)] copy(buffer, set1) @@ -54,7 +54,7 @@ func difference(set1 []uint16, set2 []uint16, buffer []uint16) int { return pos } -func exclusiveUnion2by2(set1 []uint16, set2 []uint16, buffer []uint16) int { +func localexclusiveUnion2by2(set1 []uint16, set2 []uint16, buffer []uint16) int { if 0 == len(set2) { buffer = buffer[:len(set1)] copy(buffer, set1[:]) @@ -169,34 +169,6 @@ func union2by2Cardinality(set1 []uint16, set2 []uint16) int { return pos } -func intersection2by2( - set1 []uint16, - set2 []uint16, - buffer []uint16, -) int { - if len(set1)*64 < len(set2) { - return onesidedgallopingintersect2by2(set1, set2, buffer) - } else if len(set2)*64 < len(set1) { - return onesidedgallopingintersect2by2(set2, set1, buffer) - } else { - return localintersect2by2(set1, set2, buffer) - } -} - -// intersection2by2Cardinality computes the cardinality of the intersection -func intersection2by2Cardinality( - set1 []uint16, - set2 []uint16, -) int { - if len(set1)*64 < len(set2) { - return onesidedgallopingintersect2by2Cardinality(set1, set2) - } else if len(set2)*64 < len(set1) { - return onesidedgallopingintersect2by2Cardinality(set2, set1) - } else { - return localintersect2by2Cardinality(set1, set2) - } -} - // intersects2by2 computes whether the two sets intersect func intersects2by2( set1 []uint16, diff --git a/setutil_andnot_arm64.go b/setutil_andnot_arm64.go new file mode 100644 index 00000000..3bc16f06 --- /dev/null +++ b/setutil_andnot_arm64.go @@ -0,0 +1,52 @@ +//go:build arm64 && !gccgo && !appengine +// +build arm64,!gccgo,!appengine + +package roaring + +//go:noescape +func andnotKernelNEON(set1, set2, buffer []uint16, shuf *byte, spill *[8]uint16) (outLen, pos1, pos2, spilled, accMask int) + +// The kernel needs whole 8-element blocks on both sides and cap(buffer) >= len(set1). +// Below 16 its setup usually loses to the scalar path. +// Duplicate values get unspecified results; stores stay within capacity. +const neonAndnotThreshold = 16 + +func difference(set1 []uint16, set2 []uint16, buffer []uint16) int { + if len(set1) < neonAndnotThreshold || len(set2) < neonAndnotThreshold { + return localdifference(set1, set2, buffer) + } + if set1[len(set1)-1] < set2[0] || set2[len(set2)-1] < set1[0] { + buffer = buffer[:len(set1)] + copy(buffer, set1) + return len(set1) + } + buffer = buffer[:cap(buffer)] + var spill [8]uint16 + outLen, pos1, pos2, spilled, accMask := andnotKernelNEON(set1, set2, buffer, &uniqshuf[0], &spill) + if spilled != 0 { + // set1 may alias buffer: drain the retained block from this copy, never reread. + i := 0 + for i < 8 && pos2 < len(set2) { + switch { + case spill[i] < set2[pos2]: + if accMask&(1<= len(set1), out <= +// pos1 at retirement, and each whole block is loaded before it is stored. +// +// Registers: +// R0/R1 input cursors, R2 out cursor, R3/R4 full-block ends, R5 out +// base, R6 shuffle table, R19 set1 base, R8-R11 block boundary values, +// R12/R13 constants, R14-R17 scratch. +// V0/V1 input blocks, V2 accumulated match mask, V3-V7 temps. + +// Match mask of V0's lanes against all rotations of V1, result in V3. +#define MATCH8 \ + VCMEQ V1.H8, V0.H8, V3.H8 \ + VEXT $2, V1.B16, V1.B16, V4.B16 \ + VCMEQ V4.H8, V0.H8, V4.H8 \ + VEXT $4, V1.B16, V1.B16, V5.B16 \ + VCMEQ V5.H8, V0.H8, V5.H8 \ + VEXT $6, V1.B16, V1.B16, V6.B16 \ + VCMEQ V6.H8, V0.H8, V6.H8 \ + VORR V4.B16, V3.B16, V3.B16 \ + VORR V6.B16, V5.B16, V5.B16 \ + VEXT $8, V1.B16, V1.B16, V4.B16 \ + VCMEQ V4.H8, V0.H8, V4.H8 \ + VEXT $10, V1.B16, V1.B16, V6.B16 \ + VCMEQ V6.H8, V0.H8, V6.H8 \ + VORR V6.B16, V4.B16, V4.B16 \ + VEXT $12, V1.B16, V1.B16, V6.B16 \ + VCMEQ V6.H8, V0.H8, V6.H8 \ + VEXT $14, V1.B16, V1.B16, V7.B16 \ + VCMEQ V7.H8, V0.H8, V7.H8 \ + VORR V7.B16, V6.B16, V6.B16 \ + VORR V5.B16, V3.B16, V3.B16 \ + VORR V6.B16, V4.B16, V4.B16 \ + VORR V4.B16, V3.B16, V3.B16 + +// Retire V0: uniqshuf's clear-lane polarity compacts the unmatched lanes. +#define EMITBLOCK \ + VUZP1 V2.B16, V2.B16, V4.B16 \ + VMOV V4.D[0], R14 \ + AND R12, R14, R14 \ + MUL R12, R14, R16 \ + LSR $56, R16, R16 \ + MUL R13, R14, R15 \ + LSR $56, R15, R15 \ + ADD R15<<4, R6, R17 \ + VLD1 (R17), [V4.B16] \ + VTBL V4.B16, [V0.B16], V4.B16 \ + VST1 [V4.B16], (R2) \ + ADD $16, R2 \ + SUB R16<<1, R2, R2 \ + VEOR V2.B16, V2.B16, V2.B16 + +// func andnotKernelNEON(set1, set2, buffer []uint16, shuf *byte, spill *[8]uint16) (outLen, pos1, pos2, spilled, accMask int) +TEXT ·andnotKernelNEON(SB), NOSPLIT, $0-128 + MOVD set1_base+0(FP), R0 + MOVD set2_base+24(FP), R1 + MOVD buffer_base+48(FP), R2 + MOVD set1_len+8(FP), R3 + MOVD set2_len+32(FP), R4 + MOVD shuf+72(FP), R6 + + MOVD R0, R19 + MOVD R2, R5 + + AND $~7, R3, R3 + AND $~7, R4, R4 + ADD R3<<1, R0, R3 + ADD R4<<1, R1, R4 + + MOVD $0x0101010101010101, R12 + MOVD $0x0102040810204080, R13 + + VEOR V2.B16, V2.B16, V2.B16 + VLD1 (R0), [V0.H8] + VLD1 (R1), [V1.H8] + MOVHU (R0), R8 + MOVHU 14(R0), R9 + MOVHU (R1), R10 + MOVHU 14(R1), R11 + +nloop: + // Range gates use strict <: equal boundary values may match. + CMP R10, R9 + BLO nffA + CMP R8, R11 + BLO nffB + + MATCH8 + VORR V3.B16, V2.B16, V2.B16 + + CMP R11, R9 + BHI nadvB + BLO nadvA + EMITBLOCK + ADD $16, R0 + ADD $16, R1 + CMP R3, R0 + BHS ndoneNoSpill + CMP R4, R1 + BHS ndoneNoSpill + VLD1 (R0), [V0.H8] + MOVHU (R0), R8 + MOVHU 14(R0), R9 + VLD1 (R1), [V1.H8] + MOVHU (R1), R10 + MOVHU 14(R1), R11 + B nloop + +nadvA: + EMITBLOCK + ADD $16, R0 + CMP R3, R0 + BHS ntailA + VLD1 (R0), [V0.H8] + MOVHU (R0), R8 + MOVHU 14(R0), R9 + B nloop + +nadvB: + ADD $16, R1 + CMP R4, R1 + BHS ndoneSpill + VLD1 (R1), [V1.H8] + MOVHU (R1), R10 + MOVHU 14(R1), R11 + B nloop + +nffA: + // Nothing left in set2 can match blocks wholly below it: emit them whole. + EMITBLOCK +nffAskip: + ADD $16, R0 + CMP R3, R0 + BHS ndoneNoSpill + MOVHU 14(R0), R9 + CMP R10, R9 + BHS nffAload + VLD1 (R0), [V0.H8] + VST1 [V0.H8], (R2) + ADD $16, R2 + B nffAskip + +nffAload: + VLD1 (R0), [V0.H8] + MOVHU (R0), R8 + B nloop + +nffB: + ADD $16, R1 + CMP R4, R1 + BHS ndoneSpill + MOVHU 14(R1), R11 + CMP R8, R11 + BLO nffB + VLD1 (R1), [V1.H8] + MOVHU (R1), R10 + B nloop + +ndoneSpill: + MOVD spill+80(FP), R16 + VST1 [V0.H8], (R16) + VUZP1 V2.B16, V2.B16, V4.B16 + VMOV V4.D[0], R14 + AND R12, R14, R14 + MUL R13, R14, R15 + LSR $56, R15, R15 + MOVD R15, accMask+120(FP) + MOVD $1, R16 + MOVD R16, spilled+112(FP) + ADD $16, R0 + B nout + +ndoneNoSpill: + MOVD ZR, spilled+112(FP) + MOVD ZR, accMask+120(FP) + +nout: + SUB R5, R2, R2 + LSR $1, R2, R2 + MOVD R2, outLen+88(FP) + SUB R19, R0, R0 + LSR $1, R0, R0 + MOVD R0, pos1+96(FP) + MOVD set2_base+24(FP), R8 + SUB R8, R1, R1 + LSR $1, R1, R1 + MOVD R1, pos2+104(FP) + RET + +ntailA: + // set2 lanes at or below the retired set1 maximum cannot match the + // scalar tail; skipping them avoids rescanning up to seven values. + VDUP R9, V4.H8 + VUMIN V4.H8, V1.H8, V5.H8 + VCMEQ V5.H8, V1.H8, V5.H8 + VUZP1 V5.B16, V5.B16, V5.B16 + VMOV V5.D[0], R14 + AND R12, R14, R14 + MUL R12, R14, R14 + LSR $56, R14, R14 + ADD R14<<1, R1, R1 + B ndoneNoSpill diff --git a/setutil_andnot_neon_arm64_test.go b/setutil_andnot_neon_arm64_test.go new file mode 100644 index 00000000..30baa808 --- /dev/null +++ b/setutil_andnot_neon_arm64_test.go @@ -0,0 +1,334 @@ +//go:build arm64 && !gccgo && !appengine +// +build arm64,!gccgo,!appengine + +package roaring + +import ( + "math/rand" + "testing" +) + +const andnotCanary = 0xABAB + +// checkDifference verifies the exact-capacity, in-place and spare-capacity +// buffer contracts. +func checkDifference(t *testing.T, label string, a, b []uint16) { + t.Helper() + want := make([]uint16, len(a)) + wn := localdifference(a, b, want) + want = want[:wn] + + m := len(a) + backing := make([]uint16, m+16) + for i := range backing { + backing[i] = andnotCanary + } + gn := difference(a, b, backing[0:0:m]) + if gn != wn { + t.Fatalf("%s: la=%d lb=%d want len %d got %d", label, len(a), len(b), wn, gn) + } + for i := 0; i < wn; i++ { + if backing[i] != want[i] { + t.Fatalf("%s: la=%d lb=%d idx %d want %d got %d", label, len(a), len(b), i, want[i], backing[i]) + } + } + for i := m; i < m+16; i++ { + if backing[i] != andnotCanary { + t.Fatalf("%s: overstore past cap at %d (cap %d)", label, i, m) + } + } + + inplace := make([]uint16, len(a)+16) + for i := range inplace { + inplace[i] = andnotCanary + } + copy(inplace, a) + gn = difference(inplace[:len(a)], b, inplace[0:len(a):len(a)]) + if gn != wn { + t.Fatalf("%s inplace: la=%d lb=%d want len %d got %d", label, len(a), len(b), wn, gn) + } + for i := 0; i < wn; i++ { + if inplace[i] != want[i] { + t.Fatalf("%s inplace: idx %d want %d got %d", label, i, want[i], inplace[i]) + } + } + for i := len(a); i < len(a)+16; i++ { + if inplace[i] != andnotCanary { + t.Fatalf("%s inplace: overstore past cap at %d (cap %d)", label, i, len(a)) + } + } + + spare := make([]uint16, len(a)+24) + for i := range spare { + spare[i] = andnotCanary + } + copy(spare, a) + gn = difference(spare[:len(a)], b, spare[0:len(a):len(a)+8]) + if gn != wn { + t.Fatalf("%s sparecap: la=%d lb=%d want len %d got %d", label, len(a), len(b), wn, gn) + } + for i := 0; i < wn; i++ { + if spare[i] != want[i] { + t.Fatalf("%s sparecap: idx %d want %d got %d", label, i, want[i], spare[i]) + } + } + for i := len(a) + 8; i < len(a)+24; i++ { + if spare[i] != andnotCanary { + t.Fatalf("%s sparecap: overstore past cap at %d (cap %d)", label, i, len(a)+8) + } + } +} + +func checkDifferenceBoth(t *testing.T, label string, a, b []uint16) { + t.Helper() + checkDifference(t, label, a, b) + checkDifference(t, label+" swapped", b, a) +} + +func TestAndnotNEONSmallSizes(t *testing.T) { + r := rand.New(rand.NewSource(41)) + for la := 0; la <= 40; la++ { + for lb := 12; lb <= 44; lb++ { + a := andnotSortedUnique(r, la, 160) + b := andnotSortedUnique(r, lb, 160) + checkDifferenceBoth(t, "small", a, b) + } + } +} + +func TestAndnotNEONRangeDisjoint(t *testing.T) { + for _, n := range []int{64, 512} { + a := make([]uint16, n) + b := make([]uint16, n) + for i := 0; i < n; i++ { + a[i] = uint16(2 * i) + b[i] = uint16(2*i + 1) + } + checkDifferenceBoth(t, "interleaved", a, b) + + for i := 0; i < n; i++ { + a[i] = uint16(i) + b[i] = uint16(65535 - n + 1 + i) + } + checkDifferenceBoth(t, "extremes", a, b) + + for i := 0; i < n; i++ { + b[i] = uint16(n - 1 + i) + } + checkDifferenceBoth(t, "endpoint", a, b) + } +} + +// Skewed and sparse inputs exercise retained-block spill and drain handoffs. +func TestAndnotNEONSkewSpill(t *testing.T) { + r := rand.New(rand.NewSource(43)) + small := andnotSortedUnique(r, 32, 60000) + large := andnotSortedUnique(r, 2048, 60000) + checkDifferenceBoth(t, "skew", small, large) + + a := make([]uint16, 512) + for i := range a { + a[i] = uint16(i * 3) + } + var b []uint16 + for i := 5; i < 512; i += 9 { + b = append(b, uint16(i*3)) + } + checkDifferenceBoth(t, "pickoff", a, b) +} + +// Alternating disjoint runs exercise the fast-forward store/skip loop. +func TestAndnotNEONBlockRuns(t *testing.T) { + for _, run := range []int{1, 2, 4, 8} { + n := 1024 + a := make([]uint16, n) + b := make([]uint16, n) + for i := 0; i < n; i++ { + blk, off := i/(8*run), i%(8*run) + a[i] = uint16(blk*16*run + off) + b[i] = uint16(blk*16*run + 8*run + off) + } + checkDifferenceBoth(t, "blockruns", a, b) + } +} + +func TestAndnotNEONRandom(t *testing.T) { + r := rand.New(rand.NewSource(47)) + for i := 0; i < 100; i++ { + la := r.Intn(3000) + lb := r.Intn(3000) + valRange := 512 + r.Intn(65000) + a := andnotSortedUnique(r, la, valRange) + b := andnotSortedUnique(r, lb, valRange) + checkDifferenceBoth(t, "random", a, b) + } +} + +// Three-way aliasing must return empty without writing past the capacity. +func TestAndnotNEONSelfAlias(t *testing.T) { + r := rand.New(rand.NewSource(53)) + for _, n := range []int{16, 65, 512} { + a := andnotSortedUnique(r, n, 4*n) + self := make([]uint16, n+16) + for i := range self { + self[i] = andnotCanary + } + copy(self, a) + gn := difference(self[:n], self[:n], self[0:n:n]) + if gn != 0 { + t.Fatalf("self n=%d: want empty, got len %d", n, gn) + } + for i := n; i < n+16; i++ { + if self[i] != andnotCanary { + t.Fatalf("self n=%d: overstore past cap at %d", n, i) + } + } + checkDifference(t, "equal-distinct", a, append([]uint16(nil), a...)) + } + + rb := BitmapOf() + for i := 0; i < 500; i++ { + rb.Add(uint32(i * 5)) + } + rb.AndNot(rb) + if !rb.IsEmpty() { + t.Fatalf("bitmap self AndNot: want empty, got %d", rb.GetCardinality()) + } +} + +// Duplicate-bearing inputs get unspecified values but bounded stores and no panic. +func TestAndnotNEONDuplicateInputBounded(t *testing.T) { + dup := func(vals ...uint16) []uint16 { return vals } + cases := [][2][]uint16{ + {dup(1, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15), + dup(1, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34)}, + {dup(100, 100, 100, 100, 101, 101, 101, 101, 101, 101, 102, 102, 102, 102, 103, 103), + dup(100, 101, 101, 101, 101, 101, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102)}, + } + for ci, c := range cases { + for o := 0; o < 2; o++ { + a, b := c[0], c[1] + if o == 1 { + a, b = b, a + } + m := len(a) + backing := make([]uint16, m+16) + for i := range backing { + backing[i] = andnotCanary + } + gn := difference(a, b, backing[0:0:m]) + if gn < 0 || gn > m { + t.Fatalf("dup case %d/%d: out of range length %d", ci, o, gn) + } + for i := m; i < m+16; i++ { + if backing[i] != andnotCanary { + t.Fatalf("dup case %d/%d: overstore past cap at %d", ci, o, i) + } + } + inplace := make([]uint16, m+16) + for i := range inplace { + inplace[i] = andnotCanary + } + copy(inplace, a) + gn = difference(inplace[:m], b, inplace[0:m:m]) + if gn < 0 || gn > m { + t.Fatalf("dup case %d/%d inplace: out of range length %d", ci, o, gn) + } + for i := m; i < m+16; i++ { + if inplace[i] != andnotCanary { + t.Fatalf("dup case %d/%d inplace: overstore past cap at %d", ci, o, i) + } + } + } + } +} + +// Every kernel exit hands back exact cursors, and a spilled block with its +// match mask when set2 runs out first. +func TestAndnotNEONKernelExits(t *testing.T) { + seq := func(from, n int) []uint16 { + s := make([]uint16, n) + for i := range s { + s[i] = uint16(from + i) + } + return s + } + cat := func(parts ...[]uint16) []uint16 { + var s []uint16 + for _, p := range parts { + s = append(s, p...) + } + return s + } + type exit struct { + name string + a, b []uint16 + out, pos1, pos2, spilled, mask int + } + cases := []exit{ + // equal maxima retire both blocks + {"tie", seq(0, 16), seq(0, 16), 0, 16, 16, 0, 0}, + // mask carried across two set2 advances into the fast-forward retire, + // then whole-block copies and a fresh block with a clear mask + {"ffmask", cat(seq(10, 1), seq(20, 1), seq(30, 1), seq(40, 1), seq(50, 1), seq(60, 1), seq(70, 1), seq(80, 1), seq(90, 16)), + cat(seq(0, 7), seq(10, 1), seq(11, 7), seq(20, 1), seq(100, 1), seq(110, 7)), 21, 24, 17, 0, 0}, + // set1 exhausted while set2 is held: the lane equal to the retired + // maximum is consumed, the tail value above it survives + {"exiteq", seq(0, 17), cat(seq(0, 8), seq(15, 1), seq(17, 7)), 7, 16, 9, 0, 0}, + // spill after an earlier block was emitted, same-start alias included + {"spillafter", cat(seq(0, 8), seq(100, 8), seq(200, 8)), cat(seq(7, 15), seq(103, 1)), 7, 16, 16, 1, 1 << 3}, + // all-matched retire, a skipped set2 block, a later block accepted + // with two matches, exit skips exactly those two lanes + {"reloadexit", cat(seq(0, 8), seq(100, 9)), cat(seq(0, 24), seq(103, 1), seq(107, 7)), 6, 16, 26, 0, 0}, + // 65535 as the retired maximum through a copy and an equal retire + {"top", seq(65520, 16), cat(seq(65496, 8), seq(65528, 8)), 8, 16, 16, 0, 0}, + } + // both set2-exhaustion exits, through the skip and after a match, for + // every set2 tail length + a := cat(seq(100, 8), seq(200, 8)) + for tail := 0; tail <= 7; tail++ { + cases = append(cases, + exit{"ffB", a, cat(seq(0, 16), []uint16{103, 107, 200, 201, 202, 203, 204}[:tail]), 0, 8, 16, 1, 0}, + exit{"advB", a, cat(seq(0, 15), seq(103, 1), []uint16{107, 200, 201, 202, 203, 204, 205}[:tail]), 0, 8, 16, 1, 1 << 3}) + } + // every reachable skip count on the set1-exhaust exit, every tail length + for c := 1; c <= 7; c++ { + for tail := 0; tail <= 7; tail++ { + cases = append(cases, exit{"exitskip", seq(0, 16+tail), cat(seq(0, 8+c), seq(16, 8-c)), 8 - c, 16, 8 + c, 0, 0}) + } + } + for _, c := range cases { + checkDifferenceBoth(t, c.name, c.a, c.b) + buffer := make([]uint16, len(c.a)) + var spill [8]uint16 + out, pos1, pos2, spilled, mask := andnotKernelNEON(c.a, c.b, buffer, &uniqshuf[0], &spill) + if out != c.out || pos1 != c.pos1 || pos2 != c.pos2 || spilled != c.spilled || mask != c.mask { + t.Fatalf("%s: out=%d pos1=%d pos2=%d spilled=%d mask=%08b, want %d %d %d %d %08b", + c.name, out, pos1, pos2, spilled, mask, c.out, c.pos1, c.pos2, c.spilled, c.mask) + } + if spilled != 0 { + for i := 0; i < 8; i++ { + if spill[i] != c.a[pos1-8+i] { + t.Fatalf("%s: spill %v is not the retained block %v", c.name, spill, c.a[pos1-8:pos1]) + } + } + } + } +} + +func andnotSortedUnique(r *rand.Rand, n, valRange int) []uint16 { + if n > valRange { + n = valRange + } + seen := make(map[uint16]bool, n) + for len(seen) < n { + seen[uint16(r.Intn(valRange))] = true + } + out := make([]uint16, 0, n) + for v := 0; v < valRange; v++ { + if seen[uint16(v)] { + out = append(out, uint16(v)) + } + } + return out +} diff --git a/setutil_arm64.go b/setutil_arm64.go index 3e089650..e40869fc 100644 --- a/setutil_arm64.go +++ b/setutil_arm64.go @@ -3,5 +3,27 @@ package roaring +// uniqshuf[m] compacts the lanes not set in m. Initialize it before any +// package-level bitmap unions, which can run before init functions. +var uniqshuf = buildUniqshuf() + +func buildUniqshuf() (t [256 * 16]byte) { + for m := 0; m < 256; m++ { + pos := 0 + for lane := 0; lane < 8; lane++ { + if m&(1<