diff --git a/setutil_arm64.go b/setutil_arm64.go index 3e089650..57007fc8 100644 --- a/setutil_arm64.go +++ b/setutil_arm64.go @@ -3,5 +3,47 @@ package roaring +// uniqshuf[m] compacts the lanes not set in m. Initialize it before any +// package-level bitmap unions, which can run before init functions. +var uniqshuf = buildUniqshuf() + +func buildUniqshuf() (t [256 * 16]byte) { + for m := 0; m < 256; m++ { + pos := 0 + for lane := 0; lane < 8; lane++ { + if m&(1< compare -> count -> advance. +// +// Every iteration consumes all values <= x on both sides, so no duplicate +// crosses an iteration boundary. The first merged vector holds at most two +// copies of any value, hence comparing its first lane with its last lane +// cannot drop the first output. +// +// Below 64 elements on either side, or when the 16-lane loop reaches its +// reserve, an 8-lane partition stage continues with the same invariant. +// It halves the merge and compaction work and leaves a shorter scalar tail. +// +// Register use: +// R0/R1 byte cursors into set1/set2, R3/R4 their loop end addresses, +// R2 output cursor, R5 output base, R6 uniqshuf table, +// R12 = 0x0101010101010101, R13 = 0x0102040810204080, R20 = 8. +// V0/V1 A window, V2/V3 B window, V4 am, V5 bm, V6 x, +// V16-V19 merged 32, V20-V23 compaction shuffles. + +// Sort two 8-element bitonic sequences u,v (as produced by a compare-exchange +// pair) into lo,hi. Transposed network: one TRN pair per distance, both +// halves cleaned at once, ZIP to untranspose. Clobbers V14, V15, u, v. +#define CLEAN(u, v, lo, hi) \ + VTRN1 v.D2, u.D2, V14.D2 \ + VTRN2 v.D2, u.D2, V15.D2 \ + VUMIN V15.H8, V14.H8, u.H8 \ + VUMAX V15.H8, V14.H8, v.H8 \ + VTRN1 v.S4, u.S4, V14.S4 \ + VTRN2 v.S4, u.S4, V15.S4 \ + VUMIN V15.H8, V14.H8, u.H8 \ + VUMAX V15.H8, V14.H8, v.H8 \ + VTRN1 v.H8, u.H8, V14.H8 \ + VTRN2 v.H8, u.H8, V15.H8 \ + VUMIN V15.H8, V14.H8, u.H8 \ + VUMAX V15.H8, V14.H8, v.H8 \ + VZIP1 v.H8, u.H8, lo.H8 \ + VZIP2 v.H8, u.H8, hi.H8 + +// Merge sorted (V0,V1) with sorted (V2,V3) into sorted V16..V19. +// Reversing B makes the concatenation bitonic; two compare-exchange stages +// split it into two bitonic 16-sequences, each cleaned by CLEAN. +#define MERGE32 \ + VREV64 V3.H8, V8.H8 \ + VEXT $8, V8.B16, V8.B16, V8.B16 \ + VREV64 V2.H8, V9.H8 \ + VEXT $8, V9.B16, V9.B16, V9.B16 \ + VUMIN V8.H8, V0.H8, V10.H8 \ + VUMAX V8.H8, V0.H8, V12.H8 \ + VUMIN V9.H8, V1.H8, V11.H8 \ + VUMAX V9.H8, V1.H8, V13.H8 \ + VUMIN V11.H8, V10.H8, V8.H8 \ + VUMAX V11.H8, V10.H8, V9.H8 \ + VUMIN V13.H8, V12.H8, V10.H8 \ + VUMAX V13.H8, V12.H8, V11.H8 \ + CLEAN(V8, V9, V16, V17) \ + CLEAN(V10, V11, V18, V19) + +// Count a 16-lane prefix mask in t0,t1, then advance byte cursor p. +// UZP1 keeps one byte per predicate; SHRN #4 keeps four bits per +// predicate in a single doubleword. Thus the byte advance is 32-clz(mask)/2. +// nt0 is t0's register number for SHRN, which Go's assembler does not expose. +// Clobbers t0, t1, Ra, Rc. +#define COUNTADV(t0, t1, Ra, Rc, p, nt0) \ + VUZP1 t1.B16, t0.B16, t0.B16 \ + WORD $(0x0f0c8400 | (nt0 << 5) | nt0) \ + VMOV t0.D[0], Ra \ + CLZ Ra, Ra \ + ADD $32, p, Rc \ + SUB Ra>>1, Rc, p + +// Eight predicates fit in a doubleword after keeping one byte per lane. +#define COUNTADV8(t, Ra, Rc, p) \ + VUZP1 t.B16, t.B16, t.B16 \ + VMOV t.D[0], Ra \ + CLZ Ra, Ra \ + ADD $16, p, Rc \ + SUB Ra>>2, Rc, p + +// Build the compaction shuffle sv and kept-lane count Rcnt for cur, whose +// predecessor lane is lane 7 of prv. Clobbers t, R17, R19, R21. +#define PREP(cur, prv, t, sv, Rcnt) \ + VEXT $14, cur.B16, prv.B16, t.B16 \ + VCMEQ t.H8, cur.H8, t.H8 \ + VUZP1 t.B16, t.B16, t.B16 \ + VMOV t.D[0], R17 \ + AND R12, R17, R17 \ + MUL R13, R17, R19 \ + LSR $56, R19, R19 \ + MUL R12, R17, R17 \ + LSR $56, R17, R17 \ + SUB R17, R20, Rcnt \ + ADD R19<<4, R6, R21 \ + VLD1 (R21), [sv.B16] + +// func unionPartKernelNEON(set1, set2, buffer []uint16, shuf *byte) (outLen, pos1, pos2 int) +TEXT ·unionPartKernelNEON(SB), NOSPLIT, $0-104 + MOVD set1_base+0(FP), R0 + MOVD set2_base+24(FP), R1 + MOVD buffer_base+48(FP), R2 + MOVD set1_len+8(FP), R3 + MOVD set2_len+32(FP), R4 + MOVD shuf+72(FP), R6 + MOVD R2, R5 + MOVD $0x0101010101010101, R12 + MOVD $0x0102040810204080, R13 + MOVD $8, R20 + CMP $64, R3 + BLT setup8 + CMP $64, R4 + BLT setup8 + + // Stop 24 elements short of each whole-block end. Each iteration leaves + // at least 9 elements per side unread; compaction stores extend at most + // 8 elements past the output cursor. This also protects unread set1 + // when iorArray places it at buffer[len(set2):]. + AND $~7, R3, R3 + SUB $24, R3, R3 + ADD R3<<1, R0, R3 + AND $~7, R4, R4 + SUB $24, R4, R4 + ADD R4<<1, R1, R4 + +loop: + CMP R3, R0 + BHS setup8 + CMP R4, R1 + BHS setup8 + + // Non-overlapping windows: the lower one is the union up to its own + // maximum, so it is emitted as loaded and the merge, the counts and the + // compaction are all skipped. Scalar endpoint loads resolve the branch + // without waiting on the vector loads. Boundary equality falls through + // to the general path, which dedups it. + MOVHU 30(R0), R8 + MOVHU (R1), R9 + MOVHU 30(R1), R10 + MOVHU (R0), R11 + CMP R9, R8 + BLO fast1 + CMP R11, R10 + BLO fast2 + + // Half-window retry: ownership runs shorter than the window never let + // the tests above fire, so they are repeated at 8 elements. + MOVHU 14(R0), R16 + CMP R9, R16 + BLO fast3 + MOVHU 14(R1), R17 + CMP R11, R17 + BLO fast4 + + VLD1 (R0), [V0.H8, V1.H8] + VLD1 (R1), [V2.H8, V3.H8] + VDUP V1.H[7], V4.H8 + VDUP V3.H[7], V5.H8 + + WORD $0x6e603ca8 // CMHS V8.8H, V5.8H, V0.8H + WORD $0x6e613ca9 // CMHS V9.8H, V5.8H, V1.8H + COUNTADV(V8, V9, R8, R10, R0, 8) + WORD $0x6e623c8a // CMHS V10.8H, V4.8H, V2.8H + WORD $0x6e633c8b // CMHS V11.8H, V4.8H, V3.8H + COUNTADV(V10, V11, R14, R16, R1, 10) + + VUMIN V5.H8, V4.H8, V6.H8 + + MERGE32 + + // Lanes above x are outside this iteration's emission. Clamping them to + // x makes each a repeat of its predecessor, so the dedup compare drops + // them. Only the last two vectors can hold such lanes (T >= 16). + VUMIN V6.H8, V18.H8, V18.H8 + VUMIN V6.H8, V19.H8, V19.H8 + + PREP(V16, V16, V8, V20, R22) + PREP(V17, V16, V9, V21, R23) + PREP(V18, V17, V10, V22, R24) + PREP(V19, V18, V11, V23, R25) + + VTBL V20.B16, [V16.B16], V12.B16 + VST1 [V12.B16], (R2) + ADD R22<<1, R2, R2 + VTBL V21.B16, [V17.B16], V13.B16 + VST1 [V13.B16], (R2) + ADD R23<<1, R2, R2 + VTBL V22.B16, [V18.B16], V14.B16 + VST1 [V14.B16], (R2) + ADD R24<<1, R2, R2 + VTBL V23.B16, [V19.B16], V15.B16 + VST1 [V15.B16], (R2) + ADD R25<<1, R2, R2 + + B loop + +fast1: + VLD1 (R0), [V0.H8, V1.H8] + VST1 [V0.H8, V1.H8], (R2) + ADD $32, R0, R0 + ADD $32, R2, R2 + B loop + +fast2: + VLD1 (R1), [V2.H8, V3.H8] + VST1 [V2.H8, V3.H8], (R2) + ADD $32, R1, R1 + ADD $32, R2, R2 + B loop + +fast3: + VLD1 (R0), [V0.H8] + VST1 [V0.H8], (R2) + ADD $16, R0, R0 + ADD $16, R2, R2 + B loop + +fast4: + VLD1 (R1), [V2.H8] + VST1 [V2.H8], (R2) + ADD $16, R1, R1 + ADD $16, R2, R2 + B loop + +setup8: + // Two full compaction stores need 16 unread elements per input to + // preserve the relocated-input alias. Cursors need not be aligned. + MOVD set1_len+8(FP), R3 + SUB $16, R3, R3 + MOVD set1_base+0(FP), R8 + ADD R3<<1, R8, R3 + MOVD set2_len+32(FP), R4 + SUB $16, R4, R4 + MOVD set2_base+24(FP), R8 + ADD R4<<1, R8, R4 + +loop8: + CMP R3, R0 + BHI done + CMP R4, R1 + BHI done + + MOVHU 14(R0), R8 + MOVHU (R1), R9 + CMP R9, R8 + BLO fastA8 + MOVHU 14(R1), R10 + MOVHU (R0), R11 + CMP R11, R10 + BLO fastB8 + + VLD1 (R0), [V0.H8] + VLD1 (R1), [V1.H8] + VDUP V0.H[7], V4.H8 + VDUP V1.H[7], V5.H8 + WORD $0x6e603ca8 // CMHS V8.8H, V5.8H, V0.8H + COUNTADV8(V8, R8, R10, R0) + WORD $0x6e613c89 // CMHS V9.8H, V4.8H, V1.8H + COUNTADV8(V9, R14, R16, R1) + VUMIN V5.H8, V4.H8, V6.H8 + + VREV64 V1.H8, V8.H8 + VEXT $8, V8.B16, V8.B16, V8.B16 + VUMIN V8.H8, V0.H8, V9.H8 + VUMAX V8.H8, V0.H8, V10.H8 + CLEAN(V9, V10, V16, V17) + VUMIN V6.H8, V17.H8, V17.H8 + + PREP(V16, V16, V8, V20, R22) + PREP(V17, V16, V9, V21, R23) + VTBL V20.B16, [V16.B16], V12.B16 + VST1 [V12.B16], (R2) + ADD R22<<1, R2, R2 + VTBL V21.B16, [V17.B16], V13.B16 + VST1 [V13.B16], (R2) + ADD R23<<1, R2, R2 + B loop8 + +fastA8: + VLD1 (R0), [V0.H8] + VST1 [V0.H8], (R2) + ADD $16, R0, R0 + ADD $16, R2, R2 + B loop8 + +fastB8: + VLD1 (R1), [V1.H8] + VST1 [V1.H8], (R2) + ADD $16, R1, R1 + ADD $16, R2, R2 + B loop8 + +done: + SUB R5, R2, R2 + LSR $1, R2, R2 + MOVD R2, outLen+80(FP) + + MOVD set1_base+0(FP), R8 + SUB R8, R0, R0 + LSR $1, R0, R0 + MOVD R0, pos1+88(FP) + MOVD set2_base+24(FP), R8 + SUB R8, R1, R1 + LSR $1, R1, R1 + MOVD R1, pos2+96(FP) + RET diff --git a/setutil_union_arm64_test.go b/setutil_union_arm64_test.go new file mode 100644 index 00000000..351b1a76 --- /dev/null +++ b/setutil_union_arm64_test.go @@ -0,0 +1,305 @@ +//go:build arm64 && !gccgo && !appengine +// +build arm64,!gccgo,!appengine + +package roaring + +import ( + "fmt" + "math/rand" + "slices" + "testing" +) + +func unionSortedUnique(values []uint16) []uint16 { + values = slices.Clone(values) + slices.Sort(values) + return slices.Compact(values) +} + +func unionReference(a, b []uint16) []uint16 { + return unionSortedUnique(append(slices.Clone(a), b...)) +} + +func unionTestPair(shape string, na, nb int, seed int64) (a, b []uint16) { + r := rand.New(rand.NewSource(seed)) + random := func(n, limit int) []uint16 { + values := make(map[uint16]struct{}, n) + for len(values) < n { + values[uint16(r.Intn(limit))] = struct{}{} + } + out := make([]uint16, 0, n) + for value := range values { + out = append(out, value) + } + slices.Sort(out) + return out + } + switch shape { + case "sparse": + return random(na, 65536), random(nb, 65536) + case "dense": + return random(na, 3*(na+nb)/2+1), random(nb, 3*(na+nb)/2+1) + case "identical": + pool := random(max(na, nb), 65536) + return slices.Clone(pool[:na]), slices.Clone(pool[:nb]) + case "sharedprefix": + pool := random(max(na, nb)+min(na, nb)/8+1, 65536) + return slices.Clone(pool[:na]), slices.Clone(pool[len(pool)-nb:]) + case "interleaved", "runs": + run := 1 + if shape == "runs" { + run = 4 + r.Intn(29) + } + value := r.Intn(1024) + for len(a) < na || len(b) < nb { + for i := 0; i < run && len(a) < na; i++ { + a = append(a, uint16(value)) + value++ + } + for i := 0; i < run && len(b) < nb; i++ { + b = append(b, uint16(value)) + value++ + } + } + return a, b + default: + panic("unknown union test shape") + } +} + +func checkUnionBuffers(t *testing.T, a, b []uint16) { + t.Helper() + want := unionReference(a, b) + originalA, originalB := slices.Clone(a), slices.Clone(b) + for _, relocated := range []bool{false, true} { + const guard = 8 + need := len(a) + len(b) + backing := make([]uint16, need+2*guard) + for i := range backing { + backing[i] = 0xdead + } + out := backing[guard : guard+need : guard+need] + input := a + if relocated { + copy(out[len(b):], a) + input = out[len(b):] + } + n := union2by2(input, b, out[:0]) + if n < 0 || n > need || !slices.Equal(out[:n], want) { + t.Fatalf("%dx%d relocated=%t: got cardinality %d, want %d or wrong elements", len(a), len(b), relocated, n, len(want)) + } + for i := 0; i < guard; i++ { + if backing[i] != 0xdead || backing[guard+need+i] != 0xdead { + t.Fatalf("%dx%d relocated=%t: output overwrote canary", len(a), len(b), relocated) + } + } + if !slices.Equal(a, originalA) || !slices.Equal(b, originalB) { + t.Fatal("union modified a non-output input") + } + } +} + +func TestUnion2By2Boundaries(t *testing.T) { + pairs := [][2]int{ + {0, 0}, {0, 64}, {64, 0}, {1, 128}, {15, 128}, {16, 128}, + {31, 4096}, {32, 4096}, {4096, 32}, {32, 95}, {32, 96}, {32, 97}, + {95, 32}, {96, 32}, {97, 32}, {48, 79}, {48, 80}, {48, 81}, + {63, 63}, {63, 64}, {64, 63}, {64, 64}, {64, 65}, {65, 64}, + {79, 80}, {95, 96}, {127, 128}, {255, 256}, {4096, 4096}, + } + for _, shape := range []string{"sparse", "dense", "identical", "sharedprefix", "interleaved", "runs"} { + for _, pair := range pairs { + t.Run(fmt.Sprintf("%s/%dx%d", shape, pair[0], pair[1]), func(t *testing.T) { + a, b := unionTestPair(shape, pair[0], pair[1], 42) + checkUnionBuffers(t, a, b) + if len(a) > 0 && len(b) > 0 { + a[len(a)-1], b[len(b)-1] = 0xffff, 0xffff + checkUnionBuffers(t, a, b) + } + }) + } + } +} + +func TestUnionArrayContainerPaths(t *testing.T) { + for _, pair := range [][2]int{{32, 95}, {32, 96}, {48, 80}, {63, 64}, {64, 64}, {65, 255}} { + for _, shape := range []string{"sparse", "sharedprefix", "runs"} { + a, b := unionTestPair(shape, pair[0], pair[1], 73) + want := unionReference(a, b) + fresh := func(values []uint16, capacity int) *arrayContainer { + return &arrayContainer{content: append(make([]uint16, 0, capacity), values...)} + } + for _, capacity := range []int{len(a), 2 * (len(a) + len(b))} { + left, right := fresh(a, capacity), fresh(b, len(b)) + for _, result := range []container{left.orArray(right), left.lazyorArray(right), left.iorArray(right)} { + if !slices.Equal(result.(*arrayContainer).content, want) { + t.Fatalf("%s/%dx%d capacity=%d: incorrect container union", shape, len(a), len(b), capacity) + } + } + if !slices.Equal(right.content, b) { + t.Fatal("union modified the other container") + } + self := fresh(a, capacity) + if !slices.Equal(self.iorArray(self).(*arrayContainer).content, a) { + t.Fatal("incorrect self union") + } + } + } + } +} + +func TestUnionPartitionPrefixes(t *testing.T) { + for _, size := range []int{32, 64, 127, 4096} { + for _, shape := range []string{"sparse", "identical", "interleaved", "sharedprefix"} { + a, b := unionTestPair(shape, size, size, 53) + backing := make([]uint16, len(a)+len(b)+2) + backing[0], backing[len(backing)-1] = 0xdead, 0xdead + out := backing[1 : len(backing)-1 : len(backing)-1] + n, posA, posB := unionPartKernelNEON(a, b, out, &uniqshuf[0]) + if n <= 0 || n > len(out) || posA < 0 || posA > len(a) || posB < 0 || posB > len(b) { + t.Fatalf("%s/%d: invalid kernel accounting: %d, %d, %d", shape, size, n, posA, posB) + } + if !slices.Equal(out[:n], unionReference(a[:posA], b[:posB])) { + t.Fatalf("%s/%d: output differs from consumed prefixes", shape, size) + } + if posA < len(a) && a[posA] <= out[n-1] || posB < len(b) && b[posB] <= out[n-1] { + t.Fatalf("%s/%d: unconsumed input does not follow output", shape, size) + } + if backing[0] != 0xdead || backing[len(backing)-1] != 0xdead { + t.Fatal("kernel overwrote output canary") + } + } + } +} + +func TestUnionWindowTies(t *testing.T) { + for _, run := range []int{8, 16} { + var a, b []uint16 + value := 0 + for window := 0; window < 16; window++ { + for i := 0; i < run; i++ { + a = append(a, uint16(value)) + value++ + } + value-- // Equality at a window boundary must reach deduplication. + for i := 0; i < run; i++ { + b = append(b, uint16(value)) + value++ + } + value-- + } + checkUnionBuffers(t, a, b) + checkUnionBuffers(t, b, a) + checkUnionBuffers(t, a[:48], b[:80]) + } +} + +func TestUnionMixedWindowOwnership(t *testing.T) { + r := rand.New(rand.NewSource(8080)) + for trial := 0; trial < 64; trial++ { + var a, b []uint16 + for value := 0; value < 2048; { + run, owner := 1+r.Intn(40), r.Intn(2) + for i := 0; i < run; i++ { + shared := r.Intn(8) == 0 + if owner == 0 || shared { + a = append(a, uint16(value)) + } + if owner == 1 || shared { + b = append(b, uint16(value)) + } + value++ + } + } + checkUnionBuffers(t, a, b) + } +} + +func TestUnionBitmapCopyOnWrite(t *testing.T) { + a, b := unionTestPair("sparse", 64, 96, 91) + left, right, want := NewBitmap(), NewBitmap(), NewBitmap() + for _, value := range a { + left.Add(uint32(value)) + want.Add(uint32(value)) + } + for _, value := range b { + right.Add(uint32(value)) + want.Add(uint32(value)) + } + originalLeft, originalRight := left.Clone(), right.Clone() + for _, leftCOW := range []bool{false, true} { + for _, rightCOW := range []bool{false, true} { + left.SetCopyOnWrite(leftCOW) + right.SetCopyOnWrite(rightCOW) + copyLeft, copyRight := left.Clone(), right.Clone() + result := Or(copyLeft, copyRight) + copyLeft.Or(copyRight) + if !result.Equals(want) || !copyLeft.Equals(want) || !FastOr(left, right).Equals(want) { + t.Fatal("incorrect bitmap union") + } + if !left.Equals(originalLeft) || !right.Equals(originalRight) || !copyRight.Equals(originalRight) { + t.Fatal("union modified a shared input") + } + } + } +} + +func FuzzUnion2By2(f *testing.F) { + for _, pair := range [][2]int{{0, 0}, {32, 96}, {64, 64}, {128, 255}, {4096, 32}} { + a, b := unionTestPair("dense", pair[0], pair[1], 123) + encode := func(values []uint16) []byte { + out := make([]byte, 2*len(values)) + for i, value := range values { + out[2*i], out[2*i+1] = byte(value), byte(value>>8) + } + return out + } + f.Add(encode(a), encode(b)) + } + f.Fuzz(func(t *testing.T, aBytes, bBytes []byte) { + decode := func(data []byte) []uint16 { + data = data[:min(len(data), 8192)] + values := make([]uint16, len(data)/2) + for i := range values { + values[i] = uint16(data[2*i]) | uint16(data[2*i+1])<<8 + } + return unionSortedUnique(values) + } + checkUnionBuffers(t, decode(aBytes), decode(bBytes)) + }) +} + +var unionBenchmarkSize int + +func BenchmarkUnion2By2(b *testing.B) { + for _, shape := range []string{"sparse", "dense", "interleaved", "runs", "identical", "sharedprefix"} { + for _, size := range [][2]int{{32, 32}, {64, 64}, {256, 256}, {4096, 4096}, {32, 96}} { + b.Run(fmt.Sprintf("%s/%dx%d", shape, size[0], size[1]), func(b *testing.B) { + var inputs [64][2][]uint16 + for i := range inputs { + inputs[i][0], inputs[i][1] = unionTestPair(shape, size[0], size[1], int64(i+100)) + for side, values := range inputs[i] { + if len(values) != size[side] || !slices.Equal(values, unionSortedUnique(values)) { + b.Fatal("benchmark generator returned wrong length or non-unique sorted input") + } + } + } + out := make([]uint16, size[0]+size[1]) + b.Run("dispatch", func(b *testing.B) { + b.ReportAllocs() + for i := 0; i < b.N; i++ { + pair := inputs[i&63] + unionBenchmarkSize = union2by2(pair[0], pair[1], out) + } + }) + b.Run("scalar", func(b *testing.B) { + b.ReportAllocs() + for i := 0; i < b.N; i++ { + pair := inputs[i&63] + unionBenchmarkSize = union2by2scalar(pair[0], pair[1], out) + } + }) + }) + } + } +} diff --git a/union_real_data_benchmark_test.go b/union_real_data_benchmark_test.go new file mode 100644 index 00000000..ed4acf13 --- /dev/null +++ b/union_real_data_benchmark_test.go @@ -0,0 +1,95 @@ +package roaring + +import "testing" + +var unionRealDataBitmap *Bitmap +var unionRealDataCardinality uint64 + +// BENCH_REAL_DATA=1 go test -run '^$' -bench '^BenchmarkUnionRealData$' -benchmem +// GOPATH must contain src/github.com/RoaringBitmap/real-roaring-datasets. +func BenchmarkUnionRealData(b *testing.B) { + if !benchRealData { + b.Skip("set BENCH_REAL_DATA=1 and GOPATH to enable the real-data benchmarks") + } + for _, dataset := range realDatasets { + for _, optimize := range []bool{false, true} { + mode := "raw" + if optimize { + mode = "optimized" + } + b.Run(dataset+"/"+mode, func(b *testing.B) { + bitmaps, err := retrieveRealDataBitmaps(dataset, optimize) + if err != nil { + b.Fatal(err) + } + if len(bitmaps) < 2 { + b.Fatal("dataset needs at least two bitmaps") + } + wantAll := NewBitmap() + for _, bitmap := range bitmaps { + iterator := bitmap.Iterator() + for iterator.HasNext() { + wantAll.Add(iterator.Next()) + } + } + var wantPairs uint64 + for i := 0; i+1 < len(bitmaps); i++ { + want := bitmaps[i].Clone() + iterator := bitmaps[i+1].Iterator() + for iterator.HasNext() { + want.Add(iterator.Next()) + } + if !Or(bitmaps[i], bitmaps[i+1]).Equals(want) { + b.Fatalf("incorrect pairwise union at pair %d", i) + } + wantPairs += want.GetCardinality() + } + b.Run("pairwise", func(b *testing.B) { + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + var cardinality uint64 + for j := 0; j+1 < len(bitmaps); j++ { + unionRealDataBitmap = Or(bitmaps[j], bitmaps[j+1]) + cardinality += unionRealDataBitmap.GetCardinality() + } + unionRealDataCardinality = cardinality + } + b.StopTimer() + if unionRealDataCardinality != wantPairs { + b.Fatal("pairwise union cardinality changed during benchmark") + } + b.ReportMetric(float64(len(bitmaps)-1), "unions/op") + }) + b.Run("accumulated", func(b *testing.B) { + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + result := NewBitmap() + for _, bitmap := range bitmaps { + result.Or(bitmap) + } + unionRealDataBitmap = result + unionRealDataCardinality = result.GetCardinality() + } + b.StopTimer() + if !unionRealDataBitmap.Equals(wantAll) { + b.Fatal("incorrect accumulated union") + } + }) + b.Run("fastor", func(b *testing.B) { + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + unionRealDataBitmap = FastOr(bitmaps...) + unionRealDataCardinality = unionRealDataBitmap.GetCardinality() + } + b.StopTimer() + if !unionRealDataBitmap.Equals(wantAll) { + b.Fatal("incorrect FastOr union") + } + }) + }) + } + } +}