From 3fe980c9eb64f014f3e0d8a9761da3460765fc8b Mon Sep 17 00:00:00 2001 From: tamirms Date: Tue, 15 Sep 2026 06:44:24 +0100 Subject: [PATCH 1/2] runcontainer: count or test before allocating in run-bitmap intersections MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit runContainer16.andBitmapContainer materialized the run as an 8 KiB bitmap container and then intersected, so an empty or array-sized result paid for a bitmap it never returned. It now counts the result with the existing andBitmapContainerCardinality and builds only that: an empty array container when the intersection is empty, an array of the right size when it is small, and the bitmap only when the result is one. A run with more than 64 intervals is ORed into a scratch bitmap on the stack, counted there, and extracted in one pass instead. bitmapContainer.iandRun16 did the same conversion on the argument side. It now counts the same way, returns a small result as an array built from the run's intervals, and otherwise clears the gaps between intervals in place; past 64 intervals the run is built in stack scratch and ANDed in one flat pass, since clearing gap by gap costs a call per interval. runContainer16.intersects computed rc.and(a) and tested the result for emptiness, allocating a container to answer a boolean; every mixed pairing with a run reached it through the array and bitmap dispatchers. It now walks the run's intervals against the other container, a masked word scan for bitmaps, a galloping merge for arrays or a probe per value when the array is much smaller than the interval list, an interval overlap test for runs, each returning at the first shared value. BenchmarkRunAndBitmap (added): And, in-place And and Intersects between a run-optimized bitmap and a dense one over a single key. Xeon 8375C: before after And, empty result 3.0 µs 8328 B/op 4 allocs 594 ns 104 B/op 2 allocs And, array result 8.3 µs 14504 B/op 8 allocs 4.6 µs 6280 B/op 6 allocs And, bitmap result 4.6 µs 16560 B/op 8 allocs 3.6 µs 8336 B/op 6 allocs And, 1024-interval run, array 11.9 µs 12456 B/op 8 allocs 10.0 µs 4232 B/op 6 allocs And, 1024-interval run, bitmap 9.5 µs 16560 B/op 8 allocs 7.6 µs 8336 B/op 6 allocs in-place And, bitmap result 2.4 µs 8224 B/op 2 allocs 290 ns 0 B/op 0 allocs in-place And, 1024-interval run 7.3 µs 8224 B/op 2 allocs 4.6 µs 0 B/op 0 allocs Intersects, disjoint 3.0 µs 8248 B/op 3 allocs 121 ns 0 B/op 0 allocs Intersects, overlapping 8.3 µs 14392 B/op 4 allocs 8 ns 0 B/op 0 allocs Intersects, few values, 1024 intervals 1.3 µs 32 B/op 2 allocs 62 ns 0 B/op 0 allocs Co-Authored-By: Claude Opus 5 (1M context) --- bitmapcontainer.go | 42 ++++++- runbitmap_and_test.go | 260 ++++++++++++++++++++++++++++++++++++++++++ runcontainer.go | 88 +++++++++++++- util.go | 64 +++++++++++ 4 files changed, 446 insertions(+), 8 deletions(-) create mode 100644 runbitmap_and_test.go diff --git a/bitmapcontainer.go b/bitmapcontainer.go index b5b9d226..661e4b99 100644 --- a/bitmapcontainer.go +++ b/bitmapcontainer.go @@ -798,8 +798,26 @@ func (bc *bitmapContainer) iand(a container) container { } func (bc *bitmapContainer) iandRun16(rc *runContainer16) container { - rcb := newBitmapContainerFromRun(rc) - return bc.iandBitmap(rcb) + if len(rc.iv) > runAndScratchIntervals { + // Clearing gap by gap costs a call per interval; past this many the + // run is built in scratch and ANDed in one flat pass instead. + var scratch [bitmapContainerSize]uint64 + for i := range rc.iv { + setBitmapRange(scratch[:], int(rc.iv[i].start), int(rc.iv[i].last())+1) + } + return bc.iandBitmap(&bitmapContainer{bitmap: scratch[:]}) + } + card := rc.andBitmapContainerCardinality(bc) + if card <= arrayDefaultMaxSize { + answer := newArrayContainerCapacity(card) + for i := range rc.iv { + answer.content = appendBitmapRange(answer.content, bc.bitmap, int(rc.iv[i].start), int(rc.iv[i].last())+1) + } + return answer + } + clearBitmapGaps(bc.bitmap, rc.iv, 0, maxCapacity) + bc.cardinality = card + return bc } func (bc *bitmapContainer) iandArray(ac *arrayContainer) container { @@ -831,6 +849,26 @@ func (bc *bitmapContainer) andArrayCardinality(value2 *arrayContainer) int { return pos } +func (bc *bitmapContainer) intersectsRange(start, end uint) bool { + if start >= end { + return false + } + firstword, endword := start/64, (end-1)/64 + lo, hi := ^uint64(0)<<(start%64), ^uint64(0)>>((64-end)&63) + if firstword == endword { + return bc.bitmap[firstword]&lo&hi != 0 + } + if bc.bitmap[firstword]&lo != 0 || bc.bitmap[endword]&hi != 0 { + return true + } + for _, w := range bc.bitmap[firstword+1 : endword] { + if w != 0 { + return true + } + } + return false +} + func (bc *bitmapContainer) getCardinalityInRange(start, end uint) int { if start >= end { return 0 diff --git a/runbitmap_and_test.go b/runbitmap_and_test.go new file mode 100644 index 00000000..33ffd333 --- /dev/null +++ b/runbitmap_and_test.go @@ -0,0 +1,260 @@ +package roaring + +import ( + "math/rand" + "testing" + + "github.com/stretchr/testify/require" +) + +// randomRun builds a canonical run container: up to n sorted intervals of +// at most maxLen values separated by gaps of at most maxGap, optionally +// touching 0 and 65535. +func randomRun(rng *rand.Rand, n, maxLen, maxGap int, touchEdges bool) *runContainer16 { + rc := &runContainer16{} + pos := 0 + if !touchEdges { + pos = rng.Intn(100) + } + for i := 0; i < n && pos < 65536; i++ { + length := min(1+rng.Intn(1+rng.Intn(maxLen)), 65536-pos) + rc.iv = append(rc.iv, interval16{start: uint16(pos), length: uint16(length - 1)}) + pos += length + 1 + rng.Intn(maxGap) + } + if last := int(rc.maximum()); touchEdges && last != 65535 { + if start := 65535 - rng.Intn(500); last+2 <= start { + rc.iv = append(rc.iv, interval16{start: uint16(start), length: uint16(65535 - start)}) + } + } + return rc +} + +// randomBitmapContainer ANDs ands random words per position, so the density +// is 1/2^ands. +func randomBitmapContainer(rng *rand.Rand, ands int) *bitmapContainer { + bc := newBitmapContainer() + for i := range bc.bitmap { + w := rng.Uint64() + for k := 1; k < ands; k++ { + w &= rng.Uint64() + } + bc.bitmap[i] = w + } + bc.computeCardinality() + return bc +} + +func requireAndResult(t *testing.T, want, got container) { + t.Helper() + require.True(t, want.equals(got), "want %d values, got %d", want.getCardinality(), got.getCardinality()) + _, isArray := got.(*arrayContainer) + require.Equal(t, got.getCardinality() <= arrayDefaultMaxSize, isArray, "container kind must follow its cardinality") +} + +// The reference is the old path: materialize the run, then the unchanged +// bitmap-bitmap intersection. +func TestRunAndBitmapContainer(t *testing.T) { + rng := rand.New(rand.NewSource(1)) + for trial := 0; trial < 300; trial++ { + rc := randomRun(rng, 1+rng.Intn(40), 3000, 2000, trial%5 == 0) + bc := randomBitmapContainer(rng, 1+trial%4) + want := newBitmapContainerFromRun(rc).andBitmap(bc) + requireAndResult(t, want, rc.and(bc)) + requireAndResult(t, want, bc.and(rc)) + requireAndResult(t, want, bc.clone().iand(rc)) + } +} + +// Runs with more than runAndScratchIntervals intervals take the scratch +// path; check it on regular and random runs against every density. +func TestRunAndBitmapContainerManyIntervals(t *testing.T) { + rng := rand.New(rand.NewSource(21)) + regular := func(stride, length int) *runContainer16 { + rc := &runContainer16{} + for start := 0; start+length <= 65536; start += stride { + rc.iv = append(rc.iv, interval16{start: uint16(start), length: uint16(length - 1)}) + } + return rc + } + runs := []*runContainer16{regular(32, 8), regular(64, 1), regular(3, 2), regular(200, 100)} + for i := 0; i < 60; i++ { + runs = append(runs, randomRun(rng, 65+rng.Intn(400), 60, 120, false)) + } + for i, rc := range runs { + require.Greater(t, len(rc.iv), runAndScratchIntervals, "run %d", i) + for _, bc := range []*bitmapContainer{randomBitmapContainer(rng, 6), randomBitmapContainer(rng, 2), randomBitmapContainer(rng, 1), newBitmapContainerwithRange(0, 65535)} { + want := newBitmapContainerFromRun(rc).andBitmap(bc) + requireAndResult(t, want, rc.and(bc)) + requireAndResult(t, want, bc.and(rc)) + requireAndResult(t, want, bc.clone().iand(rc)) + require.Equal(t, !want.isEmpty(), rc.intersects(bc), "run %d", i) + } + } +} + +func TestRunAndBitmapContainerEdges(t *testing.T) { + empty := newBitmapContainer() + full := newBitmapContainerwithRange(0, 65535) + single := &runContainer16{iv: []interval16{{start: 100, length: 9}}} + edges := &runContainer16{iv: []interval16{{start: 0, length: 0}, {start: 65535, length: 0}}} + atThreshold := &runContainer16{iv: []interval16{{start: 0, length: arrayDefaultMaxSize - 1}}} + pastThreshold := &runContainer16{iv: []interval16{{start: 0, length: arrayDefaultMaxSize}}} + for _, rc := range []*runContainer16{single, edges, atThreshold, pastThreshold} { + for _, bc := range []*bitmapContainer{empty, full} { + want := newBitmapContainerFromRun(rc).andBitmap(bc) + requireAndResult(t, want, rc.and(bc)) + requireAndResult(t, want, bc.clone().iand(rc)) + } + } +} + +// An empty intersection allocates only its empty array container; the old +// path allocated an 8 KiB bitmap to hold the run first. +func TestRunAndBitmapContainerEmptyAllocation(t *testing.T) { + rc := &runContainer16{iv: []interval16{{start: 0, length: 999}, {start: 30000, length: 999}}} + bc := newBitmapContainerwithRange(2000, 20000) + require.LessOrEqual(t, testing.AllocsPerRun(100, func() { + if !rc.and(bc).isEmpty() { + t.Fatal("expected empty") + } + }), 1.0) + // The clone costs two allocations, the empty result one. + require.LessOrEqual(t, testing.AllocsPerRun(100, func() { + _ = bc.clone().iand(rc) + }), 3.0) +} + +// The reference is the old definition, !rc.and(c).isEmpty(). +func TestRunContainerIntersects(t *testing.T) { + rng := rand.New(rand.NewSource(11)) + for trial := 0; trial < 400; trial++ { + rc := randomRun(rng, 1+rng.Intn(30), 3000, 2000, trial%7 == 0) + ac := newArrayContainer() + for i := 0; i < rng.Intn(300); i++ { + ac.iadd(uint16(rng.Intn(65536))) + } + others := []container{randomBitmapContainer(rng, 1+trial%3*3), ac, randomRun(rng, 1+rng.Intn(30), 3000, 2000, trial%5 == 0)} + for _, c := range others { + want := !rc.and(c).isEmpty() + require.Equal(t, want, rc.intersects(c), "trial %d run vs %T", trial, c) + require.Equal(t, want, c.intersects(rc), "trial %d %T vs run", trial, c) + } + } + // A few values against many intervals take the search branch; the + // reference is the same definition. + many := randomRun(rng, 300, 60, 120, false) + require.Greater(t, len(many.iv), 200) + for trial := 0; trial < 200; trial++ { + few := newArrayContainer() + for i := 0; i < 1+rng.Intn(6); i++ { + few.iadd(uint16(rng.Intn(65536))) + } + want := !many.and(few).isEmpty() + require.Equal(t, want, many.intersects(few), "trial %d few vs many", trial) + require.Equal(t, want, few.intersects(many), "trial %d many vs few", trial) + } + // Edges: empty array, touching intervals, single values at both ends. + empty := newArrayContainer() + edges := &runContainer16{iv: []interval16{{start: 0, length: 0}, {start: 65535, length: 0}}} + require.False(t, edges.intersects(empty)) + one := newArrayContainer() + one.iadd(65535) + require.True(t, edges.intersects(one)) + touching := &runContainer16{iv: []interval16{{start: 1, length: 65533}}} + require.False(t, edges.intersects(touching)) + require.True(t, edges.intersects(newBitmapContainerwithRange(0, 0))) +} + +func TestRunContainerIntersectsAllocatesNothing(t *testing.T) { + rc := &runContainer16{iv: []interval16{{start: 0, length: 999}, {start: 30000, length: 999}}} + bc := newBitmapContainerwithRange(2000, 20000) + ac := newArrayContainerRange(2000, 3000) + other := &runContainer16{iv: []interval16{{start: 5000, length: 100}}} + for _, c := range []container{bc, ac, other} { + require.Zero(t, testing.AllocsPerRun(100, func() { _ = rc.intersects(c) }), "%T", c) + } +} + +// runBitmap holds one run container at key 0 covering n intervals of the +// given length, spaced stride apart. +func runBitmap(n, length, stride int) *Bitmap { + bm := New() + for i := 0; i < n; i++ { + bm.AddRange(uint64(i*stride), uint64(i*stride+length)) + } + bm.RunOptimize() + return bm +} + +// BenchmarkRunAndBitmap intersects a run-optimized bitmap with a dense one +// over a single key through the public API, one case per result kind, plus +// the in-place form and the Intersects test. +func BenchmarkRunAndBitmap(b *testing.B) { + rng := rand.New(rand.NewSource(7)) + short := runBitmap(20, 300, 3000) // 6000 values + long := runBitmap(4, 10000, 16000) // 40000 values + many := runBitmap(1024, 8, 64) // 8192 values in 1024 intervals + gaps, quarter, half, dense := New(), New(), New(), New() + for v := uint32(0); v < 65536; v++ { + if !short.Contains(v) && rng.Intn(2) == 0 { + gaps.Add(v) + } + if rng.Intn(4) == 0 { + quarter.Add(v) + } + if rng.Intn(2) == 0 { + half.Add(v) + } + if rng.Intn(10) != 0 { + dense.Add(v) + } + } + for _, tc := range []struct { + name string + x, y *Bitmap + }{ + {"and/empty", short, gaps}, + {"and/array", short, half}, + {"and/bitmap", long, dense}, + {"and/many-intervals/array", many, quarter}, + {"and/many-intervals/bitmap", many, dense}, + } { + b.Run(tc.name, func(b *testing.B) { + b.ReportAllocs() + for b.Loop() { + _ = And(tc.x, tc.y) + } + }) + } + for _, tc := range []struct { + name string + x *Bitmap + }{ + {"iand/bitmap", long}, + {"iand/many-intervals", many}, + } { + b.Run(tc.name, func(b *testing.B) { + b.ReportAllocs() + x := dense.Clone() + for b.Loop() { + x.And(tc.x) + } + }) + } + few := New() + few.AddMany([]uint32{100, 30000, 60000}) + for _, tc := range []struct { + name string + x, y *Bitmap + }{ + {"intersects/disjoint", short, gaps}, + {"intersects/overlapping", short, half}, + {"intersects/few-values-many-intervals", many, few}, + } { + b.Run(tc.name, func(b *testing.B) { + for b.Loop() { + _ = tc.x.Intersects(tc.y) + } + }) + } +} diff --git a/runcontainer.go b/runcontainer.go index 7c369b75..5cf35f24 100644 --- a/runcontainer.go +++ b/runcontainer.go @@ -1855,10 +1855,42 @@ func (rc *runContainer16) andCardinality(a container) int { panic("unsupported container type") } -// andBitmapContainer finds the intersection of rc and b. +// Above this many intervals, building the intersection in scratch beats +// counting it interval by interval first (measured). +const runAndScratchIntervals = 64 + +// andBitmapContainer intersects rc with bc, counting before allocating. func (rc *runContainer16) andBitmapContainer(bc *bitmapContainer) container { - bc2 := newBitmapContainerFromRun(rc) - return bc2.andBitmap(bc) + if len(rc.iv) > runAndScratchIntervals { + return rc.andBitmapContainerScratch(bc) + } + card := rc.andBitmapContainerCardinality(bc) + if card > arrayDefaultMaxSize { + answer := newBitmapContainer() + for i := range rc.iv { + orBitmapRange(answer.bitmap, bc.bitmap, int(rc.iv[i].start), int(rc.iv[i].last())+1) + } + answer.cardinality = card + return answer + } + answer := newArrayContainerCapacity(card) + for i := range rc.iv { + answer.content = appendBitmapRange(answer.content, bc.bitmap, int(rc.iv[i].start), int(rc.iv[i].last())+1) + } + return answer +} + +func (rc *runContainer16) andBitmapContainerScratch(bc *bitmapContainer) container { + first, last := int(rc.iv[0].start)/64, int(rc.maximum())/64 + var scratch [bitmapContainerSize]uint64 + for i := range rc.iv { + orBitmapRange(scratch[:], bc.bitmap, int(rc.iv[i].start), int(rc.iv[i].last())+1) + } + card := int(popcntSlice(scratch[first : last+1])) + if card == 0 { + return newArrayContainerCapacity(0) + } + return containerFromWords(scratch[:], first, last, card) } func (rc *runContainer16) andArrayCardinality(ac *arrayContainer) int { @@ -2418,9 +2450,53 @@ func (rc *runContainer16) lazyOR(a container) container { } func (rc *runContainer16) intersects(a container) bool { - // TODO: optimize by doing inplace/less allocation - isect := rc.and(a) - return !isect.isEmpty() + switch c := a.(type) { + case *bitmapContainer: + for i := range rc.iv { + if c.intersectsRange(uint(rc.iv[i].start), uint(rc.iv[i].last())+1) { + return true + } + } + return false + case *arrayContainer: + if len(c.content)*bits.Len(uint(len(rc.iv))) < len(rc.iv) { + // Few values against many intervals: a search each beats + // walking every interval. + for _, v := range c.content { + if rc.contains(v) { + return true + } + } + return false + } + pos := 0 + for i := range rc.iv { + // advanceUntil searches from pos+1; the value at pos may still match. + pos = advanceUntil(c.content, pos-1, len(c.content), rc.iv[i].start) + if pos == len(c.content) { + return false + } + if c.content[pos] <= rc.iv[i].last() { + return true + } + } + return false + case *runContainer16: + i, j := 0, 0 + for i < len(rc.iv) && j < len(c.iv) { + x, y := rc.iv[i], c.iv[j] + if haveOverlap16(x, y) { + return true + } + if x.last() < y.last() { + i++ + } else { + j++ + } + } + return false + } + panic("unsupported container type") } func (rc *runContainer16) xor(a container) container { diff --git a/util.go b/util.go index e727d6e0..6ed3934e 100644 --- a/util.go +++ b/util.go @@ -143,6 +143,70 @@ func flipBitmapRange(bitmap []uint64, start int, end int) { bitmap[endword] ^= ^uint64(0) >> (uint(-end) % 64) } +// orBitmapRange ORs the bits of src within [start, end) into dst. +func orBitmapRange(dst, src []uint64, start int, end int) { + if start >= end { + return + } + firstword := start / 64 + endword := (end - 1) / 64 + if firstword == endword { + dst[firstword] |= src[firstword] & (^uint64(0) << uint(start%64)) & (^uint64(0) >> (uint(-end) % 64)) + return + } + dst[firstword] |= src[firstword] & (^uint64(0) << uint(start%64)) + for i := firstword + 1; i < endword; i++ { + dst[i] |= src[i] + } + dst[endword] |= src[endword] & (^uint64(0) >> (uint(-end) % 64)) +} + +// appendBitmapRange appends the positions of the bits set in [start, end). +func appendBitmapRange(out []uint16, bitmap []uint64, start int, end int) []uint16 { + if start >= end { + return out + } + firstword := start / 64 + endword := (end - 1) / 64 + for i := firstword; i <= endword; i++ { + w := bitmap[i] + if i == firstword { + w &= ^uint64(0) << uint(start%64) + } + if i == endword { + w &= ^uint64(0) >> (uint(-end) % 64) + } + for w != 0 { + out = append(out, uint16(i*64+bits.TrailingZeros64(w))) + w &= w - 1 + } + } + return out +} + +// clearBitmapGaps clears the bits between from, the sorted intervals iv, and to. +func clearBitmapGaps(bitmap []uint64, iv []interval16, from, to int) { + for i := range iv { + resetBitmapRange(bitmap, from, int(iv[i].start)) + from = int(iv[i].last()) + 1 + } + resetBitmapRange(bitmap, from, to) +} + +// containerFromWords builds an array or bitmap container from the card bits +// set in words [first, last] of w. +func containerFromWords(w []uint64, first, last, card int) container { + if card <= arrayDefaultMaxSize { + ac := newArrayContainerCapacity(card) + ac.content = appendBitmapRange(ac.content, w, first*64, (last+1)*64) + return ac + } + bc := newBitmapContainer() + copy(bc.bitmap[first:last+1], w[first:last+1]) + bc.cardinality = card + return bc +} + func resetBitmapRange(bitmap []uint64, start int, end int) { if start >= end { return From b6b237c49ec6223b9fb3e848b2e1de576d274054 Mon Sep 17 00:00:00 2001 From: tamirms Date: Mon, 14 Sep 2026 21:41:33 +0100 Subject: [PATCH 2/2] fastaggregation: count before allocating in FastAnd MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit FastAnd with three or more inputs intersected pairwise and materialized an intermediate per input per key, so a key whose result was empty still allocated up to 8 KiB per step, and a run container among bitmaps fell back to the same pairwise path. With three or more inputs the result is now counted before it is allocated. The input with the fewest containers drives; its keys are walked once, in order, against every other input with advanceUntil, and a key every input has is probed with a container intersects test before its containers are intersected, so a key with nothing in common costs no allocation and the call allocates nothing but the answer. Two inputs keep the existing And path. andK intersects one key's containers. Bitmaps are ANDed into stack scratch with the fused andCardSlice, stopping at the first empty prefix, over the words the runs leave: a run narrows the span to its extent, and only a run with gaps is folded into a mask afterwards, so a range built with AddRange costs no intersection and a result the bitmaps already rule out never walks an interval. The result is copied out as a bitmap, appended as an exact-size array, or dropped. A key with an array, or of runs only, goes through the library's own kernels instead: a full run is the identity and drops out, the smallest array leads and the rest keep the caller's order, the first pair into a fresh container and the rest in place, so nothing bigger than that first result is built. The first step for an array under a bitmap is bitmapContainer.andArray, whose loop reloaded three slice headers per element; with them hoisted it is a quarter faster on both amd64 and arm64. BenchmarkFastAndShapes (added) runs each kernel path against the pairwise chain on the same inputs, one key unless noted, Xeon 8375C: pairwise chain FastAnd array30-bitmap 231 ns 200 B 6 allocs 269 ns 200 B 6 allocs array4000-bitmap 6.6 µs 8328 B 6 allocs 6.1 µs 8328 B 6 allocs array30-runs 938 ns 288 B 8 allocs 928 ns 200 B 6 allocs array4000-runs 5.7 µs 16544 B 8 allocs 3.7 µs 8328 B 6 allocs bitmap-bitmap 2.5 µs 8336 B 6 allocs 2.6 µs 8336 B 6 allocs runs-bitmap 5.8 µs 6304 B 8 allocs 5.0 µs 4232 B 6 allocs runs-runs 8.3 µs 10400 B 8 allocs 7.5 µs 8328 B 6 allocs jointly-empty 12.5 µs 33200 B 18 allocs 1.9 µs 80 B 1 allocs eight-keys 1.4 µs 1056 B 24 allocs 1.5 µs 1056 B 24 allocs Co-Authored-By: Claude Opus 5 (1M context) --- bitmapcontainer.go | 12 +- fastaggregation.go | 187 ++++++++++++++++++++++-- fastand_kway_test.go | 277 ++++++++++++++++++++++++++++++++++++ real_data_benchmark_test.go | 6 + 4 files changed, 464 insertions(+), 18 deletions(-) create mode 100644 fastand_kway_test.go diff --git a/bitmapcontainer.go b/bitmapcontainer.go index 661e4b99..ee5f5fc3 100644 --- a/bitmapcontainer.go +++ b/bitmapcontainer.go @@ -827,15 +827,13 @@ func (bc *bitmapContainer) iandArray(ac *arrayContainer) container { func (bc *bitmapContainer) andArray(value2 *arrayContainer) *arrayContainer { answer := newArrayContainerCapacity(len(value2.content)) - answer.content = answer.content[:cap(answer.content)] - c := value2.getCardinality() + out, bitmap := answer.content[:cap(answer.content)], bc.bitmap pos := 0 - for k := 0; k < c; k++ { - v := value2.content[k] - answer.content[pos] = v - pos += int(bc.bitValue(v)) + for _, v := range value2.content { + out[pos] = v + pos += int((bitmap[v>>6] >> (v & 63)) & 1) } - answer.content = answer.content[:pos] + answer.content = out[:pos] return answer } diff --git a/fastaggregation.go b/fastaggregation.go index 3881b88a..6dfd2a16 100644 --- a/fastaggregation.go +++ b/fastaggregation.go @@ -125,26 +125,191 @@ func (x1 *Bitmap) repairAfterLazy() { } } -// FastAnd computes the intersection between many bitmaps quickly -// Compared to the And function, it can take many bitmaps as input, thus saving the trouble -// of manually calling "And" many times. +// FastAnd computes the intersection between many bitmaps quickly. +// Compared to the And function, it can take many bitmaps as input. // -// Performance hints: if you have very large and tiny bitmaps, -// it may be beneficial performance-wise to put a tiny bitmap -// in first position. +// With three or more inputs, the keys of the smallest input are walked +// against every other input and probed with a container-level intersects +// test; only a key every input shares a value on is intersected, and a key +// of bitmaps is counted before its result is allocated. func FastAnd(bitmaps ...*Bitmap) *Bitmap { - if len(bitmaps) == 0 { + switch len(bitmaps) { + case 0: return NewBitmap() - } else if len(bitmaps) == 1 { + case 1: return bitmaps[0].Clone() + case 2: + return And(bitmaps[0], bitmaps[1]) } - answer := And(bitmaps[0], bitmaps[1]) - for _, bm := range bitmaps[2:] { - answer.And(bm) + driver := 0 + for i := 1; i < len(bitmaps); i++ { + if bitmaps[i].highlowcontainer.size() < bitmaps[driver].highlowcontainer.size() { + driver = i + } + } + dra := &bitmaps[driver].highlowcontainer + // A cursor per input and the containers of one key, on the stack for up + // to eight inputs; the containers are only made once a key needs them. + var posBuf [8]int + var csBuf [8]container + var pos []int + var cs []container + if len(bitmaps) <= len(posBuf) { + pos, cs = posBuf[:len(bitmaps)], csBuf[:len(bitmaps)] + } else { + pos = make([]int, len(bitmaps)) + } + answer := NewBitmap() + // Keys come in order, so every input is walked, not searched, and a key + // is probed for a shared value with each input before its containers are + // intersected: a key with nothing in common allocates nothing. +keys: + for j := 0; j < dra.size(); j++ { + key := dra.getKeyAtIndex(j) + dc := dra.getContainerAtIndex(j) + pos[driver] = j + for i, bm := range bitmaps { + if i == driver { + continue + } + ra := &bm.highlowcontainer + if pos[i] < ra.size() && ra.getKeyAtIndex(pos[i]) < key { + pos[i] = ra.advanceUntil(key, pos[i]) // searches from pos+1 + } + if pos[i] >= ra.size() { + break keys + } + if ra.getKeyAtIndex(pos[i]) != key || !dc.intersects(ra.getContainerAtIndex(pos[i])) { + continue keys + } + } + if cs == nil { + cs = make([]container, len(bitmaps)) + } + for i, bm := range bitmaps { + cs[i] = bm.highlowcontainer.getContainerAtIndex(pos[i]) + pos[i]++ + } + if c := andK(cs); c != nil { + answer.highlowcontainer.appendContainer(key, c, false) + } } return answer } +// andK intersects the containers of one key, three or more, and returns nil +// when the intersection is empty. cs is scratch and is reordered in place. +func andK(cs []container) container { + // A full run is the identity and drops out; the smallest array leads + // and the rest keep the caller's order. + n, smallest := 0, -1 + for _, c := range cs { + switch x := c.(type) { + case *runContainer16: + if x.isFull() { + continue + } + case *arrayContainer: + if smallest < 0 || x.getCardinality() < cs[smallest].getCardinality() { + smallest = n + } + } + cs[n] = c + n++ + } + switch cs = cs[:n]; len(cs) { + case 0: + return newRunContainer16Range(0, maxCapacity-1) + case 1: + return cs[0].clone() + } + if smallest >= 0 { + a := cs[smallest] + copy(cs[1:smallest+1], cs[:smallest]) + cs[0] = a + return andKChain(cs) + } + n = 0 + for i, c := range cs { + if _, ok := c.(*runContainer16); ok { + cs[n], cs[i] = cs[i], cs[n] + n++ + } + } + if n == len(cs) { + return andKChain(cs) + } + return andKBitmaps(cs[n:], cs[:n]) +} + +// andKChain intersects cs in order with the library's own kernels, the first +// pair into a fresh container and the rest in place, so nothing bigger than +// that first result is built. +func andKChain(cs []container) container { + c := cs[0].and(cs[1]) + for i := 2; i < len(cs) && !c.isEmpty(); i++ { + c = c.iand(cs[i]) + } + if c.isEmpty() { + return nil + } + return c +} + +// andKBitmaps ANDs the bitmaps into stack scratch, stopping at the first +// empty prefix, over the words the runs leave: every run narrows the span to +// its extent, and only a run with gaps is folded into a mask afterwards, so +// a range built with AddRange costs no intersection at all. +func andKBitmaps(bms, runs []container) container { + lo, hi := 0, maxCapacity-1 + n := 0 + for _, c := range runs { + rc := c.(*runContainer16) + lo, hi = max(lo, int(rc.minimum())), min(hi, int(rc.maximum())) + if len(rc.iv) > 1 { + runs[n] = c + n++ + } + } + if lo > hi { + return nil + } + runs = runs[:n] + first, last := lo/64, hi/64 + var scratch [bitmapContainerSize]uint64 + w := scratch[first : last+1] + src := bms[0].(*bitmapContainer).bitmap[first : last+1] + card := uint64(0) + if len(bms) == 1 { + if card = andCardSlice(w, src, src); card == 0 { + return nil + } + } + for _, c := range bms[1:] { + if card = andCardSlice(w, src, c.(*bitmapContainer).bitmap[first:last+1]); card == 0 { + return nil + } + src = w + } + if len(runs) > 0 || lo%64 != 0 || (hi+1)%64 != 0 { + resetBitmapRange(scratch[:], first*64, lo) + resetBitmapRange(scratch[:], hi+1, (last+1)*64) + if len(runs) > 0 { + mask := runs[0].(*runContainer16) + for _, c := range runs[1:] { + if mask = mask.intersect(c.(*runContainer16)); len(mask.iv) == 0 { + return nil + } + } + clearBitmapGaps(scratch[:], mask.iv, lo, hi+1) + } + if card = popcntSlice(w); card == 0 { + return nil + } + } + return containerFromWords(scratch[:], first, last, int(card)) +} + // FastOr computes the union between many bitmaps quickly, as opposed to having to call Or repeatedly. // It might also be faster than calling Or repeatedly. func FastOr(bitmaps ...*Bitmap) *Bitmap { diff --git a/fastand_kway_test.go b/fastand_kway_test.go new file mode 100644 index 00000000..f7102a33 --- /dev/null +++ b/fastand_kway_test.go @@ -0,0 +1,277 @@ +package roaring + +import ( + "math/rand" + "testing" + + "github.com/stretchr/testify/require" +) + +// kwayKey picks the container key for the k-th key of an input: two inputs +// built with the same k may land on different keys, and some keys are left +// out, so key sets interleave and go missing the way real inputs do. +func kwayKey(rng *rand.Rand, k int) (uint64, bool) { + return uint64(2*k+rng.Intn(2)) << 16, rng.Intn(4) != 0 +} + +// kwayBitmap builds a bitmap over up to keys keys holding random values at +// the given density: arrays when sparse, bitmaps when dense. +func kwayBitmap(rng *rand.Rand, keys int, density float64) *Bitmap { + bm := New() + for k := 0; k < keys; k++ { + base, present := kwayKey(rng, k) + if !present { + continue + } + for i := 0; i < int(density*65536); i++ { + bm.Add(uint32(base) + uint32(rng.Intn(65536))) + } + } + return bm +} + +// kwayRuns builds a run-optimized bitmap of random ranges over up to keys keys. +func kwayRuns(rng *rand.Rand, keys int) *Bitmap { + bm := New() + for k := 0; k < keys; k++ { + base, present := kwayKey(rng, k) + if !present { + continue + } + for start := 0; start < 65536; start += 1000 + rng.Intn(3000) { + bm.AddRange(base+uint64(start), base+uint64(min(start+1+rng.Intn(800), 65536))) + } + } + bm.RunOptimize() + return bm +} + +// intervalBitmap holds one run container at key 0 made of the given +// half-open intervals. +func intervalBitmap(t *testing.T, intervals [][2]uint64) *Bitmap { + t.Helper() + bm := New() + for _, iv := range intervals { + bm.AddRange(iv[0], iv[1]) + } + bm.RunOptimize() + require.Equal(t, uint64(1), bm.Stats().RunContainers, "%v", intervals) + return bm +} + +func randomIntervals(rng *rand.Rand) [][2]uint64 { + var out [][2]uint64 + pos := uint64(rng.Intn(64)) + for n := 1 + rng.Intn(40); n > 0 && pos < 65536; n-- { + length := min(uint64(4+rng.Intn(3000)), 65536-pos) + out = append(out, [2]uint64{pos, pos + length}) + pos += length + uint64(rng.Intn(200)) + } + return out +} + +// requireFastAnd checks FastAnd against the pairwise definition, that the +// result is valid, and that it shares no storage with its inputs. +func requireFastAnd(t *testing.T, inputs ...*Bitmap) { + t.Helper() + before := make([]*Bitmap, len(inputs)) + for i, in := range inputs { + before[i] = in.Clone() + } + want := And(inputs[0], inputs[1]) + for _, bm := range inputs[2:] { + want.And(bm) + } + got := FastAnd(inputs...) + require.NoError(t, got.Validate()) + require.True(t, got.Equals(want), "%d values, want %d", got.GetCardinality(), want.GetCardinality()) + got.Flip(0, 1<<21) // rewrites every container the inputs could share + for i, in := range inputs { + require.True(t, in.Equals(before[i]), "input %d was modified through the result", i) + } +} + +func TestFastAndMatchesPairwise(t *testing.T) { + rng := rand.New(rand.NewSource(3)) + densities := []float64{0.0005, 0.01, 0.05, 0.3, 0.8, 0.98} + for trial := 0; trial < 400; trial++ { + inputs := make([]*Bitmap, 3+rng.Intn(4)) + for i := range inputs { + switch rng.Intn(5) { + case 0: + inputs[i] = kwayRuns(rng, 1+rng.Intn(4)) + case 1: + inputs[i] = intervalBitmap(t, randomIntervals(rng)) + default: + inputs[i] = kwayBitmap(rng, 1+rng.Intn(4), densities[rng.Intn(len(densities))]) + } + } + requireFastAnd(t, inputs...) + } +} + +func TestFastAndEdges(t *testing.T) { + rng := rand.New(rand.NewSource(12)) + dense := func() *Bitmap { return kwayBitmap(rng, 1, 0.5) } + full := intervalBitmap(t, [][2]uint64{{0, 65536}}) + var manyShort [][2]uint64 + for p := uint64(0); p < 65536; p += 64 { + manyShort = append(manyShort, [2]uint64{p, p + 8}) + } + for _, ivs := range [][][2]uint64{ + {{0, 65536}}, + {{777, 800}}, + {{64, 128}, {1024, 4096}}, + {{65472, 65536}}, + {{60, 70}, {130, 140}, {65530, 65536}}, + {{100, 200}, {201, 300}}, + manyShort, + } { + mask := intervalBitmap(t, ivs) + requireFastAnd(t, dense(), mask, dense()) // one run masks the bitmaps + requireFastAnd(t, dense(), mask, full, dense()) // two runs fold into one mask + requireFastAnd(t, dense(), mask, full) // one bitmap under two runs + requireFastAnd(t, mask, full, intervalBitmap(t, ivs)) // runs only + } + // Two runs whose fold, narrower than the first run, misses the one bitmap. + spot := New() + spot.Add(10) + spot.AddRange(5000, 20000) + requireFastAnd(t, intervalBitmap(t, [][2]uint64{{0, 100}}), intervalBitmap(t, [][2]uint64{{50, 100}}), spot) + // Ranges that each meet the first one but have no common value end the + // key before a bitmap is read; a range narrower than the mask's span + // clips a many-interval run. + requireFastAnd(t, intervalBitmap(t, [][2]uint64{{0, 40000}}), intervalBitmap(t, [][2]uint64{{30000, 65536}}), intervalBitmap(t, [][2]uint64{{0, 20000}}), dense()) + requireFastAnd(t, intervalBitmap(t, [][2]uint64{{100, 40000}}), intervalBitmap(t, manyShort), dense(), dense()) + requireFastAnd(t, intervalBitmap(t, [][2]uint64{{100, 40000}}), intervalBitmap(t, [][2]uint64{{50, 60}, {30000, 50000}}), dense()) + // Two ranges that each meet the bitmap but overlap only where it is + // empty, on word boundaries: the one-bitmap span counts to zero. + gap := New() + gap.Add(0) + gap.AddRange(128, 5300) + requireFastAnd(t, gap, intervalBitmap(t, [][2]uint64{{0, 128}}), intervalBitmap(t, [][2]uint64{{64, 192}})) + // Disjoint runs end the call before a bitmap is read. + requireFastAnd(t, intervalBitmap(t, [][2]uint64{{0, 1000}}), intervalBitmap(t, [][2]uint64{{2000, 3000}}), dense()) + // Keys present in some inputs only, and none shared at all. + x := kwayBitmap(rng, 6, 0.3) + requireFastAnd(t, x, AddOffset64(x, 1<<16), x) + requireFastAnd(t, x, AddOffset64(x, 40<<16), dense()) + y := New() + y.AddMany([]uint32{1, 5, 9, 70000}) + require.True(t, FastAnd(y, full, y).Equals(And(y, full))) + require.True(t, FastAnd(y, New(), full).IsEmpty()) + require.True(t, FastAnd(full, full, full).Equals(full)) + require.True(t, FastAnd(y, y, y).Equals(y)) +} + +// kwayTriple returns three dense inputs, every pair overlapping, all three +// disjoint, so only the counted k-way pass can prove the result empty. +func kwayTriple() (a, b, c *Bitmap) { + a, b, c = New(), New(), New() + for id := uint32(0); id < 4<<16; id++ { + switch id % 3 { + case 0: + a.Add(id) + b.Add(id) + case 1: + b.Add(id) + c.Add(id) + case 2: + a.Add(id) + c.Add(id) + } + } + return a, b, c +} + +func TestFastAndEmptyAllocatesNoContainer(t *testing.T) { + a, b, c := kwayTriple() + require.True(t, a.Intersects(b) && b.Intersects(c) && a.Intersects(c)) + require.True(t, FastAnd(a, b, c).IsEmpty()) + // Only the answer bitmap; never a container. + require.LessOrEqual(t, testing.AllocsPerRun(50, func() { FastAnd(a, b, c) }), 1.0) + + mask := intervalBitmap(t, [][2]uint64{{0, 65536}}) + evens, odds := New(), New() + for v := uint32(0); v < 65536; v += 2 { + evens.Add(v) + odds.Add(v + 1) + } + require.True(t, FastAnd(mask, evens, odds).IsEmpty()) + require.LessOrEqual(t, testing.AllocsPerRun(50, func() { FastAnd(mask, evens, odds) }), 1.0) +} + +// shapeKey builds one container of the given kind and cardinality on key: +// random values for arrays and bitmaps, card/8 short ranges for runs. +func shapeKey(rng *rand.Rand, kind string, card int, key uint32) *Bitmap { + bm := New() + base := uint64(key) << 16 + switch kind { + case "array", "bitmap": + for bm.GetCardinality() < uint64(card) { + bm.Add(uint32(base) + uint32(rng.Intn(65536))) + } + case "runs": + for i := 0; i < card/8; i++ { + bm.AddRange(base+uint64(i*16), base+uint64(i*16+8)) + } + bm.RunOptimize() + } + return bm +} + +func pairwiseAnd(bms []*Bitmap) *Bitmap { + r := And(bms[0], bms[1]) + for _, bm := range bms[2:] { + r.And(bm) + } + return r +} + +// BenchmarkFastAndShapes runs each kernel path once against the pairwise +// chain on the same inputs: the smallest input an array, tiny and large, +// against a bitmap, a run and a many-interval run; bitmaps under a run +// mask; runs only; dense inputs that are pairwise non-empty and jointly +// empty; and a term spanning eight keys. +func BenchmarkFastAndShapes(b *testing.B) { + rng := rand.New(rand.NewSource(9)) + window := New() + window.AddRange(0, 65536) + wide := New() + wide.AddRange(0, 8<<16) + spread := func(kind string, card int) *Bitmap { + bm := New() + for k := uint32(0); k < 8; k++ { + bm.Or(shapeKey(rng, kind, card, k)) + } + return bm + } + x, y, z := kwayTriple() + for _, tc := range []struct { + name string + inputs []*Bitmap + }{ + {"array30-bitmap", []*Bitmap{window, shapeKey(rng, "array", 30, 0), shapeKey(rng, "bitmap", 33000, 0)}}, + {"array4000-bitmap", []*Bitmap{window, shapeKey(rng, "array", 4000, 0), shapeKey(rng, "bitmap", 33000, 0)}}, + {"array30-runs", []*Bitmap{window, shapeKey(rng, "array", 30, 0), shapeKey(rng, "runs", 4000, 0)}}, + {"array4000-runs", []*Bitmap{window, shapeKey(rng, "array", 4000, 0), shapeKey(rng, "runs", 4000, 0)}}, + {"bitmap-bitmap", []*Bitmap{window, shapeKey(rng, "bitmap", 33000, 0), shapeKey(rng, "bitmap", 33000, 0)}}, + {"runs-bitmap", []*Bitmap{window, shapeKey(rng, "runs", 4000, 0), shapeKey(rng, "bitmap", 33000, 0)}}, + {"runs-runs", []*Bitmap{window, shapeKey(rng, "runs", 4000, 0), shapeKey(rng, "runs", 30000, 0)}}, + {"jointly-empty", []*Bitmap{x, y, z}}, + {"eight-keys", []*Bitmap{wide, spread("array", 30), spread("bitmap", 33000)}}, + } { + b.Run(tc.name+"/fastand", func(b *testing.B) { + b.ReportAllocs() + for b.Loop() { + _ = FastAnd(tc.inputs...) + } + }) + b.Run(tc.name+"/pairwise", func(b *testing.B) { + b.ReportAllocs() + for b.Loop() { + _ = pairwiseAnd(tc.inputs) + } + }) + } +} diff --git a/real_data_benchmark_test.go b/real_data_benchmark_test.go index 7e47a323..f40a12a4 100644 --- a/real_data_benchmark_test.go +++ b/real_data_benchmark_test.go @@ -186,3 +186,9 @@ func BenchmarkRealDataFastOr(b *testing.B) { return FastOr(bitmaps...).GetCardinality() }) } + +func BenchmarkRealDataFastAnd(b *testing.B) { + benchmarkRealDataAggregate(b, func(bitmaps []*Bitmap) uint64 { + return FastAnd(bitmaps...).GetCardinality() + }) +}