NEON-vectorized exclusiveUnion2by2

setutil.go

@@ -1,6 +1,6 @@
 package roaring
 
-func difference(set1 []uint16, set2 []uint16, buffer []uint16) int {
+func localdifference(set1 []uint16, set2 []uint16, buffer []uint16) int {
 	if len(set2) == 0 {
 		buffer = buffer[:len(set1)]
 		copy(buffer, set1)
@@ -54,7 +54,7 @@ func difference(set1 []uint16, set2 []uint16, buffer []uint16) int {
 	return pos
 }
 
-func exclusiveUnion2by2(set1 []uint16, set2 []uint16, buffer []uint16) int {
+func localexclusiveUnion2by2(set1 []uint16, set2 []uint16, buffer []uint16) int {
 	if 0 == len(set2) {
 		buffer = buffer[:len(set1)]
 		copy(buffer, set1[:])
@@ -169,34 +169,6 @@ func union2by2Cardinality(set1 []uint16, set2 []uint16) int {
 	return pos
 }
 
-func intersection2by2(
-	set1 []uint16,
-	set2 []uint16,
-	buffer []uint16,
-) int {
-	if len(set1)*64 < len(set2) {
-		return onesidedgallopingintersect2by2(set1, set2, buffer)
-	} else if len(set2)*64 < len(set1) {
-		return onesidedgallopingintersect2by2(set2, set1, buffer)
-	} else {
-		return localintersect2by2(set1, set2, buffer)
-	}
-}
-
-// intersection2by2Cardinality computes the cardinality of the intersection
-func intersection2by2Cardinality(
-	set1 []uint16,
-	set2 []uint16,
-) int {
-	if len(set1)*64 < len(set2) {
-		return onesidedgallopingintersect2by2Cardinality(set1, set2)
-	} else if len(set2)*64 < len(set1) {
-		return onesidedgallopingintersect2by2Cardinality(set2, set1)
-	} else {
-		return localintersect2by2Cardinality(set1, set2)
-	}
-}
-
 // intersects2by2 computes whether the two sets intersect
 func intersects2by2(
 	set1 []uint16,

setutil_andnot_arm64.go

@@ -0,0 +1,8 @@
+//go:build arm64 && !gccgo && !appengine
+// +build arm64,!gccgo,!appengine
+
+package roaring
+
+func difference(set1 []uint16, set2 []uint16, buffer []uint16) int {
+	return localdifference(set1, set2, buffer)
+}

setutil_arm64.go

@@ -3,5 +3,27 @@
 
 package roaring
 
+// uniqshuf[m] compacts the lanes not set in m. Initialize it before any
+// package-level bitmap unions, which can run before init functions.
+var uniqshuf = buildUniqshuf()
+
+func buildUniqshuf() (t [256 * 16]byte) {
+	for m := 0; m < 256; m++ {
+		pos := 0
+		for lane := 0; lane < 8; lane++ {
+			if m&(1<<lane) == 0 {
+				t[m*16+pos*2] = byte(2 * lane)
+				t[m*16+pos*2+1] = byte(2*lane + 1)
+				pos++
+			}
+		}
+		for ; pos < 8; pos++ {
+			t[m*16+pos*2] = 0xFF
+			t[m*16+pos*2+1] = 0xFF
+		}
+	}
+	return t
+}
+
 //go:noescape
 func union2by2(set1 []uint16, set2 []uint16, buffer []uint16) (size int)

setutil_generic.go

@@ -3,6 +3,34 @@
 
 package roaring
 
+func intersection2by2(
+	set1 []uint16,
+	set2 []uint16,
+	buffer []uint16,
+) int {
+	if len(set1)*64 < len(set2) {
+		return onesidedgallopingintersect2by2(set1, set2, buffer)
+	} else if len(set2)*64 < len(set1) {
+		return onesidedgallopingintersect2by2(set2, set1, buffer)
+	} else {
+		return localintersect2by2(set1, set2, buffer)
+	}
+}
+
+// intersection2by2Cardinality computes the cardinality of the intersection
+func intersection2by2Cardinality(
+	set1 []uint16,
+	set2 []uint16,
+) int {
+	if len(set1)*64 < len(set2) {
+		return onesidedgallopingintersect2by2Cardinality(set1, set2)
+	} else if len(set2)*64 < len(set1) {
+		return onesidedgallopingintersect2by2Cardinality(set2, set1)
+	} else {
+		return localintersect2by2Cardinality(set1, set2)
+	}
+}
+
 func union2by2(set1 []uint16, set2 []uint16, buffer []uint16) int {
 	pos := 0
 	k1 := 0
@@ -62,3 +90,11 @@ func union2by2(set1 []uint16, set2 []uint16, buffer []uint16) int {
 	}
 	return pos
 }
+
+func difference(set1 []uint16, set2 []uint16, buffer []uint16) int {
+	return localdifference(set1, set2, buffer)
+}
+
+func exclusiveUnion2by2(set1 []uint16, set2 []uint16, buffer []uint16) int {
+	return localexclusiveUnion2by2(set1, set2, buffer)
+}

setutil_intersect_arm64.go

@@ -0,0 +1,32 @@
+//go:build arm64 && !gccgo && !appengine
+// +build arm64,!gccgo,!appengine
+
+package roaring
+
+func intersection2by2(
+	set1 []uint16,
+	set2 []uint16,
+	buffer []uint16,
+) int {
+	if len(set1)*64 < len(set2) {
+		return onesidedgallopingintersect2by2(set1, set2, buffer)
+	} else if len(set2)*64 < len(set1) {
+		return onesidedgallopingintersect2by2(set2, set1, buffer)
+	} else {
+		return localintersect2by2(set1, set2, buffer)
+	}
+}
+
+// intersection2by2Cardinality computes the cardinality of the intersection
+func intersection2by2Cardinality(
+	set1 []uint16,
+	set2 []uint16,
+) int {
+	if len(set1)*64 < len(set2) {
+		return onesidedgallopingintersect2by2Cardinality(set1, set2)
+	} else if len(set2)*64 < len(set1) {
+		return onesidedgallopingintersect2by2Cardinality(set2, set1)
+	} else {
+		return localintersect2by2Cardinality(set1, set2)
+	}
+}

setutil_xor_arm64.go

@@ -0,0 +1,23 @@
+//go:build arm64 && !gccgo && !appengine
+// +build arm64,!gccgo,!appengine
+
+package roaring
+
+//go:noescape
+func xorKernelNEON(set1, set2, buffer []uint16, shuf *byte) (outLen, pos1, pos2 int)
+
+// The kernel needs 16 unread elements on both sides and a fresh buffer with
+// cap(buffer) >= len(set1)+len(set2). Below 32 its setup usually loses to the
+// scalar path. Duplicate values get unspecified results; stores stay within capacity.
+const neonXorThreshold = 32
+
+func exclusiveUnion2by2(set1 []uint16, set2 []uint16, buffer []uint16) int {
+	if len(set1) < neonXorThreshold || len(set2) < neonXorThreshold || cap(buffer) < len(set1)+len(set2) {
+		return localexclusiveUnion2by2(set1, set2, buffer)
+	}
+	// Callers such as xorArray pass a zero-length buffer with capacity.
+	buffer = buffer[:cap(buffer)]
+	outLen, pos1, pos2 := xorKernelNEON(set1, set2, buffer, &uniqshuf[0])
+	// Everything unread is above the last emitted value, so the tails need no seam check.
+	return outLen + localexclusiveUnion2by2(set1[pos1:], set2[pos2:], buffer[outLen:])
+}