Merge pull request #419 from neena/neena/faster-iandnot-bm-arr

faster iandnot between bitmap and array containers

Author: lemire

benchmark_test.go

@@ -322,7 +322,7 @@ func BenchmarkUnionRoaring(b *testing.B) {
 // The nested for-loops test a number of different scenarios
 // with respect to the ranges and densities of bitmaps.
 func BenchmarkUnionInPlaceCopyOnWrite(b *testing.B) {
-	//uint32s to maintain 1.12 compatibility, which requires unsigned shifts.
+	// uint32s to maintain 1.12 compatibility, which requires unsigned shifts.
 	startingContainerPower := uint32(4)
 	finalContainerPower := uint32(10)
 	containerIncrement := uint32(3)
@@ -1181,6 +1181,7 @@ func BenchmarkAndNot(b *testing.B) {
 				b.Run(fmt.Sprintf("left=%s", leftGen.name), func(b *testing.B) {
 					for _, rightGen := range []generator{makeRunContainer, makeArrayContainer, makeBitmapContainer} {
 						b.Run(fmt.Sprintf("right=%s", rightGen.name), func(b *testing.B) {
+							b.ReportAllocs()
 							b.StopTimer()
 							serializedLefts := make([][]byte, 1000)
 							for i := range serializedLefts {

bitmapcontainer.go

@@ -888,8 +888,42 @@ func (bc *bitmapContainer) iandNot(a container) container {
 }
 
 func (bc *bitmapContainer) iandNotArray(ac *arrayContainer) container {
-	acb := ac.toBitmapContainer()
-	return bc.iandNotBitmapSurely(acb)
+	if ac.isEmpty() || bc.isEmpty() {
+		// Nothing to do.
+		return bc
+	}
+
+	// Word by word, we remove the elements in ac from bc. The approach is to build
+	// a mask of the elements to remove, and then apply it to the bitmap.
+	wordIdx := uint16(0)
+	mask := uint64(0)
+	for i, v := range ac.content {
+		if v/64 != wordIdx {
+			// Flush the current word.
+			if i != 0 {
+				// We're removing bits that are set in the mask and in the current word.
+				// To figure out the cardinality change, we count the number of bits that
+				// are set in the mask and in the current word.
+				mask &= bc.bitmap[wordIdx]
+				bc.bitmap[wordIdx] &= ^mask
+				bc.cardinality -= int(popcount(mask))
+			}
+
+			wordIdx = v / 64
+			mask = 0
+		}
+		mask |= 1 << (v % 64)
+	}
+
+	// Flush the last word.
+	mask &= bc.bitmap[wordIdx]
+	bc.bitmap[wordIdx] &= ^mask
+	bc.cardinality -= int(popcount(mask))
+
+	if bc.getCardinality() <= arrayDefaultMaxSize {
+		return bc.toArrayContainer()
+	}
+	return bc
 }
 
 func (bc *bitmapContainer) iandNotRun16(rc *runContainer16) container {