Merge pull request #256 from RoaringBitmap/dlemire/better_tests

Trying new tests

.github/workflows/armtests.yml

@@ -0,0 +1,27 @@
+
+name: Go-ARM-CI
+
+on: [push, pull_request]
+
+jobs:
+  test:
+    strategy:
+      matrix:
+        go-version: [1.13.x, 1.14.x]
+        platform: [ubuntu-latest]
+    runs-on: ${{ matrix.platform }}
+    steps:
+    - name: Install Go
+      uses: actions/setup-go@v2
+      with:
+        go-version: ${{ matrix.go-version }}
+    - name: Checkout code
+      uses: actions/checkout@v2
+    - name: Test
+      run: |
+        GOARCH=arm go build
+        GOARCH=arm go build ./roaring64
+        GOARCH=arm go build ./BitSliceIndexing
+        GOARCH=arm64 go build
+        GOARCH=arm64 go build ./roaring64
+        GOARCH=arm64 go build ./BitSliceIndexing

.github/workflows/basictests.yml

@@ -0,0 +1,27 @@
+
+name: Go-CI
+
+on: [push, pull_request]
+
+jobs:
+  test:
+    strategy:
+      matrix:
+        go-version: [1.13.x, 1.14.x]
+        platform: [ubuntu-latest]
+    runs-on: ${{ matrix.platform }}
+    steps:
+    - name: Install Go
+      uses: actions/setup-go@v2
+      with:
+        go-version: ${{ matrix.go-version }}
+    - name: Checkout code
+      uses: actions/checkout@v2
+    - name: Test
+      run: |
+        go test -race -run TestConcurrent*
+        go build -tags appengine
+        go test -tags appengine
+        go test -v
+        go test -v ./roaring64
+        go test -v ./BitSliceIndexing

.github/workflows/legacytests.yml

@@ -0,0 +1,26 @@
+
+name: Go-legacy-CI
+
+on: [push, pull_request]
+
+jobs:
+  test:
+    strategy:
+      matrix:
+        go-version: [1.13.x, 1.14.x]
+        platform: [ubuntu-latest]
+    runs-on: ${{ matrix.platform }}
+    steps:
+    - name: Install Go
+      uses: actions/setup-go@v2
+      with:
+        go-version: ${{ matrix.go-version }}
+    - name: Checkout code
+      uses: actions/checkout@v2
+    - name: Test
+      run: |
+        GOARCH=386 go build ./roaring64
+        GOARCH=386 go build ./BitSliceIndexing
+        GOARCH=386 go test -v 
+        GOARCH=386 go test -v ./roaring64
+        GOARCH=386 go test -v ./BitSliceIndexing

.github/workflows/macostests.yml

@@ -0,0 +1,27 @@
+
+name: Go-macos-CI
+
+on: [push, pull_request]
+
+jobs:
+  test:
+    strategy:
+      matrix:
+        go-version: [1.13.x, 1.14.x]
+        platform: [macos-latest]
+    runs-on: ${{ matrix.platform }}
+    steps:
+    - name: Install Go
+      uses: actions/setup-go@v2
+      with:
+        go-version: ${{ matrix.go-version }}
+    - name: Checkout code
+      uses: actions/checkout@v2
+    - name: Test
+      run: |
+        go test -race -run TestConcurrent*
+        go build -tags appengine
+        go test -tags appengine
+        go test -v
+        go test -v ./roaring64
+        go test -v ./BitSliceIndexing

.github/workflows/windowstests.yml

@@ -0,0 +1,27 @@
+
+name: Go-Windows-CI
+
+on: [push, pull_request]
+
+jobs:
+  test:
+    strategy:
+      matrix:
+        go-version: [1.13.x, 1.14.x]
+        platform: [windows-latest]
+    runs-on: ${{ matrix.platform }}
+    steps:
+    - name: Install Go
+      uses: actions/setup-go@v2
+      with:
+        go-version: ${{ matrix.go-version }}
+    - name: Checkout code
+      uses: actions/checkout@v2
+    - name: Test
+      run: |
+        go test -race -run TestConcurrent*
+        go build -tags appengine
+        go test -tags appengine
+        go test -v
+        go test -v ./roaring64
+        go test -v ./BitSliceIndexing

roaring64/Makefile

@@ -0,0 +1,111 @@
+.PHONY: help all test format fmtcheck vet lint     qa deps clean nuke ser fetch-real-roaring-datasets
+
+
+
+
+
+
+
+
+# Display general help about this command
+help:
+	@echo ""
+	@echo "The following commands are available:"
+	@echo ""
+	@echo "    make qa          : Run all the tests"
+	@echo "    make test        : Run the unit tests"
+	@echo ""
+	@echo "    make format      : Format the source code"
+	@echo "    make fmtcheck    : Check if the source code has been formatted"
+	@echo "    make vet         : Check for suspicious constructs"
+	@echo "    make lint        : Check for style errors"
+	@echo ""
+	@echo "    make deps        : Get the dependencies"
+	@echo "    make clean       : Remove any build artifact"
+	@echo "    make nuke        : Deletes any intermediate file"
+	@echo ""
+	@echo "    make fuzz-smat   : Fuzzy testing with smat"
+	@echo "    make fuzz-stream : Fuzzy testing with stream deserialization"
+	@echo "    make fuzz-buffer : Fuzzy testing with buffer deserialization"
+	@echo ""
+
+# Alias for help target
+all: help
+test:
+	go test
+	go test -race -run TestConcurrent*
+# Format the source code
+format:
+	@find ./ -type f -name "*.go" -exec gofmt -w {} \;
+
+# Check if the source code has been formatted
+fmtcheck:
+	@mkdir -p target
+	@find ./ -type f -name "*.go" -exec gofmt -d {} \; | tee target/format.diff
+	@test ! -s target/format.diff || { echo "ERROR: the source code has not been formatted - please use 'make format' or 'gofmt'"; exit 1; }
+
+# Check for syntax errors
+vet:
+	GOPATH=$(GOPATH) go vet ./...
+
+# Check for style errors
+lint:
+	GOPATH=$(GOPATH) PATH=$(GOPATH)/bin:$(PATH) golint ./...
+
+
+
+
+
+# Alias to run all quality-assurance checks
+qa: fmtcheck test vet lint
+
+# --- INSTALL ---
+
+# Get the dependencies
+deps:
+	GOPATH=$(GOPATH) go get github.com/stretchr/testify
+	GOPATH=$(GOPATH) go get github.com/willf/bitset
+	GOPATH=$(GOPATH) go get github.com/golang/lint/golint
+	GOPATH=$(GOPATH) go get github.com/mschoch/smat
+	GOPATH=$(GOPATH) go get github.com/dvyukov/go-fuzz/go-fuzz
+	GOPATH=$(GOPATH) go get github.com/dvyukov/go-fuzz/go-fuzz-build
+	GOPATH=$(GOPATH) go get github.com/glycerine/go-unsnap-stream
+	GOPATH=$(GOPATH) go get github.com/philhofer/fwd
+	GOPATH=$(GOPATH) go get github.com/jtolds/gls
+
+fuzz-smat:
+	go test -tags=gofuzz -run=TestGenerateSmatCorpus
+	go-fuzz-build -func FuzzSmat github.com/RoaringBitmap/roaring
+	go-fuzz -bin=./roaring-fuzz.zip -workdir=workdir/ -timeout=200
+
+
+fuzz-stream:
+	go-fuzz-build -func FuzzSerializationStream github.com/RoaringBitmap/roaring
+	go-fuzz -bin=./roaring-fuzz.zip -workdir=workdir/ -timeout=200
+
+
+fuzz-buffer:
+	go-fuzz-build -func FuzzSerializationBuffer github.com/RoaringBitmap/roaring
+	go-fuzz -bin=./roaring-fuzz.zip -workdir=workdir/ -timeout=200
+
+# Remove any build artifact
+clean:
+	GOPATH=$(GOPATH) go clean ./...
+
+# Deletes any intermediate file
+nuke:
+	rm -rf ./target
+	GOPATH=$(GOPATH) go clean -i ./...
+
+
+ser: 
+	go generate
+
+cover:
+	go test -coverprofile=coverage.out
+	go tool cover -html=coverage.out
+
+fetch-real-roaring-datasets:
+	# pull github.com/RoaringBitmap/real-roaring-datasets -> testdata/real-roaring-datasets
+	git submodule init
+	git submodule update

roaring64/fastaggregation64.go

@@ -34,4 +34,3 @@ func FastOr(bitmaps ...*Bitmap) *Bitmap {
 	//answer.repairAfterLazy()
 	return answer
 }
-

roaring64/fastaggregation64_test.go

@@ -7,8 +7,6 @@ import (
 	"testing"
 )
 
-
-
 func TestFastAggregationsAdvanced_run(t *testing.T) {
 	rb1 := NewBitmap()
 	rb2 := NewBitmap()
@@ -39,4 +37,3 @@ func TestFastAggregationsAdvanced_run(t *testing.T) {
 	assert.True(t, FastOr(rb1, rb2, rb3).Equals(rb1))
 	assert.True(t, FastAnd(rb1, rb2, rb3).Equals(bigand))
 }
-

roaring64/iterables64.go

@@ -2,13 +2,13 @@ package roaring64
 
 import "github.com/RoaringBitmap/roaring"
 
-// IntIterable allows you to iterate over the values in a Bitmap
+// IntIterable64 allows you to iterate over the values in a Bitmap
 type IntIterable64 interface {
 	HasNext() bool
 	Next() uint64
 }
 
-// IntPeekable allows you to look at the next value without advancing and
+// IntPeekable64 allows you to look at the next value without advancing and
 // advance as long as the next value is smaller than minval
 type IntPeekable64 interface {
 	IntIterable64
@@ -118,7 +118,7 @@ func newIntReverseIterator(a *Bitmap) *intReverseIterator {
 	return p
 }
 
-// ManyIntIterable allows you to iterate over the values in a Bitmap
+// ManyIntIterable64 allows you to iterate over the values in a Bitmap
 type ManyIntIterable64 interface {
 	// pass in a buffer to fill up with values, returns how many values were returned
 	NextMany([]uint64) int
@@ -164,4 +164,3 @@ func newManyIntIterator(a *Bitmap) *manyIntIterator {
 	p.init()
 	return p
 }
-

roaring64/iterables64_test.go

@@ -28,7 +28,7 @@ func TestReverseIteratorCount(t *testing.T) {
 
 func TestReverseIterator(t *testing.T) {
 	t.Run("#1", func(t *testing.T) {
-		values := []uint64{0, 2, 15, 16, 31, 32, 33, 9999, roaring.MaxUint16, roaring.MaxUint32, roaring.MaxUint32*2, math.MaxUint64}
+		values := []uint64{0, 2, 15, 16, 31, 32, 33, 9999, roaring.MaxUint16, roaring.MaxUint32, roaring.MaxUint32 * 2, math.MaxUint64}
 		bm := New()
 		for n := 0; n < len(values); n++ {
 			bm.Add(values[n])
@@ -99,7 +99,7 @@ func TestReverseIterator(t *testing.T) {
 }
 
 func TestIteratorPeekNext(t *testing.T) {
-	values := []uint64{0, 2, 15, 16, 31, 32, 33, 9999, roaring.MaxUint16, roaring.MaxUint32, roaring.MaxUint32*2, math.MaxUint64}
+	values := []uint64{0, 2, 15, 16, 31, 32, 33, 9999, roaring.MaxUint16, roaring.MaxUint32, roaring.MaxUint32 * 2, math.MaxUint64}
 	bm := New()
 
 	for n := 0; n < len(values); n++ {
@@ -180,4 +180,3 @@ func TestIteratorAdvance(t *testing.T) {
 		assert.EqualValues(t, 31, i.PeekNext())
 	})
 }
-

roaring64/parallel64.go

@@ -9,12 +9,11 @@ import (
 
 var defaultWorkerCount = runtime.NumCPU()
 
-
 // ParOr computes the union (OR) of all provided bitmaps in parallel,
 // where the parameter "parallelism" determines how many workers are to be used
 // (if it is set to 0, a default number of workers is chosen)
 func ParOr(parallelism int, bitmaps ...*Bitmap) *Bitmap {
-	var lKey uint32 = MaxUint32
+	var lKey uint32 = maxUint32
 	var hKey uint32
 
 	bitmapsFiltered := bitmaps[:0]
@@ -30,7 +29,7 @@ func ParOr(parallelism int, bitmaps ...*Bitmap) *Bitmap {
 		hKey = maxOfUint32(hKey, b.highlowcontainer.keys[b.highlowcontainer.size()-1])
 	}
 
-	if lKey == MaxUint32 && hKey == 0 {
+	if lKey == maxUint32 && hKey == 0 {
 		return New()
 	} else if len(bitmaps) == 1 {
 		return bitmaps[0]
@@ -262,8 +261,8 @@ func iorOnRange(ra1, ra2 *roaringArray64, start, last uint32) *roaringArray64 {
 
 				//ra1.containers[idx1] = c1.lazyIOR(ra2.getContainerAtIndex(idx2))
 				c1.Or(ra2.getContainerAtIndex(idx2))
-                                ra1.setContainerAtIndex(idx1, c1)
-                   
+				ra1.setContainerAtIndex(idx1, c1)
+
 				ra1.needCopyOnWrite[idx1] = false
 				idx1++
 				idx2++