Add unsafe little endian loaders

.github/workflows/go.yml

@@ -32,18 +32,24 @@ jobs:
     - name: Test
       run: go test ./...
 
-    - name: Test Noasm
+    - name: Test No-asm
       run: go test -tags=noasm ./...
 
+    - name: Test No-unsafe
+      run: go test -tags=nounsafe ./...
+
+    - name: Test No-unsafe, noasm
+      run: go test -tags="nounsafe,noasm" ./...
+
     - name: Test Race 1 CPU
       env:
         CGO_ENABLED: 1
-      run: go test -cpu=1 -short -race -v ./...
+      run: go test -cpu=1 -short -race -tags=nounsafe -v ./...
 
     - name: Test Race 4 CPU
       env:
         CGO_ENABLED: 1
-      run: go test -cpu=4 -short -race -v ./...
+      run: go test -cpu=4 -short -race -tags=nounsafe -v ./...
 
   generate:
     strategy:
@@ -112,6 +118,9 @@ jobs:
     env:
       CGO_ENABLED: 0
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        tags: [ 'nounsafe', '"noasm,nounsafe"' ]
     steps:
       - name: Set up Go
         uses: actions/[email protected]
@@ -121,28 +130,23 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v4
 
-      - name: S2/FuzzDictBlocks
-        run: go test -run=none -fuzz=FuzzDictBlocks -fuzztime=100000x -test.fuzzminimizetime=10ms ./s2/.
+      - name: S2/FuzzDictBlocks/${{ matrix.tags }}
+        run: go test -tags=${{ matrix.tags }} -run=none -fuzz=FuzzDictBlocks -fuzztime=100000x -test.fuzzminimizetime=10ms ./s2/.
 
-      - name: S2/FuzzEncodingBlocks
-        run: go test -run=none -fuzz=FuzzEncodingBlocks -fuzztime=500000x -test.fuzzminimizetime=10ms ./s2/.
+      - name: S2/FuzzEncodingBlocks/${{ matrix.tags }}
+        run: go test -tags=${{ matrix.tags }} -run=none -fuzz=FuzzEncodingBlocks -fuzztime=500000x -test.fuzzminimizetime=10ms ./s2/.
 
-      - name: S2/FuzzLZ4Block
-        run: go test -run=none -fuzz=FuzzLZ4Block -fuzztime=500000x -test.fuzzminimizetime=10ms ./s2/.
+      - name: S2/FuzzLZ4Block/${{ matrix.tags }}
+        run: go test -tags=${{ matrix.tags }} -run=none -fuzz=FuzzLZ4Block -fuzztime=500000x -test.fuzzminimizetime=10ms ./s2/.
 
-      - name: S2/FuzzDictBlocks/noasm
-        run: go test -tags=noasm -run=none -fuzz=FuzzDictBlocks -fuzztime=100000x -test.fuzzminimizetime=10ms ./s2/.
-
-      - name: S2/FuzzEncodingBlocks/noasm
-        run: go test -tags=noasm -run=none -fuzz=FuzzEncodingBlocks -fuzztime=500000x -test.fuzzminimizetime=10ms ./s2/.
-
-      - name: S2/FuzzLZ4Block/noasm
-        run: go test -tags=noasm -run=none -fuzz=FuzzLZ4Block -fuzztime=500000x -test.fuzzminimizetime=10ms ./s2/.
 
   fuzz-zstd:
     env:
       CGO_ENABLED: 0
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        tags: [ 'nounsafe', '"noasm,nounsafe"' ]
     steps:
       - name: Set up Go
         uses: actions/[email protected]
@@ -152,57 +156,44 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v4
 
-      - name: zstd/FuzzDecodeAll
-        run: go test -run=none -fuzz=FuzzDecodeAll -fuzztime=500000x -test.fuzzminimizetime=10ms ./zstd/.
+      - name: zstd/FuzzDecodeAll/${{ matrix.tags }}
+        run: go test -tags=${{ matrix.tags }} -run=none -fuzz=FuzzDecodeAll -fuzztime=500000x -test.fuzzminimizetime=10ms ./zstd/.
 
-      - name: zstd/FuzzDecAllNoBMI2
-        run: go test -run=none -fuzz=FuzzDecAllNoBMI2 -fuzztime=500000x -test.fuzzminimizetime=10ms ./zstd/.
+      - name: zstd/FuzzDecAllNoBMI2/${{ matrix.tags }}
+        run: go test -tags=${{ matrix.tags }} -run=none -fuzz=FuzzDecAllNoBMI2 -fuzztime=500000x -test.fuzzminimizetime=10ms ./zstd/.
 
-      - name: zstd/FuzzDecoder
-        run: go test -run=none -fuzz=FuzzDecoder -fuzztime=500000x -test.fuzzminimizetime=10ms ./zstd/.
+      - name: zstd/FuzzDecoder/${{ matrix.tags }}
+        run: go test -tags=${{ matrix.tags }} -run=none -fuzz=FuzzDecoder -fuzztime=500000x -test.fuzzminimizetime=10ms ./zstd/.
 
-      - name: zstd/FuzzNoBMI2Dec
-        run: go test -run=none -fuzz=FuzzNoBMI2Dec -fuzztime=500000x -test.fuzzminimizetime=10ms ./zstd/.
+      - name: zstd/FuzzNoBMI2Dec/${{ matrix.tags }}
+        run: go test -tags=${{ matrix.tags }} -run=none -fuzz=FuzzNoBMI2Dec -fuzztime=500000x -test.fuzzminimizetime=10ms ./zstd/.
 
-      - name: zstd/FuzzEncoding
-        run: cd zstd&&go test -run=none -fuzz=FuzzEncoding -fuzztime=250000x -test.fuzzminimizetime=10ms -fuzz-end=3&&cd ..
-
-      - name: zstd/FuzzDecodeAll/noasm
-        run: go test -tags=noasm -run=none -fuzz=FuzzDecodeAll -fuzztime=500000x -test.fuzzminimizetime=10ms ./zstd/.
-
-      - name: zstd/FuzzDecoder/noasm
-        run: go test -tags=noasm -run=none -fuzz=FuzzDecoder -fuzztime=500000x -test.fuzzminimizetime=10ms ./zstd/.
-
-      - name: zstd/FuzzEncoding/noasm
-        run: cd zstd&&go test -tags=noasm -run=none -fuzz=FuzzEncoding -fuzztime=250000x -test.fuzzminimizetime=10ms -fuzz-end=3&&cd ..
-
-      - name: zstd/FuzzEncodingBest
-        run: cd zstd&&go test -run=none -fuzz=FuzzEncoding -fuzztime=25000x -test.fuzzminimizetime=10ms -fuzz-start=4&&cd ..
+      - name: zstd/FuzzEncoding/${{ matrix.tags }}
+        run: cd zstd&&go test -tags=${{ matrix.tags }} -run=none -fuzz=FuzzEncoding -fuzztime=250000x -test.fuzzminimizetime=10ms -fuzz-end=3&&cd ..
 
   fuzz-other:
     env:
       CGO_ENABLED: 0
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        tags: [ 'nounsafe', '"noasm,nounsafe"' ]
     steps:
       - name: Set up Go
         uses: actions/[email protected]
         with:
           go-version: 1.23.x
-
       - name: Checkout code
         uses: actions/checkout@v4
 
-      - name: flate/FuzzEncoding
-        run: go test -run=none -fuzz=FuzzEncoding -fuzztime=100000x -test.fuzzminimizetime=10ms ./flate/.
-
-      - name: flate/FuzzEncoding/noasm
-        run: go test -run=none -tags=noasm -fuzz=FuzzEncoding -fuzztime=100000x -test.fuzzminimizetime=10ms ./flate/.
+      - name: flate/FuzzEncoding/${{ matrix.tags }}
+        run: go test -tags=${{ matrix.tags }} -run=none -fuzz=FuzzEncoding -fuzztime=100000x -test.fuzzminimizetime=10ms ./flate/.
 
-      - name: zip/FuzzReader
-        run: go test -run=none -fuzz=FuzzReader -fuzztime=500000x -test.fuzzminimizetime=10ms ./zip/.
+      - name: zip/FuzzReader/${{ matrix.tags }}
+        run: go test -tags=${{ matrix.tags }} -run=none -fuzz=FuzzReader -fuzztime=500000x -test.fuzzminimizetime=10ms ./zip/.
 
-      - name: fse/FuzzCompress
-        run: go test -run=none -fuzz=FuzzCompress -fuzztime=1000000x -test.fuzzminimizetime=10ms ./fse/.
+      - name: fse/FuzzCompress/${{ matrix.tags }}
+        run: go test -tags=${{ matrix.tags }} -run=none -fuzz=FuzzCompress -fuzztime=1000000x -test.fuzzminimizetime=10ms ./fse/.
 
-      - name: fse/FuzzDecompress
-        run: go test -run=none -fuzz=FuzzDecompress -fuzztime=1000000x -test.fuzzminimizetime=10ms ./fse/.
+      - name: fse/FuzzDecompress/${{ matrix.tags }}
+        run: go test -tags=${{ matrix.tags }} -run=none -fuzz=FuzzDecompress -fuzztime=1000000x -test.fuzzminimizetime=10ms ./fse/.

README.md

@@ -14,6 +14,17 @@ This package provides various compression algorithms.
 [![Go](https://github.com/klauspost/compress/actions/workflows/go.yml/badge.svg)](https://github.com/klauspost/compress/actions/workflows/go.yml)
 [![Sourcegraph Badge](https://sourcegraph.com/github.com/klauspost/compress/-/badge.svg)](https://sourcegraph.com/github.com/klauspost/compress?badge)
 
+# package usage
+
+Use `go get github.com/klauspost/compress@latest` to add it to your project.
+
+This package will support the current Go version and 2 versions back.
+
+* Use the `nounsafe` tag to disable all use of the "unsafe" package.
+* Use the `noasm` tag to disable all assembly across packages.
+
+Use the links above for more information on each.
+
 # changelog
 
 * Sep 23rd, 2024 - [1.17.10](https://github.com/klauspost/compress/releases/tag/v1.17.10)

flate/fast_encoder.go

@@ -6,8 +6,9 @@
 package flate
 
 import (
-	"encoding/binary"
 	"fmt"
+
+	"github.com/klauspost/compress/internal/le"
 )
 
 type fastEnc interface {
@@ -58,11 +59,11 @@ const (
 )
 
 func load3232(b []byte, i int32) uint32 {
-	return binary.LittleEndian.Uint32(b[i:])
+	return le.Load32(b, i)
 }
 
 func load6432(b []byte, i int32) uint64 {
-	return binary.LittleEndian.Uint64(b[i:])
+	return le.Load64(b, i)
 }
 
 type tableEntry struct {

flate/fuzz_test.go

@@ -1,5 +1,4 @@
 //go:build go1.18
-// +build go1.18
 
 package flate
 

flate/huffman_bit_writer.go

@@ -5,10 +5,11 @@
 package flate
 
 import (
-	"encoding/binary"
 	"fmt"
 	"io"
 	"math"
+
+	"github.com/klauspost/compress/internal/le"
 )
 
 const (
@@ -438,7 +439,7 @@ func (w *huffmanBitWriter) writeOutBits() {
 	n := w.nbytes
 
 	// We over-write, but faster...
-	binary.LittleEndian.PutUint64(w.bytes[n:], bits)
+	le.Store64(w.bytes[n:], bits)
 	n += 6
 
 	if n >= bufferFlushSize {
@@ -854,7 +855,7 @@ func (w *huffmanBitWriter) writeTokens(tokens []token, leCodes, oeCodes []hcode)
 			bits |= c.code64() << (nbits & 63)
 			nbits += c.len()
 			if nbits >= 48 {
-				binary.LittleEndian.PutUint64(w.bytes[nbytes:], bits)
+				le.Store64(w.bytes[nbytes:], bits)
 				//*(*uint64)(unsafe.Pointer(&w.bytes[nbytes])) = bits
 				bits >>= 48
 				nbits -= 48
@@ -882,7 +883,7 @@ func (w *huffmanBitWriter) writeTokens(tokens []token, leCodes, oeCodes []hcode)
 			bits |= c.code64() << (nbits & 63)
 			nbits += c.len()
 			if nbits >= 48 {
-				binary.LittleEndian.PutUint64(w.bytes[nbytes:], bits)
+				le.Store64(w.bytes[nbytes:], bits)
 				//*(*uint64)(unsafe.Pointer(&w.bytes[nbytes])) = bits
 				bits >>= 48
 				nbits -= 48
@@ -905,7 +906,7 @@ func (w *huffmanBitWriter) writeTokens(tokens []token, leCodes, oeCodes []hcode)
 			bits |= uint64(extraLength) << (nbits & 63)
 			nbits += extraLengthBits
 			if nbits >= 48 {
-				binary.LittleEndian.PutUint64(w.bytes[nbytes:], bits)
+				le.Store64(w.bytes[nbytes:], bits)
 				//*(*uint64)(unsafe.Pointer(&w.bytes[nbytes])) = bits
 				bits >>= 48
 				nbits -= 48
@@ -931,7 +932,7 @@ func (w *huffmanBitWriter) writeTokens(tokens []token, leCodes, oeCodes []hcode)
 			bits |= c.code64() << (nbits & 63)
 			nbits += c.len()
 			if nbits >= 48 {
-				binary.LittleEndian.PutUint64(w.bytes[nbytes:], bits)
+				le.Store64(w.bytes[nbytes:], bits)
 				//*(*uint64)(unsafe.Pointer(&w.bytes[nbytes])) = bits
 				bits >>= 48
 				nbits -= 48
@@ -953,7 +954,7 @@ func (w *huffmanBitWriter) writeTokens(tokens []token, leCodes, oeCodes []hcode)
 			bits |= uint64((offset-(offsetComb>>8))&matchOffsetOnlyMask) << (nbits & 63)
 			nbits += uint8(offsetComb)
 			if nbits >= 48 {
-				binary.LittleEndian.PutUint64(w.bytes[nbytes:], bits)
+				le.Store64(w.bytes[nbytes:], bits)
 				//*(*uint64)(unsafe.Pointer(&w.bytes[nbytes])) = bits
 				bits >>= 48
 				nbits -= 48
@@ -1107,7 +1108,7 @@ func (w *huffmanBitWriter) writeBlockHuff(eof bool, input []byte, sync bool) {
 		// We must have at least 48 bits free.
 		if nbits >= 8 {
 			n := nbits >> 3
-			binary.LittleEndian.PutUint64(w.bytes[nbytes:], bits)
+			le.Store64(w.bytes[nbytes:], bits)
 			bits >>= (n * 8) & 63
 			nbits -= n * 8
 			nbytes += n
@@ -1136,7 +1137,7 @@ func (w *huffmanBitWriter) writeBlockHuff(eof bool, input []byte, sync bool) {
 	// Remaining...
 	for _, t := range input {
 		if nbits >= 48 {
-			binary.LittleEndian.PutUint64(w.bytes[nbytes:], bits)
+			le.Store64(w.bytes[nbytes:], bits)
 			//*(*uint64)(unsafe.Pointer(&w.bytes[nbytes])) = bits
 			bits >>= 48
 			nbits -= 48

flate/level1.go

@@ -1,9 +1,10 @@
 package flate
 
 import (
-	"encoding/binary"
 	"fmt"
 	"math/bits"
+
+	"github.com/klauspost/compress/internal/le"
 )
 
 // fastGen maintains the table for matches,
@@ -126,26 +127,26 @@ func (e *fastEncL1) Encode(dst *tokens, src []byte) {
 				l = e.matchlenLong(s+4, t+4, src) + 4
 			} else {
 				// inlined:
-				a := src[s+4:]
-				b := src[t+4:]
-				for len(a) >= 8 {
-					if diff := binary.LittleEndian.Uint64(a) ^ binary.LittleEndian.Uint64(b); diff != 0 {
+				a := src[s:]
+				b := src[t:]
+				left := len(a) - 4
+				for left >= 8 {
+					if diff := le.Load64(a, l) ^ le.Load64(b, l); diff != 0 {
 						l += int32(bits.TrailingZeros64(diff) >> 3)
-						break
+						goto endMatch
 					}
 					l += 8
-					a = a[8:]
-					b = b[8:]
+					left -= 8
 				}
-				if len(a) < 8 {
-					b = b[:len(a)]
-					for i := range a {
-						if a[i] != b[i] {
-							break
-						}
-						l++
+				a = a[l:]
+				b = b[l:]
+				for i := range a {
+					if a[i] != b[i] {
+						break
 					}
+					l++
 				}
+			endMatch:
 			}
 
 			// Extend backwards

flate/matchlen_generic.go

@@ -7,27 +7,31 @@
 package flate
 
 import (
-	"encoding/binary"
 	"math/bits"
+
+	"github.com/klauspost/compress/internal/le"
 )
 
 // matchLen returns the maximum common prefix length of a and b.
 // a must be the shortest of the two.
 func matchLen(a, b []byte) (n int) {
-	for ; len(a) >= 8 && len(b) >= 8; a, b = a[8:], b[8:] {
-		diff := binary.LittleEndian.Uint64(a) ^ binary.LittleEndian.Uint64(b)
+	left := len(a)
+	for left >= 8 {
+		diff := le.Load64(a, n) ^ le.Load64(b, n)
 		if diff != 0 {
 			return n + bits.TrailingZeros64(diff)>>3
 		}
 		n += 8
+		left -= 8
 	}
 
+	a = a[n:]
+	b = b[n:]
 	for i := range a {
 		if a[i] != b[i] {
 			break
 		}
 		n++
 	}
 	return n
-
 }