sift4

Author: nikolaydubina

.github/workflows/test.yml

@@ -0,0 +1,34 @@
+name: Test
+
+on:
+  push:
+    branches: [master]
+  pull_request:
+    branches: [master]
+
+permissions: read-all
+
+jobs:
+  build:
+    name: Test
+    runs-on: ubuntu-latest
+    steps:
+      - name: code
+        uses: actions/checkout@v4
+
+      - name: go
+        uses: actions/setup-go@v5
+        with:
+          go-version: ^1.24
+
+      - name: test
+        run: go test -coverprofile=coverage.out -covermode=atomic -cover ./...
+
+      - name: fuzz
+        run: go test -list . | grep Fuzz | xargs -P 8 -I {} go test -fuzz {} -fuzztime 30s .
+
+      - name: Upload coverage to Codecov
+        uses: codecov/[email protected]
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          files: coverage.out

README.md

@@ -1 +1,24 @@
-# sift4
+SIFT4 — fast approximate string distance algorithm
+
+* zero memory copy
+* 100% test coverage
+
+```bash
+$ go test -benchmem -bench .
+goos: darwin
+goarch: arm64
+pkg: github.com/ndx-technologies/sift4
+cpu: Apple M3 Max
+BenchmarkSIFT4Distance/empty-16                         956523358                1.109 ns/op           0 B/op          0 allocs/op
+BenchmarkSIFT4Distance/one_empty-16                     1000000000               1.093 ns/op           0 B/op          0 allocs/op
+BenchmarkSIFT4Distance/equal-16                         562000238                2.137 ns/op           0 B/op          0 allocs/op
+BenchmarkSIFT4Distance/different-16                     16788264                71.80 ns/op           48 B/op          2 allocs/op
+BenchmarkSIFT4Distance/long_different-16                 1488578               809.1 ns/op            24 B/op          1 allocs/op
+BenchmarkSIFT4Distance/buffer/empty-16                  1000000000               1.122 ns/op           0 B/op          0 allocs/op
+BenchmarkSIFT4Distance/buffer/one_empty-16              1000000000               1.116 ns/op           0 B/op          0 allocs/op
+BenchmarkSIFT4Distance/buffer/equal-16                  552020344                2.179 ns/op           0 B/op          0 allocs/op
+BenchmarkSIFT4Distance/buffer/different-16              29124799                41.24 ns/op            0 B/op          0 allocs/op
+BenchmarkSIFT4Distance/buffer/long_different-16          1520445               789.8 ns/op             0 B/op          0 allocs/op
+PASS
+ok      github.com/ndx-technologies/sift4       11.848s
+```

go.mod

@@ -0,0 +1,3 @@
+module github.com/ndx-technologies/sift4
+
+go 1.24.0

sfit4_test.go

@@ -0,0 +1,129 @@
+package sift4_test
+
+import (
+	"fmt"
+	"strings"
+	"testing"
+
+	"github.com/ndx-technologies/sift4"
+)
+
+func ExampleDistance() {
+	d := sift4.Distance("kitten", "sitting", 100, 5, nil)
+	fmt.Print(d)
+	// Output: 3
+}
+
+func ExampleDistance_buffer() {
+	var b sift4.Buffer
+	d := sift4.Distance("kitten", "sitting", 100, 5, &b)
+	fmt.Print(d)
+	// Output: 3
+}
+
+func TestDistance(t *testing.T) {
+	tests := []struct {
+		s1          string
+		s2          string
+		maxOffset   int
+		maxDistance int
+		distance    int
+	}{
+		{"kitten", "sitting", 100, 5, 3},
+		{"book", "back", 100, 5, 2},
+		{"", "abc", 100, 5, 3},
+		{"abc", "", 100, 5, 3},
+		{"", "", 100, 5, 0},
+		{"a", "a", 100, 5, 0},
+		{"a", "b", 100, 5, 1},
+		{"ab", "abc", 100, 5, 1},
+		{"abc", "ab", 100, 5, 1},
+		{"abc", "def", 100, 5, 3},
+		{"hello", "helo", 100, 5, 1},
+		{"world", "word", 100, 5, 1},
+		{"halooooxo", "hbloooogo", 100, 5, 6},
+
+		// early exit not reached
+		{"distance", "difference", 100, 5, 6},
+		{"abcdef", "xyz", 100, 2, 3},
+		{"abcdefabcdefabcdefabcdefabcdefabcdef", "xyz", 100, 2, 3},
+
+		// transposition
+		{"abc", "acb", 100, 5, 1}, // Damerau–Levenshtein distance, transposition of adjacent characters is one operation
+		{"ab", "ba", 100, 5, 1},
+		{"abcd", "badc", 100, 5, 2}, // two transpositions
+		{"abc", "acb", 100, 5, 1},
+		{"aab", "baa", 100, 5, 1},   // covers cursor adjustment when s1[c1] == s2[c2+i]
+		{"abcd", "cdab", 100, 5, 2}, // transposition test with cyclic shift
+		{"01", "11", 100, 5, 1},
+		{"00010", "000010", 100, 5, 2},
+	}
+
+	for _, tc := range tests {
+		t.Run("", func(t *testing.T) {
+			var buf sift4.Buffer
+			d := sift4.Distance(tc.s1, tc.s2, tc.maxOffset, tc.maxDistance, &buf)
+			if d != tc.distance {
+				t.Error(tc, d)
+			}
+		})
+	}
+}
+
+func Benchmark_______________________________________(b *testing.B) {}
+
+func BenchmarkSIFT4Distance(b *testing.B) {
+	testCases := []struct {
+		name string
+		s1   string
+		s2   string
+	}{
+		{"empty", "", ""},
+		{"one empty", "hello", ""},
+		{"equal", "kitten", "kitten"},
+		{"different", "kitten", "sitting"},
+		{"long different", strings.Repeat("a", 256), strings.Repeat("b", 256)},
+	}
+	for _, tc := range testCases {
+		b.Run(tc.name, func(b *testing.B) {
+			for b.Loop() {
+				sift4.Distance(tc.s1, tc.s2, 100, 5, nil)
+			}
+		})
+	}
+
+	b.Run("buffer", func(b *testing.B) {
+		for _, tc := range testCases {
+			b.Run(tc.name, func(b *testing.B) {
+				var buffer sift4.Buffer
+				for b.Loop() {
+					sift4.Distance(tc.s1, tc.s2, 100, 5, &buffer)
+				}
+			})
+		}
+	})
+}
+
+func FuzzSIFT4Distance(f *testing.F) {
+	f.Add("", "")
+	f.Add("hello", "")
+	f.Add("", "world")
+	f.Add("kitten", "sitting")
+
+	f.Fuzz(func(t *testing.T, s1, s2 string) {
+		d := sift4.Distance(s1, s2, 100, 5, nil)
+
+		if d < 0 {
+			t.Error("d < 0")
+		}
+		if s1 == s2 && d != 0 {
+			t.Error(d)
+		}
+		if s1 == "" && d != len(s2) {
+			t.Error(d)
+		}
+		if s2 == "" && d != len(s1) {
+			t.Error(d)
+		}
+	})
+}

sift4.go

@@ -0,0 +1,125 @@
+package sift4
+
+import "slices"
+
+// Buffer stores temporary structures and can be reused in multiple runs.
+// Zero value is safe to use.
+type Buffer struct {
+	offset []offset
+}
+
+type offset struct {
+	c1    int
+	c2    int
+	trans bool
+}
+
+// Distance is fast O(N) approximate string distance algorithm.
+// If pointer to buffer is provided, then it will be reused for storing temporary structures.
+//   - maxOffset is the number of characters to search for matching letters
+//   - maxDistance is the distance at which the algorithm should stop computing the value and just exit (the strings are too different anyway)
+//
+// https://siderite.dev/blog/super-fast-and-accurate-string-distance.html
+func Distance(s1, s2 string, maxOffset, maxDistance int, buffer *Buffer) int {
+	if len(s1) == 0 {
+		return len(s2)
+	}
+	if len(s2) == 0 {
+		return len(s1)
+	}
+	if s1 == s2 {
+		return 0
+	}
+
+	var (
+		c1, c2  int // cursor for string 1 and 2
+		lcss    int // largest common subsequence
+		localCS int // local common substring
+		trans   int // number of transpositions
+	)
+
+	if buffer == nil {
+		buffer = &Buffer{}
+	}
+	buffer.offset = buffer.offset[:0]
+
+	for (c1 < len(s1)) && (c2 < len(s2)) {
+		if s1[c1] == s2[c2] {
+			isTrans := false
+
+			localCS++
+
+			for i := 0; i < len(buffer.offset); {
+				ofs := buffer.offset[i]
+				if c1 <= ofs.c1 || c2 <= ofs.c2 {
+					isTrans = abs(c2-c1) >= abs(ofs.c2-ofs.c1)
+					if isTrans {
+						trans++
+					} else if !ofs.trans {
+						ofs.trans = true
+						trans++
+					}
+					break
+				} else {
+					if c1 > ofs.c2 && c2 > ofs.c1 {
+						buffer.offset = slices.Delete(buffer.offset, i, i+1)
+					} else {
+						i++
+					}
+				}
+			}
+
+			buffer.offset = append(buffer.offset, offset{c1, c2, isTrans})
+		} else {
+			lcss += localCS
+			localCS = 0
+
+			if c1 != c2 {
+				c1 = min(c1, c2) // using min allows the computation of transpositions
+				c2 = c1
+			}
+
+			// if matching characters are found, remove 1 from both cursors (they get incremented at the end of the loop)
+			// so that we can have only one code block handling matches
+			for i := 0; i < maxOffset && (c1+i < len(s1) || c2+i < len(s2)); i++ {
+				if c1+i < len(s1) && s1[c1+i] == s2[c2] {
+					c1 += i - 1
+					c2--
+					break
+				}
+				if c2+i < len(s2) && s1[c1] == s2[c2+i] {
+					c1--
+					c2 += i - 1
+					break
+				}
+			}
+		}
+
+		c1++
+		c2++
+
+		if maxDistance > 0 {
+			if d := max(c1, c2) - lcss + trans; d > maxDistance {
+				return d
+			}
+		}
+
+		// this covers the case where the last match is on the last token in list, so that it can compute transpositions correctly
+		if c1 >= len(s1) || c2 >= len(s2) {
+			lcss += localCS
+			localCS = 0
+			c1 = min(c1, c2)
+			c2 = c1
+		}
+	}
+
+	lcss += localCS
+	return max(len(s1), len(s2)) - lcss + trans // add the cost of transpositions to the final result
+}
+
+func abs[T int](x T) T {
+	if x < 0 {
+		return -x
+	}
+	return x
+}