StringZilla 4.0!

.clang-format

@@ -6,6 +6,7 @@ NamespaceIndentation: None
 ColumnLimit: 120
 ReflowComments: true
 UseTab: Never
+IndentPPDirectives: None
 
 AlignConsecutiveAssignments: false
 AlignConsecutiveDeclarations: false
@@ -44,8 +45,8 @@ BraceWrapping:
   SplitEmptyNamespace: false
   IndentBraces: false
 
-SortIncludes: true
-SortUsingDeclarations: true
+SortIncludes: false
+SortUsingDeclarations: false
 
 SpaceAfterCStyleCast: false
 SpaceAfterLogicalNot: false

.cmake-format.py

@@ -0,0 +1,19 @@
+# -----------------------------
+# Options effecting formatting.
+# -----------------------------
+with section("format"):
+    # How wide to allow formatted cmake files
+    line_width = 120
+
+    # How many spaces to tab for indent
+    tab_size = 4
+
+    # If true, separate flow control names from their parentheses with a space
+    separate_ctrl_name_with_space = True
+
+    # If true, separate function names from parentheses with a space
+    separate_fn_name_with_space = False
+
+    # If a statement is wrapped to more than one line, than dangle the closing
+    # parenthesis on its own line.
+    dangle_parens = True

.git-blame-ignore-revs

@@ -0,0 +1,42 @@
+6512f1d129aeddc8601c9df7332c135038914b68
+fc9e5d61e5fb1c5031f6f10920f6b50e2530de1e
+ad2af78f8651870727c5b39e1fea2eff26d71d2f
+49e8d9d240993bdf68715a9c87824a032752798d
+fc408fa0a0f2d947c610568bd7a5c4a60ecca443
+b835051c09a0ecfc420932de444f3c6839610764
+1ba7982559111d4fc9b58caa7bc7aa1c6e64257c
+5b55e19d1378c61da88309b30a38f9cf7c64bf79
+be4c63d926c8628451726863e4d14dbd1ea374dd
+8b401bd41e4bd9c29c8fad9a5b83d8232efa50c7
+295d49a38d66b08075357ac829ad66d80b5edab0
+2a1fcd113d217e3124f6501c38e93a318aca37f0
+2f7652141bd8dc3c2c38ab34321567bfcdb91d93
+9e3180019acffe5261f0a1713b4ea324dca79ea0
+45e57eefd796841cbd14ee7f75ec42b42b5bde0c
+66778d6b2b3aa0eed27e32fbdceef79b8c54eda5
+c357c3ea756523d3bcc8d8f25068ad08aef5456d
+9b1948b3771c21dd56954e5f43301ca8a0b8b1a9
+cbfe5c7ac6371047eae88621b092297474d0b82a
+085d2d3c8b99e0f90d320dd027040e554e410929
+3464cb428ae9a8721ab82a8c4bff214aa9ce6254
+5d0d2da422c7df96f9613ec843cd47c579a2edce
+89c46810c2f9bfafa31f8592339f9a1b45dcc245
+3f9c248fbf59add2246055462e8fc19dc9f1693b
+e23c35ff2c2d4ccb752f4ffbf9b6f39a1677b532
+7fdc58fd26e06c41052287d47a9c729c068a95ca
+10d829efcb8ed4cfa5f2db4050f8403184484423
+d74e5dca2e62eb0078cb2ebacc0dac2b8bb92d54
+1f60e6d7c81f0e285e594eb63fee6119e05a3e69
+a6768af38b40307fe66364403f141c285b3e164c
+08d0a20d35d3b29a44b9c8a826d53435c3ef839c
+9e9f2567d052d635722921a1d70ec63d69ec6669
+974ed78822dc0b519dd61bc1c4dc18d59fe4ad15
+b007ba571860e1d3737d1478c7f8d66ae1839e36
+14ba3bf3c43408438a7de9ad57118c747c1347b1
+9e577be71dcd2e20854bf55f08c54854b3e82989
+8cb0742b2d1b31b61fac5272f17017953c6677e6
+bd547453122e9f8565e5be15f137e7b0de37caca
+22e3d1e34d62d68c1e89df7c8bdc201faa18a9de
+ecb377541d0c706cf8997faff4f026b07e3f76f3
+0d982a45f842287d7e344f0d8b360f52482017f5
+467b4b81cb4bc0e9a64844748a417762378918c9

.github/workflows/prerelease.yml

@@ -37,11 +37,11 @@ jobs:
             package.json:"version": "(\d+\.\d+\.\d+)"
             CMakeLists.txt:VERSION (\d+\.\d+\.\d+)
           update-major-version-in: |
-            include/stringzilla/stringzilla.h:^#define STRINGZILLA_VERSION_MAJOR (\d+)
+            include/stringzilla/stringzilla.h:^#define STRINGZILLA_H_VERSION_MAJOR (\d+)
           update-minor-version-in: |
-            include/stringzilla/stringzilla.h:^#define STRINGZILLA_VERSION_MINOR (\d+)
+            include/stringzilla/stringzilla.h:^#define STRINGZILLA_H_VERSION_MINOR (\d+)
           update-patch-version-in: |
-            include/stringzilla/stringzilla.h:^#define STRINGZILLA_VERSION_PATCH (\d+)
+            include/stringzilla/stringzilla.h:^#define STRINGZILLA_H_VERSION_PATCH (\d+)
           dry-run: "true"
 
   test_ubuntu_gcc:
@@ -87,10 +87,11 @@ jobs:
         run: build_artifacts/stringzilla_test_cpp20
       - name: Test on Real World Data
         run: |
+          build_artifacts/stringzilla_bench_memory ${DATASET_PATH}     # for string copies and fills
           build_artifacts/stringzilla_bench_search ${DATASET_PATH}     # for substring search
           build_artifacts/stringzilla_bench_token ${DATASET_PATH}      # for hashing, equality comparisons, etc.
           build_artifacts/stringzilla_bench_similarity ${DATASET_PATH} # for edit distances and alignment scores
-          build_artifacts/stringzilla_bench_sort ${DATASET_PATH}       # for sorting arrays of strings
+          build_artifacts/stringzilla_bench_sequence ${DATASET_PATH}   # for sorting arrays of strings
           build_artifacts/stringzilla_bench_container ${DATASET_PATH}  # for STL containers with string keys
         env:
           DATASET_PATH: ./README.md
@@ -174,7 +175,7 @@ jobs:
           build_artifacts/stringzilla_bench_search ${DATASET_PATH}     # for substring search
           build_artifacts/stringzilla_bench_token ${DATASET_PATH}      # for hashing, equality comparisons, etc.
           build_artifacts/stringzilla_bench_similarity ${DATASET_PATH} # for edit distances and alignment scores
-          build_artifacts/stringzilla_bench_sort ${DATASET_PATH}       # for sorting arrays of strings
+          build_artifacts/stringzilla_bench_sequence ${DATASET_PATH}       # for sorting arrays of strings
           build_artifacts/stringzilla_bench_container ${DATASET_PATH}  # for STL containers with string keys
         env:
           DATASET_PATH: ./README.md
@@ -275,7 +276,7 @@ jobs:
         # We can't run the produced builds, but we can make sure they exist
       - name: Test artifacts presense
         run: |
-          test -e build_artifacts/libstringzillite.so
+          test -e build_artifacts/libstringzilla_bare.so
           test -e build_artifacts/libstringzilla_shared.so
           test -e build_artifacts/stringzilla_test_cpp20
 
@@ -306,7 +307,7 @@ jobs:
           build_artifacts/stringzilla_bench_search ${DATASET_PATH}     # for substring search
           build_artifacts/stringzilla_bench_token ${DATASET_PATH}      # for hashing, equality comparisons, etc.
           build_artifacts/stringzilla_bench_similarity ${DATASET_PATH} # for edit distances and alignment scores
-          build_artifacts/stringzilla_bench_sort ${DATASET_PATH}       # for sorting arrays of strings
+          build_artifacts/stringzilla_bench_sequence ${DATASET_PATH}       # for sorting arrays of strings
           build_artifacts/stringzilla_bench_container ${DATASET_PATH}  # for STL containers with string keys
         env:
           DATASET_PATH: ./README.md
@@ -379,7 +380,7 @@ jobs:
           .\build_artifacts\stringzilla_bench_search.exe ${DATASET_PATH}     # for substring search
           .\build_artifacts\stringzilla_bench_token.exe ${DATASET_PATH}      # for hashing, equality comparisons, etc.
           .\build_artifacts\stringzilla_bench_similarity.exe ${DATASET_PATH} # for edit distances and alignment scores
-          .\build_artifacts\stringzilla_bench_sort.exe ${DATASET_PATH}       # for sorting arrays of strings
+          .\build_artifacts\stringzilla_bench_sequence.exe ${DATASET_PATH}       # for sorting arrays of strings
           .\build_artifacts\stringzilla_bench_container.exe ${DATASET_PATH}  # for STL containers with string keys
         env:
           DATASET_PATH: ./README.md

.github/workflows/release.yml

@@ -37,11 +37,11 @@ jobs:
             package.json:"version": "(\d+\.\d+\.\d+)"
             CMakeLists.txt:VERSION (\d+\.\d+\.\d+)
           update-major-version-in: |
-            include/stringzilla/stringzilla.h:^#define STRINGZILLA_VERSION_MAJOR (\d+)
+            include/stringzilla/stringzilla.h:^#define STRINGZILLA_H_VERSION_MAJOR (\d+)
           update-minor-version-in: |
-            include/stringzilla/stringzilla.h:^#define STRINGZILLA_VERSION_MINOR (\d+)
+            include/stringzilla/stringzilla.h:^#define STRINGZILLA_H_VERSION_MINOR (\d+)
           update-patch-version-in: |
-            include/stringzilla/stringzilla.h:^#define STRINGZILLA_VERSION_PATCH (\d+)
+            include/stringzilla/stringzilla.h:^#define STRINGZILLA_H_VERSION_PATCH (\d+)
           dry-run: "false"
           push: "true"
           create-release: "true"
@@ -253,15 +253,15 @@ jobs:
 
           cmake --build build_release --config Release
 
-          cp build_release/libstringzillite.so "stringzillite_linux_${{ matrix.arch }}_${{ steps.set_version.outputs.version }}.so"
-          mkdir -p "stringzillite_linux_${{ matrix.arch }}_${{ steps.set_version.outputs.version }}/DEBIAN"
-          touch "stringzillite_linux_${{ matrix.arch }}_${{ steps.set_version.outputs.version }}/DEBIAN/control"
-          mkdir -p "stringzillite_linux_${{ matrix.arch }}_${{ steps.set_version.outputs.version }}/usr/local/lib"
-          mkdir "stringzillite_linux_${{ matrix.arch }}_${{ steps.set_version.outputs.version }}/usr/local/include"
-          cp include/stringzilla/stringzilla.h "stringzillite_linux_${{ matrix.arch }}_${{ steps.set_version.outputs.version }}/usr/local/include/"
-          cp build_release/libstringzillite.so "stringzillite_linux_${{ matrix.arch }}_${{ steps.set_version.outputs.version }}/usr/local/lib/"
-          echo -e "Package: stringzilla\nVersion: ${{ steps.set_version.outputs.version }}\nMaintainer: Ash Vardanian\nArchitecture: ${{ matrix.arch }}\nDescription: SIMD-accelerated string search, sort, hashes, fingerprints, & edit distances" > "stringzillite_linux_${{ matrix.arch }}_${{ steps.set_version.outputs.version }}/DEBIAN/control"
-          dpkg-deb --build "stringzillite_linux_${{ matrix.arch }}_${{ steps.set_version.outputs.version }}"
+          cp build_release/libstringzilla_bare.so "stringzilla_bare_linux_${{ matrix.arch }}_${{ steps.set_version.outputs.version }}.so"
+          mkdir -p "stringzilla_bare_linux_${{ matrix.arch }}_${{ steps.set_version.outputs.version }}/DEBIAN"
+          touch "stringzilla_bare_linux_${{ matrix.arch }}_${{ steps.set_version.outputs.version }}/DEBIAN/control"
+          mkdir -p "stringzilla_bare_linux_${{ matrix.arch }}_${{ steps.set_version.outputs.version }}/usr/local/lib"
+          mkdir "stringzilla_bare_linux_${{ matrix.arch }}_${{ steps.set_version.outputs.version }}/usr/local/include"
+          cp include/stringzilla/stringzilla.h "stringzilla_bare_linux_${{ matrix.arch }}_${{ steps.set_version.outputs.version }}/usr/local/include/"
+          cp build_release/libstringzilla_bare.so "stringzilla_bare_linux_${{ matrix.arch }}_${{ steps.set_version.outputs.version }}/usr/local/lib/"
+          echo -e "Package: stringzilla\nVersion: ${{ steps.set_version.outputs.version }}\nMaintainer: Ash Vardanian\nArchitecture: ${{ matrix.arch }}\nDescription: SIMD-accelerated string search, sort, hashes, fingerprints, & edit distances" > "stringzilla_bare_linux_${{ matrix.arch }}_${{ steps.set_version.outputs.version }}/DEBIAN/control"
+          dpkg-deb --build "stringzilla_bare_linux_${{ matrix.arch }}_${{ steps.set_version.outputs.version }}"
 
       - name: Upload library
         uses: xresloader/upload-to-github-release@v1
@@ -314,14 +314,14 @@ jobs:
         run: |
           cmake -DCMAKE_BUILD_TYPE=Release -B build_release
           cmake --build build_release --config Release
-          tar -cvf "stringzillite_windows_${{ matrix.arch }}_${{ steps.set_version.outputs.version }}.tar" "build_release/stringzillite.dll" "./include/stringzilla/stringzilla.h"
+          tar -cvf "stringzilla_bare_windows_${{ matrix.arch }}_${{ steps.set_version.outputs.version }}.tar" "build_release/stringzilla_bare.dll" "./include/stringzilla/stringzilla.h"
 
       - name: Upload archive
         uses: xresloader/upload-to-github-release@v1
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         with:
-          file: "stringzillite_windows_${{ matrix.arch }}_${{ steps.set_version.outputs.version }}.tar"
+          file: "stringzilla_bare_windows_${{ matrix.arch }}_${{ steps.set_version.outputs.version }}.tar"
           update_latest_release: true
 
   create_macos_library:
@@ -349,7 +349,7 @@ jobs:
         run: |
           cmake -DCMAKE_BUILD_TYPE=Release -B build_release
           cmake --build build_release --config Release        
-          zip -r stringzillite_macos_${{ matrix.arch }}_${{ steps.set_version.outputs.version }}.zip build_release/libstringzillite.dylib include/stringzilla/stringzilla.h
+          zip -r stringzilla_bare_macos_${{ matrix.arch }}_${{ steps.set_version.outputs.version }}.zip build_release/libstringzilla_bare.dylib include/stringzilla/stringzilla.h
 
       - name: Upload archive
         uses: xresloader/upload-to-github-release@v1

.gitignore

@@ -32,10 +32,17 @@ node_modules/
 *.tar.gz
 
 # Recommended datasets
+utf8.txt
 leipzig1M.txt
 enwik9.txt
 xlsum.csv
-human_protein_1200row_800len.txt
+acgt_100.txt
+acgt_100k.txt
+acgt_10k.txt
+acgt_10m.txt
+acgt_1k.txt
+acgt_1m.txt
 
 # StringZilla-specific log files
-/failed_sz_*
+/failed_sz_*
+/.cache/

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "fork_union"]
+	path = fork_union
+	url = https://github.com/ashvardanian/fork_union.git