Merge branch 'main' into impl/hack/implement-deadlink-checker

Signed-off-by: Yusuke Kato <[email protected]>

.gitfiles

@@ -73,7 +73,7 @@
 .github/workflows/chatops-help.yaml
 .github/workflows/chatops.yaml
 .github/workflows/check-conflict.yaml
-.github/workflows/codeql-analysis.yaml
+.github/workflows/codeql-analysis.yml
 .github/workflows/coverage.yaml
 .github/workflows/detect-internal-config-changes.yaml
 .github/workflows/dockers-agent-faiss-image.yaml
@@ -167,6 +167,8 @@ apis/grpc/v1/filter/egress/egress_filter.pb.go
 apis/grpc/v1/filter/egress/egress_filter_vtproto.pb.go
 apis/grpc/v1/filter/ingress/ingress_filter.pb.go
 apis/grpc/v1/filter/ingress/ingress_filter_vtproto.pb.go
+apis/grpc/v1/meta/meta.pb.go
+apis/grpc/v1/meta/meta_vtproto.pb.go
 apis/grpc/v1/mirror/mirror.go
 apis/grpc/v1/mirror/mirror.pb.go
 apis/grpc/v1/mirror/mirror_vtproto.pb.go
@@ -203,6 +205,7 @@ apis/proto/v1/agent/sidecar/sidecar.proto
 apis/proto/v1/discoverer/discoverer.proto
 apis/proto/v1/filter/egress/egress_filter.proto
 apis/proto/v1/filter/ingress/ingress_filter.proto
+apis/proto/v1/meta/meta.proto
 apis/proto/v1/mirror/mirror.proto
 apis/proto/v1/payload/payload.proto
 apis/proto/v1/rpc/errdetails/error_details.proto
@@ -220,6 +223,7 @@ apis/swagger/v1/agent/sidecar/sidecar.swagger.json
 apis/swagger/v1/discoverer/discoverer.swagger.json
 apis/swagger/v1/filter/egress/egress_filter.swagger.json
 apis/swagger/v1/filter/ingress/ingress_filter.swagger.json
+apis/swagger/v1/meta/meta.swagger.json
 apis/swagger/v1/mirror/mirror.swagger.json
 apis/swagger/v1/payload/payload.swagger.json
 apis/swagger/v1/rpc/errdetails/error_details.swagger.json
@@ -882,6 +886,9 @@ internal/core/algorithm/ngt/ngt_bench_test.go
 internal/core/algorithm/ngt/ngt_test.go
 internal/core/algorithm/ngt/option.go
 internal/core/algorithm/ngt/option_test.go
+internal/core/algorithm/usearch/option.go
+internal/core/algorithm/usearch/usearch.go
+internal/core/algorithm/usearch/usearch_test.go
 internal/db/kvs/bbolt/bbolt.go
 internal/db/kvs/bbolt/bbolt_test.go
 internal/db/kvs/bbolt/option.go
@@ -1024,6 +1031,7 @@ internal/errors/storage.go
 internal/errors/tls.go
 internal/errors/unit.go
 internal/errors/unit_test.go
+internal/errors/usearch.go
 internal/errors/vald.go
 internal/errors/vald_test.go
 internal/errors/vqueue.go
@@ -1911,6 +1919,10 @@ rust/bin/agent/src/handler/search.rs
 rust/bin/agent/src/handler/update.rs
 rust/bin/agent/src/handler/upsert.rs
 rust/bin/agent/src/main.rs
+rust/bin/meta/Cargo.toml
+rust/bin/meta/src/handler.rs
+rust/bin/meta/src/handler/meta.rs
+rust/bin/meta/src/main.rs
 rust/libs/algorithm/Cargo.toml
 rust/libs/algorithm/src/lib.rs
 rust/libs/algorithms/faiss/Cargo.toml
@@ -1931,6 +1943,7 @@ rust/libs/proto/src/discoverer.v1.tonic.rs
 rust/libs/proto/src/filter.egress.v1.tonic.rs
 rust/libs/proto/src/filter.ingress.v1.tonic.rs
 rust/libs/proto/src/lib.rs
+rust/libs/proto/src/meta.v1.tonic.rs
 rust/libs/proto/src/mirror.v1.tonic.rs
 rust/libs/proto/src/payload.v1.rs
 rust/libs/proto/src/rpc.v1.rs
@@ -1986,6 +1999,7 @@ versions/PROTOBUF_VERSION
 versions/REVIEWDOG_VERSION
 versions/RUST_VERSION
 versions/TELEPRESENCE_VERSION
+versions/USEARCH_VERSION
 versions/VALD_VERSION
 versions/YQ_VERSION
 versions/ZLIB_VERSION

.github/ISSUE_TEMPLATE/bug_report.md

@@ -25,8 +25,8 @@ assignees: ""
 - Vald Version: v1.7.13
 - Go Version: v1.23.1
 - Rust Version: v1.81.0
-- Docker Version: v27.2.1
-- Kubernetes Version: v1.31.0
-- Helm Version: v3.16.0
+- Docker Version: v27.3.1
+- Kubernetes Version: v1.31.1
+- Helm Version: v3.16.1
 - NGT Version: v2.2.4
 - Faiss Version: v1.8.0

.github/ISSUE_TEMPLATE/security_issue_report.md

@@ -19,8 +19,8 @@ assignees: ""
 - Vald Version: v1.7.13
 - Go Version: v1.23.1
 - Rust Version: v1.81.0
-- Docker Version: v27.2.1
-- Kubernetes Version: v1.31.0
-- Helm Version: v3.16.0
+- Docker Version: v27.3.1
+- Kubernetes Version: v1.31.1
+- Helm Version: v3.16.1
 - NGT Version: v2.2.4
 - Faiss Version: v1.8.0

.github/PULL_REQUEST_TEMPLATE.md

@@ -3,7 +3,7 @@
 ### Description
 
 <!-- Describe your changes in detail -->
-<!-- It would be better to describe the details especially What changed and Why you changed -->
+<!-- It would be better to describe the details, especially What changed and Why you changed -->
 
 ### Related Issue
 
@@ -18,9 +18,9 @@
 - Vald Version: v1.7.13
 - Go Version: v1.23.1
 - Rust Version: v1.81.0
-- Docker Version: v27.2.1
-- Kubernetes Version: v1.31.0
-- Helm Version: v3.16.0
+- Docker Version: v27.3.1
+- Kubernetes Version: v1.31.1
+- Helm Version: v3.16.1
 - NGT Version: v2.2.4
 - Faiss Version: v1.8.0
 
@@ -34,4 +34,4 @@
 
 ### Special notes for your reviewer
 
-<!-- Please tell us anything you would like to share to reviewers related this PR -->
+<!-- Please tell us anything you would like to share with reviewers related to this PR. Your thoughts and feedback are highly valued -->

.github/workflows/e2e-profiling.yaml

@@ -59,12 +59,14 @@ jobs:
           values: .github/helm/values/values-profile.yaml
           wait_for_selector: app=vald-lb-gateway
       - name: Deploy profefe
+        shell: bash
         run: |
           make k8s/metrics/profefe/deploy
           kubectl patch cronjob kprofefe -p '{"spec": {"schedule": "*/1 * * * *"}}'
           kubectl wait --for=condition=ready pod -l app=profefe --timeout=300s
       - name: Run E2E CRUD
         continue-on-error: true
+        shell: bash
         run: |
           make hack/benchmark/assets/dataset/${{ env.DATASET }}
           make E2E_BIND_PORT=8081 \
@@ -83,6 +85,7 @@ jobs:
         env:
           POD_NAME: ${{ steps.deploy_vald.outputs.POD_NAME }}
       - name: Get profiles
+        shell: bash
         run: |
           mkdir -p profiles
           kubectl port-forward deployment/profefe 10100:10100 &
@@ -151,6 +154,7 @@ jobs:
           path: graphs/*
       - name: Upload to vald-ci-images repository
         if: github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-profiling'
+        shell: bash
         run: |
           CLONE_DIR=$(mktemp -d)
           git clone --depth 1 https://${GITHUB_USER}:${GITHUB_TOKEN}@github.com/vdaas-ci/vald-ci-images.git $CLONE_DIR
@@ -166,6 +170,7 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.DISPATCH_TOKEN }}
       - name: Comment
         if: github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-profiling'
+        shell: bash
         run: |
           base="https://raw.githubusercontent.com/vdaas-ci/vald-ci-images/main/${GITHUB_SHA::6}"
           body="<table><tr><th>type</th>"

.github/workflows/e2e.yaml

@@ -220,7 +220,7 @@ jobs:
             E2E_UPDATE_COUNT=10 \
             E2E_UPSERT_COUNT=10 \
             E2E_REMOVE_COUNT=10 \
-            E2E_WAIT_FOR_CREATE_INDEX_DURATION=3m \
+            E2E_WAIT_FOR_CREATE_INDEX_DURATION=5m \
             E2E_TARGET_POD_NAME=${POD_NAME} \
             E2E_TARGET_NAMESPACE=default \
             e2e/skip

Makefile

@@ -674,7 +674,7 @@ $(LIB_PATH)/libfaiss.a:
 		-DBUILD_STATIC_EXECS=ON \
 		-DBUILD_TESTING=OFF \
 		-DFAISS_ENABLE_PYTHON=OFF \
-	        -DFAISS_ENABLE_GPU=OFF \
+		-DFAISS_ENABLE_GPU=OFF \
 		-DBLA_VENDOR=OpenBLAS \
 		-DCMAKE_C_FLAGS="$(LDFLAGS)" \
 		-DCMAKE_EXE_LINKER_FLAGS="$(FAISS_LDFLAGS)" \
@@ -688,19 +688,32 @@ $(LIB_PATH)/libfaiss.a:
 
 .PHONY: usearch/install
 ## install usearch
-usearch/install:
-ifeq ($(OS),linux)
-	curl -sSL https://github.com/unum-cloud/usearch/releases/download/v$(USEARCH_VERSION)/usearch_$(OS)_$(GOARCH)_$(USEARCH_VERSION).deb -o usearch_$(OS)_$(USEARCH_VERSION).deb
-	dpkg -i usearch_$(OS)_$(USEARCH_VERSION).deb
-	rm usearch_$(OS)_$(USEARCH_VERSION).deb
-	ldconfig
-else ifeq ($(OS),macos)
-	curl -sSL https://github.com/unum-cloud/usearch/releases/download/v$(USEARCH_VERSION)/usearch_macos_$(GOARCH)_$(USEARCH_VERSION).zip -o usearch_macos_$(OS)_$(USEARCH_VERSION).zip
-	unzip usearch_macos_$(OS)_$(USEARCH_VERSION).zip
-	sudo mv libusearch_c.dylib /usr/local/lib && sudo mv usearch.h /usr/local/include
-	rm -rf usearch_macos_$(OS)_$(USEARCH_VERSION).zip
+usearch/install: $(USR_LOCAL)/include/usearch.h
+$(USR_LOCAL)/include/usearch.h:
+	git clone --depth 1 --recursive --branch v$(USEARCH_VERSION) https://github.com/unum-cloud/usearch $(TEMP_DIR)/usearch-$(USEARCH_VERSION)
+	cd $(TEMP_DIR)/usearch-$(USEARCH_VERSION) && \
+	cmake -DCMAKE_BUILD_TYPE=Release \
+		-DBUILD_SHARED_LIBS=OFF \
+		-DBUILD_TESTING=OFF \
+		-DUSEARCH_BUILD_LIB_C=ON \
+		-DUSEARCH_USE_FP16LIB=ON \
+		-DUSEARCH_USE_OPENMP=ON \
+		-DUSEARCH_USE_SIMSIMD=ON \
+		-DUSEARCH_USE_JEMALLOC=ON \
+		-DCMAKE_C_FLAGS="$(CFLAGS)" \
+		-DCMAKE_CXX_FLAGS="$(CXXFLAGS)" \
+		-DCMAKE_INSTALL_PREFIX=$(USR_LOCAL) \
+		-DCMAKE_INSTALL_LIBDIR=$(LIB_PATH) \
+		-B $(TEMP_DIR)/usearch-$(USEARCH_VERSION)/build $(TEMP_DIR)/usearch-$(USEARCH_VERSION)
+	cmake --build $(TEMP_DIR)/usearch-$(USEARCH_VERSION)/build -j$(CORES)
+	cmake --install $(TEMP_DIR)/usearch-$(USEARCH_VERSION)/build --prefix=$(USR_LOCAL)
+	cd $(ROOTDIR)
+	cp $(TEMP_DIR)/usearch-$(USEARCH_VERSION)/build/libusearch_static_c.a $(LIB_PATH)/libusearch_c.a
+	cp $(TEMP_DIR)/usearch-$(USEARCH_VERSION)/build/libusearch_static_c.a $(LIB_PATH)/libusearch_static_c.a
+	cp $(TEMP_DIR)/usearch-$(USEARCH_VERSION)/build/libusearch_c.so $(LIB_PATH)/libusearch_c.so
+	cp $(TEMP_DIR)/usearch-$(USEARCH_VERSION)/c/usearch.h $(USR_LOCAL)/include/usearch.h
+	rm -rf $(TEMP_DIR)/usearch-$(USEARCH_VERSION)
 	ldconfig
-endif
 
 .PHONY: cmake/install
 ## install CMAKE
@@ -719,7 +732,6 @@ cmake/install:
 	cd $(ROOTDIR)
 	rm -rf $(TEMP_DIR)/CMAKE-$(CMAKE_VERSION)
 	ldconfig
-	# -DCMAKE_USE_OPENSSL=OFF
 
 .PHONY: lint
 ## run lints

apis/grpc/v1/agent/sidecar/sidecar_vtproto.pb.go

@@ -36,7 +36,7 @@ const _ = grpc.SupportPackageIsVersion7
 // SidecarClient is the client API for Sidecar service.
 //
 // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream.
-type SidecarClient interface{}
+type SidecarClient any
 
 type sidecarClient struct {
 	cc grpc.ClientConnInterface