Merge branch 'main' into feature/buf/migrate-to-v2

.github/ISSUE_TEMPLATE/bug_report.md

@@ -27,6 +27,6 @@ assignees: ""
 - Rust Version: v1.81.0
 - Docker Version: v27.3.1
 - Kubernetes Version: v1.31.1
-- Helm Version: v3.16.1
+- Helm Version: v3.16.2
 - NGT Version: v2.2.4
-- Faiss Version: v1.8.0
+- Faiss Version: v1.9.0

.github/ISSUE_TEMPLATE/security_issue_report.md

@@ -21,6 +21,6 @@ assignees: ""
 - Rust Version: v1.81.0
 - Docker Version: v27.3.1
 - Kubernetes Version: v1.31.1
-- Helm Version: v3.16.1
+- Helm Version: v3.16.2
 - NGT Version: v2.2.4
-- Faiss Version: v1.8.0
+- Faiss Version: v1.9.0

.github/PULL_REQUEST_TEMPLATE.md

@@ -20,9 +20,9 @@
 - Rust Version: v1.81.0
 - Docker Version: v27.3.1
 - Kubernetes Version: v1.31.1
-- Helm Version: v3.16.1
+- Helm Version: v3.16.2
 - NGT Version: v2.2.4
-- Faiss Version: v1.8.0
+- Faiss Version: v1.9.0
 
 ### Checklist
 

Makefile

@@ -394,6 +394,28 @@ help:
 	{ lastLine = $$0 }' $(MAKELISTS) | sort -u
 	@printf "\n"
 
+.PHONY: perm
+## set correct permissions for dirs and files
+perm:
+	find $(ROOTDIR) -type d -not -path "$(ROOTDIR)/.git*" -exec chmod 755 {} \;
+	find $(ROOTDIR) -type f -not -path "$(ROOTDIR)/.git*" -not -name ".gitignore" -exec chmod 644 {} \;
+	if [ -d "$(ROOTDIR)/.git" ]; then \
+		chmod 750 "$(ROOTDIR)/.git"; \
+		if [ -f "$(ROOTDIR)/.git/config" ]; then \
+			chmod 644 "$(ROOTDIR)/.git/config"; \
+		fi; \
+		if [ -d "$(ROOTDIR)/.git/hooks" ]; then \
+			find "$(ROOTDIR)/.git/hooks" -type f -exec chmod 755 {} \;; \
+		fi; \
+	fi
+	if [ -f "$(ROOTDIR)/.gitignore" ]; then \
+		chmod 644 "$(ROOTDIR)/.gitignore"; \
+	fi
+	if [ -f "$(ROOTDIR)/.gitattributes" ]; then \
+		chmod 644 "$(ROOTDIR)/.gitattributes"; \
+	fi
+
+
 .PHONY: all
 ## execute clean and deps
 all: clean deps

charts/vald/values.yaml

@@ -238,16 +238,31 @@ defaults:
             # @schema {"name": "defaults.server_config.servers.grpc.server.grpc.header_table_size", "type": "integer"}
             # defaults.server_config.servers.grpc.server.grpc.header_table_size -- gRPC server header table size
             header_table_size: 0
+            # @schema {"name": "defaults.server_config.servers.grpc.server.grpc.max_concurrent_streams", "type": "integer"}
+            # defaults.server_config.servers.grpc.server.grpc.max_concurrent_streams -- gRPC server max concurrent stream size
+            max_concurrent_streams: 0
+            # @schema {"name": "defaults.server_config.servers.grpc.server.grpc.num_stream_workers", "type": "integer"}
+            # defaults.server_config.servers.grpc.server.grpc.num_stream_workers -- gRPC server number of stream workers
+            num_stream_workers: 0
             # @schema {"name": "defaults.server_config.servers.grpc.server.grpc.interceptors", "type": "array", "items": {"type": "string", "enum": ["RecoverInterceptor", "AccessLogInterceptor", "TraceInterceptor", "MetricInterceptor"]}}
             # defaults.server_config.servers.grpc.server.grpc.interceptors -- gRPC server interceptors
             interceptors:
               - "RecoverInterceptor"
+            # @schema {"name": "defaults.server_config.servers.grpc.server.grpc.shared_write_buffer", "type": "boolean"}
+            # defaults.server_config.servers.grpc.server.grpc.shared_write_buffer -- gRPC server write buffer sharing option
+            shared_write_buffer: false
+            # @schema {"name": "defaults.server_config.servers.grpc.server.grpc.wait_for_handlers", "type": "boolean"}
+            # defaults.server_config.servers.grpc.server.grpc.wait_for_handlers -- gRPC server wait for handlers when stop
+            wait_for_handlers: true
             # @schema {"name": "defaults.server_config.servers.grpc.server.grpc.enable_reflection", "type": "boolean"}
             # defaults.server_config.servers.grpc.server.grpc.enable_reflection -- gRPC server reflection option
             enable_reflection: true
             # @schema {"name": "defaults.server_config.servers.grpc.server.grpc.enable_admin", "type": "boolean"}
             # defaults.server_config.servers.grpc.server.grpc.enable_admin -- gRPC server admin option
             enable_admin: true
+            # @schema {"name": "defaults.server_config.servers.grpc.server.grpc.enable_channelz", "type": "boolean"}
+            # defaults.server_config.servers.grpc.server.grpc.enable_channelz -- gRPC server channelz option
+            enable_channelz: true
           # @schema {"name": "defaults.server_config.servers.grpc.server.socket_option", "alias": "socket_option"}
           socket_option:
             # defaults.server_config.servers.grpc.server.socket_option.reuse_port -- server listen socket option for reuse_port functionality
@@ -738,6 +753,9 @@ defaults:
         # @schema {"name": "defaults.grpc.client.max_send_msg_size", "type": "integer"}
         # defaults.grpc.client.call_option.max_send_msg_size -- gRPC client call option max send message size
         max_send_msg_size: 0
+        # @schema {"name": "defaults.grpc.client.content_subtype", "type": "string"}
+        # defaults.grpc.client.call_option.content_subtype -- gRPC client call option content subtype
+        content_subtype: ""
       # @schema {"name": "defaults.grpc.client.dial_option", "type": "object"}
       dial_option:
         # @schema {"name": "defaults.grpc.client.dial_option.write_buffer_size", "type": "integer"}
@@ -755,6 +773,12 @@ defaults:
         # @schema {"name": "defaults.grpc.client.dial_option.max_msg_size", "type": "integer"}
         # defaults.grpc.client.dial_option.max_msg_size -- gRPC client dial option max message size
         max_msg_size: 0
+        # @schema {"name": "defaults.grpc.client.dial_option.max_header_list_size", "type": "integer"}
+        # defaults.grpc.client.dial_option.max_header_list_size -- gRPC client dial option max header list size
+        max_header_list_size: 0
+        # @schema {"name": "defaults.grpc.client.dial_option.max_call_attempts", "type": "integer"}
+        # defaults.grpc.client.dial_option.max_call_attempts -- gRPC client dial option number of max call attempts
+        max_call_attempts: 0
         # @schema {"name": "defaults.grpc.client.dial_option.backoff_max_delay", "type": "string"}
         # defaults.grpc.client.dial_option.backoff_max_delay -- gRPC client dial option max backoff delay
         backoff_max_delay: "120s"
@@ -773,9 +797,24 @@ defaults:
         # @schema {"name": "defaults.grpc.client.dial_option.enable_backoff", "type": "boolean"}
         # defaults.grpc.client.dial_option.enable_backoff -- gRPC client dial option backoff enabled
         enable_backoff: false
+        # @schema {"name": "defaults.grpc.client.dial_option.disable_retry", "type": "boolean"}
+        # defaults.grpc.client.dial_option.disable_retry -- gRPC client dial option disables retry
+        disable_retry: false
         # @schema {"name": "defaults.grpc.client.dial_option.insecure", "type": "boolean"}
         # defaults.grpc.client.dial_option.insecure -- gRPC client dial option insecure enabled
         insecure: true
+        # @schema {"name": "defaults.grpc.client.dial_option.shared_write_buffer", "type": "boolean"}
+        # defaults.grpc.client.dial_option.shared_write_buffer -- gRPC client dial option sharing write buffer
+        shared_write_buffer: false
+        # @schema {"name": "defaults.grpc.client.dial_option.authority", "type": "string"}
+        # defaults.grpc.client.dial_option.authority -- gRPC client dial option authority
+        authority: ""
+        # @schema {"name": "defaults.grpc.client.dial_option.idle_timeout", "type": "string"}
+        # defaults.grpc.client.dial_option.idle_timeout -- gRPC client dial option idle_timeout
+        idle_timeout: "1h"
+        # @schema {"name": "defaults.grpc.client.dial_option.user_agent", "type": "string"}
+        # defaults.grpc.client.dial_option.user_agent -- gRPC client dial option user_agent
+        user_agent: "Vald-gRPC"
         # @schema {"name": "defaults.grpc.client.dial_option.timeout", "type": "string"}
         # defaults.grpc.client.dial_option.timeout -- gRPC client dial option timeout
         timeout: ""

dockers/agent/core/agent/Dockerfile

@@ -93,4 +93,4 @@ LABEL maintainer="vdaas.org vald team <[email protected]>"
 COPY --from=builder /usr/bin/agent /usr/bin/agent
 # skipcq: DOK-DL3002
 USER nonroot:nonroot
-ENTRYPOINT ["/usr/bin/agent"]
+ENTRYPOINT ["/usr/bin/agent"]

dockers/agent/core/faiss/Dockerfile

@@ -95,4 +95,4 @@ COPY --from=builder /usr/bin/faiss /usr/bin/faiss
 COPY cmd/agent/core/faiss/sample.yaml /etc/server/config.yaml
 # skipcq: DOK-DL3002
 USER nonroot:nonroot
-ENTRYPOINT ["/usr/bin/faiss"]
+ENTRYPOINT ["/usr/bin/faiss"]

dockers/agent/core/ngt/Dockerfile

@@ -94,4 +94,4 @@ COPY --from=builder /usr/bin/ngt /usr/bin/ngt
 COPY cmd/agent/core/ngt/sample.yaml /etc/server/config.yaml
 # skipcq: DOK-DL3002
 USER nonroot:nonroot
-ENTRYPOINT ["/usr/bin/ngt"]
+ENTRYPOINT ["/usr/bin/ngt"]

dockers/agent/sidecar/Dockerfile

@@ -84,4 +84,4 @@ LABEL maintainer="vdaas.org vald team <[email protected]>"
 COPY --from=builder /usr/bin/sidecar /usr/bin/sidecar
 # skipcq: DOK-DL3002
 USER nonroot:nonroot
-ENTRYPOINT ["/usr/bin/sidecar"]
+ENTRYPOINT ["/usr/bin/sidecar"]

dockers/binfmt/Dockerfile

@@ -16,4 +16,4 @@
 #
 
 # DO_NOT_EDIT this Dockerfile is generated by https://github.com/vdaas/vald/blob/main/hack/docker/gen/main.go
-FROM tonistiigi/binfmt:master AS builder
+FROM tonistiigi/binfmt:master AS builder

dockers/buildbase/Dockerfile

@@ -16,4 +16,4 @@
 #
 
 # DO_NOT_EDIT this Dockerfile is generated by https://github.com/vdaas/vald/blob/main/hack/docker/gen/main.go
-FROM ubuntu:devel AS builder
+FROM ubuntu:devel AS builder

dockers/buildkit/Dockerfile

@@ -16,4 +16,4 @@
 #
 
 # DO_NOT_EDIT this Dockerfile is generated by https://github.com/vdaas/vald/blob/main/hack/docker/gen/main.go
-FROM moby/buildkit:master AS builder
+FROM moby/buildkit:master AS builder

dockers/buildkit/syft/scanner/Dockerfile

@@ -16,4 +16,4 @@
 #
 
 # DO_NOT_EDIT this Dockerfile is generated by https://github.com/vdaas/vald/blob/main/hack/docker/gen/main.go
-FROM docker/buildkit-syft-scanner:edge AS scanner
+FROM docker/buildkit-syft-scanner:edge AS scanner

dockers/ci/base/Dockerfile

@@ -44,8 +44,8 @@ ENV REPO=vald
 ENV RUST_HOME=/usr/local/lib/rust
 ENV TZ=Etc/UTC
 ENV USER=root
-ENV RUSTUP_HOME=${RUST_HOME}/rustup
 ENV CARGO_HOME=${RUST_HOME}/cargo
+ENV RUSTUP_HOME=${RUST_HOME}/rustup
 ENV PATH=${CARGO_HOME}/bin:${GOPATH}/bin:${GOROOT}/bin:${RUSTUP_HOME}/bin:/usr/local/bin:${PATH}
 WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}
 SHELL ["/bin/bash", "-o", "pipefail", "-c"]
@@ -127,4 +127,4 @@ RUN --mount=type=bind,target=.,rw \
     && rm -rf ${GOPATH}/src/github.com/${ORG}/${REPO}/*
 # skipcq: DOK-DL3002
 USER root:root
-ENTRYPOINT ["/bin/bash"]
+ENTRYPOINT ["/bin/bash"]

dockers/dev/Dockerfile

@@ -44,8 +44,8 @@ ENV REPO=vald
 ENV RUST_HOME=/usr/local/lib/rust
 ENV TZ=Etc/UTC
 ENV USER=root
-ENV CARGO_HOME=${RUST_HOME}/cargo
 ENV RUSTUP_HOME=${RUST_HOME}/rustup
+ENV CARGO_HOME=${RUST_HOME}/cargo
 ENV PATH=${CARGO_HOME}/bin:${GOPATH}/bin:${GOROOT}/bin:${RUSTUP_HOME}/bin:/usr/local/bin:${PATH}
 WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}
 SHELL ["/bin/bash", "-o", "pipefail", "-c"]
@@ -145,4 +145,4 @@ RUN --mount=type=bind,target=.,rw \
     && make faiss/install \
     && rm -rf ${GOPATH}/src/github.com/${ORG}/${REPO}/*
 # skipcq: DOK-DL3002
-USER root:root
+USER root:root

dockers/discoverer/k8s/Dockerfile

@@ -85,4 +85,4 @@ COPY --from=builder /usr/bin/discoverer /usr/bin/discoverer
 COPY cmd/discoverer/k8s/sample.yaml /etc/server/config.yaml
 # skipcq: DOK-DL3002
 USER nonroot:nonroot
-ENTRYPOINT ["/usr/bin/discoverer"]
+ENTRYPOINT ["/usr/bin/discoverer"]

dockers/gateway/filter/Dockerfile

@@ -85,4 +85,4 @@ COPY --from=builder /usr/bin/filter /usr/bin/filter
 COPY cmd/gateway/filter/sample.yaml /etc/server/config.yaml
 # skipcq: DOK-DL3002
 USER nonroot:nonroot
-ENTRYPOINT ["/usr/bin/filter"]
+ENTRYPOINT ["/usr/bin/filter"]

dockers/gateway/lb/Dockerfile

@@ -85,4 +85,4 @@ COPY --from=builder /usr/bin/lb /usr/bin/lb
 COPY cmd/gateway/lb/sample.yaml /etc/server/config.yaml
 # skipcq: DOK-DL3002
 USER nonroot:nonroot
-ENTRYPOINT ["/usr/bin/lb"]
+ENTRYPOINT ["/usr/bin/lb"]

dockers/gateway/mirror/Dockerfile

@@ -85,4 +85,4 @@ COPY --from=builder /usr/bin/mirror /usr/bin/mirror
 COPY cmd/gateway/mirror/sample.yaml /etc/server/config.yaml
 # skipcq: DOK-DL3002
 USER nonroot:nonroot
-ENTRYPOINT ["/usr/bin/mirror"]
+ENTRYPOINT ["/usr/bin/mirror"]

dockers/index/job/correction/Dockerfile

@@ -85,4 +85,4 @@ COPY --from=builder /usr/bin/index-correction /usr/bin/index-correction
 COPY cmd/index/job/correction/sample.yaml /etc/server/config.yaml
 # skipcq: DOK-DL3002
 USER nonroot:nonroot
-ENTRYPOINT ["/usr/bin/index-correction"]
+ENTRYPOINT ["/usr/bin/index-correction"]

dockers/index/job/creation/Dockerfile

@@ -85,4 +85,4 @@ COPY --from=builder /usr/bin/index-creation /usr/bin/index-creation
 COPY cmd/index/job/creation/sample.yaml /etc/server/config.yaml
 # skipcq: DOK-DL3002
 USER nonroot:nonroot
-ENTRYPOINT ["/usr/bin/index-creation"]
+ENTRYPOINT ["/usr/bin/index-creation"]

dockers/index/job/readreplica/rotate/Dockerfile

@@ -85,4 +85,4 @@ COPY --from=builder /usr/bin/readreplica-rotate /usr/bin/readreplica-rotate
 COPY cmd/index/job/readreplica/rotate/sample.yaml /etc/server/config.yaml
 # skipcq: DOK-DL3002
 USER nonroot:nonroot
-ENTRYPOINT ["/usr/bin/readreplica-rotate"]
+ENTRYPOINT ["/usr/bin/readreplica-rotate"]

dockers/index/job/save/Dockerfile

@@ -85,4 +85,4 @@ COPY --from=builder /usr/bin/index-save /usr/bin/index-save
 COPY cmd/index/job/save/sample.yaml /etc/server/config.yaml
 # skipcq: DOK-DL3002
 USER nonroot:nonroot
-ENTRYPOINT ["/usr/bin/index-save"]
+ENTRYPOINT ["/usr/bin/index-save"]

dockers/index/operator/Dockerfile

@@ -85,4 +85,4 @@ COPY --from=builder /usr/bin/index-operator /usr/bin/index-operator
 COPY cmd/index/operator/sample.yaml /etc/server/config.yaml
 # skipcq: DOK-DL3002
 USER nonroot:nonroot
-ENTRYPOINT ["/usr/bin/index-operator"]
+ENTRYPOINT ["/usr/bin/index-operator"]

dockers/manager/index/Dockerfile

@@ -85,4 +85,4 @@ COPY --from=builder /usr/bin/index /usr/bin/index
 COPY cmd/manager/index/sample.yaml /etc/server/config.yaml
 # skipcq: DOK-DL3002
 USER nonroot:nonroot
-ENTRYPOINT ["/usr/bin/index"]
+ENTRYPOINT ["/usr/bin/index"]

dockers/operator/helm/Dockerfile

@@ -106,4 +106,4 @@ COPY --from=builder /opt/helm/charts/vald /opt/helm/charts/vald
 COPY --from=builder /opt/helm/charts/vald-helm-operator /opt/helm/charts/vald-helm-operator
 # skipcq: DOK-DL3002
 USER nonroot:nonroot
-ENTRYPOINT ["/usr/bin/helm-operator", "run", "--watches-file=/opt/helm/watches.yaml"]
+ENTRYPOINT ["/usr/bin/helm-operator", "run", "--watches-file=/opt/helm/watches.yaml"]

dockers/tools/benchmark/job/Dockerfile

@@ -93,4 +93,4 @@ COPY --from=builder /usr/bin/job /usr/bin/job
 COPY cmd/tools/benchmark/job/sample.yaml /etc/server/config.yaml
 # skipcq: DOK-DL3002
 USER nonroot:nonroot
-ENTRYPOINT ["/usr/bin/job"]
+ENTRYPOINT ["/usr/bin/job"]

dockers/tools/benchmark/operator/Dockerfile

@@ -85,4 +85,4 @@ COPY --from=builder /usr/bin/operator /usr/bin/operator
 COPY cmd/tools/benchmark/operator/sample.yaml /etc/server/config.yaml
 # skipcq: DOK-DL3002
 USER nonroot:nonroot
-ENTRYPOINT ["/usr/bin/operator"]
+ENTRYPOINT ["/usr/bin/operator"]

dockers/tools/cli/loadtest/Dockerfile

@@ -93,4 +93,4 @@ COPY --from=builder /usr/bin/loadtest /usr/bin/loadtest
 COPY cmd/tools/cli/loadtest/sample.yaml /etc/server/config.yaml
 # skipcq: DOK-DL3002
 USER nonroot:nonroot
-ENTRYPOINT ["/usr/bin/loadtest"]
+ENTRYPOINT ["/usr/bin/loadtest"]

example/client/go.mod

@@ -8,14 +8,14 @@ replace (
 	github.com/golang/protobuf => github.com/golang/protobuf v1.5.4
 	github.com/kpango/glg => github.com/kpango/glg v1.6.15
 	github.com/pkg/sftp => github.com/pkg/sftp v1.13.6
-	golang.org/x/crypto => golang.org/x/crypto v0.27.0
-	golang.org/x/net => golang.org/x/net v0.29.0
-	golang.org/x/text => golang.org/x/text v0.18.0
-	google.golang.org/genproto => google.golang.org/genproto v0.0.0-20240930140551-af27646dc61f
-	google.golang.org/genproto/googleapis/api => google.golang.org/genproto/googleapis/api v0.0.0-20240930140551-af27646dc61f
-	google.golang.org/genproto/googleapis/rpc => google.golang.org/genproto/googleapis/rpc v0.0.0-20240930140551-af27646dc61f
+	golang.org/x/crypto => golang.org/x/crypto v0.28.0
+	golang.org/x/net => golang.org/x/net v0.30.0
+	golang.org/x/text => golang.org/x/text v0.19.0
+	google.golang.org/genproto => google.golang.org/genproto v0.0.0-20241007155032-5fefd90f89a9
+	google.golang.org/genproto/googleapis/api => google.golang.org/genproto/googleapis/api v0.0.0-20241007155032-5fefd90f89a9
+	google.golang.org/genproto/googleapis/rpc => google.golang.org/genproto/googleapis/rpc v0.0.0-20241007155032-5fefd90f89a9
 	google.golang.org/grpc => google.golang.org/grpc v1.67.1
-	google.golang.org/protobuf => google.golang.org/protobuf v1.34.2
+	google.golang.org/protobuf => google.golang.org/protobuf v1.35.1
 	gopkg.in/yaml.v2 => gopkg.in/yaml.v2 v2.4.0
 	gopkg.in/yaml.v3 => gopkg.in/yaml.v3 v3.0.1
 )
@@ -25,7 +25,7 @@ require (
 	github.com/kpango/glg v1.6.14
 	github.com/vdaas/vald-client-go v1.7.13
 	gonum.org/v1/hdf5 v0.0.0-20210714002203-8c5d23bc6946
-	google.golang.org/grpc v1.67.0
+	google.golang.org/grpc v1.67.1
 )
 
 require (
@@ -34,9 +34,9 @@ require (
 	github.com/kpango/fastime v1.1.9 // indirect
 	github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect
 	golang.org/x/net v0.28.0 // indirect
-	golang.org/x/sys v0.25.0 // indirect
-	golang.org/x/text v0.18.0 // indirect
+	golang.org/x/sys v0.26.0 // indirect
+	golang.org/x/text v0.19.0 // indirect
 	google.golang.org/genproto/googleapis/api v0.0.0-20240814211410-ddb44dafa142 // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20240924160255-9d4c2d233b61 // indirect
-	google.golang.org/protobuf v1.34.2 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20240930140551-af27646dc61f // indirect
+	google.golang.org/protobuf v1.35.1 // indirect
 )