From 075f303cf0e2744076604c2b373ed0f45342a22a Mon Sep 17 00:00:00 2001 From: kpango Date: Mon, 12 Aug 2024 17:16:29 +0900 Subject: [PATCH] Add UpdateTimestamp API Signed-off-by: kpango --- .gitfiles | 7 +- Makefile | 9 +- Makefile.d/client.mk | 45 -- Makefile.d/dependencies.mk | 6 - README.md | 2 + apis/docs/v1/docs.md | 11 +- apis/grpc/v1/vald/update.pb.go | 41 +- apis/grpc/v1/vald/update_vtproto.pb.go | 45 ++ apis/grpc/v1/vald/vald.go | 1 + apis/proto/v1/vald/update.proto | 8 + apis/swagger/v1/vald/update.swagger.json | 23 +- dockers/agent/core/agent/Dockerfile | 2 +- dockers/agent/core/faiss/Dockerfile | 2 +- dockers/agent/core/ngt/Dockerfile | 2 +- dockers/agent/sidecar/Dockerfile | 2 +- dockers/binfmt/Dockerfile | 2 +- dockers/buildbase/Dockerfile | 2 +- dockers/buildkit/Dockerfile | 2 +- dockers/ci/base/Dockerfile | 5 +- dockers/dev/Dockerfile | 5 +- dockers/discoverer/k8s/Dockerfile | 2 +- dockers/gateway/filter/Dockerfile | 2 +- dockers/gateway/lb/Dockerfile | 2 +- dockers/gateway/mirror/Dockerfile | 2 +- dockers/index/job/correction/Dockerfile | 2 +- dockers/index/job/creation/Dockerfile | 2 +- .../index/job/readreplica/rotate/Dockerfile | 2 +- dockers/index/job/save/Dockerfile | 2 +- dockers/index/operator/Dockerfile | 2 +- dockers/manager/index/Dockerfile | 2 +- dockers/operator/helm/Dockerfile | 2 +- dockers/tools/benchmark/job/Dockerfile | 2 +- dockers/tools/benchmark/operator/Dockerfile | 2 +- dockers/tools/cli/loadtest/Dockerfile | 2 +- example/client/go.mod | 6 +- example/client/go.sum | 8 +- go.mod | 40 +- go.sum | 49 +- hack/docker/gen/main.go | 1 - internal/client/v1/client/vald/vald.go | 34 ++ internal/errors/agent.go | 5 + pkg/gateway/lb/handler/grpc/handler.go | 441 ++++++++++-------- pkg/index/job/correction/service/corrector.go | 36 +- rust/Cargo.lock | 24 +- rust/libs/proto/src/vald.v1.tonic.rs | 87 ++++ versions/PROMETHEUS_STACK_VERSION | 2 +- versions/VALDCLI_VERSION | 1 - 47 files changed, 611 insertions(+), 373 deletions(-) delete mode 100644 Makefile.d/client.mk delete mode 100644 versions/VALDCLI_VERSION diff --git a/.gitfiles b/.gitfiles index 64f792a39c..56c5bfe305 100644 --- a/.gitfiles +++ b/.gitfiles @@ -140,7 +140,6 @@ Makefile Makefile.d/actions.mk Makefile.d/bench.mk Makefile.d/build.mk -Makefile.d/client.mk Makefile.d/dependencies.mk Makefile.d/docker.mk Makefile.d/e2e.mk @@ -1904,6 +1903,11 @@ rust/libs/ngt-rs/src/input.h rust/libs/ngt-rs/src/lib.rs rust/libs/ngt/Cargo.toml rust/libs/ngt/src/lib.rs +rust/libs/observability/Cargo.toml +rust/libs/observability/src/config.rs +rust/libs/observability/src/lib.rs +rust/libs/observability/src/macros.rs +rust/libs/observability/src/observability.rs rust/libs/proto/Cargo.toml rust/libs/proto/src/core.v1.tonic.rs rust/libs/proto/src/discoverer.v1.tonic.rs @@ -1965,7 +1969,6 @@ versions/PROTOBUF_VERSION versions/REVIEWDOG_VERSION versions/RUST_VERSION versions/TELEPRESENCE_VERSION -versions/VALDCLI_VERSION versions/VALD_VERSION versions/YQ_VERSION versions/ZLIB_VERSION diff --git a/Makefile b/Makefile index 4a5e001f71..edc14dbba5 100644 --- a/Makefile +++ b/Makefile @@ -103,7 +103,6 @@ PROTOBUF_VERSION := $(eval PROTOBUF_VERSION := $(shell cat versions/PRO REVIEWDOG_VERSION := $(eval REVIEWDOG_VERSION := $(shell cat versions/REVIEWDOG_VERSION))$(REVIEWDOG_VERSION) RUST_VERSION := $(eval RUST_VERSION := $(shell cat versions/RUST_VERSION))$(RUST_VERSION) TELEPRESENCE_VERSION := $(eval TELEPRESENCE_VERSION := $(shell cat versions/TELEPRESENCE_VERSION))$(TELEPRESENCE_VERSION) -VALDCLI_VERSION := $(eval VALDCLI_VERSION := $(shell cat versions/VALDCLI_VERSION))$(VALDCLI_VERSION) YQ_VERSION := $(eval YQ_VERSION := $(shell cat versions/YQ_VERSION))$(YQ_VERSION) ZLIB_VERSION := $(eval ZLIB_VERSION := $(shell cat versions/ZLIB_VERSION))$(ZLIB_VERSION) @@ -455,7 +454,6 @@ init: \ tools/install: \ helm/install \ kind/install \ - valdcli/install \ telepresence/install \ textlint/install @@ -620,10 +618,6 @@ version/helm: version/yq: @echo $(YQ_VERSION) -.PHONY: version/valdcli -version/valdcli: - @echo $(VALDCLI_VERSION) - .PHONY: version/telepresence version/telepresence: @echo $(TELEPRESENCE_VERSION) @@ -766,7 +760,6 @@ changelog/next/print: include Makefile.d/actions.mk include Makefile.d/bench.mk include Makefile.d/build.mk -include Makefile.d/client.mk include Makefile.d/dependencies.mk include Makefile.d/docker.mk include Makefile.d/e2e.mk @@ -775,7 +768,7 @@ include Makefile.d/helm.mk include Makefile.d/k3d.mk include Makefile.d/k8s.mk include Makefile.d/kind.mk +include Makefile.d/minikube.mk include Makefile.d/proto.mk include Makefile.d/test.mk include Makefile.d/tools.mk -include Makefile.d/minikube.mk diff --git a/Makefile.d/client.mk b/Makefile.d/client.mk deleted file mode 100644 index 1c5c7aa9bf..0000000000 --- a/Makefile.d/client.mk +++ /dev/null @@ -1,45 +0,0 @@ -# -# Copyright (C) 2019-2024 vdaas.org vald team -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# You may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -.PHONY: valdcli/install -## install valdcli -valdcli/install: $(BINDIR)/valdcli - -ifeq ($(UNAME),Darwin) -$(BINDIR)/valdcli: - mkdir -p $(BINDIR) - curl -fsSLO https://github.com/rinx/vald-client-clj/releases/download/$(VALDCLI_VERSION)/valdcli-macos.zip - unzip valdcli-macos.zip - rm -f valdcli-macos.zip - mv valdcli $(BINDIR)/valdcli -else -$(BINDIR)/valdcli: - mkdir -p $(BINDIR) - curl -fsSLO https://github.com/rinx/vald-client-clj/releases/download/$(VALDCLI_VERSION)/valdcli-linux-static.zip - unzip valdcli-linux-static.zip - rm -f valdcli-linux-static.zip - mv valdcli $(BINDIR)/valdcli -endif - -.PHONY: valdcli/xpanes/insert -## insert randomized vectors using valdcli and xpanes -valdcli/xpanes/insert: - xpanes -c "valdcli rand-vecs -n $(NUMBER) -d $(DIMENSION) --gaussian --gaussian-mean $(MEAN) --gaussian-stddev $(STDDEV) --with-ids | valdcli -h $(HOST) -p $(PORT) stream-insert --elapsed-time" $$(seq 1 $(NUMPANES)) - -.PHONY: valdcli/xpanes/search -## search randomized vectors using valdcli and xpanes -valdcli/xpanes/search: - xpanes -c "valdcli rand-vecs -n $(NUMBER) -d $(DIMENSION) --gaussian --gaussian-mean $(MEAN) --gaussian-stddev $(STDDEV) | valdcli -h $(HOST) -p $(PORT) stream-search --elapsed-time" $$(seq 1 $(NUMPANES)) diff --git a/Makefile.d/dependencies.mk b/Makefile.d/dependencies.mk index 928ed8c9d2..596e9ec1b2 100644 --- a/Makefile.d/dependencies.mk +++ b/Makefile.d/dependencies.mk @@ -39,7 +39,6 @@ update/libs: \ update/rust \ update/telepresence \ update/vald \ - update/valdcli \ update/yq \ update/zlib @@ -227,11 +226,6 @@ update/hdf5: update/vald: curl -fsSL https://api.github.com/repos/$(REPO)/releases/latest | grep -Po '"tag_name": "\K.*?(?=")' > $(ROOTDIR)/versions/VALD_VERSION -.PHONY: update/valdcli -## update vald client library made by clojure self version -update/valdcli: - curl -fsSL https://api.github.com/repos/$(REPO)-client-clj/releases/latest | grep -Po '"tag_name": "\K.*?(?=")' > $(ROOTDIR)/versions/VALDCLI_VERSION - .PHONY: update/template ## update PULL_REQUEST_TEMPLATE and ISSUE_TEMPLATE update/template: diff --git a/README.md b/README.md index f383078d65..a03458bf5b 100755 --- a/README.md +++ b/README.md @@ -264,7 +264,9 @@ make init ## Contributors + [![All Contributors](https://img.shields.io/badge/all_contributors-18-orange.svg?style=flat-square)](#contributors) + Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/docs/en/emoji-key)): diff --git a/apis/docs/v1/docs.md b/apis/docs/v1/docs.md index 2289344848..08bf1d81d6 100644 --- a/apis/docs/v1/docs.md +++ b/apis/docs/v1/docs.md @@ -1664,11 +1664,12 @@ Search service provides ways to search indexed vectors. Update service provides ways to update indexed vectors. -| Method Name | Request Type | Response Type | Description | -| ------------ | ------------------------------------------------------------------ | ----------------------------------------------------------------------------- | ----------------------------------------------------------------------- | -| Update | [.payload.v1.Update.Request](#payload-v1-Update-Request) | [.payload.v1.Object.Location](#payload-v1-Object-Location) | A method to update an indexed vector. | -| StreamUpdate | [.payload.v1.Update.Request](#payload-v1-Update-Request) stream | [.payload.v1.Object.StreamLocation](#payload-v1-Object-StreamLocation) stream | A method to update multiple indexed vectors by bidirectional streaming. | -| MultiUpdate | [.payload.v1.Update.MultiRequest](#payload-v1-Update-MultiRequest) | [.payload.v1.Object.Locations](#payload-v1-Object-Locations) | A method to update multiple indexed vectors in a single request. | +| Method Name | Request Type | Response Type | Description | +| --------------- | ------------------------------------------------------------------ | ----------------------------------------------------------------------------- | ----------------------------------------------------------------------- | +| Update | [.payload.v1.Update.Request](#payload-v1-Update-Request) | [.payload.v1.Object.Location](#payload-v1-Object-Location) | A method to update an indexed vector. | +| StreamUpdate | [.payload.v1.Update.Request](#payload-v1-Update-Request) stream | [.payload.v1.Object.StreamLocation](#payload-v1-Object-StreamLocation) stream | A method to update multiple indexed vectors by bidirectional streaming. | +| MultiUpdate | [.payload.v1.Update.MultiRequest](#payload-v1-Update-MultiRequest) | [.payload.v1.Object.Locations](#payload-v1-Object-Locations) | A method to update multiple indexed vectors in a single request. | +| UpdateTimestamp | [.payload.v1.Object.Timestamp](#payload-v1-Object-Timestamp) | [.payload.v1.Object.Location](#payload-v1-Object-Location) | A method to update timestamp an indexed vector. | diff --git a/apis/grpc/v1/vald/update.pb.go b/apis/grpc/v1/vald/update.pb.go index ad05fbe50c..f98b9d1b26 100644 --- a/apis/grpc/v1/vald/update.pb.go +++ b/apis/grpc/v1/vald/update.pb.go @@ -46,7 +46,7 @@ var file_v1_vald_update_proto_rawDesc = []byte{ 0x1c, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x61, 0x6e, 0x6e, 0x6f, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x18, 0x76, 0x31, 0x2f, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x2f, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, - 0x64, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x32, 0x9f, 0x02, 0x0a, 0x06, 0x55, 0x70, 0x64, 0x61, + 0x64, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x32, 0x81, 0x03, 0x0a, 0x06, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x12, 0x55, 0x0a, 0x06, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x12, 0x1a, 0x2e, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x2e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1b, 0x2e, 0x70, 0x61, 0x79, 0x6c, 0x6f, @@ -64,32 +64,41 @@ var file_v1_vald_update_proto_rawDesc = []byte{ 0x2e, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x2e, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x22, 0x1b, 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x15, 0x3a, 0x01, 0x2a, 0x22, 0x10, 0x2f, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, - 0x2f, 0x6d, 0x75, 0x6c, 0x74, 0x69, 0x70, 0x6c, 0x65, 0x42, 0x53, 0x0a, 0x1a, 0x6f, 0x72, 0x67, - 0x2e, 0x76, 0x64, 0x61, 0x61, 0x73, 0x2e, 0x76, 0x61, 0x6c, 0x64, 0x2e, 0x61, 0x70, 0x69, 0x2e, - 0x76, 0x31, 0x2e, 0x76, 0x61, 0x6c, 0x64, 0x42, 0x0a, 0x56, 0x61, 0x6c, 0x64, 0x55, 0x70, 0x64, - 0x61, 0x74, 0x65, 0x50, 0x01, 0x5a, 0x27, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, - 0x6d, 0x2f, 0x76, 0x64, 0x61, 0x61, 0x73, 0x2f, 0x76, 0x61, 0x6c, 0x64, 0x2f, 0x61, 0x70, 0x69, - 0x73, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x76, 0x31, 0x2f, 0x76, 0x61, 0x6c, 0x64, 0x62, 0x06, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x2f, 0x6d, 0x75, 0x6c, 0x74, 0x69, 0x70, 0x6c, 0x65, 0x12, 0x60, 0x0a, 0x0f, 0x55, 0x70, 0x64, + 0x61, 0x74, 0x65, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x1c, 0x2e, 0x70, + 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, + 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x1a, 0x1b, 0x2e, 0x70, 0x61, 0x79, + 0x6c, 0x6f, 0x61, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x2e, 0x4c, + 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x12, 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x0c, 0x3a, + 0x01, 0x2a, 0x22, 0x07, 0x2f, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x42, 0x53, 0x0a, 0x1a, 0x6f, + 0x72, 0x67, 0x2e, 0x76, 0x64, 0x61, 0x61, 0x73, 0x2e, 0x76, 0x61, 0x6c, 0x64, 0x2e, 0x61, 0x70, + 0x69, 0x2e, 0x76, 0x31, 0x2e, 0x76, 0x61, 0x6c, 0x64, 0x42, 0x0a, 0x56, 0x61, 0x6c, 0x64, 0x55, + 0x70, 0x64, 0x61, 0x74, 0x65, 0x50, 0x01, 0x5a, 0x27, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, + 0x63, 0x6f, 0x6d, 0x2f, 0x76, 0x64, 0x61, 0x61, 0x73, 0x2f, 0x76, 0x61, 0x6c, 0x64, 0x2f, 0x61, + 0x70, 0x69, 0x73, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x76, 0x31, 0x2f, 0x76, 0x61, 0x6c, 0x64, + 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var file_v1_vald_update_proto_goTypes = []any{ (*payload.Update_Request)(nil), // 0: payload.v1.Update.Request (*payload.Update_MultiRequest)(nil), // 1: payload.v1.Update.MultiRequest - (*payload.Object_Location)(nil), // 2: payload.v1.Object.Location - (*payload.Object_StreamLocation)(nil), // 3: payload.v1.Object.StreamLocation - (*payload.Object_Locations)(nil), // 4: payload.v1.Object.Locations + (*payload.Object_Timestamp)(nil), // 2: payload.v1.Object.Timestamp + (*payload.Object_Location)(nil), // 3: payload.v1.Object.Location + (*payload.Object_StreamLocation)(nil), // 4: payload.v1.Object.StreamLocation + (*payload.Object_Locations)(nil), // 5: payload.v1.Object.Locations } var file_v1_vald_update_proto_depIdxs = []int32{ 0, // 0: vald.v1.Update.Update:input_type -> payload.v1.Update.Request 0, // 1: vald.v1.Update.StreamUpdate:input_type -> payload.v1.Update.Request 1, // 2: vald.v1.Update.MultiUpdate:input_type -> payload.v1.Update.MultiRequest - 2, // 3: vald.v1.Update.Update:output_type -> payload.v1.Object.Location - 3, // 4: vald.v1.Update.StreamUpdate:output_type -> payload.v1.Object.StreamLocation - 4, // 5: vald.v1.Update.MultiUpdate:output_type -> payload.v1.Object.Locations - 3, // [3:6] is the sub-list for method output_type - 0, // [0:3] is the sub-list for method input_type + 2, // 3: vald.v1.Update.UpdateTimestamp:input_type -> payload.v1.Object.Timestamp + 3, // 4: vald.v1.Update.Update:output_type -> payload.v1.Object.Location + 4, // 5: vald.v1.Update.StreamUpdate:output_type -> payload.v1.Object.StreamLocation + 5, // 6: vald.v1.Update.MultiUpdate:output_type -> payload.v1.Object.Locations + 3, // 7: vald.v1.Update.UpdateTimestamp:output_type -> payload.v1.Object.Location + 4, // [4:8] is the sub-list for method output_type + 0, // [0:4] is the sub-list for method input_type 0, // [0:0] is the sub-list for extension type_name 0, // [0:0] is the sub-list for extension extendee 0, // [0:0] is the sub-list for field type_name diff --git a/apis/grpc/v1/vald/update_vtproto.pb.go b/apis/grpc/v1/vald/update_vtproto.pb.go index 60f92fd0a8..6fc00c758c 100644 --- a/apis/grpc/v1/vald/update_vtproto.pb.go +++ b/apis/grpc/v1/vald/update_vtproto.pb.go @@ -48,6 +48,8 @@ type UpdateClient interface { StreamUpdate(ctx context.Context, opts ...grpc.CallOption) (Update_StreamUpdateClient, error) // A method to update multiple indexed vectors in a single request. MultiUpdate(ctx context.Context, in *payload.Update_MultiRequest, opts ...grpc.CallOption) (*payload.Object_Locations, error) + // A method to update timestamp an indexed vector. + UpdateTimestamp(ctx context.Context, in *payload.Object_Timestamp, opts ...grpc.CallOption) (*payload.Object_Location, error) } type updateClient struct { @@ -113,6 +115,17 @@ func (c *updateClient) MultiUpdate( return out, nil } +func (c *updateClient) UpdateTimestamp( + ctx context.Context, in *payload.Object_Timestamp, opts ...grpc.CallOption, +) (*payload.Object_Location, error) { + out := new(payload.Object_Location) + err := c.cc.Invoke(ctx, "/vald.v1.Update/UpdateTimestamp", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + // UpdateServer is the server API for Update service. // All implementations must embed UnimplementedUpdateServer // for forward compatibility @@ -123,6 +136,8 @@ type UpdateServer interface { StreamUpdate(Update_StreamUpdateServer) error // A method to update multiple indexed vectors in a single request. MultiUpdate(context.Context, *payload.Update_MultiRequest) (*payload.Object_Locations, error) + // A method to update timestamp an indexed vector. + UpdateTimestamp(context.Context, *payload.Object_Timestamp) (*payload.Object_Location, error) mustEmbedUnimplementedUpdateServer() } @@ -144,6 +159,12 @@ func (UnimplementedUpdateServer) MultiUpdate( ) (*payload.Object_Locations, error) { return nil, status.Errorf(codes.Unimplemented, "method MultiUpdate not implemented") } + +func (UnimplementedUpdateServer) UpdateTimestamp( + context.Context, *payload.Object_Timestamp, +) (*payload.Object_Location, error) { + return nil, status.Errorf(codes.Unimplemented, "method UpdateTimestamp not implemented") +} func (UnimplementedUpdateServer) mustEmbedUnimplementedUpdateServer() {} // UnsafeUpdateServer may be embedded to opt out of forward compatibility for this service. @@ -223,6 +244,26 @@ func _Update_MultiUpdate_Handler( return interceptor(ctx, in, info, handler) } +func _Update_UpdateTimestamp_Handler( + srv any, ctx context.Context, dec func(any) error, interceptor grpc.UnaryServerInterceptor, +) (any, error) { + in := new(payload.Object_Timestamp) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(UpdateServer).UpdateTimestamp(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/vald.v1.Update/UpdateTimestamp", + } + handler := func(ctx context.Context, req any) (any, error) { + return srv.(UpdateServer).UpdateTimestamp(ctx, req.(*payload.Object_Timestamp)) + } + return interceptor(ctx, in, info, handler) +} + // Update_ServiceDesc is the grpc.ServiceDesc for Update service. // It's only intended for direct use with grpc.RegisterService, // and not to be introspected or modified (even as a copy) @@ -238,6 +279,10 @@ var Update_ServiceDesc = grpc.ServiceDesc{ MethodName: "MultiUpdate", Handler: _Update_MultiUpdate_Handler, }, + { + MethodName: "UpdateTimestamp", + Handler: _Update_UpdateTimestamp_Handler, + }, }, Streams: []grpc.StreamDesc{ { diff --git a/apis/grpc/v1/vald/vald.go b/apis/grpc/v1/vald/vald.go index a67e7c97ef..c1255fac42 100644 --- a/apis/grpc/v1/vald/vald.go +++ b/apis/grpc/v1/vald/vald.go @@ -95,6 +95,7 @@ const ( UpdateObjectRPCName = "UpdateObject" StreamUpdateObjectRPCName = "StreamUpdateObject" MultiUpdateObjectRPCName = "MultiUpdateObject" + UpdateTimestampRPCName = "UpdateTimestamp" UpsertRPCName = "Upsert" StreamUpsertRPCName = "StreamUpsert" diff --git a/apis/proto/v1/vald/update.proto b/apis/proto/v1/vald/update.proto index 4cab1cabf0..ccd715efd7 100644 --- a/apis/proto/v1/vald/update.proto +++ b/apis/proto/v1/vald/update.proto @@ -46,4 +46,12 @@ service Update { body: "*" }; } + + // A method to update timestamp an indexed vector. + rpc UpdateTimestamp(payload.v1.Object.Timestamp) returns (payload.v1.Object.Location) { + option (google.api.http) = { + post: "/update" + body: "*" + }; + } } diff --git a/apis/swagger/v1/vald/update.swagger.json b/apis/swagger/v1/vald/update.swagger.json index d295febc19..85c729d702 100644 --- a/apis/swagger/v1/vald/update.swagger.json +++ b/apis/swagger/v1/vald/update.swagger.json @@ -14,8 +14,8 @@ "paths": { "/update": { "post": { - "summary": "A method to update an indexed vector.", - "operationId": "Update_Update", + "summary": "A method to update timestamp an indexed vector.", + "operationId": "Update_UpdateTimestamp", "responses": { "200": { "description": "A successful response.", @@ -33,11 +33,11 @@ "parameters": [ { "name": "body", - "description": "Represent the update request.", + "description": "Represent a vector meta data.", "in": "body", "required": true, "schema": { - "$ref": "#/definitions/v1UpdateRequest" + "$ref": "#/definitions/v1ObjectTimestamp" } } ], @@ -212,6 +212,21 @@ }, "description": "Represent the vector location." }, + "v1ObjectTimestamp": { + "type": "object", + "properties": { + "id": { + "type": "string", + "description": "The vector ID." + }, + "timestamp": { + "type": "string", + "format": "int64", + "description": "timestamp represents when this vector inserted." + } + }, + "description": "Represent a vector meta data." + }, "v1UpdateConfig": { "type": "object", "properties": { diff --git a/dockers/agent/core/agent/Dockerfile b/dockers/agent/core/agent/Dockerfile index f3195dde42..ccade20803 100644 --- a/dockers/agent/core/agent/Dockerfile +++ b/dockers/agent/core/agent/Dockerfile @@ -91,4 +91,4 @@ LABEL maintainer="vdaas.org vald team " COPY --from=builder /usr/bin/agent /usr/bin/agent # skipcq: DOK-DL3002 USER nonroot:nonroot -ENTRYPOINT ["/usr/bin/agent"] \ No newline at end of file +ENTRYPOINT ["/usr/bin/agent"] diff --git a/dockers/agent/core/faiss/Dockerfile b/dockers/agent/core/faiss/Dockerfile index 463e13494b..3ee7076aad 100644 --- a/dockers/agent/core/faiss/Dockerfile +++ b/dockers/agent/core/faiss/Dockerfile @@ -93,4 +93,4 @@ COPY --from=builder /usr/bin/faiss /usr/bin/faiss COPY cmd/agent/core/faiss/sample.yaml /etc/server/config.yaml # skipcq: DOK-DL3002 USER nonroot:nonroot -ENTRYPOINT ["/usr/bin/faiss"] \ No newline at end of file +ENTRYPOINT ["/usr/bin/faiss"] diff --git a/dockers/agent/core/ngt/Dockerfile b/dockers/agent/core/ngt/Dockerfile index cf0bbcdce2..55b55d7e43 100644 --- a/dockers/agent/core/ngt/Dockerfile +++ b/dockers/agent/core/ngt/Dockerfile @@ -92,4 +92,4 @@ COPY --from=builder /usr/bin/ngt /usr/bin/ngt COPY cmd/agent/core/ngt/sample.yaml /etc/server/config.yaml # skipcq: DOK-DL3002 USER nonroot:nonroot -ENTRYPOINT ["/usr/bin/ngt"] \ No newline at end of file +ENTRYPOINT ["/usr/bin/ngt"] diff --git a/dockers/agent/sidecar/Dockerfile b/dockers/agent/sidecar/Dockerfile index 2b06e565fd..0f9c0e0760 100644 --- a/dockers/agent/sidecar/Dockerfile +++ b/dockers/agent/sidecar/Dockerfile @@ -82,4 +82,4 @@ LABEL maintainer="vdaas.org vald team " COPY --from=builder /usr/bin/sidecar /usr/bin/sidecar # skipcq: DOK-DL3002 USER nonroot:nonroot -ENTRYPOINT ["/usr/bin/sidecar"] \ No newline at end of file +ENTRYPOINT ["/usr/bin/sidecar"] diff --git a/dockers/binfmt/Dockerfile b/dockers/binfmt/Dockerfile index 47284d17e2..f2d73909bd 100644 --- a/dockers/binfmt/Dockerfile +++ b/dockers/binfmt/Dockerfile @@ -16,4 +16,4 @@ # # DO_NOT_EDIT this Dockerfile is generated by https://github.com/vdaas/vald/blob/main/hack/docker/gen/main.go -FROM tonistiigi/binfmt:master AS builder \ No newline at end of file +FROM tonistiigi/binfmt:master AS builder diff --git a/dockers/buildbase/Dockerfile b/dockers/buildbase/Dockerfile index 6457b01457..301a31e010 100644 --- a/dockers/buildbase/Dockerfile +++ b/dockers/buildbase/Dockerfile @@ -16,4 +16,4 @@ # # DO_NOT_EDIT this Dockerfile is generated by https://github.com/vdaas/vald/blob/main/hack/docker/gen/main.go -FROM ubuntu:devel AS builder \ No newline at end of file +FROM ubuntu:devel AS builder diff --git a/dockers/buildkit/Dockerfile b/dockers/buildkit/Dockerfile index 9dd722ea25..99c71c61e2 100644 --- a/dockers/buildkit/Dockerfile +++ b/dockers/buildkit/Dockerfile @@ -16,4 +16,4 @@ # # DO_NOT_EDIT this Dockerfile is generated by https://github.com/vdaas/vald/blob/main/hack/docker/gen/main.go -FROM moby/buildkit:master AS builder \ No newline at end of file +FROM moby/buildkit:master AS builder diff --git a/dockers/ci/base/Dockerfile b/dockers/ci/base/Dockerfile index 09fccb1e35..95feefc0c0 100644 --- a/dockers/ci/base/Dockerfile +++ b/dockers/ci/base/Dockerfile @@ -44,8 +44,8 @@ ENV REPO=vald ENV RUST_HOME=/usr/loacl/lib/rust ENV TZ=Etc/UTC ENV USER=root -ENV RUSTUP_HOME=${RUST_HOME}/rustup ENV CARGO_HOME=${RUST_HOME}/cargo +ENV RUSTUP_HOME=${RUST_HOME}/rustup ENV PATH=${CARGO_HOME}/bin:${GOPATH}/bin:${GOROOT}/bin:${RUSTUP_HOME}/bin:/usr/local/bin:${PATH} WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO} SHELL ["/bin/bash", "-o", "pipefail", "-c"] @@ -113,7 +113,6 @@ RUN --mount=type=bind,target=.,rw \ && make kubelinter/install \ && make reviewdog/install \ && make tparse/install \ - && make valdcli/install \ && make yq/install \ && make minikube/install \ && make stern/install \ @@ -123,4 +122,4 @@ RUN --mount=type=bind,target=.,rw \ && rm -rf ${GOPATH}/src/github.com/${ORG}/${REPO}/* # skipcq: DOK-DL3002 USER root:root -ENTRYPOINT ["/bin/bash"] \ No newline at end of file +ENTRYPOINT ["/bin/bash"] diff --git a/dockers/dev/Dockerfile b/dockers/dev/Dockerfile index 502f386e97..2a7c772846 100644 --- a/dockers/dev/Dockerfile +++ b/dockers/dev/Dockerfile @@ -44,8 +44,8 @@ ENV REPO=vald ENV RUST_HOME=/usr/loacl/lib/rust ENV TZ=Etc/UTC ENV USER=root -ENV CARGO_HOME=${RUST_HOME}/cargo ENV RUSTUP_HOME=${RUST_HOME}/rustup +ENV CARGO_HOME=${RUST_HOME}/cargo ENV PATH=${CARGO_HOME}/bin:${GOPATH}/bin:${GOROOT}/bin:${RUSTUP_HOME}/bin:/usr/local/bin:${PATH} WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO} SHELL ["/bin/bash", "-o", "pipefail", "-c"] @@ -133,7 +133,6 @@ RUN --mount=type=bind,target=.,rw \ && make kubelinter/install \ && make reviewdog/install \ && make tparse/install \ - && make valdcli/install \ && make yq/install \ && make minikube/install \ && make stern/install \ @@ -142,4 +141,4 @@ RUN --mount=type=bind,target=.,rw \ && make faiss/install \ && rm -rf ${GOPATH}/src/github.com/${ORG}/${REPO}/* # skipcq: DOK-DL3002 -USER root:root \ No newline at end of file +USER root:root diff --git a/dockers/discoverer/k8s/Dockerfile b/dockers/discoverer/k8s/Dockerfile index 19e5953ee2..42d7b476f7 100644 --- a/dockers/discoverer/k8s/Dockerfile +++ b/dockers/discoverer/k8s/Dockerfile @@ -83,4 +83,4 @@ COPY --from=builder /usr/bin/discoverer /usr/bin/discoverer COPY cmd/discoverer/k8s/sample.yaml /etc/server/config.yaml # skipcq: DOK-DL3002 USER nonroot:nonroot -ENTRYPOINT ["/usr/bin/discoverer"] \ No newline at end of file +ENTRYPOINT ["/usr/bin/discoverer"] diff --git a/dockers/gateway/filter/Dockerfile b/dockers/gateway/filter/Dockerfile index b523b5e5b8..bd6b3dadd3 100644 --- a/dockers/gateway/filter/Dockerfile +++ b/dockers/gateway/filter/Dockerfile @@ -83,4 +83,4 @@ COPY --from=builder /usr/bin/filter /usr/bin/filter COPY cmd/gateway/filter/sample.yaml /etc/server/config.yaml # skipcq: DOK-DL3002 USER nonroot:nonroot -ENTRYPOINT ["/usr/bin/filter"] \ No newline at end of file +ENTRYPOINT ["/usr/bin/filter"] diff --git a/dockers/gateway/lb/Dockerfile b/dockers/gateway/lb/Dockerfile index 2f8c91768b..f6c3f0b5c5 100644 --- a/dockers/gateway/lb/Dockerfile +++ b/dockers/gateway/lb/Dockerfile @@ -83,4 +83,4 @@ COPY --from=builder /usr/bin/lb /usr/bin/lb COPY cmd/gateway/lb/sample.yaml /etc/server/config.yaml # skipcq: DOK-DL3002 USER nonroot:nonroot -ENTRYPOINT ["/usr/bin/lb"] \ No newline at end of file +ENTRYPOINT ["/usr/bin/lb"] diff --git a/dockers/gateway/mirror/Dockerfile b/dockers/gateway/mirror/Dockerfile index 9b97231c74..f3effce23f 100644 --- a/dockers/gateway/mirror/Dockerfile +++ b/dockers/gateway/mirror/Dockerfile @@ -83,4 +83,4 @@ COPY --from=builder /usr/bin/mirror /usr/bin/mirror COPY cmd/gateway/mirror/sample.yaml /etc/server/config.yaml # skipcq: DOK-DL3002 USER nonroot:nonroot -ENTRYPOINT ["/usr/bin/mirror"] \ No newline at end of file +ENTRYPOINT ["/usr/bin/mirror"] diff --git a/dockers/index/job/correction/Dockerfile b/dockers/index/job/correction/Dockerfile index 01e3818c56..06b7642c1a 100644 --- a/dockers/index/job/correction/Dockerfile +++ b/dockers/index/job/correction/Dockerfile @@ -83,4 +83,4 @@ COPY --from=builder /usr/bin/index-correction /usr/bin/index-correction COPY cmd/index/job/correction/sample.yaml /etc/server/config.yaml # skipcq: DOK-DL3002 USER nonroot:nonroot -ENTRYPOINT ["/usr/bin/index-correction"] \ No newline at end of file +ENTRYPOINT ["/usr/bin/index-correction"] diff --git a/dockers/index/job/creation/Dockerfile b/dockers/index/job/creation/Dockerfile index d656b3ad22..6ea3eda4b6 100644 --- a/dockers/index/job/creation/Dockerfile +++ b/dockers/index/job/creation/Dockerfile @@ -83,4 +83,4 @@ COPY --from=builder /usr/bin/index-creation /usr/bin/index-creation COPY cmd/index/job/creation/sample.yaml /etc/server/config.yaml # skipcq: DOK-DL3002 USER nonroot:nonroot -ENTRYPOINT ["/usr/bin/index-creation"] \ No newline at end of file +ENTRYPOINT ["/usr/bin/index-creation"] diff --git a/dockers/index/job/readreplica/rotate/Dockerfile b/dockers/index/job/readreplica/rotate/Dockerfile index bdb0ec7664..432085dbb4 100644 --- a/dockers/index/job/readreplica/rotate/Dockerfile +++ b/dockers/index/job/readreplica/rotate/Dockerfile @@ -83,4 +83,4 @@ COPY --from=builder /usr/bin/readreplica-rotate /usr/bin/readreplica-rotate COPY cmd/index/job/readreplica/rotate/sample.yaml /etc/server/config.yaml # skipcq: DOK-DL3002 USER nonroot:nonroot -ENTRYPOINT ["/usr/bin/readreplica-rotate"] \ No newline at end of file +ENTRYPOINT ["/usr/bin/readreplica-rotate"] diff --git a/dockers/index/job/save/Dockerfile b/dockers/index/job/save/Dockerfile index fdd674abd4..1b0797a81d 100644 --- a/dockers/index/job/save/Dockerfile +++ b/dockers/index/job/save/Dockerfile @@ -83,4 +83,4 @@ COPY --from=builder /usr/bin/index-save /usr/bin/index-save COPY cmd/index/job/save/sample.yaml /etc/server/config.yaml # skipcq: DOK-DL3002 USER nonroot:nonroot -ENTRYPOINT ["/usr/bin/index-save"] \ No newline at end of file +ENTRYPOINT ["/usr/bin/index-save"] diff --git a/dockers/index/operator/Dockerfile b/dockers/index/operator/Dockerfile index 23e9aae514..c2f652283f 100644 --- a/dockers/index/operator/Dockerfile +++ b/dockers/index/operator/Dockerfile @@ -83,4 +83,4 @@ COPY --from=builder /usr/bin/index-operator /usr/bin/index-operator COPY cmd/index/operator/sample.yaml /etc/server/config.yaml # skipcq: DOK-DL3002 USER nonroot:nonroot -ENTRYPOINT ["/usr/bin/index-operator"] \ No newline at end of file +ENTRYPOINT ["/usr/bin/index-operator"] diff --git a/dockers/manager/index/Dockerfile b/dockers/manager/index/Dockerfile index edecb98a6c..8eb554a390 100644 --- a/dockers/manager/index/Dockerfile +++ b/dockers/manager/index/Dockerfile @@ -83,4 +83,4 @@ COPY --from=builder /usr/bin/index /usr/bin/index COPY cmd/manager/index/sample.yaml /etc/server/config.yaml # skipcq: DOK-DL3002 USER nonroot:nonroot -ENTRYPOINT ["/usr/bin/index"] \ No newline at end of file +ENTRYPOINT ["/usr/bin/index"] diff --git a/dockers/operator/helm/Dockerfile b/dockers/operator/helm/Dockerfile index db24094790..d5edc4f6e9 100644 --- a/dockers/operator/helm/Dockerfile +++ b/dockers/operator/helm/Dockerfile @@ -104,4 +104,4 @@ COPY --from=builder /opt/helm/charts/vald /opt/helm/charts/vald COPY --from=builder /opt/helm/charts/vald-helm-operator /opt/helm/charts/vald-helm-operator # skipcq: DOK-DL3002 USER nonroot:nonroot -ENTRYPOINT ["/usr/bin/helm-operator", "run", "--watches-file=/opt/helm/watches.yaml"] \ No newline at end of file +ENTRYPOINT ["/usr/bin/helm-operator", "run", "--watches-file=/opt/helm/watches.yaml"] diff --git a/dockers/tools/benchmark/job/Dockerfile b/dockers/tools/benchmark/job/Dockerfile index 65da31d468..d77a1181fd 100644 --- a/dockers/tools/benchmark/job/Dockerfile +++ b/dockers/tools/benchmark/job/Dockerfile @@ -91,4 +91,4 @@ COPY --from=builder /usr/bin/job /usr/bin/job COPY cmd/tools/benchmark/job/sample.yaml /etc/server/config.yaml # skipcq: DOK-DL3002 USER nonroot:nonroot -ENTRYPOINT ["/usr/bin/job"] \ No newline at end of file +ENTRYPOINT ["/usr/bin/job"] diff --git a/dockers/tools/benchmark/operator/Dockerfile b/dockers/tools/benchmark/operator/Dockerfile index 283773b11e..efe0a34e9f 100644 --- a/dockers/tools/benchmark/operator/Dockerfile +++ b/dockers/tools/benchmark/operator/Dockerfile @@ -83,4 +83,4 @@ COPY --from=builder /usr/bin/operator /usr/bin/operator COPY cmd/tools/benchmark/operator/sample.yaml /etc/server/config.yaml # skipcq: DOK-DL3002 USER nonroot:nonroot -ENTRYPOINT ["/usr/bin/operator"] \ No newline at end of file +ENTRYPOINT ["/usr/bin/operator"] diff --git a/dockers/tools/cli/loadtest/Dockerfile b/dockers/tools/cli/loadtest/Dockerfile index ffbea81b05..d061b486f9 100644 --- a/dockers/tools/cli/loadtest/Dockerfile +++ b/dockers/tools/cli/loadtest/Dockerfile @@ -91,4 +91,4 @@ COPY --from=builder /usr/bin/loadtest /usr/bin/loadtest COPY cmd/tools/cli/loadtest/sample.yaml /etc/server/config.yaml # skipcq: DOK-DL3002 USER nonroot:nonroot -ENTRYPOINT ["/usr/bin/loadtest"] \ No newline at end of file +ENTRYPOINT ["/usr/bin/loadtest"] diff --git a/example/client/go.mod b/example/client/go.mod index f64e6546d4..40915d3177 100644 --- a/example/client/go.mod +++ b/example/client/go.mod @@ -11,9 +11,9 @@ replace ( golang.org/x/crypto => golang.org/x/crypto v0.26.0 golang.org/x/net => golang.org/x/net v0.28.0 golang.org/x/text => golang.org/x/text v0.17.0 - google.golang.org/genproto => google.golang.org/genproto v0.0.0-20240805194559-2c9e96a0b5d4 - google.golang.org/genproto/googleapis/api => google.golang.org/genproto/googleapis/api v0.0.0-20240805194559-2c9e96a0b5d4 - google.golang.org/genproto/googleapis/rpc => google.golang.org/genproto/googleapis/rpc v0.0.0-20240805194559-2c9e96a0b5d4 + google.golang.org/genproto => google.golang.org/genproto v0.0.0-20240808171019-573a1156607a + google.golang.org/genproto/googleapis/api => google.golang.org/genproto/googleapis/api v0.0.0-20240808171019-573a1156607a + google.golang.org/genproto/googleapis/rpc => google.golang.org/genproto/googleapis/rpc v0.0.0-20240808171019-573a1156607a google.golang.org/grpc => google.golang.org/grpc v1.65.0 google.golang.org/protobuf => google.golang.org/protobuf v1.34.2 gopkg.in/yaml.v2 => gopkg.in/yaml.v2 v2.4.0 diff --git a/example/client/go.sum b/example/client/go.sum index d91b037bda..2467dd8ec9 100644 --- a/example/client/go.sum +++ b/example/client/go.sum @@ -30,10 +30,10 @@ golang.org/x/text v0.17.0 h1:XtiM5bkSOt+ewxlOE/aE/AKEHibwj/6gvWMl9Rsh0Qc= golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= gonum.org/v1/hdf5 v0.0.0-20210714002203-8c5d23bc6946 h1:vJpL69PeUullhJyKtTjHjENEmZU3BkO4e+fod7nKzgM= gonum.org/v1/hdf5 v0.0.0-20210714002203-8c5d23bc6946/go.mod h1:BQUWDHIAygjdt1HnUPQ0eWqLN2n5FwJycrpYUVUOx2I= -google.golang.org/genproto/googleapis/api v0.0.0-20240805194559-2c9e96a0b5d4 h1:ABEBT/sZ7We8zd7A5f3KO6zMQe+s3901H7l8Whhijt0= -google.golang.org/genproto/googleapis/api v0.0.0-20240805194559-2c9e96a0b5d4/go.mod h1:4+X6GvPs+25wZKbQq9qyAXrwIRExv7w0Ea6MgZLZiDM= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240805194559-2c9e96a0b5d4 h1:OsSGQeIIsyOEOimVxLEIL4rwGcnrjOydQaiA2bOnZUM= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240805194559-2c9e96a0b5d4/go.mod h1:Ue6ibwXGpU+dqIcODieyLOcgj7z8+IcskoNIgZxtrFY= +google.golang.org/genproto/googleapis/api v0.0.0-20240808171019-573a1156607a h1:KyUe15n7B1YCu+kMmPtlXxgkLQbp+Dw0tCRZf9Sd+CE= +google.golang.org/genproto/googleapis/api v0.0.0-20240808171019-573a1156607a/go.mod h1:4+X6GvPs+25wZKbQq9qyAXrwIRExv7w0Ea6MgZLZiDM= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240808171019-573a1156607a h1:EKiZZXueP9/T68B8Nl0GAx9cjbQnCId0yP3qPMgaaHs= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240808171019-573a1156607a/go.mod h1:Ue6ibwXGpU+dqIcODieyLOcgj7z8+IcskoNIgZxtrFY= google.golang.org/grpc v1.65.0 h1:bs/cUb4lp1G5iImFFd3u5ixQzweKizoZJAwBNLR42lc= google.golang.org/grpc v1.65.0/go.mod h1:WgYC2ypjlB0EiQi6wdKixMqukr6lBc0Vo+oOgjrM5ZQ= google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= diff --git a/go.mod b/go.mod index 333ed08f29..abf69b3445 100644 --- a/go.mod +++ b/go.mod @@ -5,17 +5,17 @@ go 1.22.6 replace ( cloud.google.com/go => cloud.google.com/go v0.115.0 cloud.google.com/go/bigquery => cloud.google.com/go/bigquery v1.62.0 - cloud.google.com/go/compute => cloud.google.com/go/compute v1.27.4 + cloud.google.com/go/compute => cloud.google.com/go/compute v1.27.5 cloud.google.com/go/datastore => cloud.google.com/go/datastore v1.17.1 cloud.google.com/go/firestore => cloud.google.com/go/firestore v1.16.0 - cloud.google.com/go/iam => cloud.google.com/go/iam v1.1.12 - cloud.google.com/go/kms => cloud.google.com/go/kms v1.18.4 - cloud.google.com/go/monitoring => cloud.google.com/go/monitoring v1.20.3 + cloud.google.com/go/iam => cloud.google.com/go/iam v1.1.13 + cloud.google.com/go/kms => cloud.google.com/go/kms v1.18.5 + cloud.google.com/go/monitoring => cloud.google.com/go/monitoring v1.20.4 cloud.google.com/go/pubsub => cloud.google.com/go/pubsub v1.41.0 - cloud.google.com/go/secretmanager => cloud.google.com/go/secretmanager v1.13.5 + cloud.google.com/go/secretmanager => cloud.google.com/go/secretmanager v1.13.6 cloud.google.com/go/storage => cloud.google.com/go/storage v1.43.0 - cloud.google.com/go/trace => cloud.google.com/go/trace v1.10.11 - code.cloudfoundry.org/bytefmt => code.cloudfoundry.org/bytefmt v0.0.0-20240806182212-6cf545ebdd6b + cloud.google.com/go/trace => cloud.google.com/go/trace v1.10.12 + code.cloudfoundry.org/bytefmt => code.cloudfoundry.org/bytefmt v0.0.0-20240808182453-a379845013d9 contrib.go.opencensus.io/exporter/aws => contrib.go.opencensus.io/exporter/aws v0.0.0-20230502192102-15967c811cec contrib.go.opencensus.io/exporter/prometheus => contrib.go.opencensus.io/exporter/prometheus v0.4.2 contrib.go.opencensus.io/integrations/ocsql => contrib.go.opencensus.io/integrations/ocsql v0.1.7 @@ -63,7 +63,7 @@ replace ( github.com/aws/aws-sdk-go-v2/service/secretsmanager => github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.32.4 github.com/aws/aws-sdk-go-v2/service/sns => github.com/aws/aws-sdk-go-v2/service/sns v1.31.3 github.com/aws/aws-sdk-go-v2/service/sqs => github.com/aws/aws-sdk-go-v2/service/sqs v1.34.3 - github.com/aws/aws-sdk-go-v2/service/ssm => github.com/aws/aws-sdk-go-v2/service/ssm v1.52.3 + github.com/aws/aws-sdk-go-v2/service/ssm => github.com/aws/aws-sdk-go-v2/service/ssm v1.52.4 github.com/aws/aws-sdk-go-v2/service/sso => github.com/aws/aws-sdk-go-v2/service/sso v1.22.4 github.com/aws/aws-sdk-go-v2/service/sts => github.com/aws/aws-sdk-go-v2/service/sts v1.30.3 github.com/aws/smithy-go => github.com/aws/smithy-go v1.20.3 @@ -295,8 +295,8 @@ replace ( go.uber.org/zap => go.uber.org/zap v1.27.0 gocloud.dev => gocloud.dev v0.38.0 golang.org/x/crypto => golang.org/x/crypto v0.26.0 - golang.org/x/exp => golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 - golang.org/x/exp/typeparams => golang.org/x/exp/typeparams v0.0.0-20240719175910-8a7402abbf56 + golang.org/x/exp => golang.org/x/exp v0.0.0-20240808152545-0cdaa3abc0fa + golang.org/x/exp/typeparams => golang.org/x/exp/typeparams v0.0.0-20240808152545-0cdaa3abc0fa golang.org/x/image => golang.org/x/image v0.19.0 golang.org/x/lint => golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 golang.org/x/mobile => golang.org/x/mobile v0.0.0-20240806205939-81131f6468ab @@ -304,7 +304,7 @@ replace ( golang.org/x/net => golang.org/x/net v0.28.0 golang.org/x/oauth2 => golang.org/x/oauth2 v0.22.0 golang.org/x/sync => golang.org/x/sync v0.8.0 - golang.org/x/sys => golang.org/x/sys v0.23.0 + golang.org/x/sys => golang.org/x/sys v0.24.0 golang.org/x/term => golang.org/x/term v0.23.0 golang.org/x/text => golang.org/x/text v0.17.0 golang.org/x/time => golang.org/x/time v0.6.0 @@ -316,9 +316,9 @@ replace ( gonum.org/v1/plot => gonum.org/v1/plot v0.14.0 google.golang.org/api => google.golang.org/api v0.191.0 google.golang.org/appengine => google.golang.org/appengine v1.6.8 - google.golang.org/genproto => google.golang.org/genproto v0.0.0-20240805194559-2c9e96a0b5d4 - google.golang.org/genproto/googleapis/api => google.golang.org/genproto/googleapis/api v0.0.0-20240805194559-2c9e96a0b5d4 - google.golang.org/genproto/googleapis/rpc => google.golang.org/genproto/googleapis/rpc v0.0.0-20240805194559-2c9e96a0b5d4 + google.golang.org/genproto => google.golang.org/genproto v0.0.0-20240808171019-573a1156607a + google.golang.org/genproto/googleapis/api => google.golang.org/genproto/googleapis/api v0.0.0-20240808171019-573a1156607a + google.golang.org/genproto/googleapis/rpc => google.golang.org/genproto/googleapis/rpc v0.0.0-20240808171019-573a1156607a google.golang.org/grpc => google.golang.org/grpc v1.65.0 google.golang.org/grpc/cmd/protoc-gen-go-grpc => google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.5.1 google.golang.org/protobuf => google.golang.org/protobuf v1.34.2 @@ -336,10 +336,10 @@ replace ( k8s.io/client-go => k8s.io/client-go v0.30.3 k8s.io/component-base => k8s.io/component-base v0.30.3 k8s.io/klog/v2 => k8s.io/klog/v2 v2.130.1 - k8s.io/kube-openapi => k8s.io/kube-openapi v0.0.0-20240730131305-7a9a4e85957e + k8s.io/kube-openapi => k8s.io/kube-openapi v0.0.0-20240808142205-8e686545bdb8 k8s.io/kubernetes => k8s.io/kubernetes v0.30.3 k8s.io/metrics => k8s.io/metrics v0.30.3 - nhooyr.io/websocket => nhooyr.io/websocket v1.8.11 + nhooyr.io/websocket => nhooyr.io/websocket v1.8.17 rsc.io/pdf => rsc.io/pdf v0.1.1 sigs.k8s.io/controller-runtime => sigs.k8s.io/controller-runtime v0.18.4 sigs.k8s.io/json => sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd @@ -399,7 +399,7 @@ require ( golang.org/x/net v0.28.0 golang.org/x/oauth2 v0.22.0 golang.org/x/sync v0.8.0 - golang.org/x/sys v0.23.0 + golang.org/x/sys v0.24.0 golang.org/x/text v0.17.0 golang.org/x/time v0.6.0 golang.org/x/tools v0.24.0 @@ -422,7 +422,7 @@ require ( require ( cloud.google.com/go v0.115.0 // indirect - cloud.google.com/go/auth v0.7.3 // indirect + cloud.google.com/go/auth v0.8.0 // indirect cloud.google.com/go/auth/oauth2adapt v0.2.3 // indirect cloud.google.com/go/compute/metadata v0.5.0 // indirect cloud.google.com/go/iam v1.1.12 // indirect @@ -506,14 +506,14 @@ require ( go.starlark.net v0.0.0-20230525235612-a134d8f9ddca // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/crypto v0.26.0 // indirect - golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect + golang.org/x/exp v0.0.0-20240808152545-0cdaa3abc0fa // indirect golang.org/x/exp/typeparams v0.0.0-20240213143201-ec583247a57a // indirect golang.org/x/image v0.19.0 // indirect golang.org/x/mod v0.20.0 // indirect golang.org/x/term v0.23.0 // indirect golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect - google.golang.org/api v0.189.0 // indirect + google.golang.org/api v0.191.0 // indirect google.golang.org/genproto v0.0.0-20240730163845-b1a4ccb954bf // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/go.sum b/go.sum index 106b02efec..b68a61b14c 100644 --- a/go.sum +++ b/go.sum @@ -25,8 +25,9 @@ cloud.google.com/go/auth v0.6.0/go.mod h1:b4acV+jLQDyjwm4OXHYjNvRi4jvGBzHWJRtJcy cloud.google.com/go/auth v0.6.1/go.mod h1:eFHG7zDzbXHKmjJddFG/rBlcGp6t25SwRUiEQSlO4x4= cloud.google.com/go/auth v0.7.0/go.mod h1:D+WqdrpcjmiCgWrXmLLxOVq1GACoE36chW6KXoEvuIw= cloud.google.com/go/auth v0.7.2/go.mod h1:VEc4p5NNxycWQTMQEDQF0bd6aTMb6VgYDXEwiJJQAbs= -cloud.google.com/go/auth v0.7.3 h1:98Vr+5jMaCZ5NZk6e/uBgf60phTk/XN84r8QEWB9yjY= cloud.google.com/go/auth v0.7.3/go.mod h1:HJtWUx1P5eqjy/f6Iq5KeytNpbAcGolPhOgyop2LlzA= +cloud.google.com/go/auth v0.8.0 h1:y8jUJLl/Fg+qNBWxP/Hox2ezJvjkrPb952PC1p0G6A4= +cloud.google.com/go/auth v0.8.0/go.mod h1:qGVp/Y3kDRSDZ5gFD/XPUfYQ9xW1iI7q8RIRoCyBbJc= cloud.google.com/go/auth/oauth2adapt v0.2.2/go.mod h1:wcYjgpZI9+Yu7LyYBg4pqSiaRkfEK3GQcpb7C/uyF1Q= cloud.google.com/go/auth/oauth2adapt v0.2.3 h1:MlxF+Pd3OmSudg/b1yZ5lJwoXCEaeedAguodky1PcKI= cloud.google.com/go/auth/oauth2adapt v0.2.3/go.mod h1:tMQXOfZzFuNuUxOypHlQEXgdfX5cuhwU+ffUuXRJE8I= @@ -45,7 +46,7 @@ cloud.google.com/go/channel v1.17.11/go.mod h1:gjWCDBcTGQce/BSMoe2lAqhlq0dIRiZuk cloud.google.com/go/cloudbuild v1.16.5/go.mod h1:HXLpZ8QeYZgmDIWpbl9Gs22p6o6uScgQ/cV9HF9cIZU= cloud.google.com/go/clouddms v1.7.10/go.mod h1:PzHELq0QDyA7VaD9z6mzh2mxeBz4kM6oDe8YxMxd4RA= cloud.google.com/go/cloudtasks v1.12.12/go.mod h1:8UmM+duMrQpzzRREo0i3x3TrFjsgI/3FQw3664/JblA= -cloud.google.com/go/compute v1.27.4/go.mod h1:7JZS+h21ERAGHOy5qb7+EPyXlQwzshzrx1x6L9JhTqU= +cloud.google.com/go/compute v1.27.5/go.mod h1:DfwDGujFTdSeiE8b8ZqadF/uxHFBz+ekGsk8Zfi9dTA= cloud.google.com/go/compute/metadata v0.2.1/go.mod h1:jgHgmJd2RKBGzXqF5LR2EZMGxBkeanZ9wwa75XHJgOM= cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= cloud.google.com/go/compute/metadata v0.3.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k= @@ -84,12 +85,12 @@ cloud.google.com/go/gkehub v0.14.11/go.mod h1:CsmDJ4qbBnSPkoBltEubK6qGOjG0xNfeeT cloud.google.com/go/gkemulticloud v1.2.4/go.mod h1:PjTtoKLQpIRztrL+eKQw8030/S4c7rx/WvHydDJlpGE= cloud.google.com/go/grafeas v0.3.6/go.mod h1:to6ECAPgRO2xeqD8ISXHc70nObJuaKZThreQOjeOH3o= cloud.google.com/go/gsuiteaddons v1.6.11/go.mod h1:U7mk5PLBzDpHhgHv5aJkuvLp9RQzZFpa8hgWAB+xVIk= -cloud.google.com/go/iam v1.1.12 h1:JixGLimRrNGcxvJEQ8+clfLxPlbeZA6MuRJ+qJNQ5Xw= -cloud.google.com/go/iam v1.1.12/go.mod h1:9LDX8J7dN5YRyzVHxwQzrQs9opFFqn0Mxs9nAeB+Hhg= +cloud.google.com/go/iam v1.1.13 h1:7zWBXG9ERbMLrzQBRhFliAV+kjcRToDTgQT3CTwYyv4= +cloud.google.com/go/iam v1.1.13/go.mod h1:K8mY0uSXwEXS30KrnVb+j54LB/ntfZu1dr+4zFMNbus= cloud.google.com/go/iap v1.9.10/go.mod h1:pO0FEirrhMOT1H0WVwpD5dD9r3oBhvsunyBQtNXzzc0= cloud.google.com/go/ids v1.4.11/go.mod h1:+ZKqWELpJm8WcRRsSvKZWUdkriu4A3XsLLzToTv3418= cloud.google.com/go/iot v1.7.11/go.mod h1:0vZJOqFy9kVLbUXwTP95e0dWHakfR4u5IWqsKMGIfHk= -cloud.google.com/go/kms v1.18.4/go.mod h1:SG1bgQ3UWW6/KdPo9uuJnzELXY5YTTMJtDYvajiQ22g= +cloud.google.com/go/kms v1.18.5/go.mod h1:yXunGUGzabH8rjUPImp2ndHiGolHeWJJ0LODLedicIY= cloud.google.com/go/language v1.13.0/go.mod h1:B9FbD17g1EkilctNGUDAdSrBHiFOlKNErLljO7jplDU= cloud.google.com/go/lifesciences v0.9.11/go.mod h1:NMxu++FYdv55TxOBEvLIhiAvah8acQwXsz79i9l9/RY= cloud.google.com/go/logging v1.11.0/go.mod h1:5LDiJC/RxTt+fHc1LAt20R9TKiUTReDg6RuuFOZ67+A= @@ -108,7 +109,7 @@ cloud.google.com/go/maps v1.11.6/go.mod h1:MOS/NN0L6b7Kumr8bLux9XTpd8+D54DYxBMUj cloud.google.com/go/mediatranslation v0.8.11/go.mod h1:3sNEm0fx61eHk7rfzBzrljVV9XKr931xI3OFacQBVFg= cloud.google.com/go/memcache v1.10.11/go.mod h1:ubJ7Gfz/xQawQY5WO5pht4Q0dhzXBFeEszAeEJnwBHU= cloud.google.com/go/metastore v1.13.10/go.mod h1:RPhMnBxUmTLT1fN7fNbPqtH5EoGHueDxubmJ1R1yT84= -cloud.google.com/go/monitoring v1.20.3/go.mod h1:GPIVIdNznIdGqEjtRKQWTLcUeRnPjZW85szouimiczU= +cloud.google.com/go/monitoring v1.20.4/go.mod h1:v7F/UcLRw15EX7xq565N7Ae5tnYEE28+Cl717aTXG4c= cloud.google.com/go/networkconnectivity v1.14.10/go.mod h1:f7ZbGl4CV08DDb7lw+NmMXQTKKjMhgCEEwFbEukWuOY= cloud.google.com/go/networkmanagement v1.13.6/go.mod h1:WXBijOnX90IFb6sberjnGrVtZbgDNcPDUYOlGXmG8+4= cloud.google.com/go/networksecurity v0.9.11/go.mod h1:4xbpOqCwplmFgymAjPFM6ZIplVC6+eQ4m7sIiEq9oJA= @@ -134,7 +135,7 @@ cloud.google.com/go/resourcesettings v1.7.4/go.mod h1:seBdLuyeq+ol2u9G2+74GkSjQa cloud.google.com/go/retail v1.17.4/go.mod h1:oPkL1FzW7D+v/hX5alYIx52ro2FY/WPAviwR1kZZTMs= cloud.google.com/go/run v1.4.0/go.mod h1:4G9iHLjdOC+CQ0CzA0+6nLeR6NezVPmlj+GULmb0zE4= cloud.google.com/go/scheduler v1.10.12/go.mod h1:6DRtOddMWJ001HJ6MS148rtLSh/S2oqd2hQC3n5n9fQ= -cloud.google.com/go/secretmanager v1.13.5/go.mod h1:/OeZ88l5Z6nBVilV0SXgv6XJ243KP2aIhSWRMrbvDCQ= +cloud.google.com/go/secretmanager v1.13.6/go.mod h1:x2ySyOrqv3WGFRFn2Xk10iHmNmvmcEVSSqc30eb1bhw= cloud.google.com/go/security v1.17.4/go.mod h1:KMuDJH+sEB3KTODd/tLJ7kZK+u2PQt+Cfu0oAxzIhgo= cloud.google.com/go/securitycenter v1.33.1/go.mod h1:jeFisdYUWHr+ig72T4g0dnNCFhRwgwGoQV6GFuEwafw= cloud.google.com/go/servicedirectory v1.11.11/go.mod h1:pnynaftaj9LmRLIc6t3r7r7rdCZZKKxui/HaF/RqYfs= @@ -147,7 +148,7 @@ cloud.google.com/go/storagetransfer v1.10.10/go.mod h1:8+nX+WgQ2ZJJnK8e+RbK/zCXk cloud.google.com/go/talent v1.6.12/go.mod h1:nT9kNVuJhZX2QgqKZS6t6eCWZs5XEBYRBv6bIMnPmo4= cloud.google.com/go/texttospeech v1.7.11/go.mod h1:Ua125HU+WT2IkIo5MzQtuNpNEk72soShJQVdorZ1SAE= cloud.google.com/go/tpu v1.6.11/go.mod h1:W0C4xaSj1Ay3VX/H96FRvLt2HDs0CgdRPVI4e7PoCDk= -cloud.google.com/go/trace v1.10.11/go.mod h1:fUr5L3wSXerNfT0f1bBg08W4axS2VbHGgYcfH4KuTXU= +cloud.google.com/go/trace v1.10.12/go.mod h1:tYkAIta/gxgbBZ/PIzFxSH5blajgX4D00RpQqCG/GZs= cloud.google.com/go/translate v1.10.3/go.mod h1:GW0vC1qvPtd3pgtypCv4k4U8B7EdgK9/QEF2aJEUovs= cloud.google.com/go/translate v1.10.7/go.mod h1:mH/+8tvcItuy1cOWqU+/Y3iFHgkVUObNIQYI/kiFFiY= cloud.google.com/go/video v1.22.0/go.mod h1:CxPshUNAb1ucnzbtruEHlAal9XY+SPG2cFqC/woJzII= @@ -159,8 +160,8 @@ cloud.google.com/go/vpcaccess v1.7.11/go.mod h1:a2cuAiSCI4TVK0Dt6/dRjf22qQvfY+po cloud.google.com/go/webrisk v1.9.11/go.mod h1:mK6M8KEO0ZI7VkrjCq3Tjzw4vYq+3c4DzlMUDVaiswE= cloud.google.com/go/websecurityscanner v1.6.11/go.mod h1:vhAZjksELSg58EZfUQ1BMExD+hxqpn0G0DuyCZQjiTg= cloud.google.com/go/workflows v1.12.10/go.mod h1:RcKqCiOmKs8wFUEf3EwWZPH5eHc7Oq0kamIyOUCk0IE= -code.cloudfoundry.org/bytefmt v0.0.0-20240806182212-6cf545ebdd6b h1:C+YxvUtePwiSzniQagI/yDdWjh1Lx1TkstHBGN7OTnA= -code.cloudfoundry.org/bytefmt v0.0.0-20240806182212-6cf545ebdd6b/go.mod h1:9aVxojRyikUaWddFMb0A9tgpGMtDPhk1pnCfhY0/fA4= +code.cloudfoundry.org/bytefmt v0.0.0-20240808182453-a379845013d9 h1:8KlrGCtoaWaaxVxi9KzED38kNIWa1qafh9bNSVZ6otk= +code.cloudfoundry.org/bytefmt v0.0.0-20240808182453-a379845013d9/go.mod h1:eF2ZbltNI7Pv+8Cuyeksu9up5FN5konuH0trDJBuscw= dmitri.shuralyov.com/gpu/mtl v0.0.0-20201218220906-28db891af037/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= dmitri.shuralyov.com/gpu/mtl v0.0.0-20221208032759-85de2813cf6b/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= eliasnaur.com/font v0.0.0-20230308162249-dd43949cb42d/go.mod h1:OYVuxibdk9OSLX8vAqydtRPP87PyTFcT9uH3MlEGBQA= @@ -684,14 +685,14 @@ gocloud.dev v0.38.0 h1:SpxfaOc/Fp4PeO8ui7wRcCZV0EgXZ+IWcVSLn6ZMSw0= gocloud.dev v0.38.0/go.mod h1:3XjKvd2E5iVNu/xFImRzjN0d/fkNHe4s0RiKidpEUMQ= golang.org/x/crypto v0.26.0 h1:RrRspgV4mU+YwB4FYnuBoKsUapNIL5cohGAmSH3azsw= golang.org/x/crypto v0.26.0/go.mod h1:GY7jblb9wI+FOo5y8/S2oY4zWP07AkOJ4+jxCqdqn54= -golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8= -golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY= +golang.org/x/exp v0.0.0-20240808152545-0cdaa3abc0fa h1:ELnwvuAXPNtPk1TJRuGkI9fDTwym6AYBu0qzT8AcHdI= +golang.org/x/exp v0.0.0-20240808152545-0cdaa3abc0fa/go.mod h1:akd2r19cwCdwSwWeIdzYQGa/EZZyqcOdwWiwj5L5eKQ= golang.org/x/exp/shiny v0.0.0-20220827204233-334a2380cb91/go.mod h1:VjAR7z0ngyATZTELrBSkxOOHhhlnVUxDye4mcjx5h/8= golang.org/x/exp/shiny v0.0.0-20230801115018-d63ba01acd4b/go.mod h1:UH99kUObWAZkDnWqppdQe5ZhPYESUw8I0zVV1uWBR+0= golang.org/x/exp/shiny v0.0.0-20230817173708-d852ddb80c63/go.mod h1:UH99kUObWAZkDnWqppdQe5ZhPYESUw8I0zVV1uWBR+0= golang.org/x/exp/shiny v0.0.0-20240707233637-46b078467d37/go.mod h1:3F+MieQB7dRYLTmnncoFbb1crS5lfQoTfDgQy6K4N0o= -golang.org/x/exp/typeparams v0.0.0-20240719175910-8a7402abbf56 h1:i+QrZdyNyfLEnWjd5T6LQZvQP3xk2XiNs3sQgN7QDGE= -golang.org/x/exp/typeparams v0.0.0-20240719175910-8a7402abbf56/go.mod h1:AbB0pIl9nAr9wVwH+Z2ZpaocVmF5I4GyWCDIsVjR0bk= +golang.org/x/exp/typeparams v0.0.0-20240808152545-0cdaa3abc0fa h1:54T+HVkPu4D3lltpEHyI3Fs2pG/GqjGkXLgyKOmifXk= +golang.org/x/exp/typeparams v0.0.0-20240808152545-0cdaa3abc0fa/go.mod h1:AbB0pIl9nAr9wVwH+Z2ZpaocVmF5I4GyWCDIsVjR0bk= golang.org/x/image v0.19.0 h1:D9FX4QWkLfkeqaC62SonffIIuYdOk/UE2XKUBgRIBIQ= golang.org/x/image v0.19.0/go.mod h1:y0zrRqlQRWQ5PXaYCOMLTW2fpsxZ8Qh9I/ohnInJEys= golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= @@ -704,8 +705,8 @@ golang.org/x/oauth2 v0.22.0 h1:BzDx2FehcG7jJwgWLELCdmLuxk2i+x9UDpSiss2u0ZA= golang.org/x/oauth2 v0.22.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= -golang.org/x/sys v0.23.0 h1:YfKFowiIMvtgl1UERQoTPPToxltDeZfbj4H7dVUCwmM= -golang.org/x/sys v0.23.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.24.0 h1:Twjiwq9dn6R1fQcyiK+wQyHWfaz/BJB+YIpzU/Cv3Xg= +golang.org/x/sys v0.24.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/telemetry v0.0.0-20240521205824-bda55230c457/go.mod h1:pRgIJT+bRLFKnoM1ldnzKoxTIn14Yxz928LQRYYgIN0= golang.org/x/term v0.23.0 h1:F6D4vR+EHoL9/sWAWgAR1H2DcHr4PareCbAaCo1RpuU= golang.org/x/term v0.23.0/go.mod h1:DgV24QBUrK6jhZXl+20l6UWznPlwAHm1Q1mGHtydmSk= @@ -728,13 +729,13 @@ gonum.org/v1/plot v0.14.0/go.mod h1:MLdR9424SJed+5VqC6MsouEpig9pZX2VZ57H9ko2bXU= google.golang.org/api v0.191.0 h1:cJcF09Z+4HAB2t5qTQM1ZtfL/PemsLFkcFG67qq2afk= google.golang.org/api v0.191.0/go.mod h1:tD5dsFGxFza0hnQveGfVk9QQYKcfp+VzgRqyXFxE0+E= google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= -google.golang.org/genproto v0.0.0-20240805194559-2c9e96a0b5d4 h1:g+rQ3aqOyXK/0qwnC5TGUXnyIeipstP5SsniB9uPJ2c= -google.golang.org/genproto v0.0.0-20240805194559-2c9e96a0b5d4/go.mod h1:7uvplUBj4RjHAxIZ//98LzOvrQ04JBkaixRmCMI29hc= -google.golang.org/genproto/googleapis/api v0.0.0-20240805194559-2c9e96a0b5d4 h1:ABEBT/sZ7We8zd7A5f3KO6zMQe+s3901H7l8Whhijt0= -google.golang.org/genproto/googleapis/api v0.0.0-20240805194559-2c9e96a0b5d4/go.mod h1:4+X6GvPs+25wZKbQq9qyAXrwIRExv7w0Ea6MgZLZiDM= +google.golang.org/genproto v0.0.0-20240808171019-573a1156607a h1:3JVv3Ujh+kGiajpSqHWnbWPuu0nQqMZ3hASNDDF9974= +google.golang.org/genproto v0.0.0-20240808171019-573a1156607a/go.mod h1:7uvplUBj4RjHAxIZ//98LzOvrQ04JBkaixRmCMI29hc= +google.golang.org/genproto/googleapis/api v0.0.0-20240808171019-573a1156607a h1:KyUe15n7B1YCu+kMmPtlXxgkLQbp+Dw0tCRZf9Sd+CE= +google.golang.org/genproto/googleapis/api v0.0.0-20240808171019-573a1156607a/go.mod h1:4+X6GvPs+25wZKbQq9qyAXrwIRExv7w0Ea6MgZLZiDM= google.golang.org/genproto/googleapis/bytestream v0.0.0-20240730163845-b1a4ccb954bf/go.mod h1:5/MT647Cn/GGhwTpXC7QqcaR5Cnee4v4MKCU1/nwnIQ= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240805194559-2c9e96a0b5d4 h1:OsSGQeIIsyOEOimVxLEIL4rwGcnrjOydQaiA2bOnZUM= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240805194559-2c9e96a0b5d4/go.mod h1:Ue6ibwXGpU+dqIcODieyLOcgj7z8+IcskoNIgZxtrFY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240808171019-573a1156607a h1:EKiZZXueP9/T68B8Nl0GAx9cjbQnCId0yP3qPMgaaHs= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240808171019-573a1156607a/go.mod h1:Ue6ibwXGpU+dqIcODieyLOcgj7z8+IcskoNIgZxtrFY= google.golang.org/grpc v1.65.0 h1:bs/cUb4lp1G5iImFFd3u5ixQzweKizoZJAwBNLR42lc= google.golang.org/grpc v1.65.0/go.mod h1:WgYC2ypjlB0EiQi6wdKixMqukr6lBc0Vo+oOgjrM5ZQ= google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= @@ -764,8 +765,8 @@ k8s.io/client-go v0.30.3 h1:bHrJu3xQZNXIi8/MoxYtZBBWQQXwy16zqJwloXXfD3k= k8s.io/client-go v0.30.3/go.mod h1:8d4pf8vYu665/kUbsxWAQ/JDBNWqfFeZnvFiVdmx89U= k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= -k8s.io/kube-openapi v0.0.0-20240730131305-7a9a4e85957e h1:OnKkExfhk4yxMqvBSPzUfhv3zQ96FWJ+UOZzLrAFyAo= -k8s.io/kube-openapi v0.0.0-20240730131305-7a9a4e85957e/go.mod h1:0CVn9SVo8PeW5/JgsBZZIFmmTk5noOM8WXf2e1tCihE= +k8s.io/kube-openapi v0.0.0-20240808142205-8e686545bdb8 h1:1Wof1cGQgA5pqgo8MxKPtf+qN6Sh/0JzznmeGPm1HnE= +k8s.io/kube-openapi v0.0.0-20240808142205-8e686545bdb8/go.mod h1:Os6V6dZwLNii3vxFpxcNaTmH8LJJBkOTg1N0tOA0fvA= k8s.io/metrics v0.30.3 h1:gKCpte5zykrOmQhZ8qmsxyJslMdiLN+sqbBfIWNpbGM= k8s.io/metrics v0.30.3/go.mod h1:W06L2nXRhOwPkFYDJYWdEIS3u6JcJy3ebIPYbndRs6A= k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 h1:pUdcCO1Lk/tbT5ztQWOBi5HBgbBP1J8+AsQnQCKsi8A= diff --git a/hack/docker/gen/main.go b/hack/docker/gen/main.go index 7750270efc..41e90af231 100644 --- a/hack/docker/gen/main.go +++ b/hack/docker/gen/main.go @@ -372,7 +372,6 @@ var ( "make kubelinter/install", "make reviewdog/install", "make tparse/install", - "make valdcli/install", "make yq/install", "make minikube/install", "make stern/install", diff --git a/internal/client/v1/client/vald/vald.go b/internal/client/v1/client/vald/vald.go index 00957b7b7b..280aa78be8 100644 --- a/internal/client/v1/client/vald/vald.go +++ b/internal/client/v1/client/vald/vald.go @@ -499,6 +499,28 @@ func (c *client) MultiUpdate( return res, nil } +func (c *client) UpdateTimestamp( + ctx context.Context, in *payload.Object_Timestamp, opts ...grpc.CallOption, +) (res *payload.Object_Location, err error) { + ctx, span := trace.StartSpan(grpc.WrapGRPCMethod(ctx, "internal/client/"+vald.UpdateTimestampRPCName), apiName+"/"+vald.UpdateTimestampRPCName) + defer func() { + if span != nil { + span.End() + } + }() + _, err = c.c.RoundRobin(ctx, func(ctx context.Context, + conn *grpc.ClientConn, + copts ...grpc.CallOption, + ) (any, error) { + res, err = vald.NewValdClient(conn).UpdateTimestamp(ctx, in, append(copts, opts...)...) + return nil, err + }) + if err != nil { + return nil, err + } + return res, nil +} + func (c *client) Upsert( ctx context.Context, in *payload.Upsert_Request, opts ...grpc.CallOption, ) (res *payload.Object_Location, err error) { @@ -1066,6 +1088,18 @@ func (c *singleClient) Update( return c.vc.Update(ctx, in, opts...) } +func (c *singleClient) UpdateTimestamp( + ctx context.Context, in *payload.Object_Timestamp, opts ...grpc.CallOption, +) (res *payload.Object_Location, err error) { + ctx, span := trace.StartSpan(grpc.WrapGRPCMethod(ctx, "internal/singleClient/"+vald.UpdateTimestampRPCName), apiName+"/"+vald.UpdateTimestampRPCName) + defer func() { + if span != nil { + span.End() + } + }() + return c.vc.UpdateTimestamp(ctx, in, opts...) +} + func (c *singleClient) StreamUpdate( ctx context.Context, opts ...grpc.CallOption, ) (res vald.Update_StreamUpdateClient, err error) { diff --git a/internal/errors/agent.go b/internal/errors/agent.go index da16986ade..a25d13d043 100644 --- a/internal/errors/agent.go +++ b/internal/errors/agent.go @@ -115,4 +115,9 @@ var ( // ErrWriteOperationToReadReplica represents an error that when a write operation is made to read replica. ErrWriteOperationToReadReplica = New("write operation to read replica is not possible") + + // ErrInvalidTimestamp represents a function to generate an error that the timestamp is invalid. + ErrInvalidTimestamp = func(ts int64) error { + return Errorf("invalid timestamp detected", ts) + } ) diff --git a/pkg/gateway/lb/handler/grpc/handler.go b/pkg/gateway/lb/handler/grpc/handler.go index acfdfad8f3..555c3d739b 100644 --- a/pkg/gateway/lb/handler/grpc/handler.go +++ b/pkg/gateway/lb/handler/grpc/handler.go @@ -361,55 +361,17 @@ func (s *server) SearchByID( } return nil, err } - vec, err := s.getObject(ctx, uuid) + vec, err := s.GetObject(ctx, &payload.Object_VectorRequest{ + Id: &payload.Object_ID{ + Id: uuid, + }, + }) if err != nil { - var ( - attrs trace.Attributes - st *status.Status - msg string - ) - switch { - case errors.Is(err, errors.ErrInvalidUUID(uuid)): - err = status.WrapWithInvalidArgument( - vald.GetObjectRPCName+" API for "+vald.SearchByIDRPCName+" API invalid argument for uuid \""+uuid+"\" detected", - err, - reqInfo, - resInfo, - &errdetails.BadRequest{ - FieldViolations: []*errdetails.BadRequestFieldViolation{ - { - Field: "uuid", - Description: err.Error(), - }, - }, - }, - ) - attrs = trace.StatusCodeInvalidArgument(err.Error()) - case errors.Is(err, errors.ErrGRPCClientConnNotFound("*")): - err = status.WrapWithInternal(vald.GetObjectRPCName+" API for "+vald.SearchByIDRPCName+" API connection not found", err, reqInfo, resInfo) - attrs = trace.StatusCodeInternal(err.Error()) - case errors.Is(err, context.Canceled): - err = status.WrapWithCanceled(vald.GetObjectRPCName+" API for "+vald.SearchByIDRPCName+" API canceled", err, reqInfo, resInfo) - attrs = trace.StatusCodeCancelled(err.Error()) - case errors.Is(err, context.DeadlineExceeded): - err = status.WrapWithDeadlineExceeded(vald.GetObjectRPCName+" API for "+vald.SearchByIDRPCName+" API deadline exceeded", err, reqInfo, resInfo) - attrs = trace.StatusCodeDeadlineExceeded(err.Error()) - case errors.Is(err, errors.ErrObjectIDNotFound(uuid)), errors.Is(err, errors.ErrObjectNotFound(nil, uuid)): - err = nil - default: - st, msg, err = status.ParseError(err, codes.Unknown, vald.GetObjectRPCName+" API for "+vald.SearchByIDRPCName+" API uuid "+uuid+"'s request returned error", reqInfo, resInfo) - attrs = trace.FromGRPCStatus(st.Code(), msg) - if st == nil || st.Code() == codes.NotFound { - err = nil - } - } - if err != nil { - if span != nil { - span.RecordError(err) - span.SetAttributes(attrs...) - span.SetStatus(trace.StatusError, err.Error()) - } - return nil, err + st, msg, err := status.ParseError(err, codes.Unknown, vald.GetObjectRPCName+" API for "+vald.SearchByIDRPCName+" API uuid "+uuid+"'s request returned error", reqInfo, resInfo) + if span != nil && st != nil && st.Code() != codes.NotFound { + span.RecordError(err) + span.SetAttributes(trace.FromGRPCStatus(st.Code(), st.Message())...) + span.SetStatus(trace.StatusError, err.Error()) } // try search by using agent's SearchByID method this operation is emergency fallback, the search quality is not same as usual SearchByID operation. res, err = s.doSearch(ctx, req.GetConfig(), func(ctx context.Context, fcfg *payload.Search_Config, vc vald.Client, copts ...grpc.CallOption) (*payload.Search_Response, error) { @@ -851,55 +813,17 @@ func (s *server) LinearSearchByID( } return nil, err } - vec, err := s.getObject(ctx, uuid) + vec, err := s.GetObject(ctx, &payload.Object_VectorRequest{ + Id: &payload.Object_ID{ + Id: uuid, + }, + }) if err != nil { - var ( - attrs trace.Attributes - st *status.Status - msg string - ) - switch { - case errors.Is(err, errors.ErrInvalidUUID(uuid)): - err = status.WrapWithInvalidArgument( - vald.GetObjectRPCName+" API for "+vald.LinearSearchByIDRPCName+" API invalid argument for uuid \""+uuid+"\" detected", - err, - reqInfo, - resInfo, - &errdetails.BadRequest{ - FieldViolations: []*errdetails.BadRequestFieldViolation{ - { - Field: "uuid", - Description: err.Error(), - }, - }, - }, - ) - attrs = trace.StatusCodeInvalidArgument(err.Error()) - case errors.Is(err, errors.ErrGRPCClientConnNotFound("*")): - err = status.WrapWithInternal(vald.GetObjectRPCName+" API for "+vald.LinearSearchByIDRPCName+" API connection not found", err, reqInfo, resInfo) - attrs = trace.StatusCodeInternal(err.Error()) - case errors.Is(err, context.Canceled): - err = status.WrapWithCanceled(vald.GetObjectRPCName+" API for "+vald.LinearSearchByIDRPCName+" API canceled", err, reqInfo, resInfo) - attrs = trace.StatusCodeCancelled(err.Error()) - case errors.Is(err, context.DeadlineExceeded): - err = status.WrapWithDeadlineExceeded(vald.GetObjectRPCName+" API for "+vald.LinearSearchByIDRPCName+" API deadline exceeded", err, reqInfo, resInfo) - attrs = trace.StatusCodeDeadlineExceeded(err.Error()) - case errors.Is(err, errors.ErrObjectIDNotFound(uuid)), errors.Is(err, errors.ErrObjectNotFound(nil, uuid)): - err = nil - default: - st, msg, err = status.ParseError(err, codes.Unknown, vald.GetObjectRPCName+" API for "+vald.LinearSearchByIDRPCName+" API uuid "+uuid+"'s request returned error", reqInfo, resInfo) - attrs = trace.FromGRPCStatus(st.Code(), msg) - if st == nil || st.Code() == codes.NotFound { - err = nil - } - } - if err != nil { - if span != nil { - span.RecordError(err) - span.SetAttributes(attrs...) - span.SetStatus(trace.StatusError, err.Error()) - } - return nil, err + st, msg, err := status.ParseError(err, codes.Unknown, vald.GetObjectRPCName+" API for "+vald.LinearSearchByIDRPCName+" API uuid "+uuid+"'s request returned error", reqInfo, resInfo) + if span != nil && st != nil && st.Code() != codes.NotFound { + span.RecordError(err) + span.SetAttributes(trace.FromGRPCStatus(st.Code(), st.Message())...) + span.SetStatus(trace.StatusError, err.Error()) } // try search by using agent's LinearSearchByID method this operation is emergency fallback, the search quality is not same as usual LinearSearchByID operation. res, err = s.doSearch(ctx, req.GetConfig(), func(ctx context.Context, fcfg *payload.Search_Config, vc vald.Client, copts ...grpc.CallOption) (*payload.Search_Response, error) { @@ -1778,54 +1702,16 @@ func (s *server) Update( } if !req.GetConfig().GetSkipStrictExistCheck() { - vec, err := s.getObject(ctx, uuid) - if err != nil || vec == nil { - var ( - attrs trace.Attributes - st *status.Status - msg string - ) - switch { - case errors.Is(err, errors.ErrInvalidUUID(uuid)): - err = status.WrapWithInvalidArgument( - vald.GetObjectRPCName+" API for "+vald.UpdateRPCName+" API invalid argument for uuid \""+uuid+"\" detected", - err, - reqInfo, - resInfo, - &errdetails.BadRequest{ - FieldViolations: []*errdetails.BadRequestFieldViolation{ - { - Field: "uuid", - Description: err.Error(), - }, - }, - }, - ) - attrs = trace.StatusCodeInvalidArgument(err.Error()) - case errors.Is(err, errors.ErrGRPCClientConnNotFound("*")): - err = status.WrapWithInternal(vald.GetObjectRPCName+" API for "+vald.UpdateRPCName+" API connection not found", err, reqInfo, resInfo) - attrs = trace.StatusCodeInternal(err.Error()) - case errors.Is(err, context.Canceled): - err = status.WrapWithCanceled(vald.GetObjectRPCName+" API for "+vald.UpdateRPCName+" API canceled", err, reqInfo, resInfo) - attrs = trace.StatusCodeCancelled(err.Error()) - case errors.Is(err, context.DeadlineExceeded): - err = status.WrapWithDeadlineExceeded(vald.GetObjectRPCName+" API for "+vald.UpdateRPCName+" API deadline exceeded", err, reqInfo, resInfo) - attrs = trace.StatusCodeDeadlineExceeded(err.Error()) - case errors.Is(err, errors.ErrObjectIDNotFound(uuid)), errors.Is(err, errors.ErrObjectNotFound(nil, uuid)): - err = status.WrapWithNotFound(vald.GetObjectRPCName+" API for "+vald.UpdateRPCName+" API uuid "+uuid+"'s object not found", err, reqInfo, resInfo) - attrs = trace.StatusCodeNotFound(err.Error()) - default: - code := codes.Unknown - if err == nil { - err = errors.ErrObjectIDNotFound(uuid) - code = codes.NotFound - } - st, msg, err = status.ParseError(err, code, vald.GetObjectRPCName+" API for "+vald.UpdateRPCName+" API uuid "+uuid+"'s request returned error", reqInfo, resInfo) - attrs = trace.FromGRPCStatus(st.Code(), msg) - } - if span != nil { + vec, err := s.GetObject(ctx, &payload.Object_VectorRequest{ + Id: &payload.Object_ID{ + Id: uuid, + }, + }) + if err != nil { + st, msg, err := status.ParseError(err, codes.Unknown, vald.GetObjectRPCName+" API for "+vald.UpdateRPCName+" API uuid "+uuid+"'s request returned error", reqInfo, resInfo) + if span != nil && st != nil { span.RecordError(err) - span.SetAttributes(attrs...) + span.SetAttributes(trace.FromGRPCStatus(st.Code(), msg)...) span.SetStatus(trace.StatusError, err.Error()) } return nil, err @@ -2121,6 +2007,221 @@ func (s *server) MultiUpdate( return locs, errs } +func (s *server) UpdateTimestamp( + ctx context.Context, req *payload.Object_Timestamp, +) (res *payload.Object_Location, err error) { + ctx, span := trace.StartSpan(grpc.WithGRPCMethod(ctx, vald.PackageName+"."+vald.UpdateRPCServiceName+"/"+vald.UpdateTimestampRPCName), apiName+"/"+vald.UpdateTimestampRPCName) + defer func() { + if span != nil { + span.End() + } + }() + uuid := req.GetId() + reqInfo := &errdetails.RequestInfo{ + RequestId: uuid, + ServingData: errdetails.Serialize(req), + } + resInfo := &errdetails.ResourceInfo{ + ResourceType: errdetails.ValdGRPCResourceTypePrefix + "/vald.v1." + vald.UpdateTimestampRPCName + "." + vald.GetObjectRPCName, + ResourceName: fmt.Sprintf("%s: %s(%s) to %v", apiName, s.name, s.ip, s.gateway.Addrs(ctx)), + } + if len(uuid) == 0 { + err = errors.ErrInvalidMetaDataConfig + err = status.WrapWithInvalidArgument(vald.UpdateTimestampRPCName+" API invalid uuid", err, reqInfo, resInfo, + &errdetails.BadRequest{ + FieldViolations: []*errdetails.BadRequestFieldViolation{ + { + Field: "invalid id", + Description: err.Error(), + }, + }, + }) + if span != nil { + span.RecordError(err) + span.SetAttributes(trace.StatusCodeInvalidArgument(err.Error())...) + span.SetStatus(trace.StatusError, err.Error()) + } + return nil, err + } + ts := req.GetTimestamp() + if ts < 0 { + err = errors.ErrInvalidTimestamp(ts) + err = status.WrapWithInvalidArgument(vald.UpdateTimestampRPCName+" API invalid vector argument", err, reqInfo, resInfo, + &errdetails.BadRequest{ + FieldViolations: []*errdetails.BadRequestFieldViolation{ + { + Field: "timestamp", + Description: err.Error(), + }, + }, + }, info.Get()) + if span != nil { + span.RecordError(err) + span.SetAttributes(trace.StatusCodeInvalidArgument(err.Error())...) + span.SetStatus(trace.StatusError, err.Error()) + } + return nil, err + } + var ( + mu sync.RWMutex + aeCount atomic.Uint64 + updated atomic.Uint64 + ls = make([]string, 0, s.replica) + visited = make(map[string]bool, s.replica) + locs = &payload.Object_Location{ + Uuid: uuid, + Ips: make([]string, 0, s.replica), + } + ) + err = s.gateway.BroadCast(ctx, service.WRITE, func(ctx context.Context, target string, vc vald.Client, copts ...grpc.CallOption) (err error) { + ctx, span := trace.StartSpan(grpc.WrapGRPCMethod(ctx, "BroadCast/"+target), apiName+"/"+vald.UpdateRPCName+"/"+target) + defer func() { + if span != nil { + span.End() + } + }() + loc, err := vc.UpdateTimestamp(ctx, req, copts...) + if err != nil { + st, ok := status.FromError(err) + if ok && st != nil { + if st.Code() != codes.AlreadyExists && + st.Code() != codes.Canceled && + st.Code() != codes.DeadlineExceeded && + st.Code() != codes.InvalidArgument && + st.Code() != codes.NotFound && + st.Code() != codes.OK && + st.Code() != codes.Unimplemented { + if span != nil { + span.RecordError(err) + span.SetAttributes(trace.FromGRPCStatus(st.Code(), fmt.Sprintf("UpdateTimestamp operation for Agent %s failed,\terror: %v", target, err))...) + span.SetStatus(trace.StatusError, err.Error()) + } + return err + } + if st.Code() == codes.AlreadyExists { + host, _, err := net.SplitHostPort(target) + if err != nil { + host = target + } + aeCount.Add(1) + mu.Lock() + visited[target] = true + locs.Ips = append(locs.GetIps(), host) + ls = append(ls, host) + mu.Unlock() + + } + } + return nil + } + if loc != nil { + updated.Add(1) + mu.Lock() + visited[target] = true + locs.Ips = append(locs.GetIps(), loc.GetIps()...) + ls = append(ls, loc.GetName()) + mu.Unlock() + } + return nil + }) + switch { + case err != nil: + st, msg, err := status.ParseError(err, codes.Internal, + "failed to parse "+vald.UpdateRPCName+" gRPC error response", reqInfo, resInfo, info.Get()) + if span != nil { + span.RecordError(err) + span.SetAttributes(trace.FromGRPCStatus(st.Code(), msg)...) + span.SetStatus(trace.StatusError, err.Error()) + } + return nil, err + case len(locs.Ips) <= 0: + err = errors.ErrIndexNotFound + err = status.WrapWithNotFound(vald.UpdateRPCName+" API update target not found", err, reqInfo, resInfo) + if span != nil { + span.RecordError(err) + span.SetAttributes(trace.StatusCodeNotFound(err.Error())...) + span.SetStatus(trace.StatusError, err.Error()) + } + return nil, err + case updated.Load()+aeCount.Load() < uint64(s.replica): + shortage := s.replica - int(updated.Load()+aeCount.Load()) + vec, err := s.GetObject(ctx, &payload.Object_VectorRequest{ + Id: &payload.Object_ID{ + Id: uuid, + }, + }) + if err != nil { + st, msg, err := status.ParseError(err, codes.Unknown, vald.GetObjectRPCName+" API for "+vald.UpdateTimestampRPCName+" API uuid "+uuid+"'s request returned error", reqInfo, resInfo) + if span != nil && st != nil { + span.RecordError(err) + span.SetAttributes(trace.FromGRPCStatus(st.Code(), msg)...) + span.SetStatus(trace.StatusError, err.Error()) + } + return nil, err + } + + err = s.gateway.DoMulti(ctx, shortage, func(ctx context.Context, target string, vc vald.Client, copts ...grpc.CallOption) (err error) { + mu.RLock() + tf, ok := visited[target] + mu.RUnlock() + if tf && ok { + return errors.Errorf("target: %s already inserted will skip", target) + } + ctx, span := trace.StartSpan(grpc.WrapGRPCMethod(ctx, "DoMulti/"+target), apiName+"/"+vald.InsertRPCName+"/"+target) + defer func() { + if span != nil { + span.End() + } + }() + loc, err := vc.Insert(ctx, &payload.Insert_Request{ + Vector: vec, + Config: &payload.Insert_Config{ + SkipStrictExistCheck: true, + Timestamp: ts, + }, + }, copts...) + if err != nil { + st, ok := status.FromError(err) + if ok && st != nil && span != nil { + span.RecordError(err) + span.SetAttributes(trace.FromGRPCStatus(st.Code(), fmt.Sprintf("Shortage index Insert for Update operation for Agent %s failed,\terror: %v", target, err))...) + span.SetStatus(trace.StatusError, err.Error()) + } + return err + } + if loc != nil { + updated.Add(1) + mu.Lock() + locs.Ips = append(locs.GetIps(), loc.GetIps()...) + ls = append(ls, loc.GetName()) + mu.Unlock() + } + return nil + }) + if err != nil { + st, msg, err := status.ParseError(err, codes.Unknown, vald.InsertRPCName+" API for "+vald.UpdateRPCName+" API uuid "+uuid+"'s request returned error", reqInfo, resInfo) + if span != nil && st != nil { + span.RecordError(err) + span.SetAttributes(trace.FromGRPCStatus(st.Code(), msg)...) + span.SetStatus(trace.StatusError, err.Error()) + } + return nil, err + } + case updated.Load() == 0 && aeCount.Load() > 0: + err = status.WrapWithAlreadyExists(vald.UpdateRPCName+" API update target same vector already exists", errors.ErrSameVectorAlreadyExists(uuid, nil, nil), reqInfo, resInfo) + if span != nil { + span.RecordError(err) + span.SetAttributes(trace.StatusCodeAlreadyExists(err.Error())...) + span.SetStatus(trace.StatusError, err.Error()) + } + return nil, err + + } + slices.Sort(ls) + locs.Name = strings.Join(ls, ",") + return locs, nil +} + func (s *server) Upsert( ctx context.Context, req *payload.Upsert_Request, ) (loc *payload.Object_Location, err error) { @@ -2180,48 +2281,23 @@ func (s *server) Upsert( } var shouldInsert bool if !req.GetConfig().GetSkipStrictExistCheck() { - vec, err := s.getObject(ctx, uuid) - var ( - attrs trace.Attributes - st *status.Status - msg string - ) + vec, err := s.GetObject(ctx, &payload.Object_VectorRequest{ + Id: &payload.Object_ID{ + Id: uuid, + }, + }) + var attrs trace.Attributes if err != nil || vec == nil { - switch { - case errors.Is(err, errors.ErrInvalidUUID(uuid)): - err = status.WrapWithInvalidArgument( - vald.GetObjectRPCName+" API for "+vald.UpsertRPCName+" API invalid argument for uuid \""+uuid+"\" detected", - err, - reqInfo, - resInfo, - &errdetails.BadRequest{ - FieldViolations: []*errdetails.BadRequestFieldViolation{ - { - Field: "uuid", - Description: err.Error(), - }, - }, - }, - ) - attrs = trace.StatusCodeInvalidArgument(err.Error()) - case errors.Is(err, errors.ErrGRPCClientConnNotFound("*")): - err = status.WrapWithInternal(vald.GetObjectRPCName+" API for "+vald.UpsertRPCName+" API connection not found", err, reqInfo, resInfo) - attrs = trace.StatusCodeInternal(err.Error()) - case errors.Is(err, context.Canceled): - err = status.WrapWithCanceled(vald.GetObjectRPCName+" API for "+vald.UpsertRPCName+" API canceled", err, reqInfo, resInfo) - attrs = trace.StatusCodeCancelled(err.Error()) - case errors.Is(err, context.DeadlineExceeded): - err = status.WrapWithDeadlineExceeded(vald.GetObjectRPCName+" API for "+vald.UpsertRPCName+" API deadline exceeded", err, reqInfo, resInfo) - attrs = trace.StatusCodeDeadlineExceeded(err.Error()) - case errors.Is(err, errors.ErrObjectIDNotFound(uuid)), errors.Is(err, errors.ErrObjectNotFound(nil, uuid)): - err = nil - shouldInsert = true - default: - st, msg, err = status.ParseError(err, codes.Unknown, vald.GetObjectRPCName+" API for "+vald.UpsertRPCName+" API uuid "+uuid+"'s request returned error", reqInfo, resInfo) + var ( + st *status.Status + msg string + ) + st, msg, err = status.ParseError(err, codes.Unknown, vald.GetObjectRPCName+" API for "+vald.UpsertRPCName+" API uuid "+uuid+"'s request returned error", reqInfo, resInfo) + if st != nil { attrs = trace.FromGRPCStatus(st.Code(), msg) - if st != nil && st.Code() == codes.NotFound { - err = nil + if st.Code() == codes.NotFound { shouldInsert = true + err = nil } } } else if conv.F32stos(vec.GetVector()) == conv.F32stos(req.GetVector().GetVector()) { @@ -2236,7 +2312,6 @@ func (s *server) Upsert( } return nil, err } - } else { id, err := s.exists(ctx, uuid) if err != nil { diff --git a/pkg/index/job/correction/service/corrector.go b/pkg/index/job/correction/service/corrector.go index 41a71b2274..c82ef65b24 100644 --- a/pkg/index/job/correction/service/corrector.go +++ b/pkg/index/job/correction/service/corrector.go @@ -136,15 +136,22 @@ func (c *correct) Start(ctx context.Context) (err error) { return err } counts := detail.GetCounts() - agents := make([]string, 0, detail.GetLiveAgents()) - for agent, count := range counts { - log.Infof("index info: addr(%s), stored(%d), uncommitted(%d), indexing=%t, saving=%t", agent, count.GetStored(), count.GetUncommitted(), count.GetIndexing(), count.GetSaving()) + agents := make([]string, 0, len(counts)) + for agent := range counts { agents = append(agents, agent) } slices.SortFunc(agents, func(left, right string) int { - return cmp.Compare(counts[left].GetStored(), counts[right].GetStored()) + return cmp.Compare(counts[right].GetStored(), counts[left].GetStored()) }) + for _, agent := range agents { + count, ok := counts[agent] + if ok && count != nil { + log.Infof("index info: addr(%s), stored(%d), uncommitted(%d), indexing=%t, saving=%t", agent, count.GetStored(), count.GetUncommitted(), count.GetIndexing(), count.GetSaving()) + } + } + log.Infof("sorted agents: %v,\tdiscovered agents: %v", agents, c.discoverer.GetAddrs(ctx)) + errs := make([]error, 0, len(agents)) emptyReq := new(payload.Object_List_Request) @@ -189,7 +196,7 @@ func (c *correct) Start(ctx context.Context) (err error) { indexing = count.GetIndexing() saving = count.GetSaving() } - debugMsg := fmt.Sprintf("agent %s (stored: %d, uncommitted: %d, indexing=%t, saving=%t), stream concurrency: %d, processing %d/%d, replicas: size(%d) = addrs%v", addr, stored, uncommitted, indexing, saving, c.streamListConcurrency, corrected, len(agents), len(replicas), replicas) + debugMsg := fmt.Sprintf("agent %s (total index detail = stored: %d, uncommitted: %d, indexing=%t, saving=%t), stream concurrency: %d, processing %d/%d, replicas: size(%d) = addrs%v", addr, stored, uncommitted, indexing, saving, c.streamListConcurrency, corrected, len(agents), len(replicas), replicas) eg, egctx := errgroup.WithContext(ctx) eg.SetLimit(c.streamListConcurrency) @@ -265,12 +272,12 @@ func (c *correct) Start(ctx context.Context) (err error) { if len(replicas) <= 0 { diff := c.indexReplica - 1 addrs := c.discoverer.GetAddrs(egctx) + if len(addrs) == 0 { + return errors.ErrNoAvailableAgentToInsert + } // correct index replica shortage if diff > 0 { - log.Infof("replica shortage(diff=%d) of vector id: %s detected from last %s. inserting to other agents = %v", diff, id, debugMsg, addrs) - if len(addrs) == 0 { - return errors.ErrNoAvailableAgentToInsert - } + log.Infof("replica shortage(configured: %d, stored: 1) of vector id: %s detected for %s. inserting to other agents = %v", c.indexReplica, id, debugMsg, addrs) req := &payload.Insert_Request{ Vector: vec, // TODO: this should be deleted after Config.Timestamp deprecation @@ -361,6 +368,13 @@ func (c *correct) Start(ctx context.Context) (err error) { // load index replica from other agents and store it to found map if err := c.discoverer.GetClient().OrderedRangeConcurrent(egctx, replicas, len(replicas), func(ctx context.Context, addr string, conn *grpc.ClientConn, copts ...grpc.CallOption) error { + count, ok := counts[addr] // counts is read-only we don't need to lock. + if ok && count != nil && count.GetStored() == 0 && count.GetUncommitted() == 0{ + log.Debugf("no object found will skip the process for replica %s for id %s", addr, id) + return nil + } + + ots, err := vald.NewObjectClient(conn).GetTimestamp(ctx, &payload.Object_TimestampRequest{ Id: &payload.Object_ID{ Id: id, @@ -550,7 +564,7 @@ func (c *correct) Start(ctx context.Context) (err error) { diff := c.indexReplica - currentNumberOfIndexReplica addrs := c.discoverer.GetAddrs(egctx) if diff > 0 { // correct index replica shortage - log.Infof("replica shortage(diff=%d) of vector id: %s detected for %s. inserting to other agents = %v", diff, id, debugMsg, addrs) + log.Infof("replica shortage(configured: %d, stored: %d, diff: %d) of vector id: %s detected for %s. inserting to other agents = %v", c.indexReplica, currentNumberOfIndexReplica, diff, id, debugMsg, addrs) if len(addrs) == 0 { return errors.ErrNoAvailableAgentToInsert } @@ -635,7 +649,7 @@ func (c *correct) Start(ctx context.Context) (err error) { } } } else if diff < 0 { // correct index replica oversupply - log.Infof("replica oversupply of vector %s. deleting...", id) + log.Infof("replica oversupply(configured: %d, stored: %d, diff: %d) of vector id: %s detected for %s. deleting from agents = %v", c.indexReplica, currentNumberOfIndexReplica, diff, id, debugMsg, found) if len(addrs) == 0 { return errors.ErrNoAvailableAgentToRemove } diff --git a/rust/Cargo.lock b/rust/Cargo.lock index c6b8a0217d..e884bb3221 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -181,9 +181,9 @@ checksum = "8318a53db07bb3f8dca91a600466bdb3f2eaadeedfdbcf02e1accbad9271ba50" [[package]] name = "cc" -version = "1.1.8" +version = "1.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "504bdec147f2cc13c8b57ed9401fd8a147cc66b67ad5cb241394244f2c947549" +checksum = "e9e8aabfac534be767c909e0690571677d49f41bd8465ae876fe043d52ba5292" [[package]] name = "cfg-if" @@ -213,9 +213,9 @@ dependencies = [ [[package]] name = "core-foundation-sys" -version = "0.8.6" +version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" [[package]] name = "cxx" @@ -1105,18 +1105,18 @@ checksum = "a3cf7c11c38cb994f3d40e8a8cde3bbd1f72a435e4c49e85d6553d8312306152" [[package]] name = "serde" -version = "1.0.204" +version = "1.0.206" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc76f558e0cbb2a839d37354c575f1dc3fdc6546b5be373ba43d95f231bf7c12" +checksum = "5b3e4cd94123dd520a128bcd11e34d9e9e423e7e3e50425cb1b4b1e3549d0284" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.204" +version = "1.0.206" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0cd7e117be63d3c3678776753929474f3b04a43a080c744d6b0ae2a8c28e222" +checksum = "fabfb6138d2383ea8208cf98ccf69cdfb1aff4088460681d84189aa259762f97" dependencies = [ "proc-macro2", "quote", @@ -1125,9 +1125,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.121" +version = "1.0.124" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ab380d7d9f22ef3f21ad3e6c1ebe8e4fc7a2000ccba2e4d71fc96f15b2cb609" +checksum = "66ad62847a56b3dba58cc891acd13884b9c61138d330c0d7b6181713d4fce38d" dependencies = [ "itoa", "memchr", @@ -1210,9 +1210,9 @@ checksum = "b7401a30af6cb5818bb64852270bb722533397edcfc7344954a38f420819ece2" [[package]] name = "syn" -version = "2.0.72" +version = "2.0.74" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc4b9b9bf2add8093d3f2c0204471e951b2285580335de42f9d2534f3ae7a8af" +checksum = "1fceb41e3d546d0bd83421d3409b1460cc7444cd389341a4c880fe7a042cb3d7" dependencies = [ "proc-macro2", "quote", diff --git a/rust/libs/proto/src/vald.v1.tonic.rs b/rust/libs/proto/src/vald.v1.tonic.rs index 1ff23e3b46..654bb5701b 100644 --- a/rust/libs/proto/src/vald.v1.tonic.rs +++ b/rust/libs/proto/src/vald.v1.tonic.rs @@ -5450,6 +5450,35 @@ pub mod update_client { .insert(GrpcMethod::new("vald.v1.Update", "MultiUpdate")); self.inner.unary(req, path, codec).await } + /** A method to update timestamp an indexed vector. +*/ + pub async fn update_timestamp( + &mut self, + request: impl tonic::IntoRequest< + super::super::super::payload::v1::object::Timestamp, + >, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { + self.inner + .ready() + .await + .map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; + let codec = tonic::codec::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static( + "/vald.v1.Update/UpdateTimestamp", + ); + let mut req = request.into_request(); + req.extensions_mut() + .insert(GrpcMethod::new("vald.v1.Update", "UpdateTimestamp")); + self.inner.unary(req, path, codec).await + } } } /// Generated server implementations. @@ -5497,6 +5526,15 @@ pub mod update_server { tonic::Response, tonic::Status, >; + /** A method to update timestamp an indexed vector. +*/ + async fn update_timestamp( + &self, + request: tonic::Request, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + >; } #[derive(Debug)] pub struct UpdateServer { @@ -5727,6 +5765,55 @@ pub mod update_server { }; Box::pin(fut) } + "/vald.v1.Update/UpdateTimestamp" => { + #[allow(non_camel_case_types)] + struct UpdateTimestampSvc(pub Arc); + impl< + T: Update, + > tonic::server::UnaryService< + super::super::super::payload::v1::object::Timestamp, + > for UpdateTimestampSvc { + type Response = super::super::super::payload::v1::object::Location; + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request< + super::super::super::payload::v1::object::Timestamp, + >, + ) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = async move { + ::update_timestamp(&inner, request).await + }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let inner = inner.0; + let method = UpdateTimestampSvc(inner); + let codec = tonic::codec::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); + let res = grpc.unary(method, req).await; + Ok(res) + }; + Box::pin(fut) + } _ => { Box::pin(async move { Ok( diff --git a/versions/PROMETHEUS_STACK_VERSION b/versions/PROMETHEUS_STACK_VERSION index 14e3460225..c40b5fb3e9 100644 --- a/versions/PROMETHEUS_STACK_VERSION +++ b/versions/PROMETHEUS_STACK_VERSION @@ -1 +1 @@ -61.7.1 +61.8.0 diff --git a/versions/VALDCLI_VERSION b/versions/VALDCLI_VERSION deleted file mode 100644 index b84efa430e..0000000000 --- a/versions/VALDCLI_VERSION +++ /dev/null @@ -1 +0,0 @@ -v1.7.12