diff --git a/.golangci.yml b/.golangci.yml index d5b4709ea2f..4807a9a4fda 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -152,6 +152,12 @@ linters: - path: internal/test/comparator/standard\.go linters: - depguard + - path: internal/sync/alias\.go + linters: + - depguard + - path: hack/benchmark/src/singleflight/singleflight_bench_test\.go + linters: + - depguard linters-settings: gocritic: enabled-checks: @@ -169,6 +175,14 @@ linters-settings: rules: main: deny: + - pkg: "sync" + desc: "sync is allowed only by internal/sync" + - pkg: "golang.org/x/sync" + desc: "golang.org/x/sync is allowed only by internal/sync" + - pkg: "golang.org/x/sync/singleflight" + desc: "golang.org/x/sync/singleflight is allowed only by internal/sync/singleflight" + - pkg: "golang.org/x/sync/semaphore" + desc: "golang.org/x/sync/semaphore is allowed only by internal/sync/semaphore" - pkg: "errors" desc: "errors is allowed only by internal/errors" - pkg: "k8s.io/apimachinery/pkg/api/errors" diff --git a/CHANGELOG.md b/CHANGELOG.md index 25a6a697d59..c2894fcc056 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4750,7 +4750,7 @@ - :white_check_mark: add internal/config/ngt test ([#554](https://github.com/vdaas/vald/pull/554)) - :white_check_mark: internal/cache/cacher test ([#553](https://github.com/vdaas/vald/pull/553)) - :white_check_mark: Add test case for `internal/file` ([#550](https://github.com/vdaas/vald/pull/550)) -- :white_check_mark: add internal/singleflight test ([#542](https://github.com/vdaas/vald/pull/542)) +- :white_check_mark: add internal/sync/singleflight test ([#542](https://github.com/vdaas/vald/pull/542)) - not to force rebuild gotests ([#548](https://github.com/vdaas/vald/pull/548)) - :pencil: Add use case document ([#482](https://github.com/vdaas/vald/pull/482)) - :white_check_mark: add internal/log/mock/retry test ([#549](https://github.com/vdaas/vald/pull/549)) @@ -5272,7 +5272,7 @@ - internal gRPC client for Vald - Cassandra NewConvictionPolicy - dicoverer now returns clone object -- new internal/singleflight package +- new internal/sync/singleflight package - new internal/net package - coding guideline diff --git a/Makefile b/Makefile index ec122ea034c..a1660b524e5 100644 --- a/Makefile +++ b/Makefile @@ -17,7 +17,6 @@ ORG ?= vdaas NAME = vald GOPKG = github.com/$(ORG)/$(NAME) -GOPRIVATE = $(GOPKG),$(GOPKG)/apis,$(GOPKG)-client-go DATETIME = $(eval DATETIME := $(shell date -u +%Y/%m/%d_%H:%M:%S%z))$(DATETIME) TAG ?= latest CRORG ?= $(ORG) @@ -39,7 +38,8 @@ VERSION ?= $(eval VERSION := $(shell cat versions/VALD_VERSION))$(VERSION) NGT_VERSION := $(eval NGT_VERSION := $(shell cat versions/NGT_VERSION))$(NGT_VERSION) NGT_REPO = github.com/yahoojapan/NGT -GOPROXY=direct +GOPRIVATE = $(GOPKG),$(GOPKG)/apis,$(GOPKG)-client-go +GOPROXY = "https://proxy.golang.org,direct" GOPATH := $(eval GOPATH := $(shell go env GOPATH))$(GOPATH) GO_VERSION := $(eval GO_VERSION := $(shell cat versions/GO_VERSION))$(GO_VERSION) GOARCH := $(eval GOARCH := $(shell go env GOARCH))$(GOARCH) @@ -361,7 +361,7 @@ clean: license: GOPRIVATE=$(GOPRIVATE) \ MAINTAINER=$(MAINTAINER) \ - go run -mod=readonly hack/license/gen/main.go ./ + go run -mod=readonly hack/license/gen/main.go $(ROOTDIR) .PHONY: init ## initialize development environment diff --git a/Makefile.d/dependencies.mk b/Makefile.d/dependencies.mk index c9fc96612df..9252716cc1e 100644 --- a/Makefile.d/dependencies.mk +++ b/Makefile.d/dependencies.mk @@ -80,7 +80,7 @@ update/chaos-mesh: .PHONY: update/go ## update go version update/go: - curl --silent https://go.dev/VERSION?m=text | sed -e 's/go//g' > $(ROOTDIR)/versions/GO_VERSION + curl --silent https://go.dev/VERSION?m=text | head -n 1 | sed -e 's/go//g' > $(ROOTDIR)/versions/GO_VERSION .PHONY: update/golangci-lint ## update golangci-lint version diff --git a/Makefile.d/proto.mk b/Makefile.d/proto.mk index a98893193b2..592c6bcc49e 100644 --- a/Makefile.d/proto.mk +++ b/Makefile.d/proto.mk @@ -138,6 +138,7 @@ $(PBGOS): \ find $(ROOTDIR)/apis/grpc/* -name '*.go' | xargs sed -i -E "s%google.golang.org/grpc/codes%github.com/vdaas/vald/internal/net/grpc/codes%g" find $(ROOTDIR)/apis/grpc/* -name '*.go' | xargs sed -i -E "s%google.golang.org/grpc/status%github.com/vdaas/vald/internal/net/grpc/status%g" find $(ROOTDIR)/apis/grpc/* -name '*.go' | xargs sed -i -E "s%\"io\"%\"github.com/vdaas/vald/internal/io\"%g" + find $(ROOTDIR)/apis/grpc/* -name '*.go' | xargs sed -i -E "s%\"sync\"%\"github.com/vdaas/vald/internal/sync\"%g" $(SWAGGERS): \ $(PROTOS) \ diff --git a/Makefile.d/test.mk b/Makefile.d/test.mk index b85e6401dd0..f591103b6ad 100644 --- a/Makefile.d/test.mk +++ b/Makefile.d/test.mk @@ -284,7 +284,7 @@ gotests/patch: @$(call green, "apply patches to go test files...") find $(ROOTDIR)/internal/k8s/* -name '*_test.go' | xargs sed -i -E "s%k8s.io/apimachinery/pkg/api/errors%github.com/vdaas/vald/internal/errors%g" find $(ROOTDIR)/* -name '*_test.go' | xargs sed -i -E "s%cockroachdb/errors%vdaas/vald/internal/errors%g" - find $(ROOTDIR)/* -name '*_test.go' | xargs sed -i -E "s%golang.org/x/sync/errgroup%github.com/vdaas/vald/internal/errgroup%g" + find $(ROOTDIR)/* -name '*_test.go' | xargs sed -i -E "s%golang.org/x/sync/errgroup%github.com/vdaas/vald/internal/sync/errgroup%g" find $(ROOTDIR)/* -name '*_test.go' | xargs sed -i -E "s%pkg/errors%vdaas/vald/internal/errors%g" find $(ROOTDIR)/* -name '*_test.go' | xargs sed -i -E "s%go-errors/errors%vdaas/vald/internal/errors%g" find $(ROOTDIR)/* -name '*_test.go' | xargs sed -i -E "s%go.uber.org/goleak%github.com/vdaas/vald/internal/test/goleak%g" diff --git a/apis/docs/v1/docs.md b/apis/docs/v1/docs.md index f93e8153f4f..88740c70316 100644 --- a/apis/docs/v1/docs.md +++ b/apis/docs/v1/docs.md @@ -4,16 +4,18 @@ ## Table of Contents +- [apis/proto/v1/agent/core/agent.proto](#apis_proto_v1_agent_core_agent-proto) + - [Agent](#core-v1-Agent) +- [apis/proto/v1/agent/sidecar/sidecar.proto](#apis_proto_v1_agent_sidecar_sidecar-proto) + - [Sidecar](#sidecar-v1-Sidecar) +- [apis/proto/v1/discoverer/discoverer.proto](#apis_proto_v1_discoverer_discoverer-proto) + - [Discoverer](#discoverer-v1-Discoverer) - [apis/proto/v1/filter/egress/egress_filter.proto](#apis_proto_v1_filter_egress_egress_filter-proto) - [Filter](#filter-egress-v1-Filter) - [apis/proto/v1/filter/ingress/ingress_filter.proto](#apis_proto_v1_filter_ingress_ingress_filter-proto) - [Filter](#filter-ingress-v1-Filter) - [apis/proto/v1/manager/index/index_manager.proto](#apis_proto_v1_manager_index_index_manager-proto) - [Index](#manager-index-v1-Index) -- [apis/proto/v1/agent/sidecar/sidecar.proto](#apis_proto_v1_agent_sidecar_sidecar-proto) - - [Sidecar](#sidecar-v1-Sidecar) -- [apis/proto/v1/agent/core/agent.proto](#apis_proto_v1_agent_core_agent-proto) - - [Agent](#core-v1-Agent) - [apis/proto/v1/payload/payload.proto](#apis_proto_v1_payload_payload-proto) - [Control](#payload-v1-Control) @@ -88,27 +90,75 @@ - [Upsert.MultiRequest](#payload-v1-Upsert-MultiRequest) - [Upsert.ObjectRequest](#payload-v1-Upsert-ObjectRequest) - [Upsert.Request](#payload-v1-Upsert-Request) - - [Search.AggregationAlgorithm](#payload-v1-Search-AggregationAlgorithm) -- [apis/proto/v1/discoverer/discoverer.proto](#apis_proto_v1_discoverer_discoverer-proto) - - [Discoverer](#discoverer-v1-Discoverer) -- [apis/proto/v1/vald/remove.proto](#apis_proto_v1_vald_remove-proto) - - [Remove](#vald-v1-Remove) +- [apis/proto/v1/vald/filter.proto](#apis_proto_v1_vald_filter-proto) + - [Filter](#vald-v1-Filter) - [apis/proto/v1/vald/insert.proto](#apis_proto_v1_vald_insert-proto) - [Insert](#vald-v1-Insert) -- [apis/proto/v1/vald/upsert.proto](#apis_proto_v1_vald_upsert-proto) - - [Upsert](#vald-v1-Upsert) - [apis/proto/v1/vald/object.proto](#apis_proto_v1_vald_object-proto) - [Object](#vald-v1-Object) -- [apis/proto/v1/vald/filter.proto](#apis_proto_v1_vald_filter-proto) - - [Filter](#vald-v1-Filter) +- [apis/proto/v1/vald/remove.proto](#apis_proto_v1_vald_remove-proto) + - [Remove](#vald-v1-Remove) - [apis/proto/v1/vald/search.proto](#apis_proto_v1_vald_search-proto) - [Search](#vald-v1-Search) - [apis/proto/v1/vald/update.proto](#apis_proto_v1_vald_update-proto) - [Update](#vald-v1-Update) +- [apis/proto/v1/vald/upsert.proto](#apis_proto_v1_vald_upsert-proto) + - [Upsert](#vald-v1-Upsert) - [Scalar Value Types](#scalar-value-types) + + +

Top

+ +## apis/proto/v1/agent/core/agent.proto + + + +### Agent + +Represent the agent service. + +| Method Name | Request Type | Response Type | Description | +| ------------------ | -------------------------------------------------------------------------------- | ------------------------------------------------------------ | ----------------------------------------------------- | +| CreateIndex | [.payload.v1.Control.CreateIndexRequest](#payload-v1-Control-CreateIndexRequest) | [.payload.v1.Empty](#payload-v1-Empty) | Represent the creating index RPC. | +| SaveIndex | [.payload.v1.Empty](#payload-v1-Empty) | [.payload.v1.Empty](#payload-v1-Empty) | Represent the saving index RPC. | +| CreateAndSaveIndex | [.payload.v1.Control.CreateIndexRequest](#payload-v1-Control-CreateIndexRequest) | [.payload.v1.Empty](#payload-v1-Empty) | Represent the creating and saving index RPC. | +| IndexInfo | [.payload.v1.Empty](#payload-v1-Empty) | [.payload.v1.Info.Index.Count](#payload-v1-Info-Index-Count) | Represent the RPC to get the agent index information. | + + + +

Top

+ +## apis/proto/v1/agent/sidecar/sidecar.proto + + + +### Sidecar + +Represent the agent sidecar service. + +| Method Name | Request Type | Response Type | Description | +| ----------- | ------------ | ------------- | ----------- | + + + +

Top

+ +## apis/proto/v1/discoverer/discoverer.proto + + + +### Discoverer + +Represent the discoverer service. + +| Method Name | Request Type | Response Type | Description | +| ----------- | ---------------------------------------------------------------- | ------------------------------------------------ | ---------------------------------------------------- | +| Pods | [.payload.v1.Discoverer.Request](#payload-v1-Discoverer-Request) | [.payload.v1.Info.Pods](#payload-v1-Info-Pods) | Represent the RPC to get the agent pods information. | +| Nodes | [.payload.v1.Discoverer.Request](#payload-v1-Discoverer-Request) | [.payload.v1.Info.Nodes](#payload-v1-Info-Nodes) | Represent the RPC to get the node information. | +

Top

@@ -159,40 +209,6 @@ Represent the index manager service. | ----------- | -------------------------------------- | ------------------------------------------------------------ | ----------------------------------------------- | | IndexInfo | [.payload.v1.Empty](#payload-v1-Empty) | [.payload.v1.Info.Index.Count](#payload-v1-Info-Index-Count) | Represent the RPC to get the index information. | - - -

Top

- -## apis/proto/v1/agent/sidecar/sidecar.proto - - - -### Sidecar - -Represent the agent sidecar service. - -| Method Name | Request Type | Response Type | Description | -| ----------- | ------------ | ------------- | ----------- | - - - -

Top

- -## apis/proto/v1/agent/core/agent.proto - - - -### Agent - -Represent the agent service. - -| Method Name | Request Type | Response Type | Description | -| ------------------ | -------------------------------------------------------------------------------- | ------------------------------------------------------------ | ----------------------------------------------------- | -| CreateIndex | [.payload.v1.Control.CreateIndexRequest](#payload-v1-Control-CreateIndexRequest) | [.payload.v1.Empty](#payload-v1-Empty) | Represent the creating index RPC. | -| SaveIndex | [.payload.v1.Empty](#payload-v1-Empty) | [.payload.v1.Empty](#payload-v1-Empty) | Represent the saving index RPC. | -| CreateAndSaveIndex | [.payload.v1.Control.CreateIndexRequest](#payload-v1-Control-CreateIndexRequest) | [.payload.v1.Empty](#payload-v1-Empty) | Represent the creating and saving index RPC. | -| IndexInfo | [.payload.v1.Empty](#payload-v1-Empty) | [.payload.v1.Info.Index.Count](#payload-v1-Info-Index-Count) | Represent the RPC to get the agent index information. | -

Top

@@ -515,6 +531,8 @@ Represent multiple vector IDs. ### Object.List +Represent the list object vector stream request and response. + ### Object.List.Request @@ -523,10 +541,10 @@ Represent multiple vector IDs. ### Object.List.Response -| Field | Type | Label | Description | -| ------ | ------------------------------------------ | ----- | ----------- | -| vector | [Object.Vector](#payload-v1-Object-Vector) | | | -| status | [google.rpc.Status](#google-rpc-Status) | | | +| Field | Type | Label | Description | +| ------ | ------------------------------------------ | ----- | --------------------- | +| vector | [Object.Vector](#payload-v1-Object-Vector) | | The vector | +| status | [google.rpc.Status](#google-rpc-Status) | | The RPC error status. | @@ -934,40 +952,32 @@ AggregationAlgorithm is enum of each aggregation algorithms | SortPoolSlice | 3 | | | PairingHeap | 4 | | - - -

Top

- -## apis/proto/v1/discoverer/discoverer.proto - - - -### Discoverer - -Represent the discoverer service. - -| Method Name | Request Type | Response Type | Description | -| ----------- | ---------------------------------------------------------------- | ------------------------------------------------ | ---------------------------------------------------- | -| Pods | [.payload.v1.Discoverer.Request](#payload-v1-Discoverer-Request) | [.payload.v1.Info.Pods](#payload-v1-Info-Pods) | Represent the RPC to get the agent pods information. | -| Nodes | [.payload.v1.Discoverer.Request](#payload-v1-Discoverer-Request) | [.payload.v1.Info.Nodes](#payload-v1-Info-Nodes) | Represent the RPC to get the node information. | - - +

Top

-## apis/proto/v1/vald/remove.proto +## apis/proto/v1/vald/filter.proto - + -### Remove +### Filter -Remove service provides ways to remove indexed vectors. +Filter service provides ways to connect to Vald through filter. -| Method Name | Request Type | Response Type | Description | -| ------------ | ------------------------------------------------------------------ | ----------------------------------------------------------------------------- | ----------------------------------------------------------------------- | -| Remove | [.payload.v1.Remove.Request](#payload-v1-Remove-Request) | [.payload.v1.Object.Location](#payload-v1-Object-Location) | A method to remove an indexed vector. | -| StreamRemove | [.payload.v1.Remove.Request](#payload-v1-Remove-Request) stream | [.payload.v1.Object.StreamLocation](#payload-v1-Object-StreamLocation) stream | A method to remove multiple indexed vectors by bidirectional streaming. | -| MultiRemove | [.payload.v1.Remove.MultiRequest](#payload-v1-Remove-MultiRequest) | [.payload.v1.Object.Locations](#payload-v1-Object-Locations) | A method to remove multiple indexed vectors in a single request. | +| Method Name | Request Type | Response Type | Description | +| ------------------ | ------------------------------------------------------------------------------ | ----------------------------------------------------------------------------- | ------------------------------------------------------------------------ | +| SearchObject | [.payload.v1.Search.ObjectRequest](#payload-v1-Search-ObjectRequest) | [.payload.v1.Search.Response](#payload-v1-Search-Response) | A method to search object. | +| MultiSearchObject | [.payload.v1.Search.MultiObjectRequest](#payload-v1-Search-MultiObjectRequest) | [.payload.v1.Search.Responses](#payload-v1-Search-Responses) | A method to search multiple objects. | +| StreamSearchObject | [.payload.v1.Search.ObjectRequest](#payload-v1-Search-ObjectRequest) stream | [.payload.v1.Search.StreamResponse](#payload-v1-Search-StreamResponse) stream | A method to search object by bidirectional streaming. | +| InsertObject | [.payload.v1.Insert.ObjectRequest](#payload-v1-Insert-ObjectRequest) | [.payload.v1.Object.Location](#payload-v1-Object-Location) | A method insert object. | +| StreamInsertObject | [.payload.v1.Insert.ObjectRequest](#payload-v1-Insert-ObjectRequest) stream | [.payload.v1.Object.StreamLocation](#payload-v1-Object-StreamLocation) stream | Represent the streaming RPC to insert object by bidirectional streaming. | +| MultiInsertObject | [.payload.v1.Insert.MultiObjectRequest](#payload-v1-Insert-MultiObjectRequest) | [.payload.v1.Object.Locations](#payload-v1-Object-Locations) | A method to insert multiple objects. | +| UpdateObject | [.payload.v1.Update.ObjectRequest](#payload-v1-Update-ObjectRequest) | [.payload.v1.Object.Location](#payload-v1-Object-Location) | A method to update object. | +| StreamUpdateObject | [.payload.v1.Update.ObjectRequest](#payload-v1-Update-ObjectRequest) stream | [.payload.v1.Object.StreamLocation](#payload-v1-Object-StreamLocation) stream | A method to update object by bidirectional streaming. | +| MultiUpdateObject | [.payload.v1.Update.MultiObjectRequest](#payload-v1-Update-MultiObjectRequest) | [.payload.v1.Object.Locations](#payload-v1-Object-Locations) | A method to update multiple objects. | +| UpsertObject | [.payload.v1.Upsert.ObjectRequest](#payload-v1-Upsert-ObjectRequest) | [.payload.v1.Object.Location](#payload-v1-Object-Location) | A method to upsert object. | +| StreamUpsertObject | [.payload.v1.Upsert.ObjectRequest](#payload-v1-Upsert-ObjectRequest) stream | [.payload.v1.Object.StreamLocation](#payload-v1-Object-StreamLocation) stream | A method to upsert object by bidirectional streaming. | +| MultiUpsertObject | [.payload.v1.Upsert.MultiObjectRequest](#payload-v1-Upsert-MultiObjectRequest) | [.payload.v1.Object.Locations](#payload-v1-Object-Locations) | A method to upsert multiple objects. | @@ -987,24 +997,6 @@ Insert service provides ways to add new vectors. | StreamInsert | [.payload.v1.Insert.Request](#payload-v1-Insert-Request) stream | [.payload.v1.Object.StreamLocation](#payload-v1-Object-StreamLocation) stream | A method to add new multiple vectors by bidirectional streaming. | | MultiInsert | [.payload.v1.Insert.MultiRequest](#payload-v1-Insert-MultiRequest) | [.payload.v1.Object.Locations](#payload-v1-Object-Locations) | A method to add new multiple vectors in a single request. | - - -

Top

- -## apis/proto/v1/vald/upsert.proto - - - -### Upsert - -Upsert service provides ways to insert/update vectors. - -| Method Name | Request Type | Response Type | Description | -| ------------ | ------------------------------------------------------------------ | ----------------------------------------------------------------------------- | ---------------------------------------------------------------------- | -| Upsert | [.payload.v1.Upsert.Request](#payload-v1-Upsert-Request) | [.payload.v1.Object.Location](#payload-v1-Object-Location) | A method to insert/update a vector. | -| StreamUpsert | [.payload.v1.Upsert.Request](#payload-v1-Upsert-Request) stream | [.payload.v1.Object.StreamLocation](#payload-v1-Object-StreamLocation) stream | A method to insert/update multiple vectors by bidirectional streaming. | -| MultiUpsert | [.payload.v1.Upsert.MultiRequest](#payload-v1-Upsert-MultiRequest) | [.payload.v1.Object.Locations](#payload-v1-Object-Locations) | A method to insert/update multiple vectors in a single request. | -

Top

@@ -1024,32 +1016,23 @@ Object service provides ways to fetch indexed vectors. | StreamGetObject | [.payload.v1.Object.VectorRequest](#payload-v1-Object-VectorRequest) stream | [.payload.v1.Object.StreamVector](#payload-v1-Object-StreamVector) stream | A method to fetch vectors by bidirectional streaming. | | StreamListObject | [.payload.v1.Object.List.Request](#payload-v1-Object-List-Request) | [.payload.v1.Object.List.Response](#payload-v1-Object-List-Response) stream | A method to get all the vectors with server streaming | - +

Top

-## apis/proto/v1/vald/filter.proto +## apis/proto/v1/vald/remove.proto - + -### Filter +### Remove -Filter service provides ways to connect to Vald through filter. +Remove service provides ways to remove indexed vectors. -| Method Name | Request Type | Response Type | Description | -| ------------------ | ------------------------------------------------------------------------------ | ----------------------------------------------------------------------------- | ------------------------------------------------------------------------ | -| SearchObject | [.payload.v1.Search.ObjectRequest](#payload-v1-Search-ObjectRequest) | [.payload.v1.Search.Response](#payload-v1-Search-Response) | A method to search object. | -| MultiSearchObject | [.payload.v1.Search.MultiObjectRequest](#payload-v1-Search-MultiObjectRequest) | [.payload.v1.Search.Responses](#payload-v1-Search-Responses) | A method to search multiple objects. | -| StreamSearchObject | [.payload.v1.Search.ObjectRequest](#payload-v1-Search-ObjectRequest) stream | [.payload.v1.Search.StreamResponse](#payload-v1-Search-StreamResponse) stream | A method to search object by bidirectional streaming. | -| InsertObject | [.payload.v1.Insert.ObjectRequest](#payload-v1-Insert-ObjectRequest) | [.payload.v1.Object.Location](#payload-v1-Object-Location) | A method insert object. | -| StreamInsertObject | [.payload.v1.Insert.ObjectRequest](#payload-v1-Insert-ObjectRequest) stream | [.payload.v1.Object.StreamLocation](#payload-v1-Object-StreamLocation) stream | Represent the streaming RPC to insert object by bidirectional streaming. | -| MultiInsertObject | [.payload.v1.Insert.MultiObjectRequest](#payload-v1-Insert-MultiObjectRequest) | [.payload.v1.Object.Locations](#payload-v1-Object-Locations) | A method to insert multiple objects. | -| UpdateObject | [.payload.v1.Update.ObjectRequest](#payload-v1-Update-ObjectRequest) | [.payload.v1.Object.Location](#payload-v1-Object-Location) | A method to update object. | -| StreamUpdateObject | [.payload.v1.Update.ObjectRequest](#payload-v1-Update-ObjectRequest) stream | [.payload.v1.Object.StreamLocation](#payload-v1-Object-StreamLocation) stream | A method to update object by bidirectional streaming. | -| MultiUpdateObject | [.payload.v1.Update.MultiObjectRequest](#payload-v1-Update-MultiObjectRequest) | [.payload.v1.Object.Locations](#payload-v1-Object-Locations) | A method to update multiple objects. | -| UpsertObject | [.payload.v1.Upsert.ObjectRequest](#payload-v1-Upsert-ObjectRequest) | [.payload.v1.Object.Location](#payload-v1-Object-Location) | A method to upsert object. | -| StreamUpsertObject | [.payload.v1.Upsert.ObjectRequest](#payload-v1-Upsert-ObjectRequest) stream | [.payload.v1.Object.StreamLocation](#payload-v1-Object-StreamLocation) stream | A method to upsert object by bidirectional streaming. | -| MultiUpsertObject | [.payload.v1.Upsert.MultiObjectRequest](#payload-v1-Upsert-MultiObjectRequest) | [.payload.v1.Object.Locations](#payload-v1-Object-Locations) | A method to upsert multiple objects. | +| Method Name | Request Type | Response Type | Description | +| ------------ | ------------------------------------------------------------------ | ----------------------------------------------------------------------------- | ----------------------------------------------------------------------- | +| Remove | [.payload.v1.Remove.Request](#payload-v1-Remove-Request) | [.payload.v1.Object.Location](#payload-v1-Object-Location) | A method to remove an indexed vector. | +| StreamRemove | [.payload.v1.Remove.Request](#payload-v1-Remove-Request) stream | [.payload.v1.Object.StreamLocation](#payload-v1-Object-StreamLocation) stream | A method to remove multiple indexed vectors by bidirectional streaming. | +| MultiRemove | [.payload.v1.Remove.MultiRequest](#payload-v1-Remove-MultiRequest) | [.payload.v1.Object.Locations](#payload-v1-Object-Locations) | A method to remove multiple indexed vectors in a single request. | @@ -1096,6 +1079,24 @@ Update service provides ways to update indexed vectors. | StreamUpdate | [.payload.v1.Update.Request](#payload-v1-Update-Request) stream | [.payload.v1.Object.StreamLocation](#payload-v1-Object-StreamLocation) stream | A method to update multiple indexed vectors by bidirectional streaming. | | MultiUpdate | [.payload.v1.Update.MultiRequest](#payload-v1-Update-MultiRequest) | [.payload.v1.Object.Locations](#payload-v1-Object-Locations) | A method to update multiple indexed vectors in a single request. | + + +

Top

+ +## apis/proto/v1/vald/upsert.proto + + + +### Upsert + +Upsert service provides ways to insert/update vectors. + +| Method Name | Request Type | Response Type | Description | +| ------------ | ------------------------------------------------------------------ | ----------------------------------------------------------------------------- | ---------------------------------------------------------------------- | +| Upsert | [.payload.v1.Upsert.Request](#payload-v1-Upsert-Request) | [.payload.v1.Object.Location](#payload-v1-Object-Location) | A method to insert/update a vector. | +| StreamUpsert | [.payload.v1.Upsert.Request](#payload-v1-Upsert-Request) stream | [.payload.v1.Object.StreamLocation](#payload-v1-Object-StreamLocation) stream | A method to insert/update multiple vectors by bidirectional streaming. | +| MultiUpsert | [.payload.v1.Upsert.MultiRequest](#payload-v1-Upsert-MultiRequest) | [.payload.v1.Object.Locations](#payload-v1-Object-Locations) | A method to insert/update multiple vectors in a single request. | + ## Scalar Value Types | .proto Type | Notes | C++ | Java | Python | Go | C# | PHP | Ruby | diff --git a/apis/grpc/v1/agent/core/agent.pb.go b/apis/grpc/v1/agent/core/agent.pb.go index 67c9b1ef6bf..854f8e1ca17 100644 --- a/apis/grpc/v1/agent/core/agent.pb.go +++ b/apis/grpc/v1/agent/core/agent.pb.go @@ -17,7 +17,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.31.0 -// protoc v4.23.3 +// protoc v4.24.0 // source: apis/proto/v1/agent/core/agent.proto package core diff --git a/apis/grpc/v1/agent/sidecar/sidecar.pb.go b/apis/grpc/v1/agent/sidecar/sidecar.pb.go index f72544b3512..5349243c77e 100644 --- a/apis/grpc/v1/agent/sidecar/sidecar.pb.go +++ b/apis/grpc/v1/agent/sidecar/sidecar.pb.go @@ -17,7 +17,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.31.0 -// protoc v4.23.3 +// protoc v4.24.0 // source: apis/proto/v1/agent/sidecar/sidecar.proto package sidecar diff --git a/apis/grpc/v1/discoverer/discoverer.pb.go b/apis/grpc/v1/discoverer/discoverer.pb.go index 3e6a96ba65c..128c23f73ce 100644 --- a/apis/grpc/v1/discoverer/discoverer.pb.go +++ b/apis/grpc/v1/discoverer/discoverer.pb.go @@ -17,7 +17,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.31.0 -// protoc v4.23.3 +// protoc v4.24.0 // source: apis/proto/v1/discoverer/discoverer.proto package discoverer diff --git a/apis/grpc/v1/filter/egress/egress_filter.pb.go b/apis/grpc/v1/filter/egress/egress_filter.pb.go index 70578312686..e2ba8624d35 100644 --- a/apis/grpc/v1/filter/egress/egress_filter.pb.go +++ b/apis/grpc/v1/filter/egress/egress_filter.pb.go @@ -17,7 +17,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.31.0 -// protoc v4.23.3 +// protoc v4.24.0 // source: apis/proto/v1/filter/egress/egress_filter.proto package egress diff --git a/apis/grpc/v1/filter/ingress/ingress_filter.pb.go b/apis/grpc/v1/filter/ingress/ingress_filter.pb.go index de6217fa682..7ede8d5e4e1 100644 --- a/apis/grpc/v1/filter/ingress/ingress_filter.pb.go +++ b/apis/grpc/v1/filter/ingress/ingress_filter.pb.go @@ -17,7 +17,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.31.0 -// protoc v4.23.3 +// protoc v4.24.0 // source: apis/proto/v1/filter/ingress/ingress_filter.proto package ingress diff --git a/apis/grpc/v1/manager/index/index_manager.pb.go b/apis/grpc/v1/manager/index/index_manager.pb.go index fb23bdc3c5c..255686686a3 100644 --- a/apis/grpc/v1/manager/index/index_manager.pb.go +++ b/apis/grpc/v1/manager/index/index_manager.pb.go @@ -17,7 +17,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.31.0 -// protoc v4.23.3 +// protoc v4.24.0 // source: apis/proto/v1/manager/index/index_manager.proto package index diff --git a/apis/grpc/v1/payload/payload.pb.go b/apis/grpc/v1/payload/payload.pb.go index ca79c764671..f08e1bb2a1a 100644 --- a/apis/grpc/v1/payload/payload.pb.go +++ b/apis/grpc/v1/payload/payload.pb.go @@ -17,16 +17,16 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.31.0 -// protoc v4.23.3 +// protoc v4.24.0 // source: apis/proto/v1/payload/payload.proto package payload import ( reflect "reflect" - sync "sync" _ "github.com/envoyproxy/protoc-gen-validate/validate" + sync "github.com/vdaas/vald/internal/sync" status "google.golang.org/genproto/googleapis/rpc/status" protoreflect "google.golang.org/protobuf/reflect/protoreflect" protoimpl "google.golang.org/protobuf/runtime/protoimpl" @@ -3222,6 +3222,7 @@ func (x *Object_Locations) GetLocations() []*Object_Location { return nil } +// Represent the list object vector stream request and response. type Object_List struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -3368,10 +3369,12 @@ type isObject_List_Response_Payload interface { } type Object_List_Response_Vector struct { + // The vector Vector *Object_Vector `protobuf:"bytes,1,opt,name=vector,proto3,oneof"` } type Object_List_Response_Status struct { + // The RPC error status. Status *status.Status `protobuf:"bytes,2,opt,name=status,proto3,oneof"` } diff --git a/apis/grpc/v1/vald/filter.pb.go b/apis/grpc/v1/vald/filter.pb.go index 51befdf4365..6b6a73e209d 100644 --- a/apis/grpc/v1/vald/filter.pb.go +++ b/apis/grpc/v1/vald/filter.pb.go @@ -17,7 +17,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.31.0 -// protoc v4.23.3 +// protoc v4.24.0 // source: apis/proto/v1/vald/filter.proto package vald diff --git a/apis/grpc/v1/vald/insert.pb.go b/apis/grpc/v1/vald/insert.pb.go index 49be357eff6..eb2c92db7a9 100644 --- a/apis/grpc/v1/vald/insert.pb.go +++ b/apis/grpc/v1/vald/insert.pb.go @@ -17,7 +17,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.31.0 -// protoc v4.23.3 +// protoc v4.24.0 // source: apis/proto/v1/vald/insert.proto package vald diff --git a/apis/grpc/v1/vald/object.pb.go b/apis/grpc/v1/vald/object.pb.go index c428c3dc533..612ca5df235 100644 --- a/apis/grpc/v1/vald/object.pb.go +++ b/apis/grpc/v1/vald/object.pb.go @@ -17,7 +17,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.31.0 -// protoc v4.23.3 +// protoc v4.24.0 // source: apis/proto/v1/vald/object.proto package vald diff --git a/apis/grpc/v1/vald/remove.pb.go b/apis/grpc/v1/vald/remove.pb.go index 2f7edd2dc91..f4e1109606e 100644 --- a/apis/grpc/v1/vald/remove.pb.go +++ b/apis/grpc/v1/vald/remove.pb.go @@ -17,7 +17,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.31.0 -// protoc v4.23.3 +// protoc v4.24.0 // source: apis/proto/v1/vald/remove.proto package vald diff --git a/apis/grpc/v1/vald/search.pb.go b/apis/grpc/v1/vald/search.pb.go index f3a25256226..8456029c651 100644 --- a/apis/grpc/v1/vald/search.pb.go +++ b/apis/grpc/v1/vald/search.pb.go @@ -17,7 +17,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.31.0 -// protoc v4.23.3 +// protoc v4.24.0 // source: apis/proto/v1/vald/search.proto package vald diff --git a/apis/grpc/v1/vald/update.pb.go b/apis/grpc/v1/vald/update.pb.go index 309e5df2cf1..21f91a673f0 100644 --- a/apis/grpc/v1/vald/update.pb.go +++ b/apis/grpc/v1/vald/update.pb.go @@ -17,7 +17,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.31.0 -// protoc v4.23.3 +// protoc v4.24.0 // source: apis/proto/v1/vald/update.proto package vald diff --git a/apis/grpc/v1/vald/upsert.pb.go b/apis/grpc/v1/vald/upsert.pb.go index 5583d957214..7aa0aaeec59 100644 --- a/apis/grpc/v1/vald/upsert.pb.go +++ b/apis/grpc/v1/vald/upsert.pb.go @@ -17,7 +17,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.31.0 -// protoc v4.23.3 +// protoc v4.24.0 // source: apis/proto/v1/vald/upsert.proto package vald diff --git a/apis/swagger/v1/agent/core/apis/proto/v1/agent/core/agent.swagger.json b/apis/swagger/v1/agent/core/apis/proto/v1/agent/core/agent.swagger.json index a802b1512b6..4910719033f 100644 --- a/apis/swagger/v1/agent/core/apis/proto/v1/agent/core/agent.swagger.json +++ b/apis/swagger/v1/agent/core/apis/proto/v1/agent/core/agent.swagger.json @@ -142,7 +142,7 @@ "properties": { "typeUrl": { "type": "string", - "description": "A URL/resource name that uniquely identifies the type of the serialized\nprotocol buffer message. This string must contain at least\none \"/\" character. The last segment of the URL's path must represent\nthe fully qualified name of the type (as in\n`path/google.protobuf.Duration`). The name should be in a canonical form\n(e.g., leading \".\" is not accepted).\n\nIn practice, teams usually precompile into the binary all types that they\nexpect it to use in the context of Any. However, for URLs which use the\nscheme `http`, `https`, or no scheme, one can optionally set up a type\nserver that maps type URLs to message definitions as follows:\n\n* If no scheme is provided, `https` is assumed.\n* An HTTP GET on the URL must yield a [google.protobuf.Type][]\n value in binary format, or produce an error.\n* Applications are allowed to cache lookup results based on the\n URL, or have them precompiled into a binary to avoid any\n lookup. Therefore, binary compatibility needs to be preserved\n on changes to types. (Use versioned type names to manage\n breaking changes.)\n\nNote: this functionality is not currently available in the official\nprotobuf release, and it is not used for type URLs beginning with\ntype.googleapis.com.\n\nSchemes other than `http`, `https` (or the empty scheme) might be\nused with implementation specific semantics." + "description": "A URL/resource name that uniquely identifies the type of the serialized\nprotocol buffer message. This string must contain at least\none \"/\" character. The last segment of the URL's path must represent\nthe fully qualified name of the type (as in\n`path/google.protobuf.Duration`). The name should be in a canonical form\n(e.g., leading \".\" is not accepted).\n\nIn practice, teams usually precompile into the binary all types that they\nexpect it to use in the context of Any. However, for URLs which use the\nscheme `http`, `https`, or no scheme, one can optionally set up a type\nserver that maps type URLs to message definitions as follows:\n\n* If no scheme is provided, `https` is assumed.\n* An HTTP GET on the URL must yield a [google.protobuf.Type][]\n value in binary format, or produce an error.\n* Applications are allowed to cache lookup results based on the\n URL, or have them precompiled into a binary to avoid any\n lookup. Therefore, binary compatibility needs to be preserved\n on changes to types. (Use versioned type names to manage\n breaking changes.)\n\nNote: this functionality is not currently available in the official\nprotobuf release, and it is not used for type URLs beginning with\ntype.googleapis.com. As of May 2023, there are no widely used type server\nimplementations and no plans to implement one.\n\nSchemes other than `http`, `https` (or the empty scheme) might be\nused with implementation specific semantics." }, "value": { "type": "string", diff --git a/apis/swagger/v1/discoverer/apis/proto/v1/discoverer/discoverer.swagger.json b/apis/swagger/v1/discoverer/apis/proto/v1/discoverer/discoverer.swagger.json index 17c90f4c729..51bee365c1c 100644 --- a/apis/swagger/v1/discoverer/apis/proto/v1/discoverer/discoverer.swagger.json +++ b/apis/swagger/v1/discoverer/apis/proto/v1/discoverer/discoverer.swagger.json @@ -208,7 +208,7 @@ "properties": { "typeUrl": { "type": "string", - "description": "A URL/resource name that uniquely identifies the type of the serialized\nprotocol buffer message. This string must contain at least\none \"/\" character. The last segment of the URL's path must represent\nthe fully qualified name of the type (as in\n`path/google.protobuf.Duration`). The name should be in a canonical form\n(e.g., leading \".\" is not accepted).\n\nIn practice, teams usually precompile into the binary all types that they\nexpect it to use in the context of Any. However, for URLs which use the\nscheme `http`, `https`, or no scheme, one can optionally set up a type\nserver that maps type URLs to message definitions as follows:\n\n* If no scheme is provided, `https` is assumed.\n* An HTTP GET on the URL must yield a [google.protobuf.Type][]\n value in binary format, or produce an error.\n* Applications are allowed to cache lookup results based on the\n URL, or have them precompiled into a binary to avoid any\n lookup. Therefore, binary compatibility needs to be preserved\n on changes to types. (Use versioned type names to manage\n breaking changes.)\n\nNote: this functionality is not currently available in the official\nprotobuf release, and it is not used for type URLs beginning with\ntype.googleapis.com.\n\nSchemes other than `http`, `https` (or the empty scheme) might be\nused with implementation specific semantics." + "description": "A URL/resource name that uniquely identifies the type of the serialized\nprotocol buffer message. This string must contain at least\none \"/\" character. The last segment of the URL's path must represent\nthe fully qualified name of the type (as in\n`path/google.protobuf.Duration`). The name should be in a canonical form\n(e.g., leading \".\" is not accepted).\n\nIn practice, teams usually precompile into the binary all types that they\nexpect it to use in the context of Any. However, for URLs which use the\nscheme `http`, `https`, or no scheme, one can optionally set up a type\nserver that maps type URLs to message definitions as follows:\n\n* If no scheme is provided, `https` is assumed.\n* An HTTP GET on the URL must yield a [google.protobuf.Type][]\n value in binary format, or produce an error.\n* Applications are allowed to cache lookup results based on the\n URL, or have them precompiled into a binary to avoid any\n lookup. Therefore, binary compatibility needs to be preserved\n on changes to types. (Use versioned type names to manage\n breaking changes.)\n\nNote: this functionality is not currently available in the official\nprotobuf release, and it is not used for type URLs beginning with\ntype.googleapis.com. As of May 2023, there are no widely used type server\nimplementations and no plans to implement one.\n\nSchemes other than `http`, `https` (or the empty scheme) might be\nused with implementation specific semantics." }, "value": { "type": "string", diff --git a/apis/swagger/v1/filter/egress/apis/proto/v1/filter/egress/egress_filter.swagger.json b/apis/swagger/v1/filter/egress/apis/proto/v1/filter/egress/egress_filter.swagger.json index 6859db29e33..b1bd9992a61 100644 --- a/apis/swagger/v1/filter/egress/apis/proto/v1/filter/egress/egress_filter.swagger.json +++ b/apis/swagger/v1/filter/egress/apis/proto/v1/filter/egress/egress_filter.swagger.json @@ -114,7 +114,7 @@ "properties": { "typeUrl": { "type": "string", - "description": "A URL/resource name that uniquely identifies the type of the serialized\nprotocol buffer message. This string must contain at least\none \"/\" character. The last segment of the URL's path must represent\nthe fully qualified name of the type (as in\n`path/google.protobuf.Duration`). The name should be in a canonical form\n(e.g., leading \".\" is not accepted).\n\nIn practice, teams usually precompile into the binary all types that they\nexpect it to use in the context of Any. However, for URLs which use the\nscheme `http`, `https`, or no scheme, one can optionally set up a type\nserver that maps type URLs to message definitions as follows:\n\n* If no scheme is provided, `https` is assumed.\n* An HTTP GET on the URL must yield a [google.protobuf.Type][]\n value in binary format, or produce an error.\n* Applications are allowed to cache lookup results based on the\n URL, or have them precompiled into a binary to avoid any\n lookup. Therefore, binary compatibility needs to be preserved\n on changes to types. (Use versioned type names to manage\n breaking changes.)\n\nNote: this functionality is not currently available in the official\nprotobuf release, and it is not used for type URLs beginning with\ntype.googleapis.com.\n\nSchemes other than `http`, `https` (or the empty scheme) might be\nused with implementation specific semantics." + "description": "A URL/resource name that uniquely identifies the type of the serialized\nprotocol buffer message. This string must contain at least\none \"/\" character. The last segment of the URL's path must represent\nthe fully qualified name of the type (as in\n`path/google.protobuf.Duration`). The name should be in a canonical form\n(e.g., leading \".\" is not accepted).\n\nIn practice, teams usually precompile into the binary all types that they\nexpect it to use in the context of Any. However, for URLs which use the\nscheme `http`, `https`, or no scheme, one can optionally set up a type\nserver that maps type URLs to message definitions as follows:\n\n* If no scheme is provided, `https` is assumed.\n* An HTTP GET on the URL must yield a [google.protobuf.Type][]\n value in binary format, or produce an error.\n* Applications are allowed to cache lookup results based on the\n URL, or have them precompiled into a binary to avoid any\n lookup. Therefore, binary compatibility needs to be preserved\n on changes to types. (Use versioned type names to manage\n breaking changes.)\n\nNote: this functionality is not currently available in the official\nprotobuf release, and it is not used for type URLs beginning with\ntype.googleapis.com. As of May 2023, there are no widely used type server\nimplementations and no plans to implement one.\n\nSchemes other than `http`, `https` (or the empty scheme) might be\nused with implementation specific semantics." }, "value": { "type": "string", diff --git a/apis/swagger/v1/filter/ingress/apis/proto/v1/filter/ingress/ingress_filter.swagger.json b/apis/swagger/v1/filter/ingress/apis/proto/v1/filter/ingress/ingress_filter.swagger.json index 6899db4647c..7e88c9e8b56 100644 --- a/apis/swagger/v1/filter/ingress/apis/proto/v1/filter/ingress/ingress_filter.swagger.json +++ b/apis/swagger/v1/filter/ingress/apis/proto/v1/filter/ingress/ingress_filter.swagger.json @@ -114,7 +114,7 @@ "properties": { "typeUrl": { "type": "string", - "description": "A URL/resource name that uniquely identifies the type of the serialized\nprotocol buffer message. This string must contain at least\none \"/\" character. The last segment of the URL's path must represent\nthe fully qualified name of the type (as in\n`path/google.protobuf.Duration`). The name should be in a canonical form\n(e.g., leading \".\" is not accepted).\n\nIn practice, teams usually precompile into the binary all types that they\nexpect it to use in the context of Any. However, for URLs which use the\nscheme `http`, `https`, or no scheme, one can optionally set up a type\nserver that maps type URLs to message definitions as follows:\n\n* If no scheme is provided, `https` is assumed.\n* An HTTP GET on the URL must yield a [google.protobuf.Type][]\n value in binary format, or produce an error.\n* Applications are allowed to cache lookup results based on the\n URL, or have them precompiled into a binary to avoid any\n lookup. Therefore, binary compatibility needs to be preserved\n on changes to types. (Use versioned type names to manage\n breaking changes.)\n\nNote: this functionality is not currently available in the official\nprotobuf release, and it is not used for type URLs beginning with\ntype.googleapis.com.\n\nSchemes other than `http`, `https` (or the empty scheme) might be\nused with implementation specific semantics." + "description": "A URL/resource name that uniquely identifies the type of the serialized\nprotocol buffer message. This string must contain at least\none \"/\" character. The last segment of the URL's path must represent\nthe fully qualified name of the type (as in\n`path/google.protobuf.Duration`). The name should be in a canonical form\n(e.g., leading \".\" is not accepted).\n\nIn practice, teams usually precompile into the binary all types that they\nexpect it to use in the context of Any. However, for URLs which use the\nscheme `http`, `https`, or no scheme, one can optionally set up a type\nserver that maps type URLs to message definitions as follows:\n\n* If no scheme is provided, `https` is assumed.\n* An HTTP GET on the URL must yield a [google.protobuf.Type][]\n value in binary format, or produce an error.\n* Applications are allowed to cache lookup results based on the\n URL, or have them precompiled into a binary to avoid any\n lookup. Therefore, binary compatibility needs to be preserved\n on changes to types. (Use versioned type names to manage\n breaking changes.)\n\nNote: this functionality is not currently available in the official\nprotobuf release, and it is not used for type URLs beginning with\ntype.googleapis.com. As of May 2023, there are no widely used type server\nimplementations and no plans to implement one.\n\nSchemes other than `http`, `https` (or the empty scheme) might be\nused with implementation specific semantics." }, "value": { "type": "string", diff --git a/apis/swagger/v1/manager/index/apis/proto/v1/manager/index/index_manager.swagger.json b/apis/swagger/v1/manager/index/apis/proto/v1/manager/index/index_manager.swagger.json index 76548a29ca6..835e1aa5b93 100644 --- a/apis/swagger/v1/manager/index/apis/proto/v1/manager/index/index_manager.swagger.json +++ b/apis/swagger/v1/manager/index/apis/proto/v1/manager/index/index_manager.swagger.json @@ -59,7 +59,7 @@ "properties": { "typeUrl": { "type": "string", - "description": "A URL/resource name that uniquely identifies the type of the serialized\nprotocol buffer message. This string must contain at least\none \"/\" character. The last segment of the URL's path must represent\nthe fully qualified name of the type (as in\n`path/google.protobuf.Duration`). The name should be in a canonical form\n(e.g., leading \".\" is not accepted).\n\nIn practice, teams usually precompile into the binary all types that they\nexpect it to use in the context of Any. However, for URLs which use the\nscheme `http`, `https`, or no scheme, one can optionally set up a type\nserver that maps type URLs to message definitions as follows:\n\n* If no scheme is provided, `https` is assumed.\n* An HTTP GET on the URL must yield a [google.protobuf.Type][]\n value in binary format, or produce an error.\n* Applications are allowed to cache lookup results based on the\n URL, or have them precompiled into a binary to avoid any\n lookup. Therefore, binary compatibility needs to be preserved\n on changes to types. (Use versioned type names to manage\n breaking changes.)\n\nNote: this functionality is not currently available in the official\nprotobuf release, and it is not used for type URLs beginning with\ntype.googleapis.com.\n\nSchemes other than `http`, `https` (or the empty scheme) might be\nused with implementation specific semantics." + "description": "A URL/resource name that uniquely identifies the type of the serialized\nprotocol buffer message. This string must contain at least\none \"/\" character. The last segment of the URL's path must represent\nthe fully qualified name of the type (as in\n`path/google.protobuf.Duration`). The name should be in a canonical form\n(e.g., leading \".\" is not accepted).\n\nIn practice, teams usually precompile into the binary all types that they\nexpect it to use in the context of Any. However, for URLs which use the\nscheme `http`, `https`, or no scheme, one can optionally set up a type\nserver that maps type URLs to message definitions as follows:\n\n* If no scheme is provided, `https` is assumed.\n* An HTTP GET on the URL must yield a [google.protobuf.Type][]\n value in binary format, or produce an error.\n* Applications are allowed to cache lookup results based on the\n URL, or have them precompiled into a binary to avoid any\n lookup. Therefore, binary compatibility needs to be preserved\n on changes to types. (Use versioned type names to manage\n breaking changes.)\n\nNote: this functionality is not currently available in the official\nprotobuf release, and it is not used for type URLs beginning with\ntype.googleapis.com. As of May 2023, there are no widely used type server\nimplementations and no plans to implement one.\n\nSchemes other than `http`, `https` (or the empty scheme) might be\nused with implementation specific semantics." }, "value": { "type": "string", diff --git a/apis/swagger/v1/payload/apis/proto/v1/payload/payload.swagger.json b/apis/swagger/v1/payload/apis/proto/v1/payload/payload.swagger.json index a63839f548e..dd665c6995d 100644 --- a/apis/swagger/v1/payload/apis/proto/v1/payload/payload.swagger.json +++ b/apis/swagger/v1/payload/apis/proto/v1/payload/payload.swagger.json @@ -13,7 +13,7 @@ "properties": { "typeUrl": { "type": "string", - "description": "A URL/resource name that uniquely identifies the type of the serialized\nprotocol buffer message. This string must contain at least\none \"/\" character. The last segment of the URL's path must represent\nthe fully qualified name of the type (as in\n`path/google.protobuf.Duration`). The name should be in a canonical form\n(e.g., leading \".\" is not accepted).\n\nIn practice, teams usually precompile into the binary all types that they\nexpect it to use in the context of Any. However, for URLs which use the\nscheme `http`, `https`, or no scheme, one can optionally set up a type\nserver that maps type URLs to message definitions as follows:\n\n* If no scheme is provided, `https` is assumed.\n* An HTTP GET on the URL must yield a [google.protobuf.Type][]\n value in binary format, or produce an error.\n* Applications are allowed to cache lookup results based on the\n URL, or have them precompiled into a binary to avoid any\n lookup. Therefore, binary compatibility needs to be preserved\n on changes to types. (Use versioned type names to manage\n breaking changes.)\n\nNote: this functionality is not currently available in the official\nprotobuf release, and it is not used for type URLs beginning with\ntype.googleapis.com.\n\nSchemes other than `http`, `https` (or the empty scheme) might be\nused with implementation specific semantics." + "description": "A URL/resource name that uniquely identifies the type of the serialized\nprotocol buffer message. This string must contain at least\none \"/\" character. The last segment of the URL's path must represent\nthe fully qualified name of the type (as in\n`path/google.protobuf.Duration`). The name should be in a canonical form\n(e.g., leading \".\" is not accepted).\n\nIn practice, teams usually precompile into the binary all types that they\nexpect it to use in the context of Any. However, for URLs which use the\nscheme `http`, `https`, or no scheme, one can optionally set up a type\nserver that maps type URLs to message definitions as follows:\n\n* If no scheme is provided, `https` is assumed.\n* An HTTP GET on the URL must yield a [google.protobuf.Type][]\n value in binary format, or produce an error.\n* Applications are allowed to cache lookup results based on the\n URL, or have them precompiled into a binary to avoid any\n lookup. Therefore, binary compatibility needs to be preserved\n on changes to types. (Use versioned type names to manage\n breaking changes.)\n\nNote: this functionality is not currently available in the official\nprotobuf release, and it is not used for type URLs beginning with\ntype.googleapis.com. As of May 2023, there are no widely used type server\nimplementations and no plans to implement one.\n\nSchemes other than `http`, `https` (or the empty scheme) might be\nused with implementation specific semantics." }, "value": { "type": "string", diff --git a/apis/swagger/v1/vald/apis/proto/v1/vald/filter.swagger.json b/apis/swagger/v1/vald/apis/proto/v1/vald/filter.swagger.json index 0e52e2800e3..ad756bf4e7c 100644 --- a/apis/swagger/v1/vald/apis/proto/v1/vald/filter.swagger.json +++ b/apis/swagger/v1/vald/apis/proto/v1/vald/filter.swagger.json @@ -372,7 +372,7 @@ "properties": { "typeUrl": { "type": "string", - "description": "A URL/resource name that uniquely identifies the type of the serialized\nprotocol buffer message. This string must contain at least\none \"/\" character. The last segment of the URL's path must represent\nthe fully qualified name of the type (as in\n`path/google.protobuf.Duration`). The name should be in a canonical form\n(e.g., leading \".\" is not accepted).\n\nIn practice, teams usually precompile into the binary all types that they\nexpect it to use in the context of Any. However, for URLs which use the\nscheme `http`, `https`, or no scheme, one can optionally set up a type\nserver that maps type URLs to message definitions as follows:\n\n* If no scheme is provided, `https` is assumed.\n* An HTTP GET on the URL must yield a [google.protobuf.Type][]\n value in binary format, or produce an error.\n* Applications are allowed to cache lookup results based on the\n URL, or have them precompiled into a binary to avoid any\n lookup. Therefore, binary compatibility needs to be preserved\n on changes to types. (Use versioned type names to manage\n breaking changes.)\n\nNote: this functionality is not currently available in the official\nprotobuf release, and it is not used for type URLs beginning with\ntype.googleapis.com.\n\nSchemes other than `http`, `https` (or the empty scheme) might be\nused with implementation specific semantics." + "description": "A URL/resource name that uniquely identifies the type of the serialized\nprotocol buffer message. This string must contain at least\none \"/\" character. The last segment of the URL's path must represent\nthe fully qualified name of the type (as in\n`path/google.protobuf.Duration`). The name should be in a canonical form\n(e.g., leading \".\" is not accepted).\n\nIn practice, teams usually precompile into the binary all types that they\nexpect it to use in the context of Any. However, for URLs which use the\nscheme `http`, `https`, or no scheme, one can optionally set up a type\nserver that maps type URLs to message definitions as follows:\n\n* If no scheme is provided, `https` is assumed.\n* An HTTP GET on the URL must yield a [google.protobuf.Type][]\n value in binary format, or produce an error.\n* Applications are allowed to cache lookup results based on the\n URL, or have them precompiled into a binary to avoid any\n lookup. Therefore, binary compatibility needs to be preserved\n on changes to types. (Use versioned type names to manage\n breaking changes.)\n\nNote: this functionality is not currently available in the official\nprotobuf release, and it is not used for type URLs beginning with\ntype.googleapis.com. As of May 2023, there are no widely used type server\nimplementations and no plans to implement one.\n\nSchemes other than `http`, `https` (or the empty scheme) might be\nused with implementation specific semantics." }, "value": { "type": "string", @@ -388,11 +388,11 @@ "code": { "type": "integer", "format": "int32", - "description": "The status code, which should be an enum value of\n[google.rpc.Code][google.rpc.Code]." + "description": "The status code, which should be an enum value of [google.rpc.Code][google.rpc.Code]." }, "message": { "type": "string", - "description": "A developer-facing error message, which should be in English. Any\nuser-facing error message should be localized and sent in the\n[google.rpc.Status.details][google.rpc.Status.details] field, or localized\nby the client." + "description": "A developer-facing error message, which should be in English. Any\nuser-facing error message should be localized and sent in the\n[google.rpc.Status.details][google.rpc.Status.details] field, or localized by the client." }, "details": { "type": "array", diff --git a/apis/swagger/v1/vald/apis/proto/v1/vald/insert.swagger.json b/apis/swagger/v1/vald/apis/proto/v1/vald/insert.swagger.json index 3154ff85286..56456efacb6 100644 --- a/apis/swagger/v1/vald/apis/proto/v1/vald/insert.swagger.json +++ b/apis/swagger/v1/vald/apis/proto/v1/vald/insert.swagger.json @@ -140,7 +140,7 @@ "properties": { "typeUrl": { "type": "string", - "description": "A URL/resource name that uniquely identifies the type of the serialized\nprotocol buffer message. This string must contain at least\none \"/\" character. The last segment of the URL's path must represent\nthe fully qualified name of the type (as in\n`path/google.protobuf.Duration`). The name should be in a canonical form\n(e.g., leading \".\" is not accepted).\n\nIn practice, teams usually precompile into the binary all types that they\nexpect it to use in the context of Any. However, for URLs which use the\nscheme `http`, `https`, or no scheme, one can optionally set up a type\nserver that maps type URLs to message definitions as follows:\n\n* If no scheme is provided, `https` is assumed.\n* An HTTP GET on the URL must yield a [google.protobuf.Type][]\n value in binary format, or produce an error.\n* Applications are allowed to cache lookup results based on the\n URL, or have them precompiled into a binary to avoid any\n lookup. Therefore, binary compatibility needs to be preserved\n on changes to types. (Use versioned type names to manage\n breaking changes.)\n\nNote: this functionality is not currently available in the official\nprotobuf release, and it is not used for type URLs beginning with\ntype.googleapis.com.\n\nSchemes other than `http`, `https` (or the empty scheme) might be\nused with implementation specific semantics." + "description": "A URL/resource name that uniquely identifies the type of the serialized\nprotocol buffer message. This string must contain at least\none \"/\" character. The last segment of the URL's path must represent\nthe fully qualified name of the type (as in\n`path/google.protobuf.Duration`). The name should be in a canonical form\n(e.g., leading \".\" is not accepted).\n\nIn practice, teams usually precompile into the binary all types that they\nexpect it to use in the context of Any. However, for URLs which use the\nscheme `http`, `https`, or no scheme, one can optionally set up a type\nserver that maps type URLs to message definitions as follows:\n\n* If no scheme is provided, `https` is assumed.\n* An HTTP GET on the URL must yield a [google.protobuf.Type][]\n value in binary format, or produce an error.\n* Applications are allowed to cache lookup results based on the\n URL, or have them precompiled into a binary to avoid any\n lookup. Therefore, binary compatibility needs to be preserved\n on changes to types. (Use versioned type names to manage\n breaking changes.)\n\nNote: this functionality is not currently available in the official\nprotobuf release, and it is not used for type URLs beginning with\ntype.googleapis.com. As of May 2023, there are no widely used type server\nimplementations and no plans to implement one.\n\nSchemes other than `http`, `https` (or the empty scheme) might be\nused with implementation specific semantics." }, "value": { "type": "string", @@ -156,11 +156,11 @@ "code": { "type": "integer", "format": "int32", - "description": "The status code, which should be an enum value of\n[google.rpc.Code][google.rpc.Code]." + "description": "The status code, which should be an enum value of [google.rpc.Code][google.rpc.Code]." }, "message": { "type": "string", - "description": "A developer-facing error message, which should be in English. Any\nuser-facing error message should be localized and sent in the\n[google.rpc.Status.details][google.rpc.Status.details] field, or localized\nby the client." + "description": "A developer-facing error message, which should be in English. Any\nuser-facing error message should be localized and sent in the\n[google.rpc.Status.details][google.rpc.Status.details] field, or localized by the client." }, "details": { "type": "array", diff --git a/apis/swagger/v1/vald/apis/proto/v1/vald/object.swagger.json b/apis/swagger/v1/vald/apis/proto/v1/vald/object.swagger.json index 36dd578ab71..389fb067f16 100644 --- a/apis/swagger/v1/vald/apis/proto/v1/vald/object.swagger.json +++ b/apis/swagger/v1/vald/apis/proto/v1/vald/object.swagger.json @@ -125,10 +125,12 @@ "type": "object", "properties": { "vector": { - "$ref": "#/definitions/ObjectVector" + "$ref": "#/definitions/ObjectVector", + "title": "The vector" }, "status": { - "$ref": "#/definitions/rpcStatus" + "$ref": "#/definitions/rpcStatus", + "description": "The RPC error status." } } }, @@ -174,7 +176,7 @@ "properties": { "typeUrl": { "type": "string", - "description": "A URL/resource name that uniquely identifies the type of the serialized\nprotocol buffer message. This string must contain at least\none \"/\" character. The last segment of the URL's path must represent\nthe fully qualified name of the type (as in\n`path/google.protobuf.Duration`). The name should be in a canonical form\n(e.g., leading \".\" is not accepted).\n\nIn practice, teams usually precompile into the binary all types that they\nexpect it to use in the context of Any. However, for URLs which use the\nscheme `http`, `https`, or no scheme, one can optionally set up a type\nserver that maps type URLs to message definitions as follows:\n\n* If no scheme is provided, `https` is assumed.\n* An HTTP GET on the URL must yield a [google.protobuf.Type][]\n value in binary format, or produce an error.\n* Applications are allowed to cache lookup results based on the\n URL, or have them precompiled into a binary to avoid any\n lookup. Therefore, binary compatibility needs to be preserved\n on changes to types. (Use versioned type names to manage\n breaking changes.)\n\nNote: this functionality is not currently available in the official\nprotobuf release, and it is not used for type URLs beginning with\ntype.googleapis.com.\n\nSchemes other than `http`, `https` (or the empty scheme) might be\nused with implementation specific semantics." + "description": "A URL/resource name that uniquely identifies the type of the serialized\nprotocol buffer message. This string must contain at least\none \"/\" character. The last segment of the URL's path must represent\nthe fully qualified name of the type (as in\n`path/google.protobuf.Duration`). The name should be in a canonical form\n(e.g., leading \".\" is not accepted).\n\nIn practice, teams usually precompile into the binary all types that they\nexpect it to use in the context of Any. However, for URLs which use the\nscheme `http`, `https`, or no scheme, one can optionally set up a type\nserver that maps type URLs to message definitions as follows:\n\n* If no scheme is provided, `https` is assumed.\n* An HTTP GET on the URL must yield a [google.protobuf.Type][]\n value in binary format, or produce an error.\n* Applications are allowed to cache lookup results based on the\n URL, or have them precompiled into a binary to avoid any\n lookup. Therefore, binary compatibility needs to be preserved\n on changes to types. (Use versioned type names to manage\n breaking changes.)\n\nNote: this functionality is not currently available in the official\nprotobuf release, and it is not used for type URLs beginning with\ntype.googleapis.com. As of May 2023, there are no widely used type server\nimplementations and no plans to implement one.\n\nSchemes other than `http`, `https` (or the empty scheme) might be\nused with implementation specific semantics." }, "value": { "type": "string", @@ -190,11 +192,11 @@ "code": { "type": "integer", "format": "int32", - "description": "The status code, which should be an enum value of\n[google.rpc.Code][google.rpc.Code]." + "description": "The status code, which should be an enum value of [google.rpc.Code][google.rpc.Code]." }, "message": { "type": "string", - "description": "A developer-facing error message, which should be in English. Any\nuser-facing error message should be localized and sent in the\n[google.rpc.Status.details][google.rpc.Status.details] field, or localized\nby the client." + "description": "A developer-facing error message, which should be in English. Any\nuser-facing error message should be localized and sent in the\n[google.rpc.Status.details][google.rpc.Status.details] field, or localized by the client." }, "details": { "type": "array", diff --git a/apis/swagger/v1/vald/apis/proto/v1/vald/remove.swagger.json b/apis/swagger/v1/vald/apis/proto/v1/vald/remove.swagger.json index ad593dfa0b5..c4084796b04 100644 --- a/apis/swagger/v1/vald/apis/proto/v1/vald/remove.swagger.json +++ b/apis/swagger/v1/vald/apis/proto/v1/vald/remove.swagger.json @@ -111,7 +111,7 @@ "properties": { "typeUrl": { "type": "string", - "description": "A URL/resource name that uniquely identifies the type of the serialized\nprotocol buffer message. This string must contain at least\none \"/\" character. The last segment of the URL's path must represent\nthe fully qualified name of the type (as in\n`path/google.protobuf.Duration`). The name should be in a canonical form\n(e.g., leading \".\" is not accepted).\n\nIn practice, teams usually precompile into the binary all types that they\nexpect it to use in the context of Any. However, for URLs which use the\nscheme `http`, `https`, or no scheme, one can optionally set up a type\nserver that maps type URLs to message definitions as follows:\n\n* If no scheme is provided, `https` is assumed.\n* An HTTP GET on the URL must yield a [google.protobuf.Type][]\n value in binary format, or produce an error.\n* Applications are allowed to cache lookup results based on the\n URL, or have them precompiled into a binary to avoid any\n lookup. Therefore, binary compatibility needs to be preserved\n on changes to types. (Use versioned type names to manage\n breaking changes.)\n\nNote: this functionality is not currently available in the official\nprotobuf release, and it is not used for type URLs beginning with\ntype.googleapis.com.\n\nSchemes other than `http`, `https` (or the empty scheme) might be\nused with implementation specific semantics." + "description": "A URL/resource name that uniquely identifies the type of the serialized\nprotocol buffer message. This string must contain at least\none \"/\" character. The last segment of the URL's path must represent\nthe fully qualified name of the type (as in\n`path/google.protobuf.Duration`). The name should be in a canonical form\n(e.g., leading \".\" is not accepted).\n\nIn practice, teams usually precompile into the binary all types that they\nexpect it to use in the context of Any. However, for URLs which use the\nscheme `http`, `https`, or no scheme, one can optionally set up a type\nserver that maps type URLs to message definitions as follows:\n\n* If no scheme is provided, `https` is assumed.\n* An HTTP GET on the URL must yield a [google.protobuf.Type][]\n value in binary format, or produce an error.\n* Applications are allowed to cache lookup results based on the\n URL, or have them precompiled into a binary to avoid any\n lookup. Therefore, binary compatibility needs to be preserved\n on changes to types. (Use versioned type names to manage\n breaking changes.)\n\nNote: this functionality is not currently available in the official\nprotobuf release, and it is not used for type URLs beginning with\ntype.googleapis.com. As of May 2023, there are no widely used type server\nimplementations and no plans to implement one.\n\nSchemes other than `http`, `https` (or the empty scheme) might be\nused with implementation specific semantics." }, "value": { "type": "string", @@ -127,11 +127,11 @@ "code": { "type": "integer", "format": "int32", - "description": "The status code, which should be an enum value of\n[google.rpc.Code][google.rpc.Code]." + "description": "The status code, which should be an enum value of [google.rpc.Code][google.rpc.Code]." }, "message": { "type": "string", - "description": "A developer-facing error message, which should be in English. Any\nuser-facing error message should be localized and sent in the\n[google.rpc.Status.details][google.rpc.Status.details] field, or localized\nby the client." + "description": "A developer-facing error message, which should be in English. Any\nuser-facing error message should be localized and sent in the\n[google.rpc.Status.details][google.rpc.Status.details] field, or localized by the client." }, "details": { "type": "array", diff --git a/apis/swagger/v1/vald/apis/proto/v1/vald/search.swagger.json b/apis/swagger/v1/vald/apis/proto/v1/vald/search.swagger.json index 9965724de9e..14c69829960 100644 --- a/apis/swagger/v1/vald/apis/proto/v1/vald/search.swagger.json +++ b/apis/swagger/v1/vald/apis/proto/v1/vald/search.swagger.json @@ -358,7 +358,7 @@ "properties": { "typeUrl": { "type": "string", - "description": "A URL/resource name that uniquely identifies the type of the serialized\nprotocol buffer message. This string must contain at least\none \"/\" character. The last segment of the URL's path must represent\nthe fully qualified name of the type (as in\n`path/google.protobuf.Duration`). The name should be in a canonical form\n(e.g., leading \".\" is not accepted).\n\nIn practice, teams usually precompile into the binary all types that they\nexpect it to use in the context of Any. However, for URLs which use the\nscheme `http`, `https`, or no scheme, one can optionally set up a type\nserver that maps type URLs to message definitions as follows:\n\n* If no scheme is provided, `https` is assumed.\n* An HTTP GET on the URL must yield a [google.protobuf.Type][]\n value in binary format, or produce an error.\n* Applications are allowed to cache lookup results based on the\n URL, or have them precompiled into a binary to avoid any\n lookup. Therefore, binary compatibility needs to be preserved\n on changes to types. (Use versioned type names to manage\n breaking changes.)\n\nNote: this functionality is not currently available in the official\nprotobuf release, and it is not used for type URLs beginning with\ntype.googleapis.com.\n\nSchemes other than `http`, `https` (or the empty scheme) might be\nused with implementation specific semantics." + "description": "A URL/resource name that uniquely identifies the type of the serialized\nprotocol buffer message. This string must contain at least\none \"/\" character. The last segment of the URL's path must represent\nthe fully qualified name of the type (as in\n`path/google.protobuf.Duration`). The name should be in a canonical form\n(e.g., leading \".\" is not accepted).\n\nIn practice, teams usually precompile into the binary all types that they\nexpect it to use in the context of Any. However, for URLs which use the\nscheme `http`, `https`, or no scheme, one can optionally set up a type\nserver that maps type URLs to message definitions as follows:\n\n* If no scheme is provided, `https` is assumed.\n* An HTTP GET on the URL must yield a [google.protobuf.Type][]\n value in binary format, or produce an error.\n* Applications are allowed to cache lookup results based on the\n URL, or have them precompiled into a binary to avoid any\n lookup. Therefore, binary compatibility needs to be preserved\n on changes to types. (Use versioned type names to manage\n breaking changes.)\n\nNote: this functionality is not currently available in the official\nprotobuf release, and it is not used for type URLs beginning with\ntype.googleapis.com. As of May 2023, there are no widely used type server\nimplementations and no plans to implement one.\n\nSchemes other than `http`, `https` (or the empty scheme) might be\nused with implementation specific semantics." }, "value": { "type": "string", @@ -374,11 +374,11 @@ "code": { "type": "integer", "format": "int32", - "description": "The status code, which should be an enum value of\n[google.rpc.Code][google.rpc.Code]." + "description": "The status code, which should be an enum value of [google.rpc.Code][google.rpc.Code]." }, "message": { "type": "string", - "description": "A developer-facing error message, which should be in English. Any\nuser-facing error message should be localized and sent in the\n[google.rpc.Status.details][google.rpc.Status.details] field, or localized\nby the client." + "description": "A developer-facing error message, which should be in English. Any\nuser-facing error message should be localized and sent in the\n[google.rpc.Status.details][google.rpc.Status.details] field, or localized by the client." }, "details": { "type": "array", diff --git a/apis/swagger/v1/vald/apis/proto/v1/vald/update.swagger.json b/apis/swagger/v1/vald/apis/proto/v1/vald/update.swagger.json index 82e80fe506b..0301c855661 100644 --- a/apis/swagger/v1/vald/apis/proto/v1/vald/update.swagger.json +++ b/apis/swagger/v1/vald/apis/proto/v1/vald/update.swagger.json @@ -140,7 +140,7 @@ "properties": { "typeUrl": { "type": "string", - "description": "A URL/resource name that uniquely identifies the type of the serialized\nprotocol buffer message. This string must contain at least\none \"/\" character. The last segment of the URL's path must represent\nthe fully qualified name of the type (as in\n`path/google.protobuf.Duration`). The name should be in a canonical form\n(e.g., leading \".\" is not accepted).\n\nIn practice, teams usually precompile into the binary all types that they\nexpect it to use in the context of Any. However, for URLs which use the\nscheme `http`, `https`, or no scheme, one can optionally set up a type\nserver that maps type URLs to message definitions as follows:\n\n* If no scheme is provided, `https` is assumed.\n* An HTTP GET on the URL must yield a [google.protobuf.Type][]\n value in binary format, or produce an error.\n* Applications are allowed to cache lookup results based on the\n URL, or have them precompiled into a binary to avoid any\n lookup. Therefore, binary compatibility needs to be preserved\n on changes to types. (Use versioned type names to manage\n breaking changes.)\n\nNote: this functionality is not currently available in the official\nprotobuf release, and it is not used for type URLs beginning with\ntype.googleapis.com.\n\nSchemes other than `http`, `https` (or the empty scheme) might be\nused with implementation specific semantics." + "description": "A URL/resource name that uniquely identifies the type of the serialized\nprotocol buffer message. This string must contain at least\none \"/\" character. The last segment of the URL's path must represent\nthe fully qualified name of the type (as in\n`path/google.protobuf.Duration`). The name should be in a canonical form\n(e.g., leading \".\" is not accepted).\n\nIn practice, teams usually precompile into the binary all types that they\nexpect it to use in the context of Any. However, for URLs which use the\nscheme `http`, `https`, or no scheme, one can optionally set up a type\nserver that maps type URLs to message definitions as follows:\n\n* If no scheme is provided, `https` is assumed.\n* An HTTP GET on the URL must yield a [google.protobuf.Type][]\n value in binary format, or produce an error.\n* Applications are allowed to cache lookup results based on the\n URL, or have them precompiled into a binary to avoid any\n lookup. Therefore, binary compatibility needs to be preserved\n on changes to types. (Use versioned type names to manage\n breaking changes.)\n\nNote: this functionality is not currently available in the official\nprotobuf release, and it is not used for type URLs beginning with\ntype.googleapis.com. As of May 2023, there are no widely used type server\nimplementations and no plans to implement one.\n\nSchemes other than `http`, `https` (or the empty scheme) might be\nused with implementation specific semantics." }, "value": { "type": "string", @@ -156,11 +156,11 @@ "code": { "type": "integer", "format": "int32", - "description": "The status code, which should be an enum value of\n[google.rpc.Code][google.rpc.Code]." + "description": "The status code, which should be an enum value of [google.rpc.Code][google.rpc.Code]." }, "message": { "type": "string", - "description": "A developer-facing error message, which should be in English. Any\nuser-facing error message should be localized and sent in the\n[google.rpc.Status.details][google.rpc.Status.details] field, or localized\nby the client." + "description": "A developer-facing error message, which should be in English. Any\nuser-facing error message should be localized and sent in the\n[google.rpc.Status.details][google.rpc.Status.details] field, or localized by the client." }, "details": { "type": "array", diff --git a/apis/swagger/v1/vald/apis/proto/v1/vald/upsert.swagger.json b/apis/swagger/v1/vald/apis/proto/v1/vald/upsert.swagger.json index a6defddc461..7e1e5c24cdb 100644 --- a/apis/swagger/v1/vald/apis/proto/v1/vald/upsert.swagger.json +++ b/apis/swagger/v1/vald/apis/proto/v1/vald/upsert.swagger.json @@ -140,7 +140,7 @@ "properties": { "typeUrl": { "type": "string", - "description": "A URL/resource name that uniquely identifies the type of the serialized\nprotocol buffer message. This string must contain at least\none \"/\" character. The last segment of the URL's path must represent\nthe fully qualified name of the type (as in\n`path/google.protobuf.Duration`). The name should be in a canonical form\n(e.g., leading \".\" is not accepted).\n\nIn practice, teams usually precompile into the binary all types that they\nexpect it to use in the context of Any. However, for URLs which use the\nscheme `http`, `https`, or no scheme, one can optionally set up a type\nserver that maps type URLs to message definitions as follows:\n\n* If no scheme is provided, `https` is assumed.\n* An HTTP GET on the URL must yield a [google.protobuf.Type][]\n value in binary format, or produce an error.\n* Applications are allowed to cache lookup results based on the\n URL, or have them precompiled into a binary to avoid any\n lookup. Therefore, binary compatibility needs to be preserved\n on changes to types. (Use versioned type names to manage\n breaking changes.)\n\nNote: this functionality is not currently available in the official\nprotobuf release, and it is not used for type URLs beginning with\ntype.googleapis.com.\n\nSchemes other than `http`, `https` (or the empty scheme) might be\nused with implementation specific semantics." + "description": "A URL/resource name that uniquely identifies the type of the serialized\nprotocol buffer message. This string must contain at least\none \"/\" character. The last segment of the URL's path must represent\nthe fully qualified name of the type (as in\n`path/google.protobuf.Duration`). The name should be in a canonical form\n(e.g., leading \".\" is not accepted).\n\nIn practice, teams usually precompile into the binary all types that they\nexpect it to use in the context of Any. However, for URLs which use the\nscheme `http`, `https`, or no scheme, one can optionally set up a type\nserver that maps type URLs to message definitions as follows:\n\n* If no scheme is provided, `https` is assumed.\n* An HTTP GET on the URL must yield a [google.protobuf.Type][]\n value in binary format, or produce an error.\n* Applications are allowed to cache lookup results based on the\n URL, or have them precompiled into a binary to avoid any\n lookup. Therefore, binary compatibility needs to be preserved\n on changes to types. (Use versioned type names to manage\n breaking changes.)\n\nNote: this functionality is not currently available in the official\nprotobuf release, and it is not used for type URLs beginning with\ntype.googleapis.com. As of May 2023, there are no widely used type server\nimplementations and no plans to implement one.\n\nSchemes other than `http`, `https` (or the empty scheme) might be\nused with implementation specific semantics." }, "value": { "type": "string", @@ -156,11 +156,11 @@ "code": { "type": "integer", "format": "int32", - "description": "The status code, which should be an enum value of\n[google.rpc.Code][google.rpc.Code]." + "description": "The status code, which should be an enum value of [google.rpc.Code][google.rpc.Code]." }, "message": { "type": "string", - "description": "A developer-facing error message, which should be in English. Any\nuser-facing error message should be localized and sent in the\n[google.rpc.Status.details][google.rpc.Status.details] field, or localized\nby the client." + "description": "A developer-facing error message, which should be in English. Any\nuser-facing error message should be localized and sent in the\n[google.rpc.Status.details][google.rpc.Status.details] field, or localized by the client." }, "details": { "type": "array", diff --git a/docs/contributing/coding-style.md b/docs/contributing/coding-style.md index da6644f1c54..50a5ee693d3 100644 --- a/docs/contributing/coding-style.md +++ b/docs/contributing/coding-style.md @@ -385,7 +385,7 @@ if err := srv.Run(); err != nil { All errors should define in [internal/errors package](https://github.com/vdaas/vald/blob/main/internal/errors). All errors should be start with `Err` prefix, and all errors should be handle if possible. -Please use [internal/errgroup](https://github.com/vdaas/vald/blob/main/internal/errgroup) for synchronized error handling on multi-goroutine processing. +Please use [internal/sync/errgroup](https://github.com/vdaas/vald/blob/main/internal/sync/errgroup) for synchronized error handling on multi-goroutine processing. ### Error checking diff --git a/go.mod b/go.mod index 38f9c817bc0..b26b881ee6c 100755 --- a/go.mod +++ b/go.mod @@ -3,13 +3,13 @@ module github.com/vdaas/vald go 1.20 replace ( - cloud.google.com/go => cloud.google.com/go v0.110.6 + cloud.google.com/go => cloud.google.com/go v0.110.7 cloud.google.com/go/bigquery => cloud.google.com/go/bigquery v1.53.0 - cloud.google.com/go/compute => cloud.google.com/go/compute v1.22.0 - cloud.google.com/go/datastore => cloud.google.com/go/datastore v1.12.1 - cloud.google.com/go/firestore => cloud.google.com/go/firestore v1.11.0 - cloud.google.com/go/iam => cloud.google.com/go/iam v1.1.1 - cloud.google.com/go/kms => cloud.google.com/go/kms v1.15.0 + cloud.google.com/go/compute => cloud.google.com/go/compute v1.23.0 + cloud.google.com/go/datastore => cloud.google.com/go/datastore v1.13.0 + cloud.google.com/go/firestore => cloud.google.com/go/firestore v1.12.0 + cloud.google.com/go/iam => cloud.google.com/go/iam v1.1.2 + cloud.google.com/go/kms => cloud.google.com/go/kms v1.15.1 cloud.google.com/go/monitoring => cloud.google.com/go/monitoring v1.15.1 cloud.google.com/go/pubsub => cloud.google.com/go/pubsub v1.33.0 cloud.google.com/go/secretmanager => cloud.google.com/go/secretmanager v1.11.1 @@ -22,7 +22,7 @@ replace ( git.sr.ht/~sbinet/gg => git.sr.ht/~sbinet/gg v0.5.0 github.com/Azure/azure-amqp-common-go/v3 => github.com/Azure/azure-amqp-common-go/v3 v3.2.3 github.com/Azure/azure-sdk-for-go => github.com/Azure/azure-sdk-for-go v68.0.0+incompatible - github.com/Azure/azure-sdk-for-go/sdk/azcore => github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.0 + github.com/Azure/azure-sdk-for-go/sdk/azcore => github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.1 github.com/Azure/azure-sdk-for-go/sdk/azidentity => github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0 github.com/Azure/azure-sdk-for-go/sdk/internal => github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 github.com/Azure/go-amqp => github.com/Azure/go-amqp v1.0.1 @@ -43,29 +43,29 @@ replace ( github.com/ajstarks/svgo => github.com/ajstarks/svgo v0.0.0-20211024235047-1546f124cd8b github.com/antihax/optional => github.com/antihax/optional v1.0.0 github.com/armon/go-socks5 => github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 - github.com/aws/aws-sdk-go => github.com/aws/aws-sdk-go v1.44.308 - github.com/aws/aws-sdk-go-v2 => github.com/aws/aws-sdk-go-v2 v1.19.0 - github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream => github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10 - github.com/aws/aws-sdk-go-v2/config => github.com/aws/aws-sdk-go-v2/config v1.18.29 - github.com/aws/aws-sdk-go-v2/credentials => github.com/aws/aws-sdk-go-v2/credentials v1.13.28 - github.com/aws/aws-sdk-go-v2/feature/ec2/imds => github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.5 - github.com/aws/aws-sdk-go-v2/feature/s3/manager => github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.73 - github.com/aws/aws-sdk-go-v2/internal/configsources => github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.35 - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 => github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.29 - github.com/aws/aws-sdk-go-v2/internal/ini => github.com/aws/aws-sdk-go-v2/internal/ini v1.3.36 - github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding => github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.11 - github.com/aws/aws-sdk-go-v2/service/internal/checksum => github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.30 - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url => github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.29 - github.com/aws/aws-sdk-go-v2/service/internal/s3shared => github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.14.4 - github.com/aws/aws-sdk-go-v2/service/kms => github.com/aws/aws-sdk-go-v2/service/kms v1.23.1 - github.com/aws/aws-sdk-go-v2/service/s3 => github.com/aws/aws-sdk-go-v2/service/s3 v1.37.0 - github.com/aws/aws-sdk-go-v2/service/secretsmanager => github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.19.11 - github.com/aws/aws-sdk-go-v2/service/sns => github.com/aws/aws-sdk-go-v2/service/sns v1.20.14 - github.com/aws/aws-sdk-go-v2/service/sqs => github.com/aws/aws-sdk-go-v2/service/sqs v1.23.3 - github.com/aws/aws-sdk-go-v2/service/ssm => github.com/aws/aws-sdk-go-v2/service/ssm v1.36.8 - github.com/aws/aws-sdk-go-v2/service/sso => github.com/aws/aws-sdk-go-v2/service/sso v1.12.13 - github.com/aws/aws-sdk-go-v2/service/sts => github.com/aws/aws-sdk-go-v2/service/sts v1.20.0 - github.com/aws/smithy-go => github.com/aws/smithy-go v1.13.5 + github.com/aws/aws-sdk-go => github.com/aws/aws-sdk-go v1.44.323 + github.com/aws/aws-sdk-go-v2 => github.com/aws/aws-sdk-go-v2 v1.20.1 + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream => github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.12 + github.com/aws/aws-sdk-go-v2/config => github.com/aws/aws-sdk-go-v2/config v1.18.33 + github.com/aws/aws-sdk-go-v2/credentials => github.com/aws/aws-sdk-go-v2/credentials v1.13.32 + github.com/aws/aws-sdk-go-v2/feature/ec2/imds => github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.8 + github.com/aws/aws-sdk-go-v2/feature/s3/manager => github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.77 + github.com/aws/aws-sdk-go-v2/internal/configsources => github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.38 + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 => github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.32 + github.com/aws/aws-sdk-go-v2/internal/ini => github.com/aws/aws-sdk-go-v2/internal/ini v1.3.39 + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding => github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.13 + github.com/aws/aws-sdk-go-v2/service/internal/checksum => github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.33 + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url => github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.32 + github.com/aws/aws-sdk-go-v2/service/internal/s3shared => github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.15.1 + github.com/aws/aws-sdk-go-v2/service/kms => github.com/aws/aws-sdk-go-v2/service/kms v1.24.2 + github.com/aws/aws-sdk-go-v2/service/s3 => github.com/aws/aws-sdk-go-v2/service/s3 v1.38.2 + github.com/aws/aws-sdk-go-v2/service/secretsmanager => github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.21.0 + github.com/aws/aws-sdk-go-v2/service/sns => github.com/aws/aws-sdk-go-v2/service/sns v1.21.2 + github.com/aws/aws-sdk-go-v2/service/sqs => github.com/aws/aws-sdk-go-v2/service/sqs v1.24.2 + github.com/aws/aws-sdk-go-v2/service/ssm => github.com/aws/aws-sdk-go-v2/service/ssm v1.37.2 + github.com/aws/aws-sdk-go-v2/service/sso => github.com/aws/aws-sdk-go-v2/service/sso v1.13.2 + github.com/aws/aws-sdk-go-v2/service/sts => github.com/aws/aws-sdk-go-v2/service/sts v1.21.2 + github.com/aws/smithy-go => github.com/aws/smithy-go v1.14.1 github.com/benbjohnson/clock => github.com/benbjohnson/clock v1.3.5 github.com/beorn7/perks => github.com/beorn7/perks v1.0.1 github.com/bmizerany/assert => github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 @@ -96,7 +96,7 @@ replace ( github.com/evanphx/json-patch => github.com/evanphx/json-patch v0.5.2 github.com/fogleman/gg => github.com/fogleman/gg v1.3.0 github.com/fortytw2/leaktest => github.com/fortytw2/leaktest v1.3.0 - github.com/frankban/quicktest => github.com/frankban/quicktest v1.14.5 + github.com/frankban/quicktest => github.com/frankban/quicktest v1.14.6 github.com/fsnotify/fsnotify => github.com/fsnotify/fsnotify v1.6.0 github.com/gin-contrib/sse => github.com/gin-contrib/sse v0.1.0 github.com/gin-gonic/gin => github.com/gin-gonic/gin v1.9.1 @@ -120,7 +120,7 @@ replace ( github.com/go-playground/assert/v2 => github.com/go-playground/assert/v2 v2.2.0 github.com/go-playground/locales => github.com/go-playground/locales v0.14.1 github.com/go-playground/universal-translator => github.com/go-playground/universal-translator v0.18.1 - github.com/go-playground/validator/v10 => github.com/go-playground/validator/v10 v10.14.1 + github.com/go-playground/validator/v10 => github.com/go-playground/validator/v10 v10.15.0 github.com/go-redis/redis/v8 => github.com/go-redis/redis/v8 v8.11.5 github.com/go-sql-driver/mysql => github.com/go-sql-driver/mysql v1.7.1 github.com/go-task/slim-sprig => github.com/go-task/slim-sprig v2.20.0+incompatible @@ -129,7 +129,7 @@ replace ( github.com/go-toolsmith/strparse => github.com/go-toolsmith/strparse v1.1.0 github.com/gobwas/httphead => github.com/gobwas/httphead v0.1.0 github.com/gobwas/pool => github.com/gobwas/pool v0.2.1 - github.com/gobwas/ws => github.com/gobwas/ws v1.2.1 + github.com/gobwas/ws => github.com/gobwas/ws v1.3.0 github.com/goccy/go-json => github.com/goccy/go-json v0.10.2 github.com/gocql/gocql => github.com/gocql/gocql v1.5.2 github.com/gocraft/dbr/v2 => github.com/gocraft/dbr/v2 v2.7.5 @@ -140,7 +140,7 @@ replace ( github.com/golang-sql/civil => github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9 github.com/golang-sql/sqlexp => github.com/golang-sql/sqlexp v0.1.0 github.com/golang/freetype => github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 - github.com/golang/glog => github.com/golang/glog v1.1.1 + github.com/golang/glog => github.com/golang/glog v1.1.2 github.com/golang/groupcache => github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da github.com/golang/mock => github.com/golang/mock v1.6.0 github.com/golang/protobuf => github.com/golang/protobuf v1.5.3 @@ -153,7 +153,7 @@ replace ( github.com/google/gofuzz => github.com/google/gofuzz v1.2.0 github.com/google/martian => github.com/google/martian v2.1.0+incompatible github.com/google/martian/v3 => github.com/google/martian/v3 v3.3.2 - github.com/google/pprof => github.com/google/pprof v0.0.0-20230705174524-200ffdc848b8 + github.com/google/pprof => github.com/google/pprof v0.0.0-20230811205829-9131a7e9cc17 github.com/google/shlex => github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 github.com/google/subcommands => github.com/google/subcommands v1.2.0 github.com/google/uuid => github.com/google/uuid v1.3.0 @@ -162,7 +162,7 @@ replace ( github.com/gorilla/mux => github.com/gorilla/mux v1.8.0 github.com/gorilla/websocket => github.com/gorilla/websocket v1.5.0 github.com/gregjones/httpcache => github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 - github.com/grpc-ecosystem/grpc-gateway/v2 => github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.1 + github.com/grpc-ecosystem/grpc-gateway/v2 => github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.2 github.com/hailocab/go-hostpool => github.com/kpango/go-hostpool v0.0.0-20210303030322-aab80263dcd0 github.com/hanwen/go-fuse/v2 => github.com/hanwen/go-fuse/v2 v2.3.0 github.com/hashicorp/go-uuid => github.com/hashicorp/go-uuid v1.0.3 @@ -190,7 +190,7 @@ replace ( github.com/jstemmer/go-junit-report => github.com/jstemmer/go-junit-report v1.0.0 github.com/kisielk/errcheck => github.com/kisielk/errcheck v1.6.3 github.com/kisielk/gotool => github.com/kisielk/gotool v1.0.0 - github.com/klauspost/compress => github.com/klauspost/compress v1.16.8-0.20230717094025-33c9bc800133 + github.com/klauspost/compress => github.com/klauspost/compress v1.16.8-0.20230809110045-b404607005bc github.com/klauspost/cpuid/v2 => github.com/klauspost/cpuid/v2 v2.2.5 github.com/kpango/fastime => github.com/kpango/fastime v1.1.9 github.com/kpango/fuid => github.com/kpango/fuid v0.0.0-20221203053508-503b5ad89aa1 @@ -231,22 +231,22 @@ replace ( github.com/pierrec/lz4/v3 => github.com/pierrec/lz4/v3 v3.3.5 github.com/pkg/browser => github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 github.com/pkg/errors => github.com/pkg/errors v0.9.1 - github.com/pkg/sftp => github.com/pkg/sftp v1.13.5 + github.com/pkg/sftp => github.com/pkg/sftp v1.13.6 github.com/pmezard/go-difflib => github.com/pmezard/go-difflib v1.0.0 github.com/prashantv/gostub => github.com/prashantv/gostub v1.1.0 github.com/prometheus/client_golang => github.com/prometheus/client_golang v1.16.0 github.com/prometheus/client_model => github.com/prometheus/client_model v0.4.0 github.com/prometheus/common => github.com/prometheus/common v0.44.0 - github.com/prometheus/procfs => github.com/prometheus/procfs v0.11.0 + github.com/prometheus/procfs => github.com/prometheus/procfs v0.11.1 github.com/prometheus/prometheus => github.com/prometheus/prometheus v1.99.0 - github.com/quasilyte/go-ruleguard => github.com/quasilyte/go-ruleguard v0.3.19 + github.com/quasilyte/go-ruleguard => github.com/quasilyte/go-ruleguard v0.4.0 github.com/quasilyte/go-ruleguard/dsl => github.com/quasilyte/go-ruleguard/dsl v0.3.22 github.com/quasilyte/gogrep => github.com/quasilyte/gogrep v0.5.0 github.com/quasilyte/stdinfo => github.com/quasilyte/stdinfo v0.0.0-20220114132959-f7386bf02567 github.com/rogpeppe/fastuuid => github.com/rogpeppe/fastuuid v1.2.0 github.com/rogpeppe/go-internal => github.com/rogpeppe/go-internal v1.11.0 github.com/rs/xid => github.com/rs/xid v1.5.0 - github.com/rs/zerolog => github.com/rs/zerolog v1.29.1 + github.com/rs/zerolog => github.com/rs/zerolog v1.30.0 github.com/russross/blackfriday/v2 => github.com/russross/blackfriday/v2 v2.1.0 github.com/ruudk/golang-pdf417 => github.com/ruudk/golang-pdf417 v0.0.0-20201230142125-a7e3863a1245 github.com/schollz/progressbar/v2 => github.com/schollz/progressbar/v2 v2.15.0 @@ -261,7 +261,7 @@ replace ( github.com/spf13/cobra => github.com/spf13/cobra v1.7.0 github.com/spf13/pflag => github.com/spf13/pflag v1.0.5 github.com/stoewer/go-strcase => github.com/stoewer/go-strcase v1.3.0 - github.com/stretchr/objx => github.com/stretchr/objx v0.5.0 + github.com/stretchr/objx => github.com/stretchr/objx v0.5.1 github.com/stretchr/testify => github.com/stretchr/testify v1.8.4 github.com/ugorji/go/codec => github.com/ugorji/go/codec v1.2.11 github.com/xeipuuv/gojsonpointer => github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb @@ -283,39 +283,39 @@ replace ( go.opentelemetry.io/otel/sdk/metric => go.opentelemetry.io/otel/sdk/metric v0.33.0 go.opentelemetry.io/otel/trace => go.opentelemetry.io/otel/trace v1.11.1 go.opentelemetry.io/proto/otlp => go.opentelemetry.io/proto/otlp v1.0.0 - go.starlark.net => go.starlark.net v0.0.0-20230725161458-0d7263928a74 + go.starlark.net => go.starlark.net v0.0.0-20230814145427-12f4cb8177e4 go.uber.org/atomic => go.uber.org/atomic v1.11.0 go.uber.org/automaxprocs => go.uber.org/automaxprocs v1.5.3 go.uber.org/goleak => go.uber.org/goleak v1.2.1 go.uber.org/multierr => go.uber.org/multierr v1.11.0 - go.uber.org/zap => go.uber.org/zap v1.24.0 - gocloud.dev => gocloud.dev v0.32.0 - golang.org/x/crypto => golang.org/x/crypto v0.11.0 + go.uber.org/zap => go.uber.org/zap v1.25.0 + gocloud.dev => gocloud.dev v0.33.0 + golang.org/x/crypto => golang.org/x/crypto v0.12.0 golang.org/x/exp => golang.org/x/exp v0.0.0-20230711153332-06a737ee72cb - golang.org/x/exp/typeparams => golang.org/x/exp/typeparams v0.0.0-20230725093048-515e97ebf090 - golang.org/x/image => golang.org/x/image v0.9.0 + golang.org/x/exp/typeparams => golang.org/x/exp/typeparams v0.0.0-20230811145659-89c5cff77bcb + golang.org/x/image => golang.org/x/image v0.11.0 golang.org/x/lint => golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 golang.org/x/mobile => golang.org/x/mobile v0.0.0-20230531173138-3c911d8e3eda golang.org/x/mod => golang.org/x/mod v0.12.0 - golang.org/x/net => golang.org/x/net v0.12.0 - golang.org/x/oauth2 => golang.org/x/oauth2 v0.10.0 + golang.org/x/net => golang.org/x/net v0.14.0 + golang.org/x/oauth2 => golang.org/x/oauth2 v0.11.0 golang.org/x/sync => golang.org/x/sync v0.3.0 - golang.org/x/sys => golang.org/x/sys v0.10.0 - golang.org/x/term => golang.org/x/term v0.10.0 - golang.org/x/text => golang.org/x/text v0.11.0 + golang.org/x/sys => golang.org/x/sys v0.11.0 + golang.org/x/term => golang.org/x/term v0.11.0 + golang.org/x/text => golang.org/x/text v0.12.0 golang.org/x/time => golang.org/x/time v0.3.0 - golang.org/x/tools => golang.org/x/tools v0.11.0 + golang.org/x/tools => golang.org/x/tools v0.12.0 golang.org/x/xerrors => golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 - gomodules.xyz/jsonpatch/v2 => gomodules.xyz/jsonpatch/v2 v2.3.0 + gomodules.xyz/jsonpatch/v2 => gomodules.xyz/jsonpatch/v2 v2.4.0 gonum.org/v1/gonum => gonum.org/v1/gonum v0.13.0 gonum.org/v1/hdf5 => gonum.org/v1/hdf5 v0.0.0-20210714002203-8c5d23bc6946 gonum.org/v1/plot => gonum.org/v1/plot v0.13.0 - google.golang.org/api => google.golang.org/api v0.133.0 + google.golang.org/api => google.golang.org/api v0.137.0 google.golang.org/appengine => google.golang.org/appengine v1.6.7 - google.golang.org/genproto => google.golang.org/genproto v0.0.0-20230725213213-b022f6e96895 - google.golang.org/genproto/googleapis/api => google.golang.org/genproto/googleapis/api v0.0.0-20230725213213-b022f6e96895 - google.golang.org/genproto/googleapis/rpc => google.golang.org/genproto/googleapis/rpc v0.0.0-20230725213213-b022f6e96895 - google.golang.org/grpc => google.golang.org/grpc v1.56.2 + google.golang.org/genproto => google.golang.org/genproto v0.0.0-20230814215434-ca7cfce7776a + google.golang.org/genproto/googleapis/api => google.golang.org/genproto/googleapis/api v0.0.0-20230814215434-ca7cfce7776a + google.golang.org/genproto/googleapis/rpc => google.golang.org/genproto/googleapis/rpc v0.0.0-20230814215434-ca7cfce7776a + google.golang.org/grpc => google.golang.org/grpc v1.57.0 google.golang.org/grpc/cmd/protoc-gen-go-grpc => google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.3.0 google.golang.org/protobuf => google.golang.org/protobuf v1.31.0 gopkg.in/check.v1 => gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c @@ -347,7 +347,7 @@ replace ( require ( cloud.google.com/go/storage v1.31.0 code.cloudfoundry.org/bytefmt v0.0.0-20190710193110-1eb035ffe2b6 - github.com/aws/aws-sdk-go v1.44.303 + github.com/aws/aws-sdk-go v1.44.314 github.com/envoyproxy/protoc-gen-validate v1.0.1 github.com/fsnotify/fsnotify v1.6.0 github.com/go-redis/redis/v8 v8.0.0-00010101000000-000000000000 @@ -386,17 +386,17 @@ require ( go.uber.org/zap v1.24.0 gocloud.dev v0.0.0-00010101000000-000000000000 golang.org/x/exp v0.0.0-20230510235704-dd950f8aeaea - golang.org/x/net v0.12.0 - golang.org/x/oauth2 v0.10.0 + golang.org/x/net v0.14.0 + golang.org/x/oauth2 v0.11.0 golang.org/x/sync v0.3.0 - golang.org/x/sys v0.10.0 - golang.org/x/text v0.11.0 - golang.org/x/tools v0.10.0 + golang.org/x/sys v0.11.0 + golang.org/x/text v0.12.0 + golang.org/x/tools v0.11.0 gonum.org/v1/hdf5 v0.0.0-00010101000000-000000000000 gonum.org/v1/plot v0.10.1 - google.golang.org/genproto/googleapis/api v0.0.0-20230724170836-66ad5b6ff146 - google.golang.org/genproto/googleapis/rpc v0.0.0-20230724170836-66ad5b6ff146 - google.golang.org/grpc v1.56.2 + google.golang.org/genproto/googleapis/api v0.0.0-20230803162519-f966b187b2e5 + google.golang.org/genproto/googleapis/rpc v0.0.0-20230807174057-1744710a1577 + google.golang.org/grpc v1.57.0 google.golang.org/protobuf v1.31.0 gopkg.in/yaml.v2 v2.4.0 k8s.io/api v0.27.3 @@ -408,8 +408,8 @@ require ( ) require ( - cloud.google.com/go v0.110.6 // indirect - cloud.google.com/go/compute v1.22.0 // indirect + cloud.google.com/go v0.110.7 // indirect + cloud.google.com/go/compute v1.23.0 // indirect cloud.google.com/go/compute/metadata v0.2.3 // indirect cloud.google.com/go/iam v1.1.1 // indirect git.sr.ht/~sbinet/gg v0.4.1 // indirect @@ -445,7 +445,7 @@ require ( github.com/google/btree v1.0.1 // indirect github.com/google/gnostic v0.5.7-v3refs // indirect github.com/google/gofuzz v1.1.0 // indirect - github.com/google/s2a-go v0.1.4 // indirect + github.com/google/s2a-go v0.1.5 // indirect github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect github.com/google/wire v0.5.0 // indirect github.com/googleapis/enterprise-certificate-proxy v0.2.5 // indirect @@ -487,19 +487,18 @@ require ( go.opentelemetry.io/otel/exporters/otlp/otlpmetric v0.33.0 // indirect go.opentelemetry.io/proto/otlp v0.19.0 // indirect go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5 // indirect - go.uber.org/atomic v1.11.0 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/crypto v0.11.0 // indirect - golang.org/x/exp/typeparams v0.0.0-20230203172020-98cc5a0785f9 // indirect + golang.org/x/crypto v0.12.0 // indirect + golang.org/x/exp/typeparams v0.0.0-20230307190834-24139beb5833 // indirect golang.org/x/image v0.7.0 // indirect golang.org/x/mod v0.12.0 // indirect - golang.org/x/term v0.10.0 // indirect + golang.org/x/term v0.11.0 // indirect golang.org/x/time v0.3.0 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect gomodules.xyz/jsonpatch/v2 v2.3.0 // indirect - google.golang.org/api v0.132.0 // indirect + google.golang.org/api v0.134.0 // indirect google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto v0.0.0-20230717213848-3f92550aa753 // indirect + google.golang.org/genproto v0.0.0-20230803162519-f966b187b2e5 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/apiextensions-apiserver v0.27.2 // indirect diff --git a/go.sum b/go.sum index 8629b356439..ecea2899a80 100644 --- a/go.sum +++ b/go.sum @@ -1,10 +1,10 @@ -cloud.google.com/go v0.110.6 h1:8uYAkj3YHTP/1iwReuHPxLSbdcyc+dSBbzFMrVwDR6Q= -cloud.google.com/go v0.110.6/go.mod h1:+EYjdK8e5RME/VY/qLCAtuyALQ9q67dvuum8i+H5xsI= +cloud.google.com/go v0.110.7 h1:rJyC7nWRg2jWGZ4wSJ5nY65GTdYJkg0cd/uXb+ACI6o= +cloud.google.com/go v0.110.7/go.mod h1:+EYjdK8e5RME/VY/qLCAtuyALQ9q67dvuum8i+H5xsI= cloud.google.com/go/accessapproval v1.7.1/go.mod h1:JYczztsHRMK7NTXb6Xw+dwbs/WnOJxbo/2mTI+Kgg68= cloud.google.com/go/accesscontextmanager v1.8.0/go.mod h1:uI+AI/r1oyWK99NN8cQ3UK76AMelMzgZCvJfsi2c+ps= cloud.google.com/go/accesscontextmanager v1.8.1/go.mod h1:JFJHfvuaTC+++1iL1coPiG1eu5D24db2wXCDWDjIrxo= -cloud.google.com/go/aiplatform v1.45.0/go.mod h1:Iu2Q7sC7QGhXUeOhAj/oCK9a+ULz1O4AotZiqjQ8MYA= -cloud.google.com/go/analytics v0.21.2/go.mod h1:U8dcUtmDmjrmUTnnnRnI4m6zKn/yaA5N9RlEkYFHpQo= +cloud.google.com/go/aiplatform v1.48.0/go.mod h1:Iu2Q7sC7QGhXUeOhAj/oCK9a+ULz1O4AotZiqjQ8MYA= +cloud.google.com/go/analytics v0.21.3/go.mod h1:U8dcUtmDmjrmUTnnnRnI4m6zKn/yaA5N9RlEkYFHpQo= cloud.google.com/go/apigateway v1.6.1/go.mod h1:ufAS3wpbRjqfZrzpvLC2oh0MFlpRJm2E/ts25yyqmXA= cloud.google.com/go/apigeeconnect v1.6.1/go.mod h1:C4awq7x0JpLtrlQCr8AzVIzAaYgngRqWf9S5Uhg+wWs= cloud.google.com/go/apigeeregistry v0.7.1/go.mod h1:1XgyjZye4Mqtw7T9TsY4NW10U7BojBvG4RMD+vRDrIw= @@ -22,52 +22,53 @@ cloud.google.com/go/billing v1.16.0/go.mod h1:y8vx09JSSJG02k5QxbycNRrN7FGZB6F3CA cloud.google.com/go/binaryauthorization v1.6.1/go.mod h1:TKt4pa8xhowwffiBmbrbcxijJRZED4zrqnwZ1lKH51U= cloud.google.com/go/certificatemanager v1.7.1/go.mod h1:iW8J3nG6SaRYImIa+wXQ0g8IgoofDFRp5UMzaNk1UqI= cloud.google.com/go/channel v1.16.0/go.mod h1:eN/q1PFSl5gyu0dYdmxNXscY/4Fi7ABmeHCJNf/oHmc= -cloud.google.com/go/cloudbuild v1.10.1/go.mod h1:lyJg7v97SUIPq4RC2sGsz/9tNczhyv2AjML/ci4ulzU= +cloud.google.com/go/cloudbuild v1.13.0/go.mod h1:lyJg7v97SUIPq4RC2sGsz/9tNczhyv2AjML/ci4ulzU= cloud.google.com/go/clouddms v1.6.1/go.mod h1:Ygo1vL52Ov4TBZQquhz5fiw2CQ58gvu+PlS6PVXCpZI= -cloud.google.com/go/cloudtasks v1.11.1/go.mod h1:a9udmnou9KO2iulGscKR0qBYjreuX8oHwpmFsKspEvM= -cloud.google.com/go/compute v1.22.0 h1:cB8R6FtUtT1TYGl5R3xuxnW6OUIc/DrT2aiR16TTG7Y= -cloud.google.com/go/compute v1.22.0/go.mod h1:4tCnrn48xsqlwSAiLf1HXMQk8CONslYbdiEZc9FEIbM= +cloud.google.com/go/cloudtasks v1.12.1/go.mod h1:a9udmnou9KO2iulGscKR0qBYjreuX8oHwpmFsKspEvM= +cloud.google.com/go/compute v1.23.0 h1:tP41Zoavr8ptEqaW6j+LQOnyBBhO7OkOMAGrgLopTwY= +cloud.google.com/go/compute v1.23.0/go.mod h1:4tCnrn48xsqlwSAiLf1HXMQk8CONslYbdiEZc9FEIbM= cloud.google.com/go/compute/metadata v0.2.1/go.mod h1:jgHgmJd2RKBGzXqF5LR2EZMGxBkeanZ9wwa75XHJgOM= cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= -cloud.google.com/go/contactcenterinsights v1.9.1/go.mod h1:bsg/R7zGLYMVxFFzfh9ooLTruLRCG9fnzhH9KznHhbM= -cloud.google.com/go/container v1.22.1/go.mod h1:lTNExE2R7f+DLbAN+rJiKTisauFCaoDq6NURZ83eVH4= +cloud.google.com/go/contactcenterinsights v1.10.0/go.mod h1:bsg/R7zGLYMVxFFzfh9ooLTruLRCG9fnzhH9KznHhbM= +cloud.google.com/go/container v1.24.0/go.mod h1:lTNExE2R7f+DLbAN+rJiKTisauFCaoDq6NURZ83eVH4= cloud.google.com/go/containeranalysis v0.10.1/go.mod h1:Ya2jiILITMY68ZLPaogjmOMNkwsDrWBSTyBubGXO7j0= cloud.google.com/go/datacatalog v1.14.0/go.mod h1:h0PrGtlihoutNMp/uvwhawLQ9+c63Kz65UFqh49Yo+E= -cloud.google.com/go/datacatalog v1.14.1/go.mod h1:d2CevwTG4yedZilwe+v3E3ZBDRMobQfSG/a6cCCN5R4= +cloud.google.com/go/datacatalog v1.16.0/go.mod h1:d2CevwTG4yedZilwe+v3E3ZBDRMobQfSG/a6cCCN5R4= cloud.google.com/go/dataflow v0.9.1/go.mod h1:Wp7s32QjYuQDWqJPFFlnBKhkAtiFpMTdg00qGbnIHVw= cloud.google.com/go/dataform v0.8.1/go.mod h1:3BhPSiw8xmppbgzeBbmDvmSWlwouuJkXsXsb8UBih9M= cloud.google.com/go/datafusion v1.7.1/go.mod h1:KpoTBbFmoToDExJUso/fcCiguGDk7MEzOWXUsJo0wsI= cloud.google.com/go/datalabeling v0.8.1/go.mod h1:XS62LBSVPbYR54GfYQsPXZjTW8UxCK2fkDciSrpRFdY= -cloud.google.com/go/dataplex v1.8.1/go.mod h1:7TyrDT6BCdI8/38Uvp0/ZxBslOslP2X2MPDucliyvSE= +cloud.google.com/go/dataplex v1.9.0/go.mod h1:7TyrDT6BCdI8/38Uvp0/ZxBslOslP2X2MPDucliyvSE= cloud.google.com/go/dataproc v1.12.0/go.mod h1:zrF3aX0uV3ikkMz6z4uBbIKyhRITnxvr4i3IjKsKrw4= +cloud.google.com/go/dataproc/v2 v2.0.1/go.mod h1:7Ez3KRHdFGcfY7GcevBbvozX+zyWGcwLJvvAMwCaoZ4= cloud.google.com/go/dataqna v0.8.1/go.mod h1:zxZM0Bl6liMePWsHA8RMGAfmTG34vJMapbHAxQ5+WA8= -cloud.google.com/go/datastore v1.12.1/go.mod h1:KjdB88W897MRITkvWWJrg2OUtrR5XVj1EoLgSp6/N70= -cloud.google.com/go/datastream v1.9.1/go.mod h1:hqnmr8kdUBmrnk65k5wNRoHSCYksvpdZIcZIEl8h43Q= -cloud.google.com/go/deploy v1.11.0/go.mod h1:tKuSUV5pXbn67KiubiUNUejqLs4f5cxxiCNCeyl0F2g= -cloud.google.com/go/dialogflow v1.38.0/go.mod h1:L7jnH+JL2mtmdChzAIcXQHXMvQkE3U4hTaNltEuxXn4= +cloud.google.com/go/datastore v1.13.0/go.mod h1:KjdB88W897MRITkvWWJrg2OUtrR5XVj1EoLgSp6/N70= +cloud.google.com/go/datastream v1.10.0/go.mod h1:hqnmr8kdUBmrnk65k5wNRoHSCYksvpdZIcZIEl8h43Q= +cloud.google.com/go/deploy v1.13.0/go.mod h1:tKuSUV5pXbn67KiubiUNUejqLs4f5cxxiCNCeyl0F2g= +cloud.google.com/go/dialogflow v1.40.0/go.mod h1:L7jnH+JL2mtmdChzAIcXQHXMvQkE3U4hTaNltEuxXn4= cloud.google.com/go/dlp v1.10.1/go.mod h1:IM8BWz1iJd8njcNcG0+Kyd9OPnqnRNkDV8j42VT5KOI= -cloud.google.com/go/documentai v1.20.0/go.mod h1:yJkInoMcK0qNAEdRnqY/D5asy73tnPe88I1YTZT+a8E= +cloud.google.com/go/documentai v1.22.0/go.mod h1:yJkInoMcK0qNAEdRnqY/D5asy73tnPe88I1YTZT+a8E= cloud.google.com/go/domains v0.9.1/go.mod h1:aOp1c0MbejQQ2Pjf1iJvnVyT+z6R6s8pX66KaCSDYfE= cloud.google.com/go/edgecontainer v1.1.1/go.mod h1:O5bYcS//7MELQZs3+7mabRqoWQhXCzenBu0R8bz2rwk= cloud.google.com/go/errorreporting v0.3.0/go.mod h1:xsP2yaAp+OAW4OIm60An2bbLpqIhKXdWR/tawvl7QzU= cloud.google.com/go/essentialcontacts v1.6.2/go.mod h1:T2tB6tX+TRak7i88Fb2N9Ok3PvY3UNbUsMag9/BARh4= -cloud.google.com/go/eventarc v1.12.1/go.mod h1:mAFCW6lukH5+IZjkvrEss+jmt2kOdYlN8aMx3sRJiAI= +cloud.google.com/go/eventarc v1.13.0/go.mod h1:mAFCW6lukH5+IZjkvrEss+jmt2kOdYlN8aMx3sRJiAI= cloud.google.com/go/filestore v1.7.1/go.mod h1:y10jsorq40JJnjR/lQ8AfFbbcGlw3g+Dp8oN7i7FjV4= -cloud.google.com/go/firestore v1.11.0/go.mod h1:b38dKhgzlmNNGTNZZwe7ZRFEuRab1Hay3/DBsIGKKy4= +cloud.google.com/go/firestore v1.12.0/go.mod h1:b38dKhgzlmNNGTNZZwe7ZRFEuRab1Hay3/DBsIGKKy4= cloud.google.com/go/functions v1.15.1/go.mod h1:P5yNWUTkyU+LvW/S9O6V+V423VZooALQlqoXdoPz5AE= cloud.google.com/go/gkebackup v1.3.0/go.mod h1:vUDOu++N0U5qs4IhG1pcOnD1Mac79xWy6GoBFlWCWBU= cloud.google.com/go/gkeconnect v0.8.1/go.mod h1:KWiK1g9sDLZqhxB2xEuPV8V9NYzrqTUmQR9shJHpOZw= cloud.google.com/go/gkehub v0.14.1/go.mod h1:VEXKIJZ2avzrbd7u+zeMtW00Y8ddk/4V9511C9CQGTY= -cloud.google.com/go/gkemulticloud v0.6.1/go.mod h1:kbZ3HKyTsiwqKX7Yw56+wUGwwNZViRnxWK2DVknXWfw= +cloud.google.com/go/gkemulticloud v1.0.0/go.mod h1:kbZ3HKyTsiwqKX7Yw56+wUGwwNZViRnxWK2DVknXWfw= cloud.google.com/go/grafeas v0.3.0/go.mod h1:P7hgN24EyONOTMyeJH6DxG4zD7fwiYa5Q6GUgyFSOU8= cloud.google.com/go/gsuiteaddons v1.6.1/go.mod h1:CodrdOqRZcLp5WOwejHWYBjZvfY0kOphkAKpF/3qdZY= -cloud.google.com/go/iam v1.1.1 h1:lW7fzj15aVIXYHREOqjRBV9PsH0Z6u8Y46a1YGvQP4Y= -cloud.google.com/go/iam v1.1.1/go.mod h1:A5avdyVL2tCppe4unb0951eI9jreack+RJ0/d+KUZOU= +cloud.google.com/go/iam v1.1.2 h1:gacbrBdWcoVmGLozRuStX45YKvJtzIjJdAolzUs1sm4= +cloud.google.com/go/iam v1.1.2/go.mod h1:A5avdyVL2tCppe4unb0951eI9jreack+RJ0/d+KUZOU= cloud.google.com/go/iap v1.8.1/go.mod h1:sJCbeqg3mvWLqjZNsI6dfAtbbV1DL2Rl7e1mTyXYREQ= cloud.google.com/go/ids v1.4.1/go.mod h1:np41ed8YMU8zOgv53MMMoCntLTn2lF+SUzlM+O3u/jw= cloud.google.com/go/iot v1.7.1/go.mod h1:46Mgw7ev1k9KqK1ao0ayW9h0lI+3hxeanz+L1zmbbbk= -cloud.google.com/go/kms v1.15.0/go.mod h1:c9J991h5DTl+kg7gi3MYomh12YEENGrf48ee/N/2CDM= +cloud.google.com/go/kms v1.15.1/go.mod h1:c9J991h5DTl+kg7gi3MYomh12YEENGrf48ee/N/2CDM= cloud.google.com/go/language v1.10.1/go.mod h1:CPp94nsdVNiQEt1CNjF5WkTcisLiHPyIbMhvR8H2AW0= cloud.google.com/go/lifesciences v0.9.1/go.mod h1:hACAOd1fFbCGLr/+weUKRAJas82Y4vrL3O5326N//Wc= cloud.google.com/go/logging v1.7.0/go.mod h1:3xjP2CjkM3ZkO73aj4ASA5wRPGGCRrPIAeNqVNkzY8M= @@ -77,10 +78,10 @@ cloud.google.com/go/longrunning v0.4.2/go.mod h1:OHrnaYyLUV6oqwh0xiS7e5sLQhP1m0Q cloud.google.com/go/longrunning v0.5.0/go.mod h1:0JNuqRShmscVAhIACGtskSAWtqtOoPkwP0YF1oVEchc= cloud.google.com/go/longrunning v0.5.1/go.mod h1:spvimkwdz6SPWKEt/XBij79E9fiTkHSQl/fRUUQJYJc= cloud.google.com/go/managedidentities v1.6.1/go.mod h1:h/irGhTN2SkZ64F43tfGPMbHnypMbu4RB3yl8YcuEak= -cloud.google.com/go/maps v1.3.0/go.mod h1:6mWTUv+WhnOwAgjVsSW2QPPECmW+s3PcRyOa9vgG/5s= +cloud.google.com/go/maps v1.4.0/go.mod h1:6mWTUv+WhnOwAgjVsSW2QPPECmW+s3PcRyOa9vgG/5s= cloud.google.com/go/mediatranslation v0.8.1/go.mod h1:L/7hBdEYbYHQJhX2sldtTO5SZZ1C1vkapubj0T2aGig= cloud.google.com/go/memcache v1.10.1/go.mod h1:47YRQIarv4I3QS5+hoETgKO40InqzLP6kpNLvyXuyaA= -cloud.google.com/go/metastore v1.11.1/go.mod h1:uZuSo80U3Wd4zi6C22ZZliOUJ3XeM/MlYi/z5OAOWRA= +cloud.google.com/go/metastore v1.12.0/go.mod h1:uZuSo80U3Wd4zi6C22ZZliOUJ3XeM/MlYi/z5OAOWRA= cloud.google.com/go/monitoring v1.15.1/go.mod h1:lADlSAlFdbqQuwwpaImhsJXu1QSdd3ojypXrFSMr2rM= cloud.google.com/go/networkconnectivity v1.12.1/go.mod h1:PelxSWYM7Sh9/guf8CFhi6vIqf19Ir/sbfZRUwXh92E= cloud.google.com/go/networkmanagement v1.8.0/go.mod h1:Ho/BUGmtyEqrttTgWEe7m+8vDdK74ibQc+Be0q7Fof0= @@ -94,7 +95,7 @@ cloud.google.com/go/osconfig v1.12.0/go.mod h1:8f/PaYzoS3JMVfdfTubkowZYGmAhUCjjw cloud.google.com/go/osconfig v1.12.1/go.mod h1:4CjBxND0gswz2gfYRCUoUzCm9zCABp91EeTtWXyz0tE= cloud.google.com/go/oslogin v1.10.1/go.mod h1:x692z7yAue5nE7CsSnoG0aaMbNoRJRXO4sn73R+ZqAs= cloud.google.com/go/phishingprotection v0.8.1/go.mod h1:AxonW7GovcA8qdEk13NfHq9hNx5KPtfxXNeUxTDxB6I= -cloud.google.com/go/policytroubleshooter v1.7.1/go.mod h1:0NaT5v3Ag1M7U5r0GfDCpUFkWd9YqpubBWsQlhanRv0= +cloud.google.com/go/policytroubleshooter v1.8.0/go.mod h1:tmn5Ir5EToWe384EuboTcVQT7nTag2+DuH3uHmKd1HU= cloud.google.com/go/privatecatalog v0.9.1/go.mod h1:0XlDXW2unJXdf9zFz968Hp35gl/bhF4twwpXZAW50JA= cloud.google.com/go/pubsub v1.33.0/go.mod h1:f+w71I33OMyxf9VpMVcZbnG5KSUkCOUHYpFd5U1GdRc= cloud.google.com/go/pubsublite v1.8.1/go.mod h1:fOLdU4f5xldK4RGJrBMm+J7zMWNj/k4PxwEZXy39QS0= @@ -110,10 +111,10 @@ cloud.google.com/go/scheduler v1.10.1/go.mod h1:R63Ldltd47Bs4gnhQkmNDse5w8gBRrhO cloud.google.com/go/secretmanager v1.11.1/go.mod h1:znq9JlXgTNdBeQk9TBW/FnR/W4uChEKGeqQWAJ8SXFw= cloud.google.com/go/security v1.15.1/go.mod h1:MvTnnbsWnehoizHi09zoiZob0iCHVcL4AUBj76h9fXA= cloud.google.com/go/securitycenter v1.23.0/go.mod h1:8pwQ4n+Y9WCWM278R8W3nF65QtY172h4S8aXyI9/hsQ= -cloud.google.com/go/servicedirectory v1.10.1/go.mod h1:Xv0YVH8s4pVOwfM/1eMTl0XJ6bzIOSLDt8f8eLaGOxQ= +cloud.google.com/go/servicedirectory v1.11.0/go.mod h1:Xv0YVH8s4pVOwfM/1eMTl0XJ6bzIOSLDt8f8eLaGOxQ= cloud.google.com/go/shell v1.7.1/go.mod h1:u1RaM+huXFaTojTbW4g9P5emOrrmLE69KrxqQahKn4g= cloud.google.com/go/spanner v1.47.0/go.mod h1:IXsJwVW2j4UKs0eYDqodab6HgGuA1bViSqW4uH9lfUI= -cloud.google.com/go/speech v1.17.1/go.mod h1:8rVNzU43tQvxDaGvqOhpDqgkJTFowBpDvCJ14kGlJYo= +cloud.google.com/go/speech v1.19.0/go.mod h1:8rVNzU43tQvxDaGvqOhpDqgkJTFowBpDvCJ14kGlJYo= cloud.google.com/go/storage v1.31.0 h1:+S3LjjEN2zZ+L5hOwj4+1OkGCsLVe0NzpXKQ1pSdTCI= cloud.google.com/go/storage v1.31.0/go.mod h1:81ams1PrhW16L4kF7qg+4mTq7SRs5HsbDTM0bWvrwJ0= cloud.google.com/go/storagetransfer v1.10.0/go.mod h1:DM4sTlSmGiNczmV6iZyceIh2dbs+7z2Ayg6YAiQlYfA= @@ -121,12 +122,12 @@ cloud.google.com/go/talent v1.6.2/go.mod h1:CbGvmKCG61mkdjcqTcLOkb2ZN1SrQI8MDyma cloud.google.com/go/texttospeech v1.7.1/go.mod h1:m7QfG5IXxeneGqTapXNxv2ItxP/FS0hCZBwXYqucgSk= cloud.google.com/go/tpu v1.6.1/go.mod h1:sOdcHVIgDEEOKuqUoi6Fq53MKHJAtOwtz0GuKsWSH3E= cloud.google.com/go/trace v1.10.1/go.mod h1:gbtL94KE5AJLH3y+WVpfWILmqgc6dXcqgNXdOPAQTYk= -cloud.google.com/go/translate v1.8.1/go.mod h1:d1ZH5aaOA0CNhWeXeC8ujd4tdCFw8XoNWRljklu5RHs= -cloud.google.com/go/video v1.17.1/go.mod h1:9qmqPqw/Ib2tLqaeHgtakU+l5TcJxCJbhFXM7UJjVzU= +cloud.google.com/go/translate v1.8.2/go.mod h1:d1ZH5aaOA0CNhWeXeC8ujd4tdCFw8XoNWRljklu5RHs= +cloud.google.com/go/video v1.19.0/go.mod h1:9qmqPqw/Ib2tLqaeHgtakU+l5TcJxCJbhFXM7UJjVzU= cloud.google.com/go/videointelligence v1.11.1/go.mod h1:76xn/8InyQHarjTWsBR058SmlPCwQjgcvoW0aZykOvo= cloud.google.com/go/vision/v2 v2.7.2/go.mod h1:jKa8oSYBWhYiXarHPvP4USxYANYUEdEsQrloLjrSwJU= cloud.google.com/go/vmmigration v1.7.1/go.mod h1:WD+5z7a/IpZ5bKK//YmT9E047AD+rjycCAvyMxGJbro= -cloud.google.com/go/vmwareengine v0.4.1/go.mod h1:Px64x+BvjPZwWuc4HdmVhoygcXqEkGHXoa7uyfTgSI0= +cloud.google.com/go/vmwareengine v1.0.0/go.mod h1:Px64x+BvjPZwWuc4HdmVhoygcXqEkGHXoa7uyfTgSI0= cloud.google.com/go/vpcaccess v1.7.1/go.mod h1:FogoD46/ZU+JUBX9D606X21EnxiszYi2tArQwLY4SXs= cloud.google.com/go/webrisk v1.9.1/go.mod h1:4GCmXKcOa2BZcZPn6DCEvE7HypmEJcJkr4mtM+sqYPc= cloud.google.com/go/websecurityscanner v1.6.1/go.mod h1:Njgaw3rttgRHXzwCB8kgCYqv5/rGpFCsBOvPbYgszpg= @@ -174,27 +175,27 @@ github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kd github.com/apache/arrow/go/v12 v12.0.0/go.mod h1:d+tV/eHZZ7Dz7RPrFKtPK02tpr+c9/PEd/zm8mDS9Vg= github.com/apache/thrift v0.16.0/go.mod h1:PHK3hniurgQaNMZYaCLEqXKsYK8upmhPbmdP2FXSqgU= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= -github.com/aws/aws-sdk-go v1.44.308 h1:XKu+76UHsD5LaiU2Zb1q42uWakw80Az7x39jJXXahos= -github.com/aws/aws-sdk-go v1.44.308/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= -github.com/aws/aws-sdk-go-v2 v1.19.0 h1:klAT+y3pGFBU/qVf1uzwttpBbiuozJYWzNLHioyDJ+k= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10 h1:dK82zF6kkPeCo8J1e+tGx4JdvDIQzj7ygIoLg8WMuGs= -github.com/aws/aws-sdk-go-v2/config v1.18.29 h1:yA+bSSRGhBwWuprG9I4VgxfK//NBLZ/0BGOHiV3f9oM= -github.com/aws/aws-sdk-go-v2/credentials v1.13.28 h1:WM9tEHgoOh5ThJZ042UKnSx7TXGSC/bz63X3fsrQL2o= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.5 h1:kP3Me6Fy3vdi+9uHd7YLr6ewPxRL+PU6y15urfTaamU= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.73 h1:ez9K7LHdBxo+g1ExpUPsWEFfHxzHhiVZriWAJAiZwKc= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.35 h1:hMUCiE3Zi5AHrRNGf5j985u0WyqI6r2NULhUfo0N/No= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.29 h1:yOpYx+FTBdpk/g+sBU6Cb1H0U/TLEcYYp66mYqsPpcc= -github.com/aws/aws-sdk-go-v2/internal/ini v1.3.36 h1:8r5m1BoAWkn0TDC34lUculryf7nUF25EgIMdjvGCkgo= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.27 h1:cZG7psLfqpkB6H+fIrgUDWmlzM474St1LP0jcz272yI= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.11 h1:y2+VQzC6Zh2ojtV2LoC0MNwHWc6qXv/j2vrQtlftkdA= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.30 h1:Bje8Xkh2OWpjBdNfXLrnn8eZg569dUQmhgtydxAYyP0= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.29 h1:IiDolu/eLmuB18DRZibj77n1hHQT7z12jnGO7Ze3pLc= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.14.4 h1:hx4WksB0NRQ9utR+2c3gEGzl6uKj3eM6PMQ6tN3lgXs= -github.com/aws/aws-sdk-go-v2/service/s3 v1.37.0 h1:PalLOEGZ/4XfQxpGZFTLaoJSmPoybnqJYotaIZEf/Rg= -github.com/aws/aws-sdk-go-v2/service/sso v1.12.13 h1:sWDv7cMITPcZ21QdreULwxOOAmE05JjEsT6fCDtDA9k= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.13 h1:BFubHS/xN5bjl818QaroN6mQdjneYQ+AOx44KNXlyH4= -github.com/aws/aws-sdk-go-v2/service/sts v1.20.0 h1:jKmIOO+dFvCPuIhhM8u0Dy3dtd590n2kEDSYiGHoI98= -github.com/aws/smithy-go v1.13.5 h1:hgz0X/DX0dGqTYpGALqXJoRKRj5oQ7150i5FdTePzO8= +github.com/aws/aws-sdk-go v1.44.323 h1:97/dn93DWrN1VfhAWQ2tV+xuE6oO/LO9rSsEsuC4PLU= +github.com/aws/aws-sdk-go v1.44.323/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= +github.com/aws/aws-sdk-go-v2 v1.20.1 h1:rZBf5DWr7YGrnlTK4kgDQGn1ltqOg5orCYb/UhOFZkg= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.12 h1:lN6L3LrYHeZ6xCxaIYtoWCx4GMLk4nRknsh29OMSqHY= +github.com/aws/aws-sdk-go-v2/config v1.18.33 h1:JKcw5SFxFW/rpM4mOPjv0VQ11E2kxW13F3exWOy7VZU= +github.com/aws/aws-sdk-go-v2/credentials v1.13.32 h1:lIH1eKPcCY1ylR4B6PkBGRWMHO3aVenOKJHWiS4/G2w= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.8 h1:DK/9C+UN/X+1+Wm8pqaDksQr2tSLzq+8X1/rI/ZxKEQ= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.77 h1:oWSNL9oQy+do911sXpJyIc2J7RiUrbm9BecyaGy1wHo= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.38 h1:c8ed/T9T2K5I+h/JzmF5tpI46+OODQ74dzmdo+QnaMg= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.32 h1:hNeAAymUY5gu11WrrmFb3CVIp9Dar9hbo44yzzcQpzA= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.39 h1:fc0ukRAiP1syoSGZYu+DaE+FulSYhTiJ8WpVu5jElU4= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.1.0 h1:U5yySdwt2HPo/pnQec04DImLzWORbeWML1fJiLkKruI= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.13 h1:iV/W5OMBys+66OeXJi/7xIRrKZNsu0ylsLGu+6nbmQE= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.33 h1:QviNkc+vGSuEHx8P+pVNKOdWLXBPIwMFv7p0fphgE4U= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.32 h1:dGAseBFEYxth10V23b5e2mAS+tX7oVbfYHD6dnDdAsg= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.15.1 h1:PT6PBCycRwhpEW5hJnRiceCeoWJ+r3bdgXtV+VKG7Pk= +github.com/aws/aws-sdk-go-v2/service/s3 v1.38.2 h1:v346f1h8sUBKXnEbrv43L37MTBlFHyKXQPIZHNAaghA= +github.com/aws/aws-sdk-go-v2/service/sso v1.13.2 h1:A2RlEMo4SJSwbNoUUgkxTAEMduAy/8wG3eB2b2lP4gY= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.15.1 h1:hd0SKLMdOL/Sl6Z0np1PX9LeH2gqNtBe0MhTedA8MGI= +github.com/aws/aws-sdk-go-v2/service/sts v1.21.2 h1:ympg1+Lnq33XLhcK/xTG4yZHPs1Oyxu+6DEWbl7qOzA= +github.com/aws/smithy-go v1.14.1 h1:EFKMUmH/iHMqLiwoEDx2rRjRQpI1YCn5jTysoaDujFs= github.com/benbjohnson/clock v1.3.5 h1:VvXlSJBzZpA/zum6Sj74hxwYI2DIxRWuNIoXAzHZz5o= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= @@ -237,8 +238,8 @@ github.com/evanphx/json-patch/v5 v5.6.0 h1:b91NhWfaz02IuVxO9faSllyAtNXHMPkC5J8sJ github.com/evanphx/json-patch/v5 v5.6.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2VvlbKOFpnXhI9Bw4= github.com/flowstack/go-jsonschema v0.1.1/go.mod h1:yL7fNggx1o8rm9RlgXv7hTBWxdBM0rVwpMwimd3F3N0= github.com/fogleman/gg v1.3.0/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= -github.com/frankban/quicktest v1.14.5 h1:dfYrrRyLtiqT9GyKXgdh+k4inNeTvmGbuSgZ3lx3GhA= -github.com/frankban/quicktest v1.14.5/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= +github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= +github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fredbi/uri v0.1.0/go.mod h1:1xC40RnIOGCaQzswaOvrzvG/3M3F0hyDVb3aO/1iGy0= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= @@ -306,8 +307,8 @@ github.com/golang-sql/sqlexp v0.1.0 h1:ZCD6MBpcuOVfGVqsEmY5/4FtYiKz6tSyUv9LPEDei github.com/golang-sql/sqlexp v0.1.0/go.mod h1:J4ad9Vo8ZCWQ2GMrC4UCQy1JpCbwU9m3EOqtpKwwwHI= github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 h1:DACJavvAHhabrF08vX0COfcOBJRhZ8lUbR+ZWIs0Y5g= github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= -github.com/golang/glog v1.1.1 h1:jxpi2eWoU84wbX9iIEyAeeoac3FLuifZpY9tcNUD9kw= -github.com/golang/glog v1.1.1/go.mod h1:zR+okUeTbrL6EL3xHUDxZuEtGv04p5shwip1+mL/rLQ= +github.com/golang/glog v1.1.2 h1:DVjP2PbBOzHyzA+dn3WhHIq4NdVu3Q+pvivFICf/7fo= +github.com/golang/glog v1.1.2/go.mod h1:zR+okUeTbrL6EL3xHUDxZuEtGv04p5shwip1+mL/rLQ= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= @@ -329,11 +330,12 @@ github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian/v3 v3.3.2 h1:IqNFLAmvJOgVlpdEBiQbDc2EwKW77amAycfTuWKdfvw= github.com/google/martian/v3 v3.3.2/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= -github.com/google/pprof v0.0.0-20230705174524-200ffdc848b8 h1:n6vlPhxsA+BW/XsS5+uqi7GyzaLa5MH7qlSLBZtRdiA= -github.com/google/pprof v0.0.0-20230705174524-200ffdc848b8/go.mod h1:Jh3hGz2jkYak8qXPD19ryItVnUgpgeqzdkY/D0EaeuA= +github.com/google/pprof v0.0.0-20230811205829-9131a7e9cc17 h1:0h35ESZ02+hN/MFZb7XZOXg+Rl9+Rk8fBIf5YLws9gA= +github.com/google/pprof v0.0.0-20230811205829-9131a7e9cc17/go.mod h1:Jh3hGz2jkYak8qXPD19ryItVnUgpgeqzdkY/D0EaeuA= github.com/google/s2a-go v0.1.0/go.mod h1:OJpEgntRZo8ugHpF9hkoLJbS5dSI20XZeXJ9JVywLlM= -github.com/google/s2a-go v0.1.4 h1:1kZ/sQM3srePvKs3tXAvQzo66XfcReoqFpIpIccE7Oc= github.com/google/s2a-go v0.1.4/go.mod h1:Ej+mSEMGRnqRzjc7VtF+jdBwYG5fuJfiZ8ELkjEwM0A= +github.com/google/s2a-go v0.1.5 h1:8IYp3w9nysqv3JH+NJgXJzGbDHzLOTj43BmSkp+O7qg= +github.com/google/s2a-go v0.1.5/go.mod h1:Ej+mSEMGRnqRzjc7VtF+jdBwYG5fuJfiZ8ELkjEwM0A= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= github.com/google/subcommands v1.2.0/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk= @@ -358,8 +360,8 @@ github.com/goxjs/gl v0.0.0-20210104184919-e3fafc6f8f2a/go.mod h1:dy/f2gjY09hwVfI github.com/goxjs/glfw v0.0.0-20191126052801-d2efb5f20838/go.mod h1:oS8P8gVOT4ywTcjV6wZlOU4GuVFQ8F5328KY3MJ79CY= github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 h1:+ngKgrYPPJrOjhax5N+uePQ0Fh1Z7PheYoUI/0nzkPA= github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.1 h1:RoziI+96HlQWrbaVhgOOdFYUHtX81pwA6tCgDS9FNRo= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.1/go.mod h1:Rj8lEaVgLiPn1jTMVXEhATiZhuyXJq167bMYPbJM1CY= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.2 h1:dygLcbEBA+t/P7ck6a8AkXv6juQ4cK0RHBoh32jxhHM= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.2/go.mod h1:Ap9RLCIJVtgQg1/BBgVEfypOAySvvlcpcVQkSzJCH4Y= github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek= github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/huandu/xstrings v1.4.0 h1:D17IlohoQq4UcpqD7fDk80P7l+lwAmlFaBHgOipl2FU= @@ -391,8 +393,8 @@ github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:C github.com/kisielk/errcheck v1.6.3/go.mod h1:nXw/i/MfnvRHqXa7XXmQMUB0oNFGuBrNI8d8NLy0LPw= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/asmfmt v1.3.2/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j0HLHbNSE= -github.com/klauspost/compress v1.16.8-0.20230717094025-33c9bc800133 h1:EHtgNm0KvRXG0aBa0+5ur6mGJXAsxVF+MLu2/YSTVXQ= -github.com/klauspost/compress v1.16.8-0.20230717094025-33c9bc800133/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/klauspost/compress v1.16.8-0.20230809110045-b404607005bc h1:5EwbwgCPqUsIcq9X7+CKH3a0NVJZFxbrY8fz9PUQEz8= +github.com/klauspost/compress v1.16.8-0.20230809110045-b404607005bc/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/qbZg= github.com/klauspost/cpuid/v2 v2.2.5/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/kpango/fastime v1.1.9 h1:xVQHcqyPt5M69DyFH7g1EPRns1YQNap9d5eLhl/Jy84= @@ -464,7 +466,7 @@ github.com/pierrec/lz4/v3 v3.3.5/go.mod h1:280XNCGS8jAcG++AHdd6SeWnzyJ1w9oow2vbO github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/sftp v1.13.5/go.mod h1:wHDZ0IZX6JcBYRK1TH9bcVq8G7TLpVHYIGJRFnmPfxg= +github.com/pkg/sftp v1.13.6/go.mod h1:tz1ryNURKu77RL+GuCzmoJYxQczL3wLNNpPWagdg4Qk= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prashantv/gostub v1.1.0 h1:BTyx3RfQjRHnUWaGF9oQos79AlQ5k8WNktv7VGvVH4g= @@ -474,10 +476,10 @@ github.com/prometheus/client_model v0.4.0 h1:5lQXD3cAg1OXBf4Wq03gTrXHeaV0TQvGfUo github.com/prometheus/client_model v0.4.0/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU= github.com/prometheus/common v0.44.0 h1:+5BrQJwiBB9xsMygAB3TNvpQKOwlkc25LbISbrdOOfY= github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO7x0VV9VvuY= -github.com/prometheus/procfs v0.11.0 h1:5EAgkfkMl659uZPbe9AS2N68a7Cc1TJbPEuGzFuRbyk= -github.com/prometheus/procfs v0.11.0/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPHWJq+XBB/FM= -github.com/quasilyte/go-ruleguard v0.3.19 h1:tfMnabXle/HzOb5Xe9CUZYWXKfkS1KwRmZyPmD9nVcc= -github.com/quasilyte/go-ruleguard v0.3.19/go.mod h1:lHSn69Scl48I7Gt9cX3VrbsZYvYiBYszZOZW4A+oTEw= +github.com/prometheus/procfs v0.11.1 h1:xRC8Iq1yyca5ypa9n1EZnWZkt7dwcoRPQwX/5gwaUuI= +github.com/prometheus/procfs v0.11.1/go.mod h1:eesXgaPo1q7lBpVMoMy0ZOFTth9hBn4W/y0/p/ScXhY= +github.com/quasilyte/go-ruleguard v0.4.0 h1:DyM6r+TKL+xbKB4Nm7Afd1IQh9kEUKQs2pboWGKtvQo= +github.com/quasilyte/go-ruleguard v0.4.0/go.mod h1:Eu76Z/R8IXtViWUIHkE3p8gdH3/PKk1eh3YGfaEof10= github.com/quasilyte/go-ruleguard/dsl v0.3.22 h1:wd8zkOhSNr+I+8Qeciml08ivDt1pSXe60+5DqOpCjPE= github.com/quasilyte/go-ruleguard/dsl v0.3.22/go.mod h1:KeCP03KrjuSO0H1kTuZQCWlQPulDV6YMIXmpQss17rU= github.com/quasilyte/gogrep v0.5.0 h1:eTKODPXbI8ffJMN+W2aE0+oL0z/nh8/5eNdiO34SOAo= @@ -513,8 +515,8 @@ github.com/srwiley/rasterx v0.0.0-20200120212402-85cb7272f5e9/go.mod h1:mvWM0+15 github.com/srwiley/rasterx v0.0.0-20210519020934-456a8d69b780/go.mod h1:mvWM0+15UqyrFKqdRjY6LuAVJR0HOVhJlEgZ5JWtSWU= github.com/srwiley/rasterx v0.0.0-20220128185129-2efea2b9ea41/go.mod h1:nXTWP6+gD5+LUJ8krVhhoeHjvHTutPxMYl5SvkcnJNE= github.com/stoewer/go-strcase v1.3.0/go.mod h1:fAH5hQ5pehh+j3nZfvwdk2RgEgQjAoM8wodgtPmh1xo= -github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= -github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.1 h1:4VhoImhV/Bm0ToFkXFi8hXNXwpDRZ/ynw3amt82mzq0= +github.com/stretchr/objx v0.5.1/go.mod h1:/iHQpkQwBD6DLUmQ4pE+s1TXdob1mORJ4/UFdrifcy0= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/tevino/abool v1.2.0/go.mod h1:qc66Pna1RiIsPa7O4Egxxs9OqkuxDX55zznh9K07Tzg= @@ -555,73 +557,71 @@ go.opentelemetry.io/otel/trace v1.11.1 h1:ofxdnzsNrGBYXbP7t7zpUK281+go5rF7dvdIZX go.opentelemetry.io/otel/trace v1.11.1/go.mod h1:f/Q9G7vzk5u91PhbmKbg1Qn0rzH1LJ4vbPHFGkTPtOk= go.opentelemetry.io/proto/otlp v1.0.0 h1:T0TX0tmXU8a3CbNXzEKGeU5mIVOdf0oykP+u2lIVU/I= go.opentelemetry.io/proto/otlp v1.0.0/go.mod h1:Sy6pihPLfYHkr3NkUbEhGHFhINUSI/v80hjKIs5JXpM= -go.starlark.net v0.0.0-20230725161458-0d7263928a74 h1:EL8MuNFlzO8vvpHgZxDGPaehP0ozoJ1j1zA768zKXUQ= -go.starlark.net v0.0.0-20230725161458-0d7263928a74/go.mod h1:jxU+3+j+71eXOW14274+SmmuW82qJzl6iZSeqEtTGds= -go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= -go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= +go.starlark.net v0.0.0-20230814145427-12f4cb8177e4 h1:Ydko8M6UfXgvSpGOnbAjRMQDIvBheUsjBjkm6Azcpf4= +go.starlark.net v0.0.0-20230814145427-12f4cb8177e4/go.mod h1:jxU+3+j+71eXOW14274+SmmuW82qJzl6iZSeqEtTGds= go.uber.org/automaxprocs v1.5.3 h1:kWazyxZUrS3Gs4qUpbwo5kEIMGe/DAvi5Z4tl2NW4j8= go.uber.org/automaxprocs v1.5.3/go.mod h1:eRbA25aqJrxAbsLO0xy5jVwPt7FQnRgjW+efnwa1WM0= go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= go.uber.org/goleak v1.2.1/go.mod h1:qlT2yGI9QafXHhZZLxlSuNsMw3FFLxBr+tBRlmO1xH4= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= -go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= -go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= -gocloud.dev v0.32.0 h1:jHf8WSkByuAuXcvFt04OiiQH+N0zaRtxI6iEph8Bq8Y= -gocloud.dev v0.32.0/go.mod h1:m/x/N9cRjDF5MD0i5TLFbKbqkGffl/qayXA9FcMT5Oc= -golang.org/x/crypto v0.11.0 h1:6Ewdq3tDic1mg5xRO4milcWCfMVQhI4NkqWWvqejpuA= -golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio= +go.uber.org/zap v1.25.0 h1:4Hvk6GtkucQ790dqmj7l1eEnRdKm3k3ZUrUMS2d5+5c= +go.uber.org/zap v1.25.0/go.mod h1:JIAUzQIH94IC4fOJQm7gMmBJP5k7wQfdcnYdPoEXJYk= +gocloud.dev v0.33.0 h1:ET5z49jm1+eUhY5BkuGk2d7czfgGeXKd4vtg1Jcg9OQ= +gocloud.dev v0.33.0/go.mod h1:z6W8qorjrfM09H8t1MDk8KLPj3Xi26aFBzDKAHWIgLU= +golang.org/x/crypto v0.12.0 h1:tFM/ta59kqch6LlvYnPa0yx5a83cL2nHflFhYKvv9Yk= +golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= golang.org/x/exp v0.0.0-20230711153332-06a737ee72cb h1:xIApU0ow1zwMa2uL1VDNeQlNVFTWMQxZUZCMDy0Q4Us= golang.org/x/exp v0.0.0-20230711153332-06a737ee72cb/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= golang.org/x/exp/shiny v0.0.0-20220827204233-334a2380cb91/go.mod h1:VjAR7z0ngyATZTELrBSkxOOHhhlnVUxDye4mcjx5h/8= golang.org/x/exp/shiny v0.0.0-20230425010034-47ecfdc1ba53/go.mod h1:UH99kUObWAZkDnWqppdQe5ZhPYESUw8I0zVV1uWBR+0= -golang.org/x/exp/typeparams v0.0.0-20230725093048-515e97ebf090 h1:qOYhjyK9OeXREdh7Zrta8JRvnmnFIzhkosQpp+852Ag= -golang.org/x/exp/typeparams v0.0.0-20230725093048-515e97ebf090/go.mod h1:AbB0pIl9nAr9wVwH+Z2ZpaocVmF5I4GyWCDIsVjR0bk= -golang.org/x/image v0.9.0 h1:QrzfX26snvCM20hIhBwuHI/ThTg18b/+kcKdXHvnR+g= -golang.org/x/image v0.9.0/go.mod h1:jtrku+n79PfroUbvDdeUWMAI+heR786BofxrbiSF+J0= +golang.org/x/exp/typeparams v0.0.0-20230811145659-89c5cff77bcb h1:v3JOchFBzuOEFQgVl0t5JnLg3yx29q2e1IjrEovxAt4= +golang.org/x/exp/typeparams v0.0.0-20230811145659-89c5cff77bcb/go.mod h1:AbB0pIl9nAr9wVwH+Z2ZpaocVmF5I4GyWCDIsVjR0bk= +golang.org/x/image v0.11.0 h1:ds2RoQvBvYTiJkwpSFDwCcDFNX7DqjL2WsUgTNk0Ooo= +golang.org/x/image v0.11.0/go.mod h1:bglhjqbqVuEb9e9+eNR45Jfu7D+T4Qan+NhQk8Ck2P8= golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20230531173138-3c911d8e3eda/go.mod h1:aAjjkJNdrh3PMckS4B10TGS2nag27cbKR1y2BpUxsiY= golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc= golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/net v0.12.0 h1:cfawfvKITfUsFCeJIHJrbSxpeu/E81khclypR0GVT50= -golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= -golang.org/x/oauth2 v0.10.0 h1:zHCpF2Khkwy4mMB4bv0U37YtJdTGW8jI0glAApi0Kh8= -golang.org/x/oauth2 v0.10.0/go.mod h1:kTpgurOux7LqtuxjuyZa4Gj2gdezIt/jQtGnNFfypQI= +golang.org/x/net v0.14.0 h1:BONx9s002vGdD9umnlX1Po8vOZmrgH34qlHcD1MfK14= +golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= +golang.org/x/oauth2 v0.11.0 h1:vPL4xzxBM4niKCW6g9whtaWVXTJf1U5e4aZxxFx/gbU= +golang.org/x/oauth2 v0.11.0/go.mod h1:LdF7O/8bLR/qWK9DrpXmbHLTouvRHK0SgJl0GmDBchk= golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= -golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= -golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/term v0.10.0 h1:3R7pNqamzBraeqj/Tj8qt1aQ2HpmlC+Cx/qL/7hn4/c= -golang.org/x/term v0.10.0/go.mod h1:lpqdcUyK/oCiQxvxVrppt5ggO2KCZ5QblwqPnfZ6d5o= -golang.org/x/text v0.11.0 h1:LAntKIrcmeSKERyiOh0XMV39LXS8IE9UL2yP7+f5ij4= -golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM= +golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.11.0 h1:F9tnn/DA/Im8nCwm+fX+1/eBwi4qFjRT++MhtVC4ZX0= +golang.org/x/term v0.11.0/go.mod h1:zC9APTIj3jG3FdV/Ons+XE1riIZXG4aZ4GTHiPZJPIU= +golang.org/x/text v0.12.0 h1:k+n5B8goJNdU7hSvEtMUz3d1Q6D/XW4COJSJR6fN0mc= +golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/tools v0.11.0 h1:EMCa6U9S2LtZXLAMoWiR/R8dAQFRqbAitmbJ2UKhoi8= -golang.org/x/tools v0.11.0/go.mod h1:anzJrxPjNtfgiYQYirP2CPGzGLxrH2u2QBhn6Bf3qY8= +golang.org/x/tools v0.12.0 h1:YW6HUoUmYBpwSgyaGaZq1fHjrBjX1rlpZ54T6mu2kss= +golang.org/x/tools v0.12.0/go.mod h1:Sc0INKfu04TlqNoRA1hgpFZbhYXHPr4V5DzpSBTPqQM= golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk= golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= -gomodules.xyz/jsonpatch/v2 v2.3.0 h1:8NFhfS6gzxNqjLIYnZxg319wZ5Qjnx4m/CcX+Klzazc= -gomodules.xyz/jsonpatch/v2 v2.3.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= +gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw= +gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= gonum.org/v1/gonum v0.13.0 h1:a0T3bh+7fhRyqeNbiC3qVHYmkiQgit3wnNan/2c0HMM= gonum.org/v1/gonum v0.13.0/go.mod h1:/WPYRckkfWrhWefxyYTfrTtQR0KH4iyHNuzxqXAKyAU= gonum.org/v1/hdf5 v0.0.0-20210714002203-8c5d23bc6946 h1:vJpL69PeUullhJyKtTjHjENEmZU3BkO4e+fod7nKzgM= gonum.org/v1/hdf5 v0.0.0-20210714002203-8c5d23bc6946/go.mod h1:BQUWDHIAygjdt1HnUPQ0eWqLN2n5FwJycrpYUVUOx2I= gonum.org/v1/plot v0.13.0 h1:yb2Z/b8bY5h/xC4uix+ujJ+ixvPUvBmUOtM73CJzpsw= gonum.org/v1/plot v0.13.0/go.mod h1:mV4Bpu4PWTgN2CETURNF8hCMg7EtlZqJYCcmYo/t4Co= -google.golang.org/api v0.133.0 h1:N7Ym5Hl0Dpn0I0o7R1z4UpVA1GCDyS8vbPu1/ObV73A= -google.golang.org/api v0.133.0/go.mod h1:sjRL3UnjTx5UqNQS9EWr9N8p7xbHpy1k0XGRLCf3Spk= +google.golang.org/api v0.137.0 h1:QrKX6uNvzJLr0Fd3vWVqcyrcmFoYi036VUAsZbiF4+s= +google.golang.org/api v0.137.0/go.mod h1:4xyob8CxC+0GChNBvEUAk8VBKNvYOTWM9T3v3UfRxuY= google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/genproto v0.0.0-20230725213213-b022f6e96895 h1:f4HtRHVw5oEuUSMwhzcRW+w4X9++1iU+MZ9cRAHbWxk= -google.golang.org/genproto v0.0.0-20230725213213-b022f6e96895/go.mod h1:0ggbjUrZYpy1q+ANUS30SEoGZ53cdfwtbuG7Ptgy108= -google.golang.org/genproto/googleapis/api v0.0.0-20230725213213-b022f6e96895 h1:9rcwSXpqHEULy96NKetvTJMCLnvnod0LcF8A/ULEBxE= -google.golang.org/genproto/googleapis/api v0.0.0-20230725213213-b022f6e96895/go.mod h1:rsr7RhLuwsDKL7RmgDDCUc6yaGr1iqceVb5Wv6f6YvQ= -google.golang.org/genproto/googleapis/bytestream v0.0.0-20230720185612-659f7aaaa771/go.mod h1:3QoBVwTHkXbY1oRGzlhwhOykfcATQN43LJ6iT8Wy8kE= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230725213213-b022f6e96895 h1:co8AMhI481nhd3WBfW2mq5msyQHNBcGn7G9GCEqz45k= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230725213213-b022f6e96895/go.mod h1:TUfxEVdsvPg18p6AslUXFoLdpED4oBnGwyqk3dV1XzM= -google.golang.org/grpc v1.56.2 h1:fVRFRnXvU+x6C4IlHZewvJOVHoOv1TUuQyoRsYnB4bI= -google.golang.org/grpc v1.56.2/go.mod h1:I9bI3vqKfayGqPUAwGdOSu7kt6oIJLixfffKrpXqQ9s= +google.golang.org/genproto v0.0.0-20230814215434-ca7cfce7776a h1:+pp9kgkk8qyqFeQyXoDBFI4U+YYB15cmRkQ3pi6v/h4= +google.golang.org/genproto v0.0.0-20230814215434-ca7cfce7776a/go.mod h1:yZTlhN0tQnXo3h00fuXNCxJdLdIdnVFVBaRJ5LWBbw4= +google.golang.org/genproto/googleapis/api v0.0.0-20230814215434-ca7cfce7776a h1:ICnet6R+XBkBfhCeApORkkEZbXfPUIHmqOC+5MdMtaQ= +google.golang.org/genproto/googleapis/api v0.0.0-20230814215434-ca7cfce7776a/go.mod h1:KjSP20unUpOx5kyQUFa7k4OJg0qeJ7DEZflGDu2p6Bk= +google.golang.org/genproto/googleapis/bytestream v0.0.0-20230807174057-1744710a1577/go.mod h1:NjCQG/D8JandXxM57PZbAJL1DCNL6EypA0vPPwfsc7c= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230814215434-ca7cfce7776a h1:5rTPHLf5eLPfqGvw3fLpEmUpko2Ky91ft14LxGs5BZc= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230814215434-ca7cfce7776a/go.mod h1:+Bk1OCOj40wS2hwAMA+aCW9ypzm63QTBBHp6lQ3p+9M= +google.golang.org/grpc v1.57.0 h1:kfzNeI/klCGD2YPMUlaGNT3pxvYfga7smW3Vth8Zsiw= +google.golang.org/grpc v1.57.0/go.mod h1:Sd+9RMTACXwmub0zcNY2c4arhtrbBYD1AUHI/dt16Mo= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.3.0/go.mod h1:Dk1tviKTvMCz5tvh7t+fh94dhmQVHuCt2OzJB3CTW9Y= google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= diff --git a/hack/benchmark/internal/core/algorithm/ngt/ngt_test.go b/hack/benchmark/internal/core/algorithm/ngt/ngt_test.go index e146bcb0b43..793622b8b39 100644 --- a/hack/benchmark/internal/core/algorithm/ngt/ngt_test.go +++ b/hack/benchmark/internal/core/algorithm/ngt/ngt_test.go @@ -111,6 +111,7 @@ package ngt // // func Test_core_Search(t *testing.T) { // type args struct { +// ctx context.Context // vec []float32 // size int // epsilon float32 @@ -151,6 +152,7 @@ package ngt // { // name: "test_case_1", // args: args { +// ctx:nil, // vec:nil, // size:0, // epsilon:0, @@ -180,6 +182,7 @@ package ngt // return test { // name: "test_case_2", // args: args { +// ctx:nil, // vec:nil, // size:0, // epsilon:0, @@ -228,7 +231,7 @@ package ngt // NGT: test.fields.NGT, // } // -// got, err := c.Search(test.args.vec, test.args.size, test.args.epsilon, test.args.radius) +// got, err := c.Search(test.args.ctx, test.args.vec, test.args.size, test.args.epsilon, test.args.radius) // if err := checkFunc(test.want, got, err); err != nil { // tt.Errorf("error = %v", err) // } diff --git a/hack/benchmark/internal/operation/insert.go b/hack/benchmark/internal/operation/insert.go index 3428b5ab734..6ecdbefe3fc 100644 --- a/hack/benchmark/internal/operation/insert.go +++ b/hack/benchmark/internal/operation/insert.go @@ -16,7 +16,6 @@ package operation import ( "context" "strconv" - "sync" "sync/atomic" "testing" @@ -25,6 +24,7 @@ import ( "github.com/vdaas/vald/internal/io" "github.com/vdaas/vald/internal/net/grpc/codes" "github.com/vdaas/vald/internal/net/grpc/status" + "github.com/vdaas/vald/internal/sync" ) func (o *operation) Insert(ctx context.Context, b *testing.B, ds assets.Dataset) (insertedNum int) { diff --git a/hack/benchmark/internal/operation/remove.go b/hack/benchmark/internal/operation/remove.go index 7c31ac8a1dc..8789e861ed9 100644 --- a/hack/benchmark/internal/operation/remove.go +++ b/hack/benchmark/internal/operation/remove.go @@ -16,11 +16,11 @@ package operation import ( "context" "strconv" - "sync" "testing" "github.com/vdaas/vald/apis/grpc/v1/payload" "github.com/vdaas/vald/internal/io" + "github.com/vdaas/vald/internal/sync" ) func (o *operation) Remove(ctx context.Context, b *testing.B, maxIdNum int) { diff --git a/hack/benchmark/internal/operation/search.go b/hack/benchmark/internal/operation/search.go index e37b8a286b2..ee2178c1aad 100644 --- a/hack/benchmark/internal/operation/search.go +++ b/hack/benchmark/internal/operation/search.go @@ -16,12 +16,12 @@ package operation import ( "context" "strconv" - "sync" "testing" "github.com/vdaas/vald/apis/grpc/v1/payload" "github.com/vdaas/vald/hack/benchmark/internal/assets" "github.com/vdaas/vald/internal/io" + "github.com/vdaas/vald/internal/sync" ) func (o *operation) Search(ctx context.Context, b *testing.B, ds assets.Dataset) { diff --git a/hack/benchmark/internal/starter/agent/core/ngt/ngt.go b/hack/benchmark/internal/starter/agent/core/ngt/ngt.go index d4fec6c59ae..2dced31d27f 100644 --- a/hack/benchmark/internal/starter/agent/core/ngt/ngt.go +++ b/hack/benchmark/internal/starter/agent/core/ngt/ngt.go @@ -19,7 +19,6 @@ package ngt import ( "context" - "sync" "testing" "time" @@ -27,6 +26,7 @@ import ( "github.com/vdaas/vald/internal/client/v1/client/vald" "github.com/vdaas/vald/internal/info" "github.com/vdaas/vald/internal/runner" + "github.com/vdaas/vald/internal/sync" "github.com/vdaas/vald/pkg/agent/core/ngt/config" "github.com/vdaas/vald/pkg/agent/core/ngt/usecase" ) diff --git a/hack/benchmark/src/singleflight/singleflight_bench_test.go b/hack/benchmark/src/singleflight/singleflight_bench_test.go index 4b183fc6a06..481d39e43de 100644 --- a/hack/benchmark/src/singleflight/singleflight_bench_test.go +++ b/hack/benchmark/src/singleflight/singleflight_bench_test.go @@ -19,13 +19,13 @@ import ( "io/fs" "math" "os" - "sync" "sync/atomic" "testing" "time" "github.com/vdaas/vald/internal/errors" - "github.com/vdaas/vald/internal/singleflight" + "github.com/vdaas/vald/internal/sync" + "github.com/vdaas/vald/internal/sync/singleflight" stdsingleflight "golang.org/x/sync/singleflight" ) @@ -36,7 +36,7 @@ type Result struct { } type helper struct { - initDoFn func() func(ctx context.Context, key string, fn func() (string, error)) + initDoFn func() func(ctx context.Context, key string, fn func(context.Context) (string, error)) sleepDur time.Duration calledCnt int64 totalCnt int64 @@ -67,7 +67,7 @@ var durs = []time.Duration{ func (h *helper) Do(parallel int, b *testing.B) { b.Helper() - fn := func() (string, error) { + fn := func(context.Context) (string, error) { atomic.AddInt64(&h.calledCnt, 1) time.Sleep(h.sleepDur) return "", nil @@ -115,10 +115,10 @@ func Benchmark_group_Do_with_sync_singleflight(b *testing.B) { results := make([]Result, 0, tryCnt) for j := 0; j < tryCnt; j++ { h := &helper{ - initDoFn: func() func(ctx context.Context, key string, fn func() (string, error)) { + initDoFn: func() func(ctx context.Context, key string, fn func(context.Context) (string, error)) { g := new(stdsingleflight.Group) - return func(ctx context.Context, key string, fn func() (string, error)) { - g.Do(key, func() (interface{}, error) { return fn() }) + return func(ctx context.Context, key string, fn func(context.Context) (string, error)) { + g.Do(key, func() (interface{}, error) { return fn(context.Background()) }) } }, sleepDur: dur, @@ -210,9 +210,9 @@ func Benchmark_group_Do_with_vald_internal_singleflight(b *testing.B) { results := make([]Result, 0, tryCnt) for j := 0; j < tryCnt; j++ { h := &helper{ - initDoFn: func() func(ctx context.Context, key string, fn func() (string, error)) { + initDoFn: func() func(ctx context.Context, key string, fn func(context.Context) (string, error)) { g := singleflight.New[string]() - return func(ctx context.Context, key string, fn func() (string, error)) { + return func(ctx context.Context, key string, fn func(context.Context) (string, error)) { g.Do(ctx, key, fn) } }, diff --git a/hack/license/gen/main.go b/hack/license/gen/main.go index 6a3d5ce9000..a7fbcc72df0 100644 --- a/hack/license/gen/main.go +++ b/hack/license/gen/main.go @@ -23,7 +23,6 @@ import ( "io/fs" "os" "path/filepath" - "sync" "text/template" "time" @@ -31,6 +30,7 @@ import ( "github.com/vdaas/vald/internal/file" "github.com/vdaas/vald/internal/log" "github.com/vdaas/vald/internal/strings" + "github.com/vdaas/vald/internal/sync" ) var ( @@ -50,6 +50,38 @@ var ( {{.Escape}} limitations under the License. {{.Escape}} `)) + goStandard = template.Must(template.New("Go License").Parse(`{{.Escape}} +{{.Escape}} Copyright (c) 2009-{{.Year}} The Go Authors. All rights resered. +{{.Escape}} Modified by {{.Maintainer}} +{{.Escape}} +{{.Escape}} Redistribution and use in source and binary forms, with or without +{{.Escape}} modification, are permitted provided that the following conditions are +{{.Escape}} met: +{{.Escape}} +{{.Escape}} * Redistributions of source code must retain the above copyright +{{.Escape}} notice, this list of conditions and the following disclaimer. +{{.Escape}} * Redistributions in binary form must reproduce the above +{{.Escape}} copyright notice, this list of conditions and the following disclaimer +{{.Escape}} in the documentation and/or other materials provided with the +{{.Escape}} distribution. +{{.Escape}} * Neither the name of Google Inc. nor the names of its +{{.Escape}} contributors may be used to endorse or promote products derived from +{{.Escape}} this software without specific prior written permission. +{{.Escape}} +{{.Escape}} THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +{{.Escape}} "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +{{.Escape}} LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +{{.Escape}} A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +{{.Escape}} OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +{{.Escape}} SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +{{.Escape}} LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +{{.Escape}} DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +{{.Escape}} THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +{{.Escape}} (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +{{.Escape}} OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +{{.Escape}} +`)) + slushEscape = "//" sharpEscape = "#" ) @@ -189,8 +221,21 @@ func readAndRewrite(path string) error { log.Fatal(err) } } else { + tmpl := apache switch filepath.Ext(path) { - case ".go", ".proto": + case ".go": + d.Escape = slushEscape + switch fi.Name() { + case "errgroup_test.go", + "singleflight.go", + "semaphore.go", + "semaphore_bench_test.go", + "semaphore_example_test.go", + "semaphore_test.go": + tmpl = goStandard + default: + } + case ".proto": d.Escape = slushEscape } lf := true @@ -236,7 +281,7 @@ func readAndRewrite(path string) error { continue } else if !bf { once.Do(func() { - err = apache.Execute(buf, d) + err = tmpl.Execute(buf, d) if err != nil { // skipcq: RVV-A0003 log.Fatal(err) diff --git a/internal/backoff/backoff.go b/internal/backoff/backoff.go index e91cf606316..1b99fbdba4d 100644 --- a/internal/backoff/backoff.go +++ b/internal/backoff/backoff.go @@ -21,7 +21,6 @@ import ( "context" "math" "strconv" - "sync" "time" "github.com/vdaas/vald/internal/errors" @@ -30,6 +29,7 @@ import ( "github.com/vdaas/vald/internal/observability/trace" "github.com/vdaas/vald/internal/rand" "github.com/vdaas/vald/internal/strings" + "github.com/vdaas/vald/internal/sync" ) // NOTE: This variable is for observability package. diff --git a/internal/cache/gache/option_test.go b/internal/cache/gache/option_test.go index 3baba7374c0..bc00aac1e9e 100644 --- a/internal/cache/gache/option_test.go +++ b/internal/cache/gache/option_test.go @@ -393,16 +393,16 @@ func TestWithExpireCheckDuration(t *testing.T) { // // func Test_defaultOptions(t *testing.T) { // type want struct { -// want []Option +// want []Option[V] // } // type test struct { // name string // want want -// checkFunc func(want, []Option) error +// checkFunc func(want, []Option[V]) error // beforeFunc func(*testing.T) // afterFunc func(*testing.T) // } -// defaultCheckFunc := func(w want, got []Option) error { +// defaultCheckFunc := func(w want, got []Option[V]) error { // if !reflect.DeepEqual(got, w.want) { // return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", got, w.want) // } diff --git a/internal/cache/option_test.go b/internal/cache/option_test.go index f41f6ccd159..a2eee8e05d7 100644 --- a/internal/cache/option_test.go +++ b/internal/cache/option_test.go @@ -342,3 +342,79 @@ func TestWithExpireCheckDuration(t *testing.T) { } // NOT IMPLEMENTED BELOW +// +// func Test_defaultOptions(t *testing.T) { +// type want struct { +// want []Option[V] +// } +// type test struct { +// name string +// want want +// checkFunc func(want, []Option[V]) error +// beforeFunc func(*testing.T) +// afterFunc func(*testing.T) +// } +// defaultCheckFunc := func(w want, got []Option[V]) error { +// if !reflect.DeepEqual(got, w.want) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", got, w.want) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T,) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T,) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T,) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T,) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// +// got := defaultOptions() +// if err := checkFunc(test.want, got); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } diff --git a/internal/circuitbreaker/manager.go b/internal/circuitbreaker/manager.go index 0cae88a809c..64338b2a4f6 100644 --- a/internal/circuitbreaker/manager.go +++ b/internal/circuitbreaker/manager.go @@ -16,11 +16,10 @@ package circuitbreaker import ( "context" "reflect" - "sync" "github.com/vdaas/vald/internal/errors" "github.com/vdaas/vald/internal/log" - valdsync "github.com/vdaas/vald/internal/sync" + "github.com/vdaas/vald/internal/sync" ) // NOTE: This variable is for observability package. @@ -37,7 +36,7 @@ type CircuitBreaker interface { } type breakerManager struct { - m valdsync.Map[string, *breaker] + m sync.Map[string, *breaker] opts []BreakerOption } diff --git a/internal/circuitbreaker/manager_test.go b/internal/circuitbreaker/manager_test.go index c082552ec56..7c007204b62 100644 --- a/internal/circuitbreaker/manager_test.go +++ b/internal/circuitbreaker/manager_test.go @@ -112,6 +112,7 @@ package circuitbreaker // fn func(ctx context.Context) (interface{}, error) // } // type fields struct { +// m sync.Map[string, *breaker] // opts []BreakerOption // } // type want struct { @@ -147,6 +148,7 @@ package circuitbreaker // fn:nil, // }, // fields: fields { +// m:nil, // opts:nil, // }, // want: want{}, @@ -171,6 +173,7 @@ package circuitbreaker // fn:nil, // }, // fields: fields { +// m:nil, // opts:nil, // }, // want: want{}, @@ -202,6 +205,7 @@ package circuitbreaker // checkFunc = defaultCheckFunc // } // bm := &breakerManager{ +// m: test.fields.m, // opts: test.fields.opts, // } // diff --git a/internal/client/v1/client/discoverer/discover.go b/internal/client/v1/client/discoverer/discover.go index f4f2e6e8e12..d859212181b 100644 --- a/internal/client/v1/client/discoverer/discover.go +++ b/internal/client/v1/client/discoverer/discover.go @@ -25,13 +25,13 @@ import ( "github.com/vdaas/vald/apis/grpc/v1/discoverer" "github.com/vdaas/vald/apis/grpc/v1/payload" - "github.com/vdaas/vald/internal/errgroup" "github.com/vdaas/vald/internal/errors" "github.com/vdaas/vald/internal/log" "github.com/vdaas/vald/internal/net" "github.com/vdaas/vald/internal/net/grpc" "github.com/vdaas/vald/internal/safety" - valdsync "github.com/vdaas/vald/internal/sync" + "github.com/vdaas/vald/internal/sync" + "github.com/vdaas/vald/internal/sync/errgroup" ) type Client interface { @@ -344,7 +344,7 @@ func (c *client) disconnectOldAddrs(ctx context.Context, oldAddrs, connectedAddr if !c.autoconn { return nil } - var cur valdsync.Map[string, any] // TODO: Does this have to be a sync.Map not a map? + var cur sync.Map[string, any] // TODO: Does this have to be a sync.Map not a map? for _, addr := range connectedAddrs { cur.Store(addr, struct{}{}) } diff --git a/internal/client/v1/client/discoverer/option.go b/internal/client/v1/client/discoverer/option.go index 794662ef373..f9d890a7452 100644 --- a/internal/client/v1/client/discoverer/option.go +++ b/internal/client/v1/client/discoverer/option.go @@ -21,8 +21,8 @@ import ( "context" "time" - "github.com/vdaas/vald/internal/errgroup" "github.com/vdaas/vald/internal/net/grpc" + "github.com/vdaas/vald/internal/sync/errgroup" "github.com/vdaas/vald/internal/timeutil" ) diff --git a/internal/client/v1/client/filter/egress/client.go b/internal/client/v1/client/filter/egress/client.go index 77ea9793cfb..5e4105ef8a8 100644 --- a/internal/client/v1/client/filter/egress/client.go +++ b/internal/client/v1/client/filter/egress/client.go @@ -27,7 +27,7 @@ import ( "github.com/vdaas/vald/internal/net/grpc" "github.com/vdaas/vald/internal/observability/trace" "github.com/vdaas/vald/internal/strings" - valdsync "github.com/vdaas/vald/internal/sync" + "github.com/vdaas/vald/internal/sync" ) type Client interface { @@ -40,7 +40,7 @@ type Client interface { type client struct { addrs []string - cl valdsync.Map[string, any] + cl sync.Map[string, any] c grpc.Client } diff --git a/internal/client/v1/client/filter/egress/client_test.go b/internal/client/v1/client/filter/egress/client_test.go index db7850cc219..61ef1db0720 100644 --- a/internal/client/v1/client/filter/egress/client_test.go +++ b/internal/client/v1/client/filter/egress/client_test.go @@ -115,6 +115,7 @@ package egress // } // type fields struct { // addrs []string +// cl sync.Map[string, any] // c grpc.Client // } // type want struct { @@ -149,6 +150,7 @@ package egress // }, // fields: fields { // addrs:nil, +// cl:nil, // c:nil, // }, // want: want{}, @@ -172,6 +174,7 @@ package egress // }, // fields: fields { // addrs:nil, +// cl:nil, // c:nil, // }, // want: want{}, @@ -204,6 +207,7 @@ package egress // } // c := &client{ // addrs: test.fields.addrs, +// cl: test.fields.cl, // c: test.fields.c, // } // @@ -222,6 +226,7 @@ package egress // } // type fields struct { // addrs []string +// cl sync.Map[string, any] // c grpc.Client // } // type want struct { @@ -252,6 +257,7 @@ package egress // }, // fields: fields { // addrs:nil, +// cl:nil, // c:nil, // }, // want: want{}, @@ -275,6 +281,7 @@ package egress // }, // fields: fields { // addrs:nil, +// cl:nil, // c:nil, // }, // want: want{}, @@ -307,6 +314,7 @@ package egress // } // c := &client{ // addrs: test.fields.addrs, +// cl: test.fields.cl, // c: test.fields.c, // } // @@ -322,6 +330,7 @@ package egress // func Test_client_GRPCClient(t *testing.T) { // type fields struct { // addrs []string +// cl sync.Map[string, any] // c grpc.Client // } // type want struct { @@ -348,6 +357,7 @@ package egress // name: "test_case_1", // fields: fields { // addrs:nil, +// cl:nil, // c:nil, // }, // want: want{}, @@ -368,6 +378,7 @@ package egress // name: "test_case_2", // fields: fields { // addrs:nil, +// cl:nil, // c:nil, // }, // want: want{}, @@ -400,6 +411,7 @@ package egress // } // c := &client{ // addrs: test.fields.addrs, +// cl: test.fields.cl, // c: test.fields.c, // } // @@ -419,6 +431,7 @@ package egress // } // type fields struct { // addrs []string +// cl sync.Map[string, any] // c grpc.Client // } // type want struct { @@ -454,6 +467,7 @@ package egress // }, // fields: fields { // addrs:nil, +// cl:nil, // c:nil, // }, // want: want{}, @@ -478,6 +492,7 @@ package egress // }, // fields: fields { // addrs:nil, +// cl:nil, // c:nil, // }, // want: want{}, @@ -510,6 +525,7 @@ package egress // } // c := &client{ // addrs: test.fields.addrs, +// cl: test.fields.cl, // c: test.fields.c, // } // @@ -530,6 +546,7 @@ package egress // } // type fields struct { // addrs []string +// cl sync.Map[string, any] // c grpc.Client // } // type want struct { @@ -566,6 +583,7 @@ package egress // }, // fields: fields { // addrs:nil, +// cl:nil, // c:nil, // }, // want: want{}, @@ -591,6 +609,7 @@ package egress // }, // fields: fields { // addrs:nil, +// cl:nil, // c:nil, // }, // want: want{}, @@ -623,6 +642,7 @@ package egress // } // c := &client{ // addrs: test.fields.addrs, +// cl: test.fields.cl, // c: test.fields.c, // } // @@ -869,6 +889,7 @@ package egress // } // type fields struct { // addrs []string +// cl sync.Map[string, any] // c grpc.Client // } // type want struct { @@ -905,6 +926,7 @@ package egress // }, // fields: fields { // addrs:nil, +// cl:nil, // c:nil, // }, // want: want{}, @@ -930,6 +952,7 @@ package egress // }, // fields: fields { // addrs:nil, +// cl:nil, // c:nil, // }, // want: want{}, @@ -962,6 +985,7 @@ package egress // } // c := &client{ // addrs: test.fields.addrs, +// cl: test.fields.cl, // c: test.fields.c, // } // diff --git a/internal/client/v1/client/filter/ingress/client.go b/internal/client/v1/client/filter/ingress/client.go index 7222e0763d3..645326b1af6 100644 --- a/internal/client/v1/client/filter/ingress/client.go +++ b/internal/client/v1/client/filter/ingress/client.go @@ -27,7 +27,7 @@ import ( "github.com/vdaas/vald/internal/net/grpc" "github.com/vdaas/vald/internal/observability/trace" "github.com/vdaas/vald/internal/strings" - valdsync "github.com/vdaas/vald/internal/sync" + "github.com/vdaas/vald/internal/sync" ) type Client interface { @@ -40,7 +40,7 @@ type Client interface { type client struct { addrs []string - cl valdsync.Map[string, any] + cl sync.Map[string, any] c grpc.Client } diff --git a/internal/client/v1/client/filter/ingress/client_test.go b/internal/client/v1/client/filter/ingress/client_test.go index 58fafcc5e9c..b1a95177708 100644 --- a/internal/client/v1/client/filter/ingress/client_test.go +++ b/internal/client/v1/client/filter/ingress/client_test.go @@ -115,6 +115,7 @@ package ingress // } // type fields struct { // addrs []string +// cl sync.Map[string, any] // c grpc.Client // } // type want struct { @@ -149,6 +150,7 @@ package ingress // }, // fields: fields { // addrs:nil, +// cl:nil, // c:nil, // }, // want: want{}, @@ -172,6 +174,7 @@ package ingress // }, // fields: fields { // addrs:nil, +// cl:nil, // c:nil, // }, // want: want{}, @@ -204,6 +207,7 @@ package ingress // } // c := &client{ // addrs: test.fields.addrs, +// cl: test.fields.cl, // c: test.fields.c, // } // @@ -222,6 +226,7 @@ package ingress // } // type fields struct { // addrs []string +// cl sync.Map[string, any] // c grpc.Client // } // type want struct { @@ -252,6 +257,7 @@ package ingress // }, // fields: fields { // addrs:nil, +// cl:nil, // c:nil, // }, // want: want{}, @@ -275,6 +281,7 @@ package ingress // }, // fields: fields { // addrs:nil, +// cl:nil, // c:nil, // }, // want: want{}, @@ -307,6 +314,7 @@ package ingress // } // c := &client{ // addrs: test.fields.addrs, +// cl: test.fields.cl, // c: test.fields.c, // } // @@ -322,6 +330,7 @@ package ingress // func Test_client_GRPCClient(t *testing.T) { // type fields struct { // addrs []string +// cl sync.Map[string, any] // c grpc.Client // } // type want struct { @@ -348,6 +357,7 @@ package ingress // name: "test_case_1", // fields: fields { // addrs:nil, +// cl:nil, // c:nil, // }, // want: want{}, @@ -368,6 +378,7 @@ package ingress // name: "test_case_2", // fields: fields { // addrs:nil, +// cl:nil, // c:nil, // }, // want: want{}, @@ -400,6 +411,7 @@ package ingress // } // c := &client{ // addrs: test.fields.addrs, +// cl: test.fields.cl, // c: test.fields.c, // } // @@ -419,6 +431,7 @@ package ingress // } // type fields struct { // addrs []string +// cl sync.Map[string, any] // c grpc.Client // } // type want struct { @@ -454,6 +467,7 @@ package ingress // }, // fields: fields { // addrs:nil, +// cl:nil, // c:nil, // }, // want: want{}, @@ -478,6 +492,7 @@ package ingress // }, // fields: fields { // addrs:nil, +// cl:nil, // c:nil, // }, // want: want{}, @@ -510,6 +525,7 @@ package ingress // } // c := &client{ // addrs: test.fields.addrs, +// cl: test.fields.cl, // c: test.fields.c, // } // @@ -530,6 +546,7 @@ package ingress // } // type fields struct { // addrs []string +// cl sync.Map[string, any] // c grpc.Client // } // type want struct { @@ -566,6 +583,7 @@ package ingress // }, // fields: fields { // addrs:nil, +// cl:nil, // c:nil, // }, // want: want{}, @@ -591,6 +609,7 @@ package ingress // }, // fields: fields { // addrs:nil, +// cl:nil, // c:nil, // }, // want: want{}, @@ -623,6 +642,7 @@ package ingress // } // c := &client{ // addrs: test.fields.addrs, +// cl: test.fields.cl, // c: test.fields.c, // } // @@ -643,6 +663,7 @@ package ingress // } // type fields struct { // addrs []string +// cl sync.Map[string, any] // c grpc.Client // } // type want struct { @@ -679,6 +700,7 @@ package ingress // }, // fields: fields { // addrs:nil, +// cl:nil, // c:nil, // }, // want: want{}, @@ -704,6 +726,7 @@ package ingress // }, // fields: fields { // addrs:nil, +// cl:nil, // c:nil, // }, // want: want{}, @@ -736,6 +759,7 @@ package ingress // } // c := &client{ // addrs: test.fields.addrs, +// cl: test.fields.cl, // c: test.fields.c, // } // diff --git a/internal/client/v1/client/vald/vald_test.go b/internal/client/v1/client/vald/vald_test.go index 3c108a0149f..80bdb91c66c 100644 --- a/internal/client/v1/client/vald/vald_test.go +++ b/internal/client/v1/client/vald/vald_test.go @@ -3522,6 +3522,119 @@ package vald // } // } // +// func Test_client_StreamListObject(t *testing.T) { +// type args struct { +// ctx context.Context +// in *payload.Object_List_Request +// opts []grpc.CallOption +// } +// type fields struct { +// addrs []string +// c grpc.Client +// } +// type want struct { +// wantRes vald.Object_StreamListObjectClient +// err error +// } +// type test struct { +// name string +// args args +// fields fields +// want want +// checkFunc func(want, vald.Object_StreamListObjectClient, error) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, gotRes vald.Object_StreamListObjectClient, err error) error { +// if !errors.Is(err, w.err) { +// return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err) +// } +// if !reflect.DeepEqual(gotRes, w.wantRes) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", gotRes, w.wantRes) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// ctx:nil, +// in:nil, +// opts:nil, +// }, +// fields: fields { +// addrs:nil, +// c:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// ctx:nil, +// in:nil, +// opts:nil, +// }, +// fields: fields { +// addrs:nil, +// c:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// c := &client{ +// addrs: test.fields.addrs, +// c: test.fields.c, +// } +// +// gotRes, err := c.StreamListObject(test.args.ctx, test.args.in, test.args.opts...) +// if err := checkFunc(test.want, gotRes, err); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } +// // func Test_singleClient_Start(t *testing.T) { // type args struct { // in0 context.Context @@ -6728,3 +6841,112 @@ package vald // }) // } // } +// +// func Test_singleClient_StreamListObject(t *testing.T) { +// type args struct { +// ctx context.Context +// in *payload.Object_List_Request +// opts []grpc.CallOption +// } +// type fields struct { +// vc vald.Client +// } +// type want struct { +// wantRes vald.Object_StreamListObjectClient +// err error +// } +// type test struct { +// name string +// args args +// fields fields +// want want +// checkFunc func(want, vald.Object_StreamListObjectClient, error) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, gotRes vald.Object_StreamListObjectClient, err error) error { +// if !errors.Is(err, w.err) { +// return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err) +// } +// if !reflect.DeepEqual(gotRes, w.wantRes) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", gotRes, w.wantRes) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// ctx:nil, +// in:nil, +// opts:nil, +// }, +// fields: fields { +// vc:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// ctx:nil, +// in:nil, +// opts:nil, +// }, +// fields: fields { +// vc:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// c := &singleClient{ +// vc: test.fields.vc, +// } +// +// gotRes, err := c.StreamListObject(test.args.ctx, test.args.in, test.args.opts...) +// if err := checkFunc(test.want, gotRes, err); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } diff --git a/internal/core/algorithm/ngt/ngt.go b/internal/core/algorithm/ngt/ngt.go index 28a40b2bfc8..80156eb2083 100644 --- a/internal/core/algorithm/ngt/ngt.go +++ b/internal/core/algorithm/ngt/ngt.go @@ -27,13 +27,13 @@ import "C" import ( "context" "reflect" - "sync" "unsafe" "github.com/vdaas/vald/internal/core/algorithm" "github.com/vdaas/vald/internal/errors" "github.com/vdaas/vald/internal/file" "github.com/vdaas/vald/internal/log" + "github.com/vdaas/vald/internal/sync" ) type ( diff --git a/internal/core/algorithm/ngt/ngt_test.go b/internal/core/algorithm/ngt/ngt_test.go index 1fe3609d2fe..0e40fa3ecab 100644 --- a/internal/core/algorithm/ngt/ngt_test.go +++ b/internal/core/algorithm/ngt/ngt_test.go @@ -25,7 +25,6 @@ import ( "path/filepath" "reflect" "strings" - "sync" "testing" "github.com/vdaas/vald/internal/core/algorithm" @@ -33,6 +32,7 @@ import ( "github.com/vdaas/vald/internal/file" "github.com/vdaas/vald/internal/log" "github.com/vdaas/vald/internal/log/logger" + "github.com/vdaas/vald/internal/sync" "github.com/vdaas/vald/internal/test/comparator" "github.com/vdaas/vald/internal/test/goleak" ) diff --git a/internal/db/storage/blob/s3/option.go b/internal/db/storage/blob/s3/option.go index 39472f8196c..92c94054dec 100644 --- a/internal/db/storage/blob/s3/option.go +++ b/internal/db/storage/blob/s3/option.go @@ -22,7 +22,7 @@ import ( "github.com/vdaas/vald/internal/backoff" "github.com/vdaas/vald/internal/db/storage/blob/s3/reader" "github.com/vdaas/vald/internal/db/storage/blob/s3/writer" - "github.com/vdaas/vald/internal/errgroup" + "github.com/vdaas/vald/internal/sync/errgroup" "github.com/vdaas/vald/internal/unit" ) diff --git a/internal/db/storage/blob/s3/option_test.go b/internal/db/storage/blob/s3/option_test.go index 2cdf592f951..beefdf10059 100644 --- a/internal/db/storage/blob/s3/option_test.go +++ b/internal/db/storage/blob/s3/option_test.go @@ -24,8 +24,8 @@ import ( "github.com/vdaas/vald/internal/backoff" "github.com/vdaas/vald/internal/db/storage/blob/s3/reader" "github.com/vdaas/vald/internal/db/storage/blob/s3/writer" - "github.com/vdaas/vald/internal/errgroup" "github.com/vdaas/vald/internal/errors" + "github.com/vdaas/vald/internal/sync/errgroup" "github.com/vdaas/vald/internal/test/comparator" "github.com/vdaas/vald/internal/test/goleak" ) diff --git a/internal/db/storage/blob/s3/reader/option.go b/internal/db/storage/blob/s3/reader/option.go index ed3cddc4a3a..96d5b210af6 100644 --- a/internal/db/storage/blob/s3/reader/option.go +++ b/internal/db/storage/blob/s3/reader/option.go @@ -20,7 +20,7 @@ import ( "github.com/vdaas/vald/internal/backoff" "github.com/vdaas/vald/internal/db/storage/blob/s3/reader/io" "github.com/vdaas/vald/internal/db/storage/blob/s3/sdk/s3/s3iface" - "github.com/vdaas/vald/internal/errgroup" + "github.com/vdaas/vald/internal/sync/errgroup" ) // Option represents the functional option for reader. diff --git a/internal/db/storage/blob/s3/reader/option_test.go b/internal/db/storage/blob/s3/reader/option_test.go index d46f0cd4c6b..544a2f7a03e 100644 --- a/internal/db/storage/blob/s3/reader/option_test.go +++ b/internal/db/storage/blob/s3/reader/option_test.go @@ -23,8 +23,8 @@ import ( "github.com/aws/aws-sdk-go/service/s3" "github.com/vdaas/vald/internal/backoff" "github.com/vdaas/vald/internal/db/storage/blob/s3/sdk/s3/s3iface" - "github.com/vdaas/vald/internal/errgroup" "github.com/vdaas/vald/internal/errors" + "github.com/vdaas/vald/internal/sync/errgroup" "github.com/vdaas/vald/internal/test/comparator" "github.com/vdaas/vald/internal/test/goleak" ) diff --git a/internal/db/storage/blob/s3/reader/reader.go b/internal/db/storage/blob/s3/reader/reader.go index fbff2f9cb13..bc8a3cc483c 100644 --- a/internal/db/storage/blob/s3/reader/reader.go +++ b/internal/db/storage/blob/s3/reader/reader.go @@ -20,7 +20,6 @@ import ( "bytes" "context" "strconv" - "sync" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/awserr" @@ -28,11 +27,12 @@ import ( ctxio "github.com/vdaas/vald/internal/db/storage/blob/s3/reader/io" "github.com/vdaas/vald/internal/db/storage/blob/s3/sdk/s3" "github.com/vdaas/vald/internal/db/storage/blob/s3/sdk/s3/s3iface" - "github.com/vdaas/vald/internal/errgroup" "github.com/vdaas/vald/internal/errors" "github.com/vdaas/vald/internal/io" "github.com/vdaas/vald/internal/log" "github.com/vdaas/vald/internal/safety" + "github.com/vdaas/vald/internal/sync" + "github.com/vdaas/vald/internal/sync/errgroup" ) type reader struct { diff --git a/internal/db/storage/blob/s3/reader/reader_test.go b/internal/db/storage/blob/s3/reader/reader_test.go index 43e7a0d9521..0fa9d48a88c 100644 --- a/internal/db/storage/blob/s3/reader/reader_test.go +++ b/internal/db/storage/blob/s3/reader/reader_test.go @@ -21,7 +21,6 @@ import ( "context" "os" "reflect" - "sync" "sync/atomic" "testing" @@ -32,11 +31,12 @@ import ( ctxio "github.com/vdaas/vald/internal/db/storage/blob/s3/reader/io" "github.com/vdaas/vald/internal/db/storage/blob/s3/sdk/s3" "github.com/vdaas/vald/internal/db/storage/blob/s3/sdk/s3/s3iface" - "github.com/vdaas/vald/internal/errgroup" "github.com/vdaas/vald/internal/errors" "github.com/vdaas/vald/internal/io" "github.com/vdaas/vald/internal/log" "github.com/vdaas/vald/internal/log/logger" + "github.com/vdaas/vald/internal/sync" + "github.com/vdaas/vald/internal/sync/errgroup" "github.com/vdaas/vald/internal/test/goleak" ) diff --git a/internal/db/storage/blob/s3/s3.go b/internal/db/storage/blob/s3/s3.go index f48422a0082..02935d557fc 100644 --- a/internal/db/storage/blob/s3/s3.go +++ b/internal/db/storage/blob/s3/s3.go @@ -26,9 +26,9 @@ import ( "github.com/vdaas/vald/internal/db/storage/blob" "github.com/vdaas/vald/internal/db/storage/blob/s3/reader" "github.com/vdaas/vald/internal/db/storage/blob/s3/writer" - "github.com/vdaas/vald/internal/errgroup" "github.com/vdaas/vald/internal/errors" "github.com/vdaas/vald/internal/io" + "github.com/vdaas/vald/internal/sync/errgroup" ) type client struct { diff --git a/internal/db/storage/blob/s3/s3_test.go b/internal/db/storage/blob/s3/s3_test.go index 3e37575d462..d486f4ca88b 100644 --- a/internal/db/storage/blob/s3/s3_test.go +++ b/internal/db/storage/blob/s3/s3_test.go @@ -30,9 +30,9 @@ import ( "github.com/vdaas/vald/internal/db/storage/blob/s3/reader" "github.com/vdaas/vald/internal/db/storage/blob/s3/sdk/s3/s3manager" "github.com/vdaas/vald/internal/db/storage/blob/s3/writer" - "github.com/vdaas/vald/internal/errgroup" "github.com/vdaas/vald/internal/errors" "github.com/vdaas/vald/internal/io" + "github.com/vdaas/vald/internal/sync/errgroup" "github.com/vdaas/vald/internal/test/comparator" "github.com/vdaas/vald/internal/test/goleak" ) diff --git a/internal/db/storage/blob/s3/writer/option.go b/internal/db/storage/blob/s3/writer/option.go index a1e7a685c56..73f8ea46b60 100644 --- a/internal/db/storage/blob/s3/writer/option.go +++ b/internal/db/storage/blob/s3/writer/option.go @@ -19,8 +19,8 @@ package writer import ( "github.com/aws/aws-sdk-go/service/s3" "github.com/aws/aws-sdk-go/service/s3/s3manager" - "github.com/vdaas/vald/internal/errgroup" "github.com/vdaas/vald/internal/errors" + "github.com/vdaas/vald/internal/sync/errgroup" ) // Option represents the functional option for writer. diff --git a/internal/db/storage/blob/s3/writer/option_test.go b/internal/db/storage/blob/s3/writer/option_test.go index 8db3c6bf1c2..a4b79c47023 100644 --- a/internal/db/storage/blob/s3/writer/option_test.go +++ b/internal/db/storage/blob/s3/writer/option_test.go @@ -22,8 +22,8 @@ import ( "github.com/aws/aws-sdk-go/service/s3" "github.com/aws/aws-sdk-go/service/s3/s3manager" - "github.com/vdaas/vald/internal/errgroup" "github.com/vdaas/vald/internal/errors" + "github.com/vdaas/vald/internal/sync/errgroup" "github.com/vdaas/vald/internal/test/goleak" ) diff --git a/internal/db/storage/blob/s3/writer/writer.go b/internal/db/storage/blob/s3/writer/writer.go index c059b49c965..6b0f8cd0e93 100644 --- a/internal/db/storage/blob/s3/writer/writer.go +++ b/internal/db/storage/blob/s3/writer/writer.go @@ -19,16 +19,16 @@ package writer import ( "context" "reflect" - "sync" "github.com/aws/aws-sdk-go/aws" "github.com/vdaas/vald/internal/db/storage/blob/s3/sdk/s3/s3iface" "github.com/vdaas/vald/internal/db/storage/blob/s3/sdk/s3/s3manager" - "github.com/vdaas/vald/internal/errgroup" "github.com/vdaas/vald/internal/errors" "github.com/vdaas/vald/internal/io" "github.com/vdaas/vald/internal/log" "github.com/vdaas/vald/internal/safety" + "github.com/vdaas/vald/internal/sync" + "github.com/vdaas/vald/internal/sync/errgroup" ) type writer struct { diff --git a/internal/db/storage/blob/s3/writer/writer_test.go b/internal/db/storage/blob/s3/writer/writer_test.go index 5d6e6d624e7..1c7545a62ad 100644 --- a/internal/db/storage/blob/s3/writer/writer_test.go +++ b/internal/db/storage/blob/s3/writer/writer_test.go @@ -20,18 +20,18 @@ import ( "context" "os" "reflect" - "sync" "testing" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/s3" "github.com/vdaas/vald/internal/db/storage/blob/s3/sdk/s3/s3iface" "github.com/vdaas/vald/internal/db/storage/blob/s3/sdk/s3/s3manager" - "github.com/vdaas/vald/internal/errgroup" "github.com/vdaas/vald/internal/errors" "github.com/vdaas/vald/internal/io" "github.com/vdaas/vald/internal/log" "github.com/vdaas/vald/internal/log/logger" + "github.com/vdaas/vald/internal/sync" + "github.com/vdaas/vald/internal/sync/errgroup" "github.com/vdaas/vald/internal/test/goleak" ) diff --git a/internal/errgroup/group.go b/internal/errgroup/group.go deleted file mode 100644 index f1071921ef8..00000000000 --- a/internal/errgroup/group.go +++ /dev/null @@ -1,202 +0,0 @@ -// -// Copyright (C) 2019-2023 vdaas.org vald team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// - -// Package errgroup provides server global wait group for graceful kill all goroutine -package errgroup - -import ( - "context" - "runtime" - "sync" - "sync/atomic" - - "github.com/vdaas/vald/internal/errors" -) - -type Group interface { - Go(func() error) - Limitation(int) - Wait() error -} - -type group struct { - egctx context.Context - cancel context.CancelFunc - - wg sync.WaitGroup - - limitation chan struct{} - enableLimitation atomic.Bool - cancelOnce sync.Once - mu sync.RWMutex - emap map[string]struct{} - errs []error - err error -} - -var ( - instance Group - once sync.Once -) - -func New(ctx context.Context) (Group, context.Context) { - egctx, cancel := context.WithCancel(ctx) - g := &group{ - egctx: egctx, - emap: make(map[string]struct{}), - cancel: cancel, - } - g.enableLimitation.Store(false) - return g, egctx -} - -func Init(ctx context.Context) (egctx context.Context) { - egctx = ctx - once.Do(func() { - instance, egctx = New(ctx) - }) - return -} - -func Get() Group { - if instance == nil { - Init(context.Background()) - } - return instance -} - -func Go(f func() error) { - instance.Go(f) -} - -func (g *group) Limitation(limit int) { - if limit > 0 { - ch := make(chan struct{}, limit) - g.closeLimitation() - g.limitation = ch - g.enableLimitation.Store(true) - } else { - g.enableLimitation.Store(false) - } -} - -func (g *group) closeLimitation() { - if g.limitation != nil { - for { - select { - case _, live := <-g.limitation: - if !live { - return - } - default: - close(g.limitation) - return - } - } - } -} - -func (g *group) Go(f func() error) { - if f != nil { - g.wg.Add(1) - go func() { - defer g.wg.Done() - var err error - if g.enableLimitation.Load() { - select { - case <-g.egctx.Done(): - err = g.egctx.Err() - if !errors.Is(err, context.Canceled) && - !errors.Is(err, context.DeadlineExceeded) { - g.mu.RLock() - _, ok := g.emap[err.Error()] - g.mu.RUnlock() - if !ok { - g.mu.Lock() - g.errs = append(g.errs, err) - g.emap[err.Error()] = struct{}{} - g.mu.Unlock() - } - } - g.doCancel() - return - case g.limitation <- struct{}{}: - } - err = f() - select { - case <-g.limitation: - case <-g.egctx.Done(): - if err != nil { - err = errors.Join(err, g.egctx.Err()) - } - } - } else { - err = f() - } - if err != nil { - if !errors.Is(err, context.Canceled) && - !errors.Is(err, context.DeadlineExceeded) { - runtime.Gosched() - g.mu.RLock() - _, ok := g.emap[err.Error()] - g.mu.RUnlock() - if !ok { - g.mu.Lock() - g.errs = append(g.errs, err) - g.emap[err.Error()] = struct{}{} - g.mu.Unlock() - } - } - g.doCancel() - return - } - }() - } -} - -func (g *group) doCancel() { - g.cancelOnce.Do(func() { - if g.cancel != nil { - g.cancel() - } - }) -} - -func Wait() error { - return instance.Wait() -} - -func (g *group) Wait() error { - g.wg.Wait() - g.doCancel() - g.closeLimitation() - g.enableLimitation.Store(false) - g.mu.RLock() - switch len(g.errs) { - case 0: - g.mu.RUnlock() - return nil - case 1: - g.err = g.errs[0] - default: - g.err = g.errs[0] - for _, err := range g.errs[1:] { - g.err = errors.Join(g.err, err) - } - } - g.mu.RUnlock() - return g.err -} diff --git a/internal/errors/errors.go b/internal/errors/errors.go index b9585230c92..d4647986f8e 100644 --- a/internal/errors/errors.go +++ b/internal/errors/errors.go @@ -23,7 +23,8 @@ import ( "reflect" "runtime" "strings" - "sync" + + "github.com/vdaas/vald/internal/sync" ) var ( diff --git a/internal/file/file.go b/internal/file/file.go index ec9e57c08d7..0d766878fde 100644 --- a/internal/file/file.go +++ b/internal/file/file.go @@ -26,13 +26,13 @@ import ( "time" "github.com/kpango/fastime" - "github.com/vdaas/vald/internal/errgroup" "github.com/vdaas/vald/internal/errors" "github.com/vdaas/vald/internal/io" "github.com/vdaas/vald/internal/log" "github.com/vdaas/vald/internal/safety" "github.com/vdaas/vald/internal/slices" "github.com/vdaas/vald/internal/strings" + "github.com/vdaas/vald/internal/sync/errgroup" ) // Open opens the file with the given path, flag and permission. diff --git a/internal/file/watch/option.go b/internal/file/watch/option.go index 3cf89bf1095..43b509982ad 100644 --- a/internal/file/watch/option.go +++ b/internal/file/watch/option.go @@ -19,7 +19,7 @@ package watch import ( "context" - "github.com/vdaas/vald/internal/errgroup" + "github.com/vdaas/vald/internal/sync/errgroup" ) type Option func(w *watch) error diff --git a/internal/file/watch/option_test.go b/internal/file/watch/option_test.go index 997737d95d4..8b5fb8ae8e9 100644 --- a/internal/file/watch/option_test.go +++ b/internal/file/watch/option_test.go @@ -21,8 +21,8 @@ import ( "reflect" "testing" - "github.com/vdaas/vald/internal/errgroup" "github.com/vdaas/vald/internal/errors" + "github.com/vdaas/vald/internal/sync/errgroup" "github.com/vdaas/vald/internal/test/goleak" ) diff --git a/internal/file/watch/watch.go b/internal/file/watch/watch.go index fab40d2bc33..ca2723b669b 100644 --- a/internal/file/watch/watch.go +++ b/internal/file/watch/watch.go @@ -19,13 +19,13 @@ package watch import ( "context" "reflect" - "sync" "github.com/fsnotify/fsnotify" - "github.com/vdaas/vald/internal/errgroup" "github.com/vdaas/vald/internal/errors" "github.com/vdaas/vald/internal/log" "github.com/vdaas/vald/internal/safety" + "github.com/vdaas/vald/internal/sync" + "github.com/vdaas/vald/internal/sync/errgroup" ) // Watcher is an interface that represents a file monitor. diff --git a/internal/file/watch/watch_test.go b/internal/file/watch/watch_test.go index 3f3a34369c4..e7bb95d21d3 100644 --- a/internal/file/watch/watch_test.go +++ b/internal/file/watch/watch_test.go @@ -26,12 +26,12 @@ import ( "testing" "github.com/fsnotify/fsnotify" - "github.com/vdaas/vald/internal/errgroup" "github.com/vdaas/vald/internal/errors" "github.com/vdaas/vald/internal/file" "github.com/vdaas/vald/internal/log" "github.com/vdaas/vald/internal/log/logger" "github.com/vdaas/vald/internal/strings" + "github.com/vdaas/vald/internal/sync/errgroup" "github.com/vdaas/vald/internal/test/goleak" ) diff --git a/internal/info/info.go b/internal/info/info.go index 7baeae5d6e5..83543025575 100644 --- a/internal/info/info.go +++ b/internal/info/info.go @@ -22,12 +22,12 @@ import ( "reflect" "runtime" "strconv" - "sync" "github.com/vdaas/vald/internal/errors" "github.com/vdaas/vald/internal/log" "github.com/vdaas/vald/internal/slices" "github.com/vdaas/vald/internal/strings" + "github.com/vdaas/vald/internal/sync" ) // Info represents an interface to get the runtime information. diff --git a/internal/info/info_test.go b/internal/info/info_test.go index 75f0174aa5a..16a2ba99b30 100644 --- a/internal/info/info_test.go +++ b/internal/info/info_test.go @@ -21,13 +21,13 @@ import ( "os" "reflect" "runtime" - "sync" "testing" "github.com/vdaas/vald/internal/errors" "github.com/vdaas/vald/internal/log" "github.com/vdaas/vald/internal/log/logger" "github.com/vdaas/vald/internal/strings" + "github.com/vdaas/vald/internal/sync" "github.com/vdaas/vald/internal/test/comparator" "github.com/vdaas/vald/internal/test/goleak" ) diff --git a/internal/info/option_test.go b/internal/info/option_test.go index 99a6802f665..1a90462e691 100644 --- a/internal/info/option_test.go +++ b/internal/info/option_test.go @@ -16,10 +16,10 @@ package info import ( "reflect" "runtime" - "sync" "testing" "github.com/vdaas/vald/internal/errors" + "github.com/vdaas/vald/internal/sync" "github.com/vdaas/vald/internal/test/comparator" "github.com/vdaas/vald/internal/test/goleak" ) diff --git a/internal/io/copy.go b/internal/io/copy.go index 8884063cec4..e22dbdb030b 100644 --- a/internal/io/copy.go +++ b/internal/io/copy.go @@ -21,10 +21,10 @@ import ( "bytes" "io" "math" - "sync" "sync/atomic" "github.com/vdaas/vald/internal/errors" + "github.com/vdaas/vald/internal/sync" ) var cio = NewCopier(0) diff --git a/internal/io/copy_test.go b/internal/io/copy_test.go index 72b4cd4e581..b88af60cf00 100644 --- a/internal/io/copy_test.go +++ b/internal/io/copy_test.go @@ -21,11 +21,11 @@ import ( "bytes" "io" "reflect" - "sync" "sync/atomic" "testing" "github.com/vdaas/vald/internal/errors" + "github.com/vdaas/vald/internal/sync" "github.com/vdaas/vald/internal/test/goleak" ) diff --git a/internal/io/io.go b/internal/io/io.go index 0e39415328a..a302f5ad128 100644 --- a/internal/io/io.go +++ b/internal/io/io.go @@ -21,11 +21,11 @@ import ( "bytes" "context" "io" - "sync" "github.com/vdaas/vald/internal/conv" "github.com/vdaas/vald/internal/errors" "github.com/vdaas/vald/internal/safety" + "github.com/vdaas/vald/internal/sync" ) type ( diff --git a/internal/k8s/option.go b/internal/k8s/option.go index 255c30a9358..6b5d93168a0 100644 --- a/internal/k8s/option.go +++ b/internal/k8s/option.go @@ -18,8 +18,8 @@ package k8s import ( - "github.com/vdaas/vald/internal/errgroup" "github.com/vdaas/vald/internal/net" + "github.com/vdaas/vald/internal/sync/errgroup" "sigs.k8s.io/controller-runtime/pkg/manager" ) diff --git a/internal/k8s/reconciler.go b/internal/k8s/reconciler.go index a329232795f..f40892b81e2 100644 --- a/internal/k8s/reconciler.go +++ b/internal/k8s/reconciler.go @@ -21,10 +21,10 @@ import ( "context" "reflect" - "github.com/vdaas/vald/internal/errgroup" "github.com/vdaas/vald/internal/errors" "github.com/vdaas/vald/internal/net" "github.com/vdaas/vald/internal/safety" + "github.com/vdaas/vald/internal/sync/errgroup" "k8s.io/apimachinery/pkg/runtime" "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client" diff --git a/internal/log/log.go b/internal/log/log.go index 21bdd0129cf..8a9b69168ba 100644 --- a/internal/log/log.go +++ b/internal/log/log.go @@ -17,8 +17,6 @@ package log import ( - "sync" - "github.com/vdaas/vald/internal/log/format" "github.com/vdaas/vald/internal/log/glg" "github.com/vdaas/vald/internal/log/level" @@ -26,6 +24,7 @@ import ( "github.com/vdaas/vald/internal/log/nop" "github.com/vdaas/vald/internal/log/retry" "github.com/vdaas/vald/internal/log/zap" + "github.com/vdaas/vald/internal/sync" ) var ( diff --git a/internal/log/log_test.go b/internal/log/log_test.go index 6b8688f34a3..735f11ae9f2 100644 --- a/internal/log/log_test.go +++ b/internal/log/log_test.go @@ -18,7 +18,6 @@ package log import ( "reflect" - "sync" "testing" "github.com/vdaas/vald/internal/errors" @@ -27,6 +26,7 @@ import ( logger "github.com/vdaas/vald/internal/log/logger" "github.com/vdaas/vald/internal/log/mock" "github.com/vdaas/vald/internal/log/nop" + "github.com/vdaas/vald/internal/sync" "github.com/vdaas/vald/internal/test/goleak" ) diff --git a/internal/net/dialer.go b/internal/net/dialer.go index 692cc1e0586..822bfcc7630 100644 --- a/internal/net/dialer.go +++ b/internal/net/dialer.go @@ -21,7 +21,6 @@ import ( "context" "net" "strconv" - "sync" "sync/atomic" "syscall" "time" @@ -33,7 +32,7 @@ import ( "github.com/vdaas/vald/internal/net/control" "github.com/vdaas/vald/internal/observability/trace" "github.com/vdaas/vald/internal/safety" - valdsync "github.com/vdaas/vald/internal/sync" + "github.com/vdaas/vald/internal/sync" "github.com/vdaas/vald/internal/tls" ) @@ -60,7 +59,7 @@ type dialer struct { ctrl control.SocketController sockFlg control.SocketFlag dialerDualStack bool - addrs valdsync.Map[string, *addrInfo] + addrs sync.Map[string, *addrInfo] der *net.Dialer dialer func(ctx context.Context, network, addr string) (Conn, error) } diff --git a/internal/net/dialer_test.go b/internal/net/dialer_test.go index 9c2ff0dc333..b7ac7de9b2a 100644 --- a/internal/net/dialer_test.go +++ b/internal/net/dialer_test.go @@ -1830,7 +1830,7 @@ func Test_dialer_tlsHandshake(t *testing.T) { // host string // } // type fields struct { -// dnsCache cacher.Cache +// dnsCache cacher.Cache[*dialerCache] // enableDNSCache bool // tlsConfig *tls.Config // dnsRefreshDurationStr string @@ -1843,6 +1843,7 @@ func Test_dialer_tlsHandshake(t *testing.T) { // ctrl control.SocketController // sockFlg control.SocketFlag // dialerDualStack bool +// addrs sync.Map[string, *addrInfo] // der *net.Dialer // dialer func(ctx context.Context, network, addr string) (Conn, error) // } @@ -1891,6 +1892,7 @@ func Test_dialer_tlsHandshake(t *testing.T) { // ctrl:nil, // sockFlg:nil, // dialerDualStack:false, +// addrs:nil, // der:net.Dialer{}, // dialer:nil, // }, @@ -1928,6 +1930,7 @@ func Test_dialer_tlsHandshake(t *testing.T) { // ctrl:nil, // sockFlg:nil, // dialerDualStack:false, +// addrs:nil, // der:net.Dialer{}, // dialer:nil, // }, @@ -1973,6 +1976,7 @@ func Test_dialer_tlsHandshake(t *testing.T) { // ctrl: test.fields.ctrl, // sockFlg: test.fields.sockFlg, // dialerDualStack: test.fields.dialerDualStack, +// addrs: test.fields.addrs, // der: test.fields.der, // dialer: test.fields.dialer, // } diff --git a/internal/net/grpc/client.go b/internal/net/grpc/client.go index 928e92ade5a..06b8b32a0ae 100644 --- a/internal/net/grpc/client.go +++ b/internal/net/grpc/client.go @@ -25,7 +25,6 @@ import ( "github.com/vdaas/vald/internal/backoff" "github.com/vdaas/vald/internal/circuitbreaker" - "github.com/vdaas/vald/internal/errgroup" "github.com/vdaas/vald/internal/errors" "github.com/vdaas/vald/internal/log" "github.com/vdaas/vald/internal/net" @@ -35,9 +34,10 @@ import ( "github.com/vdaas/vald/internal/net/grpc/status" "github.com/vdaas/vald/internal/observability/trace" "github.com/vdaas/vald/internal/safety" - "github.com/vdaas/vald/internal/singleflight" "github.com/vdaas/vald/internal/strings" - valdsync "github.com/vdaas/vald/internal/sync" + "github.com/vdaas/vald/internal/sync" + "github.com/vdaas/vald/internal/sync/errgroup" + "github.com/vdaas/vald/internal/sync/singleflight" "google.golang.org/grpc" gbackoff "google.golang.org/grpc/backoff" ) @@ -110,7 +110,7 @@ type gRPCClient struct { gbo gbackoff.Config // grpc's original backoff configuration mcd time.Duration // minimum connection timeout duration group singleflight.Group[pool.Conn] - crl valdsync.Map[string, bool] // connection request list + crl sync.Map[string, bool] // connection request list ech <-chan error monitorRunning atomic.Bool @@ -424,7 +424,7 @@ func (g *gRPCClient) RangeConcurrent(ctx context.Context, return g.Range(ctx, f) } eg, egctx := errgroup.New(sctx) - eg.Limitation(concurrency) + eg.SetLimit(concurrency) if g.conns.Len() == 0 { return errors.ErrGRPCClientConnNotFound("*") } @@ -570,11 +570,11 @@ func (g *gRPCClient) OrderedRangeConcurrent(ctx context.Context, if g.conns.Len() == 0 { return errors.ErrGRPCClientConnNotFound("*") } - if concurrency < 2 { + if concurrency == 0 || concurrency == 1 { return g.OrderedRange(ctx, orders, f) } eg, egctx := errgroup.New(sctx) - eg.Limitation(concurrency) + eg.SetLimit(concurrency) for _, order := range orders { addr := order eg.Go(safety.RecoverFunc(func() (err error) { @@ -893,7 +893,7 @@ func (g *gRPCClient) Connect(ctx context.Context, addr string, dopts ...DialOpti span.End() } }() - sconn, shared, err := g.group.Do(ctx, "connect-"+addr, func() (pool.Conn, error) { + sconn, shared, err := g.group.Do(ctx, "connect-"+addr, func(ctx context.Context) (pool.Conn, error) { var ok bool conn, ok = g.conns.Load(addr) if ok && conn != nil { @@ -978,7 +978,7 @@ func (g *gRPCClient) Disconnect(ctx context.Context, addr string) error { span.End() } }() - _, _, err := g.group.Do(ctx, "disconnect-"+addr, func() (pool.Conn, error) { + _, _, err := g.group.Do(ctx, "disconnect-"+addr, func(ctx context.Context) (pool.Conn, error) { p, ok := g.conns.Load(addr) if !ok || p == nil { g.conns.Delete(addr) diff --git a/internal/net/grpc/client_test.go b/internal/net/grpc/client_test.go index 9288153a2ff..d862d7a7c55 100644 --- a/internal/net/grpc/client_test.go +++ b/internal/net/grpc/client_test.go @@ -128,6 +128,7 @@ package grpc // gbo gbackoff.Config // mcd time.Duration // group singleflight.Group[pool.Conn] +// crl sync.Map[string, bool] // ech <-chan error // monitorRunning atomic.Bool // stopMonitor context.CancelFunc @@ -181,6 +182,7 @@ package grpc // gbo:nil, // mcd:nil, // group:nil, +// crl:nil, // ech:nil, // monitorRunning:nil, // stopMonitor:nil, @@ -223,6 +225,7 @@ package grpc // gbo:nil, // mcd:nil, // group:nil, +// crl:nil, // ech:nil, // monitorRunning:nil, // stopMonitor:nil, @@ -274,6 +277,7 @@ package grpc // gbo: test.fields.gbo, // mcd: test.fields.mcd, // group: test.fields.group, +// crl: test.fields.crl, // ech: test.fields.ech, // monitorRunning: test.fields.monitorRunning, // stopMonitor: test.fields.stopMonitor, @@ -312,6 +316,7 @@ package grpc // gbo gbackoff.Config // mcd time.Duration // group singleflight.Group[pool.Conn] +// crl sync.Map[string, bool] // ech <-chan error // monitorRunning atomic.Bool // stopMonitor context.CancelFunc @@ -362,6 +367,7 @@ package grpc // gbo:nil, // mcd:nil, // group:nil, +// crl:nil, // ech:nil, // monitorRunning:nil, // stopMonitor:nil, @@ -405,6 +411,7 @@ package grpc // gbo:nil, // mcd:nil, // group:nil, +// crl:nil, // ech:nil, // monitorRunning:nil, // stopMonitor:nil, @@ -456,6 +463,7 @@ package grpc // gbo: test.fields.gbo, // mcd: test.fields.mcd, // group: test.fields.group, +// crl: test.fields.crl, // ech: test.fields.ech, // monitorRunning: test.fields.monitorRunning, // stopMonitor: test.fields.stopMonitor, @@ -495,6 +503,7 @@ package grpc // gbo gbackoff.Config // mcd time.Duration // group singleflight.Group[pool.Conn] +// crl sync.Map[string, bool] // ech <-chan error // monitorRunning atomic.Bool // stopMonitor context.CancelFunc @@ -546,6 +555,7 @@ package grpc // gbo:nil, // mcd:nil, // group:nil, +// crl:nil, // ech:nil, // monitorRunning:nil, // stopMonitor:nil, @@ -590,6 +600,7 @@ package grpc // gbo:nil, // mcd:nil, // group:nil, +// crl:nil, // ech:nil, // monitorRunning:nil, // stopMonitor:nil, @@ -641,6 +652,7 @@ package grpc // gbo: test.fields.gbo, // mcd: test.fields.mcd, // group: test.fields.group, +// crl: test.fields.crl, // ech: test.fields.ech, // monitorRunning: test.fields.monitorRunning, // stopMonitor: test.fields.stopMonitor, @@ -680,6 +692,7 @@ package grpc // gbo gbackoff.Config // mcd time.Duration // group singleflight.Group[pool.Conn] +// crl sync.Map[string, bool] // ech <-chan error // monitorRunning atomic.Bool // stopMonitor context.CancelFunc @@ -731,6 +744,7 @@ package grpc // gbo:nil, // mcd:nil, // group:nil, +// crl:nil, // ech:nil, // monitorRunning:nil, // stopMonitor:nil, @@ -775,6 +789,7 @@ package grpc // gbo:nil, // mcd:nil, // group:nil, +// crl:nil, // ech:nil, // monitorRunning:nil, // stopMonitor:nil, @@ -826,6 +841,7 @@ package grpc // gbo: test.fields.gbo, // mcd: test.fields.mcd, // group: test.fields.group, +// crl: test.fields.crl, // ech: test.fields.ech, // monitorRunning: test.fields.monitorRunning, // stopMonitor: test.fields.stopMonitor, @@ -866,6 +882,7 @@ package grpc // gbo gbackoff.Config // mcd time.Duration // group singleflight.Group[pool.Conn] +// crl sync.Map[string, bool] // ech <-chan error // monitorRunning atomic.Bool // stopMonitor context.CancelFunc @@ -918,6 +935,7 @@ package grpc // gbo:nil, // mcd:nil, // group:nil, +// crl:nil, // ech:nil, // monitorRunning:nil, // stopMonitor:nil, @@ -963,6 +981,7 @@ package grpc // gbo:nil, // mcd:nil, // group:nil, +// crl:nil, // ech:nil, // monitorRunning:nil, // stopMonitor:nil, @@ -1014,6 +1033,7 @@ package grpc // gbo: test.fields.gbo, // mcd: test.fields.mcd, // group: test.fields.group, +// crl: test.fields.crl, // ech: test.fields.ech, // monitorRunning: test.fields.monitorRunning, // stopMonitor: test.fields.stopMonitor, @@ -1052,6 +1072,7 @@ package grpc // gbo gbackoff.Config // mcd time.Duration // group singleflight.Group[pool.Conn] +// crl sync.Map[string, bool] // ech <-chan error // monitorRunning atomic.Bool // stopMonitor context.CancelFunc @@ -1106,6 +1127,7 @@ package grpc // gbo:nil, // mcd:nil, // group:nil, +// crl:nil, // ech:nil, // monitorRunning:nil, // stopMonitor:nil, @@ -1149,6 +1171,7 @@ package grpc // gbo:nil, // mcd:nil, // group:nil, +// crl:nil, // ech:nil, // monitorRunning:nil, // stopMonitor:nil, @@ -1200,6 +1223,7 @@ package grpc // gbo: test.fields.gbo, // mcd: test.fields.mcd, // group: test.fields.group, +// crl: test.fields.crl, // ech: test.fields.ech, // monitorRunning: test.fields.monitorRunning, // stopMonitor: test.fields.stopMonitor, @@ -1239,6 +1263,7 @@ package grpc // gbo gbackoff.Config // mcd time.Duration // group singleflight.Group[pool.Conn] +// crl sync.Map[string, bool] // ech <-chan error // monitorRunning atomic.Bool // stopMonitor context.CancelFunc @@ -1294,6 +1319,7 @@ package grpc // gbo:nil, // mcd:nil, // group:nil, +// crl:nil, // ech:nil, // monitorRunning:nil, // stopMonitor:nil, @@ -1338,6 +1364,7 @@ package grpc // gbo:nil, // mcd:nil, // group:nil, +// crl:nil, // ech:nil, // monitorRunning:nil, // stopMonitor:nil, @@ -1389,6 +1416,7 @@ package grpc // gbo: test.fields.gbo, // mcd: test.fields.mcd, // group: test.fields.group, +// crl: test.fields.crl, // ech: test.fields.ech, // monitorRunning: test.fields.monitorRunning, // stopMonitor: test.fields.stopMonitor, @@ -1430,6 +1458,7 @@ package grpc // gbo gbackoff.Config // mcd time.Duration // group singleflight.Group[pool.Conn] +// crl sync.Map[string, bool] // ech <-chan error // monitorRunning atomic.Bool // stopMonitor context.CancelFunc @@ -1487,6 +1516,7 @@ package grpc // gbo:nil, // mcd:nil, // group:nil, +// crl:nil, // ech:nil, // monitorRunning:nil, // stopMonitor:nil, @@ -1533,6 +1563,7 @@ package grpc // gbo:nil, // mcd:nil, // group:nil, +// crl:nil, // ech:nil, // monitorRunning:nil, // stopMonitor:nil, @@ -1584,6 +1615,7 @@ package grpc // gbo: test.fields.gbo, // mcd: test.fields.mcd, // group: test.fields.group, +// crl: test.fields.crl, // ech: test.fields.ech, // monitorRunning: test.fields.monitorRunning, // stopMonitor: test.fields.stopMonitor, @@ -1618,6 +1650,7 @@ package grpc // gbo gbackoff.Config // mcd time.Duration // group singleflight.Group[pool.Conn] +// crl sync.Map[string, bool] // ech <-chan error // monitorRunning atomic.Bool // stopMonitor context.CancelFunc @@ -1663,6 +1696,7 @@ package grpc // gbo:nil, // mcd:nil, // group:nil, +// crl:nil, // ech:nil, // monitorRunning:nil, // stopMonitor:nil, @@ -1702,6 +1736,7 @@ package grpc // gbo:nil, // mcd:nil, // group:nil, +// crl:nil, // ech:nil, // monitorRunning:nil, // stopMonitor:nil, @@ -1753,6 +1788,7 @@ package grpc // gbo: test.fields.gbo, // mcd: test.fields.mcd, // group: test.fields.group, +// crl: test.fields.crl, // ech: test.fields.ech, // monitorRunning: test.fields.monitorRunning, // stopMonitor: test.fields.stopMonitor, @@ -1787,6 +1823,7 @@ package grpc // gbo gbackoff.Config // mcd time.Duration // group singleflight.Group[pool.Conn] +// crl sync.Map[string, bool] // ech <-chan error // monitorRunning atomic.Bool // stopMonitor context.CancelFunc @@ -1832,6 +1869,7 @@ package grpc // gbo:nil, // mcd:nil, // group:nil, +// crl:nil, // ech:nil, // monitorRunning:nil, // stopMonitor:nil, @@ -1871,6 +1909,7 @@ package grpc // gbo:nil, // mcd:nil, // group:nil, +// crl:nil, // ech:nil, // monitorRunning:nil, // stopMonitor:nil, @@ -1922,6 +1961,7 @@ package grpc // gbo: test.fields.gbo, // mcd: test.fields.mcd, // group: test.fields.group, +// crl: test.fields.crl, // ech: test.fields.ech, // monitorRunning: test.fields.monitorRunning, // stopMonitor: test.fields.stopMonitor, @@ -1956,6 +1996,7 @@ package grpc // gbo gbackoff.Config // mcd time.Duration // group singleflight.Group[pool.Conn] +// crl sync.Map[string, bool] // ech <-chan error // monitorRunning atomic.Bool // stopMonitor context.CancelFunc @@ -2001,6 +2042,7 @@ package grpc // gbo:nil, // mcd:nil, // group:nil, +// crl:nil, // ech:nil, // monitorRunning:nil, // stopMonitor:nil, @@ -2040,6 +2082,7 @@ package grpc // gbo:nil, // mcd:nil, // group:nil, +// crl:nil, // ech:nil, // monitorRunning:nil, // stopMonitor:nil, @@ -2091,6 +2134,7 @@ package grpc // gbo: test.fields.gbo, // mcd: test.fields.mcd, // group: test.fields.group, +// crl: test.fields.crl, // ech: test.fields.ech, // monitorRunning: test.fields.monitorRunning, // stopMonitor: test.fields.stopMonitor, @@ -2130,6 +2174,7 @@ package grpc // gbo gbackoff.Config // mcd time.Duration // group singleflight.Group[pool.Conn] +// crl sync.Map[string, bool] // ech <-chan error // monitorRunning atomic.Bool // stopMonitor context.CancelFunc @@ -2185,6 +2230,7 @@ package grpc // gbo:nil, // mcd:nil, // group:nil, +// crl:nil, // ech:nil, // monitorRunning:nil, // stopMonitor:nil, @@ -2229,6 +2275,7 @@ package grpc // gbo:nil, // mcd:nil, // group:nil, +// crl:nil, // ech:nil, // monitorRunning:nil, // stopMonitor:nil, @@ -2280,6 +2327,7 @@ package grpc // gbo: test.fields.gbo, // mcd: test.fields.mcd, // group: test.fields.group, +// crl: test.fields.crl, // ech: test.fields.ech, // monitorRunning: test.fields.monitorRunning, // stopMonitor: test.fields.stopMonitor, @@ -2318,6 +2366,7 @@ package grpc // gbo gbackoff.Config // mcd time.Duration // group singleflight.Group[pool.Conn] +// crl sync.Map[string, bool] // ech <-chan error // monitorRunning atomic.Bool // stopMonitor context.CancelFunc @@ -2368,6 +2417,7 @@ package grpc // gbo:nil, // mcd:nil, // group:nil, +// crl:nil, // ech:nil, // monitorRunning:nil, // stopMonitor:nil, @@ -2411,6 +2461,7 @@ package grpc // gbo:nil, // mcd:nil, // group:nil, +// crl:nil, // ech:nil, // monitorRunning:nil, // stopMonitor:nil, @@ -2462,6 +2513,7 @@ package grpc // gbo: test.fields.gbo, // mcd: test.fields.mcd, // group: test.fields.group, +// crl: test.fields.crl, // ech: test.fields.ech, // monitorRunning: test.fields.monitorRunning, // stopMonitor: test.fields.stopMonitor, @@ -2500,6 +2552,7 @@ package grpc // gbo gbackoff.Config // mcd time.Duration // group singleflight.Group[pool.Conn] +// crl sync.Map[string, bool] // ech <-chan error // monitorRunning atomic.Bool // stopMonitor context.CancelFunc @@ -2550,6 +2603,7 @@ package grpc // gbo:nil, // mcd:nil, // group:nil, +// crl:nil, // ech:nil, // monitorRunning:nil, // stopMonitor:nil, @@ -2593,6 +2647,7 @@ package grpc // gbo:nil, // mcd:nil, // group:nil, +// crl:nil, // ech:nil, // monitorRunning:nil, // stopMonitor:nil, @@ -2644,6 +2699,7 @@ package grpc // gbo: test.fields.gbo, // mcd: test.fields.mcd, // group: test.fields.group, +// crl: test.fields.crl, // ech: test.fields.ech, // monitorRunning: test.fields.monitorRunning, // stopMonitor: test.fields.stopMonitor, @@ -2678,6 +2734,7 @@ package grpc // gbo gbackoff.Config // mcd time.Duration // group singleflight.Group[pool.Conn] +// crl sync.Map[string, bool] // ech <-chan error // monitorRunning atomic.Bool // stopMonitor context.CancelFunc @@ -2723,6 +2780,7 @@ package grpc // gbo:nil, // mcd:nil, // group:nil, +// crl:nil, // ech:nil, // monitorRunning:nil, // stopMonitor:nil, @@ -2762,6 +2820,7 @@ package grpc // gbo:nil, // mcd:nil, // group:nil, +// crl:nil, // ech:nil, // monitorRunning:nil, // stopMonitor:nil, @@ -2813,6 +2872,7 @@ package grpc // gbo: test.fields.gbo, // mcd: test.fields.mcd, // group: test.fields.group, +// crl: test.fields.crl, // ech: test.fields.ech, // monitorRunning: test.fields.monitorRunning, // stopMonitor: test.fields.stopMonitor, @@ -2850,6 +2910,7 @@ package grpc // gbo gbackoff.Config // mcd time.Duration // group singleflight.Group[pool.Conn] +// crl sync.Map[string, bool] // ech <-chan error // monitorRunning atomic.Bool // stopMonitor context.CancelFunc @@ -2899,6 +2960,7 @@ package grpc // gbo:nil, // mcd:nil, // group:nil, +// crl:nil, // ech:nil, // monitorRunning:nil, // stopMonitor:nil, @@ -2941,6 +3003,7 @@ package grpc // gbo:nil, // mcd:nil, // group:nil, +// crl:nil, // ech:nil, // monitorRunning:nil, // stopMonitor:nil, @@ -2992,6 +3055,7 @@ package grpc // gbo: test.fields.gbo, // mcd: test.fields.mcd, // group: test.fields.group, +// crl: test.fields.crl, // ech: test.fields.ech, // monitorRunning: test.fields.monitorRunning, // stopMonitor: test.fields.stopMonitor, diff --git a/internal/net/grpc/grpcconns.go b/internal/net/grpc/grpcconns.go index 955b83fed5b..4d19a994f7b 100644 --- a/internal/net/grpc/grpcconns.go +++ b/internal/net/grpc/grpcconns.go @@ -18,12 +18,12 @@ package grpc import ( - "sync" "sync/atomic" "unsafe" "github.com/vdaas/vald/internal/errors" "github.com/vdaas/vald/internal/net/grpc/pool" + "github.com/vdaas/vald/internal/sync" ) type grpcConns struct { diff --git a/internal/net/grpc/logger/logger.go b/internal/net/grpc/logger/logger.go index d8b81aa8a4f..72b311eab47 100644 --- a/internal/net/grpc/logger/logger.go +++ b/internal/net/grpc/logger/logger.go @@ -16,9 +16,9 @@ package logger import ( "os" "strconv" - "sync" "github.com/vdaas/vald/internal/log" + "github.com/vdaas/vald/internal/sync" glog "google.golang.org/grpc/grpclog" ) diff --git a/internal/net/grpc/logger/logger_test.go b/internal/net/grpc/logger/logger_test.go index 6fac492791a..be230770fca 100644 --- a/internal/net/grpc/logger/logger_test.go +++ b/internal/net/grpc/logger/logger_test.go @@ -17,10 +17,10 @@ import ( "os" "os/exec" "reflect" - "sync" "testing" "github.com/vdaas/vald/internal/errors" + "github.com/vdaas/vald/internal/sync" "github.com/vdaas/vald/internal/test/goleak" "google.golang.org/grpc/grpclog" ) diff --git a/internal/net/grpc/option.go b/internal/net/grpc/option.go index 14b90e9cb68..12e7fd6db09 100644 --- a/internal/net/grpc/option.go +++ b/internal/net/grpc/option.go @@ -24,11 +24,11 @@ import ( "github.com/vdaas/vald/internal/backoff" "github.com/vdaas/vald/internal/circuitbreaker" - "github.com/vdaas/vald/internal/errgroup" "github.com/vdaas/vald/internal/log" "github.com/vdaas/vald/internal/net" "github.com/vdaas/vald/internal/net/grpc/interceptor/client/trace" "github.com/vdaas/vald/internal/strings" + "github.com/vdaas/vald/internal/sync/errgroup" "github.com/vdaas/vald/internal/timeutil" "google.golang.org/grpc" gbackoff "google.golang.org/grpc/backoff" diff --git a/internal/net/grpc/pool/option.go b/internal/net/grpc/pool/option.go index c544a6520dd..8126d195198 100644 --- a/internal/net/grpc/pool/option.go +++ b/internal/net/grpc/pool/option.go @@ -19,7 +19,7 @@ package pool import ( "github.com/vdaas/vald/internal/backoff" - "github.com/vdaas/vald/internal/errgroup" + "github.com/vdaas/vald/internal/sync/errgroup" "github.com/vdaas/vald/internal/timeutil" ) diff --git a/internal/net/grpc/pool/pool.go b/internal/net/grpc/pool/pool.go index fba9f0d2678..942b8db6a98 100644 --- a/internal/net/grpc/pool/pool.go +++ b/internal/net/grpc/pool/pool.go @@ -22,18 +22,18 @@ import ( "fmt" "math" "strconv" - "sync" "sync/atomic" "time" "github.com/vdaas/vald/internal/backoff" - "github.com/vdaas/vald/internal/errgroup" "github.com/vdaas/vald/internal/errors" "github.com/vdaas/vald/internal/log" "github.com/vdaas/vald/internal/net" "github.com/vdaas/vald/internal/safety" "github.com/vdaas/vald/internal/slices" "github.com/vdaas/vald/internal/strings" + "github.com/vdaas/vald/internal/sync" + "github.com/vdaas/vald/internal/sync/errgroup" "google.golang.org/grpc" "google.golang.org/grpc/connectivity" "google.golang.org/grpc/credentials/insecure" diff --git a/internal/net/grpc/pool/pool_bench_test.go b/internal/net/grpc/pool/pool_bench_test.go index a8d7515c1f1..71f15499b43 100644 --- a/internal/net/grpc/pool/pool_bench_test.go +++ b/internal/net/grpc/pool/pool_bench_test.go @@ -19,7 +19,6 @@ package pool import ( "context" - "sync" "testing" "github.com/vdaas/vald/apis/grpc/v1/discoverer" @@ -27,7 +26,7 @@ import ( "github.com/vdaas/vald/internal/log" "github.com/vdaas/vald/internal/log/level" "github.com/vdaas/vald/internal/net" - valdsync "github.com/vdaas/vald/internal/sync" + "github.com/vdaas/vald/internal/sync" "google.golang.org/grpc" "google.golang.org/grpc/credentials/insecure" ) @@ -134,7 +133,7 @@ func Benchmark_StaticDial(b *testing.B) { b.Error(err) } - conns := new(valdsync.Map[string, *grpc.ClientConn]) + conns := new(sync.Map[string, *grpc.ClientConn]) conns.Store(DefaultServerAddr, conn) b.StopTimer() @@ -191,7 +190,7 @@ func BenchmarkParallel_StaticDial(b *testing.B) { b.Error(err) } - conns := new(valdsync.Map[string, *grpc.ClientConn]) + conns := new(sync.Map[string, *grpc.ClientConn]) conns.Store(DefaultServerAddr, conn) b.StopTimer() diff --git a/internal/net/grpc/status/status_test.go b/internal/net/grpc/status/status_test.go index c3fb020bcde..7823ba3f491 100644 --- a/internal/net/grpc/status/status_test.go +++ b/internal/net/grpc/status/status_test.go @@ -1810,6 +1810,98 @@ func TestParseError(t *testing.T) { // } // } // +// func TestCreateWithNotFound(t *testing.T) { +// type args struct { +// msg string +// err error +// details []interface{} +// } +// type want struct { +// want *Status +// } +// type test struct { +// name string +// args args +// want want +// checkFunc func(want, *Status) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, got *Status) error { +// if !reflect.DeepEqual(got, w.want) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", got, w.want) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// msg:"", +// err:nil, +// details:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// msg:"", +// err:nil, +// details:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// +// got := CreateWithNotFound(test.args.msg, test.args.err, test.args.details...) +// if err := checkFunc(test.want, got); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } +// // func TestError(t *testing.T) { // type args struct { // code codes.Code diff --git a/internal/net/grpc/stream.go b/internal/net/grpc/stream.go index 8ef34f82f3d..d6b09c15990 100644 --- a/internal/net/grpc/stream.go +++ b/internal/net/grpc/stream.go @@ -21,10 +21,8 @@ import ( "context" "fmt" "runtime" - "sync" "sync/atomic" - "github.com/vdaas/vald/internal/errgroup" "github.com/vdaas/vald/internal/errors" "github.com/vdaas/vald/internal/io" "github.com/vdaas/vald/internal/log" @@ -34,6 +32,8 @@ import ( "github.com/vdaas/vald/internal/observability/trace" "github.com/vdaas/vald/internal/safety" "github.com/vdaas/vald/internal/slices" + "github.com/vdaas/vald/internal/sync" + "github.com/vdaas/vald/internal/sync/errgroup" "google.golang.org/grpc" ) @@ -58,7 +58,7 @@ func BidirectionalStream[Q any, R any](ctx context.Context, stream ServerStream, }() eg, ctx := errgroup.New(ctx) if concurrency > 0 { - eg.Limitation(concurrency) + eg.SetLimit(concurrency) } var ( diff --git a/internal/net/http/client/client_test.go b/internal/net/http/client/client_test.go index b248b72621e..ab43883056d 100644 --- a/internal/net/http/client/client_test.go +++ b/internal/net/http/client/client_test.go @@ -22,7 +22,6 @@ import ( "net/http" "net/url" "reflect" - "sync" "sync/atomic" "testing" @@ -32,6 +31,7 @@ import ( "github.com/vdaas/vald/internal/log/logger" "github.com/vdaas/vald/internal/net" htr "github.com/vdaas/vald/internal/net/http/transport" + "github.com/vdaas/vald/internal/sync" "github.com/vdaas/vald/internal/test/comparator" "github.com/vdaas/vald/internal/test/goleak" "golang.org/x/net/http2" diff --git a/internal/net/http/middleware/option.go b/internal/net/http/middleware/option.go index cfb7750cc7f..150a2e7afec 100644 --- a/internal/net/http/middleware/option.go +++ b/internal/net/http/middleware/option.go @@ -20,7 +20,7 @@ package middleware import ( "time" - "github.com/vdaas/vald/internal/errgroup" + "github.com/vdaas/vald/internal/sync/errgroup" "github.com/vdaas/vald/internal/timeutil" ) diff --git a/internal/net/http/middleware/option_test.go b/internal/net/http/middleware/option_test.go index 140c61b985d..eb132767c24 100644 --- a/internal/net/http/middleware/option_test.go +++ b/internal/net/http/middleware/option_test.go @@ -19,8 +19,8 @@ import ( "testing" "time" - "github.com/vdaas/vald/internal/errgroup" "github.com/vdaas/vald/internal/errors" + "github.com/vdaas/vald/internal/sync/errgroup" ) func TestWithErrorGroup(t *testing.T) { diff --git a/internal/net/http/middleware/timeout.go b/internal/net/http/middleware/timeout.go index 001f9fe917d..28af2307e82 100644 --- a/internal/net/http/middleware/timeout.go +++ b/internal/net/http/middleware/timeout.go @@ -24,10 +24,10 @@ import ( "time" "github.com/kpango/fastime" - "github.com/vdaas/vald/internal/errgroup" "github.com/vdaas/vald/internal/errors" "github.com/vdaas/vald/internal/net/http/rest" "github.com/vdaas/vald/internal/safety" + "github.com/vdaas/vald/internal/sync/errgroup" ) type timeout struct { diff --git a/internal/net/http/middleware/timeout_test.go b/internal/net/http/middleware/timeout_test.go index 56e661717ef..2ad64f48e62 100644 --- a/internal/net/http/middleware/timeout_test.go +++ b/internal/net/http/middleware/timeout_test.go @@ -22,10 +22,10 @@ import ( "testing" "time" - "github.com/vdaas/vald/internal/errgroup" "github.com/vdaas/vald/internal/errors" "github.com/vdaas/vald/internal/net/http/rest" "github.com/vdaas/vald/internal/strings" + "github.com/vdaas/vald/internal/sync/errgroup" ) func TestNewTimeout(t *testing.T) { diff --git a/internal/net/net.go b/internal/net/net.go index 58cb705a76b..3108954d406 100644 --- a/internal/net/net.go +++ b/internal/net/net.go @@ -23,14 +23,14 @@ import ( "net" "net/netip" "strconv" - "sync" "syscall" - "github.com/vdaas/vald/internal/errgroup" "github.com/vdaas/vald/internal/errors" "github.com/vdaas/vald/internal/log" "github.com/vdaas/vald/internal/safety" "github.com/vdaas/vald/internal/strings" + "github.com/vdaas/vald/internal/sync" + "github.com/vdaas/vald/internal/sync/errgroup" ) type ( @@ -234,7 +234,7 @@ func ScanPorts(ctx context.Context, start, end uint16, host string) (ports []uin return nil, err } eg, egctx := errgroup.New(ctx) - eg.Limitation(int(rl.Max) / 2) + eg.SetLimit(int(rl.Max) / 2) var mu sync.Mutex diff --git a/internal/observability/metrics/agent/sidecar/sidecar.go b/internal/observability/metrics/agent/sidecar/sidecar.go index 34ec7934ea2..684a5897cfe 100644 --- a/internal/observability/metrics/agent/sidecar/sidecar.go +++ b/internal/observability/metrics/agent/sidecar/sidecar.go @@ -15,11 +15,11 @@ package sidecar import ( "context" - "sync" "time" "github.com/vdaas/vald/internal/observability/attribute" "github.com/vdaas/vald/internal/observability/metrics" + "github.com/vdaas/vald/internal/sync" "github.com/vdaas/vald/pkg/agent/sidecar/service/observer" "go.opentelemetry.io/otel/sdk/metric/aggregation" "go.opentelemetry.io/otel/sdk/metric/view" diff --git a/internal/observability/observability.go b/internal/observability/observability.go index f80fcb92d2d..3df322bf51c 100644 --- a/internal/observability/observability.go +++ b/internal/observability/observability.go @@ -18,7 +18,6 @@ import ( "reflect" "github.com/vdaas/vald/internal/config" - "github.com/vdaas/vald/internal/errgroup" "github.com/vdaas/vald/internal/errors" "github.com/vdaas/vald/internal/log" "github.com/vdaas/vald/internal/observability/exporter" @@ -30,6 +29,7 @@ import ( "github.com/vdaas/vald/internal/observability/metrics/runtime/goroutine" "github.com/vdaas/vald/internal/observability/metrics/version" "github.com/vdaas/vald/internal/observability/trace" + "github.com/vdaas/vald/internal/sync/errgroup" ) type Observability interface { diff --git a/internal/observability/option.go b/internal/observability/option.go index aa9d1d5479a..e2246f578c8 100644 --- a/internal/observability/option.go +++ b/internal/observability/option.go @@ -14,11 +14,11 @@ package observability import ( - "github.com/vdaas/vald/internal/errgroup" "github.com/vdaas/vald/internal/errors" "github.com/vdaas/vald/internal/observability/exporter" "github.com/vdaas/vald/internal/observability/metrics" "github.com/vdaas/vald/internal/observability/trace" + "github.com/vdaas/vald/internal/sync/errgroup" ) type Option func(*observability) error diff --git a/internal/rand/rand.go b/internal/rand/rand.go index e0235cb695d..38f7e5172fc 100644 --- a/internal/rand/rand.go +++ b/internal/rand/rand.go @@ -18,10 +18,10 @@ package rand import ( - "sync" "sync/atomic" "github.com/kpango/fastime" + "github.com/vdaas/vald/internal/sync" ) type rand struct { diff --git a/internal/rand/rand_test.go b/internal/rand/rand_test.go index a1cd8cc2fa1..dc4155770e3 100644 --- a/internal/rand/rand_test.go +++ b/internal/rand/rand_test.go @@ -15,11 +15,11 @@ package rand import ( "fmt" - "sync" "sync/atomic" "testing" "github.com/vdaas/vald/internal/errors" + "github.com/vdaas/vald/internal/sync" "github.com/vdaas/vald/internal/test/goleak" ) diff --git a/internal/runner/runner.go b/internal/runner/runner.go index fd6da03e7b2..56ab49efc59 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -26,7 +26,6 @@ import ( "github.com/vdaas/vald/internal/config" "github.com/vdaas/vald/internal/conv" "github.com/vdaas/vald/internal/encoding/json" - "github.com/vdaas/vald/internal/errgroup" "github.com/vdaas/vald/internal/errors" "github.com/vdaas/vald/internal/info" "github.com/vdaas/vald/internal/log" @@ -34,6 +33,7 @@ import ( "github.com/vdaas/vald/internal/params" "github.com/vdaas/vald/internal/safety" "github.com/vdaas/vald/internal/strings" + "github.com/vdaas/vald/internal/sync/errgroup" "github.com/vdaas/vald/internal/timeutil/location" ver "github.com/vdaas/vald/internal/version" "go.uber.org/automaxprocs/maxprocs" diff --git a/internal/servers/option.go b/internal/servers/option.go index f6d1e88ca33..399a14cc7f4 100644 --- a/internal/servers/option.go +++ b/internal/servers/option.go @@ -20,8 +20,8 @@ package servers import ( "time" - "github.com/vdaas/vald/internal/errgroup" "github.com/vdaas/vald/internal/servers/server" + "github.com/vdaas/vald/internal/sync/errgroup" "github.com/vdaas/vald/internal/timeutil" ) diff --git a/internal/servers/option_test.go b/internal/servers/option_test.go index 46629861092..a80a4850e4e 100644 --- a/internal/servers/option_test.go +++ b/internal/servers/option_test.go @@ -19,9 +19,9 @@ import ( "testing" "time" - "github.com/vdaas/vald/internal/errgroup" "github.com/vdaas/vald/internal/errors" "github.com/vdaas/vald/internal/servers/server" + "github.com/vdaas/vald/internal/sync/errgroup" ) func TestWithServer(t *testing.T) { diff --git a/internal/servers/server/option.go b/internal/servers/server/option.go index ac0e44053b2..7d94220ad2c 100644 --- a/internal/servers/server/option.go +++ b/internal/servers/server/option.go @@ -23,7 +23,6 @@ import ( "net/http" "time" - "github.com/vdaas/vald/internal/errgroup" "github.com/vdaas/vald/internal/errors" "github.com/vdaas/vald/internal/info" "github.com/vdaas/vald/internal/log" @@ -36,6 +35,7 @@ import ( "github.com/vdaas/vald/internal/net/grpc/interceptor/server/trace" "github.com/vdaas/vald/internal/net/http/rest" "github.com/vdaas/vald/internal/strings" + "github.com/vdaas/vald/internal/sync/errgroup" "github.com/vdaas/vald/internal/timeutil" ) diff --git a/internal/servers/server/option_test.go b/internal/servers/server/option_test.go index 721603813cc..3d6de488804 100644 --- a/internal/servers/server/option_test.go +++ b/internal/servers/server/option_test.go @@ -20,10 +20,10 @@ import ( "testing" "time" - "github.com/vdaas/vald/internal/errgroup" "github.com/vdaas/vald/internal/errors" "github.com/vdaas/vald/internal/net" "github.com/vdaas/vald/internal/net/grpc" + "github.com/vdaas/vald/internal/sync/errgroup" ) func TestWithHost(t *testing.T) { diff --git a/internal/servers/server/server.go b/internal/servers/server/server.go index 8e32ea7debf..3b7a7939a15 100644 --- a/internal/servers/server/server.go +++ b/internal/servers/server/server.go @@ -24,11 +24,9 @@ import ( "os" "reflect" "strconv" - "sync" "syscall" "time" - "github.com/vdaas/vald/internal/errgroup" "github.com/vdaas/vald/internal/errors" "github.com/vdaas/vald/internal/log" "github.com/vdaas/vald/internal/net" @@ -39,6 +37,8 @@ import ( glog "github.com/vdaas/vald/internal/net/grpc/logger" "github.com/vdaas/vald/internal/safety" "github.com/vdaas/vald/internal/strings" + "github.com/vdaas/vald/internal/sync" + "github.com/vdaas/vald/internal/sync/errgroup" ) type Server interface { diff --git a/internal/servers/server/server_test.go b/internal/servers/server/server_test.go index bedb6028c83..2b7e67d2caf 100644 --- a/internal/servers/server/server_test.go +++ b/internal/servers/server/server_test.go @@ -21,12 +21,12 @@ import ( "testing" "time" - "github.com/vdaas/vald/internal/errgroup" "github.com/vdaas/vald/internal/errors" "github.com/vdaas/vald/internal/log" "github.com/vdaas/vald/internal/log/logger" "github.com/vdaas/vald/internal/net" "github.com/vdaas/vald/internal/net/grpc" + "github.com/vdaas/vald/internal/sync/errgroup" ) func TestServerMode_String(t *testing.T) { diff --git a/internal/servers/servers.go b/internal/servers/servers.go index 0416d174775..6dd625567a6 100644 --- a/internal/servers/servers.go +++ b/internal/servers/servers.go @@ -23,11 +23,11 @@ import ( "sort" "time" - "github.com/vdaas/vald/internal/errgroup" "github.com/vdaas/vald/internal/errors" "github.com/vdaas/vald/internal/safety" "github.com/vdaas/vald/internal/servers/server" "github.com/vdaas/vald/internal/strings" + "github.com/vdaas/vald/internal/sync/errgroup" ) type Listener interface { diff --git a/internal/servers/servers_test.go b/internal/servers/servers_test.go index 3bf383844d8..5c3f9aa419f 100644 --- a/internal/servers/servers_test.go +++ b/internal/servers/servers_test.go @@ -19,9 +19,9 @@ import ( "testing" "time" - "github.com/vdaas/vald/internal/errgroup" "github.com/vdaas/vald/internal/errors" "github.com/vdaas/vald/internal/servers/server" + "github.com/vdaas/vald/internal/sync/errgroup" ) func TestNew(t *testing.T) { diff --git a/internal/singleflight/singleflight.go b/internal/singleflight/singleflight.go deleted file mode 100644 index 94e72b149cc..00000000000 --- a/internal/singleflight/singleflight.go +++ /dev/null @@ -1,68 +0,0 @@ -// -// Copyright (C) 2019-2023 vdaas.org vald team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// - -// Package singleflight represents zero time caching. -package singleflight - -import ( - "context" - "sync" - "sync/atomic" - - valdsync "github.com/vdaas/vald/internal/sync" -) - -type call[V any] struct { - wg sync.WaitGroup - val V - err error - dups uint64 -} - -// Group represents interface for zero time cache. -type Group[V any] interface { - Do(ctx context.Context, key string, fn func() (V, error)) (v V, shared bool, err error) -} - -type group[V any] struct { - m valdsync.Map[string, *call[V]] -} - -// New returns Group implementation. -func New[V any]() Group[V] { - return new(group[V]) -} - -// Do execute the given function and return the result. -// It makes sure only one execution of the function for each given key. -// If duplicate comes, the duplicated call with the same key will wait for the first caller return. -// It returns the result and the error of the given function, and whether the result is shared from the first caller. -func (g *group[V]) Do(_ context.Context, key string, fn func() (V, error)) (v V, shared bool, err error) { - c, loaded := g.m.LoadOrStore(key, new(call[V])) - if loaded { - atomic.AddUint64(&c.dups, 1) - c.wg.Wait() - v, err = c.val, c.err - return v, true, err - } - c.wg.Add(1) - c.val, c.err = fn() - c.wg.Done() - - g.m.LoadAndDelete(key) - - return c.val, atomic.LoadUint64(&c.dups) > 0, c.err -} diff --git a/internal/singleflight/singleflight_test.go b/internal/singleflight/singleflight_test.go deleted file mode 100644 index b283db915f2..00000000000 --- a/internal/singleflight/singleflight_test.go +++ /dev/null @@ -1,294 +0,0 @@ -//go:build !race - -// -// Copyright (C) 2019-2023 vdaas.org vald team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// - -// Package singleflight represents zero time caching -package singleflight - -import ( - "context" - "reflect" - "sync" - "sync/atomic" - "testing" - "time" - - "github.com/vdaas/vald/internal/errors" - "github.com/vdaas/vald/internal/test/goleak" -) - -func TestNew(t *testing.T) { - type want struct { - want Group[any] - } - type test struct { - name string - want want - checkFunc func(want, Group[any]) error - beforeFunc func() - afterFunc func() - } - defaultCheckFunc := func(w want, got Group[any]) error { - if !reflect.DeepEqual(got, w.want) { - return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", got, w.want) - } - return nil - } - tests := []test{ - { - name: "returns Group implementation", - want: want{ - want: &group[any]{}, - }, - }, - } - - for _, tc := range tests { - test := tc - t.Run(test.name, func(tt *testing.T) { - defer goleak.VerifyNone(tt) - if test.beforeFunc != nil { - test.beforeFunc() - } - if test.afterFunc != nil { - defer test.afterFunc() - } - checkFunc := test.checkFunc - if test.checkFunc == nil { - checkFunc = defaultCheckFunc - } - - got := New[any]() - if err := checkFunc(test.want, got); err != nil { - tt.Errorf("error = %v", err) - } - }) - } -} - -func Test_group_Do(t *testing.T) { - type args[V any] struct { - ctx context.Context - key string - fn func() (V, error) - } - type want[V any] struct { - wantV V - wantShared bool - err error - } - type test[V any] struct { - name string - args args[V] - want want[V] - beforeFunc func(args[V]) - execFunc func(*testing.T, args[V]) (V, bool, error) - checkFunc func(want[V], V, bool, error) error - afterFunc func(args[V]) - } - tests := []test[string]{ - func() test[string] { - // routine1 - key1 := "req_1" - var cnt1 uint32 - - // the unparam lint rule is disabled here because we need to match the interface to singleflight implementation. - // if this rule is not disabled, if will warns that the error will always return null. - //nolint:unparam - fn1 := func() (string, error) { - atomic.AddUint32(&cnt1, 1) - return "res_1", nil - } - - // routine 2 - key2 := "req_2" - var cnt2 uint32 - - // the unparam lint rule is disabled here because we need to match the interface to singleflight implementation. - // if this rule is not disabled, if will warns that the error will always return null. - //nolint:unparam - fn2 := func() (string, error) { - atomic.AddUint32(&cnt2, 1) - return "res_2", nil - } - - return test[string]{ - name: "returns (v, false, nil) when Do is called with another key", - args: args[string]{ - key: key1, - ctx: context.Background(), - fn: fn1, - }, - want: want[string]{ - wantV: "res_1", - wantShared: false, - err: nil, - }, - execFunc: func(t *testing.T, a args[string]) (got string, gotShared bool, err error) { - t.Helper() - g := New[string]() - - wg := new(sync.WaitGroup) - wg.Add(1) - go func() { - got, gotShared, err = g.Do(a.ctx, a.key, a.fn) - wg.Done() - }() - - wg.Add(1) - go func() { - _, _, _ = g.Do(a.ctx, key2, fn2) - wg.Done() - }() - - wg.Wait() - return got, gotShared, err - }, - checkFunc: func(w want[string], gotV string, gotShared bool, err error) error { - if got, want := int(atomic.LoadUint32(&cnt1)), 1; got != want { - return errors.Errorf("cnt got = %d, want = %d", got, want) - } - if got, want := int(atomic.LoadUint32(&cnt2)), 1; got != want { - return errors.Errorf("cnt got = %d, want = %d", got, want) - } - if !errors.Is(err, w.err) { - return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err) - } - if !reflect.DeepEqual(gotV, w.wantV) { - return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", gotV, w.wantV) - } - if !reflect.DeepEqual(gotShared, w.wantShared) { - return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", gotShared, w.wantShared) - } - return nil - }, - } - }(), - func() test[string] { - // routine1 - var cnt1 uint32 - - // the unparam lint rule is disabled here because we need to match the interface to singleflight implementation. - // if this rule is not disabled, if will warns that the error will always return null. - //nolint:unparam - fn1 := func() (string, error) { - atomic.AddUint32(&cnt1, 1) - time.Sleep(time.Millisecond * 500) - return "res_1", nil - } - - // routine 2 - var cnt2 uint32 - - // the unparam lint rule is disabled here because we need to match the interface to singleflight implementation. - // if this rule is not disabled, if will warns that the error will always return null. - //nolint:unparam - fn2 := func() (string, error) { - atomic.AddUint32(&cnt2, 1) - return "res_2", nil - } - - w := want[string]{ - wantV: "res_1", - wantShared: true, - err: nil, - } - - checkFunc := func(w want[string], gotV string, gotShared bool, err error) error { - c1 := int(atomic.LoadUint32(&cnt1)) - c2 := int(atomic.LoadUint32(&cnt2)) - // since there is a chance that the go routine 2 is executed before routine 1, we need to check if either one is executed - if !((c1 == 1 && c2 == 0) || (c1 == 0 && c2 == 1)) { - return errors.Errorf("cnt1 and cnt2 is executed, %d, %d", c1, c2) - } - if !errors.Is(err, w.err) { - return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err) - } - if !reflect.DeepEqual(gotV, w.wantV) { - return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", gotV, w.wantV) - } - if !reflect.DeepEqual(gotShared, w.wantShared) { - return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", gotShared, w.wantShared) - } - return nil - } - - return test[string]{ - name: "returns (v, true, nil) when Do is called with the same key", - args: args[string]{ - key: "req_1", - ctx: context.Background(), - fn: fn1, - }, - want: w, - execFunc: func(t *testing.T, a args[string]) (string, bool, error) { - t.Helper() - - g := New[string]() - wg := new(sync.WaitGroup) - var got, got1 string - var gotShared, gotShared1 bool - var err, err1 error - - wg.Add(1) - go func() { - got, gotShared, err = g.Do(a.ctx, a.key, fn1) - wg.Done() - }() - - // call with the same key but with another function - wg.Add(1) - time.Sleep(time.Millisecond * 100) - go func() { - got1, gotShared1, err1 = g.Do(a.ctx, a.key, fn2) - wg.Done() - }() - - wg.Wait() - - if err := checkFunc(w, got1, gotShared1, err1); err != nil { - t.Fatal(err) - } - - return got, gotShared, err - }, - checkFunc: checkFunc, - } - }(), - } - - for _, tc := range tests { - test := tc - t.Run(test.name, func(tt *testing.T) { - defer goleak.VerifyNone(tt) - if test.afterFunc != nil { - defer test.afterFunc(test.args) - } - if test.beforeFunc != nil { - test.beforeFunc(test.args) - } - - gotV, gotShared, err := test.execFunc(t, test.args) - - if err := test.checkFunc(test.want, gotV, gotShared, err); err != nil { - tt.Errorf("error = %v", err) - } - }) - } -} - -// NOT IMPLEMENTED BELOW diff --git a/internal/strings/strings.go b/internal/strings/strings.go index da4a76f9df2..2031fdefe86 100644 --- a/internal/strings/strings.go +++ b/internal/strings/strings.go @@ -16,7 +16,8 @@ package strings import ( "bytes" "strings" - "sync" + + "github.com/vdaas/vald/internal/sync" ) type ( diff --git a/internal/sync/alias.go b/internal/sync/alias.go new file mode 100644 index 00000000000..238935dcfb6 --- /dev/null +++ b/internal/sync/alias.go @@ -0,0 +1,28 @@ +// Copyright (C) 2019-2023 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package sync + +import "sync" + +type ( + Cond = sync.Cond + Locker = sync.Locker + Mutex = sync.Mutex + RWMutex = sync.RWMutex + Once = sync.Once + Pool = sync.Pool + WaitGroup = sync.WaitGroup +) + +var NewCond = sync.NewCond diff --git a/internal/sync/errgroup/errgroup_test.go b/internal/sync/errgroup/errgroup_test.go new file mode 100644 index 00000000000..be65578be8c --- /dev/null +++ b/internal/sync/errgroup/errgroup_test.go @@ -0,0 +1,291 @@ +// +// Copyright (c) 2009-2023 The Go Authors. All rights resered. +// Modified by vdaas.org vald team +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// + +package errgroup_test + +import ( + "context" + "fmt" + "net/http" + "os" + "sync/atomic" + "testing" + "time" + + "github.com/vdaas/vald/internal/errors" + "github.com/vdaas/vald/internal/sync/errgroup" +) + +var ( + Web = fakeSearch("web") + Image = fakeSearch("image") + Video = fakeSearch("video") +) + +type ( + Result string + Search func(ctx context.Context, query string) (Result, error) +) + +func fakeSearch(kind string) Search { + return func(_ context.Context, query string) (Result, error) { + return Result(fmt.Sprintf("%s result for %q", kind, query)), nil + } +} + +// JustErrors illustrates the use of a Group in place of a sync.WaitGroup to +// simplify goroutine counting and error handling. This example is derived from +// the sync.WaitGroup example at https://golang.org/pkg/sync/#example_WaitGroup. +func ExampleGroup_justErrors() { + g, _ := errgroup.New(context.Background()) + urls := []string{ + "http://www.golang.org/", + "http://www.google.com/", + "http://www.somestupidname.com/", + } + for _, url := range urls { + // Launch a goroutine to fetch the URL. + url := url // https://golang.org/doc/faq#closures_and_goroutines + g.Go(func() error { + // Fetch the URL. + resp, err := http.Get(url) + if err == nil { + resp.Body.Close() + } + return err + }) + } + // Wait for all HTTP fetches to complete. + if err := g.Wait(); err == nil { + fmt.Println("Successfully fetched all URLs.") + } +} + +// Parallel illustrates the use of a Group for synchronizing a simple parallel +// task: the "Google Search 2.0" function from +// https://talks.golang.org/2012/concurrency.slide#46, augmented with a Context +// and error-handling. +func ExampleGroup_parallel() { + Google := func(ctx context.Context, query string) ([]Result, error) { + g, ctx := errgroup.WithContext(ctx) + + searches := []Search{Web, Image, Video} + results := make([]Result, len(searches)) + for i, search := range searches { + i, search := i, search // https://golang.org/doc/faq#closures_and_goroutines + g.Go(func() error { + result, err := search(ctx, query) + if err == nil { + results[i] = result + } + return err + }) + } + if err := g.Wait(); err != nil { + return nil, err + } + return results, nil + } + + results, err := Google(context.Background(), "golang") + if err != nil { + fmt.Fprintln(os.Stderr, err) + return + } + for _, result := range results { + fmt.Println(result) + } + + // Output: + // web result for "golang" + // image result for "golang" + // video result for "golang" +} + +func TestZeroGroup(t *testing.T) { + err1 := errors.New("errgroup_test: 1") + err2 := errors.New("errgroup_test: 2") + + cases := []struct { + errs []error + }{ + {errs: []error{}}, + {errs: []error{nil}}, + {errs: []error{err1}}, + {errs: []error{err1, nil}}, + {errs: []error{err1, nil, err2}}, + } + + for _, tc := range cases { + g, _ := errgroup.New(context.Background()) + + var firstErr error + for i, err := range tc.errs { + err := err + g.Go(func() error { return err }) + + if firstErr == nil && err != nil { + firstErr = err + } + + if gErr := g.Wait(); !errors.Is(gErr, firstErr) { + t.Errorf("after %T.Go(func() error { return err }) for err in %v\n"+ + "g.Wait() = %v; want %v", + g, tc.errs[:i+1], err, firstErr) + } + } + } +} + +func TestWithContext(t *testing.T) { + errDoom := errors.New("group_test: doomed") + + cases := []struct { + errs []error + want error + }{ + {want: nil}, + {errs: []error{nil}, want: nil}, + {errs: []error{errDoom}, want: errDoom}, + {errs: []error{errDoom, nil}, want: errDoom}, + } + + for _, tc := range cases { + g, ctx := errgroup.WithContext(context.Background()) + + for _, err := range tc.errs { + err := err + g.Go(func() error { return err }) + } + + if err := g.Wait(); err != tc.want { + t.Errorf("after %T.Go(func() error { return err }) for err in %v\n"+ + "g.Wait() = %v; want %v", + g, tc.errs, err, tc.want) + } + + canceled := false + select { + case <-ctx.Done(): + canceled = true + default: + } + if !canceled { + t.Errorf("after %T.Go(func() error { return err }) for err in %v\n"+ + "ctx.Done() was not closed", + g, tc.errs) + } + } +} + +func TestTryGo(t *testing.T) { + g, _ := errgroup.New(context.Background()) + n := 42 + g.SetLimit(42) + ch := make(chan struct{}) + fn := func() error { + ch <- struct{}{} + return nil + } + for i := 0; i < n; i++ { + if !g.TryGo(fn) { + t.Fatalf("TryGo should succeed but got fail at %d-th call.", i) + } + } + if g.TryGo(fn) { + t.Fatalf("TryGo is expected to fail but succeeded.") + } + go func() { + for i := 0; i < n; i++ { + <-ch + } + }() + g.Wait() + + if !g.TryGo(fn) { + t.Fatalf("TryGo should success but got fail after all goroutines.") + } + go func() { <-ch }() + g.Wait() + + // Switch limit. + g.SetLimit(1) + if !g.TryGo(fn) { + t.Fatalf("TryGo should success but got failed.") + } + if g.TryGo(fn) { + t.Fatalf("TryGo should fail but succeeded.") + } + go func() { <-ch }() + g.Wait() + + // Block all calls. + g.SetLimit(0) + for i := 0; i < 1<<10; i++ { + if g.TryGo(fn) { + t.Fatalf("TryGo should fail but got succeded.") + } + } + g.Wait() +} + +func TestGoLimit(t *testing.T) { + const limit = 10 + + g, _ := errgroup.New(context.Background()) + g.SetLimit(limit) + var active int32 + for i := 0; i <= 1<<10; i++ { + g.Go(func() error { + n := atomic.AddInt32(&active, 1) + if n > limit { + return fmt.Errorf("saw %d active goroutines; want ≤ %d", n, limit) + } + time.Sleep(1 * time.Microsecond) // Give other goroutines a chance to increment active. + atomic.AddInt32(&active, -1) + return nil + }) + } + if err := g.Wait(); err != nil { + t.Fatal(err) + } +} + +func BenchmarkGo(b *testing.B) { + fn := func() {} + g, _ := errgroup.New(context.Background()) + b.ResetTimer() + b.ReportAllocs() + for i := 0; i < b.N; i++ { + g.Go(func() error { fn(); return nil }) + } + g.Wait() +} diff --git a/internal/sync/errgroup/group.go b/internal/sync/errgroup/group.go new file mode 100644 index 00000000000..7a4850c051f --- /dev/null +++ b/internal/sync/errgroup/group.go @@ -0,0 +1,220 @@ +// +// Copyright (C) 2019-2023 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +// Package errgroup provides server global wait group for graceful kill all goroutine +package errgroup + +import ( + "context" + "runtime" + + "github.com/vdaas/vald/internal/sync" + + "github.com/vdaas/vald/internal/errors" + "github.com/vdaas/vald/internal/sync/semaphore" +) + +// A Group is a collection of goroutines working on subtasks that are part of +// the same overall task. +// +// A zero Group is valid, has no limit on the number of active goroutines, +// and does not cancel on error. +type Group interface { + Go(func() error) + SetLimit(limit int) + TryGo(func() error) bool + Wait() error +} + +type group struct { + egctx context.Context + cancel context.CancelCauseFunc + + wg sync.WaitGroup + + sem *semaphore.Weighted + + cancelOnce sync.Once + mu sync.RWMutex + emap map[string]struct{} + errs []error +} + +var ( + instance Group + once sync.Once +) + +func New(ctx context.Context) (Group, context.Context) { + g := &group{emap: make(map[string]struct{})} + g.egctx, g.cancel = context.WithCancelCause(ctx) + return g, g.egctx +} + +// WithContext returns a new Group and an associated Context derived from ctx. +// +// The derived Context is canceled the first time a function passed to Go +// returns a non-nil error or the first time Wait returns, whichever occurs +// first. +func WithContext(ctx context.Context) (Group, context.Context) { + return New(ctx) +} + +func Init(ctx context.Context) (egctx context.Context) { + egctx = ctx + once.Do(func() { + instance, egctx = New(ctx) + }) + return +} + +func Get() Group { + if instance == nil { + Init(context.Background()) + } + return instance +} + +func Go(f func() error) { + instance.Go(f) +} + +func TryGo(f func() error) bool { + return instance.TryGo(f) +} + +// SetLimit limits the number of active goroutines in this group to at most n. +// A negative value indicates no limit. +// +// Any subsequent call to the Go method will block until it can add an active +// goroutine without exceeding the configured limit. +// +// The limit must not be modified while any goroutines in the group are active. +func (g *group) SetLimit(limit int) { + if limit < 0 { + g.sem = nil + return + } + + if g.sem == nil { + g.sem = semaphore.NewWeighted(int64(limit)) + } else { + g.sem.Resize(int64(limit)) + } +} + +// Go calls the given function in a new goroutine. +// It blocks until the new goroutine can be added without the number of +// active goroutines in the group exceeding the configured limit. +// +// The first call to return a non-nil error cancels the group's context, if the +// group was created by calling WithContext. The error will be returned by Wait. +func (g *group) Go(f func() error) { + if f == nil { + return + } + if g.sem != nil { + err := g.sem.Acquire(g.egctx, 1) + if err != nil { + if !errors.Is(err, context.Canceled) && + !errors.Is(err, context.DeadlineExceeded) { + g.appendErr(err) + } + return + } + } + g.run(f) +} + +// TryGo calls the given function in a new goroutine only if the number of +// active goroutines in the group is currently below the configured limit. +// +// The return value reports whether the goroutine was started. +func (g *group) TryGo(f func() error) bool { + if f == nil { + return false + } + if g.sem != nil && !g.sem.TryAcquire(1) { + return false + } + g.run(f) + return true +} + +func (g *group) run(f func() error) { + g.wg.Add(1) + go func() { + defer g.done() + err := f() + if err != nil { + if !errors.Is(err, context.Canceled) && + !errors.Is(err, context.DeadlineExceeded) { + runtime.Gosched() + g.appendErr(err) + } + g.doCancel(err) + return + } + }() +} + +func (g *group) appendErr(err error) { + g.mu.RLock() + _, ok := g.emap[err.Error()] + g.mu.RUnlock() + if !ok { + g.mu.Lock() + g.errs = append(g.errs, err) + g.emap[err.Error()] = struct{}{} + g.mu.Unlock() + } +} + +func (g *group) done() { + defer g.wg.Done() + if g.sem != nil { + g.sem.Release(1) + } +} + +func (g *group) doCancel(err error) { + g.cancelOnce.Do(func() { + if g.cancel != nil { + g.cancel(err) + } + }) +} + +func Wait() error { + return instance.Wait() +} + +// Wait blocks until all function calls from the Go method have returned, then +// returns the first non-nil error (if any) from them. +func (g *group) Wait() (err error) { + g.wg.Wait() + g.doCancel(context.Canceled) + g.mu.RLock() + defer g.mu.RUnlock() + switch len(g.errs) { + case 0: + return nil + case 1: + return g.errs[0] + default: + return errors.Join(g.errs...) + } +} diff --git a/internal/errgroup/group_test.go b/internal/sync/errgroup/group_test.go similarity index 53% rename from internal/errgroup/group_test.go rename to internal/sync/errgroup/group_test.go index f4d89cc29fc..ecd22f85237 100644 --- a/internal/errgroup/group_test.go +++ b/internal/sync/errgroup/group_test.go @@ -21,7 +21,6 @@ import ( "context" "reflect" "runtime" - "sync" "sync/atomic" "testing" "time" @@ -29,6 +28,8 @@ import ( "github.com/vdaas/vald/internal/errors" "github.com/vdaas/vald/internal/log" "github.com/vdaas/vald/internal/log/logger" + "github.com/vdaas/vald/internal/sync" + "github.com/vdaas/vald/internal/sync/semaphore" "github.com/vdaas/vald/internal/test/goleak" ) @@ -54,9 +55,7 @@ func TestNew(t *testing.T) { afterFunc func(*testing.T, args) } defaultCheckFunc := func(w want, got Group, got1 context.Context) error { - if got, want := got.(*group), w.want.(*group); !reflect.DeepEqual(got.emap, want.emap) && - !reflect.DeepEqual(got.enableLimitation, want.enableLimitation) && - got.cancel != nil { + if got, want := got.(*group), w.want.(*group); !reflect.DeepEqual(got.emap, want.emap) && got.cancel != nil { return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", got, w.want) } if !reflect.DeepEqual(got1, w.want1) { @@ -67,7 +66,7 @@ func TestNew(t *testing.T) { tests := []test{ func() test { ctx := context.Background() - egctx, cancel := context.WithCancel(ctx) + egctx, cancel := context.WithCancelCause(ctx) return test{ name: "returns (g, ctx)", @@ -78,11 +77,7 @@ func TestNew(t *testing.T) { want: &group{ egctx: egctx, cancel: cancel, - enableLimitation: func() (el atomic.Bool) { - el.Store(false) - return - }(), - emap: make(map[string]struct{}), + emap: make(map[string]struct{}), }, want1: egctx, }, @@ -142,7 +137,7 @@ func TestInit(t *testing.T) { tests := []test{ func() test { ctx := context.Background() - egctx, cancel := context.WithCancel(ctx) + egctx, cancel := context.WithCancelCause(ctx) return test{ name: "returns egctx when once.Do is called", @@ -154,7 +149,7 @@ func TestInit(t *testing.T) { }, afterFunc: func(t *testing.T, a args) { t.Helper() - cancel() + cancel(context.Canceled) defaultBeforeFunc(a) }, } @@ -217,7 +212,7 @@ func TestGet(t *testing.T) { tests := []test{ func() test { ctx := context.Background() - egctx, cancel := context.WithCancel(ctx) + egctx, cancel := context.WithCancelCause(ctx) return test{ name: "returns instance when instance is nil", @@ -302,7 +297,6 @@ func TestGo(t *testing.T) { beforeFunc: func(t *testing.T, _ args) { t.Helper() g := new(group) - g.enableLimitation.Store(false) instance = g }, checkFunc: func(got Group) error { @@ -341,13 +335,12 @@ func TestGo(t *testing.T) { } } -func Test_group_Limitation(t *testing.T) { +func Test_group_SetLimit(t *testing.T) { type args struct { limit int } type fields struct { - limitation chan struct{} - enableLimitation atomic.Bool + sem *semaphore.Weighted } type want struct { want Group @@ -363,27 +356,29 @@ func Test_group_Limitation(t *testing.T) { } defaultCheckFunc := func(w want, g Group) error { got, want := g.(*group), w.want.(*group) - if !reflect.DeepEqual(got.enableLimitation, want.enableLimitation) { - return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", got, w.want) - } - if got.limitation != nil && want.limitation != nil { - return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", got, w.want) + if (got.sem == nil) != (want.sem == nil) { + return errors.Errorf("got: \"%#v\",\n\t\t\twant: \"%#v\"", got, w.want) } return nil } tests := []test{ { - name: "set disable when limit is 0", + name: "set disable when limit is -1", + args: args{ + limit: -1, + }, + want: want{ + want: &group{}, + }, + }, + { + name: "set enable but block all acquireings when limit is 0", args: args{ limit: 0, }, want: want{ want: &group{ - enableLimitation: func() atomic.Bool { - var el atomic.Bool - el.Store(false) - return el - }(), + sem: semaphore.NewWeighted(0), }, }, }, @@ -393,16 +388,10 @@ func Test_group_Limitation(t *testing.T) { args: args{ limit: 1, }, - fields: fields{ - limitation: make(chan struct{}, 1), - }, + fields: fields{}, want: want{ want: &group{ - enableLimitation: func() atomic.Bool { - var el atomic.Bool - el.Store(true) - return el - }(), + sem: semaphore.NewWeighted(1), }, }, }, @@ -421,12 +410,9 @@ func Test_group_Limitation(t *testing.T) { if test.checkFunc == nil { checkFunc = defaultCheckFunc } - g := &group{ - limitation: test.fields.limitation, - enableLimitation: test.fields.enableLimitation, - } + g := &group{} - g.Limitation(test.args.limit) + g.SetLimit(test.args.limit) if err := checkFunc(test.want, g); err != nil { tt.Errorf("error = %v", err) } @@ -439,11 +425,10 @@ func Test_group_Go(t *testing.T) { f func() error } type fields struct { - egctx context.Context - cancel context.CancelFunc - limitation chan struct{} - enableLimitation atomic.Bool - emap map[string]struct{} + egctx context.Context + cancel context.CancelCauseFunc + sem *semaphore.Weighted + emap map[string]struct{} } type test struct { name string @@ -460,7 +445,7 @@ func Test_group_Go(t *testing.T) { func() test { var calledCnt int32 - egctx, cancel := context.WithCancel(context.Background()) + egctx, cancel := context.WithCancelCause(context.Background()) limit := 3 @@ -473,12 +458,8 @@ func Test_group_Go(t *testing.T) { }, }, fields: fields{ - egctx: egctx, - limitation: make(chan struct{}, limit), - enableLimitation: func() (el atomic.Bool) { - el.Store(true) - return - }(), + egctx: egctx, + sem: semaphore.NewWeighted(int64(limit)), }, beforeFunc: func(t *testing.T, _ args, g Group) { t.Helper() @@ -489,10 +470,9 @@ func Test_group_Go(t *testing.T) { }) } time.Sleep(time.Second) + cancel(context.Canceled) }, checkFunc: func(got Group) error { - cancel() - if err := got.Wait(); err != nil { return err } @@ -508,7 +488,7 @@ func Test_group_Go(t *testing.T) { func() test { var calledCnt int32 - egctx, cancel := context.WithCancel(context.Background()) + egctx, cancel := context.WithCancelCause(context.Background()) return test{ name: "f is called but f returns error and previous process also returns error", @@ -521,11 +501,7 @@ func Test_group_Go(t *testing.T) { fields: fields{ egctx: egctx, cancel: cancel, - enableLimitation: func() (el atomic.Bool) { - el.Store(false) - return - }(), - emap: make(map[string]struct{}), + emap: make(map[string]struct{}), }, beforeFunc: func(t *testing.T, a args, g Group) { t.Helper() @@ -569,11 +545,10 @@ func Test_group_Go(t *testing.T) { } g := &group{ - egctx: test.fields.egctx, - cancel: test.fields.cancel, - limitation: test.fields.limitation, - enableLimitation: test.fields.enableLimitation, - emap: test.fields.emap, + egctx: test.fields.egctx, + cancel: test.fields.cancel, + sem: test.fields.sem, + emap: test.fields.emap, } if test.beforeFunc != nil { @@ -590,7 +565,7 @@ func Test_group_Go(t *testing.T) { func Test_group_doCancel(t *testing.T) { type fields struct { - cancel context.CancelFunc + cancel context.CancelCauseFunc } type test struct { name string @@ -609,7 +584,7 @@ func Test_group_doCancel(t *testing.T) { return test{ name: "g.cancel is called when g.cancel is not nil", fields: fields{ - cancel: func() { + cancel: func(err error) { called = true }, }, @@ -640,7 +615,7 @@ func Test_group_doCancel(t *testing.T) { cancel: test.fields.cancel, } - g.doCancel() + g.doCancel(context.Canceled) if err := checkFunc(); err != nil { tt.Errorf("error = %v", err) } @@ -704,9 +679,8 @@ func TestWait(t *testing.T) { func Test_group_Wait(t *testing.T) { type fields struct { - limitation chan struct{} - enableLimitation atomic.Bool - errs []error + sem *semaphore.Weighted + errs []error } type want struct { err error @@ -729,13 +703,8 @@ func Test_group_Wait(t *testing.T) { func() test { var num int32 return test{ - name: "returns nil after all goroutne returns", - fields: fields{ - enableLimitation: func() (el atomic.Bool) { - el.Store(false) - return - }(), - }, + name: "returns nil after all goroutne returns", + fields: fields{}, beforeFunc: func(t *testing.T, g Group) { t.Helper() g.Go(func() error { @@ -782,9 +751,8 @@ func Test_group_Wait(t *testing.T) { checkFunc = defaultCheckFunc } g := &group{ - limitation: test.fields.limitation, - errs: test.fields.errs, - enableLimitation: test.fields.enableLimitation, + sem: test.fields.sem, + errs: test.fields.errs, } if test.beforeFunc != nil { @@ -801,15 +769,524 @@ func Test_group_Wait(t *testing.T) { // NOT IMPLEMENTED BELOW // -// func Test_group_closeLimitation(t *testing.T) { +// func TestWithContext(t *testing.T) { +// type args struct { +// ctx context.Context +// } +// type want struct { +// want Group +// want1 context.Context +// } +// type test struct { +// name string +// args args +// want want +// checkFunc func(want, Group, context.Context) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, got Group, got1 context.Context) error { +// if !reflect.DeepEqual(got, w.want) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", got, w.want) +// } +// if !reflect.DeepEqual(got1, w.want1) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", got1, w.want1) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// ctx:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// ctx:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// +// got, got1 := WithContext(test.args.ctx) +// if err := checkFunc(test.want, got, got1); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } +// +// func TestTryGo(t *testing.T) { +// type args struct { +// f func() error +// } +// type want struct { +// want bool +// } +// type test struct { +// name string +// args args +// want want +// checkFunc func(want, bool) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, got bool) error { +// if !reflect.DeepEqual(got, w.want) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", got, w.want) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// f:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// f:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// +// got := TryGo(test.args.f) +// if err := checkFunc(test.want, got); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } +// +// func Test_group_TryGo(t *testing.T) { +// type args struct { +// f func() error +// } +// type fields struct { +// egctx context.Context +// cancel context.CancelCauseFunc +// sem *semaphore.Weighted +// emap map[string]struct{} +// errs []error +// } +// type want struct { +// want bool +// } +// type test struct { +// name string +// args args +// fields fields +// want want +// checkFunc func(want, bool) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, got bool) error { +// if !reflect.DeepEqual(got, w.want) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", got, w.want) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// f:nil, +// }, +// fields: fields { +// egctx:nil, +// cancel:nil, +// sem:nil, +// emap:nil, +// errs:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// f:nil, +// }, +// fields: fields { +// egctx:nil, +// cancel:nil, +// sem:nil, +// emap:nil, +// errs:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// g := &group{ +// egctx: test.fields.egctx, +// cancel: test.fields.cancel, +// sem: test.fields.sem, +// emap: test.fields.emap, +// errs: test.fields.errs, +// } +// +// got := g.TryGo(test.args.f) +// if err := checkFunc(test.want, got); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } +// +// func Test_group_run(t *testing.T) { +// type args struct { +// f func() error +// } // type fields struct { -// egctx context.Context -// cancel context.CancelFunc -// limitation chan struct{} -// enableLimitation atomic.Bool -// emap map[string]struct{} -// errs []error -// err error +// egctx context.Context +// cancel context.CancelCauseFunc +// sem *semaphore.Weighted +// emap map[string]struct{} +// errs []error +// } +// type want struct { +// } +// type test struct { +// name string +// args args +// fields fields +// want want +// checkFunc func(want) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want) error { +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// f:nil, +// }, +// fields: fields { +// egctx:nil, +// cancel:nil, +// sem:nil, +// emap:nil, +// errs:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// f:nil, +// }, +// fields: fields { +// egctx:nil, +// cancel:nil, +// sem:nil, +// emap:nil, +// errs:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// g := &group{ +// egctx: test.fields.egctx, +// cancel: test.fields.cancel, +// sem: test.fields.sem, +// emap: test.fields.emap, +// errs: test.fields.errs, +// } +// +// g.run(test.args.f) +// if err := checkFunc(test.want); err != nil { +// tt.Errorf("error = %v", err) +// } +// }) +// } +// } +// +// func Test_group_appendErr(t *testing.T) { +// type args struct { +// err error +// } +// type fields struct { +// egctx context.Context +// cancel context.CancelCauseFunc +// sem *semaphore.Weighted +// emap map[string]struct{} +// errs []error +// } +// type want struct { +// } +// type test struct { +// name string +// args args +// fields fields +// want want +// checkFunc func(want) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want) error { +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// err:nil, +// }, +// fields: fields { +// egctx:nil, +// cancel:nil, +// sem:nil, +// emap:nil, +// errs:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// err:nil, +// }, +// fields: fields { +// egctx:nil, +// cancel:nil, +// sem:nil, +// emap:nil, +// errs:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// g := &group{ +// egctx: test.fields.egctx, +// cancel: test.fields.cancel, +// sem: test.fields.sem, +// emap: test.fields.emap, +// errs: test.fields.errs, +// } +// +// g.appendErr(test.args.err) +// if err := checkFunc(test.want); err != nil { +// tt.Errorf("error = %v", err) +// } +// }) +// } +// } +// +// func Test_group_done(t *testing.T) { +// type fields struct { +// egctx context.Context +// cancel context.CancelCauseFunc +// sem *semaphore.Weighted +// emap map[string]struct{} +// errs []error // } // type want struct { // } @@ -832,11 +1309,9 @@ func Test_group_Wait(t *testing.T) { // fields: fields { // egctx:nil, // cancel:nil, -// limitation:nil, -// enableLimitation:nil, +// sem:nil, // emap:nil, // errs:nil, -// err:nil, // }, // want: want{}, // checkFunc: defaultCheckFunc, @@ -857,11 +1332,9 @@ func Test_group_Wait(t *testing.T) { // fields: fields { // egctx:nil, // cancel:nil, -// limitation:nil, -// enableLimitation:nil, +// sem:nil, // emap:nil, // errs:nil, -// err:nil, // }, // want: want{}, // checkFunc: defaultCheckFunc, @@ -892,16 +1365,14 @@ func Test_group_Wait(t *testing.T) { // checkFunc = defaultCheckFunc // } // g := &group{ -// egctx: test.fields.egctx, -// cancel: test.fields.cancel, -// limitation: test.fields.limitation, -// enableLimitation: test.fields.enableLimitation, -// emap: test.fields.emap, -// errs: test.fields.errs, -// err: test.fields.err, +// egctx: test.fields.egctx, +// cancel: test.fields.cancel, +// sem: test.fields.sem, +// emap: test.fields.emap, +// errs: test.fields.errs, // } // -// g.closeLimitation() +// g.done() // if err := checkFunc(test.want); err != nil { // tt.Errorf("error = %v", err) // } diff --git a/internal/sync/semaphore/semaphore.go b/internal/sync/semaphore/semaphore.go new file mode 100644 index 00000000000..1ec284e1903 --- /dev/null +++ b/internal/sync/semaphore/semaphore.go @@ -0,0 +1,173 @@ +// +// Copyright (c) 2009-2023 The Go Authors. All rights resered. +// Modified by vdaas.org vald team +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// + +// Package semaphore provides a weighted semaphore implementation. +package semaphore + +import ( + "container/list" + "context" + + "github.com/vdaas/vald/internal/sync" +) + +type waiter struct { + n int64 + ready chan<- struct{} // Closed when semaphore acquired. +} + +// NewWeighted creates a new weighted semaphore with the given +// maximum combined weight for concurrent access. +func NewWeighted(n int64) *Weighted { + w := &Weighted{size: n} + return w +} + +// Weighted provides a way to bound concurrent access to a resource. +// The callers can request access with a given weight. +type Weighted struct { + size int64 + cur int64 + mu sync.Mutex + waiters list.List +} + +// Acquire acquires the semaphore with a weight of n, blocking until resources +// are available or ctx is done. On success, returns nil. On failure, returns +// ctx.Err() and leaves the semaphore unchanged. +// +// If ctx is already done, Acquire may still succeed without blocking. +func (s *Weighted) Acquire(ctx context.Context, n int64) error { + s.mu.Lock() + if s.size-s.cur >= n && s.waiters.Len() == 0 { + s.cur += n + s.mu.Unlock() + return nil + } + + if n > s.size { + // Don't make other Acquire calls block on one that's doomed to fail. + s.mu.Unlock() + <-ctx.Done() + return ctx.Err() + } + + ready := make(chan struct{}) + w := waiter{n: n, ready: ready} + elem := s.waiters.PushBack(w) + s.mu.Unlock() + + select { + case <-ctx.Done(): + err := ctx.Err() + s.mu.Lock() + select { + case <-ready: + // Acquired the semaphore after we were canceled. Rather than trying to + // fix up the queue, just pretend we didn't notice the cancelation. + err = nil + default: + isFront := s.waiters.Front() == elem + s.waiters.Remove(elem) + // If we're at the front and there're extra tokens left, notify other waiters. + if isFront && s.size > s.cur { + s.notifyWaiters() + } + } + s.mu.Unlock() + return err + + case <-ready: + return nil + } +} + +// TryAcquire acquires the semaphore with a weight of n without blocking. +// On success, returns true. On failure, returns false and leaves the semaphore unchanged. +func (s *Weighted) TryAcquire(n int64) bool { + s.mu.Lock() + success := s.size-s.cur >= n && s.waiters.Len() == 0 + if success { + s.cur += n + } + s.mu.Unlock() + return success +} + +// Release releases the semaphore with a weight of n. +func (s *Weighted) Release(n int64) { + s.mu.Lock() + s.cur -= n + if s.cur < 0 { + s.mu.Unlock() + panic("semaphore: released more than held") + } + s.notifyWaiters() + s.mu.Unlock() +} + +func (s *Weighted) Resize(n int64) { + if n < 0 { + n = 0 + } + s.mu.Lock() + s.size = n + s.mu.Unlock() +} + +func (s *Weighted) notifyWaiters() { + for { + next := s.waiters.Front() + if next == nil { + break // No more waiters blocked. + } + + w := next.Value.(waiter) + if s.size-s.cur < w.n { + // Not enough tokens for the next waiter. We could keep going (to try to + // find a waiter with a smaller request), but under load that could cause + // starvation for large requests; instead, we leave all remaining waiters + // blocked. + // + // Consider a semaphore used as a read-write lock, with N tokens, N + // readers, and one writer. Each reader can Acquire(1) to obtain a read + // lock. The writer can Acquire(N) to obtain a write lock, excluding all + // of the readers. If we allow the readers to jump ahead in the queue, + // the writer will starve — there is always one token available for every + // reader. + break + } + + s.cur += w.n + s.waiters.Remove(next) + close(w.ready) + } +} diff --git a/internal/sync/semaphore/semaphore_bench_test.go b/internal/sync/semaphore/semaphore_bench_test.go new file mode 100644 index 00000000000..3bab8eb893a --- /dev/null +++ b/internal/sync/semaphore/semaphore_bench_test.go @@ -0,0 +1,159 @@ +// +// Copyright (c) 2009-2023 The Go Authors. All rights resered. +// Modified by vdaas.org vald team +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// + +//go:build go1.7 +// +build go1.7 + +package semaphore_test + +import ( + "context" + "fmt" + "testing" + + "github.com/vdaas/vald/internal/sync/semaphore" +) + +// weighted is an interface matching a subset of *Weighted. It allows +// alternate implementations for testing and benchmarking. +type weighted interface { + Acquire(context.Context, int64) error + TryAcquire(int64) bool + Release(int64) +} + +// semChan implements Weighted using a channel for +// comparing against the condition variable-based implementation. +type semChan chan struct{} + +func newSemChan(n int64) semChan { + return semChan(make(chan struct{}, n)) +} + +func (s semChan) Acquire(_ context.Context, n int64) error { + for i := int64(0); i < n; i++ { + s <- struct{}{} + } + return nil +} + +func (s semChan) TryAcquire(n int64) bool { + if int64(len(s))+n > int64(cap(s)) { + return false + } + + for i := int64(0); i < n; i++ { + s <- struct{}{} + } + return true +} + +func (s semChan) Release(n int64) { + for i := int64(0); i < n; i++ { + <-s + } +} + +// acquireN calls Acquire(size) on sem N times and then calls Release(size) N times. +func acquireN(b *testing.B, sem weighted, size int64, N int) { + b.ResetTimer() + for i := 0; i < b.N; i++ { + for j := 0; j < N; j++ { + sem.Acquire(context.Background(), size) + } + for j := 0; j < N; j++ { + sem.Release(size) + } + } +} + +// tryAcquireN calls TryAcquire(size) on sem N times and then calls Release(size) N times. +func tryAcquireN(b *testing.B, sem weighted, size int64, N int) { + b.ResetTimer() + for i := 0; i < b.N; i++ { + for j := 0; j < N; j++ { + if !sem.TryAcquire(size) { + b.Fatalf("TryAcquire(%v) = false, want true", size) + } + } + for j := 0; j < N; j++ { + sem.Release(size) + } + } +} + +func BenchmarkNewSeq(b *testing.B) { + for _, cap := range []int64{1, 128} { + b.Run(fmt.Sprintf("Weighted-%d", cap), func(b *testing.B) { + for i := 0; i < b.N; i++ { + _ = semaphore.NewWeighted(cap) + } + }) + b.Run(fmt.Sprintf("semChan-%d", cap), func(b *testing.B) { + for i := 0; i < b.N; i++ { + _ = newSemChan(cap) + } + }) + } +} + +func BenchmarkAcquireSeq(b *testing.B) { + for _, c := range []struct { + cap, size int64 + N int + }{ + {1, 1, 1}, + {2, 1, 1}, + {16, 1, 1}, + {128, 1, 1}, + {2, 2, 1}, + {16, 2, 8}, + {128, 2, 64}, + {2, 1, 2}, + {16, 8, 2}, + {128, 64, 2}, + } { + for _, w := range []struct { + name string + w weighted + }{ + {"Weighted", semaphore.NewWeighted(c.cap)}, + {"semChan", newSemChan(c.cap)}, + } { + b.Run(fmt.Sprintf("%s-acquire-%d-%d-%d", w.name, c.cap, c.size, c.N), func(b *testing.B) { + acquireN(b, w.w, c.size, c.N) + }) + b.Run(fmt.Sprintf("%s-tryAcquire-%d-%d-%d", w.name, c.cap, c.size, c.N), func(b *testing.B) { + tryAcquireN(b, w.w, c.size, c.N) + }) + } + } +} diff --git a/internal/sync/semaphore/semaphore_example_test.go b/internal/sync/semaphore/semaphore_example_test.go new file mode 100644 index 00000000000..abc5798736b --- /dev/null +++ b/internal/sync/semaphore/semaphore_example_test.go @@ -0,0 +1,111 @@ +// +// Copyright (c) 2009-2023 The Go Authors. All rights resered. +// Modified by vdaas.org vald team +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// + +package semaphore_test + +import ( + "context" + "fmt" + "log" + "runtime" + + "github.com/vdaas/vald/internal/sync/semaphore" +) + +// Example_workerPool demonstrates how to use a semaphore to limit the number of +// goroutines working on parallel tasks. +// +// This use of a semaphore mimics a typical “worker pool” pattern, but without +// the need to explicitly shut down idle workers when the work is done. +func Example_workerPool() { + ctx := context.TODO() + + var ( + maxWorkers = runtime.GOMAXPROCS(0) + sem = semaphore.NewWeighted(int64(maxWorkers)) + out = make([]int, 32) + ) + + // Compute the output using up to maxWorkers goroutines at a time. + for i := range out { + // When maxWorkers goroutines are in flight, Acquire blocks until one of the + // workers finishes. + if err := sem.Acquire(ctx, 1); err != nil { + log.Printf("Failed to acquire semaphore: %v", err) + break + } + + go func(i int) { + defer sem.Release(1) + out[i] = collatzSteps(i + 1) + }(i) + } + + // Acquire all of the tokens to wait for any remaining workers to finish. + // + // If you are already waiting for the workers by some other means (such as an + // errgroup.Group), you can omit this final Acquire call. + if err := sem.Acquire(ctx, int64(maxWorkers)); err != nil { + log.Printf("Failed to acquire semaphore: %v", err) + } + + fmt.Println(out) + + // Output: + // [0 1 7 2 5 8 16 3 19 6 14 9 9 17 17 4 12 20 20 7 7 15 15 10 23 10 111 18 18 18 106 5] +} + +// collatzSteps computes the number of steps to reach 1 under the Collatz +// conjecture. (See https://en.wikipedia.org/wiki/Collatz_conjecture.) +func collatzSteps(n int) (steps int) { + if n <= 0 { + panic("nonpositive input") + } + + for ; n > 1; steps++ { + if steps < 0 { + panic("too many steps") + } + + if n%2 == 0 { + n /= 2 + continue + } + + const maxInt = int(^uint(0) >> 1) + if n > (maxInt-1)/3 { + panic("overflow") + } + n = 3*n + 1 + } + + return steps +} diff --git a/internal/sync/semaphore/semaphore_test.go b/internal/sync/semaphore/semaphore_test.go new file mode 100644 index 00000000000..6290b6affa7 --- /dev/null +++ b/internal/sync/semaphore/semaphore_test.go @@ -0,0 +1,830 @@ +// +// Copyright (c) 2009-2023 The Go Authors. All rights resered. +// Modified by vdaas.org vald team +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// + +package semaphore_test + +import ( + "context" + "math/rand" + "runtime" + "testing" + "time" + + "github.com/vdaas/vald/internal/sync" + "github.com/vdaas/vald/internal/sync/errgroup" + "github.com/vdaas/vald/internal/sync/semaphore" +) + +const maxSleep = 1 * time.Millisecond + +func HammerWeighted(sem *semaphore.Weighted, n int64, loops int) { + for i := 0; i < loops; i++ { + sem.Acquire(context.Background(), n) + time.Sleep(time.Duration(rand.Int63n(int64(maxSleep/time.Nanosecond))) * time.Nanosecond) + sem.Release(n) + } +} + +func TestWeighted(t *testing.T) { + t.Parallel() + + n := runtime.GOMAXPROCS(0) + loops := 10000 / n + sem := semaphore.NewWeighted(int64(n)) + var wg sync.WaitGroup + wg.Add(n) + for i := 0; i < n; i++ { + i := i + go func() { + defer wg.Done() + HammerWeighted(sem, int64(i), loops) + }() + } + wg.Wait() +} + +func TestWeightedPanic(t *testing.T) { + t.Parallel() + + defer func() { + if recover() == nil { + t.Fatal("release of an unacquired weighted semaphore did not panic") + } + }() + w := semaphore.NewWeighted(1) + w.Release(1) +} + +func TestWeightedTryAcquire(t *testing.T) { + t.Parallel() + + ctx := context.Background() + sem := semaphore.NewWeighted(2) + tries := []bool{} + sem.Acquire(ctx, 1) + tries = append(tries, sem.TryAcquire(1)) + tries = append(tries, sem.TryAcquire(1)) + + sem.Release(2) + + tries = append(tries, sem.TryAcquire(1)) + sem.Acquire(ctx, 1) + tries = append(tries, sem.TryAcquire(1)) + + want := []bool{true, false, true, false} + for i := range tries { + if tries[i] != want[i] { + t.Errorf("tries[%d]: got %t, want %t", i, tries[i], want[i]) + } + } +} + +func TestWeightedAcquire(t *testing.T) { + t.Parallel() + + ctx := context.Background() + sem := semaphore.NewWeighted(2) + tryAcquire := func(n int64) bool { + ctx, cancel := context.WithTimeout(ctx, 10*time.Millisecond) + defer cancel() + return sem.Acquire(ctx, n) == nil + } + + tries := []bool{} + sem.Acquire(ctx, 1) + tries = append(tries, tryAcquire(1)) + tries = append(tries, tryAcquire(1)) + + sem.Release(2) + + tries = append(tries, tryAcquire(1)) + sem.Acquire(ctx, 1) + tries = append(tries, tryAcquire(1)) + + want := []bool{true, false, true, false} + for i := range tries { + if tries[i] != want[i] { + t.Errorf("tries[%d]: got %t, want %t", i, tries[i], want[i]) + } + } +} + +func TestWeightedDoesntBlockIfTooBig(t *testing.T) { + t.Parallel() + + const n = 2 + sem := semaphore.NewWeighted(n) + { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + go sem.Acquire(ctx, n+1) + } + + g, ctx := errgroup.WithContext(context.Background()) + for i := n * 3; i > 0; i-- { + g.Go(func() error { + err := sem.Acquire(ctx, 1) + if err == nil { + time.Sleep(1 * time.Millisecond) + sem.Release(1) + } + return err + }) + } + if err := g.Wait(); err != nil { + t.Errorf("semaphore.NewWeighted(%v) failed to AcquireCtx(_, 1) with AcquireCtx(_, %v) pending", n, n+1) + } +} + +// TestLargeAcquireDoesntStarve times out if a large call to Acquire starves. +// Merely returning from the test function indicates success. +func TestLargeAcquireDoesntStarve(t *testing.T) { + t.Parallel() + + ctx := context.Background() + n := int64(runtime.GOMAXPROCS(0)) + sem := semaphore.NewWeighted(n) + running := true + + var wg sync.WaitGroup + wg.Add(int(n)) + for i := n; i > 0; i-- { + sem.Acquire(ctx, 1) + go func() { + defer func() { + sem.Release(1) + wg.Done() + }() + for running { + time.Sleep(1 * time.Millisecond) + sem.Release(1) + sem.Acquire(ctx, 1) + } + }() + } + + sem.Acquire(ctx, n) + running = false + sem.Release(n) + wg.Wait() +} + +// translated from https://github.com/zhiqiangxu/util/blob/master/mutex/crwmutex_test.go#L43 +func TestAllocCancelDoesntStarve(t *testing.T) { + sem := semaphore.NewWeighted(10) + + // Block off a portion of the semaphore so that Acquire(_, 10) can eventually succeed. + sem.Acquire(context.Background(), 1) + + // In the background, Acquire(_, 10). + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + go func() { + sem.Acquire(ctx, 10) + }() + + // Wait until the Acquire(_, 10) call blocks. + for sem.TryAcquire(1) { + sem.Release(1) + runtime.Gosched() + } + + // Now try to grab a read lock, and simultaneously unblock the Acquire(_, 10) call. + // Both Acquire calls should unblock and return, in either order. + go cancel() + + err := sem.Acquire(context.Background(), 1) + if err != nil { + t.Fatalf("Acquire(_, 1) failed unexpectedly: %v", err) + } + sem.Release(1) +} + +// NOT IMPLEMENTED BELOW +// +// func TestNewWeighted(t *testing.T) { +// type args struct { +// n int64 +// } +// type want struct { +// want *Weighted +// } +// type test struct { +// name string +// args args +// want want +// checkFunc func(want, *Weighted) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, got *Weighted) error { +// if !reflect.DeepEqual(got, w.want) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", got, w.want) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// n:0, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// n:0, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// +// got := NewWeighted(test.args.n) +// if err := checkFunc(test.want, got); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } +// +// func TestWeighted_Acquire(t *testing.T) { +// type args struct { +// ctx context.Context +// n int64 +// } +// type fields struct { +// size int64 +// cur int64 +// waiters list.List +// } +// type want struct { +// err error +// } +// type test struct { +// name string +// args args +// fields fields +// want want +// checkFunc func(want, error) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, err error) error { +// if !errors.Is(err, w.err) { +// return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// ctx:nil, +// n:0, +// }, +// fields: fields { +// size:0, +// cur:0, +// waiters:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// ctx:nil, +// n:0, +// }, +// fields: fields { +// size:0, +// cur:0, +// waiters:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// s := &Weighted{ +// size: test.fields.size, +// cur: test.fields.cur, +// waiters: test.fields.waiters, +// } +// +// err := s.Acquire(test.args.ctx, test.args.n) +// if err := checkFunc(test.want, err); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } +// +// func TestWeighted_TryAcquire(t *testing.T) { +// type args struct { +// n int64 +// } +// type fields struct { +// size int64 +// cur int64 +// waiters list.List +// } +// type want struct { +// want bool +// } +// type test struct { +// name string +// args args +// fields fields +// want want +// checkFunc func(want, bool) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, got bool) error { +// if !reflect.DeepEqual(got, w.want) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", got, w.want) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// n:0, +// }, +// fields: fields { +// size:0, +// cur:0, +// waiters:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// n:0, +// }, +// fields: fields { +// size:0, +// cur:0, +// waiters:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// s := &Weighted{ +// size: test.fields.size, +// cur: test.fields.cur, +// waiters: test.fields.waiters, +// } +// +// got := s.TryAcquire(test.args.n) +// if err := checkFunc(test.want, got); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } +// +// func TestWeighted_Release(t *testing.T) { +// type args struct { +// n int64 +// } +// type fields struct { +// size int64 +// cur int64 +// waiters list.List +// } +// type want struct { +// } +// type test struct { +// name string +// args args +// fields fields +// want want +// checkFunc func(want) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want) error { +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// n:0, +// }, +// fields: fields { +// size:0, +// cur:0, +// waiters:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// n:0, +// }, +// fields: fields { +// size:0, +// cur:0, +// waiters:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// s := &Weighted{ +// size: test.fields.size, +// cur: test.fields.cur, +// waiters: test.fields.waiters, +// } +// +// s.Release(test.args.n) +// if err := checkFunc(test.want); err != nil { +// tt.Errorf("error = %v", err) +// } +// }) +// } +// } +// +// func TestWeighted_Resize(t *testing.T) { +// type args struct { +// n int64 +// } +// type fields struct { +// size int64 +// cur int64 +// waiters list.List +// } +// type want struct { +// } +// type test struct { +// name string +// args args +// fields fields +// want want +// checkFunc func(want) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want) error { +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// n:0, +// }, +// fields: fields { +// size:0, +// cur:0, +// waiters:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// n:0, +// }, +// fields: fields { +// size:0, +// cur:0, +// waiters:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// s := &Weighted{ +// size: test.fields.size, +// cur: test.fields.cur, +// waiters: test.fields.waiters, +// } +// +// s.Resize(test.args.n) +// if err := checkFunc(test.want); err != nil { +// tt.Errorf("error = %v", err) +// } +// }) +// } +// } +// +// func TestWeighted_notifyWaiters(t *testing.T) { +// type fields struct { +// size int64 +// cur int64 +// waiters list.List +// } +// type want struct { +// } +// type test struct { +// name string +// fields fields +// want want +// checkFunc func(want) error +// beforeFunc func(*testing.T) +// afterFunc func(*testing.T) +// } +// defaultCheckFunc := func(w want) error { +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// fields: fields { +// size:0, +// cur:0, +// waiters:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T,) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T,) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// fields: fields { +// size:0, +// cur:0, +// waiters:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T,) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T,) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// s := &Weighted{ +// size: test.fields.size, +// cur: test.fields.cur, +// waiters: test.fields.waiters, +// } +// +// s.notifyWaiters() +// if err := checkFunc(test.want); err != nil { +// tt.Errorf("error = %v", err) +// } +// }) +// } +// } diff --git a/internal/sync/singleflight/singleflight.go b/internal/sync/singleflight/singleflight.go new file mode 100644 index 00000000000..c20730ab9f8 --- /dev/null +++ b/internal/sync/singleflight/singleflight.go @@ -0,0 +1,240 @@ +// +// Copyright (c) 2009-2023 The Go Authors. All rights resered. +// Modified by vdaas.org vald team +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// + +// Package singleflight provides a duplicate function call suppression +// mechanism. +package singleflight + +import ( + "bytes" + "context" + "errors" + "fmt" + "runtime/debug" + "sync/atomic" + + "github.com/vdaas/vald/internal/sync" +) + +// errGoexit indicates the runtime.Goexit was called in +// the user given function. +var errGoexit = errors.New("runtime.Goexit was called") + +// A panicError is an arbitrary value recovered from a panic +// with the stack trace during the execution of given function. +type panicError struct { + value interface{} + stack []byte +} + +// Error implements error interface. +func (p *panicError) Error() string { + return fmt.Sprintf("%v\n\n%s", p.value, p.stack) +} + +func newPanicError(v interface{}) error { + stack := debug.Stack() + + // The first line of the stack trace is of the form "goroutine N [status]:" + // but by the time the panic reaches Do the goroutine may no longer exist + // and its status will have changed. Trim out the misleading line. + if line := bytes.IndexByte(stack[:], '\n'); line >= 0 { + stack = stack[line+1:] + } + return &panicError{value: v, stack: stack} +} + +// call is an in-flight or completed singleflight.Do call +type call[V any] struct { + mu sync.Mutex + + // These fields are written once before the WaitGroup is done + // and are only read after the WaitGroup is done. + val V + err error + + // These fields are read and written with the singleflight + // mutex held before the WaitGroup is done, and are read but + // not written after the WaitGroup is done. + dups uint64 + chans []chan<- Result[V] + + chmu sync.RWMutex +} + +func (c *call[V]) appendChans(chs ...chan<- Result[V]) { + c.chmu.Lock() + defer c.chmu.Unlock() + if len(c.chans) == 0 { + c.chans = make([]chan<- Result[V], 0, len(chs)) + } + c.chans = append(c.chans, chs...) +} + +// Group represents interface for zero time cache. +type Group[V any] interface { + Do(ctx context.Context, key string, fn func(context.Context) (V, error)) (v V, shared bool, err error) + DoChan(ctx context.Context, key string, fn func(context.Context) (V, error)) <-chan Result[V] + Forget(key string) +} + +// group represents a class of work and forms a namespace in +// which units of work can be executed with duplicate suppression. +type group[V any] struct { + m sync.Map[string, *call[V]] +} + +// Result holds the results of Do, so they can be passed +// on a channel. +type Result[V any] struct { + Val V + Err error + Shared bool +} + +// New returns Group implementation. +func New[V any]() Group[V] { + return new(group[V]) +} + +// Do executes and returns the results of the given function, making +// sure that only one execution is in-flight for a given key at a +// time. If a duplicate comes in, the duplicate caller waits for the +// original to complete and receives the same results. +// The return value shared indicates whether v was given to multiple callers. +func (g *group[V]) Do(ctx context.Context, key string, fn func(context.Context) (V, error)) (v V, shared bool, err error) { + c, loaded := g.m.LoadOrStore(key, new(call[V])) + if loaded { + atomic.AddUint64(&c.dups, 1) + c.mu.Lock() + c.mu.Unlock() + v, err = c.val, c.err + return v, true, err + } + c.mu.Lock() + g.doCall(ctx, c, key, fn) + c.mu.Unlock() + return c.val, atomic.LoadUint64(&c.dups) > 0, c.err +} + +// DoChan is like Do but returns a channel that will receive the +// results when they are ready. +// +// The returned channel will not be closed. +func (g *group[V]) DoChan(ctx context.Context, key string, fn func(context.Context) (V, error)) <-chan Result[V] { + c, loaded := g.m.LoadOrStore(key, new(call[V])) + ch := make(chan Result[V], 1) + c.appendChans(ch) + if loaded { + atomic.AddUint64(&c.dups, 1) + return ch + } + go g.doCall(ctx, c, key, fn) + + return ch +} + +// doCall handles the single call for a key. +func (g *group[V]) doCall(ctx context.Context, c *call[V], key string, fn func(ctx context.Context) (V, error)) { + normalReturn := false + recovered := false + + // use double-defer to distinguish panic from runtime.Goexit, + // more details see https://golang.org/cl/134395 + defer func() { + // the given function invoked runtime.Goexit + if !normalReturn && !recovered { + c.err = errGoexit + } + + g.m.LoadAndDelete(key) + if e, ok := c.err.(*panicError); ok { + // In order to prevent the waiting channels from being blocked forever, + // needs to ensure that this panic cannot be recovered. + c.chmu.RLock() + defer c.chmu.RUnlock() + if len(c.chans) > 0 { + go panic(e) + select {} // Keep this goroutine around so that it will appear in the crash dump. + } else { + panic(e) + } + } else if c.err == errGoexit { + // Already in the process of goexit, no need to call again + } else { + // Normal return + c.chmu.RLock() + defer c.chmu.RUnlock() + for _, ch := range c.chans { + select { + case <-ctx.Done(): + return + case ch <- Result[V]{ + Val: c.val, + Err: c.err, + Shared: atomic.LoadUint64(&c.dups) > 0, + }: + } + } + } + }() + + func() { + defer func() { + if !normalReturn { + // Ideally, we would wait to take a stack trace until we've determined + // whether this is a panic or a runtime.Goexit. + // + // Unfortunately, the only way we can distinguish the two is to see + // whether the recover stopped the goroutine from terminating, and by + // the time we know that, the part of the stack trace relevant to the + // panic has been discarded. + if r := recover(); r != nil { + c.err = newPanicError(r) + } + } + }() + + c.val, c.err = fn(ctx) + normalReturn = true + }() + + if !normalReturn { + recovered = true + } +} + +// Forget tells the singleflight to forget about a key. Future calls +// to Do for this key will call the function rather than waiting for +// an earlier call to complete. +func (g *group[V]) Forget(key string) { + g.m.LoadAndDelete(key) +} diff --git a/internal/sync/singleflight/singleflight_test.go b/internal/sync/singleflight/singleflight_test.go new file mode 100644 index 00000000000..28ace0609f3 --- /dev/null +++ b/internal/sync/singleflight/singleflight_test.go @@ -0,0 +1,640 @@ +// +// Copyright (C) 2019-2023 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +// Package singleflight represents zero time caching +package singleflight + +import ( + "bytes" + "context" + "fmt" + "os" + "os/exec" + "reflect" + "runtime" + "runtime/debug" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/vdaas/vald/internal/errors" + "github.com/vdaas/vald/internal/sync" + "github.com/vdaas/vald/internal/test/goleak" +) + +func TestNew(t *testing.T) { + type want struct { + want Group[any] + } + type test struct { + name string + want want + checkFunc func(want, Group[any]) error + beforeFunc func() + afterFunc func() + } + defaultCheckFunc := func(w want, got Group[any]) error { + if !reflect.DeepEqual(got, w.want) { + return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", got, w.want) + } + return nil + } + tests := []test{ + { + name: "returns Group implementation", + want: want{ + want: &group[any]{}, + }, + }, + } + + for _, tc := range tests { + test := tc + t.Run(test.name, func(tt *testing.T) { + defer goleak.VerifyNone(tt) + if test.beforeFunc != nil { + test.beforeFunc() + } + if test.afterFunc != nil { + defer test.afterFunc() + } + checkFunc := test.checkFunc + if test.checkFunc == nil { + checkFunc = defaultCheckFunc + } + + got := New[any]() + if err := checkFunc(test.want, got); err != nil { + tt.Errorf("error = %v", err) + } + }) + } +} + +func Test_group_Do(t *testing.T) { + type args[V any] struct { + ctx context.Context + key string + fn func(context.Context) (V, error) + } + type want[V any] struct { + wantV V + wantShared bool + err error + } + type test[V any] struct { + name string + args args[V] + want want[V] + beforeFunc func(args[V]) + execFunc func(*testing.T, args[V]) (V, bool, error) + checkFunc func(want[V], V, bool, error) error + afterFunc func(args[V]) + } + tests := []test[string]{ + func() test[string] { + // routine1 + key1 := "req_1" + var cnt1 uint32 + + // the unparam lint rule is disabled here because we need to match the interface to singleflight implementation. + // if this rule is not disabled, if will warns that the error will always return null. + //nolint:unparam + fn1 := func(context.Context) (string, error) { + atomic.AddUint32(&cnt1, 1) + return "res_1", nil + } + + // routine 2 + key2 := "req_2" + var cnt2 uint32 + + // the unparam lint rule is disabled here because we need to match the interface to singleflight implementation. + // if this rule is not disabled, if will warns that the error will always return null. + //nolint:unparam + fn2 := func(context.Context) (string, error) { + atomic.AddUint32(&cnt2, 1) + return "res_2", nil + } + + return test[string]{ + name: "returns (v, false, nil) when Do is called with another key", + args: args[string]{ + key: key1, + ctx: context.Background(), + fn: fn1, + }, + want: want[string]{ + wantV: "res_1", + wantShared: false, + err: nil, + }, + execFunc: func(t *testing.T, a args[string]) (got string, gotShared bool, err error) { + t.Helper() + g := New[string]() + + wg := new(sync.WaitGroup) + wg.Add(1) + go func() { + got, gotShared, err = g.Do(a.ctx, a.key, a.fn) + wg.Done() + }() + + wg.Add(1) + go func() { + _, _, _ = g.Do(a.ctx, key2, fn2) + wg.Done() + }() + + wg.Wait() + return got, gotShared, err + }, + checkFunc: func(w want[string], gotV string, gotShared bool, err error) error { + if got, want := int(atomic.LoadUint32(&cnt1)), 1; got != want { + return errors.Errorf("cnt got = %d, want = %d", got, want) + } + if got, want := int(atomic.LoadUint32(&cnt2)), 1; got != want { + return errors.Errorf("cnt got = %d, want = %d", got, want) + } + if !errors.Is(err, w.err) { + return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err) + } + if !reflect.DeepEqual(gotV, w.wantV) { + return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", gotV, w.wantV) + } + if !reflect.DeepEqual(gotShared, w.wantShared) { + return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", gotShared, w.wantShared) + } + return nil + }, + } + }(), + func() test[string] { + // routine1 + var cnt1 uint32 + + // the unparam lint rule is disabled here because we need to match the interface to singleflight implementation. + // if this rule is not disabled, if will warns that the error will always return null. + //nolint:unparam + fn1 := func(context.Context) (string, error) { + atomic.AddUint32(&cnt1, 1) + time.Sleep(time.Millisecond * 500) + return "res_1", nil + } + + // routine 2 + var cnt2 uint32 + + // the unparam lint rule is disabled here because we need to match the interface to singleflight implementation. + // if this rule is not disabled, if will warns that the error will always return null. + //nolint:unparam + fn2 := func(context.Context) (string, error) { + atomic.AddUint32(&cnt2, 1) + return "res_2", nil + } + + w := want[string]{ + wantV: "res_1", + wantShared: true, + err: nil, + } + + checkFunc := func(w want[string], gotV string, gotShared bool, err error) error { + c1 := int(atomic.LoadUint32(&cnt1)) + c2 := int(atomic.LoadUint32(&cnt2)) + // since there is a chance that the go routine 2 is executed before routine 1, we need to check if either one is executed + if !((c1 == 1 && c2 == 0) || (c1 == 0 && c2 == 1)) { + return errors.Errorf("cnt1 and cnt2 is executed, %d, %d", c1, c2) + } + if !errors.Is(err, w.err) { + return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err) + } + if !reflect.DeepEqual(gotV, w.wantV) { + return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", gotV, w.wantV) + } + if !reflect.DeepEqual(gotShared, w.wantShared) { + return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", gotShared, w.wantShared) + } + return nil + } + + return test[string]{ + name: "returns (v, true, nil) when Do is called with the same key", + args: args[string]{ + key: "req_1", + ctx: context.Background(), + fn: fn1, + }, + want: w, + execFunc: func(t *testing.T, a args[string]) (string, bool, error) { + t.Helper() + + g := New[string]() + wg := new(sync.WaitGroup) + var got, got1 string + var gotShared, gotShared1 bool + var err, err1 error + + wg.Add(1) + go func() { + got, gotShared, err = g.Do(a.ctx, a.key, fn1) + wg.Done() + }() + + // call with the same key but with another function + wg.Add(1) + time.Sleep(time.Millisecond * 100) + go func() { + got1, gotShared1, err1 = g.Do(a.ctx, a.key, fn2) + wg.Done() + }() + + wg.Wait() + + if err := checkFunc(w, got1, gotShared1, err1); err != nil { + t.Fatal(err) + } + + return got, gotShared, err + }, + checkFunc: checkFunc, + } + }(), + } + + for _, tc := range tests { + test := tc + t.Run(test.name, func(tt *testing.T) { + defer goleak.VerifyNone(tt) + if test.afterFunc != nil { + defer test.afterFunc(test.args) + } + if test.beforeFunc != nil { + test.beforeFunc(test.args) + } + + gotV, gotShared, err := test.execFunc(t, test.args) + + if err := test.checkFunc(test.want, gotV, gotShared, err); err != nil { + tt.Errorf("error = %v", err) + } + }) + } +} + +func TestDo(t *testing.T) { + g := New[string]() + v, _, err := g.Do(context.Background(), "key", func(context.Context) (string, error) { + return "bar", nil + }) + if got, want := fmt.Sprintf("%v (%T)", v, v), "bar (string)"; got != want { + t.Errorf("Do = %v; want %v", got, want) + } + if err != nil { + t.Errorf("Do error = %v", err) + } +} + +func TestDoErr(t *testing.T) { + g := New[any]() + someErr := errors.New("Some error") + v, _, err := g.Do(context.Background(), "key", func(context.Context) (any, error) { + return nil, someErr + }) + if err != someErr { + t.Errorf("Do error = %v; want someErr %v", err, someErr) + } + if v != nil { + t.Errorf("unexpected non-nil value %#v", v) + } +} + +func TestDoDupSuppress(t *testing.T) { + g := New[string]() + var wg1, wg2 sync.WaitGroup + c := make(chan string, 1) + var calls int32 + fn := func(ctx context.Context) (string, error) { + if atomic.AddInt32(&calls, 1) == 1 { + // First invocation. + wg1.Done() + } + v := <-c + c <- v // pump; make available for any future calls + + time.Sleep(10 * time.Millisecond) // let more goroutines enter Do + + return v, nil + } + + const n = 10 + wg1.Add(1) + for i := 0; i < n; i++ { + wg1.Add(1) + wg2.Add(1) + go func() { + defer wg2.Done() + wg1.Done() + s, _, err := g.Do(context.Background(), "key", fn) + if err != nil { + t.Errorf("Do error: %v", err) + return + } + if s != "bar" { + t.Errorf("Do = %T %v; want %q", s, s, "bar") + } + }() + } + wg1.Wait() + // At least one goroutine is in fn now and all of them have at + // least reached the line before the Do. + c <- "bar" + wg2.Wait() + if got := atomic.LoadInt32(&calls); got <= 0 || got >= n { + t.Errorf("number of calls = %d; want over 0 and less than %d", got, n) + } +} + +// Test that singleflight behaves correctly after Forget called. +// See https://github.com/golang/go/issues/31420 +func TestForget(t *testing.T) { + g := New[int]() + + var ( + firstStarted = make(chan struct{}) + unblockFirst = make(chan struct{}) + firstFinished = make(chan struct{}) + ) + + go func() { + g.Do(context.Background(), "key", func(ctx context.Context) (i int, e error) { + close(firstStarted) + <-unblockFirst + close(firstFinished) + return + }) + }() + <-firstStarted + g.Forget("key") + + unblockSecond := make(chan struct{}) + secondResult := g.DoChan(context.Background(), "key", func(ctx context.Context) (i int, e error) { + t.Log(2, "key") + <-unblockSecond + return 2, nil + }) + + close(unblockFirst) + <-firstFinished + + thirdResult := g.DoChan(context.Background(), "key", func(ctx context.Context) (i int, e error) { + t.Log(3, "key") + return 3, nil + }) + + close(unblockSecond) + <-secondResult + r := <-thirdResult + if r.Val != 2 { + t.Errorf("We should receive result produced by second call, expected: 2, got %d", r.Val) + } +} + +func TestDoChan(t *testing.T) { + g := New[string]() + ch := g.DoChan(context.Background(), "key", func(ctx context.Context) (string, error) { + return "bar", nil + }) + + res := <-ch + v := res.Val + err := res.Err + if got, want := fmt.Sprintf("%v (%T)", v, v), "bar (string)"; got != want { + t.Errorf("Do = %v; want %v", got, want) + } + if err != nil { + t.Errorf("Do error = %v", err) + } +} + +// Test singleflight behaves correctly after Do panic. +// See https://github.com/golang/go/issues/41133 +func TestPanicDo(t *testing.T) { + g := New[any]() + fn := func(ctx context.Context) (any, error) { + panic("invalid memory address or nil pointer dereference") + } + + const n = 5 + waited := int32(n) + panicCount := int32(0) + done := make(chan struct{}) + for i := 0; i < n; i++ { + go func() { + defer func() { + if err := recover(); err != nil { + t.Logf("Got panic: %v\n%s", err, debug.Stack()) + atomic.AddInt32(&panicCount, 1) + } + + if atomic.AddInt32(&waited, -1) == 0 { + close(done) + } + }() + + g.Do(context.Background(), "key", fn) + }() + } + + select { + case <-done: + if panicCount != n { + t.Errorf("Expect %d panic, but got %d", n, panicCount) + } + case <-time.After(time.Second): + t.Fatalf("Do hangs") + } +} + +func TestGoexitDo(t *testing.T) { + g := New[any]() + fn := func(context.Context) (interface{}, error) { + runtime.Goexit() + return nil, nil + } + + const n = 5 + waited := int32(n) + done := make(chan struct{}) + for i := 0; i < n; i++ { + go func() { + var err error + defer func() { + if err != nil { + t.Errorf("Error should be nil, but got: %v", err) + } + if atomic.AddInt32(&waited, -1) == 0 { + close(done) + } + }() + _, _, err = g.Do(context.Background(), "key", fn) + }() + } + + select { + case <-done: + case <-time.After(time.Second): + t.Fatalf("Do hangs") + } +} + +func executable(t testing.TB) string { + exe, err := os.Executable() + if err != nil { + t.Skipf("skipping: test executable not found") + } + + // Control case: check whether exec.Command works at all. + // (For example, it might fail with a permission error on iOS.) + cmd := exec.Command(exe, "-test.list=^$") + cmd.Env = []string{} + if err := cmd.Run(); err != nil { + t.Skipf("skipping: exec appears not to work on %s: %v", runtime.GOOS, err) + } + + return exe +} + +func TestPanicDoChan(t *testing.T) { + if os.Getenv("TEST_PANIC_DOCHAN") != "" { + defer func() { + recover() + }() + + g := New[any]() + ch := g.DoChan(context.Background(), "", func(context.Context) (interface{}, error) { + panic("Panicking in DoChan") + }) + <-ch + t.Fatalf("DoChan unexpectedly returned") + } + + t.Parallel() + + cmd := exec.Command(executable(t), "-test.run="+t.Name(), "-test.v") + cmd.Env = append(os.Environ(), "TEST_PANIC_DOCHAN=1") + out := new(bytes.Buffer) + cmd.Stdout = out + cmd.Stderr = out + if err := cmd.Start(); err != nil { + t.Fatal(err) + } + + err := cmd.Wait() + t.Logf("%s:\n%s", strings.Join(cmd.Args, " "), out) + if err == nil { + t.Errorf("Test subprocess passed; want a crash due to panic in DoChan") + } + if bytes.Contains(out.Bytes(), []byte("DoChan unexpectedly")) { + t.Errorf("Test subprocess failed with an unexpected failure mode.") + } + if !bytes.Contains(out.Bytes(), []byte("Panicking in DoChan")) { + t.Errorf("Test subprocess failed, but the crash isn't caused by panicking in DoChan") + } +} + +func TestPanicDoSharedByDoChan(t *testing.T) { + if os.Getenv("TEST_PANIC_DOCHAN") != "" { + blocked := make(chan struct{}) + unblock := make(chan struct{}) + + g := New[any]() + go func() { + defer func() { + recover() + }() + g.Do(context.Background(), "", func(context.Context) (interface{}, error) { + close(blocked) + <-unblock + panic("Panicking in Do") + }) + }() + + <-blocked + ch := g.DoChan(context.Background(), "", func(context.Context) (interface{}, error) { + panic("DoChan unexpectedly executed callback") + }) + close(unblock) + <-ch + t.Fatalf("DoChan unexpectedly returned") + } + + t.Parallel() + + cmd := exec.Command(executable(t), "-test.run="+t.Name(), "-test.v") + cmd.Env = append(os.Environ(), "TEST_PANIC_DOCHAN=1") + out := new(bytes.Buffer) + cmd.Stdout = out + cmd.Stderr = out + if err := cmd.Start(); err != nil { + t.Fatal(err) + } + + err := cmd.Wait() + t.Logf("%s:\n%s", strings.Join(cmd.Args, " "), out) + if err == nil { + t.Errorf("Test subprocess passed; want a crash due to panic in Do shared by DoChan") + } + if bytes.Contains(out.Bytes(), []byte("DoChan unexpectedly")) { + t.Errorf("Test subprocess failed with an unexpected failure mode.") + } + if !bytes.Contains(out.Bytes(), []byte("Panicking in Do")) { + t.Errorf("Test subprocess failed, but the crash isn't caused by panicking in Do") + } +} + +func ExampleGroup() { + g := New[string]() + + block := make(chan struct{}) + res1c := g.DoChan(context.Background(), "key", func(context.Context) (string, error) { + <-block + return "func 1", nil + }) + res2c := g.DoChan(context.Background(), "key", func(context.Context) (string, error) { + <-block + return "func 2", nil + }) + close(block) + + res1 := <-res1c + res2 := <-res2c + + // Results are shared by functions executed with duplicate keys. + fmt.Println("Shared:", res2.Shared) + // Only the first function is executed: it is registered and started with "key", + // and doesn't complete before the second funtion is registered with a duplicate key. + fmt.Println("Equal results:", res1.Val == res2.Val) + fmt.Println("Result:", res1.Val) + + // Output: + // Shared: true + // Equal results: true + // Result: func 1 +} + +// NOT IMPLEMENTED BELOW diff --git a/internal/test/comparator/comparators.go b/internal/test/comparator/comparators.go index dcfcf7fb06d..085f7903261 100644 --- a/internal/test/comparator/comparators.go +++ b/internal/test/comparator/comparators.go @@ -15,9 +15,9 @@ package comparator import ( "reflect" - "sync" "github.com/vdaas/vald/internal/errors" + "github.com/vdaas/vald/internal/sync" ) var ( diff --git a/internal/timeutil/time_test.go b/internal/timeutil/time_test.go index 577ba2a6a95..3f343cd46d0 100644 --- a/internal/timeutil/time_test.go +++ b/internal/timeutil/time_test.go @@ -80,6 +80,12 @@ func TestParse(t *testing.T) { want: 0, wantErr: true, }, + { + name: "returns 0 and incorrect string error when t is minus value", + t: "-1", + want: 0, + wantErr: true, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/internal/worker/queue.go b/internal/worker/queue.go index 12fb5fc7a1c..ccdc5ef02f5 100644 --- a/internal/worker/queue.go +++ b/internal/worker/queue.go @@ -23,9 +23,9 @@ import ( "sync/atomic" "time" - "github.com/vdaas/vald/internal/errgroup" "github.com/vdaas/vald/internal/errors" "github.com/vdaas/vald/internal/safety" + "github.com/vdaas/vald/internal/sync/errgroup" ) // Queue represents the interface of queue. diff --git a/internal/worker/queue_option.go b/internal/worker/queue_option.go index 38c031bfc33..069c06e5ca9 100644 --- a/internal/worker/queue_option.go +++ b/internal/worker/queue_option.go @@ -18,7 +18,7 @@ package worker import ( - "github.com/vdaas/vald/internal/errgroup" + "github.com/vdaas/vald/internal/sync/errgroup" "github.com/vdaas/vald/internal/timeutil" ) diff --git a/internal/worker/queue_option_test.go b/internal/worker/queue_option_test.go index 52e82b0a32c..fb1e13aa403 100644 --- a/internal/worker/queue_option_test.go +++ b/internal/worker/queue_option_test.go @@ -22,8 +22,8 @@ import ( "testing" "time" - "github.com/vdaas/vald/internal/errgroup" "github.com/vdaas/vald/internal/errors" + "github.com/vdaas/vald/internal/sync/errgroup" "github.com/vdaas/vald/internal/test/goleak" ) diff --git a/internal/worker/queue_test.go b/internal/worker/queue_test.go index fe1c7e4ab06..87e0be11d47 100644 --- a/internal/worker/queue_test.go +++ b/internal/worker/queue_test.go @@ -24,8 +24,8 @@ import ( "testing" "time" - "github.com/vdaas/vald/internal/errgroup" "github.com/vdaas/vald/internal/errors" + "github.com/vdaas/vald/internal/sync/errgroup" "github.com/vdaas/vald/internal/test/comparator" "github.com/vdaas/vald/internal/test/goleak" ) diff --git a/internal/worker/worker.go b/internal/worker/worker.go index 3118903071d..fad9060e895 100644 --- a/internal/worker/worker.go +++ b/internal/worker/worker.go @@ -22,11 +22,11 @@ import ( "reflect" "sync/atomic" - "github.com/vdaas/vald/internal/errgroup" "github.com/vdaas/vald/internal/errors" "github.com/vdaas/vald/internal/log" "github.com/vdaas/vald/internal/observability/trace" "github.com/vdaas/vald/internal/safety" + "github.com/vdaas/vald/internal/sync/errgroup" ) // JobFunc represents the function of a job that works in the worker. @@ -133,7 +133,7 @@ func (w *worker) startJobLoop(ctx context.Context) <-chan error { w.eg.Go(safety.RecoverFunc(func() (err error) { defer close(ech) eg, ctx := errgroup.New(ctx) - eg.Limitation(w.limitation) + eg.SetLimit(w.limitation) limitation := make(chan struct{}, w.limitation) defer close(limitation) diff --git a/internal/worker/worker_option.go b/internal/worker/worker_option.go index b105b82b094..3ec117f7e3b 100644 --- a/internal/worker/worker_option.go +++ b/internal/worker/worker_option.go @@ -18,7 +18,7 @@ package worker import ( - "github.com/vdaas/vald/internal/errgroup" + "github.com/vdaas/vald/internal/sync/errgroup" ) type WorkerOption func(w *worker) error diff --git a/internal/worker/worker_test.go b/internal/worker/worker_test.go index 0e0e9fe18ae..575e70aa21d 100644 --- a/internal/worker/worker_test.go +++ b/internal/worker/worker_test.go @@ -24,10 +24,10 @@ import ( "testing" "time" - "github.com/vdaas/vald/internal/errgroup" "github.com/vdaas/vald/internal/errors" "github.com/vdaas/vald/internal/log" "github.com/vdaas/vald/internal/log/logger" + "github.com/vdaas/vald/internal/sync/errgroup" "github.com/vdaas/vald/internal/test/comparator" "github.com/vdaas/vald/internal/test/goleak" ) diff --git a/k8s/gateway/lb/hpa.yaml b/k8s/gateway/lb/hpa.yaml index 39e47a14950..426b3254b43 100644 --- a/k8s/gateway/lb/hpa.yaml +++ b/k8s/gateway/lb/hpa.yaml @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # -apiVersion: autoscaling/v1 +apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler metadata: name: vald-lb-gateway @@ -32,5 +32,11 @@ spec: apiVersion: apps/v1 kind: Deployment name: vald-lb-gateway - targetCPUUtilizationPercentage: 80 + metrics: + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: 80 status: diff --git a/pkg/agent/core/ngt/handler/grpc/handler.go b/pkg/agent/core/ngt/handler/grpc/handler.go index ef041a3e251..f8b84a70bbb 100644 --- a/pkg/agent/core/ngt/handler/grpc/handler.go +++ b/pkg/agent/core/ngt/handler/grpc/handler.go @@ -23,9 +23,9 @@ import ( agent "github.com/vdaas/vald/apis/grpc/v1/agent/core" "github.com/vdaas/vald/apis/grpc/v1/payload" "github.com/vdaas/vald/apis/grpc/v1/vald" - "github.com/vdaas/vald/internal/errgroup" "github.com/vdaas/vald/internal/errors" "github.com/vdaas/vald/internal/log" + "github.com/vdaas/vald/internal/sync/errgroup" "github.com/vdaas/vald/pkg/agent/core/ngt/service" ) diff --git a/pkg/agent/core/ngt/handler/grpc/handler_test.go b/pkg/agent/core/ngt/handler/grpc/handler_test.go index 17e8b6de693..266d69f87f6 100644 --- a/pkg/agent/core/ngt/handler/grpc/handler_test.go +++ b/pkg/agent/core/ngt/handler/grpc/handler_test.go @@ -23,10 +23,10 @@ import ( "github.com/vdaas/vald/apis/grpc/v1/payload" "github.com/vdaas/vald/internal/config" - "github.com/vdaas/vald/internal/errgroup" "github.com/vdaas/vald/internal/info" "github.com/vdaas/vald/internal/log" "github.com/vdaas/vald/internal/log/logger" + "github.com/vdaas/vald/internal/sync/errgroup" "github.com/vdaas/vald/internal/test/data/request" "github.com/vdaas/vald/internal/test/data/vector" "github.com/vdaas/vald/internal/test/goleak" diff --git a/pkg/agent/core/ngt/handler/grpc/index_test.go b/pkg/agent/core/ngt/handler/grpc/index_test.go index aece9ec8d05..fdfb85d0c57 100644 --- a/pkg/agent/core/ngt/handler/grpc/index_test.go +++ b/pkg/agent/core/ngt/handler/grpc/index_test.go @@ -17,19 +17,19 @@ import ( "context" "os" "reflect" - "sync" "testing" "time" "github.com/vdaas/vald/apis/grpc/v1/payload" "github.com/vdaas/vald/internal/config" "github.com/vdaas/vald/internal/core/algorithm/ngt" - "github.com/vdaas/vald/internal/errgroup" "github.com/vdaas/vald/internal/errors" "github.com/vdaas/vald/internal/file" "github.com/vdaas/vald/internal/net" "github.com/vdaas/vald/internal/net/grpc/codes" "github.com/vdaas/vald/internal/net/grpc/status" + "github.com/vdaas/vald/internal/sync" + "github.com/vdaas/vald/internal/sync/errgroup" "github.com/vdaas/vald/internal/test/comparator" "github.com/vdaas/vald/internal/test/data/request" "github.com/vdaas/vald/internal/test/data/vector" diff --git a/pkg/agent/core/ngt/handler/grpc/insert_test.go b/pkg/agent/core/ngt/handler/grpc/insert_test.go index 3bc376223af..757b03aac03 100644 --- a/pkg/agent/core/ngt/handler/grpc/insert_test.go +++ b/pkg/agent/core/ngt/handler/grpc/insert_test.go @@ -23,7 +23,6 @@ import ( "github.com/vdaas/vald/apis/grpc/v1/payload" "github.com/vdaas/vald/internal/config" "github.com/vdaas/vald/internal/core/algorithm/ngt" - "github.com/vdaas/vald/internal/errgroup" "github.com/vdaas/vald/internal/errors" "github.com/vdaas/vald/internal/io" "github.com/vdaas/vald/internal/net" @@ -31,6 +30,7 @@ import ( "github.com/vdaas/vald/internal/net/grpc/errdetails" "github.com/vdaas/vald/internal/net/grpc/status" "github.com/vdaas/vald/internal/strings" + "github.com/vdaas/vald/internal/sync/errgroup" "github.com/vdaas/vald/internal/test/data/request" "github.com/vdaas/vald/internal/test/data/vector" "github.com/vdaas/vald/internal/test/mock" diff --git a/pkg/agent/core/ngt/handler/grpc/linear_search.go b/pkg/agent/core/ngt/handler/grpc/linear_search.go index f26c48b5b35..554fb9d07fc 100644 --- a/pkg/agent/core/ngt/handler/grpc/linear_search.go +++ b/pkg/agent/core/ngt/handler/grpc/linear_search.go @@ -16,7 +16,6 @@ package grpc import ( "context" "fmt" - "sync" "github.com/vdaas/vald/apis/grpc/v1/payload" "github.com/vdaas/vald/apis/grpc/v1/vald" @@ -30,6 +29,7 @@ import ( "github.com/vdaas/vald/internal/observability/trace" "github.com/vdaas/vald/internal/safety" "github.com/vdaas/vald/internal/strings" + "github.com/vdaas/vald/internal/sync" "go.opentelemetry.io/otel/attribute" ) diff --git a/pkg/agent/core/ngt/handler/grpc/object_test.go b/pkg/agent/core/ngt/handler/grpc/object_test.go index edd5a044cfa..7db8e36e57a 100644 --- a/pkg/agent/core/ngt/handler/grpc/object_test.go +++ b/pkg/agent/core/ngt/handler/grpc/object_test.go @@ -24,11 +24,11 @@ import ( "github.com/vdaas/vald/internal/config" "github.com/vdaas/vald/internal/conv" "github.com/vdaas/vald/internal/core/algorithm/ngt" - "github.com/vdaas/vald/internal/errgroup" "github.com/vdaas/vald/internal/errors" "github.com/vdaas/vald/internal/net" "github.com/vdaas/vald/internal/net/grpc/codes" "github.com/vdaas/vald/internal/net/grpc/status" + "github.com/vdaas/vald/internal/sync/errgroup" "github.com/vdaas/vald/internal/test/data/request" "github.com/vdaas/vald/internal/test/data/vector" "github.com/vdaas/vald/internal/test/mock" @@ -1339,9 +1339,10 @@ func Test_server_StreamGetObject(t *testing.T) { // NOT IMPLEMENTED BELOW // -// func Test_server_StreamGetObject(t *testing.T) { +// func Test_server_StreamListObject(t *testing.T) { // type args struct { -// stream vald.Object_StreamGetObjectServer +// in0 *payload.Object_List_Request +// stream vald.Object_StreamListObjectServer // } // type fields struct { // name string @@ -1376,6 +1377,7 @@ func Test_server_StreamGetObject(t *testing.T) { // { // name: "test_case_1", // args: args { +// in0:nil, // stream:nil, // }, // fields: fields { @@ -1404,6 +1406,7 @@ func Test_server_StreamGetObject(t *testing.T) { // return test { // name: "test_case_2", // args: args { +// in0:nil, // stream:nil, // }, // fields: fields { @@ -1453,7 +1456,7 @@ func Test_server_StreamGetObject(t *testing.T) { // UnimplementedValdServer: test.fields.UnimplementedValdServer, // } // -// err := s.StreamGetObject(test.args.stream) +// err := s.StreamListObject(test.args.in0, test.args.stream) // if err := checkFunc(test.want, err); err != nil { // tt.Errorf("error = %v", err) // } diff --git a/pkg/agent/core/ngt/handler/grpc/option.go b/pkg/agent/core/ngt/handler/grpc/option.go index 10f08130ad3..165a5433d24 100644 --- a/pkg/agent/core/ngt/handler/grpc/option.go +++ b/pkg/agent/core/ngt/handler/grpc/option.go @@ -21,10 +21,10 @@ import ( "os" "runtime" - "github.com/vdaas/vald/internal/errgroup" "github.com/vdaas/vald/internal/errors" "github.com/vdaas/vald/internal/log" "github.com/vdaas/vald/internal/net" + "github.com/vdaas/vald/internal/sync/errgroup" "github.com/vdaas/vald/pkg/agent/core/ngt/service" ) diff --git a/pkg/agent/core/ngt/handler/grpc/option_test.go b/pkg/agent/core/ngt/handler/grpc/option_test.go index 9a2e39f2b37..c6c4d8652a3 100644 --- a/pkg/agent/core/ngt/handler/grpc/option_test.go +++ b/pkg/agent/core/ngt/handler/grpc/option_test.go @@ -22,8 +22,8 @@ import ( "testing" "github.com/vdaas/vald/internal/config" - "github.com/vdaas/vald/internal/errgroup" "github.com/vdaas/vald/internal/errors" + "github.com/vdaas/vald/internal/sync/errgroup" "github.com/vdaas/vald/pkg/agent/core/ngt/service" ) diff --git a/pkg/agent/core/ngt/handler/grpc/remove_test.go b/pkg/agent/core/ngt/handler/grpc/remove_test.go index 308087420f7..2b924b3a00d 100644 --- a/pkg/agent/core/ngt/handler/grpc/remove_test.go +++ b/pkg/agent/core/ngt/handler/grpc/remove_test.go @@ -22,10 +22,10 @@ import ( "github.com/vdaas/vald/internal/config" "github.com/vdaas/vald/internal/conv" "github.com/vdaas/vald/internal/core/algorithm/ngt" - "github.com/vdaas/vald/internal/errgroup" "github.com/vdaas/vald/internal/errors" "github.com/vdaas/vald/internal/net/grpc/codes" "github.com/vdaas/vald/internal/net/grpc/status" + "github.com/vdaas/vald/internal/sync/errgroup" "github.com/vdaas/vald/internal/test/data/request" "github.com/vdaas/vald/internal/test/data/vector" ) diff --git a/pkg/agent/core/ngt/handler/grpc/search.go b/pkg/agent/core/ngt/handler/grpc/search.go index 2a9c9d6fdcd..c6941f15dcf 100644 --- a/pkg/agent/core/ngt/handler/grpc/search.go +++ b/pkg/agent/core/ngt/handler/grpc/search.go @@ -16,7 +16,6 @@ package grpc import ( "context" "fmt" - "sync" "github.com/vdaas/vald/apis/grpc/v1/payload" "github.com/vdaas/vald/apis/grpc/v1/vald" @@ -30,6 +29,7 @@ import ( "github.com/vdaas/vald/internal/observability/trace" "github.com/vdaas/vald/internal/safety" "github.com/vdaas/vald/internal/strings" + "github.com/vdaas/vald/internal/sync" "github.com/vdaas/vald/pkg/agent/core/ngt/model" "go.opentelemetry.io/otel/attribute" ) diff --git a/pkg/agent/core/ngt/handler/grpc/search_test.go b/pkg/agent/core/ngt/handler/grpc/search_test.go index d42cb2c754d..b70f123c319 100644 --- a/pkg/agent/core/ngt/handler/grpc/search_test.go +++ b/pkg/agent/core/ngt/handler/grpc/search_test.go @@ -23,10 +23,10 @@ import ( "github.com/vdaas/vald/internal/config" "github.com/vdaas/vald/internal/conv" "github.com/vdaas/vald/internal/core/algorithm/ngt" - "github.com/vdaas/vald/internal/errgroup" "github.com/vdaas/vald/internal/errors" "github.com/vdaas/vald/internal/net/grpc/codes" "github.com/vdaas/vald/internal/net/grpc/status" + "github.com/vdaas/vald/internal/sync/errgroup" "github.com/vdaas/vald/internal/test/data/request" "github.com/vdaas/vald/internal/test/data/vector" "github.com/vdaas/vald/pkg/agent/core/ngt/service" diff --git a/pkg/agent/core/ngt/handler/grpc/update_test.go b/pkg/agent/core/ngt/handler/grpc/update_test.go index 11935a2e9be..c0967d23cd0 100644 --- a/pkg/agent/core/ngt/handler/grpc/update_test.go +++ b/pkg/agent/core/ngt/handler/grpc/update_test.go @@ -22,10 +22,10 @@ import ( "github.com/vdaas/vald/internal/config" "github.com/vdaas/vald/internal/conv" "github.com/vdaas/vald/internal/core/algorithm/ngt" - "github.com/vdaas/vald/internal/errgroup" "github.com/vdaas/vald/internal/errors" "github.com/vdaas/vald/internal/net/grpc/codes" "github.com/vdaas/vald/internal/net/grpc/status" + "github.com/vdaas/vald/internal/sync/errgroup" "github.com/vdaas/vald/internal/test/data/request" "github.com/vdaas/vald/internal/test/data/vector" ) diff --git a/pkg/agent/core/ngt/handler/grpc/upsert.go b/pkg/agent/core/ngt/handler/grpc/upsert.go index e94849d1ff7..41c6990186a 100644 --- a/pkg/agent/core/ngt/handler/grpc/upsert.go +++ b/pkg/agent/core/ngt/handler/grpc/upsert.go @@ -16,7 +16,6 @@ package grpc import ( "context" "fmt" - "sync" "github.com/vdaas/vald/apis/grpc/v1/payload" "github.com/vdaas/vald/apis/grpc/v1/vald" @@ -30,6 +29,7 @@ import ( "github.com/vdaas/vald/internal/observability/trace" "github.com/vdaas/vald/internal/safety" "github.com/vdaas/vald/internal/strings" + "github.com/vdaas/vald/internal/sync" ) func (s *server) Upsert(ctx context.Context, req *payload.Upsert_Request) (loc *payload.Object_Location, err error) { diff --git a/pkg/agent/core/ngt/handler/grpc/upsert_test.go b/pkg/agent/core/ngt/handler/grpc/upsert_test.go index 8f067d86494..a9b2ce37bf4 100644 --- a/pkg/agent/core/ngt/handler/grpc/upsert_test.go +++ b/pkg/agent/core/ngt/handler/grpc/upsert_test.go @@ -22,10 +22,10 @@ import ( "github.com/vdaas/vald/internal/config" "github.com/vdaas/vald/internal/conv" "github.com/vdaas/vald/internal/core/algorithm/ngt" - "github.com/vdaas/vald/internal/errgroup" "github.com/vdaas/vald/internal/errors" "github.com/vdaas/vald/internal/net/grpc/codes" "github.com/vdaas/vald/internal/net/grpc/status" + "github.com/vdaas/vald/internal/sync/errgroup" "github.com/vdaas/vald/internal/test/data/request" "github.com/vdaas/vald/internal/test/data/vector" ) diff --git a/pkg/agent/core/ngt/router/option.go b/pkg/agent/core/ngt/router/option.go index 3b437c028b5..6b62d72e567 100644 --- a/pkg/agent/core/ngt/router/option.go +++ b/pkg/agent/core/ngt/router/option.go @@ -18,7 +18,7 @@ package router import ( - "github.com/vdaas/vald/internal/errgroup" + "github.com/vdaas/vald/internal/sync/errgroup" "github.com/vdaas/vald/pkg/agent/core/ngt/handler/rest" ) diff --git a/pkg/agent/core/ngt/router/option_test.go b/pkg/agent/core/ngt/router/option_test.go index 91076571be9..b48dc902b39 100644 --- a/pkg/agent/core/ngt/router/option_test.go +++ b/pkg/agent/core/ngt/router/option_test.go @@ -21,8 +21,8 @@ import ( "reflect" "testing" - "github.com/vdaas/vald/internal/errgroup" "github.com/vdaas/vald/internal/errors" + "github.com/vdaas/vald/internal/sync/errgroup" "github.com/vdaas/vald/internal/test/goleak" "github.com/vdaas/vald/pkg/agent/core/ngt/handler/rest" ) diff --git a/pkg/agent/core/ngt/router/router.go b/pkg/agent/core/ngt/router/router.go index 582e6431ed3..7b10891e883 100644 --- a/pkg/agent/core/ngt/router/router.go +++ b/pkg/agent/core/ngt/router/router.go @@ -20,9 +20,9 @@ package router import ( "net/http" - "github.com/vdaas/vald/internal/errgroup" "github.com/vdaas/vald/internal/net/http/middleware" "github.com/vdaas/vald/internal/net/http/routing" + "github.com/vdaas/vald/internal/sync/errgroup" "github.com/vdaas/vald/pkg/agent/core/ngt/handler/rest" ) diff --git a/pkg/agent/core/ngt/router/router_test.go b/pkg/agent/core/ngt/router/router_test.go index f892ee707de..d576da84cd3 100644 --- a/pkg/agent/core/ngt/router/router_test.go +++ b/pkg/agent/core/ngt/router/router_test.go @@ -22,9 +22,9 @@ import ( "testing" "github.com/gorilla/mux" - "github.com/vdaas/vald/internal/errgroup" "github.com/vdaas/vald/internal/errors" "github.com/vdaas/vald/internal/net/http/routing" + "github.com/vdaas/vald/internal/sync/errgroup" "github.com/vdaas/vald/internal/test/goleak" "github.com/vdaas/vald/pkg/agent/core/ngt/handler/rest" ) diff --git a/pkg/agent/core/ngt/service/kvs/kvs.go b/pkg/agent/core/ngt/service/kvs/kvs.go index 7524110ef0c..90fc2c4c5d7 100644 --- a/pkg/agent/core/ngt/service/kvs/kvs.go +++ b/pkg/agent/core/ngt/service/kvs/kvs.go @@ -18,12 +18,11 @@ package kvs import ( "context" - "sync" "sync/atomic" - "github.com/vdaas/vald/internal/errgroup" "github.com/vdaas/vald/internal/safety" - valdsync "github.com/vdaas/vald/internal/sync" + "github.com/vdaas/vald/internal/sync" + "github.com/vdaas/vald/internal/sync/errgroup" "github.com/zeebo/xxh3" ) @@ -52,8 +51,8 @@ type ValueStructUo struct { type bidi struct { concurrency int l uint64 - ou [slen]*valdsync.Map[uint32, valueStructOu] - uo [slen]*valdsync.Map[string, ValueStructUo] + ou [slen]*sync.Map[uint32, valueStructOu] + uo [slen]*sync.Map[string, ValueStructUo] eg errgroup.Group } @@ -76,8 +75,8 @@ func New(opts ...Option) BidiMap { opt(b) } for i := range b.ou { - b.ou[i] = new(valdsync.Map[uint32, valueStructOu]) - b.uo[i] = new(valdsync.Map[string, ValueStructUo]) + b.ou[i] = new(sync.Map[uint32, valueStructOu]) + b.uo[i] = new(sync.Map[string, ValueStructUo]) } if b.eg == nil { @@ -85,7 +84,7 @@ func New(opts ...Option) BidiMap { } if b.concurrency > 0 { - b.eg.Limitation(b.concurrency) + b.eg.SetLimit(b.concurrency) } return b @@ -94,7 +93,7 @@ func New(opts ...Option) BidiMap { // Get returns the value and boolean from the given key. // If the value does not exist, it returns nil and false. func (b *bidi) Get(key string) (oid uint32, timestamp int64, exists bool) { - vs, ok := b.uo[xxh3.HashString(key)&mask].Load(key) + vs, ok := b.uo[getShardID(key)].Load(key) if !ok { return 0, 0, false } @@ -114,7 +113,7 @@ func (b *bidi) GetInverse(val uint32) (string, int64, bool) { // Set sets the key and val to the bidi. func (b *bidi) Set(key string, val uint32, ts int64) { - id := xxh3.HashString(key) & mask + id := getShardID(key) vs, loaded := b.uo[id].LoadOrStore(key, ValueStructUo{value: val, timestamp: ts}) old := vs.value if !loaded { // increase the count only if the key is not exists before @@ -129,7 +128,7 @@ func (b *bidi) Set(key string, val uint32, ts int64) { // Delete deletes the key and the value from the bidi by the given key and returns val and true. // If the value for the key does not exist, it returns nil and false. func (b *bidi) Delete(key string) (val uint32, ok bool) { - vs, ok := b.uo[xxh3.HashString(key)&mask].LoadAndDelete(key) + vs, ok := b.uo[getShardID(key)].LoadAndDelete(key) val = vs.value if ok { b.ou[val&mask].Delete(val) @@ -144,7 +143,7 @@ func (b *bidi) DeleteInverse(val uint32) (key string, ok bool) { vs, ok := b.ou[val&mask].LoadAndDelete(val) key = vs.value if ok { - b.uo[xxh3.HashString(key)&mask].Delete(key) + b.uo[getShardID(key)].LoadAndDelete(key) atomic.AddUint64(&b.l, ^uint64(0)) } return key, ok @@ -158,12 +157,11 @@ func (b *bidi) Range(ctx context.Context, f func(string, uint32, int64) bool) { wg.Add(1) b.eg.Go(safety.RecoverFunc(func() (err error) { b.uo[idx].Range(func(uuid string, val ValueStructUo) bool { - f(uuid, val.value, val.timestamp) select { case <-ctx.Done(): return false default: - return true + return f(uuid, val.value, val.timestamp) } }) wg.Done() @@ -187,3 +185,10 @@ func (b *bidi) Close() error { } return b.eg.Wait() } + +func getShardID(key string) (id uint64) { + if len(key) > 128 { + return xxh3.HashString(key[:128]) & mask + } + return xxh3.HashString(key) & mask +} diff --git a/pkg/agent/core/ngt/service/kvs/kvs_test.go b/pkg/agent/core/ngt/service/kvs/kvs_test.go index 60d811ac9f9..243b1c99b64 100644 --- a/pkg/agent/core/ngt/service/kvs/kvs_test.go +++ b/pkg/agent/core/ngt/service/kvs/kvs_test.go @@ -21,13 +21,12 @@ import ( "math" "reflect" "runtime" - "sync" "sync/atomic" "testing" - "github.com/vdaas/vald/internal/errgroup" "github.com/vdaas/vald/internal/errors" - valdsync "github.com/vdaas/vald/internal/sync" + "github.com/vdaas/vald/internal/sync" + "github.com/vdaas/vald/internal/sync/errgroup" "github.com/vdaas/vald/internal/test/goleak" ) @@ -52,12 +51,12 @@ func TestNew(t *testing.T) { tests := []test{ func() test { var ( - wantOu [slen]*valdsync.Map[uint32, valueStructOu] - wantUo [slen]*valdsync.Map[string, ValueStructUo] + wantOu [slen]*sync.Map[uint32, valueStructOu] + wantUo [slen]*sync.Map[string, ValueStructUo] ) for i := 0; i < slen; i++ { - wantOu[i] = new(valdsync.Map[uint32, valueStructOu]) - wantUo[i] = new(valdsync.Map[string, ValueStructUo]) + wantOu[i] = new(sync.Map[uint32, valueStructOu]) + wantUo[i] = new(sync.Map[string, ValueStructUo]) } return test{ name: "return the bidi struct", @@ -104,8 +103,8 @@ func Test_bidi_Get(t *testing.T) { key string } type fields struct { - ou [slen]*valdsync.Map[uint32, valueStructOu] - uo [slen]*valdsync.Map[string, ValueStructUo] + ou [slen]*sync.Map[uint32, valueStructOu] + uo [slen]*sync.Map[string, ValueStructUo] l uint64 } type want struct { @@ -144,8 +143,8 @@ func Test_bidi_Get(t *testing.T) { l: 0, } for i := 0; i < slen; i++ { - fields.ou[i] = new(valdsync.Map[uint32, valueStructOu]) - fields.uo[i] = new(valdsync.Map[string, ValueStructUo]) + fields.ou[i] = new(sync.Map[uint32, valueStructOu]) + fields.uo[i] = new(sync.Map[string, ValueStructUo]) } var ( @@ -177,8 +176,8 @@ func Test_bidi_Get(t *testing.T) { l: 100, } for i := 0; i < slen; i++ { - fields.ou[i] = new(valdsync.Map[uint32, valueStructOu]) - fields.uo[i] = new(valdsync.Map[string, ValueStructUo]) + fields.ou[i] = new(sync.Map[uint32, valueStructOu]) + fields.uo[i] = new(sync.Map[string, ValueStructUo]) } var ( @@ -210,8 +209,8 @@ func Test_bidi_Get(t *testing.T) { l: math.MaxUint64, } for i := 0; i < slen; i++ { - fields.ou[i] = new(valdsync.Map[uint32, valueStructOu]) - fields.uo[i] = new(valdsync.Map[string, ValueStructUo]) + fields.ou[i] = new(sync.Map[uint32, valueStructOu]) + fields.uo[i] = new(sync.Map[string, ValueStructUo]) } var ( @@ -243,8 +242,8 @@ func Test_bidi_Get(t *testing.T) { l: 0, } for i := 0; i < slen; i++ { - fields.ou[i] = new(valdsync.Map[uint32, valueStructOu]) - fields.uo[i] = new(valdsync.Map[string, ValueStructUo]) + fields.ou[i] = new(sync.Map[uint32, valueStructOu]) + fields.uo[i] = new(sync.Map[string, ValueStructUo]) } var ( @@ -276,8 +275,8 @@ func Test_bidi_Get(t *testing.T) { l: 0, } for i := 0; i < slen; i++ { - fields.ou[i] = new(valdsync.Map[uint32, valueStructOu]) - fields.uo[i] = new(valdsync.Map[string, ValueStructUo]) + fields.ou[i] = new(sync.Map[uint32, valueStructOu]) + fields.uo[i] = new(sync.Map[string, ValueStructUo]) } var ( @@ -307,8 +306,8 @@ func Test_bidi_Get(t *testing.T) { l: 0, } for i := 0; i < slen; i++ { - fields.ou[i] = new(valdsync.Map[uint32, valueStructOu]) - fields.uo[i] = new(valdsync.Map[string, ValueStructUo]) + fields.ou[i] = new(sync.Map[uint32, valueStructOu]) + fields.uo[i] = new(sync.Map[string, ValueStructUo]) } var ( @@ -370,8 +369,8 @@ func Test_bidi_GetInverse(t *testing.T) { val uint32 } type fields struct { - ou [slen]*valdsync.Map[uint32, valueStructOu] - uo [slen]*valdsync.Map[string, ValueStructUo] + ou [slen]*sync.Map[uint32, valueStructOu] + uo [slen]*sync.Map[string, ValueStructUo] l uint64 } type want struct { @@ -410,8 +409,8 @@ func Test_bidi_GetInverse(t *testing.T) { l: 0, } for i := 0; i < slen; i++ { - fields.ou[i] = new(valdsync.Map[uint32, valueStructOu]) - fields.uo[i] = new(valdsync.Map[string, ValueStructUo]) + fields.ou[i] = new(sync.Map[uint32, valueStructOu]) + fields.uo[i] = new(sync.Map[string, ValueStructUo]) } var ( @@ -443,8 +442,8 @@ func Test_bidi_GetInverse(t *testing.T) { l: 100, } for i := 0; i < slen; i++ { - fields.ou[i] = new(valdsync.Map[uint32, valueStructOu]) - fields.uo[i] = new(valdsync.Map[string, ValueStructUo]) + fields.ou[i] = new(sync.Map[uint32, valueStructOu]) + fields.uo[i] = new(sync.Map[string, ValueStructUo]) } var ( @@ -476,8 +475,8 @@ func Test_bidi_GetInverse(t *testing.T) { l: math.MaxUint64, } for i := 0; i < slen; i++ { - fields.ou[i] = new(valdsync.Map[uint32, valueStructOu]) - fields.uo[i] = new(valdsync.Map[string, ValueStructUo]) + fields.ou[i] = new(sync.Map[uint32, valueStructOu]) + fields.uo[i] = new(sync.Map[string, ValueStructUo]) } var ( @@ -509,8 +508,8 @@ func Test_bidi_GetInverse(t *testing.T) { l: 0, } for i := 0; i < slen; i++ { - fields.ou[i] = new(valdsync.Map[uint32, valueStructOu]) - fields.uo[i] = new(valdsync.Map[string, ValueStructUo]) + fields.ou[i] = new(sync.Map[uint32, valueStructOu]) + fields.uo[i] = new(sync.Map[string, ValueStructUo]) } var ( @@ -542,8 +541,8 @@ func Test_bidi_GetInverse(t *testing.T) { l: 0, } for i := 0; i < slen; i++ { - fields.ou[i] = new(valdsync.Map[uint32, valueStructOu]) - fields.uo[i] = new(valdsync.Map[string, ValueStructUo]) + fields.ou[i] = new(sync.Map[uint32, valueStructOu]) + fields.uo[i] = new(sync.Map[string, ValueStructUo]) } var ( @@ -573,8 +572,8 @@ func Test_bidi_GetInverse(t *testing.T) { l: 0, } for i := 0; i < slen; i++ { - fields.ou[i] = new(valdsync.Map[uint32, valueStructOu]) - fields.uo[i] = new(valdsync.Map[string, ValueStructUo]) + fields.ou[i] = new(sync.Map[uint32, valueStructOu]) + fields.uo[i] = new(sync.Map[string, ValueStructUo]) } var ( @@ -639,8 +638,8 @@ func Test_bidi_Set(t *testing.T) { ts int64 } type fields struct { - ou [slen]*valdsync.Map[uint32, valueStructOu] - uo [slen]*valdsync.Map[string, ValueStructUo] + ou [slen]*sync.Map[uint32, valueStructOu] + uo [slen]*sync.Map[string, ValueStructUo] l uint64 } type want struct { @@ -687,8 +686,8 @@ func Test_bidi_Set(t *testing.T) { l: 0, } for i := 0; i < slen; i++ { - fields.ou[i] = new(valdsync.Map[uint32, valueStructOu]) - fields.uo[i] = new(valdsync.Map[string, ValueStructUo]) + fields.ou[i] = new(sync.Map[uint32, valueStructOu]) + fields.uo[i] = new(sync.Map[string, ValueStructUo]) } var ( @@ -718,8 +717,8 @@ func Test_bidi_Set(t *testing.T) { l: 100, } for i := 0; i < slen; i++ { - fields.ou[i] = new(valdsync.Map[uint32, valueStructOu]) - fields.uo[i] = new(valdsync.Map[string, ValueStructUo]) + fields.ou[i] = new(sync.Map[uint32, valueStructOu]) + fields.uo[i] = new(sync.Map[string, ValueStructUo]) } var ( @@ -749,8 +748,8 @@ func Test_bidi_Set(t *testing.T) { l: math.MaxUint64, } for i := 0; i < slen; i++ { - fields.ou[i] = new(valdsync.Map[uint32, valueStructOu]) - fields.uo[i] = new(valdsync.Map[string, ValueStructUo]) + fields.ou[i] = new(sync.Map[uint32, valueStructOu]) + fields.uo[i] = new(sync.Map[string, ValueStructUo]) } var ( @@ -780,8 +779,8 @@ func Test_bidi_Set(t *testing.T) { l: 0, } for i := 0; i < slen; i++ { - fields.ou[i] = new(valdsync.Map[uint32, valueStructOu]) - fields.uo[i] = new(valdsync.Map[string, ValueStructUo]) + fields.ou[i] = new(sync.Map[uint32, valueStructOu]) + fields.uo[i] = new(sync.Map[string, ValueStructUo]) } var ( @@ -820,8 +819,8 @@ func Test_bidi_Set(t *testing.T) { l: 0, } for i := 0; i < slen; i++ { - fields.ou[i] = new(valdsync.Map[uint32, valueStructOu]) - fields.uo[i] = new(valdsync.Map[string, ValueStructUo]) + fields.ou[i] = new(sync.Map[uint32, valueStructOu]) + fields.uo[i] = new(sync.Map[string, ValueStructUo]) } key := "45637ec4-c85f-11ea-87d0" @@ -845,8 +844,8 @@ func Test_bidi_Set(t *testing.T) { l: 0, } for i := 0; i < slen; i++ { - fields.ou[i] = new(valdsync.Map[uint32, valueStructOu]) - fields.uo[i] = new(valdsync.Map[string, ValueStructUo]) + fields.ou[i] = new(sync.Map[uint32, valueStructOu]) + fields.uo[i] = new(sync.Map[string, ValueStructUo]) } var val uint32 = 14438 @@ -869,8 +868,8 @@ func Test_bidi_Set(t *testing.T) { l: 0, } for i := 0; i < slen; i++ { - fields.ou[i] = new(valdsync.Map[uint32, valueStructOu]) - fields.uo[i] = new(valdsync.Map[string, ValueStructUo]) + fields.ou[i] = new(sync.Map[uint32, valueStructOu]) + fields.uo[i] = new(sync.Map[string, ValueStructUo]) } return test{ @@ -922,8 +921,8 @@ func Test_bidi_Delete(t *testing.T) { key string } type fields struct { - ou [slen]*valdsync.Map[uint32, valueStructOu] - uo [slen]*valdsync.Map[string, ValueStructUo] + ou [slen]*sync.Map[uint32, valueStructOu] + uo [slen]*sync.Map[string, ValueStructUo] l uint64 } type want struct { @@ -955,8 +954,8 @@ func Test_bidi_Delete(t *testing.T) { l: 0, } for i := 0; i < slen; i++ { - fields.ou[i] = new(valdsync.Map[uint32, valueStructOu]) - fields.uo[i] = new(valdsync.Map[string, ValueStructUo]) + fields.ou[i] = new(sync.Map[uint32, valueStructOu]) + fields.uo[i] = new(sync.Map[string, ValueStructUo]) } var ( @@ -1002,8 +1001,8 @@ func Test_bidi_Delete(t *testing.T) { l: 100, } for i := 0; i < slen; i++ { - fields.ou[i] = new(valdsync.Map[uint32, valueStructOu]) - fields.uo[i] = new(valdsync.Map[string, ValueStructUo]) + fields.ou[i] = new(sync.Map[uint32, valueStructOu]) + fields.uo[i] = new(sync.Map[string, ValueStructUo]) } var ( @@ -1049,8 +1048,8 @@ func Test_bidi_Delete(t *testing.T) { l: math.MaxUint64, } for i := 0; i < slen; i++ { - fields.ou[i] = new(valdsync.Map[uint32, valueStructOu]) - fields.uo[i] = new(valdsync.Map[string, ValueStructUo]) + fields.ou[i] = new(sync.Map[uint32, valueStructOu]) + fields.uo[i] = new(sync.Map[string, ValueStructUo]) } var ( @@ -1096,8 +1095,8 @@ func Test_bidi_Delete(t *testing.T) { l: 0, } for i := 0; i < slen; i++ { - fields.ou[i] = new(valdsync.Map[uint32, valueStructOu]) - fields.uo[i] = new(valdsync.Map[string, ValueStructUo]) + fields.ou[i] = new(sync.Map[uint32, valueStructOu]) + fields.uo[i] = new(sync.Map[string, ValueStructUo]) } var ( @@ -1140,8 +1139,8 @@ func Test_bidi_Delete(t *testing.T) { l: 0, } for i := 0; i < slen; i++ { - fields.ou[i] = new(valdsync.Map[uint32, valueStructOu]) - fields.uo[i] = new(valdsync.Map[string, ValueStructUo]) + fields.ou[i] = new(sync.Map[uint32, valueStructOu]) + fields.uo[i] = new(sync.Map[string, ValueStructUo]) } var ( @@ -1203,8 +1202,8 @@ func Test_bidi_DeleteInverse(t *testing.T) { val uint32 } type fields struct { - ou [slen]*valdsync.Map[uint32, valueStructOu] - uo [slen]*valdsync.Map[string, ValueStructUo] + ou [slen]*sync.Map[uint32, valueStructOu] + uo [slen]*sync.Map[string, ValueStructUo] l uint64 } type want struct { @@ -1236,8 +1235,8 @@ func Test_bidi_DeleteInverse(t *testing.T) { l: 0, } for i := 0; i < slen; i++ { - fields.ou[i] = new(valdsync.Map[uint32, valueStructOu]) - fields.uo[i] = new(valdsync.Map[string, ValueStructUo]) + fields.ou[i] = new(sync.Map[uint32, valueStructOu]) + fields.uo[i] = new(sync.Map[string, ValueStructUo]) } var ( @@ -1283,8 +1282,8 @@ func Test_bidi_DeleteInverse(t *testing.T) { l: 100, } for i := 0; i < slen; i++ { - fields.ou[i] = new(valdsync.Map[uint32, valueStructOu]) - fields.uo[i] = new(valdsync.Map[string, ValueStructUo]) + fields.ou[i] = new(sync.Map[uint32, valueStructOu]) + fields.uo[i] = new(sync.Map[string, ValueStructUo]) } var ( @@ -1330,8 +1329,8 @@ func Test_bidi_DeleteInverse(t *testing.T) { l: math.MaxUint64, } for i := 0; i < slen; i++ { - fields.ou[i] = new(valdsync.Map[uint32, valueStructOu]) - fields.uo[i] = new(valdsync.Map[string, ValueStructUo]) + fields.ou[i] = new(sync.Map[uint32, valueStructOu]) + fields.uo[i] = new(sync.Map[string, ValueStructUo]) } var ( @@ -1377,8 +1376,8 @@ func Test_bidi_DeleteInverse(t *testing.T) { l: 0, } for i := 0; i < slen; i++ { - fields.ou[i] = new(valdsync.Map[uint32, valueStructOu]) - fields.uo[i] = new(valdsync.Map[string, ValueStructUo]) + fields.ou[i] = new(sync.Map[uint32, valueStructOu]) + fields.uo[i] = new(sync.Map[string, ValueStructUo]) } var ( @@ -1421,8 +1420,8 @@ func Test_bidi_DeleteInverse(t *testing.T) { l: 0, } for i := 0; i < slen; i++ { - fields.ou[i] = new(valdsync.Map[uint32, valueStructOu]) - fields.uo[i] = new(valdsync.Map[string, ValueStructUo]) + fields.ou[i] = new(sync.Map[uint32, valueStructOu]) + fields.uo[i] = new(sync.Map[string, ValueStructUo]) } var ( @@ -1485,8 +1484,8 @@ func Test_bidi_Range(t *testing.T) { f func(string, uint32, int64) bool } type fields struct { - ou [slen]*valdsync.Map[uint32, valueStructOu] - uo [slen]*valdsync.Map[string, ValueStructUo] + ou [slen]*sync.Map[uint32, valueStructOu] + uo [slen]*sync.Map[string, ValueStructUo] l uint64 } type want struct { @@ -1511,8 +1510,8 @@ func Test_bidi_Range(t *testing.T) { l: 0, } for i := 0; i < slen; i++ { - fields.ou[i] = new(valdsync.Map[uint32, valueStructOu]) - fields.uo[i] = new(valdsync.Map[string, ValueStructUo]) + fields.ou[i] = new(sync.Map[uint32, valueStructOu]) + fields.uo[i] = new(sync.Map[string, ValueStructUo]) } var ( @@ -1563,8 +1562,8 @@ func Test_bidi_Range(t *testing.T) { l: 100, } for i := 0; i < slen; i++ { - fields.ou[i] = new(valdsync.Map[uint32, valueStructOu]) - fields.uo[i] = new(valdsync.Map[string, ValueStructUo]) + fields.ou[i] = new(sync.Map[uint32, valueStructOu]) + fields.uo[i] = new(sync.Map[string, ValueStructUo]) } var ( @@ -1615,8 +1614,8 @@ func Test_bidi_Range(t *testing.T) { l: math.MaxUint64, } for i := 0; i < slen; i++ { - fields.ou[i] = new(valdsync.Map[uint32, valueStructOu]) - fields.uo[i] = new(valdsync.Map[string, ValueStructUo]) + fields.ou[i] = new(sync.Map[uint32, valueStructOu]) + fields.uo[i] = new(sync.Map[string, ValueStructUo]) } var ( @@ -1700,8 +1699,8 @@ func Test_bidi_Range(t *testing.T) { func Test_bidi_Len(t *testing.T) { t.Parallel() type fields struct { - ou [slen]*valdsync.Map[uint32, valueStructOu] - uo [slen]*valdsync.Map[string, ValueStructUo] + ou [slen]*sync.Map[uint32, valueStructOu] + uo [slen]*sync.Map[string, ValueStructUo] l uint64 } type want struct { @@ -1784,8 +1783,8 @@ func Test_bidi_Len(t *testing.T) { // type fields struct { // concurrency int // l uint64 -// ou [slen]*ou -// uo [slen]*uo +// ou [slen]*sync.Map[uint32, valueStructOu] +// uo [slen]*sync.Map[string, ValueStructUo] // eg errgroup.Group // } // type want struct { diff --git a/pkg/agent/core/ngt/service/kvs/option.go b/pkg/agent/core/ngt/service/kvs/option.go index 30eb68724de..b2a9d3dd473 100644 --- a/pkg/agent/core/ngt/service/kvs/option.go +++ b/pkg/agent/core/ngt/service/kvs/option.go @@ -19,7 +19,7 @@ package kvs import ( "runtime" - "github.com/vdaas/vald/internal/errgroup" + "github.com/vdaas/vald/internal/sync/errgroup" ) // Option represents the functional option for bidi. diff --git a/pkg/agent/core/ngt/service/ngt.go b/pkg/agent/core/ngt/service/ngt.go index 60cbcd589db..ea144eb9fcb 100644 --- a/pkg/agent/core/ngt/service/ngt.go +++ b/pkg/agent/core/ngt/service/ngt.go @@ -27,14 +27,12 @@ import ( "path/filepath" "reflect" "runtime" - "sync" "sync/atomic" "time" "github.com/vdaas/vald/internal/config" "github.com/vdaas/vald/internal/conv" core "github.com/vdaas/vald/internal/core/algorithm/ngt" - "github.com/vdaas/vald/internal/errgroup" "github.com/vdaas/vald/internal/errors" "github.com/vdaas/vald/internal/file" "github.com/vdaas/vald/internal/log" @@ -42,6 +40,8 @@ import ( "github.com/vdaas/vald/internal/safety" "github.com/vdaas/vald/internal/slices" "github.com/vdaas/vald/internal/strings" + "github.com/vdaas/vald/internal/sync" + "github.com/vdaas/vald/internal/sync/errgroup" "github.com/vdaas/vald/pkg/agent/core/ngt/model" "github.com/vdaas/vald/pkg/agent/core/ngt/service/kvs" "github.com/vdaas/vald/pkg/agent/core/ngt/service/vqueue" diff --git a/pkg/agent/core/ngt/service/ngt_test.go b/pkg/agent/core/ngt/service/ngt_test.go index 5e9e066e5d9..2d76fc7e403 100644 --- a/pkg/agent/core/ngt/service/ngt_test.go +++ b/pkg/agent/core/ngt/service/ngt_test.go @@ -24,7 +24,6 @@ import ( "math" "os" "path/filepath" - "sync" "sync/atomic" "testing" "time" @@ -35,12 +34,13 @@ import ( "github.com/vdaas/vald/internal/client/v1/client/vald" "github.com/vdaas/vald/internal/config" core "github.com/vdaas/vald/internal/core/algorithm/ngt" - "github.com/vdaas/vald/internal/errgroup" "github.com/vdaas/vald/internal/errors" "github.com/vdaas/vald/internal/file" "github.com/vdaas/vald/internal/log" "github.com/vdaas/vald/internal/safety" "github.com/vdaas/vald/internal/strings" + "github.com/vdaas/vald/internal/sync" + "github.com/vdaas/vald/internal/sync/errgroup" testdata "github.com/vdaas/vald/internal/test" "github.com/vdaas/vald/internal/test/data/vector" "github.com/vdaas/vald/internal/test/goleak" @@ -1508,6 +1508,7 @@ func createRandomData(num int, cfg *createRandomDataConfig) []index { // // func Test_ngt_Search(t *testing.T) { // type args struct { +// ctx context.Context // vec []float32 // size uint32 // epsilon float32 @@ -1579,6 +1580,7 @@ func createRandomData(num int, cfg *createRandomDataConfig) []index { // { // name: "test_case_1", // args: args { +// ctx:nil, // vec:nil, // size:0, // epsilon:0, @@ -1639,6 +1641,7 @@ func createRandomData(num int, cfg *createRandomDataConfig) []index { // return test { // name: "test_case_2", // args: args { +// ctx:nil, // vec:nil, // size:0, // epsilon:0, @@ -1749,7 +1752,7 @@ func createRandomData(num int, cfg *createRandomDataConfig) []index { // historyLimit: test.fields.historyLimit, // } // -// got, err := n.Search(test.args.vec, test.args.size, test.args.epsilon, test.args.radius) +// got, err := n.Search(test.args.ctx, test.args.vec, test.args.size, test.args.epsilon, test.args.radius) // if err := checkFunc(test.want, got, err); err != nil { // tt.Errorf("error = %v", err) // } @@ -1760,6 +1763,7 @@ func createRandomData(num int, cfg *createRandomDataConfig) []index { // // func Test_ngt_SearchByID(t *testing.T) { // type args struct { +// ctx context.Context // uuid string // size uint32 // epsilon float32 @@ -1835,6 +1839,7 @@ func createRandomData(num int, cfg *createRandomDataConfig) []index { // { // name: "test_case_1", // args: args { +// ctx:nil, // uuid:"", // size:0, // epsilon:0, @@ -1895,6 +1900,7 @@ func createRandomData(num int, cfg *createRandomDataConfig) []index { // return test { // name: "test_case_2", // args: args { +// ctx:nil, // uuid:"", // size:0, // epsilon:0, @@ -2005,7 +2011,7 @@ func createRandomData(num int, cfg *createRandomDataConfig) []index { // historyLimit: test.fields.historyLimit, // } // -// gotVec, gotDst, err := n.SearchByID(test.args.uuid, test.args.size, test.args.epsilon, test.args.radius) +// gotVec, gotDst, err := n.SearchByID(test.args.ctx, test.args.uuid, test.args.size, test.args.epsilon, test.args.radius) // if err := checkFunc(test.want, gotVec, gotDst, err); err != nil { // tt.Errorf("error = %v", err) // } @@ -6417,25 +6423,29 @@ func createRandomData(num int, cfg *createRandomDataConfig) []index { // historyLimit int // } // type want struct { -// wantVec []float32 -// err error +// wantVec []float32 +// wantTimestamp int64 +// err error // } // type test struct { // name string // args args // fields fields // want want -// checkFunc func(want, []float32, error) error +// checkFunc func(want, []float32, int64, error) error // beforeFunc func(*testing.T, args) // afterFunc func(*testing.T, args) // } -// defaultCheckFunc := func(w want, gotVec []float32, err error) error { +// defaultCheckFunc := func(w want, gotVec []float32, gotTimestamp int64, err error) error { // if !errors.Is(err, w.err) { // return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err) // } // if !reflect.DeepEqual(gotVec, w.wantVec) { // return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", gotVec, w.wantVec) // } +// if !reflect.DeepEqual(gotTimestamp, w.wantTimestamp) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", gotTimestamp, w.wantTimestamp) +// } // return nil // } // tests := []test{ @@ -6608,8 +6618,8 @@ func createRandomData(num int, cfg *createRandomDataConfig) []index { // historyLimit: test.fields.historyLimit, // } // -// gotVec, err := n.GetObject(test.args.uuid) -// if err := checkFunc(test.want, gotVec, err); err != nil { +// gotVec, gotTimestamp, err := n.GetObject(test.args.uuid) +// if err := checkFunc(test.want, gotVec, gotTimestamp, err); err != nil { // tt.Errorf("error = %v", err) // } // @@ -9155,3 +9165,240 @@ func createRandomData(num int, cfg *createRandomDataConfig) []index { // }) // } // } +// +// func Test_ngt_ListObjectFunc(t *testing.T) { +// type args struct { +// ctx context.Context +// f func(uuid string, oid uint32, ts int64) bool +// } +// type fields struct { +// core core.NGT +// eg errgroup.Group +// kvs kvs.BidiMap +// fmap map[string]int64 +// vq vqueue.Queue +// indexing atomic.Value +// saving atomic.Value +// lastNocie uint64 +// nocie uint64 +// nogce uint64 +// wfci uint64 +// nobic uint64 +// inMem bool +// dim int +// alen int +// lim time.Duration +// dur time.Duration +// sdur time.Duration +// minLit time.Duration +// maxLit time.Duration +// litFactor time.Duration +// enableProactiveGC bool +// enableCopyOnWrite bool +// path string +// tmpPath atomic.Value +// oldPath string +// basePath string +// brokenPath string +// backupGen uint64 +// poolSize uint32 +// radius float32 +// epsilon float32 +// idelay time.Duration +// dcd bool +// kvsdbConcurrency int +// historyLimit int +// } +// type want struct { +// } +// type test struct { +// name string +// args args +// fields fields +// want want +// checkFunc func(want) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want) error { +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// ctx:nil, +// f:nil, +// }, +// fields: fields { +// core:nil, +// eg:nil, +// kvs:nil, +// fmap:nil, +// vq:nil, +// indexing:nil, +// saving:nil, +// lastNocie:0, +// nocie:0, +// nogce:0, +// wfci:0, +// nobic:0, +// inMem:false, +// dim:0, +// alen:0, +// lim:nil, +// dur:nil, +// sdur:nil, +// minLit:nil, +// maxLit:nil, +// litFactor:nil, +// enableProactiveGC:false, +// enableCopyOnWrite:false, +// path:"", +// tmpPath:nil, +// oldPath:"", +// basePath:"", +// brokenPath:"", +// backupGen:0, +// poolSize:0, +// radius:0, +// epsilon:0, +// idelay:nil, +// dcd:false, +// kvsdbConcurrency:0, +// historyLimit:0, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// ctx:nil, +// f:nil, +// }, +// fields: fields { +// core:nil, +// eg:nil, +// kvs:nil, +// fmap:nil, +// vq:nil, +// indexing:nil, +// saving:nil, +// lastNocie:0, +// nocie:0, +// nogce:0, +// wfci:0, +// nobic:0, +// inMem:false, +// dim:0, +// alen:0, +// lim:nil, +// dur:nil, +// sdur:nil, +// minLit:nil, +// maxLit:nil, +// litFactor:nil, +// enableProactiveGC:false, +// enableCopyOnWrite:false, +// path:"", +// tmpPath:nil, +// oldPath:"", +// basePath:"", +// brokenPath:"", +// backupGen:0, +// poolSize:0, +// radius:0, +// epsilon:0, +// idelay:nil, +// dcd:false, +// kvsdbConcurrency:0, +// historyLimit:0, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// n := &ngt{ +// core: test.fields.core, +// eg: test.fields.eg, +// kvs: test.fields.kvs, +// fmap: test.fields.fmap, +// vq: test.fields.vq, +// indexing: test.fields.indexing, +// saving: test.fields.saving, +// lastNocie: test.fields.lastNocie, +// nocie: test.fields.nocie, +// nogce: test.fields.nogce, +// wfci: test.fields.wfci, +// nobic: test.fields.nobic, +// inMem: test.fields.inMem, +// dim: test.fields.dim, +// alen: test.fields.alen, +// lim: test.fields.lim, +// dur: test.fields.dur, +// sdur: test.fields.sdur, +// minLit: test.fields.minLit, +// maxLit: test.fields.maxLit, +// litFactor: test.fields.litFactor, +// enableProactiveGC: test.fields.enableProactiveGC, +// enableCopyOnWrite: test.fields.enableCopyOnWrite, +// path: test.fields.path, +// tmpPath: test.fields.tmpPath, +// oldPath: test.fields.oldPath, +// basePath: test.fields.basePath, +// brokenPath: test.fields.brokenPath, +// backupGen: test.fields.backupGen, +// poolSize: test.fields.poolSize, +// radius: test.fields.radius, +// epsilon: test.fields.epsilon, +// idelay: test.fields.idelay, +// dcd: test.fields.dcd, +// kvsdbConcurrency: test.fields.kvsdbConcurrency, +// historyLimit: test.fields.historyLimit, +// } +// +// n.ListObjectFunc(test.args.ctx, test.args.f) +// if err := checkFunc(test.want); err != nil { +// tt.Errorf("error = %v", err) +// } +// }) +// } +// } diff --git a/pkg/agent/core/ngt/service/option.go b/pkg/agent/core/ngt/service/option.go index 1d0fcfb3163..1dc4a8c574b 100644 --- a/pkg/agent/core/ngt/service/option.go +++ b/pkg/agent/core/ngt/service/option.go @@ -23,10 +23,10 @@ import ( "time" core "github.com/vdaas/vald/internal/core/algorithm/ngt" - "github.com/vdaas/vald/internal/errgroup" "github.com/vdaas/vald/internal/file" "github.com/vdaas/vald/internal/rand" "github.com/vdaas/vald/internal/strings" + "github.com/vdaas/vald/internal/sync/errgroup" "github.com/vdaas/vald/internal/timeutil" ) diff --git a/pkg/agent/core/ngt/service/option_test.go b/pkg/agent/core/ngt/service/option_test.go index 326264012d5..dbea1fff566 100644 --- a/pkg/agent/core/ngt/service/option_test.go +++ b/pkg/agent/core/ngt/service/option_test.go @@ -22,8 +22,8 @@ import ( "testing" "time" - "github.com/vdaas/vald/internal/errgroup" "github.com/vdaas/vald/internal/errors" + "github.com/vdaas/vald/internal/sync/errgroup" "github.com/vdaas/vald/internal/test/goleak" ) diff --git a/pkg/agent/core/ngt/service/vqueue/queue.go b/pkg/agent/core/ngt/service/vqueue/queue.go index e20c6d91fab..78d8837d96a 100644 --- a/pkg/agent/core/ngt/service/vqueue/queue.go +++ b/pkg/agent/core/ngt/service/vqueue/queue.go @@ -26,7 +26,7 @@ import ( "github.com/vdaas/vald/internal/errors" "github.com/vdaas/vald/internal/log" "github.com/vdaas/vald/internal/slices" - valdsync "github.com/vdaas/vald/internal/sync" + "github.com/vdaas/vald/internal/sync" ) // Queue represents vector queue cache interface @@ -43,7 +43,7 @@ type Queue interface { } type vqueue struct { - il, dl valdsync.Map[string, *index] + il, dl sync.Map[string, *index] ic, dc uint64 } diff --git a/pkg/agent/core/ngt/service/vqueue/queue_test.go b/pkg/agent/core/ngt/service/vqueue/queue_test.go index 520a2437ba3..de9b139e0b1 100644 --- a/pkg/agent/core/ngt/service/vqueue/queue_test.go +++ b/pkg/agent/core/ngt/service/vqueue/queue_test.go @@ -178,8 +178,8 @@ func TestGetVector(t *testing.T) { // date int64 // } // type fields struct { -// il indexMap -// dl indexMap +// il sync.Map[string, *index] +// dl sync.Map[string, *index] // ic uint64 // dc uint64 // } @@ -212,8 +212,8 @@ func TestGetVector(t *testing.T) { // date:0, // }, // fields: fields { -// il:indexMap{}, -// dl:indexMap{}, +// il:nil, +// dl:nil, // ic:0, // dc:0, // }, @@ -239,8 +239,8 @@ func TestGetVector(t *testing.T) { // date:0, // }, // fields: fields { -// il:indexMap{}, -// dl:indexMap{}, +// il:nil, +// dl:nil, // ic:0, // dc:0, // }, @@ -294,8 +294,8 @@ func TestGetVector(t *testing.T) { // date int64 // } // type fields struct { -// il indexMap -// dl indexMap +// il sync.Map[string, *index] +// dl sync.Map[string, *index] // ic uint64 // dc uint64 // } @@ -327,8 +327,8 @@ func TestGetVector(t *testing.T) { // date:0, // }, // fields: fields { -// il:indexMap{}, -// dl:indexMap{}, +// il:nil, +// dl:nil, // ic:0, // dc:0, // }, @@ -353,8 +353,8 @@ func TestGetVector(t *testing.T) { // date:0, // }, // fields: fields { -// il:indexMap{}, -// dl:indexMap{}, +// il:nil, +// dl:nil, // ic:0, // dc:0, // }, @@ -407,30 +407,34 @@ func TestGetVector(t *testing.T) { // uuid string // } // type fields struct { -// il indexMap -// dl indexMap +// il sync.Map[string, *index] +// dl sync.Map[string, *index] // ic uint64 // dc uint64 // } // type want struct { -// want []float32 -// want1 bool +// wantVec []float32 +// wantTimestamp int64 +// wantExists bool // } // type test struct { // name string // args args // fields fields // want want -// checkFunc func(want, []float32, bool) error +// checkFunc func(want, []float32, int64, bool) error // beforeFunc func(*testing.T, args) // afterFunc func(*testing.T, args) // } -// defaultCheckFunc := func(w want, got []float32, got1 bool) error { -// if !reflect.DeepEqual(got, w.want) { -// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", got, w.want) +// defaultCheckFunc := func(w want, gotVec []float32, gotTimestamp int64, gotExists bool) error { +// if !reflect.DeepEqual(gotVec, w.wantVec) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", gotVec, w.wantVec) +// } +// if !reflect.DeepEqual(gotTimestamp, w.wantTimestamp) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", gotTimestamp, w.wantTimestamp) // } -// if !reflect.DeepEqual(got1, w.want1) { -// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", got1, w.want1) +// if !reflect.DeepEqual(gotExists, w.wantExists) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", gotExists, w.wantExists) // } // return nil // } @@ -443,8 +447,8 @@ func TestGetVector(t *testing.T) { // uuid:"", // }, // fields: fields { -// il:indexMap{}, -// dl:indexMap{}, +// il:nil, +// dl:nil, // ic:0, // dc:0, // }, @@ -468,8 +472,8 @@ func TestGetVector(t *testing.T) { // uuid:"", // }, // fields: fields { -// il:indexMap{}, -// dl:indexMap{}, +// il:nil, +// dl:nil, // ic:0, // dc:0, // }, @@ -508,8 +512,8 @@ func TestGetVector(t *testing.T) { // dc: test.fields.dc, // } // -// got, got1 := v.GetVector(test.args.uuid) -// if err := checkFunc(test.want, got, got1); err != nil { +// gotVec, gotTimestamp, gotExists := v.GetVector(test.args.uuid) +// if err := checkFunc(test.want, gotVec, gotTimestamp, gotExists); err != nil { // tt.Errorf("error = %v", err) // } // @@ -522,8 +526,8 @@ func TestGetVector(t *testing.T) { // uuid string // } // type fields struct { -// il indexMap -// dl indexMap +// il sync.Map[string, *index] +// dl sync.Map[string, *index] // ic uint64 // dc uint64 // } @@ -554,8 +558,8 @@ func TestGetVector(t *testing.T) { // uuid:"", // }, // fields: fields { -// il:indexMap{}, -// dl:indexMap{}, +// il:nil, +// dl:nil, // ic:0, // dc:0, // }, @@ -579,8 +583,8 @@ func TestGetVector(t *testing.T) { // uuid:"", // }, // fields: fields { -// il:indexMap{}, -// dl:indexMap{}, +// il:nil, +// dl:nil, // ic:0, // dc:0, // }, @@ -633,8 +637,8 @@ func TestGetVector(t *testing.T) { // uuid string // } // type fields struct { -// il indexMap -// dl indexMap +// il sync.Map[string, *index] +// dl sync.Map[string, *index] // ic uint64 // dc uint64 // } @@ -665,8 +669,8 @@ func TestGetVector(t *testing.T) { // uuid:"", // }, // fields: fields { -// il:indexMap{}, -// dl:indexMap{}, +// il:nil, +// dl:nil, // ic:0, // dc:0, // }, @@ -690,8 +694,8 @@ func TestGetVector(t *testing.T) { // uuid:"", // }, // fields: fields { -// il:indexMap{}, -// dl:indexMap{}, +// il:nil, +// dl:nil, // ic:0, // dc:0, // }, @@ -746,8 +750,8 @@ func TestGetVector(t *testing.T) { // f func(uuid string, vector []float32, date int64) bool // } // type fields struct { -// il indexMap -// dl indexMap +// il sync.Map[string, *index] +// dl sync.Map[string, *index] // ic uint64 // dc uint64 // } @@ -776,8 +780,8 @@ func TestGetVector(t *testing.T) { // f:nil, // }, // fields: fields { -// il:indexMap{}, -// dl:indexMap{}, +// il:nil, +// dl:nil, // ic:0, // dc:0, // }, @@ -803,8 +807,8 @@ func TestGetVector(t *testing.T) { // f:nil, // }, // fields: fields { -// il:indexMap{}, -// dl:indexMap{}, +// il:nil, +// dl:nil, // ic:0, // dc:0, // }, @@ -858,8 +862,8 @@ func TestGetVector(t *testing.T) { // f func(uuid string) bool // } // type fields struct { -// il indexMap -// dl indexMap +// il sync.Map[string, *index] +// dl sync.Map[string, *index] // ic uint64 // dc uint64 // } @@ -888,8 +892,8 @@ func TestGetVector(t *testing.T) { // f:nil, // }, // fields: fields { -// il:indexMap{}, -// dl:indexMap{}, +// il:nil, +// dl:nil, // ic:0, // dc:0, // }, @@ -915,8 +919,8 @@ func TestGetVector(t *testing.T) { // f:nil, // }, // fields: fields { -// il:indexMap{}, -// dl:indexMap{}, +// il:nil, +// dl:nil, // ic:0, // dc:0, // }, @@ -965,8 +969,8 @@ func TestGetVector(t *testing.T) { // // func Test_vqueue_IVQLen(t *testing.T) { // type fields struct { -// il indexMap -// dl indexMap +// il sync.Map[string, *index] +// dl sync.Map[string, *index] // ic uint64 // dc uint64 // } @@ -993,8 +997,8 @@ func TestGetVector(t *testing.T) { // { // name: "test_case_1", // fields: fields { -// il:indexMap{}, -// dl:indexMap{}, +// il:nil, +// dl:nil, // ic:0, // dc:0, // }, @@ -1015,8 +1019,8 @@ func TestGetVector(t *testing.T) { // return test { // name: "test_case_2", // fields: fields { -// il:indexMap{}, -// dl:indexMap{}, +// il:nil, +// dl:nil, // ic:0, // dc:0, // }, @@ -1066,8 +1070,8 @@ func TestGetVector(t *testing.T) { // // func Test_vqueue_DVQLen(t *testing.T) { // type fields struct { -// il indexMap -// dl indexMap +// il sync.Map[string, *index] +// dl sync.Map[string, *index] // ic uint64 // dc uint64 // } @@ -1094,8 +1098,8 @@ func TestGetVector(t *testing.T) { // { // name: "test_case_1", // fields: fields { -// il:indexMap{}, -// dl:indexMap{}, +// il:nil, +// dl:nil, // ic:0, // dc:0, // }, @@ -1116,8 +1120,8 @@ func TestGetVector(t *testing.T) { // return test { // name: "test_case_2", // fields: fields { -// il:indexMap{}, -// dl:indexMap{}, +// il:nil, +// dl:nil, // ic:0, // dc:0, // }, diff --git a/pkg/agent/core/ngt/usecase/agentd.go b/pkg/agent/core/ngt/usecase/agentd.go index 2e198edea48..99a877d3182 100644 --- a/pkg/agent/core/ngt/usecase/agentd.go +++ b/pkg/agent/core/ngt/usecase/agentd.go @@ -22,7 +22,6 @@ import ( agent "github.com/vdaas/vald/apis/grpc/v1/agent/core" vald "github.com/vdaas/vald/apis/grpc/v1/vald" iconf "github.com/vdaas/vald/internal/config" - "github.com/vdaas/vald/internal/errgroup" "github.com/vdaas/vald/internal/net/grpc" "github.com/vdaas/vald/internal/observability" ngtmetrics "github.com/vdaas/vald/internal/observability/metrics/agent/core/ngt" @@ -31,6 +30,7 @@ import ( "github.com/vdaas/vald/internal/safety" "github.com/vdaas/vald/internal/servers/server" "github.com/vdaas/vald/internal/servers/starter" + "github.com/vdaas/vald/internal/sync/errgroup" "github.com/vdaas/vald/pkg/agent/core/ngt/config" handler "github.com/vdaas/vald/pkg/agent/core/ngt/handler/grpc" "github.com/vdaas/vald/pkg/agent/core/ngt/handler/rest" diff --git a/pkg/agent/sidecar/service/observer/observer.go b/pkg/agent/sidecar/service/observer/observer.go index d7d60e1df0c..301ba59af63 100644 --- a/pkg/agent/sidecar/service/observer/observer.go +++ b/pkg/agent/sidecar/service/observer/observer.go @@ -24,11 +24,9 @@ import ( "os" "path/filepath" "reflect" - "sync" "syscall" "time" - "github.com/vdaas/vald/internal/errgroup" "github.com/vdaas/vald/internal/errors" "github.com/vdaas/vald/internal/file" "github.com/vdaas/vald/internal/file/watch" @@ -37,6 +35,8 @@ import ( "github.com/vdaas/vald/internal/observability/attribute" "github.com/vdaas/vald/internal/observability/trace" "github.com/vdaas/vald/internal/safety" + "github.com/vdaas/vald/internal/sync" + "github.com/vdaas/vald/internal/sync/errgroup" "github.com/vdaas/vald/pkg/agent/internal/metadata" "github.com/vdaas/vald/pkg/agent/sidecar/service/storage" ) diff --git a/pkg/agent/sidecar/service/observer/option.go b/pkg/agent/sidecar/service/observer/option.go index 9c69d41a4f3..9e0a1cbd93c 100644 --- a/pkg/agent/sidecar/service/observer/option.go +++ b/pkg/agent/sidecar/service/observer/option.go @@ -18,8 +18,8 @@ package observer import ( - "github.com/vdaas/vald/internal/errgroup" "github.com/vdaas/vald/internal/file" + "github.com/vdaas/vald/internal/sync/errgroup" "github.com/vdaas/vald/internal/timeutil" "github.com/vdaas/vald/pkg/agent/internal/metadata" "github.com/vdaas/vald/pkg/agent/sidecar/service/storage" diff --git a/pkg/agent/sidecar/service/restorer/option.go b/pkg/agent/sidecar/service/restorer/option.go index 5d6d8cb63de..7215d571198 100644 --- a/pkg/agent/sidecar/service/restorer/option.go +++ b/pkg/agent/sidecar/service/restorer/option.go @@ -19,7 +19,7 @@ package restorer import ( "github.com/vdaas/vald/internal/backoff" - "github.com/vdaas/vald/internal/errgroup" + "github.com/vdaas/vald/internal/sync/errgroup" "github.com/vdaas/vald/pkg/agent/sidecar/service/storage" ) diff --git a/pkg/agent/sidecar/service/restorer/restorer.go b/pkg/agent/sidecar/service/restorer/restorer.go index ea2d6739c39..d3af05ad8a7 100644 --- a/pkg/agent/sidecar/service/restorer/restorer.go +++ b/pkg/agent/sidecar/service/restorer/restorer.go @@ -26,7 +26,6 @@ import ( "syscall" "github.com/vdaas/vald/internal/backoff" - "github.com/vdaas/vald/internal/errgroup" "github.com/vdaas/vald/internal/errors" "github.com/vdaas/vald/internal/file" "github.com/vdaas/vald/internal/io" @@ -34,6 +33,7 @@ import ( "github.com/vdaas/vald/internal/observability/trace" "github.com/vdaas/vald/internal/safety" "github.com/vdaas/vald/internal/strings" + "github.com/vdaas/vald/internal/sync/errgroup" "github.com/vdaas/vald/pkg/agent/sidecar/service/storage" ) diff --git a/pkg/agent/sidecar/service/storage/option.go b/pkg/agent/sidecar/service/storage/option.go index 9e722a25164..d78e1e232db 100644 --- a/pkg/agent/sidecar/service/storage/option.go +++ b/pkg/agent/sidecar/service/storage/option.go @@ -22,7 +22,7 @@ import ( "github.com/vdaas/vald/internal/db/storage/blob/cloudstorage/urlopener" "github.com/vdaas/vald/internal/db/storage/blob/s3" "github.com/vdaas/vald/internal/db/storage/blob/s3/session" - "github.com/vdaas/vald/internal/errgroup" + "github.com/vdaas/vald/internal/sync/errgroup" ) type Option func(b *bs) error diff --git a/pkg/agent/sidecar/service/storage/storage.go b/pkg/agent/sidecar/service/storage/storage.go index 56b53031de5..5bcda32517f 100644 --- a/pkg/agent/sidecar/service/storage/storage.go +++ b/pkg/agent/sidecar/service/storage/storage.go @@ -28,9 +28,9 @@ import ( "github.com/vdaas/vald/internal/db/storage/blob/cloudstorage/urlopener" "github.com/vdaas/vald/internal/db/storage/blob/s3" "github.com/vdaas/vald/internal/db/storage/blob/s3/session" - "github.com/vdaas/vald/internal/errgroup" "github.com/vdaas/vald/internal/errors" "github.com/vdaas/vald/internal/io" + "github.com/vdaas/vald/internal/sync/errgroup" ) type Storage interface { diff --git a/pkg/agent/sidecar/usecase/initcontainer/initcontainer.go b/pkg/agent/sidecar/usecase/initcontainer/initcontainer.go index e670914921a..e153df6801a 100644 --- a/pkg/agent/sidecar/usecase/initcontainer/initcontainer.go +++ b/pkg/agent/sidecar/usecase/initcontainer/initcontainer.go @@ -25,7 +25,6 @@ import ( "github.com/vdaas/vald/internal/db/storage/blob/cloudstorage/urlopener" "github.com/vdaas/vald/internal/db/storage/blob/s3" "github.com/vdaas/vald/internal/db/storage/blob/s3/session" - "github.com/vdaas/vald/internal/errgroup" "github.com/vdaas/vald/internal/errors" "github.com/vdaas/vald/internal/io" "github.com/vdaas/vald/internal/log" @@ -37,6 +36,7 @@ import ( "github.com/vdaas/vald/internal/safety" "github.com/vdaas/vald/internal/servers/server" "github.com/vdaas/vald/internal/servers/starter" + "github.com/vdaas/vald/internal/sync/errgroup" "github.com/vdaas/vald/pkg/agent/sidecar/config" handler "github.com/vdaas/vald/pkg/agent/sidecar/handler/grpc" "github.com/vdaas/vald/pkg/agent/sidecar/handler/rest" diff --git a/pkg/agent/sidecar/usecase/sidecar/sidecar.go b/pkg/agent/sidecar/usecase/sidecar/sidecar.go index 2a4bb9b2bf5..9bcdfe9cee2 100644 --- a/pkg/agent/sidecar/usecase/sidecar/sidecar.go +++ b/pkg/agent/sidecar/usecase/sidecar/sidecar.go @@ -25,7 +25,6 @@ import ( "github.com/vdaas/vald/internal/db/storage/blob/cloudstorage/urlopener" "github.com/vdaas/vald/internal/db/storage/blob/s3" "github.com/vdaas/vald/internal/db/storage/blob/s3/session" - "github.com/vdaas/vald/internal/errgroup" "github.com/vdaas/vald/internal/log" "github.com/vdaas/vald/internal/net" "github.com/vdaas/vald/internal/net/grpc" @@ -36,6 +35,7 @@ import ( "github.com/vdaas/vald/internal/safety" "github.com/vdaas/vald/internal/servers/server" "github.com/vdaas/vald/internal/servers/starter" + "github.com/vdaas/vald/internal/sync/errgroup" "github.com/vdaas/vald/pkg/agent/sidecar/config" handler "github.com/vdaas/vald/pkg/agent/sidecar/handler/grpc" "github.com/vdaas/vald/pkg/agent/sidecar/handler/rest" diff --git a/pkg/discoverer/k8s/handler/grpc/handler.go b/pkg/discoverer/k8s/handler/grpc/handler.go index da5344bd892..6336ce9b5e3 100644 --- a/pkg/discoverer/k8s/handler/grpc/handler.go +++ b/pkg/discoverer/k8s/handler/grpc/handler.go @@ -28,8 +28,8 @@ import ( "github.com/vdaas/vald/internal/net/grpc/errdetails" "github.com/vdaas/vald/internal/net/grpc/status" "github.com/vdaas/vald/internal/observability/trace" - "github.com/vdaas/vald/internal/singleflight" "github.com/vdaas/vald/internal/strings" + "github.com/vdaas/vald/internal/sync/singleflight" "github.com/vdaas/vald/pkg/discoverer/k8s/service" ) @@ -81,7 +81,7 @@ func (s *server) Pods(ctx context.Context, req *payload.Discoverer_Request) (*pa } }() key := singleflightKey(podPrefix, req) - res, _, err := s.pgroup.Do(ctx, key, func() (*payload.Info_Pods, error) { + res, _, err := s.pgroup.Do(ctx, key, func(context.Context) (*payload.Info_Pods, error) { return s.dsc.GetPods(req) }) if err != nil { @@ -154,7 +154,7 @@ func (s *server) Nodes(ctx context.Context, req *payload.Discoverer_Request) (*p }() key := singleflightKey(nodePrefix, req) - res, _, err := s.ngroup.Do(ctx, key, func() (*payload.Info_Nodes, error) { + res, _, err := s.ngroup.Do(ctx, key, func(context.Context) (*payload.Info_Nodes, error) { return s.dsc.GetNodes(req) }) if err != nil { diff --git a/pkg/discoverer/k8s/router/option.go b/pkg/discoverer/k8s/router/option.go index 6565c6409b9..7eb7bfef693 100644 --- a/pkg/discoverer/k8s/router/option.go +++ b/pkg/discoverer/k8s/router/option.go @@ -18,7 +18,7 @@ package router import ( - "github.com/vdaas/vald/internal/errgroup" + "github.com/vdaas/vald/internal/sync/errgroup" "github.com/vdaas/vald/pkg/discoverer/k8s/handler/rest" ) diff --git a/pkg/discoverer/k8s/router/router.go b/pkg/discoverer/k8s/router/router.go index 8503a504dbb..c3f727be1cf 100644 --- a/pkg/discoverer/k8s/router/router.go +++ b/pkg/discoverer/k8s/router/router.go @@ -20,9 +20,9 @@ package router import ( "net/http" - "github.com/vdaas/vald/internal/errgroup" "github.com/vdaas/vald/internal/net/http/middleware" "github.com/vdaas/vald/internal/net/http/routing" + "github.com/vdaas/vald/internal/sync/errgroup" "github.com/vdaas/vald/pkg/discoverer/k8s/handler/rest" ) diff --git a/pkg/discoverer/k8s/service/discover.go b/pkg/discoverer/k8s/service/discover.go index c6450f5e4a0..4f3a8b0eced 100644 --- a/pkg/discoverer/k8s/service/discover.go +++ b/pkg/discoverer/k8s/service/discover.go @@ -20,13 +20,11 @@ package service import ( "context" "reflect" - "sync" "sync/atomic" "time" "github.com/vdaas/vald/apis/grpc/v1/payload" "github.com/vdaas/vald/internal/config" - "github.com/vdaas/vald/internal/errgroup" "github.com/vdaas/vald/internal/errors" "github.com/vdaas/vald/internal/k8s" mnode "github.com/vdaas/vald/internal/k8s/metrics/node" @@ -37,7 +35,8 @@ import ( "github.com/vdaas/vald/internal/net" "github.com/vdaas/vald/internal/safety" "github.com/vdaas/vald/internal/slices" - valdsync "github.com/vdaas/vald/internal/sync" + "github.com/vdaas/vald/internal/sync" + "github.com/vdaas/vald/internal/sync/errgroup" ) type Discoverer interface { @@ -48,10 +47,10 @@ type Discoverer interface { type discoverer struct { maxPods int - nodes valdsync.Map[string, *node.Node] - nodeMetrics valdsync.Map[string, mnode.Node] - pods valdsync.Map[string, *[]pod.Pod] - podMetrics valdsync.Map[string, mpod.Pod] + nodes sync.Map[string, *node.Node] + nodeMetrics sync.Map[string, mnode.Node] + pods sync.Map[string, *[]pod.Pod] + podMetrics sync.Map[string, mpod.Pod] podsByNode atomic.Value podsByNamespace atomic.Value podsByName atomic.Value diff --git a/pkg/discoverer/k8s/service/discover_test.go b/pkg/discoverer/k8s/service/discover_test.go index f7f68f5e8e3..4e8659477ab 100644 --- a/pkg/discoverer/k8s/service/discover_test.go +++ b/pkg/discoverer/k8s/service/discover_test.go @@ -118,10 +118,10 @@ package service // } // type fields struct { // maxPods int -// nodes nodeMap -// nodeMetrics nodeMetricsMap -// pods podsMap -// podMetrics podMetricsMap +// nodes sync.Map[string, *node.Node] +// nodeMetrics sync.Map[string, mnode.Node] +// pods sync.Map[string, *[]pod.Pod] +// podMetrics sync.Map[string, mpod.Pod] // podsByNode atomic.Value // podsByNamespace atomic.Value // podsByName atomic.Value @@ -165,10 +165,10 @@ package service // }, // fields: fields { // maxPods:0, -// nodes:nodeMap{}, -// nodeMetrics:nodeMetricsMap{}, -// pods:podsMap{}, -// podMetrics:podMetricsMap{}, +// nodes:nil, +// nodeMetrics:nil, +// pods:nil, +// podMetrics:nil, // podsByNode:nil, // podsByNamespace:nil, // podsByName:nil, @@ -201,10 +201,10 @@ package service // }, // fields: fields { // maxPods:0, -// nodes:nodeMap{}, -// nodeMetrics:nodeMetricsMap{}, -// pods:podsMap{}, -// podMetrics:podMetricsMap{}, +// nodes:nil, +// nodeMetrics:nil, +// pods:nil, +// podMetrics:nil, // podsByNode:nil, // podsByNamespace:nil, // podsByName:nil, @@ -277,10 +277,10 @@ package service // } // type fields struct { // maxPods int -// nodes nodeMap -// nodeMetrics nodeMetricsMap -// pods podsMap -// podMetrics podMetricsMap +// nodes sync.Map[string, *node.Node] +// nodeMetrics sync.Map[string, mnode.Node] +// pods sync.Map[string, *[]pod.Pod] +// podMetrics sync.Map[string, mpod.Pod] // podsByNode atomic.Value // podsByNamespace atomic.Value // podsByName atomic.Value @@ -324,10 +324,10 @@ package service // }, // fields: fields { // maxPods:0, -// nodes:nodeMap{}, -// nodeMetrics:nodeMetricsMap{}, -// pods:podsMap{}, -// podMetrics:podMetricsMap{}, +// nodes:nil, +// nodeMetrics:nil, +// pods:nil, +// podMetrics:nil, // podsByNode:nil, // podsByNamespace:nil, // podsByName:nil, @@ -360,10 +360,10 @@ package service // }, // fields: fields { // maxPods:0, -// nodes:nodeMap{}, -// nodeMetrics:nodeMetricsMap{}, -// pods:podsMap{}, -// podMetrics:podMetricsMap{}, +// nodes:nil, +// nodeMetrics:nil, +// pods:nil, +// podMetrics:nil, // podsByNode:nil, // podsByNamespace:nil, // podsByName:nil, @@ -436,10 +436,10 @@ package service // } // type fields struct { // maxPods int -// nodes nodeMap -// nodeMetrics nodeMetricsMap -// pods podsMap -// podMetrics podMetricsMap +// nodes sync.Map[string, *node.Node] +// nodeMetrics sync.Map[string, mnode.Node] +// pods sync.Map[string, *[]pod.Pod] +// podMetrics sync.Map[string, mpod.Pod] // podsByNode atomic.Value // podsByNamespace atomic.Value // podsByName atomic.Value @@ -483,10 +483,10 @@ package service // }, // fields: fields { // maxPods:0, -// nodes:nodeMap{}, -// nodeMetrics:nodeMetricsMap{}, -// pods:podsMap{}, -// podMetrics:podMetricsMap{}, +// nodes:nil, +// nodeMetrics:nil, +// pods:nil, +// podMetrics:nil, // podsByNode:nil, // podsByNamespace:nil, // podsByName:nil, @@ -519,10 +519,10 @@ package service // }, // fields: fields { // maxPods:0, -// nodes:nodeMap{}, -// nodeMetrics:nodeMetricsMap{}, -// pods:podsMap{}, -// podMetrics:podMetricsMap{}, +// nodes:nil, +// nodeMetrics:nil, +// pods:nil, +// podMetrics:nil, // podsByNode:nil, // podsByNamespace:nil, // podsByName:nil, diff --git a/pkg/discoverer/k8s/service/option.go b/pkg/discoverer/k8s/service/option.go index 635bdd2f544..8174679d297 100644 --- a/pkg/discoverer/k8s/service/option.go +++ b/pkg/discoverer/k8s/service/option.go @@ -20,8 +20,8 @@ package service import ( "time" - "github.com/vdaas/vald/internal/errgroup" "github.com/vdaas/vald/internal/net" + "github.com/vdaas/vald/internal/sync/errgroup" "github.com/vdaas/vald/internal/timeutil" ) diff --git a/pkg/discoverer/k8s/usecase/discovered.go b/pkg/discoverer/k8s/usecase/discovered.go index fa0f0326389..9d564fa17dd 100644 --- a/pkg/discoverer/k8s/usecase/discovered.go +++ b/pkg/discoverer/k8s/usecase/discovered.go @@ -21,7 +21,6 @@ import ( "github.com/vdaas/vald/apis/grpc/v1/discoverer" iconf "github.com/vdaas/vald/internal/config" - "github.com/vdaas/vald/internal/errgroup" "github.com/vdaas/vald/internal/log" "github.com/vdaas/vald/internal/net" "github.com/vdaas/vald/internal/net/grpc" @@ -32,6 +31,7 @@ import ( "github.com/vdaas/vald/internal/safety" "github.com/vdaas/vald/internal/servers/server" "github.com/vdaas/vald/internal/servers/starter" + "github.com/vdaas/vald/internal/sync/errgroup" "github.com/vdaas/vald/pkg/discoverer/k8s/config" handler "github.com/vdaas/vald/pkg/discoverer/k8s/handler/grpc" "github.com/vdaas/vald/pkg/discoverer/k8s/handler/rest" diff --git a/pkg/gateway/filter/handler/grpc/handler.go b/pkg/gateway/filter/handler/grpc/handler.go index 1edba3227dc..45ccf9da5a4 100644 --- a/pkg/gateway/filter/handler/grpc/handler.go +++ b/pkg/gateway/filter/handler/grpc/handler.go @@ -21,7 +21,6 @@ import ( "context" "fmt" "strconv" - "sync" "github.com/vdaas/vald/apis/grpc/v1/payload" "github.com/vdaas/vald/apis/grpc/v1/vald" @@ -29,7 +28,6 @@ import ( "github.com/vdaas/vald/internal/client/v1/client/filter/ingress" client "github.com/vdaas/vald/internal/client/v1/client/vald" "github.com/vdaas/vald/internal/core/algorithm" - "github.com/vdaas/vald/internal/errgroup" "github.com/vdaas/vald/internal/errors" "github.com/vdaas/vald/internal/info" "github.com/vdaas/vald/internal/log" @@ -39,6 +37,8 @@ import ( "github.com/vdaas/vald/internal/net/grpc/status" "github.com/vdaas/vald/internal/observability/trace" "github.com/vdaas/vald/internal/safety" + "github.com/vdaas/vald/internal/sync" + "github.com/vdaas/vald/internal/sync/errgroup" ) type server struct { diff --git a/pkg/gateway/filter/handler/grpc/option.go b/pkg/gateway/filter/handler/grpc/option.go index a0bd58a4e3b..f2694c09666 100644 --- a/pkg/gateway/filter/handler/grpc/option.go +++ b/pkg/gateway/filter/handler/grpc/option.go @@ -24,9 +24,9 @@ import ( "github.com/vdaas/vald/internal/client/v1/client/filter/egress" "github.com/vdaas/vald/internal/client/v1/client/filter/ingress" "github.com/vdaas/vald/internal/client/v1/client/vald" - "github.com/vdaas/vald/internal/errgroup" "github.com/vdaas/vald/internal/log" "github.com/vdaas/vald/internal/net" + "github.com/vdaas/vald/internal/sync/errgroup" ) type Option func(*server) diff --git a/pkg/gateway/filter/usecase/vald.go b/pkg/gateway/filter/usecase/vald.go index 447671b99bf..88fa49372e9 100644 --- a/pkg/gateway/filter/usecase/vald.go +++ b/pkg/gateway/filter/usecase/vald.go @@ -23,7 +23,6 @@ import ( "github.com/vdaas/vald/internal/client/v1/client/filter/egress" "github.com/vdaas/vald/internal/client/v1/client/filter/ingress" client "github.com/vdaas/vald/internal/client/v1/client/vald" - "github.com/vdaas/vald/internal/errgroup" "github.com/vdaas/vald/internal/errors" "github.com/vdaas/vald/internal/net/grpc" "github.com/vdaas/vald/internal/observability" @@ -34,6 +33,7 @@ import ( "github.com/vdaas/vald/internal/servers/server" "github.com/vdaas/vald/internal/servers/starter" "github.com/vdaas/vald/internal/slices" + "github.com/vdaas/vald/internal/sync/errgroup" "github.com/vdaas/vald/pkg/gateway/filter/config" handler "github.com/vdaas/vald/pkg/gateway/filter/handler/grpc" "github.com/vdaas/vald/pkg/gateway/filter/handler/rest" diff --git a/pkg/gateway/lb/handler/grpc/aggregation.go b/pkg/gateway/lb/handler/grpc/aggregation.go index 3779201327b..702abc34e67 100644 --- a/pkg/gateway/lb/handler/grpc/aggregation.go +++ b/pkg/gateway/lb/handler/grpc/aggregation.go @@ -18,13 +18,11 @@ import ( "fmt" "math" "math/big" - "sync" "sync/atomic" "time" "github.com/vdaas/vald/apis/grpc/v1/payload" "github.com/vdaas/vald/apis/grpc/v1/vald" - "github.com/vdaas/vald/internal/errgroup" "github.com/vdaas/vald/internal/errors" "github.com/vdaas/vald/internal/info" "github.com/vdaas/vald/internal/log" @@ -35,7 +33,8 @@ import ( "github.com/vdaas/vald/internal/observability/trace" "github.com/vdaas/vald/internal/safety" "github.com/vdaas/vald/internal/slices" - valdsync "github.com/vdaas/vald/internal/sync" + "github.com/vdaas/vald/internal/sync" + "github.com/vdaas/vald/internal/sync/errgroup" ) type Aggregator interface { @@ -366,7 +365,7 @@ type valdStdAggr struct { dch chan DistPayload closed atomic.Bool maxDist atomic.Value - visited valdsync.Map[string, any] + visited sync.Map[string, any] result []*payload.Object_Distance cancel context.CancelFunc } @@ -496,7 +495,7 @@ type valdPairingHeapAggr struct { num int ph *PairingHeap mu sync.Mutex - visited valdsync.Map[string, any] + visited sync.Map[string, any] result []*payload.Object_Distance } diff --git a/pkg/gateway/lb/handler/grpc/aggregation_test.go b/pkg/gateway/lb/handler/grpc/aggregation_test.go index c7a1ef1e8a3..d36e1c4de42 100644 --- a/pkg/gateway/lb/handler/grpc/aggregation_test.go +++ b/pkg/gateway/lb/handler/grpc/aggregation_test.go @@ -24,6 +24,7 @@ package grpc // dch chan DistPayload // closed atomic.Bool // maxDist atomic.Value +// visited sync.Map[string, any] // result []*payload.Object_Distance // cancel context.CancelFunc // } @@ -54,6 +55,7 @@ package grpc // dch:nil, // closed:nil, // maxDist:nil, +// visited:nil, // result:nil, // cancel:nil, // }, @@ -81,6 +83,7 @@ package grpc // dch:nil, // closed:nil, // maxDist:nil, +// visited:nil, // result:nil, // cancel:nil, // }, @@ -117,6 +120,7 @@ package grpc // dch: test.fields.dch, // closed: test.fields.closed, // maxDist: test.fields.maxDist, +// visited: test.fields.visited, // result: test.fields.result, // cancel: test.fields.cancel, // } @@ -139,6 +143,7 @@ package grpc // dch chan DistPayload // closed atomic.Bool // maxDist atomic.Value +// visited sync.Map[string, any] // result []*payload.Object_Distance // cancel context.CancelFunc // } @@ -170,6 +175,7 @@ package grpc // dch:nil, // closed:nil, // maxDist:nil, +// visited:nil, // result:nil, // cancel:nil, // }, @@ -198,6 +204,7 @@ package grpc // dch:nil, // closed:nil, // maxDist:nil, +// visited:nil, // result:nil, // cancel:nil, // }, @@ -234,6 +241,7 @@ package grpc // dch: test.fields.dch, // closed: test.fields.closed, // maxDist: test.fields.maxDist, +// visited: test.fields.visited, // result: test.fields.result, // cancel: test.fields.cancel, // } @@ -252,6 +260,7 @@ package grpc // dch chan DistPayload // closed atomic.Bool // maxDist atomic.Value +// visited sync.Map[string, any] // result []*payload.Object_Distance // cancel context.CancelFunc // } @@ -282,6 +291,7 @@ package grpc // dch:nil, // closed:nil, // maxDist:nil, +// visited:nil, // result:nil, // cancel:nil, // }, @@ -306,6 +316,7 @@ package grpc // dch:nil, // closed:nil, // maxDist:nil, +// visited:nil, // result:nil, // cancel:nil, // }, @@ -342,6 +353,7 @@ package grpc // dch: test.fields.dch, // closed: test.fields.closed, // maxDist: test.fields.maxDist, +// visited: test.fields.visited, // result: test.fields.result, // cancel: test.fields.cancel, // } @@ -360,9 +372,10 @@ package grpc // in0 context.Context // } // type fields struct { -// num int -// ph *PairingHeap -// result []*payload.Object_Distance +// num int +// ph *PairingHeap +// visited sync.Map[string, any] +// result []*payload.Object_Distance // } // type want struct { // } @@ -389,6 +402,7 @@ package grpc // fields: fields { // num:0, // ph:PairingHeap{}, +// visited:nil, // result:nil, // }, // want: want{}, @@ -413,6 +427,7 @@ package grpc // fields: fields { // num:0, // ph:PairingHeap{}, +// visited:nil, // result:nil, // }, // want: want{}, @@ -444,9 +459,10 @@ package grpc // checkFunc = defaultCheckFunc // } // v := &valdPairingHeapAggr{ -// num: test.fields.num, -// ph: test.fields.ph, -// result: test.fields.result, +// num: test.fields.num, +// ph: test.fields.ph, +// visited: test.fields.visited, +// result: test.fields.result, // } // // v.Start(test.args.in0) @@ -463,9 +479,10 @@ package grpc // data *payload.Search_Response // } // type fields struct { -// num int -// ph *PairingHeap -// result []*payload.Object_Distance +// num int +// ph *PairingHeap +// visited sync.Map[string, any] +// result []*payload.Object_Distance // } // type want struct { // } @@ -493,6 +510,7 @@ package grpc // fields: fields { // num:0, // ph:PairingHeap{}, +// visited:nil, // result:nil, // }, // want: want{}, @@ -518,6 +536,7 @@ package grpc // fields: fields { // num:0, // ph:PairingHeap{}, +// visited:nil, // result:nil, // }, // want: want{}, @@ -549,9 +568,10 @@ package grpc // checkFunc = defaultCheckFunc // } // v := &valdPairingHeapAggr{ -// num: test.fields.num, -// ph: test.fields.ph, -// result: test.fields.result, +// num: test.fields.num, +// ph: test.fields.ph, +// visited: test.fields.visited, +// result: test.fields.result, // } // // v.Send(test.args.ctx, test.args.data) @@ -564,9 +584,10 @@ package grpc // // func Test_valdPairingHeapAggr_Result(t *testing.T) { // type fields struct { -// num int -// ph *PairingHeap -// result []*payload.Object_Distance +// num int +// ph *PairingHeap +// visited sync.Map[string, any] +// result []*payload.Object_Distance // } // type want struct { // want *payload.Search_Response @@ -593,6 +614,7 @@ package grpc // fields: fields { // num:0, // ph:PairingHeap{}, +// visited:nil, // result:nil, // }, // want: want{}, @@ -614,6 +636,7 @@ package grpc // fields: fields { // num:0, // ph:PairingHeap{}, +// visited:nil, // result:nil, // }, // want: want{}, @@ -645,9 +668,10 @@ package grpc // checkFunc = defaultCheckFunc // } // v := &valdPairingHeapAggr{ -// num: test.fields.num, -// ph: test.fields.ph, -// result: test.fields.result, +// num: test.fields.num, +// ph: test.fields.ph, +// visited: test.fields.visited, +// result: test.fields.result, // } // // got := v.Result() diff --git a/pkg/gateway/lb/handler/grpc/handler.go b/pkg/gateway/lb/handler/grpc/handler.go index 30f14f00052..2f25951b132 100644 --- a/pkg/gateway/lb/handler/grpc/handler.go +++ b/pkg/gateway/lb/handler/grpc/handler.go @@ -21,7 +21,6 @@ import ( "context" "fmt" "strconv" - "sync" "sync/atomic" "time" @@ -29,7 +28,6 @@ import ( "github.com/vdaas/vald/apis/grpc/v1/vald" "github.com/vdaas/vald/internal/conv" "github.com/vdaas/vald/internal/core/algorithm" - "github.com/vdaas/vald/internal/errgroup" "github.com/vdaas/vald/internal/errors" "github.com/vdaas/vald/internal/info" "github.com/vdaas/vald/internal/log" @@ -42,6 +40,8 @@ import ( "github.com/vdaas/vald/internal/safety" "github.com/vdaas/vald/internal/slices" "github.com/vdaas/vald/internal/strings" + "github.com/vdaas/vald/internal/sync" + "github.com/vdaas/vald/internal/sync/errgroup" "github.com/vdaas/vald/pkg/gateway/lb/service" ) @@ -1484,7 +1484,7 @@ func (s *server) MultiInsert(ctx context.Context, reqs *payload.Insert_MultiRequ lmu sync.Mutex ) eg, ectx := errgroup.New(ctx) - eg.Limitation(s.multiConcurrency) + eg.SetLimit(s.multiConcurrency) locs = &payload.Object_Locations{ Locations: make([]*payload.Object_Location, len(reqs.GetRequests())), } @@ -2021,7 +2021,7 @@ func (s *server) MultiUpdate(ctx context.Context, reqs *payload.Update_MultiRequ lmu sync.Mutex ) eg, ectx := errgroup.New(ctx) - eg.Limitation(s.multiConcurrency) + eg.SetLimit(s.multiConcurrency) locs = &payload.Object_Locations{ Locations: make([]*payload.Object_Location, len(reqs.GetRequests())), } @@ -2370,7 +2370,7 @@ func (s *server) MultiUpsert(ctx context.Context, reqs *payload.Upsert_MultiRequ lmu sync.Mutex ) eg, ectx := errgroup.New(ctx) - eg.Limitation(s.multiConcurrency) + eg.SetLimit(s.multiConcurrency) locs = &payload.Object_Locations{ Locations: make([]*payload.Object_Location, len(reqs.GetRequests())), } @@ -2674,7 +2674,7 @@ func (s *server) MultiRemove(ctx context.Context, reqs *payload.Remove_MultiRequ lmu sync.Mutex ) eg, ectx := errgroup.New(ctx) - eg.Limitation(s.multiConcurrency) + eg.SetLimit(s.multiConcurrency) locs = &payload.Object_Locations{ Locations: make([]*payload.Object_Location, len(reqs.GetRequests())), } diff --git a/pkg/gateway/lb/handler/grpc/option.go b/pkg/gateway/lb/handler/grpc/option.go index d1a5d01e930..cbb0ef7f6cb 100644 --- a/pkg/gateway/lb/handler/grpc/option.go +++ b/pkg/gateway/lb/handler/grpc/option.go @@ -22,9 +22,9 @@ import ( "runtime" "time" - "github.com/vdaas/vald/internal/errgroup" "github.com/vdaas/vald/internal/log" "github.com/vdaas/vald/internal/net" + "github.com/vdaas/vald/internal/sync/errgroup" "github.com/vdaas/vald/internal/timeutil" "github.com/vdaas/vald/pkg/gateway/lb/service" ) diff --git a/pkg/gateway/lb/handler/grpc/search_benchmark_test.go b/pkg/gateway/lb/handler/grpc/search_benchmark_test.go index 91029300878..834d9827192 100644 --- a/pkg/gateway/lb/handler/grpc/search_benchmark_test.go +++ b/pkg/gateway/lb/handler/grpc/search_benchmark_test.go @@ -18,15 +18,15 @@ import ( "flag" "fmt" "runtime" - "sync" "sync/atomic" "testing" "time" "github.com/vdaas/vald/apis/grpc/v1/payload" - "github.com/vdaas/vald/internal/errgroup" "github.com/vdaas/vald/internal/rand" "github.com/vdaas/vald/internal/slices" + "github.com/vdaas/vald/internal/sync" + "github.com/vdaas/vald/internal/sync/errgroup" "github.com/vdaas/vald/internal/test/data/strings" ) @@ -116,7 +116,7 @@ func doSearchWithAggregator(ctx context.Context, k, concurrency int, anew func(n f func(ctx context.Context) *payload.Search_Response, ) (res *payload.Search_Response, err error) { eg, ectx := errgroup.New(ctx) - eg.Limitation(concurrency) + eg.SetLimit(concurrency) aggr := anew(k, concurrency) aggr.Start(ectx) for i := 0; i < concurrency; i++ { diff --git a/pkg/gateway/lb/service/gateway.go b/pkg/gateway/lb/service/gateway.go index 1b046637e6d..47ea5c75ffc 100644 --- a/pkg/gateway/lb/service/gateway.go +++ b/pkg/gateway/lb/service/gateway.go @@ -24,11 +24,11 @@ import ( "github.com/vdaas/vald/apis/grpc/v1/vald" "github.com/vdaas/vald/internal/client/v1/client/discoverer" - "github.com/vdaas/vald/internal/errgroup" "github.com/vdaas/vald/internal/errors" "github.com/vdaas/vald/internal/net/grpc" "github.com/vdaas/vald/internal/observability/trace" - valdsync "github.com/vdaas/vald/internal/sync" + "github.com/vdaas/vald/internal/sync" + "github.com/vdaas/vald/internal/sync/errgroup" ) type Gateway interface { @@ -102,7 +102,7 @@ func (g *gateway) DoMulti(ctx context.Context, num int, } else { limit = uint32(num) } - var visited valdsync.Map[string, any] + var visited sync.Map[string, any] err = g.client.GetClient().OrderedRange(sctx, addrs, func(ictx context.Context, addr string, conn *grpc.ClientConn, diff --git a/pkg/gateway/lb/service/option.go b/pkg/gateway/lb/service/option.go index c88d2107d95..988e7fa7322 100644 --- a/pkg/gateway/lb/service/option.go +++ b/pkg/gateway/lb/service/option.go @@ -19,7 +19,7 @@ package service import ( "github.com/vdaas/vald/internal/client/v1/client/discoverer" - "github.com/vdaas/vald/internal/errgroup" + "github.com/vdaas/vald/internal/sync/errgroup" ) type Option func(g *gateway) error diff --git a/pkg/gateway/lb/usecase/vald.go b/pkg/gateway/lb/usecase/vald.go index baa08e94d93..e5b8abce48d 100644 --- a/pkg/gateway/lb/usecase/vald.go +++ b/pkg/gateway/lb/usecase/vald.go @@ -22,7 +22,6 @@ import ( "github.com/vdaas/vald/apis/grpc/v1/vald" "github.com/vdaas/vald/internal/client/v1/client/discoverer" - "github.com/vdaas/vald/internal/errgroup" "github.com/vdaas/vald/internal/net/grpc" "github.com/vdaas/vald/internal/observability" backoffmetrics "github.com/vdaas/vald/internal/observability/metrics/backoff" @@ -31,6 +30,7 @@ import ( "github.com/vdaas/vald/internal/safety" "github.com/vdaas/vald/internal/servers/server" "github.com/vdaas/vald/internal/servers/starter" + "github.com/vdaas/vald/internal/sync/errgroup" "github.com/vdaas/vald/pkg/gateway/lb/config" handler "github.com/vdaas/vald/pkg/gateway/lb/handler/grpc" "github.com/vdaas/vald/pkg/gateway/lb/handler/rest" diff --git a/pkg/manager/index/service/indexer.go b/pkg/manager/index/service/indexer.go index 365db870c6f..ca6eb8440ba 100644 --- a/pkg/manager/index/service/indexer.go +++ b/pkg/manager/index/service/indexer.go @@ -27,7 +27,6 @@ import ( agent "github.com/vdaas/vald/apis/grpc/v1/agent/core" "github.com/vdaas/vald/apis/grpc/v1/payload" "github.com/vdaas/vald/internal/client/v1/client/discoverer" - "github.com/vdaas/vald/internal/errgroup" "github.com/vdaas/vald/internal/errors" "github.com/vdaas/vald/internal/log" "github.com/vdaas/vald/internal/net/grpc" @@ -35,7 +34,8 @@ import ( "github.com/vdaas/vald/internal/net/grpc/status" "github.com/vdaas/vald/internal/observability/trace" "github.com/vdaas/vald/internal/safety" - valdsync "github.com/vdaas/vald/internal/sync" + "github.com/vdaas/vald/internal/sync" + "github.com/vdaas/vald/internal/sync/errgroup" ) type Indexer interface { @@ -54,9 +54,9 @@ type index struct { saveIndexDurationLimit time.Duration saveIndexWaitDuration time.Duration saveIndexTargetAddrCh chan string - schMap valdsync.Map[string, any] + schMap sync.Map[string, any] concurrency int - indexInfos valdsync.Map[string, *payload.Info_Index_Count] + indexInfos sync.Map[string, *payload.Info_Index_Count] indexing atomic.Value // bool minUncommitted uint32 uuidsCount uint32 @@ -276,7 +276,7 @@ func (idx *index) loadInfos(ctx context.Context) (err error) { }() var u, ucu uint32 - var infoMap valdsync.Map[string, *payload.Info_Index_Count] + var infoMap sync.Map[string, *payload.Info_Index_Count] err = idx.client.GetClient().RangeConcurrent(ctx, len(idx.client.GetAddrs(ctx)), func(ctx context.Context, addr string, conn *grpc.ClientConn, copts ...grpc.CallOption, diff --git a/pkg/manager/index/service/indexer_test.go b/pkg/manager/index/service/indexer_test.go index 226e926c072..a5847b45e53 100644 --- a/pkg/manager/index/service/indexer_test.go +++ b/pkg/manager/index/service/indexer_test.go @@ -122,8 +122,9 @@ package service // saveIndexDurationLimit time.Duration // saveIndexWaitDuration time.Duration // saveIndexTargetAddrCh chan string +// schMap sync.Map[string, any] // concurrency int -// indexInfos indexInfos +// indexInfos sync.Map[string, *payload.Info_Index_Count] // indexing atomic.Value // minUncommitted uint32 // uuidsCount uint32 @@ -168,8 +169,9 @@ package service // saveIndexDurationLimit:nil, // saveIndexWaitDuration:nil, // saveIndexTargetAddrCh:nil, +// schMap:nil, // concurrency:0, -// indexInfos:indexInfos{}, +// indexInfos:nil, // indexing:nil, // minUncommitted:0, // uuidsCount:0, @@ -203,8 +205,9 @@ package service // saveIndexDurationLimit:nil, // saveIndexWaitDuration:nil, // saveIndexTargetAddrCh:nil, +// schMap:nil, // concurrency:0, -// indexInfos:indexInfos{}, +// indexInfos:nil, // indexing:nil, // minUncommitted:0, // uuidsCount:0, @@ -247,6 +250,7 @@ package service // saveIndexDurationLimit: test.fields.saveIndexDurationLimit, // saveIndexWaitDuration: test.fields.saveIndexWaitDuration, // saveIndexTargetAddrCh: test.fields.saveIndexTargetAddrCh, +// schMap: test.fields.schMap, // concurrency: test.fields.concurrency, // indexInfos: test.fields.indexInfos, // indexing: test.fields.indexing, @@ -274,8 +278,9 @@ package service // saveIndexDurationLimit time.Duration // saveIndexWaitDuration time.Duration // saveIndexTargetAddrCh chan string +// schMap sync.Map[string, any] // concurrency int -// indexInfos indexInfos +// indexInfos sync.Map[string, *payload.Info_Index_Count] // indexing atomic.Value // minUncommitted uint32 // uuidsCount uint32 @@ -312,8 +317,9 @@ package service // saveIndexDurationLimit:nil, // saveIndexWaitDuration:nil, // saveIndexTargetAddrCh:nil, +// schMap:nil, // concurrency:0, -// indexInfos:indexInfos{}, +// indexInfos:nil, // indexing:nil, // minUncommitted:0, // uuidsCount:0, @@ -344,8 +350,9 @@ package service // saveIndexDurationLimit:nil, // saveIndexWaitDuration:nil, // saveIndexTargetAddrCh:nil, +// schMap:nil, // concurrency:0, -// indexInfos:indexInfos{}, +// indexInfos:nil, // indexing:nil, // minUncommitted:0, // uuidsCount:0, @@ -388,6 +395,7 @@ package service // saveIndexDurationLimit: test.fields.saveIndexDurationLimit, // saveIndexWaitDuration: test.fields.saveIndexWaitDuration, // saveIndexTargetAddrCh: test.fields.saveIndexTargetAddrCh, +// schMap: test.fields.schMap, // concurrency: test.fields.concurrency, // indexInfos: test.fields.indexInfos, // indexing: test.fields.indexing, @@ -415,8 +423,9 @@ package service // saveIndexDurationLimit time.Duration // saveIndexWaitDuration time.Duration // saveIndexTargetAddrCh chan string +// schMap sync.Map[string, any] // concurrency int -// indexInfos indexInfos +// indexInfos sync.Map[string, *payload.Info_Index_Count] // indexing atomic.Value // minUncommitted uint32 // uuidsCount uint32 @@ -453,8 +462,9 @@ package service // saveIndexDurationLimit:nil, // saveIndexWaitDuration:nil, // saveIndexTargetAddrCh:nil, +// schMap:nil, // concurrency:0, -// indexInfos:indexInfos{}, +// indexInfos:nil, // indexing:nil, // minUncommitted:0, // uuidsCount:0, @@ -485,8 +495,9 @@ package service // saveIndexDurationLimit:nil, // saveIndexWaitDuration:nil, // saveIndexTargetAddrCh:nil, +// schMap:nil, // concurrency:0, -// indexInfos:indexInfos{}, +// indexInfos:nil, // indexing:nil, // minUncommitted:0, // uuidsCount:0, @@ -529,6 +540,7 @@ package service // saveIndexDurationLimit: test.fields.saveIndexDurationLimit, // saveIndexWaitDuration: test.fields.saveIndexWaitDuration, // saveIndexTargetAddrCh: test.fields.saveIndexTargetAddrCh, +// schMap: test.fields.schMap, // concurrency: test.fields.concurrency, // indexInfos: test.fields.indexInfos, // indexing: test.fields.indexing, @@ -556,8 +568,9 @@ package service // saveIndexDurationLimit time.Duration // saveIndexWaitDuration time.Duration // saveIndexTargetAddrCh chan string +// schMap sync.Map[string, any] // concurrency int -// indexInfos indexInfos +// indexInfos sync.Map[string, *payload.Info_Index_Count] // indexing atomic.Value // minUncommitted uint32 // uuidsCount uint32 @@ -594,8 +607,9 @@ package service // saveIndexDurationLimit:nil, // saveIndexWaitDuration:nil, // saveIndexTargetAddrCh:nil, +// schMap:nil, // concurrency:0, -// indexInfos:indexInfos{}, +// indexInfos:nil, // indexing:nil, // minUncommitted:0, // uuidsCount:0, @@ -626,8 +640,9 @@ package service // saveIndexDurationLimit:nil, // saveIndexWaitDuration:nil, // saveIndexTargetAddrCh:nil, +// schMap:nil, // concurrency:0, -// indexInfos:indexInfos{}, +// indexInfos:nil, // indexing:nil, // minUncommitted:0, // uuidsCount:0, @@ -670,6 +685,7 @@ package service // saveIndexDurationLimit: test.fields.saveIndexDurationLimit, // saveIndexWaitDuration: test.fields.saveIndexWaitDuration, // saveIndexTargetAddrCh: test.fields.saveIndexTargetAddrCh, +// schMap: test.fields.schMap, // concurrency: test.fields.concurrency, // indexInfos: test.fields.indexInfos, // indexing: test.fields.indexing, diff --git a/pkg/manager/index/service/option.go b/pkg/manager/index/service/option.go index a675c656efa..d10f86dbfed 100644 --- a/pkg/manager/index/service/option.go +++ b/pkg/manager/index/service/option.go @@ -19,7 +19,7 @@ package service import ( "github.com/vdaas/vald/internal/client/v1/client/discoverer" - "github.com/vdaas/vald/internal/errgroup" + "github.com/vdaas/vald/internal/sync/errgroup" "github.com/vdaas/vald/internal/timeutil" ) diff --git a/pkg/manager/index/usecase/indexer.go b/pkg/manager/index/usecase/indexer.go index 11aa90054d4..9619bddec19 100644 --- a/pkg/manager/index/usecase/indexer.go +++ b/pkg/manager/index/usecase/indexer.go @@ -22,7 +22,6 @@ import ( "github.com/vdaas/vald/apis/grpc/v1/manager/index" "github.com/vdaas/vald/internal/client/v1/client/discoverer" iconf "github.com/vdaas/vald/internal/config" - "github.com/vdaas/vald/internal/errgroup" "github.com/vdaas/vald/internal/net/grpc" "github.com/vdaas/vald/internal/observability" backoffmetrics "github.com/vdaas/vald/internal/observability/metrics/backoff" @@ -32,6 +31,7 @@ import ( "github.com/vdaas/vald/internal/safety" "github.com/vdaas/vald/internal/servers/server" "github.com/vdaas/vald/internal/servers/starter" + "github.com/vdaas/vald/internal/sync/errgroup" "github.com/vdaas/vald/pkg/manager/index/config" handler "github.com/vdaas/vald/pkg/manager/index/handler/grpc" "github.com/vdaas/vald/pkg/manager/index/handler/rest" diff --git a/pkg/tools/cli/loadtest/service/loader.go b/pkg/tools/cli/loadtest/service/loader.go index 216aa72a47a..be1adf71974 100644 --- a/pkg/tools/cli/loadtest/service/loader.go +++ b/pkg/tools/cli/loadtest/service/loader.go @@ -22,11 +22,11 @@ import ( "time" "github.com/vdaas/vald/apis/grpc/v1/payload" - "github.com/vdaas/vald/internal/errgroup" "github.com/vdaas/vald/internal/errors" "github.com/vdaas/vald/internal/log" "github.com/vdaas/vald/internal/net/grpc" "github.com/vdaas/vald/internal/safety" + "github.com/vdaas/vald/internal/sync/errgroup" "github.com/vdaas/vald/pkg/tools/cli/loadtest/assets" "github.com/vdaas/vald/pkg/tools/cli/loadtest/config" ) @@ -217,7 +217,7 @@ func (l *loader) do(ctx context.Context, f func(interface{}, error), notify func })) err = eg.Wait() case config.Insert, config.Search: - eg.Limitation(l.concurrency) + eg.SetLimit(l.concurrency) for { r := l.dataProvider() diff --git a/pkg/tools/cli/loadtest/service/loader_option.go b/pkg/tools/cli/loadtest/service/loader_option.go index e96fc944e41..0ac342d2ad3 100644 --- a/pkg/tools/cli/loadtest/service/loader_option.go +++ b/pkg/tools/cli/loadtest/service/loader_option.go @@ -14,9 +14,9 @@ package service import ( - "github.com/vdaas/vald/internal/errgroup" "github.com/vdaas/vald/internal/errors" "github.com/vdaas/vald/internal/net/grpc" + "github.com/vdaas/vald/internal/sync/errgroup" "github.com/vdaas/vald/internal/timeutil" "github.com/vdaas/vald/pkg/tools/cli/loadtest/config" ) diff --git a/pkg/tools/cli/loadtest/usecase/load.go b/pkg/tools/cli/loadtest/usecase/load.go index dc67c02c0d3..501a45f0bb6 100644 --- a/pkg/tools/cli/loadtest/usecase/load.go +++ b/pkg/tools/cli/loadtest/usecase/load.go @@ -19,12 +19,12 @@ package usecase import ( "context" - "github.com/vdaas/vald/internal/errgroup" "github.com/vdaas/vald/internal/errors" "github.com/vdaas/vald/internal/log" "github.com/vdaas/vald/internal/net/grpc" "github.com/vdaas/vald/internal/runner" "github.com/vdaas/vald/internal/safety" + "github.com/vdaas/vald/internal/sync/errgroup" "github.com/vdaas/vald/pkg/tools/cli/loadtest/config" "github.com/vdaas/vald/pkg/tools/cli/loadtest/service" ) diff --git a/tests/e2e/operation/stream.go b/tests/e2e/operation/stream.go index 221b41aa62d..143399191a7 100644 --- a/tests/e2e/operation/stream.go +++ b/tests/e2e/operation/stream.go @@ -19,7 +19,6 @@ import ( "context" "reflect" "strconv" - "sync" "testing" "github.com/vdaas/vald/apis/grpc/v1/payload" @@ -29,6 +28,7 @@ import ( "github.com/vdaas/vald/internal/net/grpc/errdetails" "github.com/vdaas/vald/internal/net/grpc/status" "github.com/vdaas/vald/internal/strings" + "github.com/vdaas/vald/internal/sync" ) type ( diff --git a/tests/e2e/pkg/agent/core/ngt/service/ngt_e2s_test.go b/tests/e2e/pkg/agent/core/ngt/service/ngt_e2s_test.go index e4c5e7d8324..d16f47bcdad 100644 --- a/tests/e2e/pkg/agent/core/ngt/service/ngt_e2s_test.go +++ b/tests/e2e/pkg/agent/core/ngt/service/ngt_e2s_test.go @@ -20,12 +20,12 @@ package service import ( "context" "strconv" - "sync" "testing" "time" "github.com/vdaas/vald/internal/config" "github.com/vdaas/vald/internal/errors" + "github.com/vdaas/vald/internal/sync" "github.com/vdaas/vald/pkg/agent/core/ngt/service" ) diff --git a/tests/performance/max_vector_dim_test.go b/tests/performance/max_vector_dim_test.go index d6a1a9eb22c..89a9c5c5406 100644 --- a/tests/performance/max_vector_dim_test.go +++ b/tests/performance/max_vector_dim_test.go @@ -19,7 +19,6 @@ import ( "flag" "os" "strconv" - "sync" "testing" "time" @@ -27,11 +26,12 @@ import ( "github.com/vdaas/vald/internal/config" "github.com/vdaas/vald/internal/core/algorithm" "github.com/vdaas/vald/internal/core/algorithm/ngt" - "github.com/vdaas/vald/internal/errgroup" "github.com/vdaas/vald/internal/errors" "github.com/vdaas/vald/internal/log" "github.com/vdaas/vald/internal/safety" "github.com/vdaas/vald/internal/strings" + "github.com/vdaas/vald/internal/sync" + "github.com/vdaas/vald/internal/sync/errgroup" "github.com/vdaas/vald/internal/test/data/vector" "github.com/vdaas/vald/pkg/agent/core/ngt/handler/grpc" "github.com/vdaas/vald/pkg/agent/core/ngt/service" diff --git a/versions/GOLANGCILINT_VERSION b/versions/GOLANGCILINT_VERSION index cdbd266b869..853ab7965a2 100644 --- a/versions/GOLANGCILINT_VERSION +++ b/versions/GOLANGCILINT_VERSION @@ -1 +1 @@ -v1.53.3 +v1.54.1 diff --git a/versions/GO_VERSION b/versions/GO_VERSION index 7ad32779d83..3500250a4b0 100644 --- a/versions/GO_VERSION +++ b/versions/GO_VERSION @@ -1 +1 @@ -1.20.6 \ No newline at end of file +1.21.0 diff --git a/versions/HELM_VERSION b/versions/HELM_VERSION index 10d3e4ccb11..db8770749b3 100644 --- a/versions/HELM_VERSION +++ b/versions/HELM_VERSION @@ -1 +1 @@ -v3.12.1 +v3.12.3 diff --git a/versions/JAEGER_OPERATOR_VERSION b/versions/JAEGER_OPERATOR_VERSION index b8c7fd6aca7..64f878193d4 100644 --- a/versions/JAEGER_OPERATOR_VERSION +++ b/versions/JAEGER_OPERATOR_VERSION @@ -1 +1 @@ -2.46.1 +2.46.2 diff --git a/versions/KUBECTL_VERSION b/versions/KUBECTL_VERSION index 6c6ba5f810b..6f6fa3c2702 100644 --- a/versions/KUBECTL_VERSION +++ b/versions/KUBECTL_VERSION @@ -1 +1 @@ -v1.27.3 \ No newline at end of file +v1.27.4 \ No newline at end of file diff --git a/versions/NGT_VERSION b/versions/NGT_VERSION index a14da2902b1..3e3c2f1e5ed 100644 --- a/versions/NGT_VERSION +++ b/versions/NGT_VERSION @@ -1 +1 @@ -2.0.16 +2.1.1 diff --git a/versions/OPERATOR_SDK_VERSION b/versions/OPERATOR_SDK_VERSION index 17e22489a72..f3f644dcf8d 100644 --- a/versions/OPERATOR_SDK_VERSION +++ b/versions/OPERATOR_SDK_VERSION @@ -1 +1 @@ -v1.30 +v1.31 diff --git a/versions/PROMETHEUS_STACK_VERSION b/versions/PROMETHEUS_STACK_VERSION index b48484e16a4..57fef3c8825 100644 --- a/versions/PROMETHEUS_STACK_VERSION +++ b/versions/PROMETHEUS_STACK_VERSION @@ -1 +1 @@ -47.6.1 +48.3.1 diff --git a/versions/PROTOBUF_VERSION b/versions/PROTOBUF_VERSION index d3013b2e0f5..d9133a54b63 100644 --- a/versions/PROTOBUF_VERSION +++ b/versions/PROTOBUF_VERSION @@ -1 +1 @@ -23.4 +24.0 diff --git a/versions/TELEPRESENCE_VERSION b/versions/TELEPRESENCE_VERSION index 8b5925249b1..5378f221e11 100644 --- a/versions/TELEPRESENCE_VERSION +++ b/versions/TELEPRESENCE_VERSION @@ -1 +1 @@ -2.14.1-rc.3 +2.15.0-rc.1 diff --git a/versions/YQ_VERSION b/versions/YQ_VERSION index d8b501ab260..5c4432ea28e 100644 --- a/versions/YQ_VERSION +++ b/versions/YQ_VERSION @@ -1 +1 @@ -v4.34.1 +v4.35.1