diff --git a/.golangci.yml b/.golangci.yml
index d5b4709ea2f..45dc7f56af2 100644
--- a/.golangci.yml
+++ b/.golangci.yml
@@ -149,7 +149,14 @@ linters:
- path: internal/errors/errors_benchmark_test\.go
linters:
- depguard
+<<<<<<< HEAD
- path: internal/test/comparator/standard\.go
+=======
+ - path: internal/sync/alias\.go
+ linters:
+ - depguard
+ - path: hack/benchmark/src/singleflight/singleflight_bench_test\.go
+>>>>>>> refactor/manager-index/small-refactor
linters:
- depguard
linters-settings:
@@ -169,6 +176,14 @@ linters-settings:
rules:
main:
deny:
+ - pkg: "sync"
+ desc: "sync is allowed only by internal/sync"
+ - pkg: "golang.org/x/sync"
+ desc: "golang.org/x/sync is allowed only by internal/sync"
+ - pkg: "golang.org/x/sync/singleflight"
+ desc: "golang.org/x/sync/singleflight is allowed only by internal/sync/singleflight"
+ - pkg: "golang.org/x/sync/semaphore"
+ desc: "golang.org/x/sync/semaphore is allowed only by internal/sync/semaphore"
- pkg: "errors"
desc: "errors is allowed only by internal/errors"
- pkg: "k8s.io/apimachinery/pkg/api/errors"
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 25a6a697d59..c2894fcc056 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -4750,7 +4750,7 @@
- :white_check_mark: add internal/config/ngt test ([#554](https://github.com/vdaas/vald/pull/554))
- :white_check_mark: internal/cache/cacher test ([#553](https://github.com/vdaas/vald/pull/553))
- :white_check_mark: Add test case for `internal/file` ([#550](https://github.com/vdaas/vald/pull/550))
-- :white_check_mark: add internal/singleflight test ([#542](https://github.com/vdaas/vald/pull/542))
+- :white_check_mark: add internal/sync/singleflight test ([#542](https://github.com/vdaas/vald/pull/542))
- not to force rebuild gotests ([#548](https://github.com/vdaas/vald/pull/548))
- :pencil: Add use case document ([#482](https://github.com/vdaas/vald/pull/482))
- :white_check_mark: add internal/log/mock/retry test ([#549](https://github.com/vdaas/vald/pull/549))
@@ -5272,7 +5272,7 @@
- internal gRPC client for Vald
- Cassandra NewConvictionPolicy
- dicoverer now returns clone object
-- new internal/singleflight package
+- new internal/sync/singleflight package
- new internal/net package
- coding guideline
diff --git a/Makefile b/Makefile
index ec122ea034c..6e799374ea2 100644
--- a/Makefile
+++ b/Makefile
@@ -361,7 +361,7 @@ clean:
license:
GOPRIVATE=$(GOPRIVATE) \
MAINTAINER=$(MAINTAINER) \
- go run -mod=readonly hack/license/gen/main.go ./
+ go run -mod=readonly hack/license/gen/main.go $(ROOTDIR)
.PHONY: init
## initialize development environment
diff --git a/Makefile.d/proto.mk b/Makefile.d/proto.mk
index a98893193b2..592c6bcc49e 100644
--- a/Makefile.d/proto.mk
+++ b/Makefile.d/proto.mk
@@ -138,6 +138,7 @@ $(PBGOS): \
find $(ROOTDIR)/apis/grpc/* -name '*.go' | xargs sed -i -E "s%google.golang.org/grpc/codes%github.com/vdaas/vald/internal/net/grpc/codes%g"
find $(ROOTDIR)/apis/grpc/* -name '*.go' | xargs sed -i -E "s%google.golang.org/grpc/status%github.com/vdaas/vald/internal/net/grpc/status%g"
find $(ROOTDIR)/apis/grpc/* -name '*.go' | xargs sed -i -E "s%\"io\"%\"github.com/vdaas/vald/internal/io\"%g"
+ find $(ROOTDIR)/apis/grpc/* -name '*.go' | xargs sed -i -E "s%\"sync\"%\"github.com/vdaas/vald/internal/sync\"%g"
$(SWAGGERS): \
$(PROTOS) \
diff --git a/Makefile.d/test.mk b/Makefile.d/test.mk
index b85e6401dd0..f591103b6ad 100644
--- a/Makefile.d/test.mk
+++ b/Makefile.d/test.mk
@@ -284,7 +284,7 @@ gotests/patch:
@$(call green, "apply patches to go test files...")
find $(ROOTDIR)/internal/k8s/* -name '*_test.go' | xargs sed -i -E "s%k8s.io/apimachinery/pkg/api/errors%github.com/vdaas/vald/internal/errors%g"
find $(ROOTDIR)/* -name '*_test.go' | xargs sed -i -E "s%cockroachdb/errors%vdaas/vald/internal/errors%g"
- find $(ROOTDIR)/* -name '*_test.go' | xargs sed -i -E "s%golang.org/x/sync/errgroup%github.com/vdaas/vald/internal/errgroup%g"
+ find $(ROOTDIR)/* -name '*_test.go' | xargs sed -i -E "s%golang.org/x/sync/errgroup%github.com/vdaas/vald/internal/sync/errgroup%g"
find $(ROOTDIR)/* -name '*_test.go' | xargs sed -i -E "s%pkg/errors%vdaas/vald/internal/errors%g"
find $(ROOTDIR)/* -name '*_test.go' | xargs sed -i -E "s%go-errors/errors%vdaas/vald/internal/errors%g"
find $(ROOTDIR)/* -name '*_test.go' | xargs sed -i -E "s%go.uber.org/goleak%github.com/vdaas/vald/internal/test/goleak%g"
diff --git a/apis/docs/v1/docs.md b/apis/docs/v1/docs.md
index 24bf7ebea9c..e4e3a31c4f8 100644
--- a/apis/docs/v1/docs.md
+++ b/apis/docs/v1/docs.md
@@ -4,16 +4,18 @@
## Table of Contents
+- [apis/proto/v1/agent/core/agent.proto](#apis_proto_v1_agent_core_agent-proto)
+ - [Agent](#core-v1-Agent)
+- [apis/proto/v1/agent/sidecar/sidecar.proto](#apis_proto_v1_agent_sidecar_sidecar-proto)
+ - [Sidecar](#sidecar-v1-Sidecar)
+- [apis/proto/v1/discoverer/discoverer.proto](#apis_proto_v1_discoverer_discoverer-proto)
+ - [Discoverer](#discoverer-v1-Discoverer)
- [apis/proto/v1/filter/egress/egress_filter.proto](#apis_proto_v1_filter_egress_egress_filter-proto)
- [Filter](#filter-egress-v1-Filter)
- [apis/proto/v1/filter/ingress/ingress_filter.proto](#apis_proto_v1_filter_ingress_ingress_filter-proto)
- [Filter](#filter-ingress-v1-Filter)
- [apis/proto/v1/manager/index/index_manager.proto](#apis_proto_v1_manager_index_index_manager-proto)
- [Index](#manager-index-v1-Index)
-- [apis/proto/v1/agent/sidecar/sidecar.proto](#apis_proto_v1_agent_sidecar_sidecar-proto)
- - [Sidecar](#sidecar-v1-Sidecar)
-- [apis/proto/v1/agent/core/agent.proto](#apis_proto_v1_agent_core_agent-proto)
- - [Agent](#core-v1-Agent)
- [apis/proto/v1/payload/payload.proto](#apis_proto_v1_payload_payload-proto)
- [Control](#payload-v1-Control)
@@ -29,6 +31,8 @@
- [Info.IPs](#payload-v1-Info-IPs)
- [Info.Index](#payload-v1-Info-Index)
- [Info.Index.Count](#payload-v1-Info-Index-Count)
+ - [Info.Index.Detail](#payload-v1-Info-Index-Detail)
+ - [Info.Index.Detail.CountsEntry](#payload-v1-Info-Index-Detail-CountsEntry)
- [Info.Index.UUID](#payload-v1-Info-Index-UUID)
- [Info.Index.UUID.Committed](#payload-v1-Info-Index-UUID-Committed)
- [Info.Index.UUID.Uncommitted](#payload-v1-Info-Index-UUID-Uncommitted)
@@ -85,27 +89,75 @@
- [Upsert.MultiRequest](#payload-v1-Upsert-MultiRequest)
- [Upsert.ObjectRequest](#payload-v1-Upsert-ObjectRequest)
- [Upsert.Request](#payload-v1-Upsert-Request)
-
- [Search.AggregationAlgorithm](#payload-v1-Search-AggregationAlgorithm)
-- [apis/proto/v1/discoverer/discoverer.proto](#apis_proto_v1_discoverer_discoverer-proto)
- - [Discoverer](#discoverer-v1-Discoverer)
-- [apis/proto/v1/vald/remove.proto](#apis_proto_v1_vald_remove-proto)
- - [Remove](#vald-v1-Remove)
+- [apis/proto/v1/vald/filter.proto](#apis_proto_v1_vald_filter-proto)
+ - [Filter](#vald-v1-Filter)
- [apis/proto/v1/vald/insert.proto](#apis_proto_v1_vald_insert-proto)
- [Insert](#vald-v1-Insert)
-- [apis/proto/v1/vald/upsert.proto](#apis_proto_v1_vald_upsert-proto)
- - [Upsert](#vald-v1-Upsert)
- [apis/proto/v1/vald/object.proto](#apis_proto_v1_vald_object-proto)
- [Object](#vald-v1-Object)
-- [apis/proto/v1/vald/filter.proto](#apis_proto_v1_vald_filter-proto)
- - [Filter](#vald-v1-Filter)
+- [apis/proto/v1/vald/remove.proto](#apis_proto_v1_vald_remove-proto)
+ - [Remove](#vald-v1-Remove)
- [apis/proto/v1/vald/search.proto](#apis_proto_v1_vald_search-proto)
- [Search](#vald-v1-Search)
- [apis/proto/v1/vald/update.proto](#apis_proto_v1_vald_update-proto)
- [Update](#vald-v1-Update)
+- [apis/proto/v1/vald/upsert.proto](#apis_proto_v1_vald_upsert-proto)
+ - [Upsert](#vald-v1-Upsert)
- [Scalar Value Types](#scalar-value-types)
+
+
+Top
+
+## apis/proto/v1/agent/core/agent.proto
+
+
+
+### Agent
+
+Represent the agent service.
+
+| Method Name | Request Type | Response Type | Description |
+| ------------------ | -------------------------------------------------------------------------------- | ------------------------------------------------------------ | ----------------------------------------------------- |
+| CreateIndex | [.payload.v1.Control.CreateIndexRequest](#payload-v1-Control-CreateIndexRequest) | [.payload.v1.Empty](#payload-v1-Empty) | Represent the creating index RPC. |
+| SaveIndex | [.payload.v1.Empty](#payload-v1-Empty) | [.payload.v1.Empty](#payload-v1-Empty) | Represent the saving index RPC. |
+| CreateAndSaveIndex | [.payload.v1.Control.CreateIndexRequest](#payload-v1-Control-CreateIndexRequest) | [.payload.v1.Empty](#payload-v1-Empty) | Represent the creating and saving index RPC. |
+| IndexInfo | [.payload.v1.Empty](#payload-v1-Empty) | [.payload.v1.Info.Index.Count](#payload-v1-Info-Index-Count) | Represent the RPC to get the agent index information. |
+
+
+
+Top
+
+## apis/proto/v1/agent/sidecar/sidecar.proto
+
+
+
+### Sidecar
+
+Represent the agent sidecar service.
+
+| Method Name | Request Type | Response Type | Description |
+| ----------- | ------------ | ------------- | ----------- |
+
+
+
+Top
+
+## apis/proto/v1/discoverer/discoverer.proto
+
+
+
+### Discoverer
+
+Represent the discoverer service.
+
+| Method Name | Request Type | Response Type | Description |
+| ----------- | ---------------------------------------------------------------- | ------------------------------------------------ | ---------------------------------------------------- |
+| Pods | [.payload.v1.Discoverer.Request](#payload-v1-Discoverer-Request) | [.payload.v1.Info.Pods](#payload-v1-Info-Pods) | Represent the RPC to get the agent pods information. |
+| Nodes | [.payload.v1.Discoverer.Request](#payload-v1-Discoverer-Request) | [.payload.v1.Info.Nodes](#payload-v1-Info-Nodes) | Represent the RPC to get the node information. |
+
Top
@@ -152,43 +204,10 @@ Represent the ingress filter service.
Represent the index manager service.
-| Method Name | Request Type | Response Type | Description |
-| ----------- | -------------------------------------- | ------------------------------------------------------------ | ----------------------------------------------- |
-| IndexInfo | [.payload.v1.Empty](#payload-v1-Empty) | [.payload.v1.Info.Index.Count](#payload-v1-Info-Index-Count) | Represent the RPC to get the index information. |
-
-
-
-Top
-
-## apis/proto/v1/agent/sidecar/sidecar.proto
-
-
-
-### Sidecar
-
-Represent the agent sidecar service.
-
-| Method Name | Request Type | Response Type | Description |
-| ----------- | ------------ | ------------- | ----------- |
-
-
-
-Top
-
-## apis/proto/v1/agent/core/agent.proto
-
-
-
-### Agent
-
-Represent the agent service.
-
-| Method Name | Request Type | Response Type | Description |
-| ------------------ | -------------------------------------------------------------------------------- | ------------------------------------------------------------ | ----------------------------------------------------- |
-| CreateIndex | [.payload.v1.Control.CreateIndexRequest](#payload-v1-Control-CreateIndexRequest) | [.payload.v1.Empty](#payload-v1-Empty) | Represent the creating index RPC. |
-| SaveIndex | [.payload.v1.Empty](#payload-v1-Empty) | [.payload.v1.Empty](#payload-v1-Empty) | Represent the saving index RPC. |
-| CreateAndSaveIndex | [.payload.v1.Control.CreateIndexRequest](#payload-v1-Control-CreateIndexRequest) | [.payload.v1.Empty](#payload-v1-Empty) | Represent the creating and saving index RPC. |
-| IndexInfo | [.payload.v1.Empty](#payload-v1-Empty) | [.payload.v1.Info.Index.Count](#payload-v1-Info-Index-Count) | Represent the RPC to get the agent index information. |
+| Method Name | Request Type | Response Type | Description |
+| ----------- | -------------------------------------- | -------------------------------------------------------------- | --------------------------------------------------------------- |
+| IndexInfo | [.payload.v1.Empty](#payload-v1-Empty) | [.payload.v1.Info.Index.Count](#payload-v1-Info-Index-Count) | Represent the RPC to get the index information. |
+| IndexDetail | [.payload.v1.Empty](#payload-v1-Empty) | [.payload.v1.Info.Index.Detail](#payload-v1-Info-Index-Detail) | Represent the RPC to get the index information for each agents. |
@@ -310,6 +329,25 @@ Represent the index count message.
| indexing | [bool](#bool) | | The indexing index count. |
| saving | [bool](#bool) | | The saving index count. |
+
+
+### Info.Index.Detail
+
+Represent the index count for each Agents message.
+
+| Field | Type | Label | Description |
+| ------ | -------------------------------------------------------------------------- | -------- | --------------------------- |
+| counts | [Info.Index.Detail.CountsEntry](#payload-v1-Info-Index-Detail-CountsEntry) | repeated | count infos for each agents |
+
+
+
+### Info.Index.Detail.CountsEntry
+
+| Field | Type | Label | Description |
+| ----- | ------------------------------------------------ | ----- | ----------- |
+| key | [string](#string) | | |
+| value | [Info.Index.Count](#payload-v1-Info-Index-Count) | | |
+
### Info.Index.UUID
@@ -914,40 +952,32 @@ AggregationAlgorithm is enum of each aggregation algorithms
| SortPoolSlice | 3 | |
| PairingHeap | 4 | |
-
-
-Top
-
-## apis/proto/v1/discoverer/discoverer.proto
-
-
-
-### Discoverer
-
-Represent the discoverer service.
-
-| Method Name | Request Type | Response Type | Description |
-| ----------- | ---------------------------------------------------------------- | ------------------------------------------------ | ---------------------------------------------------- |
-| Pods | [.payload.v1.Discoverer.Request](#payload-v1-Discoverer-Request) | [.payload.v1.Info.Pods](#payload-v1-Info-Pods) | Represent the RPC to get the agent pods information. |
-| Nodes | [.payload.v1.Discoverer.Request](#payload-v1-Discoverer-Request) | [.payload.v1.Info.Nodes](#payload-v1-Info-Nodes) | Represent the RPC to get the node information. |
-
-
+
Top
-## apis/proto/v1/vald/remove.proto
+## apis/proto/v1/vald/filter.proto
-
+
-### Remove
+### Filter
-Remove service provides ways to remove indexed vectors.
+Filter service provides ways to connect to Vald through filter.
-| Method Name | Request Type | Response Type | Description |
-| ------------ | ------------------------------------------------------------------ | ----------------------------------------------------------------------------- | ----------------------------------------------------------------------- |
-| Remove | [.payload.v1.Remove.Request](#payload-v1-Remove-Request) | [.payload.v1.Object.Location](#payload-v1-Object-Location) | A method to remove an indexed vector. |
-| StreamRemove | [.payload.v1.Remove.Request](#payload-v1-Remove-Request) stream | [.payload.v1.Object.StreamLocation](#payload-v1-Object-StreamLocation) stream | A method to remove multiple indexed vectors by bidirectional streaming. |
-| MultiRemove | [.payload.v1.Remove.MultiRequest](#payload-v1-Remove-MultiRequest) | [.payload.v1.Object.Locations](#payload-v1-Object-Locations) | A method to remove multiple indexed vectors in a single request. |
+| Method Name | Request Type | Response Type | Description |
+| ------------------ | ------------------------------------------------------------------------------ | ----------------------------------------------------------------------------- | ------------------------------------------------------------------------ |
+| SearchObject | [.payload.v1.Search.ObjectRequest](#payload-v1-Search-ObjectRequest) | [.payload.v1.Search.Response](#payload-v1-Search-Response) | A method to search object. |
+| MultiSearchObject | [.payload.v1.Search.MultiObjectRequest](#payload-v1-Search-MultiObjectRequest) | [.payload.v1.Search.Responses](#payload-v1-Search-Responses) | A method to search multiple objects. |
+| StreamSearchObject | [.payload.v1.Search.ObjectRequest](#payload-v1-Search-ObjectRequest) stream | [.payload.v1.Search.StreamResponse](#payload-v1-Search-StreamResponse) stream | A method to search object by bidirectional streaming. |
+| InsertObject | [.payload.v1.Insert.ObjectRequest](#payload-v1-Insert-ObjectRequest) | [.payload.v1.Object.Location](#payload-v1-Object-Location) | A method insert object. |
+| StreamInsertObject | [.payload.v1.Insert.ObjectRequest](#payload-v1-Insert-ObjectRequest) stream | [.payload.v1.Object.StreamLocation](#payload-v1-Object-StreamLocation) stream | Represent the streaming RPC to insert object by bidirectional streaming. |
+| MultiInsertObject | [.payload.v1.Insert.MultiObjectRequest](#payload-v1-Insert-MultiObjectRequest) | [.payload.v1.Object.Locations](#payload-v1-Object-Locations) | A method to insert multiple objects. |
+| UpdateObject | [.payload.v1.Update.ObjectRequest](#payload-v1-Update-ObjectRequest) | [.payload.v1.Object.Location](#payload-v1-Object-Location) | A method to update object. |
+| StreamUpdateObject | [.payload.v1.Update.ObjectRequest](#payload-v1-Update-ObjectRequest) stream | [.payload.v1.Object.StreamLocation](#payload-v1-Object-StreamLocation) stream | A method to update object by bidirectional streaming. |
+| MultiUpdateObject | [.payload.v1.Update.MultiObjectRequest](#payload-v1-Update-MultiObjectRequest) | [.payload.v1.Object.Locations](#payload-v1-Object-Locations) | A method to update multiple objects. |
+| UpsertObject | [.payload.v1.Upsert.ObjectRequest](#payload-v1-Upsert-ObjectRequest) | [.payload.v1.Object.Location](#payload-v1-Object-Location) | A method to upsert object. |
+| StreamUpsertObject | [.payload.v1.Upsert.ObjectRequest](#payload-v1-Upsert-ObjectRequest) stream | [.payload.v1.Object.StreamLocation](#payload-v1-Object-StreamLocation) stream | A method to upsert object by bidirectional streaming. |
+| MultiUpsertObject | [.payload.v1.Upsert.MultiObjectRequest](#payload-v1-Upsert-MultiObjectRequest) | [.payload.v1.Object.Locations](#payload-v1-Object-Locations) | A method to upsert multiple objects. |
@@ -967,24 +997,6 @@ Insert service provides ways to add new vectors.
| StreamInsert | [.payload.v1.Insert.Request](#payload-v1-Insert-Request) stream | [.payload.v1.Object.StreamLocation](#payload-v1-Object-StreamLocation) stream | A method to add new multiple vectors by bidirectional streaming. |
| MultiInsert | [.payload.v1.Insert.MultiRequest](#payload-v1-Insert-MultiRequest) | [.payload.v1.Object.Locations](#payload-v1-Object-Locations) | A method to add new multiple vectors in a single request. |
-
-
-Top
-
-## apis/proto/v1/vald/upsert.proto
-
-
-
-### Upsert
-
-Upsert service provides ways to insert/update vectors.
-
-| Method Name | Request Type | Response Type | Description |
-| ------------ | ------------------------------------------------------------------ | ----------------------------------------------------------------------------- | ---------------------------------------------------------------------- |
-| Upsert | [.payload.v1.Upsert.Request](#payload-v1-Upsert-Request) | [.payload.v1.Object.Location](#payload-v1-Object-Location) | A method to insert/update a vector. |
-| StreamUpsert | [.payload.v1.Upsert.Request](#payload-v1-Upsert-Request) stream | [.payload.v1.Object.StreamLocation](#payload-v1-Object-StreamLocation) stream | A method to insert/update multiple vectors by bidirectional streaming. |
-| MultiUpsert | [.payload.v1.Upsert.MultiRequest](#payload-v1-Upsert-MultiRequest) | [.payload.v1.Object.Locations](#payload-v1-Object-Locations) | A method to insert/update multiple vectors in a single request. |
-
Top
@@ -1003,32 +1015,23 @@ Object service provides ways to fetch indexed vectors.
| GetObject | [.payload.v1.Object.VectorRequest](#payload-v1-Object-VectorRequest) | [.payload.v1.Object.Vector](#payload-v1-Object-Vector) | A method to fetch a vector. |
| StreamGetObject | [.payload.v1.Object.VectorRequest](#payload-v1-Object-VectorRequest) stream | [.payload.v1.Object.StreamVector](#payload-v1-Object-StreamVector) stream | A method to fetch vectors by bidirectional streaming. |
-
+
Top
-## apis/proto/v1/vald/filter.proto
+## apis/proto/v1/vald/remove.proto
-
+
-### Filter
+### Remove
-Filter service provides ways to connect to Vald through filter.
+Remove service provides ways to remove indexed vectors.
-| Method Name | Request Type | Response Type | Description |
-| ------------------ | ------------------------------------------------------------------------------ | ----------------------------------------------------------------------------- | ------------------------------------------------------------------------ |
-| SearchObject | [.payload.v1.Search.ObjectRequest](#payload-v1-Search-ObjectRequest) | [.payload.v1.Search.Response](#payload-v1-Search-Response) | A method to search object. |
-| MultiSearchObject | [.payload.v1.Search.MultiObjectRequest](#payload-v1-Search-MultiObjectRequest) | [.payload.v1.Search.Responses](#payload-v1-Search-Responses) | A method to search multiple objects. |
-| StreamSearchObject | [.payload.v1.Search.ObjectRequest](#payload-v1-Search-ObjectRequest) stream | [.payload.v1.Search.StreamResponse](#payload-v1-Search-StreamResponse) stream | A method to search object by bidirectional streaming. |
-| InsertObject | [.payload.v1.Insert.ObjectRequest](#payload-v1-Insert-ObjectRequest) | [.payload.v1.Object.Location](#payload-v1-Object-Location) | A method insert object. |
-| StreamInsertObject | [.payload.v1.Insert.ObjectRequest](#payload-v1-Insert-ObjectRequest) stream | [.payload.v1.Object.StreamLocation](#payload-v1-Object-StreamLocation) stream | Represent the streaming RPC to insert object by bidirectional streaming. |
-| MultiInsertObject | [.payload.v1.Insert.MultiObjectRequest](#payload-v1-Insert-MultiObjectRequest) | [.payload.v1.Object.Locations](#payload-v1-Object-Locations) | A method to insert multiple objects. |
-| UpdateObject | [.payload.v1.Update.ObjectRequest](#payload-v1-Update-ObjectRequest) | [.payload.v1.Object.Location](#payload-v1-Object-Location) | A method to update object. |
-| StreamUpdateObject | [.payload.v1.Update.ObjectRequest](#payload-v1-Update-ObjectRequest) stream | [.payload.v1.Object.StreamLocation](#payload-v1-Object-StreamLocation) stream | A method to update object by bidirectional streaming. |
-| MultiUpdateObject | [.payload.v1.Update.MultiObjectRequest](#payload-v1-Update-MultiObjectRequest) | [.payload.v1.Object.Locations](#payload-v1-Object-Locations) | A method to update multiple objects. |
-| UpsertObject | [.payload.v1.Upsert.ObjectRequest](#payload-v1-Upsert-ObjectRequest) | [.payload.v1.Object.Location](#payload-v1-Object-Location) | A method to upsert object. |
-| StreamUpsertObject | [.payload.v1.Upsert.ObjectRequest](#payload-v1-Upsert-ObjectRequest) stream | [.payload.v1.Object.StreamLocation](#payload-v1-Object-StreamLocation) stream | A method to upsert object by bidirectional streaming. |
-| MultiUpsertObject | [.payload.v1.Upsert.MultiObjectRequest](#payload-v1-Upsert-MultiObjectRequest) | [.payload.v1.Object.Locations](#payload-v1-Object-Locations) | A method to upsert multiple objects. |
+| Method Name | Request Type | Response Type | Description |
+| ------------ | ------------------------------------------------------------------ | ----------------------------------------------------------------------------- | ----------------------------------------------------------------------- |
+| Remove | [.payload.v1.Remove.Request](#payload-v1-Remove-Request) | [.payload.v1.Object.Location](#payload-v1-Object-Location) | A method to remove an indexed vector. |
+| StreamRemove | [.payload.v1.Remove.Request](#payload-v1-Remove-Request) stream | [.payload.v1.Object.StreamLocation](#payload-v1-Object-StreamLocation) stream | A method to remove multiple indexed vectors by bidirectional streaming. |
+| MultiRemove | [.payload.v1.Remove.MultiRequest](#payload-v1-Remove-MultiRequest) | [.payload.v1.Object.Locations](#payload-v1-Object-Locations) | A method to remove multiple indexed vectors in a single request. |
@@ -1075,6 +1078,24 @@ Update service provides ways to update indexed vectors.
| StreamUpdate | [.payload.v1.Update.Request](#payload-v1-Update-Request) stream | [.payload.v1.Object.StreamLocation](#payload-v1-Object-StreamLocation) stream | A method to update multiple indexed vectors by bidirectional streaming. |
| MultiUpdate | [.payload.v1.Update.MultiRequest](#payload-v1-Update-MultiRequest) | [.payload.v1.Object.Locations](#payload-v1-Object-Locations) | A method to update multiple indexed vectors in a single request. |
+
+
+Top
+
+## apis/proto/v1/vald/upsert.proto
+
+
+
+### Upsert
+
+Upsert service provides ways to insert/update vectors.
+
+| Method Name | Request Type | Response Type | Description |
+| ------------ | ------------------------------------------------------------------ | ----------------------------------------------------------------------------- | ---------------------------------------------------------------------- |
+| Upsert | [.payload.v1.Upsert.Request](#payload-v1-Upsert-Request) | [.payload.v1.Object.Location](#payload-v1-Object-Location) | A method to insert/update a vector. |
+| StreamUpsert | [.payload.v1.Upsert.Request](#payload-v1-Upsert-Request) stream | [.payload.v1.Object.StreamLocation](#payload-v1-Object-StreamLocation) stream | A method to insert/update multiple vectors by bidirectional streaming. |
+| MultiUpsert | [.payload.v1.Upsert.MultiRequest](#payload-v1-Upsert-MultiRequest) | [.payload.v1.Object.Locations](#payload-v1-Object-Locations) | A method to insert/update multiple vectors in a single request. |
+
## Scalar Value Types
| .proto Type | Notes | C++ | Java | Python | Go | C# | PHP | Ruby |
diff --git a/apis/grpc/v1/agent/core/agent.pb.go b/apis/grpc/v1/agent/core/agent.pb.go
index 67c9b1ef6bf..02c1126c37a 100644
--- a/apis/grpc/v1/agent/core/agent.pb.go
+++ b/apis/grpc/v1/agent/core/agent.pb.go
@@ -17,7 +17,7 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.31.0
-// protoc v4.23.3
+// protoc v4.23.4
// source: apis/proto/v1/agent/core/agent.proto
package core
diff --git a/apis/grpc/v1/agent/sidecar/sidecar.pb.go b/apis/grpc/v1/agent/sidecar/sidecar.pb.go
index f72544b3512..a602b133ee9 100644
--- a/apis/grpc/v1/agent/sidecar/sidecar.pb.go
+++ b/apis/grpc/v1/agent/sidecar/sidecar.pb.go
@@ -17,7 +17,7 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.31.0
-// protoc v4.23.3
+// protoc v4.23.4
// source: apis/proto/v1/agent/sidecar/sidecar.proto
package sidecar
diff --git a/apis/grpc/v1/discoverer/discoverer.pb.go b/apis/grpc/v1/discoverer/discoverer.pb.go
index 3e6a96ba65c..168c8ae6136 100644
--- a/apis/grpc/v1/discoverer/discoverer.pb.go
+++ b/apis/grpc/v1/discoverer/discoverer.pb.go
@@ -17,7 +17,7 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.31.0
-// protoc v4.23.3
+// protoc v4.23.4
// source: apis/proto/v1/discoverer/discoverer.proto
package discoverer
diff --git a/apis/grpc/v1/filter/egress/egress_filter.pb.go b/apis/grpc/v1/filter/egress/egress_filter.pb.go
index 70578312686..4d45b5d5da9 100644
--- a/apis/grpc/v1/filter/egress/egress_filter.pb.go
+++ b/apis/grpc/v1/filter/egress/egress_filter.pb.go
@@ -17,7 +17,7 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.31.0
-// protoc v4.23.3
+// protoc v4.23.4
// source: apis/proto/v1/filter/egress/egress_filter.proto
package egress
diff --git a/apis/grpc/v1/filter/ingress/ingress_filter.pb.go b/apis/grpc/v1/filter/ingress/ingress_filter.pb.go
index de6217fa682..9211486344f 100644
--- a/apis/grpc/v1/filter/ingress/ingress_filter.pb.go
+++ b/apis/grpc/v1/filter/ingress/ingress_filter.pb.go
@@ -17,7 +17,7 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.31.0
-// protoc v4.23.3
+// protoc v4.23.4
// source: apis/proto/v1/filter/ingress/ingress_filter.proto
package ingress
diff --git a/apis/grpc/v1/manager/index/index_manager.pb.go b/apis/grpc/v1/manager/index/index_manager.pb.go
index fb23bdc3c5c..5fe1b02b33a 100644
--- a/apis/grpc/v1/manager/index/index_manager.pb.go
+++ b/apis/grpc/v1/manager/index/index_manager.pb.go
@@ -17,7 +17,7 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.31.0
-// protoc v4.23.3
+// protoc v4.23.4
// source: apis/proto/v1/manager/index/index_manager.proto
package index
@@ -51,31 +51,40 @@ var file_apis_proto_v1_manager_index_index_manager_proto_rawDesc = []byte{
0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x61, 0x70, 0x69, 0x73, 0x2f,
0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x61, 0x70, 0x69, 0x73, 0x2f, 0x67, 0x6f, 0x6f, 0x67, 0x6c,
0x65, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x61, 0x6e, 0x6e, 0x6f, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e,
- 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x32, 0x5a, 0x0a, 0x05, 0x49, 0x6e, 0x64, 0x65, 0x78,
- 0x12, 0x51, 0x0a, 0x09, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x11, 0x2e,
- 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79,
- 0x1a, 0x1c, 0x2e, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x49, 0x6e,
- 0x66, 0x6f, 0x2e, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x2e, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x22, 0x13,
- 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x0d, 0x12, 0x0b, 0x2f, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x2f, 0x69,
- 0x6e, 0x66, 0x6f, 0x42, 0x6b, 0x0a, 0x23, 0x6f, 0x72, 0x67, 0x2e, 0x76, 0x64, 0x61, 0x61, 0x73,
- 0x2e, 0x76, 0x61, 0x6c, 0x64, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x2e, 0x6d, 0x61, 0x6e,
- 0x61, 0x67, 0x65, 0x72, 0x2e, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x42, 0x10, 0x56, 0x61, 0x6c, 0x64,
- 0x49, 0x6e, 0x64, 0x65, 0x78, 0x4d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x50, 0x01, 0x5a, 0x30,
- 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x76, 0x64, 0x61, 0x61, 0x73,
- 0x2f, 0x76, 0x61, 0x6c, 0x64, 0x2f, 0x61, 0x70, 0x69, 0x73, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x2f,
- 0x76, 0x31, 0x2f, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2f, 0x69, 0x6e, 0x64, 0x65, 0x78,
- 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
+ 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x32, 0xb2, 0x01, 0x0a, 0x05, 0x49, 0x6e, 0x64, 0x65,
+ 0x78, 0x12, 0x51, 0x0a, 0x09, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x11,
+ 0x2e, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x45, 0x6d, 0x70, 0x74,
+ 0x79, 0x1a, 0x1c, 0x2e, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x49,
+ 0x6e, 0x66, 0x6f, 0x2e, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x2e, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x22,
+ 0x13, 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x0d, 0x12, 0x0b, 0x2f, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x2f,
+ 0x69, 0x6e, 0x66, 0x6f, 0x12, 0x56, 0x0a, 0x0b, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x44, 0x65, 0x74,
+ 0x61, 0x69, 0x6c, 0x12, 0x11, 0x2e, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x2e, 0x76, 0x31,
+ 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x1d, 0x2e, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64,
+ 0x2e, 0x76, 0x31, 0x2e, 0x49, 0x6e, 0x66, 0x6f, 0x2e, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x2e, 0x44,
+ 0x65, 0x74, 0x61, 0x69, 0x6c, 0x22, 0x15, 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x0f, 0x12, 0x0d, 0x2f,
+ 0x69, 0x6e, 0x64, 0x65, 0x78, 0x2f, 0x64, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x42, 0x6b, 0x0a, 0x23,
+ 0x6f, 0x72, 0x67, 0x2e, 0x76, 0x64, 0x61, 0x61, 0x73, 0x2e, 0x76, 0x61, 0x6c, 0x64, 0x2e, 0x61,
+ 0x70, 0x69, 0x2e, 0x76, 0x31, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x69, 0x6e,
+ 0x64, 0x65, 0x78, 0x42, 0x10, 0x56, 0x61, 0x6c, 0x64, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x4d, 0x61,
+ 0x6e, 0x61, 0x67, 0x65, 0x72, 0x50, 0x01, 0x5a, 0x30, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e,
+ 0x63, 0x6f, 0x6d, 0x2f, 0x76, 0x64, 0x61, 0x61, 0x73, 0x2f, 0x76, 0x61, 0x6c, 0x64, 0x2f, 0x61,
+ 0x70, 0x69, 0x73, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x76, 0x31, 0x2f, 0x6d, 0x61, 0x6e, 0x61,
+ 0x67, 0x65, 0x72, 0x2f, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f,
+ 0x33,
}
var file_apis_proto_v1_manager_index_index_manager_proto_goTypes = []interface{}{
- (*payload.Empty)(nil), // 0: payload.v1.Empty
- (*payload.Info_Index_Count)(nil), // 1: payload.v1.Info.Index.Count
+ (*payload.Empty)(nil), // 0: payload.v1.Empty
+ (*payload.Info_Index_Count)(nil), // 1: payload.v1.Info.Index.Count
+ (*payload.Info_Index_Detail)(nil), // 2: payload.v1.Info.Index.Detail
}
var file_apis_proto_v1_manager_index_index_manager_proto_depIdxs = []int32{
0, // 0: manager.index.v1.Index.IndexInfo:input_type -> payload.v1.Empty
- 1, // 1: manager.index.v1.Index.IndexInfo:output_type -> payload.v1.Info.Index.Count
- 1, // [1:2] is the sub-list for method output_type
- 0, // [0:1] is the sub-list for method input_type
+ 0, // 1: manager.index.v1.Index.IndexDetail:input_type -> payload.v1.Empty
+ 1, // 2: manager.index.v1.Index.IndexInfo:output_type -> payload.v1.Info.Index.Count
+ 2, // 3: manager.index.v1.Index.IndexDetail:output_type -> payload.v1.Info.Index.Detail
+ 2, // [2:4] is the sub-list for method output_type
+ 0, // [0:2] is the sub-list for method input_type
0, // [0:0] is the sub-list for extension type_name
0, // [0:0] is the sub-list for extension extendee
0, // [0:0] is the sub-list for field type_name
diff --git a/apis/grpc/v1/manager/index/index_manager_vtproto.pb.go b/apis/grpc/v1/manager/index/index_manager_vtproto.pb.go
index 219ea820f6b..d17db0fddc0 100644
--- a/apis/grpc/v1/manager/index/index_manager_vtproto.pb.go
+++ b/apis/grpc/v1/manager/index/index_manager_vtproto.pb.go
@@ -44,6 +44,8 @@ const _ = grpc.SupportPackageIsVersion7
type IndexClient interface {
// Represent the RPC to get the index information.
IndexInfo(ctx context.Context, in *payload.Empty, opts ...grpc.CallOption) (*payload.Info_Index_Count, error)
+ // Represent the RPC to get the index information for each agents.
+ IndexDetail(ctx context.Context, in *payload.Empty, opts ...grpc.CallOption) (*payload.Info_Index_Detail, error)
}
type indexClient struct {
@@ -63,12 +65,23 @@ func (c *indexClient) IndexInfo(ctx context.Context, in *payload.Empty, opts ...
return out, nil
}
+func (c *indexClient) IndexDetail(ctx context.Context, in *payload.Empty, opts ...grpc.CallOption) (*payload.Info_Index_Detail, error) {
+ out := new(payload.Info_Index_Detail)
+ err := c.cc.Invoke(ctx, "/manager.index.v1.Index/IndexDetail", in, out, opts...)
+ if err != nil {
+ return nil, err
+ }
+ return out, nil
+}
+
// IndexServer is the server API for Index service.
// All implementations must embed UnimplementedIndexServer
// for forward compatibility
type IndexServer interface {
// Represent the RPC to get the index information.
IndexInfo(context.Context, *payload.Empty) (*payload.Info_Index_Count, error)
+ // Represent the RPC to get the index information for each agents.
+ IndexDetail(context.Context, *payload.Empty) (*payload.Info_Index_Detail, error)
mustEmbedUnimplementedIndexServer()
}
@@ -79,6 +92,9 @@ type UnimplementedIndexServer struct {
func (UnimplementedIndexServer) IndexInfo(context.Context, *payload.Empty) (*payload.Info_Index_Count, error) {
return nil, status.Errorf(codes.Unimplemented, "method IndexInfo not implemented")
}
+func (UnimplementedIndexServer) IndexDetail(context.Context, *payload.Empty) (*payload.Info_Index_Detail, error) {
+ return nil, status.Errorf(codes.Unimplemented, "method IndexDetail not implemented")
+}
func (UnimplementedIndexServer) mustEmbedUnimplementedIndexServer() {}
// UnsafeIndexServer may be embedded to opt out of forward compatibility for this service.
@@ -110,6 +126,24 @@ func _Index_IndexInfo_Handler(srv interface{}, ctx context.Context, dec func(int
return interceptor(ctx, in, info, handler)
}
+func _Index_IndexDetail_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+ in := new(payload.Empty)
+ if err := dec(in); err != nil {
+ return nil, err
+ }
+ if interceptor == nil {
+ return srv.(IndexServer).IndexDetail(ctx, in)
+ }
+ info := &grpc.UnaryServerInfo{
+ Server: srv,
+ FullMethod: "/manager.index.v1.Index/IndexDetail",
+ }
+ handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+ return srv.(IndexServer).IndexDetail(ctx, req.(*payload.Empty))
+ }
+ return interceptor(ctx, in, info, handler)
+}
+
// Index_ServiceDesc is the grpc.ServiceDesc for Index service.
// It's only intended for direct use with grpc.RegisterService,
// and not to be introspected or modified (even as a copy)
@@ -121,6 +155,10 @@ var Index_ServiceDesc = grpc.ServiceDesc{
MethodName: "IndexInfo",
Handler: _Index_IndexInfo_Handler,
},
+ {
+ MethodName: "IndexDetail",
+ Handler: _Index_IndexDetail_Handler,
+ },
},
Streams: []grpc.StreamDesc{},
Metadata: "apis/proto/v1/manager/index/index_manager.proto",
diff --git a/apis/grpc/v1/payload/payload.pb.go b/apis/grpc/v1/payload/payload.pb.go
index 710d65aced6..6b5ef8cd5ec 100644
--- a/apis/grpc/v1/payload/payload.pb.go
+++ b/apis/grpc/v1/payload/payload.pb.go
@@ -17,16 +17,16 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.31.0
-// protoc v4.23.3
+// protoc v4.23.4
// source: apis/proto/v1/payload/payload.proto
package payload
import (
reflect "reflect"
- sync "sync"
_ "github.com/envoyproxy/protoc-gen-validate/validate"
+ sync "github.com/vdaas/vald/internal/sync"
status "google.golang.org/genproto/googleapis/rpc/status"
protoreflect "google.golang.org/protobuf/reflect/protoreflect"
protoimpl "google.golang.org/protobuf/runtime/protoimpl"
@@ -3930,6 +3930,55 @@ func (x *Info_Index_Count) GetSaving() bool {
return false
}
+// Represent the index count for each Agents message.
+type Info_Index_Detail struct {
+ state protoimpl.MessageState
+ sizeCache protoimpl.SizeCache
+ unknownFields protoimpl.UnknownFields
+
+ // count infos for each agents
+ Counts map[string]*Info_Index_Count `protobuf:"bytes,1,rep,name=counts,proto3" json:"counts,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
+}
+
+func (x *Info_Index_Detail) Reset() {
+ *x = Info_Index_Detail{}
+ if protoimpl.UnsafeEnabled {
+ mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[66]
+ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+ ms.StoreMessageInfo(mi)
+ }
+}
+
+func (x *Info_Index_Detail) String() string {
+ return protoimpl.X.MessageStringOf(x)
+}
+
+func (*Info_Index_Detail) ProtoMessage() {}
+
+func (x *Info_Index_Detail) ProtoReflect() protoreflect.Message {
+ mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[66]
+ if protoimpl.UnsafeEnabled && x != nil {
+ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+ if ms.LoadMessageInfo() == nil {
+ ms.StoreMessageInfo(mi)
+ }
+ return ms
+ }
+ return mi.MessageOf(x)
+}
+
+// Deprecated: Use Info_Index_Detail.ProtoReflect.Descriptor instead.
+func (*Info_Index_Detail) Descriptor() ([]byte, []int) {
+ return file_apis_proto_v1_payload_payload_proto_rawDescGZIP(), []int{9, 0, 1}
+}
+
+func (x *Info_Index_Detail) GetCounts() map[string]*Info_Index_Count {
+ if x != nil {
+ return x.Counts
+ }
+ return nil
+}
+
// Represent the UUID message.
type Info_Index_UUID struct {
state protoimpl.MessageState
@@ -3940,7 +3989,7 @@ type Info_Index_UUID struct {
func (x *Info_Index_UUID) Reset() {
*x = Info_Index_UUID{}
if protoimpl.UnsafeEnabled {
- mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[66]
+ mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[67]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -3953,7 +4002,7 @@ func (x *Info_Index_UUID) String() string {
func (*Info_Index_UUID) ProtoMessage() {}
func (x *Info_Index_UUID) ProtoReflect() protoreflect.Message {
- mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[66]
+ mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[67]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -3966,7 +4015,7 @@ func (x *Info_Index_UUID) ProtoReflect() protoreflect.Message {
// Deprecated: Use Info_Index_UUID.ProtoReflect.Descriptor instead.
func (*Info_Index_UUID) Descriptor() ([]byte, []int) {
- return file_apis_proto_v1_payload_payload_proto_rawDescGZIP(), []int{9, 0, 1}
+ return file_apis_proto_v1_payload_payload_proto_rawDescGZIP(), []int{9, 0, 2}
}
// The committed UUID.
@@ -3981,7 +4030,7 @@ type Info_Index_UUID_Committed struct {
func (x *Info_Index_UUID_Committed) Reset() {
*x = Info_Index_UUID_Committed{}
if protoimpl.UnsafeEnabled {
- mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[67]
+ mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[69]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -3994,7 +4043,7 @@ func (x *Info_Index_UUID_Committed) String() string {
func (*Info_Index_UUID_Committed) ProtoMessage() {}
func (x *Info_Index_UUID_Committed) ProtoReflect() protoreflect.Message {
- mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[67]
+ mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[69]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -4007,7 +4056,7 @@ func (x *Info_Index_UUID_Committed) ProtoReflect() protoreflect.Message {
// Deprecated: Use Info_Index_UUID_Committed.ProtoReflect.Descriptor instead.
func (*Info_Index_UUID_Committed) Descriptor() ([]byte, []int) {
- return file_apis_proto_v1_payload_payload_proto_rawDescGZIP(), []int{9, 0, 1, 0}
+ return file_apis_proto_v1_payload_payload_proto_rawDescGZIP(), []int{9, 0, 2, 0}
}
func (x *Info_Index_UUID_Committed) GetUuid() string {
@@ -4029,7 +4078,7 @@ type Info_Index_UUID_Uncommitted struct {
func (x *Info_Index_UUID_Uncommitted) Reset() {
*x = Info_Index_UUID_Uncommitted{}
if protoimpl.UnsafeEnabled {
- mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[68]
+ mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[70]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -4042,7 +4091,7 @@ func (x *Info_Index_UUID_Uncommitted) String() string {
func (*Info_Index_UUID_Uncommitted) ProtoMessage() {}
func (x *Info_Index_UUID_Uncommitted) ProtoReflect() protoreflect.Message {
- mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[68]
+ mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[70]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -4055,7 +4104,7 @@ func (x *Info_Index_UUID_Uncommitted) ProtoReflect() protoreflect.Message {
// Deprecated: Use Info_Index_UUID_Uncommitted.ProtoReflect.Descriptor instead.
func (*Info_Index_UUID_Uncommitted) Descriptor() ([]byte, []int) {
- return file_apis_proto_v1_payload_payload_proto_rawDescGZIP(), []int{9, 0, 1, 1}
+ return file_apis_proto_v1_payload_payload_proto_rawDescGZIP(), []int{9, 0, 2, 1}
}
func (x *Info_Index_UUID_Uncommitted) GetUuid() string {
@@ -4398,7 +4447,7 @@ var file_apis_proto_v1_payload_payload_proto_rawDesc = []byte{
0x6d, 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x18,
0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65,
0x12, 0x12, 0x0a, 0x04, 0x6e, 0x6f, 0x64, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04,
- 0x6e, 0x6f, 0x64, 0x65, 0x22, 0xe0, 0x07, 0x0a, 0x04, 0x49, 0x6e, 0x66, 0x6f, 0x1a, 0xca, 0x01,
+ 0x6e, 0x6f, 0x64, 0x65, 0x22, 0x87, 0x09, 0x0a, 0x04, 0x49, 0x6e, 0x66, 0x6f, 0x1a, 0xf1, 0x02,
0x0a, 0x05, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x1a, 0x75, 0x0a, 0x05, 0x43, 0x6f, 0x75, 0x6e, 0x74,
0x12, 0x16, 0x0a, 0x06, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d,
0x52, 0x06, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x64, 0x12, 0x20, 0x0a, 0x0b, 0x75, 0x6e, 0x63, 0x6f,
@@ -4406,68 +4455,78 @@ var file_apis_proto_v1_payload_payload_proto_rawDesc = []byte{
0x6e, 0x63, 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x74, 0x65, 0x64, 0x12, 0x1a, 0x0a, 0x08, 0x69, 0x6e,
0x64, 0x65, 0x78, 0x69, 0x6e, 0x67, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x08, 0x69, 0x6e,
0x64, 0x65, 0x78, 0x69, 0x6e, 0x67, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x61, 0x76, 0x69, 0x6e, 0x67,
- 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, 0x06, 0x73, 0x61, 0x76, 0x69, 0x6e, 0x67, 0x1a, 0x4a,
- 0x0a, 0x04, 0x55, 0x55, 0x49, 0x44, 0x1a, 0x1f, 0x0a, 0x09, 0x43, 0x6f, 0x6d, 0x6d, 0x69, 0x74,
- 0x74, 0x65, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x75, 0x75, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28,
- 0x09, 0x52, 0x04, 0x75, 0x75, 0x69, 0x64, 0x1a, 0x21, 0x0a, 0x0b, 0x55, 0x6e, 0x63, 0x6f, 0x6d,
- 0x6d, 0x69, 0x74, 0x74, 0x65, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x75, 0x75, 0x69, 0x64, 0x18, 0x01,
- 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x75, 0x75, 0x69, 0x64, 0x1a, 0xef, 0x01, 0x0a, 0x03, 0x50,
- 0x6f, 0x64, 0x12, 0x19, 0x0a, 0x08, 0x61, 0x70, 0x70, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01,
- 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x61, 0x70, 0x70, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x12, 0x0a,
- 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d,
- 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x18, 0x03,
- 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x12,
- 0x17, 0x0a, 0x02, 0x69, 0x70, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x42, 0x07, 0xfa, 0x42, 0x04,
- 0x72, 0x02, 0x78, 0x01, 0x52, 0x02, 0x69, 0x70, 0x12, 0x26, 0x0a, 0x03, 0x63, 0x70, 0x75, 0x18,
- 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x2e,
- 0x76, 0x31, 0x2e, 0x49, 0x6e, 0x66, 0x6f, 0x2e, 0x43, 0x50, 0x55, 0x52, 0x03, 0x63, 0x70, 0x75,
- 0x12, 0x2f, 0x0a, 0x06, 0x6d, 0x65, 0x6d, 0x6f, 0x72, 0x79, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b,
- 0x32, 0x17, 0x2e, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x49, 0x6e,
- 0x66, 0x6f, 0x2e, 0x4d, 0x65, 0x6d, 0x6f, 0x72, 0x79, 0x52, 0x06, 0x6d, 0x65, 0x6d, 0x6f, 0x72,
- 0x79, 0x12, 0x29, 0x0a, 0x04, 0x6e, 0x6f, 0x64, 0x65, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32,
+ 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, 0x06, 0x73, 0x61, 0x76, 0x69, 0x6e, 0x67, 0x1a, 0xa4,
+ 0x01, 0x0a, 0x06, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x12, 0x41, 0x0a, 0x06, 0x63, 0x6f, 0x75,
+ 0x6e, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x29, 0x2e, 0x70, 0x61, 0x79, 0x6c,
+ 0x6f, 0x61, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x49, 0x6e, 0x66, 0x6f, 0x2e, 0x49, 0x6e, 0x64, 0x65,
+ 0x78, 0x2e, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x2e, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x73, 0x45,
+ 0x6e, 0x74, 0x72, 0x79, 0x52, 0x06, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x73, 0x1a, 0x57, 0x0a, 0x0b,
+ 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b,
+ 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x32, 0x0a,
+ 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x70,
+ 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x49, 0x6e, 0x66, 0x6f, 0x2e, 0x49,
+ 0x6e, 0x64, 0x65, 0x78, 0x2e, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75,
+ 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x4a, 0x0a, 0x04, 0x55, 0x55, 0x49, 0x44, 0x1a, 0x1f, 0x0a,
+ 0x09, 0x43, 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x74, 0x65, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x75, 0x75,
+ 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x75, 0x75, 0x69, 0x64, 0x1a, 0x21,
+ 0x0a, 0x0b, 0x55, 0x6e, 0x63, 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x74, 0x65, 0x64, 0x12, 0x12, 0x0a,
+ 0x04, 0x75, 0x75, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x75, 0x75, 0x69,
+ 0x64, 0x1a, 0xef, 0x01, 0x0a, 0x03, 0x50, 0x6f, 0x64, 0x12, 0x19, 0x0a, 0x08, 0x61, 0x70, 0x70,
+ 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x61, 0x70, 0x70,
+ 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01,
+ 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x6e, 0x61, 0x6d, 0x65,
+ 0x73, 0x70, 0x61, 0x63, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x6e, 0x61, 0x6d,
+ 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x12, 0x17, 0x0a, 0x02, 0x69, 0x70, 0x18, 0x04, 0x20, 0x01,
+ 0x28, 0x09, 0x42, 0x07, 0xfa, 0x42, 0x04, 0x72, 0x02, 0x78, 0x01, 0x52, 0x02, 0x69, 0x70, 0x12,
+ 0x26, 0x0a, 0x03, 0x63, 0x70, 0x75, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x70,
+ 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x49, 0x6e, 0x66, 0x6f, 0x2e, 0x43,
+ 0x50, 0x55, 0x52, 0x03, 0x63, 0x70, 0x75, 0x12, 0x2f, 0x0a, 0x06, 0x6d, 0x65, 0x6d, 0x6f, 0x72,
+ 0x79, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61,
+ 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x49, 0x6e, 0x66, 0x6f, 0x2e, 0x4d, 0x65, 0x6d, 0x6f, 0x72, 0x79,
+ 0x52, 0x06, 0x6d, 0x65, 0x6d, 0x6f, 0x72, 0x79, 0x12, 0x29, 0x0a, 0x04, 0x6e, 0x6f, 0x64, 0x65,
+ 0x18, 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64,
+ 0x2e, 0x76, 0x31, 0x2e, 0x49, 0x6e, 0x66, 0x6f, 0x2e, 0x4e, 0x6f, 0x64, 0x65, 0x52, 0x04, 0x6e,
+ 0x6f, 0x64, 0x65, 0x1a, 0xe8, 0x01, 0x0a, 0x04, 0x4e, 0x6f, 0x64, 0x65, 0x12, 0x12, 0x0a, 0x04,
+ 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65,
+ 0x12, 0x23, 0x0a, 0x0d, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x5f, 0x61, 0x64, 0x64,
+ 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61,
+ 0x6c, 0x41, 0x64, 0x64, 0x72, 0x12, 0x23, 0x0a, 0x0d, 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61,
+ 0x6c, 0x5f, 0x61, 0x64, 0x64, 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x65, 0x78,
+ 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x41, 0x64, 0x64, 0x72, 0x12, 0x26, 0x0a, 0x03, 0x63, 0x70,
+ 0x75, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61,
+ 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x49, 0x6e, 0x66, 0x6f, 0x2e, 0x43, 0x50, 0x55, 0x52, 0x03, 0x63,
+ 0x70, 0x75, 0x12, 0x2f, 0x0a, 0x06, 0x6d, 0x65, 0x6d, 0x6f, 0x72, 0x79, 0x18, 0x05, 0x20, 0x01,
+ 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x2e, 0x76, 0x31, 0x2e,
+ 0x49, 0x6e, 0x66, 0x6f, 0x2e, 0x4d, 0x65, 0x6d, 0x6f, 0x72, 0x79, 0x52, 0x06, 0x6d, 0x65, 0x6d,
+ 0x6f, 0x72, 0x79, 0x12, 0x29, 0x0a, 0x04, 0x50, 0x6f, 0x64, 0x73, 0x18, 0x06, 0x20, 0x01, 0x28,
+ 0x0b, 0x32, 0x15, 0x2e, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x49,
+ 0x6e, 0x66, 0x6f, 0x2e, 0x50, 0x6f, 0x64, 0x73, 0x52, 0x04, 0x50, 0x6f, 0x64, 0x73, 0x1a, 0x4b,
+ 0x0a, 0x03, 0x43, 0x50, 0x55, 0x12, 0x14, 0x0a, 0x05, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x18, 0x01,
+ 0x20, 0x01, 0x28, 0x01, 0x52, 0x05, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x12, 0x18, 0x0a, 0x07, 0x72,
+ 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x01, 0x52, 0x07, 0x72, 0x65,
+ 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x75, 0x73, 0x61, 0x67, 0x65, 0x18, 0x03,
+ 0x20, 0x01, 0x28, 0x01, 0x52, 0x05, 0x75, 0x73, 0x61, 0x67, 0x65, 0x1a, 0x4e, 0x0a, 0x06, 0x4d,
+ 0x65, 0x6d, 0x6f, 0x72, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x18, 0x01,
+ 0x20, 0x01, 0x28, 0x01, 0x52, 0x05, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x12, 0x18, 0x0a, 0x07, 0x72,
+ 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x01, 0x52, 0x07, 0x72, 0x65,
+ 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x75, 0x73, 0x61, 0x67, 0x65, 0x18, 0x03,
+ 0x20, 0x01, 0x28, 0x01, 0x52, 0x05, 0x75, 0x73, 0x61, 0x67, 0x65, 0x1a, 0x3a, 0x0a, 0x04, 0x50,
+ 0x6f, 0x64, 0x73, 0x12, 0x32, 0x0a, 0x04, 0x70, 0x6f, 0x64, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28,
+ 0x0b, 0x32, 0x14, 0x2e, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x49,
+ 0x6e, 0x66, 0x6f, 0x2e, 0x50, 0x6f, 0x64, 0x42, 0x08, 0xfa, 0x42, 0x05, 0x92, 0x01, 0x02, 0x08,
+ 0x01, 0x52, 0x04, 0x70, 0x6f, 0x64, 0x73, 0x1a, 0x3e, 0x0a, 0x05, 0x4e, 0x6f, 0x64, 0x65, 0x73,
+ 0x12, 0x35, 0x0a, 0x05, 0x6e, 0x6f, 0x64, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32,
0x15, 0x2e, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x49, 0x6e, 0x66,
- 0x6f, 0x2e, 0x4e, 0x6f, 0x64, 0x65, 0x52, 0x04, 0x6e, 0x6f, 0x64, 0x65, 0x1a, 0xe8, 0x01, 0x0a,
- 0x04, 0x4e, 0x6f, 0x64, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20,
- 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x23, 0x0a, 0x0d, 0x69, 0x6e, 0x74,
- 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x5f, 0x61, 0x64, 0x64, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09,
- 0x52, 0x0c, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x41, 0x64, 0x64, 0x72, 0x12, 0x23,
- 0x0a, 0x0d, 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x5f, 0x61, 0x64, 0x64, 0x72, 0x18,
- 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x41,
- 0x64, 0x64, 0x72, 0x12, 0x26, 0x0a, 0x03, 0x63, 0x70, 0x75, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b,
- 0x32, 0x14, 0x2e, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x49, 0x6e,
- 0x66, 0x6f, 0x2e, 0x43, 0x50, 0x55, 0x52, 0x03, 0x63, 0x70, 0x75, 0x12, 0x2f, 0x0a, 0x06, 0x6d,
- 0x65, 0x6d, 0x6f, 0x72, 0x79, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x70, 0x61,
- 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x49, 0x6e, 0x66, 0x6f, 0x2e, 0x4d, 0x65,
- 0x6d, 0x6f, 0x72, 0x79, 0x52, 0x06, 0x6d, 0x65, 0x6d, 0x6f, 0x72, 0x79, 0x12, 0x29, 0x0a, 0x04,
- 0x50, 0x6f, 0x64, 0x73, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x70, 0x61, 0x79,
- 0x6c, 0x6f, 0x61, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x49, 0x6e, 0x66, 0x6f, 0x2e, 0x50, 0x6f, 0x64,
- 0x73, 0x52, 0x04, 0x50, 0x6f, 0x64, 0x73, 0x1a, 0x4b, 0x0a, 0x03, 0x43, 0x50, 0x55, 0x12, 0x14,
- 0x0a, 0x05, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x01, 0x52, 0x05, 0x6c,
- 0x69, 0x6d, 0x69, 0x74, 0x12, 0x18, 0x0a, 0x07, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x18,
- 0x02, 0x20, 0x01, 0x28, 0x01, 0x52, 0x07, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x14,
- 0x0a, 0x05, 0x75, 0x73, 0x61, 0x67, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x01, 0x52, 0x05, 0x75,
- 0x73, 0x61, 0x67, 0x65, 0x1a, 0x4e, 0x0a, 0x06, 0x4d, 0x65, 0x6d, 0x6f, 0x72, 0x79, 0x12, 0x14,
- 0x0a, 0x05, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x01, 0x52, 0x05, 0x6c,
- 0x69, 0x6d, 0x69, 0x74, 0x12, 0x18, 0x0a, 0x07, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x18,
- 0x02, 0x20, 0x01, 0x28, 0x01, 0x52, 0x07, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x14,
- 0x0a, 0x05, 0x75, 0x73, 0x61, 0x67, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x01, 0x52, 0x05, 0x75,
- 0x73, 0x61, 0x67, 0x65, 0x1a, 0x3a, 0x0a, 0x04, 0x50, 0x6f, 0x64, 0x73, 0x12, 0x32, 0x0a, 0x04,
- 0x70, 0x6f, 0x64, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x70, 0x61, 0x79,
- 0x6c, 0x6f, 0x61, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x49, 0x6e, 0x66, 0x6f, 0x2e, 0x50, 0x6f, 0x64,
- 0x42, 0x08, 0xfa, 0x42, 0x05, 0x92, 0x01, 0x02, 0x08, 0x01, 0x52, 0x04, 0x70, 0x6f, 0x64, 0x73,
- 0x1a, 0x3e, 0x0a, 0x05, 0x4e, 0x6f, 0x64, 0x65, 0x73, 0x12, 0x35, 0x0a, 0x05, 0x6e, 0x6f, 0x64,
- 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x70, 0x61, 0x79, 0x6c, 0x6f,
- 0x61, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x49, 0x6e, 0x66, 0x6f, 0x2e, 0x4e, 0x6f, 0x64, 0x65, 0x42,
- 0x08, 0xfa, 0x42, 0x05, 0x92, 0x01, 0x02, 0x08, 0x01, 0x52, 0x05, 0x6e, 0x6f, 0x64, 0x65, 0x73,
- 0x1a, 0x15, 0x0a, 0x03, 0x49, 0x50, 0x73, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x70, 0x18, 0x01, 0x20,
- 0x03, 0x28, 0x09, 0x52, 0x02, 0x69, 0x70, 0x22, 0x07, 0x0a, 0x05, 0x45, 0x6d, 0x70, 0x74, 0x79,
- 0x42, 0x5a, 0x0a, 0x1d, 0x6f, 0x72, 0x67, 0x2e, 0x76, 0x64, 0x61, 0x61, 0x73, 0x2e, 0x76, 0x61,
- 0x6c, 0x64, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x2e, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61,
- 0x64, 0x42, 0x0b, 0x56, 0x61, 0x6c, 0x64, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x50, 0x01,
- 0x5a, 0x2a, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x76, 0x64, 0x61,
- 0x61, 0x73, 0x2f, 0x76, 0x61, 0x6c, 0x64, 0x2f, 0x61, 0x70, 0x69, 0x73, 0x2f, 0x67, 0x72, 0x70,
- 0x63, 0x2f, 0x76, 0x31, 0x2f, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x62, 0x06, 0x70, 0x72,
- 0x6f, 0x74, 0x6f, 0x33,
+ 0x6f, 0x2e, 0x4e, 0x6f, 0x64, 0x65, 0x42, 0x08, 0xfa, 0x42, 0x05, 0x92, 0x01, 0x02, 0x08, 0x01,
+ 0x52, 0x05, 0x6e, 0x6f, 0x64, 0x65, 0x73, 0x1a, 0x15, 0x0a, 0x03, 0x49, 0x50, 0x73, 0x12, 0x0e,
+ 0x0a, 0x02, 0x69, 0x70, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x02, 0x69, 0x70, 0x22, 0x07,
+ 0x0a, 0x05, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x42, 0x5a, 0x0a, 0x1d, 0x6f, 0x72, 0x67, 0x2e, 0x76,
+ 0x64, 0x61, 0x61, 0x73, 0x2e, 0x76, 0x61, 0x6c, 0x64, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31,
+ 0x2e, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x42, 0x0b, 0x56, 0x61, 0x6c, 0x64, 0x50, 0x61,
+ 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x50, 0x01, 0x5a, 0x2a, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e,
+ 0x63, 0x6f, 0x6d, 0x2f, 0x76, 0x64, 0x61, 0x61, 0x73, 0x2f, 0x76, 0x61, 0x6c, 0x64, 0x2f, 0x61,
+ 0x70, 0x69, 0x73, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x76, 0x31, 0x2f, 0x70, 0x61, 0x79, 0x6c,
+ 0x6f, 0x61, 0x64, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
}
var (
@@ -4483,7 +4542,7 @@ func file_apis_proto_v1_payload_payload_proto_rawDescGZIP() []byte {
}
var file_apis_proto_v1_payload_payload_proto_enumTypes = make([]protoimpl.EnumInfo, 1)
-var file_apis_proto_v1_payload_payload_proto_msgTypes = make([]protoimpl.MessageInfo, 69)
+var file_apis_proto_v1_payload_payload_proto_msgTypes = make([]protoimpl.MessageInfo, 71)
var file_apis_proto_v1_payload_payload_proto_goTypes = []interface{}{
(Search_AggregationAlgorithm)(0), // 0: payload.v1.Search.AggregationAlgorithm
(*Search)(nil), // 1: payload.v1.Search
@@ -4552,10 +4611,12 @@ var file_apis_proto_v1_payload_payload_proto_goTypes = []interface{}{
(*Info_Nodes)(nil), // 64: payload.v1.Info.Nodes
(*Info_IPs)(nil), // 65: payload.v1.Info.IPs
(*Info_Index_Count)(nil), // 66: payload.v1.Info.Index.Count
- (*Info_Index_UUID)(nil), // 67: payload.v1.Info.Index.UUID
- (*Info_Index_UUID_Committed)(nil), // 68: payload.v1.Info.Index.UUID.Committed
- (*Info_Index_UUID_Uncommitted)(nil), // 69: payload.v1.Info.Index.UUID.Uncommitted
- (*status.Status)(nil), // 70: google.rpc.Status
+ (*Info_Index_Detail)(nil), // 67: payload.v1.Info.Index.Detail
+ (*Info_Index_UUID)(nil), // 68: payload.v1.Info.Index.UUID
+ nil, // 69: payload.v1.Info.Index.Detail.CountsEntry
+ (*Info_Index_UUID_Committed)(nil), // 70: payload.v1.Info.Index.UUID.Committed
+ (*Info_Index_UUID_Uncommitted)(nil), // 71: payload.v1.Info.Index.UUID.Uncommitted
+ (*status.Status)(nil), // 72: google.rpc.Status
}
var file_apis_proto_v1_payload_payload_proto_depIdxs = []int32{
18, // 0: payload.v1.Search.Request.config:type_name -> payload.v1.Search.Config
@@ -4571,7 +4632,7 @@ var file_apis_proto_v1_payload_payload_proto_depIdxs = []int32{
43, // 10: payload.v1.Search.Response.results:type_name -> payload.v1.Object.Distance
19, // 11: payload.v1.Search.Responses.responses:type_name -> payload.v1.Search.Response
19, // 12: payload.v1.Search.StreamResponse.response:type_name -> payload.v1.Search.Response
- 70, // 13: payload.v1.Search.StreamResponse.status:type_name -> google.rpc.Status
+ 72, // 13: payload.v1.Search.StreamResponse.status:type_name -> google.rpc.Status
22, // 14: payload.v1.Filter.Config.targets:type_name -> payload.v1.Filter.Target
47, // 15: payload.v1.Insert.Request.vector:type_name -> payload.v1.Object.Vector
28, // 16: payload.v1.Insert.Request.config:type_name -> payload.v1.Insert.Config
@@ -4603,14 +4664,14 @@ var file_apis_proto_v1_payload_payload_proto_depIdxs = []int32{
45, // 42: payload.v1.Object.VectorRequest.id:type_name -> payload.v1.Object.ID
23, // 43: payload.v1.Object.VectorRequest.filters:type_name -> payload.v1.Filter.Config
43, // 44: payload.v1.Object.StreamDistance.distance:type_name -> payload.v1.Object.Distance
- 70, // 45: payload.v1.Object.StreamDistance.status:type_name -> google.rpc.Status
+ 72, // 45: payload.v1.Object.StreamDistance.status:type_name -> google.rpc.Status
47, // 46: payload.v1.Object.Vectors.vectors:type_name -> payload.v1.Object.Vector
47, // 47: payload.v1.Object.StreamVector.vector:type_name -> payload.v1.Object.Vector
- 70, // 48: payload.v1.Object.StreamVector.status:type_name -> google.rpc.Status
+ 72, // 48: payload.v1.Object.StreamVector.status:type_name -> google.rpc.Status
51, // 49: payload.v1.Object.StreamBlob.blob:type_name -> payload.v1.Object.Blob
- 70, // 50: payload.v1.Object.StreamBlob.status:type_name -> google.rpc.Status
+ 72, // 50: payload.v1.Object.StreamBlob.status:type_name -> google.rpc.Status
53, // 51: payload.v1.Object.StreamLocation.location:type_name -> payload.v1.Object.Location
- 70, // 52: payload.v1.Object.StreamLocation.status:type_name -> google.rpc.Status
+ 72, // 52: payload.v1.Object.StreamLocation.status:type_name -> google.rpc.Status
53, // 53: payload.v1.Object.Locations.locations:type_name -> payload.v1.Object.Location
61, // 54: payload.v1.Info.Pod.cpu:type_name -> payload.v1.Info.CPU
62, // 55: payload.v1.Info.Pod.memory:type_name -> payload.v1.Info.Memory
@@ -4620,11 +4681,13 @@ var file_apis_proto_v1_payload_payload_proto_depIdxs = []int32{
63, // 59: payload.v1.Info.Node.Pods:type_name -> payload.v1.Info.Pods
59, // 60: payload.v1.Info.Pods.pods:type_name -> payload.v1.Info.Pod
60, // 61: payload.v1.Info.Nodes.nodes:type_name -> payload.v1.Info.Node
- 62, // [62:62] is the sub-list for method output_type
- 62, // [62:62] is the sub-list for method input_type
- 62, // [62:62] is the sub-list for extension type_name
- 62, // [62:62] is the sub-list for extension extendee
- 0, // [0:62] is the sub-list for field type_name
+ 69, // 62: payload.v1.Info.Index.Detail.counts:type_name -> payload.v1.Info.Index.Detail.CountsEntry
+ 66, // 63: payload.v1.Info.Index.Detail.CountsEntry.value:type_name -> payload.v1.Info.Index.Count
+ 64, // [64:64] is the sub-list for method output_type
+ 64, // [64:64] is the sub-list for method input_type
+ 64, // [64:64] is the sub-list for extension type_name
+ 64, // [64:64] is the sub-list for extension extendee
+ 0, // [0:64] is the sub-list for field type_name
}
func init() { file_apis_proto_v1_payload_payload_proto_init() }
@@ -5426,7 +5489,7 @@ func file_apis_proto_v1_payload_payload_proto_init() {
}
}
file_apis_proto_v1_payload_payload_proto_msgTypes[66].Exporter = func(v interface{}, i int) interface{} {
- switch v := v.(*Info_Index_UUID); i {
+ switch v := v.(*Info_Index_Detail); i {
case 0:
return &v.state
case 1:
@@ -5438,6 +5501,18 @@ func file_apis_proto_v1_payload_payload_proto_init() {
}
}
file_apis_proto_v1_payload_payload_proto_msgTypes[67].Exporter = func(v interface{}, i int) interface{} {
+ switch v := v.(*Info_Index_UUID); i {
+ case 0:
+ return &v.state
+ case 1:
+ return &v.sizeCache
+ case 2:
+ return &v.unknownFields
+ default:
+ return nil
+ }
+ }
+ file_apis_proto_v1_payload_payload_proto_msgTypes[69].Exporter = func(v interface{}, i int) interface{} {
switch v := v.(*Info_Index_UUID_Committed); i {
case 0:
return &v.state
@@ -5449,7 +5524,7 @@ func file_apis_proto_v1_payload_payload_proto_init() {
return nil
}
}
- file_apis_proto_v1_payload_payload_proto_msgTypes[68].Exporter = func(v interface{}, i int) interface{} {
+ file_apis_proto_v1_payload_payload_proto_msgTypes[70].Exporter = func(v interface{}, i int) interface{} {
switch v := v.(*Info_Index_UUID_Uncommitted); i {
case 0:
return &v.state
@@ -5488,7 +5563,7 @@ func file_apis_proto_v1_payload_payload_proto_init() {
GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
RawDescriptor: file_apis_proto_v1_payload_payload_proto_rawDesc,
NumEnums: 1,
- NumMessages: 69,
+ NumMessages: 71,
NumExtensions: 0,
NumServices: 0,
},
diff --git a/apis/grpc/v1/payload/payload_vtproto.pb.go b/apis/grpc/v1/payload/payload_vtproto.pb.go
index ac22ec9fbab..16bf1243a00 100644
--- a/apis/grpc/v1/payload/payload_vtproto.pb.go
+++ b/apis/grpc/v1/payload/payload_vtproto.pb.go
@@ -1317,6 +1317,29 @@ func (m *Info_Index_Count) CloneMessageVT() proto.Message {
return m.CloneVT()
}
+func (m *Info_Index_Detail) CloneVT() *Info_Index_Detail {
+ if m == nil {
+ return (*Info_Index_Detail)(nil)
+ }
+ r := &Info_Index_Detail{}
+ if rhs := m.Counts; rhs != nil {
+ tmpContainer := make(map[string]*Info_Index_Count, len(rhs))
+ for k, v := range rhs {
+ tmpContainer[k] = v.CloneVT()
+ }
+ r.Counts = tmpContainer
+ }
+ if len(m.unknownFields) > 0 {
+ r.unknownFields = make([]byte, len(m.unknownFields))
+ copy(r.unknownFields, m.unknownFields)
+ }
+ return r
+}
+
+func (m *Info_Index_Detail) CloneMessageVT() proto.Message {
+ return m.CloneVT()
+}
+
func (m *Info_Index_UUID_Committed) CloneVT() *Info_Index_UUID_Committed {
if m == nil {
return (*Info_Index_UUID_Committed)(nil)
@@ -3301,6 +3324,42 @@ func (this *Info_Index_Count) EqualMessageVT(thatMsg proto.Message) bool {
}
return this.EqualVT(that)
}
+func (this *Info_Index_Detail) EqualVT(that *Info_Index_Detail) bool {
+ if this == that {
+ return true
+ } else if this == nil || that == nil {
+ return false
+ }
+ if len(this.Counts) != len(that.Counts) {
+ return false
+ }
+ for i, vx := range this.Counts {
+ vy, ok := that.Counts[i]
+ if !ok {
+ return false
+ }
+ if p, q := vx, vy; p != q {
+ if p == nil {
+ p = &Info_Index_Count{}
+ }
+ if q == nil {
+ q = &Info_Index_Count{}
+ }
+ if !p.EqualVT(q) {
+ return false
+ }
+ }
+ }
+ return string(this.unknownFields) == string(that.unknownFields)
+}
+
+func (this *Info_Index_Detail) EqualMessageVT(thatMsg proto.Message) bool {
+ that, ok := thatMsg.(*Info_Index_Detail)
+ if !ok {
+ return false
+ }
+ return this.EqualVT(that)
+}
func (this *Info_Index_UUID_Committed) EqualVT(that *Info_Index_UUID_Committed) bool {
if this == that {
return true
@@ -6552,6 +6611,61 @@ func (m *Info_Index_Count) MarshalToSizedBufferVT(dAtA []byte) (int, error) {
return len(dAtA) - i, nil
}
+func (m *Info_Index_Detail) MarshalVT() (dAtA []byte, err error) {
+ if m == nil {
+ return nil, nil
+ }
+ size := m.SizeVT()
+ dAtA = make([]byte, size)
+ n, err := m.MarshalToSizedBufferVT(dAtA[:size])
+ if err != nil {
+ return nil, err
+ }
+ return dAtA[:n], nil
+}
+
+func (m *Info_Index_Detail) MarshalToVT(dAtA []byte) (int, error) {
+ size := m.SizeVT()
+ return m.MarshalToSizedBufferVT(dAtA[:size])
+}
+
+func (m *Info_Index_Detail) MarshalToSizedBufferVT(dAtA []byte) (int, error) {
+ if m == nil {
+ return 0, nil
+ }
+ i := len(dAtA)
+ _ = i
+ var l int
+ _ = l
+ if m.unknownFields != nil {
+ i -= len(m.unknownFields)
+ copy(dAtA[i:], m.unknownFields)
+ }
+ if len(m.Counts) > 0 {
+ for k := range m.Counts {
+ v := m.Counts[k]
+ baseI := i
+ size, err := v.MarshalToSizedBufferVT(dAtA[:i])
+ if err != nil {
+ return 0, err
+ }
+ i -= size
+ i = encodeVarint(dAtA, i, uint64(size))
+ i--
+ dAtA[i] = 0x12
+ i -= len(k)
+ copy(dAtA[i:], k)
+ i = encodeVarint(dAtA, i, uint64(len(k)))
+ i--
+ dAtA[i] = 0xa
+ i = encodeVarint(dAtA, i, uint64(baseI-i))
+ i--
+ dAtA[i] = 0xa
+ }
+ }
+ return len(dAtA) - i, nil
+}
+
func (m *Info_Index_UUID_Committed) MarshalVT() (dAtA []byte, err error) {
if m == nil {
return nil, nil
@@ -8272,6 +8386,29 @@ func (m *Info_Index_Count) SizeVT() (n int) {
return n
}
+func (m *Info_Index_Detail) SizeVT() (n int) {
+ if m == nil {
+ return 0
+ }
+ var l int
+ _ = l
+ if len(m.Counts) > 0 {
+ for k, v := range m.Counts {
+ _ = k
+ _ = v
+ l = 0
+ if v != nil {
+ l = v.SizeVT()
+ }
+ l += 1 + sov(uint64(l))
+ mapEntrySize := 1 + len(k) + sov(uint64(len(k))) + l
+ n += mapEntrySize + 1 + sov(uint64(mapEntrySize))
+ }
+ }
+ n += len(m.unknownFields)
+ return n
+}
+
func (m *Info_Index_UUID_Committed) SizeVT() (n int) {
if m == nil {
return 0
@@ -14574,6 +14711,186 @@ func (m *Info_Index_Count) UnmarshalVT(dAtA []byte) error {
}
return nil
}
+func (m *Info_Index_Detail) UnmarshalVT(dAtA []byte) error {
+ l := len(dAtA)
+ iNdEx := 0
+ for iNdEx < l {
+ preIndex := iNdEx
+ var wire uint64
+ for shift := uint(0); ; shift += 7 {
+ if shift >= 64 {
+ return ErrIntOverflow
+ }
+ if iNdEx >= l {
+ return io.ErrUnexpectedEOF
+ }
+ b := dAtA[iNdEx]
+ iNdEx++
+ wire |= uint64(b&0x7F) << shift
+ if b < 0x80 {
+ break
+ }
+ }
+ fieldNum := int32(wire >> 3)
+ wireType := int(wire & 0x7)
+ if wireType == 4 {
+ return fmt.Errorf("proto: Info_Index_Detail: wiretype end group for non-group")
+ }
+ if fieldNum <= 0 {
+ return fmt.Errorf("proto: Info_Index_Detail: illegal tag %d (wire type %d)", fieldNum, wire)
+ }
+ switch fieldNum {
+ case 1:
+ if wireType != 2 {
+ return fmt.Errorf("proto: wrong wireType = %d for field Counts", wireType)
+ }
+ var msglen int
+ for shift := uint(0); ; shift += 7 {
+ if shift >= 64 {
+ return ErrIntOverflow
+ }
+ if iNdEx >= l {
+ return io.ErrUnexpectedEOF
+ }
+ b := dAtA[iNdEx]
+ iNdEx++
+ msglen |= int(b&0x7F) << shift
+ if b < 0x80 {
+ break
+ }
+ }
+ if msglen < 0 {
+ return ErrInvalidLength
+ }
+ postIndex := iNdEx + msglen
+ if postIndex < 0 {
+ return ErrInvalidLength
+ }
+ if postIndex > l {
+ return io.ErrUnexpectedEOF
+ }
+ if m.Counts == nil {
+ m.Counts = make(map[string]*Info_Index_Count)
+ }
+ var mapkey string
+ var mapvalue *Info_Index_Count
+ for iNdEx < postIndex {
+ entryPreIndex := iNdEx
+ var wire uint64
+ for shift := uint(0); ; shift += 7 {
+ if shift >= 64 {
+ return ErrIntOverflow
+ }
+ if iNdEx >= l {
+ return io.ErrUnexpectedEOF
+ }
+ b := dAtA[iNdEx]
+ iNdEx++
+ wire |= uint64(b&0x7F) << shift
+ if b < 0x80 {
+ break
+ }
+ }
+ fieldNum := int32(wire >> 3)
+ if fieldNum == 1 {
+ var stringLenmapkey uint64
+ for shift := uint(0); ; shift += 7 {
+ if shift >= 64 {
+ return ErrIntOverflow
+ }
+ if iNdEx >= l {
+ return io.ErrUnexpectedEOF
+ }
+ b := dAtA[iNdEx]
+ iNdEx++
+ stringLenmapkey |= uint64(b&0x7F) << shift
+ if b < 0x80 {
+ break
+ }
+ }
+ intStringLenmapkey := int(stringLenmapkey)
+ if intStringLenmapkey < 0 {
+ return ErrInvalidLength
+ }
+ postStringIndexmapkey := iNdEx + intStringLenmapkey
+ if postStringIndexmapkey < 0 {
+ return ErrInvalidLength
+ }
+ if postStringIndexmapkey > l {
+ return io.ErrUnexpectedEOF
+ }
+ mapkey = string(dAtA[iNdEx:postStringIndexmapkey])
+ iNdEx = postStringIndexmapkey
+ } else if fieldNum == 2 {
+ var mapmsglen int
+ for shift := uint(0); ; shift += 7 {
+ if shift >= 64 {
+ return ErrIntOverflow
+ }
+ if iNdEx >= l {
+ return io.ErrUnexpectedEOF
+ }
+ b := dAtA[iNdEx]
+ iNdEx++
+ mapmsglen |= int(b&0x7F) << shift
+ if b < 0x80 {
+ break
+ }
+ }
+ if mapmsglen < 0 {
+ return ErrInvalidLength
+ }
+ postmsgIndex := iNdEx + mapmsglen
+ if postmsgIndex < 0 {
+ return ErrInvalidLength
+ }
+ if postmsgIndex > l {
+ return io.ErrUnexpectedEOF
+ }
+ mapvalue = &Info_Index_Count{}
+ if err := mapvalue.UnmarshalVT(dAtA[iNdEx:postmsgIndex]); err != nil {
+ return err
+ }
+ iNdEx = postmsgIndex
+ } else {
+ iNdEx = entryPreIndex
+ skippy, err := skip(dAtA[iNdEx:])
+ if err != nil {
+ return err
+ }
+ if (skippy < 0) || (iNdEx+skippy) < 0 {
+ return ErrInvalidLength
+ }
+ if (iNdEx + skippy) > postIndex {
+ return io.ErrUnexpectedEOF
+ }
+ iNdEx += skippy
+ }
+ }
+ m.Counts[mapkey] = mapvalue
+ iNdEx = postIndex
+ default:
+ iNdEx = preIndex
+ skippy, err := skip(dAtA[iNdEx:])
+ if err != nil {
+ return err
+ }
+ if (skippy < 0) || (iNdEx+skippy) < 0 {
+ return ErrInvalidLength
+ }
+ if (iNdEx + skippy) > l {
+ return io.ErrUnexpectedEOF
+ }
+ m.unknownFields = append(m.unknownFields, dAtA[iNdEx:iNdEx+skippy]...)
+ iNdEx += skippy
+ }
+ }
+
+ if iNdEx > l {
+ return io.ErrUnexpectedEOF
+ }
+ return nil
+}
func (m *Info_Index_UUID_Committed) UnmarshalVT(dAtA []byte) error {
l := len(dAtA)
iNdEx := 0
diff --git a/apis/grpc/v1/vald/filter.pb.go b/apis/grpc/v1/vald/filter.pb.go
index 51befdf4365..b08d19dcbfe 100644
--- a/apis/grpc/v1/vald/filter.pb.go
+++ b/apis/grpc/v1/vald/filter.pb.go
@@ -17,7 +17,7 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.31.0
-// protoc v4.23.3
+// protoc v4.23.4
// source: apis/proto/v1/vald/filter.proto
package vald
diff --git a/apis/grpc/v1/vald/insert.pb.go b/apis/grpc/v1/vald/insert.pb.go
index 49be357eff6..96e84b2e5e3 100644
--- a/apis/grpc/v1/vald/insert.pb.go
+++ b/apis/grpc/v1/vald/insert.pb.go
@@ -17,7 +17,7 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.31.0
-// protoc v4.23.3
+// protoc v4.23.4
// source: apis/proto/v1/vald/insert.proto
package vald
diff --git a/apis/grpc/v1/vald/object.pb.go b/apis/grpc/v1/vald/object.pb.go
index 6c992f0345d..d7c79d6c5a6 100644
--- a/apis/grpc/v1/vald/object.pb.go
+++ b/apis/grpc/v1/vald/object.pb.go
@@ -17,7 +17,7 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.31.0
-// protoc v4.23.3
+// protoc v4.23.4
// source: apis/proto/v1/vald/object.proto
package vald
diff --git a/apis/grpc/v1/vald/remove.pb.go b/apis/grpc/v1/vald/remove.pb.go
index 2f7edd2dc91..cf8ba0ca735 100644
--- a/apis/grpc/v1/vald/remove.pb.go
+++ b/apis/grpc/v1/vald/remove.pb.go
@@ -17,7 +17,7 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.31.0
-// protoc v4.23.3
+// protoc v4.23.4
// source: apis/proto/v1/vald/remove.proto
package vald
diff --git a/apis/grpc/v1/vald/search.pb.go b/apis/grpc/v1/vald/search.pb.go
index f3a25256226..5f32ea623e9 100644
--- a/apis/grpc/v1/vald/search.pb.go
+++ b/apis/grpc/v1/vald/search.pb.go
@@ -17,7 +17,7 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.31.0
-// protoc v4.23.3
+// protoc v4.23.4
// source: apis/proto/v1/vald/search.proto
package vald
diff --git a/apis/grpc/v1/vald/update.pb.go b/apis/grpc/v1/vald/update.pb.go
index 309e5df2cf1..6db85bb7317 100644
--- a/apis/grpc/v1/vald/update.pb.go
+++ b/apis/grpc/v1/vald/update.pb.go
@@ -17,7 +17,7 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.31.0
-// protoc v4.23.3
+// protoc v4.23.4
// source: apis/proto/v1/vald/update.proto
package vald
diff --git a/apis/grpc/v1/vald/upsert.pb.go b/apis/grpc/v1/vald/upsert.pb.go
index 5583d957214..491fc403128 100644
--- a/apis/grpc/v1/vald/upsert.pb.go
+++ b/apis/grpc/v1/vald/upsert.pb.go
@@ -17,7 +17,7 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.31.0
-// protoc v4.23.3
+// protoc v4.23.4
// source: apis/proto/v1/vald/upsert.proto
package vald
diff --git a/apis/proto/v1/manager/index/index_manager.proto b/apis/proto/v1/manager/index/index_manager.proto
index 40a1679d6b6..5bfa43433c1 100644
--- a/apis/proto/v1/manager/index/index_manager.proto
+++ b/apis/proto/v1/manager/index/index_manager.proto
@@ -32,4 +32,8 @@ service Index {
rpc IndexInfo(payload.v1.Empty) returns (payload.v1.Info.Index.Count) {
option (google.api.http).get = "/index/info";
}
+ // Represent the RPC to get the index information for each agents.
+ rpc IndexDetail(payload.v1.Empty) returns (payload.v1.Info.Index.Detail) {
+ option (google.api.http).get = "/index/detail";
+ }
}
diff --git a/apis/proto/v1/payload/payload.proto b/apis/proto/v1/payload/payload.proto
index 574503f8538..a76ec99cc64 100644
--- a/apis/proto/v1/payload/payload.proto
+++ b/apis/proto/v1/payload/payload.proto
@@ -457,6 +457,11 @@ message Info {
// The saving index count.
bool saving = 4;
}
+ // Represent the index count for each Agents message.
+ message Detail {
+ // count infos for each agents
+ map counts = 1;
+ }
// Represent the UUID message.
message UUID {
diff --git a/apis/swagger/v1/manager/index/apis/proto/v1/manager/index/index_manager.swagger.json b/apis/swagger/v1/manager/index/apis/proto/v1/manager/index/index_manager.swagger.json
index 76548a29ca6..e3563a89aab 100644
--- a/apis/swagger/v1/manager/index/apis/proto/v1/manager/index/index_manager.swagger.json
+++ b/apis/swagger/v1/manager/index/apis/proto/v1/manager/index/index_manager.swagger.json
@@ -7,6 +7,27 @@
"consumes": ["application/json"],
"produces": ["application/json"],
"paths": {
+ "/index/detail": {
+ "get": {
+ "summary": "Represent the RPC to get the index information for each agents.",
+ "operationId": "Index_IndexDetail",
+ "responses": {
+ "200": {
+ "description": "A successful response.",
+ "schema": {
+ "$ref": "#/definitions/IndexDetail"
+ }
+ },
+ "default": {
+ "description": "An unexpected error response.",
+ "schema": {
+ "$ref": "#/definitions/runtimeError"
+ }
+ }
+ },
+ "tags": ["Index"]
+ }
+ },
"/index/info": {
"get": {
"summary": "Represent the RPC to get the index information.",
@@ -54,6 +75,19 @@
},
"description": "Represent the index count message."
},
+ "IndexDetail": {
+ "type": "object",
+ "properties": {
+ "counts": {
+ "type": "object",
+ "additionalProperties": {
+ "$ref": "#/definitions/IndexCount"
+ },
+ "title": "count infos for each agents"
+ }
+ },
+ "description": "Represent the index count for each Agents message."
+ },
"protobufAny": {
"type": "object",
"properties": {
diff --git a/apis/swagger/v1/vald/apis/proto/v1/vald/filter.swagger.json b/apis/swagger/v1/vald/apis/proto/v1/vald/filter.swagger.json
index d98f15559e2..506dea78946 100644
--- a/apis/swagger/v1/vald/apis/proto/v1/vald/filter.swagger.json
+++ b/apis/swagger/v1/vald/apis/proto/v1/vald/filter.swagger.json
@@ -405,11 +405,11 @@
"code": {
"type": "integer",
"format": "int32",
- "description": "The status code, which should be an enum value of\n[google.rpc.Code][google.rpc.Code]."
+ "description": "The status code, which should be an enum value of [google.rpc.Code][google.rpc.Code]."
},
"message": {
"type": "string",
- "description": "A developer-facing error message, which should be in English. Any\nuser-facing error message should be localized and sent in the\n[google.rpc.Status.details][google.rpc.Status.details] field, or localized\nby the client."
+ "description": "A developer-facing error message, which should be in English. Any\nuser-facing error message should be localized and sent in the\n[google.rpc.Status.details][google.rpc.Status.details] field, or localized by the client."
},
"details": {
"type": "array",
diff --git a/apis/swagger/v1/vald/apis/proto/v1/vald/insert.swagger.json b/apis/swagger/v1/vald/apis/proto/v1/vald/insert.swagger.json
index 3154ff85286..9b3d15944f1 100644
--- a/apis/swagger/v1/vald/apis/proto/v1/vald/insert.swagger.json
+++ b/apis/swagger/v1/vald/apis/proto/v1/vald/insert.swagger.json
@@ -156,11 +156,11 @@
"code": {
"type": "integer",
"format": "int32",
- "description": "The status code, which should be an enum value of\n[google.rpc.Code][google.rpc.Code]."
+ "description": "The status code, which should be an enum value of [google.rpc.Code][google.rpc.Code]."
},
"message": {
"type": "string",
- "description": "A developer-facing error message, which should be in English. Any\nuser-facing error message should be localized and sent in the\n[google.rpc.Status.details][google.rpc.Status.details] field, or localized\nby the client."
+ "description": "A developer-facing error message, which should be in English. Any\nuser-facing error message should be localized and sent in the\n[google.rpc.Status.details][google.rpc.Status.details] field, or localized by the client."
},
"details": {
"type": "array",
diff --git a/apis/swagger/v1/vald/apis/proto/v1/vald/object.swagger.json b/apis/swagger/v1/vald/apis/proto/v1/vald/object.swagger.json
index da1aa1cd142..a8568f59c92 100644
--- a/apis/swagger/v1/vald/apis/proto/v1/vald/object.swagger.json
+++ b/apis/swagger/v1/vald/apis/proto/v1/vald/object.swagger.json
@@ -149,11 +149,11 @@
"code": {
"type": "integer",
"format": "int32",
- "description": "The status code, which should be an enum value of\n[google.rpc.Code][google.rpc.Code]."
+ "description": "The status code, which should be an enum value of [google.rpc.Code][google.rpc.Code]."
},
"message": {
"type": "string",
- "description": "A developer-facing error message, which should be in English. Any\nuser-facing error message should be localized and sent in the\n[google.rpc.Status.details][google.rpc.Status.details] field, or localized\nby the client."
+ "description": "A developer-facing error message, which should be in English. Any\nuser-facing error message should be localized and sent in the\n[google.rpc.Status.details][google.rpc.Status.details] field, or localized by the client."
},
"details": {
"type": "array",
diff --git a/apis/swagger/v1/vald/apis/proto/v1/vald/remove.swagger.json b/apis/swagger/v1/vald/apis/proto/v1/vald/remove.swagger.json
index ad593dfa0b5..36b4cf8f674 100644
--- a/apis/swagger/v1/vald/apis/proto/v1/vald/remove.swagger.json
+++ b/apis/swagger/v1/vald/apis/proto/v1/vald/remove.swagger.json
@@ -127,11 +127,11 @@
"code": {
"type": "integer",
"format": "int32",
- "description": "The status code, which should be an enum value of\n[google.rpc.Code][google.rpc.Code]."
+ "description": "The status code, which should be an enum value of [google.rpc.Code][google.rpc.Code]."
},
"message": {
"type": "string",
- "description": "A developer-facing error message, which should be in English. Any\nuser-facing error message should be localized and sent in the\n[google.rpc.Status.details][google.rpc.Status.details] field, or localized\nby the client."
+ "description": "A developer-facing error message, which should be in English. Any\nuser-facing error message should be localized and sent in the\n[google.rpc.Status.details][google.rpc.Status.details] field, or localized by the client."
},
"details": {
"type": "array",
diff --git a/apis/swagger/v1/vald/apis/proto/v1/vald/search.swagger.json b/apis/swagger/v1/vald/apis/proto/v1/vald/search.swagger.json
index 4a193c0d849..4345256f6c0 100644
--- a/apis/swagger/v1/vald/apis/proto/v1/vald/search.swagger.json
+++ b/apis/swagger/v1/vald/apis/proto/v1/vald/search.swagger.json
@@ -391,11 +391,11 @@
"code": {
"type": "integer",
"format": "int32",
- "description": "The status code, which should be an enum value of\n[google.rpc.Code][google.rpc.Code]."
+ "description": "The status code, which should be an enum value of [google.rpc.Code][google.rpc.Code]."
},
"message": {
"type": "string",
- "description": "A developer-facing error message, which should be in English. Any\nuser-facing error message should be localized and sent in the\n[google.rpc.Status.details][google.rpc.Status.details] field, or localized\nby the client."
+ "description": "A developer-facing error message, which should be in English. Any\nuser-facing error message should be localized and sent in the\n[google.rpc.Status.details][google.rpc.Status.details] field, or localized by the client."
},
"details": {
"type": "array",
diff --git a/apis/swagger/v1/vald/apis/proto/v1/vald/update.swagger.json b/apis/swagger/v1/vald/apis/proto/v1/vald/update.swagger.json
index 82e80fe506b..4f0b54cd14e 100644
--- a/apis/swagger/v1/vald/apis/proto/v1/vald/update.swagger.json
+++ b/apis/swagger/v1/vald/apis/proto/v1/vald/update.swagger.json
@@ -156,11 +156,11 @@
"code": {
"type": "integer",
"format": "int32",
- "description": "The status code, which should be an enum value of\n[google.rpc.Code][google.rpc.Code]."
+ "description": "The status code, which should be an enum value of [google.rpc.Code][google.rpc.Code]."
},
"message": {
"type": "string",
- "description": "A developer-facing error message, which should be in English. Any\nuser-facing error message should be localized and sent in the\n[google.rpc.Status.details][google.rpc.Status.details] field, or localized\nby the client."
+ "description": "A developer-facing error message, which should be in English. Any\nuser-facing error message should be localized and sent in the\n[google.rpc.Status.details][google.rpc.Status.details] field, or localized by the client."
},
"details": {
"type": "array",
diff --git a/apis/swagger/v1/vald/apis/proto/v1/vald/upsert.swagger.json b/apis/swagger/v1/vald/apis/proto/v1/vald/upsert.swagger.json
index a6defddc461..0108ed17619 100644
--- a/apis/swagger/v1/vald/apis/proto/v1/vald/upsert.swagger.json
+++ b/apis/swagger/v1/vald/apis/proto/v1/vald/upsert.swagger.json
@@ -156,11 +156,11 @@
"code": {
"type": "integer",
"format": "int32",
- "description": "The status code, which should be an enum value of\n[google.rpc.Code][google.rpc.Code]."
+ "description": "The status code, which should be an enum value of [google.rpc.Code][google.rpc.Code]."
},
"message": {
"type": "string",
- "description": "A developer-facing error message, which should be in English. Any\nuser-facing error message should be localized and sent in the\n[google.rpc.Status.details][google.rpc.Status.details] field, or localized\nby the client."
+ "description": "A developer-facing error message, which should be in English. Any\nuser-facing error message should be localized and sent in the\n[google.rpc.Status.details][google.rpc.Status.details] field, or localized by the client."
},
"details": {
"type": "array",
diff --git a/docs/contributing/coding-style.md b/docs/contributing/coding-style.md
index da6644f1c54..50a5ee693d3 100644
--- a/docs/contributing/coding-style.md
+++ b/docs/contributing/coding-style.md
@@ -385,7 +385,7 @@ if err := srv.Run(); err != nil {
All errors should define in [internal/errors package](https://github.com/vdaas/vald/blob/main/internal/errors). All errors should be start with `Err` prefix, and all errors should be handle if possible.
-Please use [internal/errgroup](https://github.com/vdaas/vald/blob/main/internal/errgroup) for synchronized error handling on multi-goroutine processing.
+Please use [internal/sync/errgroup](https://github.com/vdaas/vald/blob/main/internal/sync/errgroup) for synchronized error handling on multi-goroutine processing.
### Error checking
diff --git a/go.mod b/go.mod
index 7aa07867e19..5fc6cd16737 100755
--- a/go.mod
+++ b/go.mod
@@ -5,8 +5,8 @@ go 1.20
replace (
cloud.google.com/go => cloud.google.com/go v0.110.6
cloud.google.com/go/bigquery => cloud.google.com/go/bigquery v1.53.0
- cloud.google.com/go/compute => cloud.google.com/go/compute v1.22.0
- cloud.google.com/go/datastore => cloud.google.com/go/datastore v1.12.1
+ cloud.google.com/go/compute => cloud.google.com/go/compute v1.23.0
+ cloud.google.com/go/datastore => cloud.google.com/go/datastore v1.13.0
cloud.google.com/go/firestore => cloud.google.com/go/firestore v1.11.0
cloud.google.com/go/iam => cloud.google.com/go/iam v1.1.1
cloud.google.com/go/kms => cloud.google.com/go/kms v1.15.0
@@ -43,28 +43,28 @@ replace (
github.com/ajstarks/svgo => github.com/ajstarks/svgo v0.0.0-20211024235047-1546f124cd8b
github.com/antihax/optional => github.com/antihax/optional v1.0.0
github.com/armon/go-socks5 => github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5
- github.com/aws/aws-sdk-go => github.com/aws/aws-sdk-go v1.44.308
- github.com/aws/aws-sdk-go-v2 => github.com/aws/aws-sdk-go-v2 v1.19.0
+ github.com/aws/aws-sdk-go => github.com/aws/aws-sdk-go v1.44.312
+ github.com/aws/aws-sdk-go-v2 => github.com/aws/aws-sdk-go-v2 v1.19.1
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream => github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10
- github.com/aws/aws-sdk-go-v2/config => github.com/aws/aws-sdk-go-v2/config v1.18.29
- github.com/aws/aws-sdk-go-v2/credentials => github.com/aws/aws-sdk-go-v2/credentials v1.13.28
- github.com/aws/aws-sdk-go-v2/feature/ec2/imds => github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.5
- github.com/aws/aws-sdk-go-v2/feature/s3/manager => github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.73
- github.com/aws/aws-sdk-go-v2/internal/configsources => github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.35
- github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 => github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.29
- github.com/aws/aws-sdk-go-v2/internal/ini => github.com/aws/aws-sdk-go-v2/internal/ini v1.3.36
+ github.com/aws/aws-sdk-go-v2/config => github.com/aws/aws-sdk-go-v2/config v1.18.30
+ github.com/aws/aws-sdk-go-v2/credentials => github.com/aws/aws-sdk-go-v2/credentials v1.13.29
+ github.com/aws/aws-sdk-go-v2/feature/ec2/imds => github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.6
+ github.com/aws/aws-sdk-go-v2/feature/s3/manager => github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.74
+ github.com/aws/aws-sdk-go-v2/internal/configsources => github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.36
+ github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 => github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.30
+ github.com/aws/aws-sdk-go-v2/internal/ini => github.com/aws/aws-sdk-go-v2/internal/ini v1.3.37
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding => github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.11
- github.com/aws/aws-sdk-go-v2/service/internal/checksum => github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.30
- github.com/aws/aws-sdk-go-v2/service/internal/presigned-url => github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.29
- github.com/aws/aws-sdk-go-v2/service/internal/s3shared => github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.14.4
- github.com/aws/aws-sdk-go-v2/service/kms => github.com/aws/aws-sdk-go-v2/service/kms v1.23.1
- github.com/aws/aws-sdk-go-v2/service/s3 => github.com/aws/aws-sdk-go-v2/service/s3 v1.37.0
- github.com/aws/aws-sdk-go-v2/service/secretsmanager => github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.19.11
- github.com/aws/aws-sdk-go-v2/service/sns => github.com/aws/aws-sdk-go-v2/service/sns v1.20.14
- github.com/aws/aws-sdk-go-v2/service/sqs => github.com/aws/aws-sdk-go-v2/service/sqs v1.23.3
- github.com/aws/aws-sdk-go-v2/service/ssm => github.com/aws/aws-sdk-go-v2/service/ssm v1.36.8
- github.com/aws/aws-sdk-go-v2/service/sso => github.com/aws/aws-sdk-go-v2/service/sso v1.12.13
- github.com/aws/aws-sdk-go-v2/service/sts => github.com/aws/aws-sdk-go-v2/service/sts v1.20.0
+ github.com/aws/aws-sdk-go-v2/service/internal/checksum => github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.31
+ github.com/aws/aws-sdk-go-v2/service/internal/presigned-url => github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.30
+ github.com/aws/aws-sdk-go-v2/service/internal/s3shared => github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.14.5
+ github.com/aws/aws-sdk-go-v2/service/kms => github.com/aws/aws-sdk-go-v2/service/kms v1.23.2
+ github.com/aws/aws-sdk-go-v2/service/s3 => github.com/aws/aws-sdk-go-v2/service/s3 v1.37.1
+ github.com/aws/aws-sdk-go-v2/service/secretsmanager => github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.19.12
+ github.com/aws/aws-sdk-go-v2/service/sns => github.com/aws/aws-sdk-go-v2/service/sns v1.20.15
+ github.com/aws/aws-sdk-go-v2/service/sqs => github.com/aws/aws-sdk-go-v2/service/sqs v1.23.4
+ github.com/aws/aws-sdk-go-v2/service/ssm => github.com/aws/aws-sdk-go-v2/service/ssm v1.36.9
+ github.com/aws/aws-sdk-go-v2/service/sso => github.com/aws/aws-sdk-go-v2/service/sso v1.12.14
+ github.com/aws/aws-sdk-go-v2/service/sts => github.com/aws/aws-sdk-go-v2/service/sts v1.20.1
github.com/aws/smithy-go => github.com/aws/smithy-go v1.13.5
github.com/benbjohnson/clock => github.com/benbjohnson/clock v1.3.5
github.com/beorn7/perks => github.com/beorn7/perks v1.0.1
@@ -153,7 +153,7 @@ replace (
github.com/google/gofuzz => github.com/google/gofuzz v1.2.0
github.com/google/martian => github.com/google/martian v2.1.0+incompatible
github.com/google/martian/v3 => github.com/google/martian/v3 v3.3.2
- github.com/google/pprof => github.com/google/pprof v0.0.0-20230705174524-200ffdc848b8
+ github.com/google/pprof => github.com/google/pprof v0.0.0-20230728192033-2ba5b33183c6
github.com/google/shlex => github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510
github.com/google/subcommands => github.com/google/subcommands v1.2.0
github.com/google/uuid => github.com/google/uuid v1.3.0
@@ -162,7 +162,7 @@ replace (
github.com/gorilla/mux => github.com/gorilla/mux v1.8.0
github.com/gorilla/websocket => github.com/gorilla/websocket v1.5.0
github.com/gregjones/httpcache => github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79
- github.com/grpc-ecosystem/grpc-gateway/v2 => github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.1
+ github.com/grpc-ecosystem/grpc-gateway/v2 => github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.2
github.com/hailocab/go-hostpool => github.com/kpango/go-hostpool v0.0.0-20210303030322-aab80263dcd0
github.com/hanwen/go-fuse/v2 => github.com/hanwen/go-fuse/v2 v2.3.0
github.com/hashicorp/go-uuid => github.com/hashicorp/go-uuid v1.0.3
@@ -190,7 +190,7 @@ replace (
github.com/jstemmer/go-junit-report => github.com/jstemmer/go-junit-report v1.0.0
github.com/kisielk/errcheck => github.com/kisielk/errcheck v1.6.3
github.com/kisielk/gotool => github.com/kisielk/gotool v1.0.0
- github.com/klauspost/compress => github.com/klauspost/compress v1.16.8-0.20230717094025-33c9bc800133
+ github.com/klauspost/compress => github.com/klauspost/compress v1.16.8-0.20230727132629-c1dcc38258f0
github.com/klauspost/cpuid/v2 => github.com/klauspost/cpuid/v2 v2.2.5
github.com/kpango/fastime => github.com/kpango/fastime v1.1.9
github.com/kpango/fuid => github.com/kpango/fuid v0.0.0-20221203053508-503b5ad89aa1
@@ -237,7 +237,7 @@ replace (
github.com/prometheus/client_golang => github.com/prometheus/client_golang v1.16.0
github.com/prometheus/client_model => github.com/prometheus/client_model v0.4.0
github.com/prometheus/common => github.com/prometheus/common v0.44.0
- github.com/prometheus/procfs => github.com/prometheus/procfs v0.11.0
+ github.com/prometheus/procfs => github.com/prometheus/procfs v0.11.1
github.com/prometheus/prometheus => github.com/prometheus/prometheus v1.99.0
github.com/quasilyte/go-ruleguard => github.com/quasilyte/go-ruleguard v0.3.19
github.com/quasilyte/go-ruleguard/dsl => github.com/quasilyte/go-ruleguard/dsl v0.3.22
@@ -246,7 +246,7 @@ replace (
github.com/rogpeppe/fastuuid => github.com/rogpeppe/fastuuid v1.2.0
github.com/rogpeppe/go-internal => github.com/rogpeppe/go-internal v1.11.0
github.com/rs/xid => github.com/rs/xid v1.5.0
- github.com/rs/zerolog => github.com/rs/zerolog v1.29.1
+ github.com/rs/zerolog => github.com/rs/zerolog v1.30.0
github.com/russross/blackfriday/v2 => github.com/russross/blackfriday/v2 v2.1.0
github.com/ruudk/golang-pdf417 => github.com/ruudk/golang-pdf417 v0.0.0-20201230142125-a7e3863a1245
github.com/schollz/progressbar/v2 => github.com/schollz/progressbar/v2 v2.15.0
@@ -283,7 +283,7 @@ replace (
go.opentelemetry.io/otel/sdk/metric => go.opentelemetry.io/otel/sdk/metric v0.33.0
go.opentelemetry.io/otel/trace => go.opentelemetry.io/otel/trace v1.11.1
go.opentelemetry.io/proto/otlp => go.opentelemetry.io/proto/otlp v1.0.0
- go.starlark.net => go.starlark.net v0.0.0-20230725161458-0d7263928a74
+ go.starlark.net => go.starlark.net v0.0.0-20230726094710-7dadff395006
go.uber.org/atomic => go.uber.org/atomic v1.11.0
go.uber.org/automaxprocs => go.uber.org/automaxprocs v1.5.3
go.uber.org/goleak => go.uber.org/goleak v1.2.1
@@ -292,7 +292,7 @@ replace (
gocloud.dev => gocloud.dev v0.32.0
golang.org/x/crypto => golang.org/x/crypto v0.11.0
golang.org/x/exp => golang.org/x/exp v0.0.0-20230711153332-06a737ee72cb
- golang.org/x/exp/typeparams => golang.org/x/exp/typeparams v0.0.0-20230725093048-515e97ebf090
+ golang.org/x/exp/typeparams => golang.org/x/exp/typeparams v0.0.0-20230728194245-b0cb94b80691
golang.org/x/image => golang.org/x/image v0.9.0
golang.org/x/lint => golang.org/x/lint v0.0.0-20210508222113-6edffad5e616
golang.org/x/mobile => golang.org/x/mobile v0.0.0-20230531173138-3c911d8e3eda
@@ -310,12 +310,12 @@ replace (
gonum.org/v1/gonum => gonum.org/v1/gonum v0.13.0
gonum.org/v1/hdf5 => gonum.org/v1/hdf5 v0.0.0-20210714002203-8c5d23bc6946
gonum.org/v1/plot => gonum.org/v1/plot v0.13.0
- google.golang.org/api => google.golang.org/api v0.133.0
+ google.golang.org/api => google.golang.org/api v0.134.0
google.golang.org/appengine => google.golang.org/appengine v1.6.7
- google.golang.org/genproto => google.golang.org/genproto v0.0.0-20230725213213-b022f6e96895
- google.golang.org/genproto/googleapis/api => google.golang.org/genproto/googleapis/api v0.0.0-20230725213213-b022f6e96895
- google.golang.org/genproto/googleapis/rpc => google.golang.org/genproto/googleapis/rpc v0.0.0-20230725213213-b022f6e96895
- google.golang.org/grpc => google.golang.org/grpc v1.56.2
+ google.golang.org/genproto => google.golang.org/genproto v0.0.0-20230726155614-23370e0ffb3e
+ google.golang.org/genproto/googleapis/api => google.golang.org/genproto/googleapis/api v0.0.0-20230726155614-23370e0ffb3e
+ google.golang.org/genproto/googleapis/rpc => google.golang.org/genproto/googleapis/rpc v0.0.0-20230726155614-23370e0ffb3e
+ google.golang.org/grpc => google.golang.org/grpc v1.57.0
google.golang.org/grpc/cmd/protoc-gen-go-grpc => google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.3.0
google.golang.org/protobuf => google.golang.org/protobuf v1.31.0
gopkg.in/check.v1 => gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c
@@ -394,9 +394,9 @@ require (
golang.org/x/tools v0.10.0
gonum.org/v1/hdf5 v0.0.0-00010101000000-000000000000
gonum.org/v1/plot v0.10.1
- google.golang.org/genproto/googleapis/api v0.0.0-20230724170836-66ad5b6ff146
- google.golang.org/genproto/googleapis/rpc v0.0.0-20230724170836-66ad5b6ff146
- google.golang.org/grpc v1.56.2
+ google.golang.org/genproto/googleapis/api v0.0.0-20230726155614-23370e0ffb3e
+ google.golang.org/genproto/googleapis/rpc v0.0.0-20230726155614-23370e0ffb3e
+ google.golang.org/grpc v1.57.0
google.golang.org/protobuf v1.31.0
gopkg.in/yaml.v2 v2.4.0
k8s.io/api v0.27.3
diff --git a/go.sum b/go.sum
index 8629b356439..a7c77dc82c3 100644
--- a/go.sum
+++ b/go.sum
@@ -25,8 +25,8 @@ cloud.google.com/go/channel v1.16.0/go.mod h1:eN/q1PFSl5gyu0dYdmxNXscY/4Fi7ABmeH
cloud.google.com/go/cloudbuild v1.10.1/go.mod h1:lyJg7v97SUIPq4RC2sGsz/9tNczhyv2AjML/ci4ulzU=
cloud.google.com/go/clouddms v1.6.1/go.mod h1:Ygo1vL52Ov4TBZQquhz5fiw2CQ58gvu+PlS6PVXCpZI=
cloud.google.com/go/cloudtasks v1.11.1/go.mod h1:a9udmnou9KO2iulGscKR0qBYjreuX8oHwpmFsKspEvM=
-cloud.google.com/go/compute v1.22.0 h1:cB8R6FtUtT1TYGl5R3xuxnW6OUIc/DrT2aiR16TTG7Y=
-cloud.google.com/go/compute v1.22.0/go.mod h1:4tCnrn48xsqlwSAiLf1HXMQk8CONslYbdiEZc9FEIbM=
+cloud.google.com/go/compute v1.23.0 h1:tP41Zoavr8ptEqaW6j+LQOnyBBhO7OkOMAGrgLopTwY=
+cloud.google.com/go/compute v1.23.0/go.mod h1:4tCnrn48xsqlwSAiLf1HXMQk8CONslYbdiEZc9FEIbM=
cloud.google.com/go/compute/metadata v0.2.1/go.mod h1:jgHgmJd2RKBGzXqF5LR2EZMGxBkeanZ9wwa75XHJgOM=
cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY=
cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA=
@@ -42,7 +42,7 @@ cloud.google.com/go/datalabeling v0.8.1/go.mod h1:XS62LBSVPbYR54GfYQsPXZjTW8UxCK
cloud.google.com/go/dataplex v1.8.1/go.mod h1:7TyrDT6BCdI8/38Uvp0/ZxBslOslP2X2MPDucliyvSE=
cloud.google.com/go/dataproc v1.12.0/go.mod h1:zrF3aX0uV3ikkMz6z4uBbIKyhRITnxvr4i3IjKsKrw4=
cloud.google.com/go/dataqna v0.8.1/go.mod h1:zxZM0Bl6liMePWsHA8RMGAfmTG34vJMapbHAxQ5+WA8=
-cloud.google.com/go/datastore v1.12.1/go.mod h1:KjdB88W897MRITkvWWJrg2OUtrR5XVj1EoLgSp6/N70=
+cloud.google.com/go/datastore v1.13.0/go.mod h1:KjdB88W897MRITkvWWJrg2OUtrR5XVj1EoLgSp6/N70=
cloud.google.com/go/datastream v1.9.1/go.mod h1:hqnmr8kdUBmrnk65k5wNRoHSCYksvpdZIcZIEl8h43Q=
cloud.google.com/go/deploy v1.11.0/go.mod h1:tKuSUV5pXbn67KiubiUNUejqLs4f5cxxiCNCeyl0F2g=
cloud.google.com/go/dialogflow v1.38.0/go.mod h1:L7jnH+JL2mtmdChzAIcXQHXMvQkE3U4hTaNltEuxXn4=
@@ -174,26 +174,26 @@ github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kd
github.com/apache/arrow/go/v12 v12.0.0/go.mod h1:d+tV/eHZZ7Dz7RPrFKtPK02tpr+c9/PEd/zm8mDS9Vg=
github.com/apache/thrift v0.16.0/go.mod h1:PHK3hniurgQaNMZYaCLEqXKsYK8upmhPbmdP2FXSqgU=
github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=
-github.com/aws/aws-sdk-go v1.44.308 h1:XKu+76UHsD5LaiU2Zb1q42uWakw80Az7x39jJXXahos=
-github.com/aws/aws-sdk-go v1.44.308/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
-github.com/aws/aws-sdk-go-v2 v1.19.0 h1:klAT+y3pGFBU/qVf1uzwttpBbiuozJYWzNLHioyDJ+k=
+github.com/aws/aws-sdk-go v1.44.312 h1:llrElfzeqG/YOLFFKjg1xNpZCFJ2xraIi3PqSuP+95k=
+github.com/aws/aws-sdk-go v1.44.312/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
+github.com/aws/aws-sdk-go-v2 v1.19.1 h1:STs0lbbpXu3byTPcnRLghs2DH0yk9qKDo27TyyJSKsM=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10 h1:dK82zF6kkPeCo8J1e+tGx4JdvDIQzj7ygIoLg8WMuGs=
-github.com/aws/aws-sdk-go-v2/config v1.18.29 h1:yA+bSSRGhBwWuprG9I4VgxfK//NBLZ/0BGOHiV3f9oM=
-github.com/aws/aws-sdk-go-v2/credentials v1.13.28 h1:WM9tEHgoOh5ThJZ042UKnSx7TXGSC/bz63X3fsrQL2o=
-github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.5 h1:kP3Me6Fy3vdi+9uHd7YLr6ewPxRL+PU6y15urfTaamU=
-github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.73 h1:ez9K7LHdBxo+g1ExpUPsWEFfHxzHhiVZriWAJAiZwKc=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.35 h1:hMUCiE3Zi5AHrRNGf5j985u0WyqI6r2NULhUfo0N/No=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.29 h1:yOpYx+FTBdpk/g+sBU6Cb1H0U/TLEcYYp66mYqsPpcc=
-github.com/aws/aws-sdk-go-v2/internal/ini v1.3.36 h1:8r5m1BoAWkn0TDC34lUculryf7nUF25EgIMdjvGCkgo=
+github.com/aws/aws-sdk-go-v2/config v1.18.30 h1:TTAXQIn31qYFUQjkW6siVrRTX1ux+sADZDOe3jsZcMg=
+github.com/aws/aws-sdk-go-v2/credentials v1.13.29 h1:KNgCpThGuZyCjq9EuuqoLDenKKMwO/x1Xx01ckDa7VI=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.6 h1:kortK122LvTU34CGX/F9oJpelXKkEA2j/MW48II+8+8=
+github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.74 h1:5iIuHdeN3/x3kFBENHgYQl1ZtD+ZhLBXy6IgXflUtSI=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.36 h1:kbk81RlPoC6e4co7cQx2FAvH9TgbzxIqCqiosAFiB+w=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.30 h1:lMl8S5SB8jNCB+Sty2Em4lnu3IJytceHQd7qbmfqKL0=
+github.com/aws/aws-sdk-go-v2/internal/ini v1.3.37 h1:BXiqvN7WuV/pMhz8CivhO8cG8icJcjnjHumif4ukQ0c=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.27 h1:cZG7psLfqpkB6H+fIrgUDWmlzM474St1LP0jcz272yI=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.11 h1:y2+VQzC6Zh2ojtV2LoC0MNwHWc6qXv/j2vrQtlftkdA=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.30 h1:Bje8Xkh2OWpjBdNfXLrnn8eZg569dUQmhgtydxAYyP0=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.29 h1:IiDolu/eLmuB18DRZibj77n1hHQT7z12jnGO7Ze3pLc=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.14.4 h1:hx4WksB0NRQ9utR+2c3gEGzl6uKj3eM6PMQ6tN3lgXs=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.37.0 h1:PalLOEGZ/4XfQxpGZFTLaoJSmPoybnqJYotaIZEf/Rg=
-github.com/aws/aws-sdk-go-v2/service/sso v1.12.13 h1:sWDv7cMITPcZ21QdreULwxOOAmE05JjEsT6fCDtDA9k=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.31 h1:TGjmYwqqE6dMDSUSyQNct4MyTAgz95bPnDAjBOEgwOI=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.30 h1:UcVZxLVNY4yayCmiG94Ge3l2qbc5WEB/oa4RmjoQEi0=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.14.5 h1:B90htdoSv7OMH6QzzZ9cuZUoXVwFml0fTCDOpcGakCw=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.37.1 h1:OoFnDN7ZixctMX/Do4DgQXFvjtzQynz0p0ErQrOCeAs=
+github.com/aws/aws-sdk-go-v2/service/sso v1.12.14 h1:gUjz7trfz9qBm0AlkKTvJHBXELi1wvw+2LA9GfD2AsM=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.13 h1:BFubHS/xN5bjl818QaroN6mQdjneYQ+AOx44KNXlyH4=
-github.com/aws/aws-sdk-go-v2/service/sts v1.20.0 h1:jKmIOO+dFvCPuIhhM8u0Dy3dtd590n2kEDSYiGHoI98=
+github.com/aws/aws-sdk-go-v2/service/sts v1.20.1 h1:U7h9CPoyMfVoN5jUglB0LglCMP10AK4vMBsbsCKM8Yw=
github.com/aws/smithy-go v1.13.5 h1:hgz0X/DX0dGqTYpGALqXJoRKRj5oQ7150i5FdTePzO8=
github.com/benbjohnson/clock v1.3.5 h1:VvXlSJBzZpA/zum6Sj74hxwYI2DIxRWuNIoXAzHZz5o=
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
@@ -329,8 +329,8 @@ github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0=
github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/martian/v3 v3.3.2 h1:IqNFLAmvJOgVlpdEBiQbDc2EwKW77amAycfTuWKdfvw=
github.com/google/martian/v3 v3.3.2/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk=
-github.com/google/pprof v0.0.0-20230705174524-200ffdc848b8 h1:n6vlPhxsA+BW/XsS5+uqi7GyzaLa5MH7qlSLBZtRdiA=
-github.com/google/pprof v0.0.0-20230705174524-200ffdc848b8/go.mod h1:Jh3hGz2jkYak8qXPD19ryItVnUgpgeqzdkY/D0EaeuA=
+github.com/google/pprof v0.0.0-20230728192033-2ba5b33183c6 h1:ZgoomqkdjGbQ3+qQXCkvYMCDvGDNg2k5JJDjjdTB6jY=
+github.com/google/pprof v0.0.0-20230728192033-2ba5b33183c6/go.mod h1:Jh3hGz2jkYak8qXPD19ryItVnUgpgeqzdkY/D0EaeuA=
github.com/google/s2a-go v0.1.0/go.mod h1:OJpEgntRZo8ugHpF9hkoLJbS5dSI20XZeXJ9JVywLlM=
github.com/google/s2a-go v0.1.4 h1:1kZ/sQM3srePvKs3tXAvQzo66XfcReoqFpIpIccE7Oc=
github.com/google/s2a-go v0.1.4/go.mod h1:Ej+mSEMGRnqRzjc7VtF+jdBwYG5fuJfiZ8ELkjEwM0A=
@@ -358,8 +358,8 @@ github.com/goxjs/gl v0.0.0-20210104184919-e3fafc6f8f2a/go.mod h1:dy/f2gjY09hwVfI
github.com/goxjs/glfw v0.0.0-20191126052801-d2efb5f20838/go.mod h1:oS8P8gVOT4ywTcjV6wZlOU4GuVFQ8F5328KY3MJ79CY=
github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 h1:+ngKgrYPPJrOjhax5N+uePQ0Fh1Z7PheYoUI/0nzkPA=
github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
-github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.1 h1:RoziI+96HlQWrbaVhgOOdFYUHtX81pwA6tCgDS9FNRo=
-github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.1/go.mod h1:Rj8lEaVgLiPn1jTMVXEhATiZhuyXJq167bMYPbJM1CY=
+github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.2 h1:dygLcbEBA+t/P7ck6a8AkXv6juQ4cK0RHBoh32jxhHM=
+github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.2/go.mod h1:Ap9RLCIJVtgQg1/BBgVEfypOAySvvlcpcVQkSzJCH4Y=
github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek=
github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
github.com/huandu/xstrings v1.4.0 h1:D17IlohoQq4UcpqD7fDk80P7l+lwAmlFaBHgOipl2FU=
@@ -391,8 +391,8 @@ github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:C
github.com/kisielk/errcheck v1.6.3/go.mod h1:nXw/i/MfnvRHqXa7XXmQMUB0oNFGuBrNI8d8NLy0LPw=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/klauspost/asmfmt v1.3.2/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j0HLHbNSE=
-github.com/klauspost/compress v1.16.8-0.20230717094025-33c9bc800133 h1:EHtgNm0KvRXG0aBa0+5ur6mGJXAsxVF+MLu2/YSTVXQ=
-github.com/klauspost/compress v1.16.8-0.20230717094025-33c9bc800133/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
+github.com/klauspost/compress v1.16.8-0.20230727132629-c1dcc38258f0 h1:8hLbH6j6nbONm75mRwDw2zTBp+DqEW3xGG91XiFR+aY=
+github.com/klauspost/compress v1.16.8-0.20230727132629-c1dcc38258f0/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/qbZg=
github.com/klauspost/cpuid/v2 v2.2.5/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=
github.com/kpango/fastime v1.1.9 h1:xVQHcqyPt5M69DyFH7g1EPRns1YQNap9d5eLhl/Jy84=
@@ -474,8 +474,8 @@ github.com/prometheus/client_model v0.4.0 h1:5lQXD3cAg1OXBf4Wq03gTrXHeaV0TQvGfUo
github.com/prometheus/client_model v0.4.0/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU=
github.com/prometheus/common v0.44.0 h1:+5BrQJwiBB9xsMygAB3TNvpQKOwlkc25LbISbrdOOfY=
github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO7x0VV9VvuY=
-github.com/prometheus/procfs v0.11.0 h1:5EAgkfkMl659uZPbe9AS2N68a7Cc1TJbPEuGzFuRbyk=
-github.com/prometheus/procfs v0.11.0/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPHWJq+XBB/FM=
+github.com/prometheus/procfs v0.11.1 h1:xRC8Iq1yyca5ypa9n1EZnWZkt7dwcoRPQwX/5gwaUuI=
+github.com/prometheus/procfs v0.11.1/go.mod h1:eesXgaPo1q7lBpVMoMy0ZOFTth9hBn4W/y0/p/ScXhY=
github.com/quasilyte/go-ruleguard v0.3.19 h1:tfMnabXle/HzOb5Xe9CUZYWXKfkS1KwRmZyPmD9nVcc=
github.com/quasilyte/go-ruleguard v0.3.19/go.mod h1:lHSn69Scl48I7Gt9cX3VrbsZYvYiBYszZOZW4A+oTEw=
github.com/quasilyte/go-ruleguard/dsl v0.3.22 h1:wd8zkOhSNr+I+8Qeciml08ivDt1pSXe60+5DqOpCjPE=
@@ -555,8 +555,8 @@ go.opentelemetry.io/otel/trace v1.11.1 h1:ofxdnzsNrGBYXbP7t7zpUK281+go5rF7dvdIZX
go.opentelemetry.io/otel/trace v1.11.1/go.mod h1:f/Q9G7vzk5u91PhbmKbg1Qn0rzH1LJ4vbPHFGkTPtOk=
go.opentelemetry.io/proto/otlp v1.0.0 h1:T0TX0tmXU8a3CbNXzEKGeU5mIVOdf0oykP+u2lIVU/I=
go.opentelemetry.io/proto/otlp v1.0.0/go.mod h1:Sy6pihPLfYHkr3NkUbEhGHFhINUSI/v80hjKIs5JXpM=
-go.starlark.net v0.0.0-20230725161458-0d7263928a74 h1:EL8MuNFlzO8vvpHgZxDGPaehP0ozoJ1j1zA768zKXUQ=
-go.starlark.net v0.0.0-20230725161458-0d7263928a74/go.mod h1:jxU+3+j+71eXOW14274+SmmuW82qJzl6iZSeqEtTGds=
+go.starlark.net v0.0.0-20230726094710-7dadff395006 h1:hGOLDtPfO4cy1P66VUXt++umv2lWLXtt+qgc2uxr1Ss=
+go.starlark.net v0.0.0-20230726094710-7dadff395006/go.mod h1:jxU+3+j+71eXOW14274+SmmuW82qJzl6iZSeqEtTGds=
go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE=
go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0=
go.uber.org/automaxprocs v1.5.3 h1:kWazyxZUrS3Gs4qUpbwo5kEIMGe/DAvi5Z4tl2NW4j8=
@@ -575,8 +575,8 @@ golang.org/x/exp v0.0.0-20230711153332-06a737ee72cb h1:xIApU0ow1zwMa2uL1VDNeQlNV
golang.org/x/exp v0.0.0-20230711153332-06a737ee72cb/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc=
golang.org/x/exp/shiny v0.0.0-20220827204233-334a2380cb91/go.mod h1:VjAR7z0ngyATZTELrBSkxOOHhhlnVUxDye4mcjx5h/8=
golang.org/x/exp/shiny v0.0.0-20230425010034-47ecfdc1ba53/go.mod h1:UH99kUObWAZkDnWqppdQe5ZhPYESUw8I0zVV1uWBR+0=
-golang.org/x/exp/typeparams v0.0.0-20230725093048-515e97ebf090 h1:qOYhjyK9OeXREdh7Zrta8JRvnmnFIzhkosQpp+852Ag=
-golang.org/x/exp/typeparams v0.0.0-20230725093048-515e97ebf090/go.mod h1:AbB0pIl9nAr9wVwH+Z2ZpaocVmF5I4GyWCDIsVjR0bk=
+golang.org/x/exp/typeparams v0.0.0-20230728194245-b0cb94b80691 h1:b3L/AEbLJCHPoRzf5gQA/tL4O/XY2glLjzmtoZ+raXs=
+golang.org/x/exp/typeparams v0.0.0-20230728194245-b0cb94b80691/go.mod h1:AbB0pIl9nAr9wVwH+Z2ZpaocVmF5I4GyWCDIsVjR0bk=
golang.org/x/image v0.9.0 h1:QrzfX26snvCM20hIhBwuHI/ThTg18b/+kcKdXHvnR+g=
golang.org/x/image v0.9.0/go.mod h1:jtrku+n79PfroUbvDdeUWMAI+heR786BofxrbiSF+J0=
golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
@@ -609,19 +609,19 @@ gonum.org/v1/hdf5 v0.0.0-20210714002203-8c5d23bc6946 h1:vJpL69PeUullhJyKtTjHjENE
gonum.org/v1/hdf5 v0.0.0-20210714002203-8c5d23bc6946/go.mod h1:BQUWDHIAygjdt1HnUPQ0eWqLN2n5FwJycrpYUVUOx2I=
gonum.org/v1/plot v0.13.0 h1:yb2Z/b8bY5h/xC4uix+ujJ+ixvPUvBmUOtM73CJzpsw=
gonum.org/v1/plot v0.13.0/go.mod h1:mV4Bpu4PWTgN2CETURNF8hCMg7EtlZqJYCcmYo/t4Co=
-google.golang.org/api v0.133.0 h1:N7Ym5Hl0Dpn0I0o7R1z4UpVA1GCDyS8vbPu1/ObV73A=
-google.golang.org/api v0.133.0/go.mod h1:sjRL3UnjTx5UqNQS9EWr9N8p7xbHpy1k0XGRLCf3Spk=
+google.golang.org/api v0.134.0 h1:ktL4Goua+UBgoP1eL1/60LwZJqa1sIzkLmvoR3hR6Gw=
+google.golang.org/api v0.134.0/go.mod h1:sjRL3UnjTx5UqNQS9EWr9N8p7xbHpy1k0XGRLCf3Spk=
google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c=
google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
-google.golang.org/genproto v0.0.0-20230725213213-b022f6e96895 h1:f4HtRHVw5oEuUSMwhzcRW+w4X9++1iU+MZ9cRAHbWxk=
-google.golang.org/genproto v0.0.0-20230725213213-b022f6e96895/go.mod h1:0ggbjUrZYpy1q+ANUS30SEoGZ53cdfwtbuG7Ptgy108=
-google.golang.org/genproto/googleapis/api v0.0.0-20230725213213-b022f6e96895 h1:9rcwSXpqHEULy96NKetvTJMCLnvnod0LcF8A/ULEBxE=
-google.golang.org/genproto/googleapis/api v0.0.0-20230725213213-b022f6e96895/go.mod h1:rsr7RhLuwsDKL7RmgDDCUc6yaGr1iqceVb5Wv6f6YvQ=
+google.golang.org/genproto v0.0.0-20230726155614-23370e0ffb3e h1:xIXmWJ303kJCuogpj0bHq+dcjcZHU+XFyc1I0Yl9cRg=
+google.golang.org/genproto v0.0.0-20230726155614-23370e0ffb3e/go.mod h1:0ggbjUrZYpy1q+ANUS30SEoGZ53cdfwtbuG7Ptgy108=
+google.golang.org/genproto/googleapis/api v0.0.0-20230726155614-23370e0ffb3e h1:z3vDksarJxsAKM5dmEGv0GHwE2hKJ096wZra71Vs4sw=
+google.golang.org/genproto/googleapis/api v0.0.0-20230726155614-23370e0ffb3e/go.mod h1:rsr7RhLuwsDKL7RmgDDCUc6yaGr1iqceVb5Wv6f6YvQ=
google.golang.org/genproto/googleapis/bytestream v0.0.0-20230720185612-659f7aaaa771/go.mod h1:3QoBVwTHkXbY1oRGzlhwhOykfcATQN43LJ6iT8Wy8kE=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20230725213213-b022f6e96895 h1:co8AMhI481nhd3WBfW2mq5msyQHNBcGn7G9GCEqz45k=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20230725213213-b022f6e96895/go.mod h1:TUfxEVdsvPg18p6AslUXFoLdpED4oBnGwyqk3dV1XzM=
-google.golang.org/grpc v1.56.2 h1:fVRFRnXvU+x6C4IlHZewvJOVHoOv1TUuQyoRsYnB4bI=
-google.golang.org/grpc v1.56.2/go.mod h1:I9bI3vqKfayGqPUAwGdOSu7kt6oIJLixfffKrpXqQ9s=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20230726155614-23370e0ffb3e h1:S83+ibolgyZ0bqz7KEsUOPErxcv4VzlszxY+31OfB/E=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20230726155614-23370e0ffb3e/go.mod h1:TUfxEVdsvPg18p6AslUXFoLdpED4oBnGwyqk3dV1XzM=
+google.golang.org/grpc v1.57.0 h1:kfzNeI/klCGD2YPMUlaGNT3pxvYfga7smW3Vth8Zsiw=
+google.golang.org/grpc v1.57.0/go.mod h1:Sd+9RMTACXwmub0zcNY2c4arhtrbBYD1AUHI/dt16Mo=
google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.3.0/go.mod h1:Dk1tviKTvMCz5tvh7t+fh94dhmQVHuCt2OzJB3CTW9Y=
google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8=
google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
diff --git a/hack/benchmark/internal/core/algorithm/ngt/ngt_test.go b/hack/benchmark/internal/core/algorithm/ngt/ngt_test.go
index e146bcb0b43..793622b8b39 100644
--- a/hack/benchmark/internal/core/algorithm/ngt/ngt_test.go
+++ b/hack/benchmark/internal/core/algorithm/ngt/ngt_test.go
@@ -111,6 +111,7 @@ package ngt
//
// func Test_core_Search(t *testing.T) {
// type args struct {
+// ctx context.Context
// vec []float32
// size int
// epsilon float32
@@ -151,6 +152,7 @@ package ngt
// {
// name: "test_case_1",
// args: args {
+// ctx:nil,
// vec:nil,
// size:0,
// epsilon:0,
@@ -180,6 +182,7 @@ package ngt
// return test {
// name: "test_case_2",
// args: args {
+// ctx:nil,
// vec:nil,
// size:0,
// epsilon:0,
@@ -228,7 +231,7 @@ package ngt
// NGT: test.fields.NGT,
// }
//
-// got, err := c.Search(test.args.vec, test.args.size, test.args.epsilon, test.args.radius)
+// got, err := c.Search(test.args.ctx, test.args.vec, test.args.size, test.args.epsilon, test.args.radius)
// if err := checkFunc(test.want, got, err); err != nil {
// tt.Errorf("error = %v", err)
// }
diff --git a/hack/benchmark/internal/operation/insert.go b/hack/benchmark/internal/operation/insert.go
index 3428b5ab734..6ecdbefe3fc 100644
--- a/hack/benchmark/internal/operation/insert.go
+++ b/hack/benchmark/internal/operation/insert.go
@@ -16,7 +16,6 @@ package operation
import (
"context"
"strconv"
- "sync"
"sync/atomic"
"testing"
@@ -25,6 +24,7 @@ import (
"github.com/vdaas/vald/internal/io"
"github.com/vdaas/vald/internal/net/grpc/codes"
"github.com/vdaas/vald/internal/net/grpc/status"
+ "github.com/vdaas/vald/internal/sync"
)
func (o *operation) Insert(ctx context.Context, b *testing.B, ds assets.Dataset) (insertedNum int) {
diff --git a/hack/benchmark/internal/operation/remove.go b/hack/benchmark/internal/operation/remove.go
index 7c31ac8a1dc..8789e861ed9 100644
--- a/hack/benchmark/internal/operation/remove.go
+++ b/hack/benchmark/internal/operation/remove.go
@@ -16,11 +16,11 @@ package operation
import (
"context"
"strconv"
- "sync"
"testing"
"github.com/vdaas/vald/apis/grpc/v1/payload"
"github.com/vdaas/vald/internal/io"
+ "github.com/vdaas/vald/internal/sync"
)
func (o *operation) Remove(ctx context.Context, b *testing.B, maxIdNum int) {
diff --git a/hack/benchmark/internal/operation/search.go b/hack/benchmark/internal/operation/search.go
index e37b8a286b2..ee2178c1aad 100644
--- a/hack/benchmark/internal/operation/search.go
+++ b/hack/benchmark/internal/operation/search.go
@@ -16,12 +16,12 @@ package operation
import (
"context"
"strconv"
- "sync"
"testing"
"github.com/vdaas/vald/apis/grpc/v1/payload"
"github.com/vdaas/vald/hack/benchmark/internal/assets"
"github.com/vdaas/vald/internal/io"
+ "github.com/vdaas/vald/internal/sync"
)
func (o *operation) Search(ctx context.Context, b *testing.B, ds assets.Dataset) {
diff --git a/hack/benchmark/internal/starter/agent/core/ngt/ngt.go b/hack/benchmark/internal/starter/agent/core/ngt/ngt.go
index d4fec6c59ae..2dced31d27f 100644
--- a/hack/benchmark/internal/starter/agent/core/ngt/ngt.go
+++ b/hack/benchmark/internal/starter/agent/core/ngt/ngt.go
@@ -19,7 +19,6 @@ package ngt
import (
"context"
- "sync"
"testing"
"time"
@@ -27,6 +26,7 @@ import (
"github.com/vdaas/vald/internal/client/v1/client/vald"
"github.com/vdaas/vald/internal/info"
"github.com/vdaas/vald/internal/runner"
+ "github.com/vdaas/vald/internal/sync"
"github.com/vdaas/vald/pkg/agent/core/ngt/config"
"github.com/vdaas/vald/pkg/agent/core/ngt/usecase"
)
diff --git a/hack/benchmark/src/singleflight/singleflight_bench_test.go b/hack/benchmark/src/singleflight/singleflight_bench_test.go
index 4b183fc6a06..481d39e43de 100644
--- a/hack/benchmark/src/singleflight/singleflight_bench_test.go
+++ b/hack/benchmark/src/singleflight/singleflight_bench_test.go
@@ -19,13 +19,13 @@ import (
"io/fs"
"math"
"os"
- "sync"
"sync/atomic"
"testing"
"time"
"github.com/vdaas/vald/internal/errors"
- "github.com/vdaas/vald/internal/singleflight"
+ "github.com/vdaas/vald/internal/sync"
+ "github.com/vdaas/vald/internal/sync/singleflight"
stdsingleflight "golang.org/x/sync/singleflight"
)
@@ -36,7 +36,7 @@ type Result struct {
}
type helper struct {
- initDoFn func() func(ctx context.Context, key string, fn func() (string, error))
+ initDoFn func() func(ctx context.Context, key string, fn func(context.Context) (string, error))
sleepDur time.Duration
calledCnt int64
totalCnt int64
@@ -67,7 +67,7 @@ var durs = []time.Duration{
func (h *helper) Do(parallel int, b *testing.B) {
b.Helper()
- fn := func() (string, error) {
+ fn := func(context.Context) (string, error) {
atomic.AddInt64(&h.calledCnt, 1)
time.Sleep(h.sleepDur)
return "", nil
@@ -115,10 +115,10 @@ func Benchmark_group_Do_with_sync_singleflight(b *testing.B) {
results := make([]Result, 0, tryCnt)
for j := 0; j < tryCnt; j++ {
h := &helper{
- initDoFn: func() func(ctx context.Context, key string, fn func() (string, error)) {
+ initDoFn: func() func(ctx context.Context, key string, fn func(context.Context) (string, error)) {
g := new(stdsingleflight.Group)
- return func(ctx context.Context, key string, fn func() (string, error)) {
- g.Do(key, func() (interface{}, error) { return fn() })
+ return func(ctx context.Context, key string, fn func(context.Context) (string, error)) {
+ g.Do(key, func() (interface{}, error) { return fn(context.Background()) })
}
},
sleepDur: dur,
@@ -210,9 +210,9 @@ func Benchmark_group_Do_with_vald_internal_singleflight(b *testing.B) {
results := make([]Result, 0, tryCnt)
for j := 0; j < tryCnt; j++ {
h := &helper{
- initDoFn: func() func(ctx context.Context, key string, fn func() (string, error)) {
+ initDoFn: func() func(ctx context.Context, key string, fn func(context.Context) (string, error)) {
g := singleflight.New[string]()
- return func(ctx context.Context, key string, fn func() (string, error)) {
+ return func(ctx context.Context, key string, fn func(context.Context) (string, error)) {
g.Do(ctx, key, fn)
}
},
diff --git a/hack/benchmark/src/singleflight/sync_singleflight_average.csv b/hack/benchmark/src/singleflight/sync_singleflight_average.csv
new file mode 100644
index 00000000000..18ea1cf68f8
--- /dev/null
+++ b/hack/benchmark/src/singleflight/sync_singleflight_average.csv
@@ -0,0 +1,64 @@
+#
+# Copyright (C) 2019-2023 vdaas.org vald team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+goroutine,duration,hit_rate
+10,10000,0.999841
+10,100000,0.999979
+10,200000,0.999986
+10,500000,0.999982
+10,1000000,0.999987
+10,5000000,0.999988
+10,10000000,0.999990
+10,25000000,0.999991
+10,50000000,0.999994
+10,100000000,0.999996
+10,250000000,0.999998
+10,500000000,0.999999
+100,10000,0.999930
+100,100000,0.999988
+100,200000,0.999986
+100,500000,0.999984
+100,1000000,0.999972
+100,5000000,0.999986
+100,10000000,0.999990
+100,25000000,0.999988
+100,50000000,0.999994
+100,100000000,0.999995
+100,250000000,0.999999
+100,500000000,0.999999
+1000,10000,0.999787
+1000,100000,0.999975
+1000,200000,0.999979
+1000,500000,0.999981
+1000,1000000,0.999971
+1000,5000000,0.999984
+1000,10000000,0.999981
+1000,25000000,0.999991
+1000,50000000,0.999994
+1000,100000000,0.999995
+1000,250000000,0.999998
+1000,500000000,0.999999
+10000,10000,0.999170
+10000,100000,0.999859
+10000,200000,0.999905
+10000,500000,0.999913
+10000,1000000,0.999924
+10000,5000000,0.999940
+10000,10000000,0.999958
+10000,25000000,0.999977
+10000,50000000,0.999990
+10000,100000000,0.999996
+10000,250000000,1.000000
+10000,500000000,1.000000
diff --git a/hack/benchmark/src/singleflight/sync_singleflight_variance.csv b/hack/benchmark/src/singleflight/sync_singleflight_variance.csv
new file mode 100644
index 00000000000..d8ac9ae55ed
--- /dev/null
+++ b/hack/benchmark/src/singleflight/sync_singleflight_variance.csv
@@ -0,0 +1,64 @@
+#
+# Copyright (C) 2019-2023 vdaas.org vald team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+goroutine,duration,hit_rate
+10,10000,0.000000
+10,100000,0.000000
+10,200000,0.000000
+10,500000,0.000000
+10,1000000,0.000000
+10,5000000,0.000000
+10,10000000,0.000000
+10,25000000,0.000000
+10,50000000,0.000000
+10,100000000,0.000000
+10,250000000,0.000000
+10,500000000,0.000000
+100,10000,0.000000
+100,100000,0.000000
+100,200000,0.000000
+100,500000,0.000000
+100,1000000,0.000000
+100,5000000,0.000000
+100,10000000,0.000000
+100,25000000,0.000000
+100,50000000,0.000000
+100,100000000,0.000000
+100,250000000,0.000000
+100,500000000,0.000000
+1000,10000,0.000000
+1000,100000,0.000000
+1000,200000,0.000000
+1000,500000,0.000000
+1000,1000000,0.000000
+1000,5000000,0.000000
+1000,10000000,0.000000
+1000,25000000,0.000000
+1000,50000000,0.000000
+1000,100000000,0.000000
+1000,250000000,0.000000
+1000,500000000,0.000000
+10000,10000,0.000000
+10000,100000,0.000000
+10000,200000,0.000000
+10000,500000,0.000000
+10000,1000000,0.000000
+10000,5000000,0.000000
+10000,10000000,0.000000
+10000,25000000,0.000000
+10000,50000000,0.000000
+10000,100000000,0.000000
+10000,250000000,0.000000
+10000,500000000,0.000000
diff --git a/hack/benchmark/src/singleflight/vald_internal_singlefligh_average.csv b/hack/benchmark/src/singleflight/vald_internal_singlefligh_average.csv
new file mode 100644
index 00000000000..1d6e08df44c
--- /dev/null
+++ b/hack/benchmark/src/singleflight/vald_internal_singlefligh_average.csv
@@ -0,0 +1,64 @@
+#
+# Copyright (C) 2019-2023 vdaas.org vald team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+goroutine,duration,hit_rate
+10,10000,0.788773
+10,100000,0.843351
+10,200000,0.865148
+10,500000,0.831613
+10,1000000,0.780246
+10,5000000,0.787488
+10,10000000,0.809159
+10,25000000,0.904879
+10,50000000,0.995670
+10,100000000,0.999927
+10,250000000,0.999994
+10,500000000,0.999999
+100,10000,0.751182
+100,100000,0.819617
+100,200000,0.762678
+100,500000,0.770103
+100,1000000,0.769985
+100,5000000,0.789949
+100,10000000,0.819131
+100,25000000,0.910536
+100,50000000,0.993586
+100,100000000,0.999953
+100,250000000,0.999995
+100,500000000,0.999999
+1000,10000,0.766264
+1000,100000,0.778658
+1000,200000,0.762406
+1000,500000,0.796780
+1000,1000000,0.766803
+1000,5000000,0.792517
+1000,10000000,0.828376
+1000,25000000,0.913363
+1000,50000000,0.993471
+1000,100000000,0.999946
+1000,250000000,0.999994
+1000,500000000,0.999999
+10000,10000,0.823414
+10000,100000,0.854507
+10000,200000,0.865419
+10000,500000,0.885624
+10000,1000000,0.905540
+10000,5000000,0.982323
+10000,10000000,0.995278
+10000,25000000,0.999844
+10000,50000000,0.999966
+10000,100000000,0.999991
+10000,250000000,1.000000
+10000,500000000,1.000000
diff --git a/hack/benchmark/src/singleflight/vald_internal_singlefligh_variance.csv b/hack/benchmark/src/singleflight/vald_internal_singlefligh_variance.csv
new file mode 100644
index 00000000000..60328bdf256
--- /dev/null
+++ b/hack/benchmark/src/singleflight/vald_internal_singlefligh_variance.csv
@@ -0,0 +1,64 @@
+#
+# Copyright (C) 2019-2023 vdaas.org vald team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+goroutine,duration,hit_rate
+10,10000,0.002508
+10,100000,0.011504
+10,200000,0.004989
+10,500000,0.003654
+10,1000000,0.000584
+10,5000000,0.000001
+10,10000000,0.000018
+10,25000000,0.000186
+10,50000000,0.000001
+10,100000000,0.000000
+10,250000000,0.000000
+10,500000000,0.000000
+100,10000,0.000219
+100,100000,0.007903
+100,200000,0.000032
+100,500000,0.000099
+100,1000000,0.000002
+100,5000000,0.000037
+100,10000000,0.000059
+100,25000000,0.000026
+100,50000000,0.000001
+100,100000000,0.000000
+100,250000000,0.000000
+100,500000000,0.000000
+1000,10000,0.001102
+1000,100000,0.000719
+1000,200000,0.000081
+1000,500000,0.001153
+1000,1000000,0.000045
+1000,5000000,0.000023
+1000,10000000,0.000013
+1000,25000000,0.000178
+1000,50000000,0.000009
+1000,100000000,0.000000
+1000,250000000,0.000000
+1000,500000000,0.000000
+10000,10000,0.000581
+10000,100000,0.001038
+10000,200000,0.001101
+10000,500000,0.000539
+10000,1000000,0.000312
+10000,5000000,0.000027
+10000,10000000,0.000002
+10000,25000000,0.000000
+10000,50000000,0.000000
+10000,100000000,0.000000
+10000,250000000,0.000000
+10000,500000000,0.000000
diff --git a/hack/license/gen/main.go b/hack/license/gen/main.go
index 6a3d5ce9000..a7fbcc72df0 100644
--- a/hack/license/gen/main.go
+++ b/hack/license/gen/main.go
@@ -23,7 +23,6 @@ import (
"io/fs"
"os"
"path/filepath"
- "sync"
"text/template"
"time"
@@ -31,6 +30,7 @@ import (
"github.com/vdaas/vald/internal/file"
"github.com/vdaas/vald/internal/log"
"github.com/vdaas/vald/internal/strings"
+ "github.com/vdaas/vald/internal/sync"
)
var (
@@ -50,6 +50,38 @@ var (
{{.Escape}} limitations under the License.
{{.Escape}}
`))
+ goStandard = template.Must(template.New("Go License").Parse(`{{.Escape}}
+{{.Escape}} Copyright (c) 2009-{{.Year}} The Go Authors. All rights resered.
+{{.Escape}} Modified by {{.Maintainer}}
+{{.Escape}}
+{{.Escape}} Redistribution and use in source and binary forms, with or without
+{{.Escape}} modification, are permitted provided that the following conditions are
+{{.Escape}} met:
+{{.Escape}}
+{{.Escape}} * Redistributions of source code must retain the above copyright
+{{.Escape}} notice, this list of conditions and the following disclaimer.
+{{.Escape}} * Redistributions in binary form must reproduce the above
+{{.Escape}} copyright notice, this list of conditions and the following disclaimer
+{{.Escape}} in the documentation and/or other materials provided with the
+{{.Escape}} distribution.
+{{.Escape}} * Neither the name of Google Inc. nor the names of its
+{{.Escape}} contributors may be used to endorse or promote products derived from
+{{.Escape}} this software without specific prior written permission.
+{{.Escape}}
+{{.Escape}} THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+{{.Escape}} "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+{{.Escape}} LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+{{.Escape}} A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+{{.Escape}} OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+{{.Escape}} SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+{{.Escape}} LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+{{.Escape}} DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+{{.Escape}} THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+{{.Escape}} (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+{{.Escape}} OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+{{.Escape}}
+`))
+
slushEscape = "//"
sharpEscape = "#"
)
@@ -189,8 +221,21 @@ func readAndRewrite(path string) error {
log.Fatal(err)
}
} else {
+ tmpl := apache
switch filepath.Ext(path) {
- case ".go", ".proto":
+ case ".go":
+ d.Escape = slushEscape
+ switch fi.Name() {
+ case "errgroup_test.go",
+ "singleflight.go",
+ "semaphore.go",
+ "semaphore_bench_test.go",
+ "semaphore_example_test.go",
+ "semaphore_test.go":
+ tmpl = goStandard
+ default:
+ }
+ case ".proto":
d.Escape = slushEscape
}
lf := true
@@ -236,7 +281,7 @@ func readAndRewrite(path string) error {
continue
} else if !bf {
once.Do(func() {
- err = apache.Execute(buf, d)
+ err = tmpl.Execute(buf, d)
if err != nil {
// skipcq: RVV-A0003
log.Fatal(err)
diff --git a/internal/backoff/backoff.go b/internal/backoff/backoff.go
index e91cf606316..1b99fbdba4d 100644
--- a/internal/backoff/backoff.go
+++ b/internal/backoff/backoff.go
@@ -21,7 +21,6 @@ import (
"context"
"math"
"strconv"
- "sync"
"time"
"github.com/vdaas/vald/internal/errors"
@@ -30,6 +29,7 @@ import (
"github.com/vdaas/vald/internal/observability/trace"
"github.com/vdaas/vald/internal/rand"
"github.com/vdaas/vald/internal/strings"
+ "github.com/vdaas/vald/internal/sync"
)
// NOTE: This variable is for observability package.
diff --git a/internal/cache/gache/option_test.go b/internal/cache/gache/option_test.go
index 3baba7374c0..bc00aac1e9e 100644
--- a/internal/cache/gache/option_test.go
+++ b/internal/cache/gache/option_test.go
@@ -393,16 +393,16 @@ func TestWithExpireCheckDuration(t *testing.T) {
//
// func Test_defaultOptions(t *testing.T) {
// type want struct {
-// want []Option
+// want []Option[V]
// }
// type test struct {
// name string
// want want
-// checkFunc func(want, []Option) error
+// checkFunc func(want, []Option[V]) error
// beforeFunc func(*testing.T)
// afterFunc func(*testing.T)
// }
-// defaultCheckFunc := func(w want, got []Option) error {
+// defaultCheckFunc := func(w want, got []Option[V]) error {
// if !reflect.DeepEqual(got, w.want) {
// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", got, w.want)
// }
diff --git a/internal/cache/option_test.go b/internal/cache/option_test.go
index f41f6ccd159..a2eee8e05d7 100644
--- a/internal/cache/option_test.go
+++ b/internal/cache/option_test.go
@@ -342,3 +342,79 @@ func TestWithExpireCheckDuration(t *testing.T) {
}
// NOT IMPLEMENTED BELOW
+//
+// func Test_defaultOptions(t *testing.T) {
+// type want struct {
+// want []Option[V]
+// }
+// type test struct {
+// name string
+// want want
+// checkFunc func(want, []Option[V]) error
+// beforeFunc func(*testing.T)
+// afterFunc func(*testing.T)
+// }
+// defaultCheckFunc := func(w want, got []Option[V]) error {
+// if !reflect.DeepEqual(got, w.want) {
+// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", got, w.want)
+// }
+// return nil
+// }
+// tests := []test{
+// // TODO test cases
+// /*
+// {
+// name: "test_case_1",
+// want: want{},
+// checkFunc: defaultCheckFunc,
+// beforeFunc: func(t *testing.T,) {
+// t.Helper()
+// },
+// afterFunc: func(t *testing.T,) {
+// t.Helper()
+// },
+// },
+// */
+//
+// // TODO test cases
+// /*
+// func() test {
+// return test {
+// name: "test_case_2",
+// want: want{},
+// checkFunc: defaultCheckFunc,
+// beforeFunc: func(t *testing.T,) {
+// t.Helper()
+// },
+// afterFunc: func(t *testing.T,) {
+// t.Helper()
+// },
+// }
+// }(),
+// */
+// }
+//
+// for _, tc := range tests {
+// test := tc
+// t.Run(test.name, func(tt *testing.T) {
+// tt.Parallel()
+// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent())
+// if test.beforeFunc != nil {
+// test.beforeFunc(tt)
+// }
+// if test.afterFunc != nil {
+// defer test.afterFunc(tt)
+// }
+// checkFunc := test.checkFunc
+// if test.checkFunc == nil {
+// checkFunc = defaultCheckFunc
+// }
+//
+// got := defaultOptions()
+// if err := checkFunc(test.want, got); err != nil {
+// tt.Errorf("error = %v", err)
+// }
+//
+// })
+// }
+// }
diff --git a/internal/circuitbreaker/manager.go b/internal/circuitbreaker/manager.go
index 0cae88a809c..64338b2a4f6 100644
--- a/internal/circuitbreaker/manager.go
+++ b/internal/circuitbreaker/manager.go
@@ -16,11 +16,10 @@ package circuitbreaker
import (
"context"
"reflect"
- "sync"
"github.com/vdaas/vald/internal/errors"
"github.com/vdaas/vald/internal/log"
- valdsync "github.com/vdaas/vald/internal/sync"
+ "github.com/vdaas/vald/internal/sync"
)
// NOTE: This variable is for observability package.
@@ -37,7 +36,7 @@ type CircuitBreaker interface {
}
type breakerManager struct {
- m valdsync.Map[string, *breaker]
+ m sync.Map[string, *breaker]
opts []BreakerOption
}
diff --git a/internal/circuitbreaker/manager_test.go b/internal/circuitbreaker/manager_test.go
index c082552ec56..7c007204b62 100644
--- a/internal/circuitbreaker/manager_test.go
+++ b/internal/circuitbreaker/manager_test.go
@@ -112,6 +112,7 @@ package circuitbreaker
// fn func(ctx context.Context) (interface{}, error)
// }
// type fields struct {
+// m sync.Map[string, *breaker]
// opts []BreakerOption
// }
// type want struct {
@@ -147,6 +148,7 @@ package circuitbreaker
// fn:nil,
// },
// fields: fields {
+// m:nil,
// opts:nil,
// },
// want: want{},
@@ -171,6 +173,7 @@ package circuitbreaker
// fn:nil,
// },
// fields: fields {
+// m:nil,
// opts:nil,
// },
// want: want{},
@@ -202,6 +205,7 @@ package circuitbreaker
// checkFunc = defaultCheckFunc
// }
// bm := &breakerManager{
+// m: test.fields.m,
// opts: test.fields.opts,
// }
//
diff --git a/internal/client/v1/client/discoverer/discover.go b/internal/client/v1/client/discoverer/discover.go
index f4f2e6e8e12..d859212181b 100644
--- a/internal/client/v1/client/discoverer/discover.go
+++ b/internal/client/v1/client/discoverer/discover.go
@@ -25,13 +25,13 @@ import (
"github.com/vdaas/vald/apis/grpc/v1/discoverer"
"github.com/vdaas/vald/apis/grpc/v1/payload"
- "github.com/vdaas/vald/internal/errgroup"
"github.com/vdaas/vald/internal/errors"
"github.com/vdaas/vald/internal/log"
"github.com/vdaas/vald/internal/net"
"github.com/vdaas/vald/internal/net/grpc"
"github.com/vdaas/vald/internal/safety"
- valdsync "github.com/vdaas/vald/internal/sync"
+ "github.com/vdaas/vald/internal/sync"
+ "github.com/vdaas/vald/internal/sync/errgroup"
)
type Client interface {
@@ -344,7 +344,7 @@ func (c *client) disconnectOldAddrs(ctx context.Context, oldAddrs, connectedAddr
if !c.autoconn {
return nil
}
- var cur valdsync.Map[string, any] // TODO: Does this have to be a sync.Map not a map?
+ var cur sync.Map[string, any] // TODO: Does this have to be a sync.Map not a map?
for _, addr := range connectedAddrs {
cur.Store(addr, struct{}{})
}
diff --git a/internal/client/v1/client/discoverer/option.go b/internal/client/v1/client/discoverer/option.go
index 794662ef373..f9d890a7452 100644
--- a/internal/client/v1/client/discoverer/option.go
+++ b/internal/client/v1/client/discoverer/option.go
@@ -21,8 +21,8 @@ import (
"context"
"time"
- "github.com/vdaas/vald/internal/errgroup"
"github.com/vdaas/vald/internal/net/grpc"
+ "github.com/vdaas/vald/internal/sync/errgroup"
"github.com/vdaas/vald/internal/timeutil"
)
diff --git a/internal/client/v1/client/filter/egress/client.go b/internal/client/v1/client/filter/egress/client.go
index 77ea9793cfb..5e4105ef8a8 100644
--- a/internal/client/v1/client/filter/egress/client.go
+++ b/internal/client/v1/client/filter/egress/client.go
@@ -27,7 +27,7 @@ import (
"github.com/vdaas/vald/internal/net/grpc"
"github.com/vdaas/vald/internal/observability/trace"
"github.com/vdaas/vald/internal/strings"
- valdsync "github.com/vdaas/vald/internal/sync"
+ "github.com/vdaas/vald/internal/sync"
)
type Client interface {
@@ -40,7 +40,7 @@ type Client interface {
type client struct {
addrs []string
- cl valdsync.Map[string, any]
+ cl sync.Map[string, any]
c grpc.Client
}
diff --git a/internal/client/v1/client/filter/egress/client_test.go b/internal/client/v1/client/filter/egress/client_test.go
index db7850cc219..61ef1db0720 100644
--- a/internal/client/v1/client/filter/egress/client_test.go
+++ b/internal/client/v1/client/filter/egress/client_test.go
@@ -115,6 +115,7 @@ package egress
// }
// type fields struct {
// addrs []string
+// cl sync.Map[string, any]
// c grpc.Client
// }
// type want struct {
@@ -149,6 +150,7 @@ package egress
// },
// fields: fields {
// addrs:nil,
+// cl:nil,
// c:nil,
// },
// want: want{},
@@ -172,6 +174,7 @@ package egress
// },
// fields: fields {
// addrs:nil,
+// cl:nil,
// c:nil,
// },
// want: want{},
@@ -204,6 +207,7 @@ package egress
// }
// c := &client{
// addrs: test.fields.addrs,
+// cl: test.fields.cl,
// c: test.fields.c,
// }
//
@@ -222,6 +226,7 @@ package egress
// }
// type fields struct {
// addrs []string
+// cl sync.Map[string, any]
// c grpc.Client
// }
// type want struct {
@@ -252,6 +257,7 @@ package egress
// },
// fields: fields {
// addrs:nil,
+// cl:nil,
// c:nil,
// },
// want: want{},
@@ -275,6 +281,7 @@ package egress
// },
// fields: fields {
// addrs:nil,
+// cl:nil,
// c:nil,
// },
// want: want{},
@@ -307,6 +314,7 @@ package egress
// }
// c := &client{
// addrs: test.fields.addrs,
+// cl: test.fields.cl,
// c: test.fields.c,
// }
//
@@ -322,6 +330,7 @@ package egress
// func Test_client_GRPCClient(t *testing.T) {
// type fields struct {
// addrs []string
+// cl sync.Map[string, any]
// c grpc.Client
// }
// type want struct {
@@ -348,6 +357,7 @@ package egress
// name: "test_case_1",
// fields: fields {
// addrs:nil,
+// cl:nil,
// c:nil,
// },
// want: want{},
@@ -368,6 +378,7 @@ package egress
// name: "test_case_2",
// fields: fields {
// addrs:nil,
+// cl:nil,
// c:nil,
// },
// want: want{},
@@ -400,6 +411,7 @@ package egress
// }
// c := &client{
// addrs: test.fields.addrs,
+// cl: test.fields.cl,
// c: test.fields.c,
// }
//
@@ -419,6 +431,7 @@ package egress
// }
// type fields struct {
// addrs []string
+// cl sync.Map[string, any]
// c grpc.Client
// }
// type want struct {
@@ -454,6 +467,7 @@ package egress
// },
// fields: fields {
// addrs:nil,
+// cl:nil,
// c:nil,
// },
// want: want{},
@@ -478,6 +492,7 @@ package egress
// },
// fields: fields {
// addrs:nil,
+// cl:nil,
// c:nil,
// },
// want: want{},
@@ -510,6 +525,7 @@ package egress
// }
// c := &client{
// addrs: test.fields.addrs,
+// cl: test.fields.cl,
// c: test.fields.c,
// }
//
@@ -530,6 +546,7 @@ package egress
// }
// type fields struct {
// addrs []string
+// cl sync.Map[string, any]
// c grpc.Client
// }
// type want struct {
@@ -566,6 +583,7 @@ package egress
// },
// fields: fields {
// addrs:nil,
+// cl:nil,
// c:nil,
// },
// want: want{},
@@ -591,6 +609,7 @@ package egress
// },
// fields: fields {
// addrs:nil,
+// cl:nil,
// c:nil,
// },
// want: want{},
@@ -623,6 +642,7 @@ package egress
// }
// c := &client{
// addrs: test.fields.addrs,
+// cl: test.fields.cl,
// c: test.fields.c,
// }
//
@@ -869,6 +889,7 @@ package egress
// }
// type fields struct {
// addrs []string
+// cl sync.Map[string, any]
// c grpc.Client
// }
// type want struct {
@@ -905,6 +926,7 @@ package egress
// },
// fields: fields {
// addrs:nil,
+// cl:nil,
// c:nil,
// },
// want: want{},
@@ -930,6 +952,7 @@ package egress
// },
// fields: fields {
// addrs:nil,
+// cl:nil,
// c:nil,
// },
// want: want{},
@@ -962,6 +985,7 @@ package egress
// }
// c := &client{
// addrs: test.fields.addrs,
+// cl: test.fields.cl,
// c: test.fields.c,
// }
//
diff --git a/internal/client/v1/client/filter/ingress/client.go b/internal/client/v1/client/filter/ingress/client.go
index 7222e0763d3..645326b1af6 100644
--- a/internal/client/v1/client/filter/ingress/client.go
+++ b/internal/client/v1/client/filter/ingress/client.go
@@ -27,7 +27,7 @@ import (
"github.com/vdaas/vald/internal/net/grpc"
"github.com/vdaas/vald/internal/observability/trace"
"github.com/vdaas/vald/internal/strings"
- valdsync "github.com/vdaas/vald/internal/sync"
+ "github.com/vdaas/vald/internal/sync"
)
type Client interface {
@@ -40,7 +40,7 @@ type Client interface {
type client struct {
addrs []string
- cl valdsync.Map[string, any]
+ cl sync.Map[string, any]
c grpc.Client
}
diff --git a/internal/client/v1/client/filter/ingress/client_test.go b/internal/client/v1/client/filter/ingress/client_test.go
index 58fafcc5e9c..b1a95177708 100644
--- a/internal/client/v1/client/filter/ingress/client_test.go
+++ b/internal/client/v1/client/filter/ingress/client_test.go
@@ -115,6 +115,7 @@ package ingress
// }
// type fields struct {
// addrs []string
+// cl sync.Map[string, any]
// c grpc.Client
// }
// type want struct {
@@ -149,6 +150,7 @@ package ingress
// },
// fields: fields {
// addrs:nil,
+// cl:nil,
// c:nil,
// },
// want: want{},
@@ -172,6 +174,7 @@ package ingress
// },
// fields: fields {
// addrs:nil,
+// cl:nil,
// c:nil,
// },
// want: want{},
@@ -204,6 +207,7 @@ package ingress
// }
// c := &client{
// addrs: test.fields.addrs,
+// cl: test.fields.cl,
// c: test.fields.c,
// }
//
@@ -222,6 +226,7 @@ package ingress
// }
// type fields struct {
// addrs []string
+// cl sync.Map[string, any]
// c grpc.Client
// }
// type want struct {
@@ -252,6 +257,7 @@ package ingress
// },
// fields: fields {
// addrs:nil,
+// cl:nil,
// c:nil,
// },
// want: want{},
@@ -275,6 +281,7 @@ package ingress
// },
// fields: fields {
// addrs:nil,
+// cl:nil,
// c:nil,
// },
// want: want{},
@@ -307,6 +314,7 @@ package ingress
// }
// c := &client{
// addrs: test.fields.addrs,
+// cl: test.fields.cl,
// c: test.fields.c,
// }
//
@@ -322,6 +330,7 @@ package ingress
// func Test_client_GRPCClient(t *testing.T) {
// type fields struct {
// addrs []string
+// cl sync.Map[string, any]
// c grpc.Client
// }
// type want struct {
@@ -348,6 +357,7 @@ package ingress
// name: "test_case_1",
// fields: fields {
// addrs:nil,
+// cl:nil,
// c:nil,
// },
// want: want{},
@@ -368,6 +378,7 @@ package ingress
// name: "test_case_2",
// fields: fields {
// addrs:nil,
+// cl:nil,
// c:nil,
// },
// want: want{},
@@ -400,6 +411,7 @@ package ingress
// }
// c := &client{
// addrs: test.fields.addrs,
+// cl: test.fields.cl,
// c: test.fields.c,
// }
//
@@ -419,6 +431,7 @@ package ingress
// }
// type fields struct {
// addrs []string
+// cl sync.Map[string, any]
// c grpc.Client
// }
// type want struct {
@@ -454,6 +467,7 @@ package ingress
// },
// fields: fields {
// addrs:nil,
+// cl:nil,
// c:nil,
// },
// want: want{},
@@ -478,6 +492,7 @@ package ingress
// },
// fields: fields {
// addrs:nil,
+// cl:nil,
// c:nil,
// },
// want: want{},
@@ -510,6 +525,7 @@ package ingress
// }
// c := &client{
// addrs: test.fields.addrs,
+// cl: test.fields.cl,
// c: test.fields.c,
// }
//
@@ -530,6 +546,7 @@ package ingress
// }
// type fields struct {
// addrs []string
+// cl sync.Map[string, any]
// c grpc.Client
// }
// type want struct {
@@ -566,6 +583,7 @@ package ingress
// },
// fields: fields {
// addrs:nil,
+// cl:nil,
// c:nil,
// },
// want: want{},
@@ -591,6 +609,7 @@ package ingress
// },
// fields: fields {
// addrs:nil,
+// cl:nil,
// c:nil,
// },
// want: want{},
@@ -623,6 +642,7 @@ package ingress
// }
// c := &client{
// addrs: test.fields.addrs,
+// cl: test.fields.cl,
// c: test.fields.c,
// }
//
@@ -643,6 +663,7 @@ package ingress
// }
// type fields struct {
// addrs []string
+// cl sync.Map[string, any]
// c grpc.Client
// }
// type want struct {
@@ -679,6 +700,7 @@ package ingress
// },
// fields: fields {
// addrs:nil,
+// cl:nil,
// c:nil,
// },
// want: want{},
@@ -704,6 +726,7 @@ package ingress
// },
// fields: fields {
// addrs:nil,
+// cl:nil,
// c:nil,
// },
// want: want{},
@@ -736,6 +759,7 @@ package ingress
// }
// c := &client{
// addrs: test.fields.addrs,
+// cl: test.fields.cl,
// c: test.fields.c,
// }
//
diff --git a/internal/core/algorithm/ngt/ngt.go b/internal/core/algorithm/ngt/ngt.go
index 28a40b2bfc8..80156eb2083 100644
--- a/internal/core/algorithm/ngt/ngt.go
+++ b/internal/core/algorithm/ngt/ngt.go
@@ -27,13 +27,13 @@ import "C"
import (
"context"
"reflect"
- "sync"
"unsafe"
"github.com/vdaas/vald/internal/core/algorithm"
"github.com/vdaas/vald/internal/errors"
"github.com/vdaas/vald/internal/file"
"github.com/vdaas/vald/internal/log"
+ "github.com/vdaas/vald/internal/sync"
)
type (
diff --git a/internal/core/algorithm/ngt/ngt_test.go b/internal/core/algorithm/ngt/ngt_test.go
index 1fe3609d2fe..0e40fa3ecab 100644
--- a/internal/core/algorithm/ngt/ngt_test.go
+++ b/internal/core/algorithm/ngt/ngt_test.go
@@ -25,7 +25,6 @@ import (
"path/filepath"
"reflect"
"strings"
- "sync"
"testing"
"github.com/vdaas/vald/internal/core/algorithm"
@@ -33,6 +32,7 @@ import (
"github.com/vdaas/vald/internal/file"
"github.com/vdaas/vald/internal/log"
"github.com/vdaas/vald/internal/log/logger"
+ "github.com/vdaas/vald/internal/sync"
"github.com/vdaas/vald/internal/test/comparator"
"github.com/vdaas/vald/internal/test/goleak"
)
diff --git a/internal/db/storage/blob/s3/option.go b/internal/db/storage/blob/s3/option.go
index 39472f8196c..92c94054dec 100644
--- a/internal/db/storage/blob/s3/option.go
+++ b/internal/db/storage/blob/s3/option.go
@@ -22,7 +22,7 @@ import (
"github.com/vdaas/vald/internal/backoff"
"github.com/vdaas/vald/internal/db/storage/blob/s3/reader"
"github.com/vdaas/vald/internal/db/storage/blob/s3/writer"
- "github.com/vdaas/vald/internal/errgroup"
+ "github.com/vdaas/vald/internal/sync/errgroup"
"github.com/vdaas/vald/internal/unit"
)
diff --git a/internal/db/storage/blob/s3/option_test.go b/internal/db/storage/blob/s3/option_test.go
index 2cdf592f951..beefdf10059 100644
--- a/internal/db/storage/blob/s3/option_test.go
+++ b/internal/db/storage/blob/s3/option_test.go
@@ -24,8 +24,8 @@ import (
"github.com/vdaas/vald/internal/backoff"
"github.com/vdaas/vald/internal/db/storage/blob/s3/reader"
"github.com/vdaas/vald/internal/db/storage/blob/s3/writer"
- "github.com/vdaas/vald/internal/errgroup"
"github.com/vdaas/vald/internal/errors"
+ "github.com/vdaas/vald/internal/sync/errgroup"
"github.com/vdaas/vald/internal/test/comparator"
"github.com/vdaas/vald/internal/test/goleak"
)
diff --git a/internal/db/storage/blob/s3/reader/option.go b/internal/db/storage/blob/s3/reader/option.go
index ed3cddc4a3a..96d5b210af6 100644
--- a/internal/db/storage/blob/s3/reader/option.go
+++ b/internal/db/storage/blob/s3/reader/option.go
@@ -20,7 +20,7 @@ import (
"github.com/vdaas/vald/internal/backoff"
"github.com/vdaas/vald/internal/db/storage/blob/s3/reader/io"
"github.com/vdaas/vald/internal/db/storage/blob/s3/sdk/s3/s3iface"
- "github.com/vdaas/vald/internal/errgroup"
+ "github.com/vdaas/vald/internal/sync/errgroup"
)
// Option represents the functional option for reader.
diff --git a/internal/db/storage/blob/s3/reader/option_test.go b/internal/db/storage/blob/s3/reader/option_test.go
index d46f0cd4c6b..544a2f7a03e 100644
--- a/internal/db/storage/blob/s3/reader/option_test.go
+++ b/internal/db/storage/blob/s3/reader/option_test.go
@@ -23,8 +23,8 @@ import (
"github.com/aws/aws-sdk-go/service/s3"
"github.com/vdaas/vald/internal/backoff"
"github.com/vdaas/vald/internal/db/storage/blob/s3/sdk/s3/s3iface"
- "github.com/vdaas/vald/internal/errgroup"
"github.com/vdaas/vald/internal/errors"
+ "github.com/vdaas/vald/internal/sync/errgroup"
"github.com/vdaas/vald/internal/test/comparator"
"github.com/vdaas/vald/internal/test/goleak"
)
diff --git a/internal/db/storage/blob/s3/reader/reader.go b/internal/db/storage/blob/s3/reader/reader.go
index fbff2f9cb13..bc8a3cc483c 100644
--- a/internal/db/storage/blob/s3/reader/reader.go
+++ b/internal/db/storage/blob/s3/reader/reader.go
@@ -20,7 +20,6 @@ import (
"bytes"
"context"
"strconv"
- "sync"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/awserr"
@@ -28,11 +27,12 @@ import (
ctxio "github.com/vdaas/vald/internal/db/storage/blob/s3/reader/io"
"github.com/vdaas/vald/internal/db/storage/blob/s3/sdk/s3"
"github.com/vdaas/vald/internal/db/storage/blob/s3/sdk/s3/s3iface"
- "github.com/vdaas/vald/internal/errgroup"
"github.com/vdaas/vald/internal/errors"
"github.com/vdaas/vald/internal/io"
"github.com/vdaas/vald/internal/log"
"github.com/vdaas/vald/internal/safety"
+ "github.com/vdaas/vald/internal/sync"
+ "github.com/vdaas/vald/internal/sync/errgroup"
)
type reader struct {
diff --git a/internal/db/storage/blob/s3/reader/reader_test.go b/internal/db/storage/blob/s3/reader/reader_test.go
index 43e7a0d9521..0fa9d48a88c 100644
--- a/internal/db/storage/blob/s3/reader/reader_test.go
+++ b/internal/db/storage/blob/s3/reader/reader_test.go
@@ -21,7 +21,6 @@ import (
"context"
"os"
"reflect"
- "sync"
"sync/atomic"
"testing"
@@ -32,11 +31,12 @@ import (
ctxio "github.com/vdaas/vald/internal/db/storage/blob/s3/reader/io"
"github.com/vdaas/vald/internal/db/storage/blob/s3/sdk/s3"
"github.com/vdaas/vald/internal/db/storage/blob/s3/sdk/s3/s3iface"
- "github.com/vdaas/vald/internal/errgroup"
"github.com/vdaas/vald/internal/errors"
"github.com/vdaas/vald/internal/io"
"github.com/vdaas/vald/internal/log"
"github.com/vdaas/vald/internal/log/logger"
+ "github.com/vdaas/vald/internal/sync"
+ "github.com/vdaas/vald/internal/sync/errgroup"
"github.com/vdaas/vald/internal/test/goleak"
)
diff --git a/internal/db/storage/blob/s3/s3.go b/internal/db/storage/blob/s3/s3.go
index f48422a0082..02935d557fc 100644
--- a/internal/db/storage/blob/s3/s3.go
+++ b/internal/db/storage/blob/s3/s3.go
@@ -26,9 +26,9 @@ import (
"github.com/vdaas/vald/internal/db/storage/blob"
"github.com/vdaas/vald/internal/db/storage/blob/s3/reader"
"github.com/vdaas/vald/internal/db/storage/blob/s3/writer"
- "github.com/vdaas/vald/internal/errgroup"
"github.com/vdaas/vald/internal/errors"
"github.com/vdaas/vald/internal/io"
+ "github.com/vdaas/vald/internal/sync/errgroup"
)
type client struct {
diff --git a/internal/db/storage/blob/s3/s3_test.go b/internal/db/storage/blob/s3/s3_test.go
index 3e37575d462..d486f4ca88b 100644
--- a/internal/db/storage/blob/s3/s3_test.go
+++ b/internal/db/storage/blob/s3/s3_test.go
@@ -30,9 +30,9 @@ import (
"github.com/vdaas/vald/internal/db/storage/blob/s3/reader"
"github.com/vdaas/vald/internal/db/storage/blob/s3/sdk/s3/s3manager"
"github.com/vdaas/vald/internal/db/storage/blob/s3/writer"
- "github.com/vdaas/vald/internal/errgroup"
"github.com/vdaas/vald/internal/errors"
"github.com/vdaas/vald/internal/io"
+ "github.com/vdaas/vald/internal/sync/errgroup"
"github.com/vdaas/vald/internal/test/comparator"
"github.com/vdaas/vald/internal/test/goleak"
)
diff --git a/internal/db/storage/blob/s3/writer/option.go b/internal/db/storage/blob/s3/writer/option.go
index a1e7a685c56..73f8ea46b60 100644
--- a/internal/db/storage/blob/s3/writer/option.go
+++ b/internal/db/storage/blob/s3/writer/option.go
@@ -19,8 +19,8 @@ package writer
import (
"github.com/aws/aws-sdk-go/service/s3"
"github.com/aws/aws-sdk-go/service/s3/s3manager"
- "github.com/vdaas/vald/internal/errgroup"
"github.com/vdaas/vald/internal/errors"
+ "github.com/vdaas/vald/internal/sync/errgroup"
)
// Option represents the functional option for writer.
diff --git a/internal/db/storage/blob/s3/writer/option_test.go b/internal/db/storage/blob/s3/writer/option_test.go
index 8db3c6bf1c2..a4b79c47023 100644
--- a/internal/db/storage/blob/s3/writer/option_test.go
+++ b/internal/db/storage/blob/s3/writer/option_test.go
@@ -22,8 +22,8 @@ import (
"github.com/aws/aws-sdk-go/service/s3"
"github.com/aws/aws-sdk-go/service/s3/s3manager"
- "github.com/vdaas/vald/internal/errgroup"
"github.com/vdaas/vald/internal/errors"
+ "github.com/vdaas/vald/internal/sync/errgroup"
"github.com/vdaas/vald/internal/test/goleak"
)
diff --git a/internal/db/storage/blob/s3/writer/writer.go b/internal/db/storage/blob/s3/writer/writer.go
index c059b49c965..6b0f8cd0e93 100644
--- a/internal/db/storage/blob/s3/writer/writer.go
+++ b/internal/db/storage/blob/s3/writer/writer.go
@@ -19,16 +19,16 @@ package writer
import (
"context"
"reflect"
- "sync"
"github.com/aws/aws-sdk-go/aws"
"github.com/vdaas/vald/internal/db/storage/blob/s3/sdk/s3/s3iface"
"github.com/vdaas/vald/internal/db/storage/blob/s3/sdk/s3/s3manager"
- "github.com/vdaas/vald/internal/errgroup"
"github.com/vdaas/vald/internal/errors"
"github.com/vdaas/vald/internal/io"
"github.com/vdaas/vald/internal/log"
"github.com/vdaas/vald/internal/safety"
+ "github.com/vdaas/vald/internal/sync"
+ "github.com/vdaas/vald/internal/sync/errgroup"
)
type writer struct {
diff --git a/internal/db/storage/blob/s3/writer/writer_test.go b/internal/db/storage/blob/s3/writer/writer_test.go
index 5d6e6d624e7..1c7545a62ad 100644
--- a/internal/db/storage/blob/s3/writer/writer_test.go
+++ b/internal/db/storage/blob/s3/writer/writer_test.go
@@ -20,18 +20,18 @@ import (
"context"
"os"
"reflect"
- "sync"
"testing"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/vdaas/vald/internal/db/storage/blob/s3/sdk/s3/s3iface"
"github.com/vdaas/vald/internal/db/storage/blob/s3/sdk/s3/s3manager"
- "github.com/vdaas/vald/internal/errgroup"
"github.com/vdaas/vald/internal/errors"
"github.com/vdaas/vald/internal/io"
"github.com/vdaas/vald/internal/log"
"github.com/vdaas/vald/internal/log/logger"
+ "github.com/vdaas/vald/internal/sync"
+ "github.com/vdaas/vald/internal/sync/errgroup"
"github.com/vdaas/vald/internal/test/goleak"
)
diff --git a/internal/errgroup/group.go b/internal/errgroup/group.go
deleted file mode 100644
index f1071921ef8..00000000000
--- a/internal/errgroup/group.go
+++ /dev/null
@@ -1,202 +0,0 @@
-//
-// Copyright (C) 2019-2023 vdaas.org vald team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// https://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-// Package errgroup provides server global wait group for graceful kill all goroutine
-package errgroup
-
-import (
- "context"
- "runtime"
- "sync"
- "sync/atomic"
-
- "github.com/vdaas/vald/internal/errors"
-)
-
-type Group interface {
- Go(func() error)
- Limitation(int)
- Wait() error
-}
-
-type group struct {
- egctx context.Context
- cancel context.CancelFunc
-
- wg sync.WaitGroup
-
- limitation chan struct{}
- enableLimitation atomic.Bool
- cancelOnce sync.Once
- mu sync.RWMutex
- emap map[string]struct{}
- errs []error
- err error
-}
-
-var (
- instance Group
- once sync.Once
-)
-
-func New(ctx context.Context) (Group, context.Context) {
- egctx, cancel := context.WithCancel(ctx)
- g := &group{
- egctx: egctx,
- emap: make(map[string]struct{}),
- cancel: cancel,
- }
- g.enableLimitation.Store(false)
- return g, egctx
-}
-
-func Init(ctx context.Context) (egctx context.Context) {
- egctx = ctx
- once.Do(func() {
- instance, egctx = New(ctx)
- })
- return
-}
-
-func Get() Group {
- if instance == nil {
- Init(context.Background())
- }
- return instance
-}
-
-func Go(f func() error) {
- instance.Go(f)
-}
-
-func (g *group) Limitation(limit int) {
- if limit > 0 {
- ch := make(chan struct{}, limit)
- g.closeLimitation()
- g.limitation = ch
- g.enableLimitation.Store(true)
- } else {
- g.enableLimitation.Store(false)
- }
-}
-
-func (g *group) closeLimitation() {
- if g.limitation != nil {
- for {
- select {
- case _, live := <-g.limitation:
- if !live {
- return
- }
- default:
- close(g.limitation)
- return
- }
- }
- }
-}
-
-func (g *group) Go(f func() error) {
- if f != nil {
- g.wg.Add(1)
- go func() {
- defer g.wg.Done()
- var err error
- if g.enableLimitation.Load() {
- select {
- case <-g.egctx.Done():
- err = g.egctx.Err()
- if !errors.Is(err, context.Canceled) &&
- !errors.Is(err, context.DeadlineExceeded) {
- g.mu.RLock()
- _, ok := g.emap[err.Error()]
- g.mu.RUnlock()
- if !ok {
- g.mu.Lock()
- g.errs = append(g.errs, err)
- g.emap[err.Error()] = struct{}{}
- g.mu.Unlock()
- }
- }
- g.doCancel()
- return
- case g.limitation <- struct{}{}:
- }
- err = f()
- select {
- case <-g.limitation:
- case <-g.egctx.Done():
- if err != nil {
- err = errors.Join(err, g.egctx.Err())
- }
- }
- } else {
- err = f()
- }
- if err != nil {
- if !errors.Is(err, context.Canceled) &&
- !errors.Is(err, context.DeadlineExceeded) {
- runtime.Gosched()
- g.mu.RLock()
- _, ok := g.emap[err.Error()]
- g.mu.RUnlock()
- if !ok {
- g.mu.Lock()
- g.errs = append(g.errs, err)
- g.emap[err.Error()] = struct{}{}
- g.mu.Unlock()
- }
- }
- g.doCancel()
- return
- }
- }()
- }
-}
-
-func (g *group) doCancel() {
- g.cancelOnce.Do(func() {
- if g.cancel != nil {
- g.cancel()
- }
- })
-}
-
-func Wait() error {
- return instance.Wait()
-}
-
-func (g *group) Wait() error {
- g.wg.Wait()
- g.doCancel()
- g.closeLimitation()
- g.enableLimitation.Store(false)
- g.mu.RLock()
- switch len(g.errs) {
- case 0:
- g.mu.RUnlock()
- return nil
- case 1:
- g.err = g.errs[0]
- default:
- g.err = g.errs[0]
- for _, err := range g.errs[1:] {
- g.err = errors.Join(g.err, err)
- }
- }
- g.mu.RUnlock()
- return g.err
-}
diff --git a/internal/errors/errors.go b/internal/errors/errors.go
index b9585230c92..d4647986f8e 100644
--- a/internal/errors/errors.go
+++ b/internal/errors/errors.go
@@ -23,7 +23,8 @@ import (
"reflect"
"runtime"
"strings"
- "sync"
+
+ "github.com/vdaas/vald/internal/sync"
)
var (
diff --git a/internal/file/file.go b/internal/file/file.go
index ec9e57c08d7..0d766878fde 100644
--- a/internal/file/file.go
+++ b/internal/file/file.go
@@ -26,13 +26,13 @@ import (
"time"
"github.com/kpango/fastime"
- "github.com/vdaas/vald/internal/errgroup"
"github.com/vdaas/vald/internal/errors"
"github.com/vdaas/vald/internal/io"
"github.com/vdaas/vald/internal/log"
"github.com/vdaas/vald/internal/safety"
"github.com/vdaas/vald/internal/slices"
"github.com/vdaas/vald/internal/strings"
+ "github.com/vdaas/vald/internal/sync/errgroup"
)
// Open opens the file with the given path, flag and permission.
diff --git a/internal/file/watch/option.go b/internal/file/watch/option.go
index 3cf89bf1095..43b509982ad 100644
--- a/internal/file/watch/option.go
+++ b/internal/file/watch/option.go
@@ -19,7 +19,7 @@ package watch
import (
"context"
- "github.com/vdaas/vald/internal/errgroup"
+ "github.com/vdaas/vald/internal/sync/errgroup"
)
type Option func(w *watch) error
diff --git a/internal/file/watch/option_test.go b/internal/file/watch/option_test.go
index 997737d95d4..8b5fb8ae8e9 100644
--- a/internal/file/watch/option_test.go
+++ b/internal/file/watch/option_test.go
@@ -21,8 +21,8 @@ import (
"reflect"
"testing"
- "github.com/vdaas/vald/internal/errgroup"
"github.com/vdaas/vald/internal/errors"
+ "github.com/vdaas/vald/internal/sync/errgroup"
"github.com/vdaas/vald/internal/test/goleak"
)
diff --git a/internal/file/watch/watch.go b/internal/file/watch/watch.go
index fab40d2bc33..ca2723b669b 100644
--- a/internal/file/watch/watch.go
+++ b/internal/file/watch/watch.go
@@ -19,13 +19,13 @@ package watch
import (
"context"
"reflect"
- "sync"
"github.com/fsnotify/fsnotify"
- "github.com/vdaas/vald/internal/errgroup"
"github.com/vdaas/vald/internal/errors"
"github.com/vdaas/vald/internal/log"
"github.com/vdaas/vald/internal/safety"
+ "github.com/vdaas/vald/internal/sync"
+ "github.com/vdaas/vald/internal/sync/errgroup"
)
// Watcher is an interface that represents a file monitor.
diff --git a/internal/file/watch/watch_test.go b/internal/file/watch/watch_test.go
index 3f3a34369c4..e7bb95d21d3 100644
--- a/internal/file/watch/watch_test.go
+++ b/internal/file/watch/watch_test.go
@@ -26,12 +26,12 @@ import (
"testing"
"github.com/fsnotify/fsnotify"
- "github.com/vdaas/vald/internal/errgroup"
"github.com/vdaas/vald/internal/errors"
"github.com/vdaas/vald/internal/file"
"github.com/vdaas/vald/internal/log"
"github.com/vdaas/vald/internal/log/logger"
"github.com/vdaas/vald/internal/strings"
+ "github.com/vdaas/vald/internal/sync/errgroup"
"github.com/vdaas/vald/internal/test/goleak"
)
diff --git a/internal/info/info.go b/internal/info/info.go
index 7baeae5d6e5..83543025575 100644
--- a/internal/info/info.go
+++ b/internal/info/info.go
@@ -22,12 +22,12 @@ import (
"reflect"
"runtime"
"strconv"
- "sync"
"github.com/vdaas/vald/internal/errors"
"github.com/vdaas/vald/internal/log"
"github.com/vdaas/vald/internal/slices"
"github.com/vdaas/vald/internal/strings"
+ "github.com/vdaas/vald/internal/sync"
)
// Info represents an interface to get the runtime information.
diff --git a/internal/info/info_test.go b/internal/info/info_test.go
index 75f0174aa5a..16a2ba99b30 100644
--- a/internal/info/info_test.go
+++ b/internal/info/info_test.go
@@ -21,13 +21,13 @@ import (
"os"
"reflect"
"runtime"
- "sync"
"testing"
"github.com/vdaas/vald/internal/errors"
"github.com/vdaas/vald/internal/log"
"github.com/vdaas/vald/internal/log/logger"
"github.com/vdaas/vald/internal/strings"
+ "github.com/vdaas/vald/internal/sync"
"github.com/vdaas/vald/internal/test/comparator"
"github.com/vdaas/vald/internal/test/goleak"
)
diff --git a/internal/info/option_test.go b/internal/info/option_test.go
index 99a6802f665..1a90462e691 100644
--- a/internal/info/option_test.go
+++ b/internal/info/option_test.go
@@ -16,10 +16,10 @@ package info
import (
"reflect"
"runtime"
- "sync"
"testing"
"github.com/vdaas/vald/internal/errors"
+ "github.com/vdaas/vald/internal/sync"
"github.com/vdaas/vald/internal/test/comparator"
"github.com/vdaas/vald/internal/test/goleak"
)
diff --git a/internal/io/copy.go b/internal/io/copy.go
index 8884063cec4..e22dbdb030b 100644
--- a/internal/io/copy.go
+++ b/internal/io/copy.go
@@ -21,10 +21,10 @@ import (
"bytes"
"io"
"math"
- "sync"
"sync/atomic"
"github.com/vdaas/vald/internal/errors"
+ "github.com/vdaas/vald/internal/sync"
)
var cio = NewCopier(0)
diff --git a/internal/io/copy_test.go b/internal/io/copy_test.go
index 72b4cd4e581..b88af60cf00 100644
--- a/internal/io/copy_test.go
+++ b/internal/io/copy_test.go
@@ -21,11 +21,11 @@ import (
"bytes"
"io"
"reflect"
- "sync"
"sync/atomic"
"testing"
"github.com/vdaas/vald/internal/errors"
+ "github.com/vdaas/vald/internal/sync"
"github.com/vdaas/vald/internal/test/goleak"
)
diff --git a/internal/io/io.go b/internal/io/io.go
index 0e39415328a..a302f5ad128 100644
--- a/internal/io/io.go
+++ b/internal/io/io.go
@@ -21,11 +21,11 @@ import (
"bytes"
"context"
"io"
- "sync"
"github.com/vdaas/vald/internal/conv"
"github.com/vdaas/vald/internal/errors"
"github.com/vdaas/vald/internal/safety"
+ "github.com/vdaas/vald/internal/sync"
)
type (
diff --git a/internal/k8s/option.go b/internal/k8s/option.go
index 255c30a9358..6b5d93168a0 100644
--- a/internal/k8s/option.go
+++ b/internal/k8s/option.go
@@ -18,8 +18,8 @@
package k8s
import (
- "github.com/vdaas/vald/internal/errgroup"
"github.com/vdaas/vald/internal/net"
+ "github.com/vdaas/vald/internal/sync/errgroup"
"sigs.k8s.io/controller-runtime/pkg/manager"
)
diff --git a/internal/k8s/reconciler.go b/internal/k8s/reconciler.go
index a329232795f..f40892b81e2 100644
--- a/internal/k8s/reconciler.go
+++ b/internal/k8s/reconciler.go
@@ -21,10 +21,10 @@ import (
"context"
"reflect"
- "github.com/vdaas/vald/internal/errgroup"
"github.com/vdaas/vald/internal/errors"
"github.com/vdaas/vald/internal/net"
"github.com/vdaas/vald/internal/safety"
+ "github.com/vdaas/vald/internal/sync/errgroup"
"k8s.io/apimachinery/pkg/runtime"
"sigs.k8s.io/controller-runtime/pkg/builder"
"sigs.k8s.io/controller-runtime/pkg/client"
diff --git a/internal/log/log.go b/internal/log/log.go
index 21bdd0129cf..8a9b69168ba 100644
--- a/internal/log/log.go
+++ b/internal/log/log.go
@@ -17,8 +17,6 @@
package log
import (
- "sync"
-
"github.com/vdaas/vald/internal/log/format"
"github.com/vdaas/vald/internal/log/glg"
"github.com/vdaas/vald/internal/log/level"
@@ -26,6 +24,7 @@ import (
"github.com/vdaas/vald/internal/log/nop"
"github.com/vdaas/vald/internal/log/retry"
"github.com/vdaas/vald/internal/log/zap"
+ "github.com/vdaas/vald/internal/sync"
)
var (
diff --git a/internal/log/log_test.go b/internal/log/log_test.go
index 6b8688f34a3..735f11ae9f2 100644
--- a/internal/log/log_test.go
+++ b/internal/log/log_test.go
@@ -18,7 +18,6 @@ package log
import (
"reflect"
- "sync"
"testing"
"github.com/vdaas/vald/internal/errors"
@@ -27,6 +26,7 @@ import (
logger "github.com/vdaas/vald/internal/log/logger"
"github.com/vdaas/vald/internal/log/mock"
"github.com/vdaas/vald/internal/log/nop"
+ "github.com/vdaas/vald/internal/sync"
"github.com/vdaas/vald/internal/test/goleak"
)
diff --git a/internal/net/dialer.go b/internal/net/dialer.go
index 692cc1e0586..822bfcc7630 100644
--- a/internal/net/dialer.go
+++ b/internal/net/dialer.go
@@ -21,7 +21,6 @@ import (
"context"
"net"
"strconv"
- "sync"
"sync/atomic"
"syscall"
"time"
@@ -33,7 +32,7 @@ import (
"github.com/vdaas/vald/internal/net/control"
"github.com/vdaas/vald/internal/observability/trace"
"github.com/vdaas/vald/internal/safety"
- valdsync "github.com/vdaas/vald/internal/sync"
+ "github.com/vdaas/vald/internal/sync"
"github.com/vdaas/vald/internal/tls"
)
@@ -60,7 +59,7 @@ type dialer struct {
ctrl control.SocketController
sockFlg control.SocketFlag
dialerDualStack bool
- addrs valdsync.Map[string, *addrInfo]
+ addrs sync.Map[string, *addrInfo]
der *net.Dialer
dialer func(ctx context.Context, network, addr string) (Conn, error)
}
diff --git a/internal/net/dialer_test.go b/internal/net/dialer_test.go
index 9c2ff0dc333..b7ac7de9b2a 100644
--- a/internal/net/dialer_test.go
+++ b/internal/net/dialer_test.go
@@ -1830,7 +1830,7 @@ func Test_dialer_tlsHandshake(t *testing.T) {
// host string
// }
// type fields struct {
-// dnsCache cacher.Cache
+// dnsCache cacher.Cache[*dialerCache]
// enableDNSCache bool
// tlsConfig *tls.Config
// dnsRefreshDurationStr string
@@ -1843,6 +1843,7 @@ func Test_dialer_tlsHandshake(t *testing.T) {
// ctrl control.SocketController
// sockFlg control.SocketFlag
// dialerDualStack bool
+// addrs sync.Map[string, *addrInfo]
// der *net.Dialer
// dialer func(ctx context.Context, network, addr string) (Conn, error)
// }
@@ -1891,6 +1892,7 @@ func Test_dialer_tlsHandshake(t *testing.T) {
// ctrl:nil,
// sockFlg:nil,
// dialerDualStack:false,
+// addrs:nil,
// der:net.Dialer{},
// dialer:nil,
// },
@@ -1928,6 +1930,7 @@ func Test_dialer_tlsHandshake(t *testing.T) {
// ctrl:nil,
// sockFlg:nil,
// dialerDualStack:false,
+// addrs:nil,
// der:net.Dialer{},
// dialer:nil,
// },
@@ -1973,6 +1976,7 @@ func Test_dialer_tlsHandshake(t *testing.T) {
// ctrl: test.fields.ctrl,
// sockFlg: test.fields.sockFlg,
// dialerDualStack: test.fields.dialerDualStack,
+// addrs: test.fields.addrs,
// der: test.fields.der,
// dialer: test.fields.dialer,
// }
diff --git a/internal/net/grpc/client.go b/internal/net/grpc/client.go
index 928e92ade5a..06b8b32a0ae 100644
--- a/internal/net/grpc/client.go
+++ b/internal/net/grpc/client.go
@@ -25,7 +25,6 @@ import (
"github.com/vdaas/vald/internal/backoff"
"github.com/vdaas/vald/internal/circuitbreaker"
- "github.com/vdaas/vald/internal/errgroup"
"github.com/vdaas/vald/internal/errors"
"github.com/vdaas/vald/internal/log"
"github.com/vdaas/vald/internal/net"
@@ -35,9 +34,10 @@ import (
"github.com/vdaas/vald/internal/net/grpc/status"
"github.com/vdaas/vald/internal/observability/trace"
"github.com/vdaas/vald/internal/safety"
- "github.com/vdaas/vald/internal/singleflight"
"github.com/vdaas/vald/internal/strings"
- valdsync "github.com/vdaas/vald/internal/sync"
+ "github.com/vdaas/vald/internal/sync"
+ "github.com/vdaas/vald/internal/sync/errgroup"
+ "github.com/vdaas/vald/internal/sync/singleflight"
"google.golang.org/grpc"
gbackoff "google.golang.org/grpc/backoff"
)
@@ -110,7 +110,7 @@ type gRPCClient struct {
gbo gbackoff.Config // grpc's original backoff configuration
mcd time.Duration // minimum connection timeout duration
group singleflight.Group[pool.Conn]
- crl valdsync.Map[string, bool] // connection request list
+ crl sync.Map[string, bool] // connection request list
ech <-chan error
monitorRunning atomic.Bool
@@ -424,7 +424,7 @@ func (g *gRPCClient) RangeConcurrent(ctx context.Context,
return g.Range(ctx, f)
}
eg, egctx := errgroup.New(sctx)
- eg.Limitation(concurrency)
+ eg.SetLimit(concurrency)
if g.conns.Len() == 0 {
return errors.ErrGRPCClientConnNotFound("*")
}
@@ -570,11 +570,11 @@ func (g *gRPCClient) OrderedRangeConcurrent(ctx context.Context,
if g.conns.Len() == 0 {
return errors.ErrGRPCClientConnNotFound("*")
}
- if concurrency < 2 {
+ if concurrency == 0 || concurrency == 1 {
return g.OrderedRange(ctx, orders, f)
}
eg, egctx := errgroup.New(sctx)
- eg.Limitation(concurrency)
+ eg.SetLimit(concurrency)
for _, order := range orders {
addr := order
eg.Go(safety.RecoverFunc(func() (err error) {
@@ -893,7 +893,7 @@ func (g *gRPCClient) Connect(ctx context.Context, addr string, dopts ...DialOpti
span.End()
}
}()
- sconn, shared, err := g.group.Do(ctx, "connect-"+addr, func() (pool.Conn, error) {
+ sconn, shared, err := g.group.Do(ctx, "connect-"+addr, func(ctx context.Context) (pool.Conn, error) {
var ok bool
conn, ok = g.conns.Load(addr)
if ok && conn != nil {
@@ -978,7 +978,7 @@ func (g *gRPCClient) Disconnect(ctx context.Context, addr string) error {
span.End()
}
}()
- _, _, err := g.group.Do(ctx, "disconnect-"+addr, func() (pool.Conn, error) {
+ _, _, err := g.group.Do(ctx, "disconnect-"+addr, func(ctx context.Context) (pool.Conn, error) {
p, ok := g.conns.Load(addr)
if !ok || p == nil {
g.conns.Delete(addr)
diff --git a/internal/net/grpc/client_test.go b/internal/net/grpc/client_test.go
index 9288153a2ff..d862d7a7c55 100644
--- a/internal/net/grpc/client_test.go
+++ b/internal/net/grpc/client_test.go
@@ -128,6 +128,7 @@ package grpc
// gbo gbackoff.Config
// mcd time.Duration
// group singleflight.Group[pool.Conn]
+// crl sync.Map[string, bool]
// ech <-chan error
// monitorRunning atomic.Bool
// stopMonitor context.CancelFunc
@@ -181,6 +182,7 @@ package grpc
// gbo:nil,
// mcd:nil,
// group:nil,
+// crl:nil,
// ech:nil,
// monitorRunning:nil,
// stopMonitor:nil,
@@ -223,6 +225,7 @@ package grpc
// gbo:nil,
// mcd:nil,
// group:nil,
+// crl:nil,
// ech:nil,
// monitorRunning:nil,
// stopMonitor:nil,
@@ -274,6 +277,7 @@ package grpc
// gbo: test.fields.gbo,
// mcd: test.fields.mcd,
// group: test.fields.group,
+// crl: test.fields.crl,
// ech: test.fields.ech,
// monitorRunning: test.fields.monitorRunning,
// stopMonitor: test.fields.stopMonitor,
@@ -312,6 +316,7 @@ package grpc
// gbo gbackoff.Config
// mcd time.Duration
// group singleflight.Group[pool.Conn]
+// crl sync.Map[string, bool]
// ech <-chan error
// monitorRunning atomic.Bool
// stopMonitor context.CancelFunc
@@ -362,6 +367,7 @@ package grpc
// gbo:nil,
// mcd:nil,
// group:nil,
+// crl:nil,
// ech:nil,
// monitorRunning:nil,
// stopMonitor:nil,
@@ -405,6 +411,7 @@ package grpc
// gbo:nil,
// mcd:nil,
// group:nil,
+// crl:nil,
// ech:nil,
// monitorRunning:nil,
// stopMonitor:nil,
@@ -456,6 +463,7 @@ package grpc
// gbo: test.fields.gbo,
// mcd: test.fields.mcd,
// group: test.fields.group,
+// crl: test.fields.crl,
// ech: test.fields.ech,
// monitorRunning: test.fields.monitorRunning,
// stopMonitor: test.fields.stopMonitor,
@@ -495,6 +503,7 @@ package grpc
// gbo gbackoff.Config
// mcd time.Duration
// group singleflight.Group[pool.Conn]
+// crl sync.Map[string, bool]
// ech <-chan error
// monitorRunning atomic.Bool
// stopMonitor context.CancelFunc
@@ -546,6 +555,7 @@ package grpc
// gbo:nil,
// mcd:nil,
// group:nil,
+// crl:nil,
// ech:nil,
// monitorRunning:nil,
// stopMonitor:nil,
@@ -590,6 +600,7 @@ package grpc
// gbo:nil,
// mcd:nil,
// group:nil,
+// crl:nil,
// ech:nil,
// monitorRunning:nil,
// stopMonitor:nil,
@@ -641,6 +652,7 @@ package grpc
// gbo: test.fields.gbo,
// mcd: test.fields.mcd,
// group: test.fields.group,
+// crl: test.fields.crl,
// ech: test.fields.ech,
// monitorRunning: test.fields.monitorRunning,
// stopMonitor: test.fields.stopMonitor,
@@ -680,6 +692,7 @@ package grpc
// gbo gbackoff.Config
// mcd time.Duration
// group singleflight.Group[pool.Conn]
+// crl sync.Map[string, bool]
// ech <-chan error
// monitorRunning atomic.Bool
// stopMonitor context.CancelFunc
@@ -731,6 +744,7 @@ package grpc
// gbo:nil,
// mcd:nil,
// group:nil,
+// crl:nil,
// ech:nil,
// monitorRunning:nil,
// stopMonitor:nil,
@@ -775,6 +789,7 @@ package grpc
// gbo:nil,
// mcd:nil,
// group:nil,
+// crl:nil,
// ech:nil,
// monitorRunning:nil,
// stopMonitor:nil,
@@ -826,6 +841,7 @@ package grpc
// gbo: test.fields.gbo,
// mcd: test.fields.mcd,
// group: test.fields.group,
+// crl: test.fields.crl,
// ech: test.fields.ech,
// monitorRunning: test.fields.monitorRunning,
// stopMonitor: test.fields.stopMonitor,
@@ -866,6 +882,7 @@ package grpc
// gbo gbackoff.Config
// mcd time.Duration
// group singleflight.Group[pool.Conn]
+// crl sync.Map[string, bool]
// ech <-chan error
// monitorRunning atomic.Bool
// stopMonitor context.CancelFunc
@@ -918,6 +935,7 @@ package grpc
// gbo:nil,
// mcd:nil,
// group:nil,
+// crl:nil,
// ech:nil,
// monitorRunning:nil,
// stopMonitor:nil,
@@ -963,6 +981,7 @@ package grpc
// gbo:nil,
// mcd:nil,
// group:nil,
+// crl:nil,
// ech:nil,
// monitorRunning:nil,
// stopMonitor:nil,
@@ -1014,6 +1033,7 @@ package grpc
// gbo: test.fields.gbo,
// mcd: test.fields.mcd,
// group: test.fields.group,
+// crl: test.fields.crl,
// ech: test.fields.ech,
// monitorRunning: test.fields.monitorRunning,
// stopMonitor: test.fields.stopMonitor,
@@ -1052,6 +1072,7 @@ package grpc
// gbo gbackoff.Config
// mcd time.Duration
// group singleflight.Group[pool.Conn]
+// crl sync.Map[string, bool]
// ech <-chan error
// monitorRunning atomic.Bool
// stopMonitor context.CancelFunc
@@ -1106,6 +1127,7 @@ package grpc
// gbo:nil,
// mcd:nil,
// group:nil,
+// crl:nil,
// ech:nil,
// monitorRunning:nil,
// stopMonitor:nil,
@@ -1149,6 +1171,7 @@ package grpc
// gbo:nil,
// mcd:nil,
// group:nil,
+// crl:nil,
// ech:nil,
// monitorRunning:nil,
// stopMonitor:nil,
@@ -1200,6 +1223,7 @@ package grpc
// gbo: test.fields.gbo,
// mcd: test.fields.mcd,
// group: test.fields.group,
+// crl: test.fields.crl,
// ech: test.fields.ech,
// monitorRunning: test.fields.monitorRunning,
// stopMonitor: test.fields.stopMonitor,
@@ -1239,6 +1263,7 @@ package grpc
// gbo gbackoff.Config
// mcd time.Duration
// group singleflight.Group[pool.Conn]
+// crl sync.Map[string, bool]
// ech <-chan error
// monitorRunning atomic.Bool
// stopMonitor context.CancelFunc
@@ -1294,6 +1319,7 @@ package grpc
// gbo:nil,
// mcd:nil,
// group:nil,
+// crl:nil,
// ech:nil,
// monitorRunning:nil,
// stopMonitor:nil,
@@ -1338,6 +1364,7 @@ package grpc
// gbo:nil,
// mcd:nil,
// group:nil,
+// crl:nil,
// ech:nil,
// monitorRunning:nil,
// stopMonitor:nil,
@@ -1389,6 +1416,7 @@ package grpc
// gbo: test.fields.gbo,
// mcd: test.fields.mcd,
// group: test.fields.group,
+// crl: test.fields.crl,
// ech: test.fields.ech,
// monitorRunning: test.fields.monitorRunning,
// stopMonitor: test.fields.stopMonitor,
@@ -1430,6 +1458,7 @@ package grpc
// gbo gbackoff.Config
// mcd time.Duration
// group singleflight.Group[pool.Conn]
+// crl sync.Map[string, bool]
// ech <-chan error
// monitorRunning atomic.Bool
// stopMonitor context.CancelFunc
@@ -1487,6 +1516,7 @@ package grpc
// gbo:nil,
// mcd:nil,
// group:nil,
+// crl:nil,
// ech:nil,
// monitorRunning:nil,
// stopMonitor:nil,
@@ -1533,6 +1563,7 @@ package grpc
// gbo:nil,
// mcd:nil,
// group:nil,
+// crl:nil,
// ech:nil,
// monitorRunning:nil,
// stopMonitor:nil,
@@ -1584,6 +1615,7 @@ package grpc
// gbo: test.fields.gbo,
// mcd: test.fields.mcd,
// group: test.fields.group,
+// crl: test.fields.crl,
// ech: test.fields.ech,
// monitorRunning: test.fields.monitorRunning,
// stopMonitor: test.fields.stopMonitor,
@@ -1618,6 +1650,7 @@ package grpc
// gbo gbackoff.Config
// mcd time.Duration
// group singleflight.Group[pool.Conn]
+// crl sync.Map[string, bool]
// ech <-chan error
// monitorRunning atomic.Bool
// stopMonitor context.CancelFunc
@@ -1663,6 +1696,7 @@ package grpc
// gbo:nil,
// mcd:nil,
// group:nil,
+// crl:nil,
// ech:nil,
// monitorRunning:nil,
// stopMonitor:nil,
@@ -1702,6 +1736,7 @@ package grpc
// gbo:nil,
// mcd:nil,
// group:nil,
+// crl:nil,
// ech:nil,
// monitorRunning:nil,
// stopMonitor:nil,
@@ -1753,6 +1788,7 @@ package grpc
// gbo: test.fields.gbo,
// mcd: test.fields.mcd,
// group: test.fields.group,
+// crl: test.fields.crl,
// ech: test.fields.ech,
// monitorRunning: test.fields.monitorRunning,
// stopMonitor: test.fields.stopMonitor,
@@ -1787,6 +1823,7 @@ package grpc
// gbo gbackoff.Config
// mcd time.Duration
// group singleflight.Group[pool.Conn]
+// crl sync.Map[string, bool]
// ech <-chan error
// monitorRunning atomic.Bool
// stopMonitor context.CancelFunc
@@ -1832,6 +1869,7 @@ package grpc
// gbo:nil,
// mcd:nil,
// group:nil,
+// crl:nil,
// ech:nil,
// monitorRunning:nil,
// stopMonitor:nil,
@@ -1871,6 +1909,7 @@ package grpc
// gbo:nil,
// mcd:nil,
// group:nil,
+// crl:nil,
// ech:nil,
// monitorRunning:nil,
// stopMonitor:nil,
@@ -1922,6 +1961,7 @@ package grpc
// gbo: test.fields.gbo,
// mcd: test.fields.mcd,
// group: test.fields.group,
+// crl: test.fields.crl,
// ech: test.fields.ech,
// monitorRunning: test.fields.monitorRunning,
// stopMonitor: test.fields.stopMonitor,
@@ -1956,6 +1996,7 @@ package grpc
// gbo gbackoff.Config
// mcd time.Duration
// group singleflight.Group[pool.Conn]
+// crl sync.Map[string, bool]
// ech <-chan error
// monitorRunning atomic.Bool
// stopMonitor context.CancelFunc
@@ -2001,6 +2042,7 @@ package grpc
// gbo:nil,
// mcd:nil,
// group:nil,
+// crl:nil,
// ech:nil,
// monitorRunning:nil,
// stopMonitor:nil,
@@ -2040,6 +2082,7 @@ package grpc
// gbo:nil,
// mcd:nil,
// group:nil,
+// crl:nil,
// ech:nil,
// monitorRunning:nil,
// stopMonitor:nil,
@@ -2091,6 +2134,7 @@ package grpc
// gbo: test.fields.gbo,
// mcd: test.fields.mcd,
// group: test.fields.group,
+// crl: test.fields.crl,
// ech: test.fields.ech,
// monitorRunning: test.fields.monitorRunning,
// stopMonitor: test.fields.stopMonitor,
@@ -2130,6 +2174,7 @@ package grpc
// gbo gbackoff.Config
// mcd time.Duration
// group singleflight.Group[pool.Conn]
+// crl sync.Map[string, bool]
// ech <-chan error
// monitorRunning atomic.Bool
// stopMonitor context.CancelFunc
@@ -2185,6 +2230,7 @@ package grpc
// gbo:nil,
// mcd:nil,
// group:nil,
+// crl:nil,
// ech:nil,
// monitorRunning:nil,
// stopMonitor:nil,
@@ -2229,6 +2275,7 @@ package grpc
// gbo:nil,
// mcd:nil,
// group:nil,
+// crl:nil,
// ech:nil,
// monitorRunning:nil,
// stopMonitor:nil,
@@ -2280,6 +2327,7 @@ package grpc
// gbo: test.fields.gbo,
// mcd: test.fields.mcd,
// group: test.fields.group,
+// crl: test.fields.crl,
// ech: test.fields.ech,
// monitorRunning: test.fields.monitorRunning,
// stopMonitor: test.fields.stopMonitor,
@@ -2318,6 +2366,7 @@ package grpc
// gbo gbackoff.Config
// mcd time.Duration
// group singleflight.Group[pool.Conn]
+// crl sync.Map[string, bool]
// ech <-chan error
// monitorRunning atomic.Bool
// stopMonitor context.CancelFunc
@@ -2368,6 +2417,7 @@ package grpc
// gbo:nil,
// mcd:nil,
// group:nil,
+// crl:nil,
// ech:nil,
// monitorRunning:nil,
// stopMonitor:nil,
@@ -2411,6 +2461,7 @@ package grpc
// gbo:nil,
// mcd:nil,
// group:nil,
+// crl:nil,
// ech:nil,
// monitorRunning:nil,
// stopMonitor:nil,
@@ -2462,6 +2513,7 @@ package grpc
// gbo: test.fields.gbo,
// mcd: test.fields.mcd,
// group: test.fields.group,
+// crl: test.fields.crl,
// ech: test.fields.ech,
// monitorRunning: test.fields.monitorRunning,
// stopMonitor: test.fields.stopMonitor,
@@ -2500,6 +2552,7 @@ package grpc
// gbo gbackoff.Config
// mcd time.Duration
// group singleflight.Group[pool.Conn]
+// crl sync.Map[string, bool]
// ech <-chan error
// monitorRunning atomic.Bool
// stopMonitor context.CancelFunc
@@ -2550,6 +2603,7 @@ package grpc
// gbo:nil,
// mcd:nil,
// group:nil,
+// crl:nil,
// ech:nil,
// monitorRunning:nil,
// stopMonitor:nil,
@@ -2593,6 +2647,7 @@ package grpc
// gbo:nil,
// mcd:nil,
// group:nil,
+// crl:nil,
// ech:nil,
// monitorRunning:nil,
// stopMonitor:nil,
@@ -2644,6 +2699,7 @@ package grpc
// gbo: test.fields.gbo,
// mcd: test.fields.mcd,
// group: test.fields.group,
+// crl: test.fields.crl,
// ech: test.fields.ech,
// monitorRunning: test.fields.monitorRunning,
// stopMonitor: test.fields.stopMonitor,
@@ -2678,6 +2734,7 @@ package grpc
// gbo gbackoff.Config
// mcd time.Duration
// group singleflight.Group[pool.Conn]
+// crl sync.Map[string, bool]
// ech <-chan error
// monitorRunning atomic.Bool
// stopMonitor context.CancelFunc
@@ -2723,6 +2780,7 @@ package grpc
// gbo:nil,
// mcd:nil,
// group:nil,
+// crl:nil,
// ech:nil,
// monitorRunning:nil,
// stopMonitor:nil,
@@ -2762,6 +2820,7 @@ package grpc
// gbo:nil,
// mcd:nil,
// group:nil,
+// crl:nil,
// ech:nil,
// monitorRunning:nil,
// stopMonitor:nil,
@@ -2813,6 +2872,7 @@ package grpc
// gbo: test.fields.gbo,
// mcd: test.fields.mcd,
// group: test.fields.group,
+// crl: test.fields.crl,
// ech: test.fields.ech,
// monitorRunning: test.fields.monitorRunning,
// stopMonitor: test.fields.stopMonitor,
@@ -2850,6 +2910,7 @@ package grpc
// gbo gbackoff.Config
// mcd time.Duration
// group singleflight.Group[pool.Conn]
+// crl sync.Map[string, bool]
// ech <-chan error
// monitorRunning atomic.Bool
// stopMonitor context.CancelFunc
@@ -2899,6 +2960,7 @@ package grpc
// gbo:nil,
// mcd:nil,
// group:nil,
+// crl:nil,
// ech:nil,
// monitorRunning:nil,
// stopMonitor:nil,
@@ -2941,6 +3003,7 @@ package grpc
// gbo:nil,
// mcd:nil,
// group:nil,
+// crl:nil,
// ech:nil,
// monitorRunning:nil,
// stopMonitor:nil,
@@ -2992,6 +3055,7 @@ package grpc
// gbo: test.fields.gbo,
// mcd: test.fields.mcd,
// group: test.fields.group,
+// crl: test.fields.crl,
// ech: test.fields.ech,
// monitorRunning: test.fields.monitorRunning,
// stopMonitor: test.fields.stopMonitor,
diff --git a/internal/net/grpc/grpcconns.go b/internal/net/grpc/grpcconns.go
index 955b83fed5b..4d19a994f7b 100644
--- a/internal/net/grpc/grpcconns.go
+++ b/internal/net/grpc/grpcconns.go
@@ -18,12 +18,12 @@
package grpc
import (
- "sync"
"sync/atomic"
"unsafe"
"github.com/vdaas/vald/internal/errors"
"github.com/vdaas/vald/internal/net/grpc/pool"
+ "github.com/vdaas/vald/internal/sync"
)
type grpcConns struct {
diff --git a/internal/net/grpc/logger/logger.go b/internal/net/grpc/logger/logger.go
index d8b81aa8a4f..72b311eab47 100644
--- a/internal/net/grpc/logger/logger.go
+++ b/internal/net/grpc/logger/logger.go
@@ -16,9 +16,9 @@ package logger
import (
"os"
"strconv"
- "sync"
"github.com/vdaas/vald/internal/log"
+ "github.com/vdaas/vald/internal/sync"
glog "google.golang.org/grpc/grpclog"
)
diff --git a/internal/net/grpc/logger/logger_test.go b/internal/net/grpc/logger/logger_test.go
index 6fac492791a..be230770fca 100644
--- a/internal/net/grpc/logger/logger_test.go
+++ b/internal/net/grpc/logger/logger_test.go
@@ -17,10 +17,10 @@ import (
"os"
"os/exec"
"reflect"
- "sync"
"testing"
"github.com/vdaas/vald/internal/errors"
+ "github.com/vdaas/vald/internal/sync"
"github.com/vdaas/vald/internal/test/goleak"
"google.golang.org/grpc/grpclog"
)
diff --git a/internal/net/grpc/option.go b/internal/net/grpc/option.go
index 14b90e9cb68..12e7fd6db09 100644
--- a/internal/net/grpc/option.go
+++ b/internal/net/grpc/option.go
@@ -24,11 +24,11 @@ import (
"github.com/vdaas/vald/internal/backoff"
"github.com/vdaas/vald/internal/circuitbreaker"
- "github.com/vdaas/vald/internal/errgroup"
"github.com/vdaas/vald/internal/log"
"github.com/vdaas/vald/internal/net"
"github.com/vdaas/vald/internal/net/grpc/interceptor/client/trace"
"github.com/vdaas/vald/internal/strings"
+ "github.com/vdaas/vald/internal/sync/errgroup"
"github.com/vdaas/vald/internal/timeutil"
"google.golang.org/grpc"
gbackoff "google.golang.org/grpc/backoff"
diff --git a/internal/net/grpc/pool/option.go b/internal/net/grpc/pool/option.go
index c544a6520dd..8126d195198 100644
--- a/internal/net/grpc/pool/option.go
+++ b/internal/net/grpc/pool/option.go
@@ -19,7 +19,7 @@ package pool
import (
"github.com/vdaas/vald/internal/backoff"
- "github.com/vdaas/vald/internal/errgroup"
+ "github.com/vdaas/vald/internal/sync/errgroup"
"github.com/vdaas/vald/internal/timeutil"
)
diff --git a/internal/net/grpc/pool/pool.go b/internal/net/grpc/pool/pool.go
index fba9f0d2678..942b8db6a98 100644
--- a/internal/net/grpc/pool/pool.go
+++ b/internal/net/grpc/pool/pool.go
@@ -22,18 +22,18 @@ import (
"fmt"
"math"
"strconv"
- "sync"
"sync/atomic"
"time"
"github.com/vdaas/vald/internal/backoff"
- "github.com/vdaas/vald/internal/errgroup"
"github.com/vdaas/vald/internal/errors"
"github.com/vdaas/vald/internal/log"
"github.com/vdaas/vald/internal/net"
"github.com/vdaas/vald/internal/safety"
"github.com/vdaas/vald/internal/slices"
"github.com/vdaas/vald/internal/strings"
+ "github.com/vdaas/vald/internal/sync"
+ "github.com/vdaas/vald/internal/sync/errgroup"
"google.golang.org/grpc"
"google.golang.org/grpc/connectivity"
"google.golang.org/grpc/credentials/insecure"
diff --git a/internal/net/grpc/pool/pool_bench_test.go b/internal/net/grpc/pool/pool_bench_test.go
index a8d7515c1f1..71f15499b43 100644
--- a/internal/net/grpc/pool/pool_bench_test.go
+++ b/internal/net/grpc/pool/pool_bench_test.go
@@ -19,7 +19,6 @@ package pool
import (
"context"
- "sync"
"testing"
"github.com/vdaas/vald/apis/grpc/v1/discoverer"
@@ -27,7 +26,7 @@ import (
"github.com/vdaas/vald/internal/log"
"github.com/vdaas/vald/internal/log/level"
"github.com/vdaas/vald/internal/net"
- valdsync "github.com/vdaas/vald/internal/sync"
+ "github.com/vdaas/vald/internal/sync"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials/insecure"
)
@@ -134,7 +133,7 @@ func Benchmark_StaticDial(b *testing.B) {
b.Error(err)
}
- conns := new(valdsync.Map[string, *grpc.ClientConn])
+ conns := new(sync.Map[string, *grpc.ClientConn])
conns.Store(DefaultServerAddr, conn)
b.StopTimer()
@@ -191,7 +190,7 @@ func BenchmarkParallel_StaticDial(b *testing.B) {
b.Error(err)
}
- conns := new(valdsync.Map[string, *grpc.ClientConn])
+ conns := new(sync.Map[string, *grpc.ClientConn])
conns.Store(DefaultServerAddr, conn)
b.StopTimer()
diff --git a/internal/net/grpc/stream.go b/internal/net/grpc/stream.go
index dcafda62bb5..0ed745916d2 100644
--- a/internal/net/grpc/stream.go
+++ b/internal/net/grpc/stream.go
@@ -21,10 +21,8 @@ import (
"context"
"fmt"
"runtime"
- "sync"
"sync/atomic"
- "github.com/vdaas/vald/internal/errgroup"
"github.com/vdaas/vald/internal/errors"
"github.com/vdaas/vald/internal/io"
"github.com/vdaas/vald/internal/log"
@@ -34,6 +32,8 @@ import (
"github.com/vdaas/vald/internal/observability/trace"
"github.com/vdaas/vald/internal/safety"
"github.com/vdaas/vald/internal/slices"
+ "github.com/vdaas/vald/internal/sync"
+ "github.com/vdaas/vald/internal/sync/errgroup"
"google.golang.org/grpc"
)
@@ -55,7 +55,7 @@ func BidirectionalStream[Q any, R any](ctx context.Context, stream ServerStream,
}()
eg, ctx := errgroup.New(ctx)
if concurrency > 0 {
- eg.Limitation(concurrency)
+ eg.SetLimit(concurrency)
}
var (
diff --git a/internal/net/http/client/client_test.go b/internal/net/http/client/client_test.go
index b248b72621e..ab43883056d 100644
--- a/internal/net/http/client/client_test.go
+++ b/internal/net/http/client/client_test.go
@@ -22,7 +22,6 @@ import (
"net/http"
"net/url"
"reflect"
- "sync"
"sync/atomic"
"testing"
@@ -32,6 +31,7 @@ import (
"github.com/vdaas/vald/internal/log/logger"
"github.com/vdaas/vald/internal/net"
htr "github.com/vdaas/vald/internal/net/http/transport"
+ "github.com/vdaas/vald/internal/sync"
"github.com/vdaas/vald/internal/test/comparator"
"github.com/vdaas/vald/internal/test/goleak"
"golang.org/x/net/http2"
diff --git a/internal/net/http/middleware/option.go b/internal/net/http/middleware/option.go
index cfb7750cc7f..150a2e7afec 100644
--- a/internal/net/http/middleware/option.go
+++ b/internal/net/http/middleware/option.go
@@ -20,7 +20,7 @@ package middleware
import (
"time"
- "github.com/vdaas/vald/internal/errgroup"
+ "github.com/vdaas/vald/internal/sync/errgroup"
"github.com/vdaas/vald/internal/timeutil"
)
diff --git a/internal/net/http/middleware/option_test.go b/internal/net/http/middleware/option_test.go
index 140c61b985d..eb132767c24 100644
--- a/internal/net/http/middleware/option_test.go
+++ b/internal/net/http/middleware/option_test.go
@@ -19,8 +19,8 @@ import (
"testing"
"time"
- "github.com/vdaas/vald/internal/errgroup"
"github.com/vdaas/vald/internal/errors"
+ "github.com/vdaas/vald/internal/sync/errgroup"
)
func TestWithErrorGroup(t *testing.T) {
diff --git a/internal/net/http/middleware/timeout.go b/internal/net/http/middleware/timeout.go
index 001f9fe917d..28af2307e82 100644
--- a/internal/net/http/middleware/timeout.go
+++ b/internal/net/http/middleware/timeout.go
@@ -24,10 +24,10 @@ import (
"time"
"github.com/kpango/fastime"
- "github.com/vdaas/vald/internal/errgroup"
"github.com/vdaas/vald/internal/errors"
"github.com/vdaas/vald/internal/net/http/rest"
"github.com/vdaas/vald/internal/safety"
+ "github.com/vdaas/vald/internal/sync/errgroup"
)
type timeout struct {
diff --git a/internal/net/http/middleware/timeout_test.go b/internal/net/http/middleware/timeout_test.go
index 56e661717ef..2ad64f48e62 100644
--- a/internal/net/http/middleware/timeout_test.go
+++ b/internal/net/http/middleware/timeout_test.go
@@ -22,10 +22,10 @@ import (
"testing"
"time"
- "github.com/vdaas/vald/internal/errgroup"
"github.com/vdaas/vald/internal/errors"
"github.com/vdaas/vald/internal/net/http/rest"
"github.com/vdaas/vald/internal/strings"
+ "github.com/vdaas/vald/internal/sync/errgroup"
)
func TestNewTimeout(t *testing.T) {
diff --git a/internal/net/net.go b/internal/net/net.go
index 58cb705a76b..3108954d406 100644
--- a/internal/net/net.go
+++ b/internal/net/net.go
@@ -23,14 +23,14 @@ import (
"net"
"net/netip"
"strconv"
- "sync"
"syscall"
- "github.com/vdaas/vald/internal/errgroup"
"github.com/vdaas/vald/internal/errors"
"github.com/vdaas/vald/internal/log"
"github.com/vdaas/vald/internal/safety"
"github.com/vdaas/vald/internal/strings"
+ "github.com/vdaas/vald/internal/sync"
+ "github.com/vdaas/vald/internal/sync/errgroup"
)
type (
@@ -234,7 +234,7 @@ func ScanPorts(ctx context.Context, start, end uint16, host string) (ports []uin
return nil, err
}
eg, egctx := errgroup.New(ctx)
- eg.Limitation(int(rl.Max) / 2)
+ eg.SetLimit(int(rl.Max) / 2)
var mu sync.Mutex
diff --git a/internal/observability/metrics/agent/sidecar/sidecar.go b/internal/observability/metrics/agent/sidecar/sidecar.go
index 34ec7934ea2..684a5897cfe 100644
--- a/internal/observability/metrics/agent/sidecar/sidecar.go
+++ b/internal/observability/metrics/agent/sidecar/sidecar.go
@@ -15,11 +15,11 @@ package sidecar
import (
"context"
- "sync"
"time"
"github.com/vdaas/vald/internal/observability/attribute"
"github.com/vdaas/vald/internal/observability/metrics"
+ "github.com/vdaas/vald/internal/sync"
"github.com/vdaas/vald/pkg/agent/sidecar/service/observer"
"go.opentelemetry.io/otel/sdk/metric/aggregation"
"go.opentelemetry.io/otel/sdk/metric/view"
diff --git a/internal/observability/observability.go b/internal/observability/observability.go
index f80fcb92d2d..3df322bf51c 100644
--- a/internal/observability/observability.go
+++ b/internal/observability/observability.go
@@ -18,7 +18,6 @@ import (
"reflect"
"github.com/vdaas/vald/internal/config"
- "github.com/vdaas/vald/internal/errgroup"
"github.com/vdaas/vald/internal/errors"
"github.com/vdaas/vald/internal/log"
"github.com/vdaas/vald/internal/observability/exporter"
@@ -30,6 +29,7 @@ import (
"github.com/vdaas/vald/internal/observability/metrics/runtime/goroutine"
"github.com/vdaas/vald/internal/observability/metrics/version"
"github.com/vdaas/vald/internal/observability/trace"
+ "github.com/vdaas/vald/internal/sync/errgroup"
)
type Observability interface {
diff --git a/internal/observability/option.go b/internal/observability/option.go
index aa9d1d5479a..e2246f578c8 100644
--- a/internal/observability/option.go
+++ b/internal/observability/option.go
@@ -14,11 +14,11 @@
package observability
import (
- "github.com/vdaas/vald/internal/errgroup"
"github.com/vdaas/vald/internal/errors"
"github.com/vdaas/vald/internal/observability/exporter"
"github.com/vdaas/vald/internal/observability/metrics"
"github.com/vdaas/vald/internal/observability/trace"
+ "github.com/vdaas/vald/internal/sync/errgroup"
)
type Option func(*observability) error
diff --git a/internal/rand/rand.go b/internal/rand/rand.go
index e0235cb695d..38f7e5172fc 100644
--- a/internal/rand/rand.go
+++ b/internal/rand/rand.go
@@ -18,10 +18,10 @@
package rand
import (
- "sync"
"sync/atomic"
"github.com/kpango/fastime"
+ "github.com/vdaas/vald/internal/sync"
)
type rand struct {
diff --git a/internal/rand/rand_test.go b/internal/rand/rand_test.go
index a1cd8cc2fa1..dc4155770e3 100644
--- a/internal/rand/rand_test.go
+++ b/internal/rand/rand_test.go
@@ -15,11 +15,11 @@ package rand
import (
"fmt"
- "sync"
"sync/atomic"
"testing"
"github.com/vdaas/vald/internal/errors"
+ "github.com/vdaas/vald/internal/sync"
"github.com/vdaas/vald/internal/test/goleak"
)
diff --git a/internal/runner/runner.go b/internal/runner/runner.go
index fd6da03e7b2..56ab49efc59 100644
--- a/internal/runner/runner.go
+++ b/internal/runner/runner.go
@@ -26,7 +26,6 @@ import (
"github.com/vdaas/vald/internal/config"
"github.com/vdaas/vald/internal/conv"
"github.com/vdaas/vald/internal/encoding/json"
- "github.com/vdaas/vald/internal/errgroup"
"github.com/vdaas/vald/internal/errors"
"github.com/vdaas/vald/internal/info"
"github.com/vdaas/vald/internal/log"
@@ -34,6 +33,7 @@ import (
"github.com/vdaas/vald/internal/params"
"github.com/vdaas/vald/internal/safety"
"github.com/vdaas/vald/internal/strings"
+ "github.com/vdaas/vald/internal/sync/errgroup"
"github.com/vdaas/vald/internal/timeutil/location"
ver "github.com/vdaas/vald/internal/version"
"go.uber.org/automaxprocs/maxprocs"
diff --git a/internal/servers/option.go b/internal/servers/option.go
index f6d1e88ca33..399a14cc7f4 100644
--- a/internal/servers/option.go
+++ b/internal/servers/option.go
@@ -20,8 +20,8 @@ package servers
import (
"time"
- "github.com/vdaas/vald/internal/errgroup"
"github.com/vdaas/vald/internal/servers/server"
+ "github.com/vdaas/vald/internal/sync/errgroup"
"github.com/vdaas/vald/internal/timeutil"
)
diff --git a/internal/servers/option_test.go b/internal/servers/option_test.go
index 46629861092..a80a4850e4e 100644
--- a/internal/servers/option_test.go
+++ b/internal/servers/option_test.go
@@ -19,9 +19,9 @@ import (
"testing"
"time"
- "github.com/vdaas/vald/internal/errgroup"
"github.com/vdaas/vald/internal/errors"
"github.com/vdaas/vald/internal/servers/server"
+ "github.com/vdaas/vald/internal/sync/errgroup"
)
func TestWithServer(t *testing.T) {
diff --git a/internal/servers/server/option.go b/internal/servers/server/option.go
index ac0e44053b2..7d94220ad2c 100644
--- a/internal/servers/server/option.go
+++ b/internal/servers/server/option.go
@@ -23,7 +23,6 @@ import (
"net/http"
"time"
- "github.com/vdaas/vald/internal/errgroup"
"github.com/vdaas/vald/internal/errors"
"github.com/vdaas/vald/internal/info"
"github.com/vdaas/vald/internal/log"
@@ -36,6 +35,7 @@ import (
"github.com/vdaas/vald/internal/net/grpc/interceptor/server/trace"
"github.com/vdaas/vald/internal/net/http/rest"
"github.com/vdaas/vald/internal/strings"
+ "github.com/vdaas/vald/internal/sync/errgroup"
"github.com/vdaas/vald/internal/timeutil"
)
diff --git a/internal/servers/server/option_test.go b/internal/servers/server/option_test.go
index 721603813cc..3d6de488804 100644
--- a/internal/servers/server/option_test.go
+++ b/internal/servers/server/option_test.go
@@ -20,10 +20,10 @@ import (
"testing"
"time"
- "github.com/vdaas/vald/internal/errgroup"
"github.com/vdaas/vald/internal/errors"
"github.com/vdaas/vald/internal/net"
"github.com/vdaas/vald/internal/net/grpc"
+ "github.com/vdaas/vald/internal/sync/errgroup"
)
func TestWithHost(t *testing.T) {
diff --git a/internal/servers/server/server.go b/internal/servers/server/server.go
index 8e32ea7debf..3b7a7939a15 100644
--- a/internal/servers/server/server.go
+++ b/internal/servers/server/server.go
@@ -24,11 +24,9 @@ import (
"os"
"reflect"
"strconv"
- "sync"
"syscall"
"time"
- "github.com/vdaas/vald/internal/errgroup"
"github.com/vdaas/vald/internal/errors"
"github.com/vdaas/vald/internal/log"
"github.com/vdaas/vald/internal/net"
@@ -39,6 +37,8 @@ import (
glog "github.com/vdaas/vald/internal/net/grpc/logger"
"github.com/vdaas/vald/internal/safety"
"github.com/vdaas/vald/internal/strings"
+ "github.com/vdaas/vald/internal/sync"
+ "github.com/vdaas/vald/internal/sync/errgroup"
)
type Server interface {
diff --git a/internal/servers/server/server_test.go b/internal/servers/server/server_test.go
index bedb6028c83..2b7e67d2caf 100644
--- a/internal/servers/server/server_test.go
+++ b/internal/servers/server/server_test.go
@@ -21,12 +21,12 @@ import (
"testing"
"time"
- "github.com/vdaas/vald/internal/errgroup"
"github.com/vdaas/vald/internal/errors"
"github.com/vdaas/vald/internal/log"
"github.com/vdaas/vald/internal/log/logger"
"github.com/vdaas/vald/internal/net"
"github.com/vdaas/vald/internal/net/grpc"
+ "github.com/vdaas/vald/internal/sync/errgroup"
)
func TestServerMode_String(t *testing.T) {
diff --git a/internal/servers/servers.go b/internal/servers/servers.go
index 0416d174775..6dd625567a6 100644
--- a/internal/servers/servers.go
+++ b/internal/servers/servers.go
@@ -23,11 +23,11 @@ import (
"sort"
"time"
- "github.com/vdaas/vald/internal/errgroup"
"github.com/vdaas/vald/internal/errors"
"github.com/vdaas/vald/internal/safety"
"github.com/vdaas/vald/internal/servers/server"
"github.com/vdaas/vald/internal/strings"
+ "github.com/vdaas/vald/internal/sync/errgroup"
)
type Listener interface {
diff --git a/internal/servers/servers_test.go b/internal/servers/servers_test.go
index 3bf383844d8..5c3f9aa419f 100644
--- a/internal/servers/servers_test.go
+++ b/internal/servers/servers_test.go
@@ -19,9 +19,9 @@ import (
"testing"
"time"
- "github.com/vdaas/vald/internal/errgroup"
"github.com/vdaas/vald/internal/errors"
"github.com/vdaas/vald/internal/servers/server"
+ "github.com/vdaas/vald/internal/sync/errgroup"
)
func TestNew(t *testing.T) {
diff --git a/internal/singleflight/singleflight.go b/internal/singleflight/singleflight.go
deleted file mode 100644
index 94e72b149cc..00000000000
--- a/internal/singleflight/singleflight.go
+++ /dev/null
@@ -1,68 +0,0 @@
-//
-// Copyright (C) 2019-2023 vdaas.org vald team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// https://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-// Package singleflight represents zero time caching.
-package singleflight
-
-import (
- "context"
- "sync"
- "sync/atomic"
-
- valdsync "github.com/vdaas/vald/internal/sync"
-)
-
-type call[V any] struct {
- wg sync.WaitGroup
- val V
- err error
- dups uint64
-}
-
-// Group represents interface for zero time cache.
-type Group[V any] interface {
- Do(ctx context.Context, key string, fn func() (V, error)) (v V, shared bool, err error)
-}
-
-type group[V any] struct {
- m valdsync.Map[string, *call[V]]
-}
-
-// New returns Group implementation.
-func New[V any]() Group[V] {
- return new(group[V])
-}
-
-// Do execute the given function and return the result.
-// It makes sure only one execution of the function for each given key.
-// If duplicate comes, the duplicated call with the same key will wait for the first caller return.
-// It returns the result and the error of the given function, and whether the result is shared from the first caller.
-func (g *group[V]) Do(_ context.Context, key string, fn func() (V, error)) (v V, shared bool, err error) {
- c, loaded := g.m.LoadOrStore(key, new(call[V]))
- if loaded {
- atomic.AddUint64(&c.dups, 1)
- c.wg.Wait()
- v, err = c.val, c.err
- return v, true, err
- }
- c.wg.Add(1)
- c.val, c.err = fn()
- c.wg.Done()
-
- g.m.LoadAndDelete(key)
-
- return c.val, atomic.LoadUint64(&c.dups) > 0, c.err
-}
diff --git a/internal/singleflight/singleflight_test.go b/internal/singleflight/singleflight_test.go
deleted file mode 100644
index b283db915f2..00000000000
--- a/internal/singleflight/singleflight_test.go
+++ /dev/null
@@ -1,294 +0,0 @@
-//go:build !race
-
-//
-// Copyright (C) 2019-2023 vdaas.org vald team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// https://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-// Package singleflight represents zero time caching
-package singleflight
-
-import (
- "context"
- "reflect"
- "sync"
- "sync/atomic"
- "testing"
- "time"
-
- "github.com/vdaas/vald/internal/errors"
- "github.com/vdaas/vald/internal/test/goleak"
-)
-
-func TestNew(t *testing.T) {
- type want struct {
- want Group[any]
- }
- type test struct {
- name string
- want want
- checkFunc func(want, Group[any]) error
- beforeFunc func()
- afterFunc func()
- }
- defaultCheckFunc := func(w want, got Group[any]) error {
- if !reflect.DeepEqual(got, w.want) {
- return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", got, w.want)
- }
- return nil
- }
- tests := []test{
- {
- name: "returns Group implementation",
- want: want{
- want: &group[any]{},
- },
- },
- }
-
- for _, tc := range tests {
- test := tc
- t.Run(test.name, func(tt *testing.T) {
- defer goleak.VerifyNone(tt)
- if test.beforeFunc != nil {
- test.beforeFunc()
- }
- if test.afterFunc != nil {
- defer test.afterFunc()
- }
- checkFunc := test.checkFunc
- if test.checkFunc == nil {
- checkFunc = defaultCheckFunc
- }
-
- got := New[any]()
- if err := checkFunc(test.want, got); err != nil {
- tt.Errorf("error = %v", err)
- }
- })
- }
-}
-
-func Test_group_Do(t *testing.T) {
- type args[V any] struct {
- ctx context.Context
- key string
- fn func() (V, error)
- }
- type want[V any] struct {
- wantV V
- wantShared bool
- err error
- }
- type test[V any] struct {
- name string
- args args[V]
- want want[V]
- beforeFunc func(args[V])
- execFunc func(*testing.T, args[V]) (V, bool, error)
- checkFunc func(want[V], V, bool, error) error
- afterFunc func(args[V])
- }
- tests := []test[string]{
- func() test[string] {
- // routine1
- key1 := "req_1"
- var cnt1 uint32
-
- // the unparam lint rule is disabled here because we need to match the interface to singleflight implementation.
- // if this rule is not disabled, if will warns that the error will always return null.
- //nolint:unparam
- fn1 := func() (string, error) {
- atomic.AddUint32(&cnt1, 1)
- return "res_1", nil
- }
-
- // routine 2
- key2 := "req_2"
- var cnt2 uint32
-
- // the unparam lint rule is disabled here because we need to match the interface to singleflight implementation.
- // if this rule is not disabled, if will warns that the error will always return null.
- //nolint:unparam
- fn2 := func() (string, error) {
- atomic.AddUint32(&cnt2, 1)
- return "res_2", nil
- }
-
- return test[string]{
- name: "returns (v, false, nil) when Do is called with another key",
- args: args[string]{
- key: key1,
- ctx: context.Background(),
- fn: fn1,
- },
- want: want[string]{
- wantV: "res_1",
- wantShared: false,
- err: nil,
- },
- execFunc: func(t *testing.T, a args[string]) (got string, gotShared bool, err error) {
- t.Helper()
- g := New[string]()
-
- wg := new(sync.WaitGroup)
- wg.Add(1)
- go func() {
- got, gotShared, err = g.Do(a.ctx, a.key, a.fn)
- wg.Done()
- }()
-
- wg.Add(1)
- go func() {
- _, _, _ = g.Do(a.ctx, key2, fn2)
- wg.Done()
- }()
-
- wg.Wait()
- return got, gotShared, err
- },
- checkFunc: func(w want[string], gotV string, gotShared bool, err error) error {
- if got, want := int(atomic.LoadUint32(&cnt1)), 1; got != want {
- return errors.Errorf("cnt got = %d, want = %d", got, want)
- }
- if got, want := int(atomic.LoadUint32(&cnt2)), 1; got != want {
- return errors.Errorf("cnt got = %d, want = %d", got, want)
- }
- if !errors.Is(err, w.err) {
- return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err)
- }
- if !reflect.DeepEqual(gotV, w.wantV) {
- return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", gotV, w.wantV)
- }
- if !reflect.DeepEqual(gotShared, w.wantShared) {
- return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", gotShared, w.wantShared)
- }
- return nil
- },
- }
- }(),
- func() test[string] {
- // routine1
- var cnt1 uint32
-
- // the unparam lint rule is disabled here because we need to match the interface to singleflight implementation.
- // if this rule is not disabled, if will warns that the error will always return null.
- //nolint:unparam
- fn1 := func() (string, error) {
- atomic.AddUint32(&cnt1, 1)
- time.Sleep(time.Millisecond * 500)
- return "res_1", nil
- }
-
- // routine 2
- var cnt2 uint32
-
- // the unparam lint rule is disabled here because we need to match the interface to singleflight implementation.
- // if this rule is not disabled, if will warns that the error will always return null.
- //nolint:unparam
- fn2 := func() (string, error) {
- atomic.AddUint32(&cnt2, 1)
- return "res_2", nil
- }
-
- w := want[string]{
- wantV: "res_1",
- wantShared: true,
- err: nil,
- }
-
- checkFunc := func(w want[string], gotV string, gotShared bool, err error) error {
- c1 := int(atomic.LoadUint32(&cnt1))
- c2 := int(atomic.LoadUint32(&cnt2))
- // since there is a chance that the go routine 2 is executed before routine 1, we need to check if either one is executed
- if !((c1 == 1 && c2 == 0) || (c1 == 0 && c2 == 1)) {
- return errors.Errorf("cnt1 and cnt2 is executed, %d, %d", c1, c2)
- }
- if !errors.Is(err, w.err) {
- return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err)
- }
- if !reflect.DeepEqual(gotV, w.wantV) {
- return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", gotV, w.wantV)
- }
- if !reflect.DeepEqual(gotShared, w.wantShared) {
- return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", gotShared, w.wantShared)
- }
- return nil
- }
-
- return test[string]{
- name: "returns (v, true, nil) when Do is called with the same key",
- args: args[string]{
- key: "req_1",
- ctx: context.Background(),
- fn: fn1,
- },
- want: w,
- execFunc: func(t *testing.T, a args[string]) (string, bool, error) {
- t.Helper()
-
- g := New[string]()
- wg := new(sync.WaitGroup)
- var got, got1 string
- var gotShared, gotShared1 bool
- var err, err1 error
-
- wg.Add(1)
- go func() {
- got, gotShared, err = g.Do(a.ctx, a.key, fn1)
- wg.Done()
- }()
-
- // call with the same key but with another function
- wg.Add(1)
- time.Sleep(time.Millisecond * 100)
- go func() {
- got1, gotShared1, err1 = g.Do(a.ctx, a.key, fn2)
- wg.Done()
- }()
-
- wg.Wait()
-
- if err := checkFunc(w, got1, gotShared1, err1); err != nil {
- t.Fatal(err)
- }
-
- return got, gotShared, err
- },
- checkFunc: checkFunc,
- }
- }(),
- }
-
- for _, tc := range tests {
- test := tc
- t.Run(test.name, func(tt *testing.T) {
- defer goleak.VerifyNone(tt)
- if test.afterFunc != nil {
- defer test.afterFunc(test.args)
- }
- if test.beforeFunc != nil {
- test.beforeFunc(test.args)
- }
-
- gotV, gotShared, err := test.execFunc(t, test.args)
-
- if err := test.checkFunc(test.want, gotV, gotShared, err); err != nil {
- tt.Errorf("error = %v", err)
- }
- })
- }
-}
-
-// NOT IMPLEMENTED BELOW
diff --git a/internal/strings/strings.go b/internal/strings/strings.go
index da4a76f9df2..2031fdefe86 100644
--- a/internal/strings/strings.go
+++ b/internal/strings/strings.go
@@ -16,7 +16,8 @@ package strings
import (
"bytes"
"strings"
- "sync"
+
+ "github.com/vdaas/vald/internal/sync"
)
type (
diff --git a/internal/sync/alias.go b/internal/sync/alias.go
new file mode 100644
index 00000000000..238935dcfb6
--- /dev/null
+++ b/internal/sync/alias.go
@@ -0,0 +1,28 @@
+// Copyright (C) 2019-2023 vdaas.org vald team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package sync
+
+import "sync"
+
+type (
+ Cond = sync.Cond
+ Locker = sync.Locker
+ Mutex = sync.Mutex
+ RWMutex = sync.RWMutex
+ Once = sync.Once
+ Pool = sync.Pool
+ WaitGroup = sync.WaitGroup
+)
+
+var NewCond = sync.NewCond
diff --git a/internal/sync/errgroup/errgroup_test.go b/internal/sync/errgroup/errgroup_test.go
new file mode 100644
index 00000000000..be65578be8c
--- /dev/null
+++ b/internal/sync/errgroup/errgroup_test.go
@@ -0,0 +1,291 @@
+//
+// Copyright (c) 2009-2023 The Go Authors. All rights resered.
+// Modified by vdaas.org vald team
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+// * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+//
+
+package errgroup_test
+
+import (
+ "context"
+ "fmt"
+ "net/http"
+ "os"
+ "sync/atomic"
+ "testing"
+ "time"
+
+ "github.com/vdaas/vald/internal/errors"
+ "github.com/vdaas/vald/internal/sync/errgroup"
+)
+
+var (
+ Web = fakeSearch("web")
+ Image = fakeSearch("image")
+ Video = fakeSearch("video")
+)
+
+type (
+ Result string
+ Search func(ctx context.Context, query string) (Result, error)
+)
+
+func fakeSearch(kind string) Search {
+ return func(_ context.Context, query string) (Result, error) {
+ return Result(fmt.Sprintf("%s result for %q", kind, query)), nil
+ }
+}
+
+// JustErrors illustrates the use of a Group in place of a sync.WaitGroup to
+// simplify goroutine counting and error handling. This example is derived from
+// the sync.WaitGroup example at https://golang.org/pkg/sync/#example_WaitGroup.
+func ExampleGroup_justErrors() {
+ g, _ := errgroup.New(context.Background())
+ urls := []string{
+ "http://www.golang.org/",
+ "http://www.google.com/",
+ "http://www.somestupidname.com/",
+ }
+ for _, url := range urls {
+ // Launch a goroutine to fetch the URL.
+ url := url // https://golang.org/doc/faq#closures_and_goroutines
+ g.Go(func() error {
+ // Fetch the URL.
+ resp, err := http.Get(url)
+ if err == nil {
+ resp.Body.Close()
+ }
+ return err
+ })
+ }
+ // Wait for all HTTP fetches to complete.
+ if err := g.Wait(); err == nil {
+ fmt.Println("Successfully fetched all URLs.")
+ }
+}
+
+// Parallel illustrates the use of a Group for synchronizing a simple parallel
+// task: the "Google Search 2.0" function from
+// https://talks.golang.org/2012/concurrency.slide#46, augmented with a Context
+// and error-handling.
+func ExampleGroup_parallel() {
+ Google := func(ctx context.Context, query string) ([]Result, error) {
+ g, ctx := errgroup.WithContext(ctx)
+
+ searches := []Search{Web, Image, Video}
+ results := make([]Result, len(searches))
+ for i, search := range searches {
+ i, search := i, search // https://golang.org/doc/faq#closures_and_goroutines
+ g.Go(func() error {
+ result, err := search(ctx, query)
+ if err == nil {
+ results[i] = result
+ }
+ return err
+ })
+ }
+ if err := g.Wait(); err != nil {
+ return nil, err
+ }
+ return results, nil
+ }
+
+ results, err := Google(context.Background(), "golang")
+ if err != nil {
+ fmt.Fprintln(os.Stderr, err)
+ return
+ }
+ for _, result := range results {
+ fmt.Println(result)
+ }
+
+ // Output:
+ // web result for "golang"
+ // image result for "golang"
+ // video result for "golang"
+}
+
+func TestZeroGroup(t *testing.T) {
+ err1 := errors.New("errgroup_test: 1")
+ err2 := errors.New("errgroup_test: 2")
+
+ cases := []struct {
+ errs []error
+ }{
+ {errs: []error{}},
+ {errs: []error{nil}},
+ {errs: []error{err1}},
+ {errs: []error{err1, nil}},
+ {errs: []error{err1, nil, err2}},
+ }
+
+ for _, tc := range cases {
+ g, _ := errgroup.New(context.Background())
+
+ var firstErr error
+ for i, err := range tc.errs {
+ err := err
+ g.Go(func() error { return err })
+
+ if firstErr == nil && err != nil {
+ firstErr = err
+ }
+
+ if gErr := g.Wait(); !errors.Is(gErr, firstErr) {
+ t.Errorf("after %T.Go(func() error { return err }) for err in %v\n"+
+ "g.Wait() = %v; want %v",
+ g, tc.errs[:i+1], err, firstErr)
+ }
+ }
+ }
+}
+
+func TestWithContext(t *testing.T) {
+ errDoom := errors.New("group_test: doomed")
+
+ cases := []struct {
+ errs []error
+ want error
+ }{
+ {want: nil},
+ {errs: []error{nil}, want: nil},
+ {errs: []error{errDoom}, want: errDoom},
+ {errs: []error{errDoom, nil}, want: errDoom},
+ }
+
+ for _, tc := range cases {
+ g, ctx := errgroup.WithContext(context.Background())
+
+ for _, err := range tc.errs {
+ err := err
+ g.Go(func() error { return err })
+ }
+
+ if err := g.Wait(); err != tc.want {
+ t.Errorf("after %T.Go(func() error { return err }) for err in %v\n"+
+ "g.Wait() = %v; want %v",
+ g, tc.errs, err, tc.want)
+ }
+
+ canceled := false
+ select {
+ case <-ctx.Done():
+ canceled = true
+ default:
+ }
+ if !canceled {
+ t.Errorf("after %T.Go(func() error { return err }) for err in %v\n"+
+ "ctx.Done() was not closed",
+ g, tc.errs)
+ }
+ }
+}
+
+func TestTryGo(t *testing.T) {
+ g, _ := errgroup.New(context.Background())
+ n := 42
+ g.SetLimit(42)
+ ch := make(chan struct{})
+ fn := func() error {
+ ch <- struct{}{}
+ return nil
+ }
+ for i := 0; i < n; i++ {
+ if !g.TryGo(fn) {
+ t.Fatalf("TryGo should succeed but got fail at %d-th call.", i)
+ }
+ }
+ if g.TryGo(fn) {
+ t.Fatalf("TryGo is expected to fail but succeeded.")
+ }
+ go func() {
+ for i := 0; i < n; i++ {
+ <-ch
+ }
+ }()
+ g.Wait()
+
+ if !g.TryGo(fn) {
+ t.Fatalf("TryGo should success but got fail after all goroutines.")
+ }
+ go func() { <-ch }()
+ g.Wait()
+
+ // Switch limit.
+ g.SetLimit(1)
+ if !g.TryGo(fn) {
+ t.Fatalf("TryGo should success but got failed.")
+ }
+ if g.TryGo(fn) {
+ t.Fatalf("TryGo should fail but succeeded.")
+ }
+ go func() { <-ch }()
+ g.Wait()
+
+ // Block all calls.
+ g.SetLimit(0)
+ for i := 0; i < 1<<10; i++ {
+ if g.TryGo(fn) {
+ t.Fatalf("TryGo should fail but got succeded.")
+ }
+ }
+ g.Wait()
+}
+
+func TestGoLimit(t *testing.T) {
+ const limit = 10
+
+ g, _ := errgroup.New(context.Background())
+ g.SetLimit(limit)
+ var active int32
+ for i := 0; i <= 1<<10; i++ {
+ g.Go(func() error {
+ n := atomic.AddInt32(&active, 1)
+ if n > limit {
+ return fmt.Errorf("saw %d active goroutines; want ≤ %d", n, limit)
+ }
+ time.Sleep(1 * time.Microsecond) // Give other goroutines a chance to increment active.
+ atomic.AddInt32(&active, -1)
+ return nil
+ })
+ }
+ if err := g.Wait(); err != nil {
+ t.Fatal(err)
+ }
+}
+
+func BenchmarkGo(b *testing.B) {
+ fn := func() {}
+ g, _ := errgroup.New(context.Background())
+ b.ResetTimer()
+ b.ReportAllocs()
+ for i := 0; i < b.N; i++ {
+ g.Go(func() error { fn(); return nil })
+ }
+ g.Wait()
+}
diff --git a/internal/sync/errgroup/group.go b/internal/sync/errgroup/group.go
new file mode 100644
index 00000000000..7a4850c051f
--- /dev/null
+++ b/internal/sync/errgroup/group.go
@@ -0,0 +1,220 @@
+//
+// Copyright (C) 2019-2023 vdaas.org vald team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+// Package errgroup provides server global wait group for graceful kill all goroutine
+package errgroup
+
+import (
+ "context"
+ "runtime"
+
+ "github.com/vdaas/vald/internal/sync"
+
+ "github.com/vdaas/vald/internal/errors"
+ "github.com/vdaas/vald/internal/sync/semaphore"
+)
+
+// A Group is a collection of goroutines working on subtasks that are part of
+// the same overall task.
+//
+// A zero Group is valid, has no limit on the number of active goroutines,
+// and does not cancel on error.
+type Group interface {
+ Go(func() error)
+ SetLimit(limit int)
+ TryGo(func() error) bool
+ Wait() error
+}
+
+type group struct {
+ egctx context.Context
+ cancel context.CancelCauseFunc
+
+ wg sync.WaitGroup
+
+ sem *semaphore.Weighted
+
+ cancelOnce sync.Once
+ mu sync.RWMutex
+ emap map[string]struct{}
+ errs []error
+}
+
+var (
+ instance Group
+ once sync.Once
+)
+
+func New(ctx context.Context) (Group, context.Context) {
+ g := &group{emap: make(map[string]struct{})}
+ g.egctx, g.cancel = context.WithCancelCause(ctx)
+ return g, g.egctx
+}
+
+// WithContext returns a new Group and an associated Context derived from ctx.
+//
+// The derived Context is canceled the first time a function passed to Go
+// returns a non-nil error or the first time Wait returns, whichever occurs
+// first.
+func WithContext(ctx context.Context) (Group, context.Context) {
+ return New(ctx)
+}
+
+func Init(ctx context.Context) (egctx context.Context) {
+ egctx = ctx
+ once.Do(func() {
+ instance, egctx = New(ctx)
+ })
+ return
+}
+
+func Get() Group {
+ if instance == nil {
+ Init(context.Background())
+ }
+ return instance
+}
+
+func Go(f func() error) {
+ instance.Go(f)
+}
+
+func TryGo(f func() error) bool {
+ return instance.TryGo(f)
+}
+
+// SetLimit limits the number of active goroutines in this group to at most n.
+// A negative value indicates no limit.
+//
+// Any subsequent call to the Go method will block until it can add an active
+// goroutine without exceeding the configured limit.
+//
+// The limit must not be modified while any goroutines in the group are active.
+func (g *group) SetLimit(limit int) {
+ if limit < 0 {
+ g.sem = nil
+ return
+ }
+
+ if g.sem == nil {
+ g.sem = semaphore.NewWeighted(int64(limit))
+ } else {
+ g.sem.Resize(int64(limit))
+ }
+}
+
+// Go calls the given function in a new goroutine.
+// It blocks until the new goroutine can be added without the number of
+// active goroutines in the group exceeding the configured limit.
+//
+// The first call to return a non-nil error cancels the group's context, if the
+// group was created by calling WithContext. The error will be returned by Wait.
+func (g *group) Go(f func() error) {
+ if f == nil {
+ return
+ }
+ if g.sem != nil {
+ err := g.sem.Acquire(g.egctx, 1)
+ if err != nil {
+ if !errors.Is(err, context.Canceled) &&
+ !errors.Is(err, context.DeadlineExceeded) {
+ g.appendErr(err)
+ }
+ return
+ }
+ }
+ g.run(f)
+}
+
+// TryGo calls the given function in a new goroutine only if the number of
+// active goroutines in the group is currently below the configured limit.
+//
+// The return value reports whether the goroutine was started.
+func (g *group) TryGo(f func() error) bool {
+ if f == nil {
+ return false
+ }
+ if g.sem != nil && !g.sem.TryAcquire(1) {
+ return false
+ }
+ g.run(f)
+ return true
+}
+
+func (g *group) run(f func() error) {
+ g.wg.Add(1)
+ go func() {
+ defer g.done()
+ err := f()
+ if err != nil {
+ if !errors.Is(err, context.Canceled) &&
+ !errors.Is(err, context.DeadlineExceeded) {
+ runtime.Gosched()
+ g.appendErr(err)
+ }
+ g.doCancel(err)
+ return
+ }
+ }()
+}
+
+func (g *group) appendErr(err error) {
+ g.mu.RLock()
+ _, ok := g.emap[err.Error()]
+ g.mu.RUnlock()
+ if !ok {
+ g.mu.Lock()
+ g.errs = append(g.errs, err)
+ g.emap[err.Error()] = struct{}{}
+ g.mu.Unlock()
+ }
+}
+
+func (g *group) done() {
+ defer g.wg.Done()
+ if g.sem != nil {
+ g.sem.Release(1)
+ }
+}
+
+func (g *group) doCancel(err error) {
+ g.cancelOnce.Do(func() {
+ if g.cancel != nil {
+ g.cancel(err)
+ }
+ })
+}
+
+func Wait() error {
+ return instance.Wait()
+}
+
+// Wait blocks until all function calls from the Go method have returned, then
+// returns the first non-nil error (if any) from them.
+func (g *group) Wait() (err error) {
+ g.wg.Wait()
+ g.doCancel(context.Canceled)
+ g.mu.RLock()
+ defer g.mu.RUnlock()
+ switch len(g.errs) {
+ case 0:
+ return nil
+ case 1:
+ return g.errs[0]
+ default:
+ return errors.Join(g.errs...)
+ }
+}
diff --git a/internal/errgroup/group_test.go b/internal/sync/errgroup/group_test.go
similarity index 53%
rename from internal/errgroup/group_test.go
rename to internal/sync/errgroup/group_test.go
index f4d89cc29fc..ecd22f85237 100644
--- a/internal/errgroup/group_test.go
+++ b/internal/sync/errgroup/group_test.go
@@ -21,7 +21,6 @@ import (
"context"
"reflect"
"runtime"
- "sync"
"sync/atomic"
"testing"
"time"
@@ -29,6 +28,8 @@ import (
"github.com/vdaas/vald/internal/errors"
"github.com/vdaas/vald/internal/log"
"github.com/vdaas/vald/internal/log/logger"
+ "github.com/vdaas/vald/internal/sync"
+ "github.com/vdaas/vald/internal/sync/semaphore"
"github.com/vdaas/vald/internal/test/goleak"
)
@@ -54,9 +55,7 @@ func TestNew(t *testing.T) {
afterFunc func(*testing.T, args)
}
defaultCheckFunc := func(w want, got Group, got1 context.Context) error {
- if got, want := got.(*group), w.want.(*group); !reflect.DeepEqual(got.emap, want.emap) &&
- !reflect.DeepEqual(got.enableLimitation, want.enableLimitation) &&
- got.cancel != nil {
+ if got, want := got.(*group), w.want.(*group); !reflect.DeepEqual(got.emap, want.emap) && got.cancel != nil {
return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", got, w.want)
}
if !reflect.DeepEqual(got1, w.want1) {
@@ -67,7 +66,7 @@ func TestNew(t *testing.T) {
tests := []test{
func() test {
ctx := context.Background()
- egctx, cancel := context.WithCancel(ctx)
+ egctx, cancel := context.WithCancelCause(ctx)
return test{
name: "returns (g, ctx)",
@@ -78,11 +77,7 @@ func TestNew(t *testing.T) {
want: &group{
egctx: egctx,
cancel: cancel,
- enableLimitation: func() (el atomic.Bool) {
- el.Store(false)
- return
- }(),
- emap: make(map[string]struct{}),
+ emap: make(map[string]struct{}),
},
want1: egctx,
},
@@ -142,7 +137,7 @@ func TestInit(t *testing.T) {
tests := []test{
func() test {
ctx := context.Background()
- egctx, cancel := context.WithCancel(ctx)
+ egctx, cancel := context.WithCancelCause(ctx)
return test{
name: "returns egctx when once.Do is called",
@@ -154,7 +149,7 @@ func TestInit(t *testing.T) {
},
afterFunc: func(t *testing.T, a args) {
t.Helper()
- cancel()
+ cancel(context.Canceled)
defaultBeforeFunc(a)
},
}
@@ -217,7 +212,7 @@ func TestGet(t *testing.T) {
tests := []test{
func() test {
ctx := context.Background()
- egctx, cancel := context.WithCancel(ctx)
+ egctx, cancel := context.WithCancelCause(ctx)
return test{
name: "returns instance when instance is nil",
@@ -302,7 +297,6 @@ func TestGo(t *testing.T) {
beforeFunc: func(t *testing.T, _ args) {
t.Helper()
g := new(group)
- g.enableLimitation.Store(false)
instance = g
},
checkFunc: func(got Group) error {
@@ -341,13 +335,12 @@ func TestGo(t *testing.T) {
}
}
-func Test_group_Limitation(t *testing.T) {
+func Test_group_SetLimit(t *testing.T) {
type args struct {
limit int
}
type fields struct {
- limitation chan struct{}
- enableLimitation atomic.Bool
+ sem *semaphore.Weighted
}
type want struct {
want Group
@@ -363,27 +356,29 @@ func Test_group_Limitation(t *testing.T) {
}
defaultCheckFunc := func(w want, g Group) error {
got, want := g.(*group), w.want.(*group)
- if !reflect.DeepEqual(got.enableLimitation, want.enableLimitation) {
- return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", got, w.want)
- }
- if got.limitation != nil && want.limitation != nil {
- return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", got, w.want)
+ if (got.sem == nil) != (want.sem == nil) {
+ return errors.Errorf("got: \"%#v\",\n\t\t\twant: \"%#v\"", got, w.want)
}
return nil
}
tests := []test{
{
- name: "set disable when limit is 0",
+ name: "set disable when limit is -1",
+ args: args{
+ limit: -1,
+ },
+ want: want{
+ want: &group{},
+ },
+ },
+ {
+ name: "set enable but block all acquireings when limit is 0",
args: args{
limit: 0,
},
want: want{
want: &group{
- enableLimitation: func() atomic.Bool {
- var el atomic.Bool
- el.Store(false)
- return el
- }(),
+ sem: semaphore.NewWeighted(0),
},
},
},
@@ -393,16 +388,10 @@ func Test_group_Limitation(t *testing.T) {
args: args{
limit: 1,
},
- fields: fields{
- limitation: make(chan struct{}, 1),
- },
+ fields: fields{},
want: want{
want: &group{
- enableLimitation: func() atomic.Bool {
- var el atomic.Bool
- el.Store(true)
- return el
- }(),
+ sem: semaphore.NewWeighted(1),
},
},
},
@@ -421,12 +410,9 @@ func Test_group_Limitation(t *testing.T) {
if test.checkFunc == nil {
checkFunc = defaultCheckFunc
}
- g := &group{
- limitation: test.fields.limitation,
- enableLimitation: test.fields.enableLimitation,
- }
+ g := &group{}
- g.Limitation(test.args.limit)
+ g.SetLimit(test.args.limit)
if err := checkFunc(test.want, g); err != nil {
tt.Errorf("error = %v", err)
}
@@ -439,11 +425,10 @@ func Test_group_Go(t *testing.T) {
f func() error
}
type fields struct {
- egctx context.Context
- cancel context.CancelFunc
- limitation chan struct{}
- enableLimitation atomic.Bool
- emap map[string]struct{}
+ egctx context.Context
+ cancel context.CancelCauseFunc
+ sem *semaphore.Weighted
+ emap map[string]struct{}
}
type test struct {
name string
@@ -460,7 +445,7 @@ func Test_group_Go(t *testing.T) {
func() test {
var calledCnt int32
- egctx, cancel := context.WithCancel(context.Background())
+ egctx, cancel := context.WithCancelCause(context.Background())
limit := 3
@@ -473,12 +458,8 @@ func Test_group_Go(t *testing.T) {
},
},
fields: fields{
- egctx: egctx,
- limitation: make(chan struct{}, limit),
- enableLimitation: func() (el atomic.Bool) {
- el.Store(true)
- return
- }(),
+ egctx: egctx,
+ sem: semaphore.NewWeighted(int64(limit)),
},
beforeFunc: func(t *testing.T, _ args, g Group) {
t.Helper()
@@ -489,10 +470,9 @@ func Test_group_Go(t *testing.T) {
})
}
time.Sleep(time.Second)
+ cancel(context.Canceled)
},
checkFunc: func(got Group) error {
- cancel()
-
if err := got.Wait(); err != nil {
return err
}
@@ -508,7 +488,7 @@ func Test_group_Go(t *testing.T) {
func() test {
var calledCnt int32
- egctx, cancel := context.WithCancel(context.Background())
+ egctx, cancel := context.WithCancelCause(context.Background())
return test{
name: "f is called but f returns error and previous process also returns error",
@@ -521,11 +501,7 @@ func Test_group_Go(t *testing.T) {
fields: fields{
egctx: egctx,
cancel: cancel,
- enableLimitation: func() (el atomic.Bool) {
- el.Store(false)
- return
- }(),
- emap: make(map[string]struct{}),
+ emap: make(map[string]struct{}),
},
beforeFunc: func(t *testing.T, a args, g Group) {
t.Helper()
@@ -569,11 +545,10 @@ func Test_group_Go(t *testing.T) {
}
g := &group{
- egctx: test.fields.egctx,
- cancel: test.fields.cancel,
- limitation: test.fields.limitation,
- enableLimitation: test.fields.enableLimitation,
- emap: test.fields.emap,
+ egctx: test.fields.egctx,
+ cancel: test.fields.cancel,
+ sem: test.fields.sem,
+ emap: test.fields.emap,
}
if test.beforeFunc != nil {
@@ -590,7 +565,7 @@ func Test_group_Go(t *testing.T) {
func Test_group_doCancel(t *testing.T) {
type fields struct {
- cancel context.CancelFunc
+ cancel context.CancelCauseFunc
}
type test struct {
name string
@@ -609,7 +584,7 @@ func Test_group_doCancel(t *testing.T) {
return test{
name: "g.cancel is called when g.cancel is not nil",
fields: fields{
- cancel: func() {
+ cancel: func(err error) {
called = true
},
},
@@ -640,7 +615,7 @@ func Test_group_doCancel(t *testing.T) {
cancel: test.fields.cancel,
}
- g.doCancel()
+ g.doCancel(context.Canceled)
if err := checkFunc(); err != nil {
tt.Errorf("error = %v", err)
}
@@ -704,9 +679,8 @@ func TestWait(t *testing.T) {
func Test_group_Wait(t *testing.T) {
type fields struct {
- limitation chan struct{}
- enableLimitation atomic.Bool
- errs []error
+ sem *semaphore.Weighted
+ errs []error
}
type want struct {
err error
@@ -729,13 +703,8 @@ func Test_group_Wait(t *testing.T) {
func() test {
var num int32
return test{
- name: "returns nil after all goroutne returns",
- fields: fields{
- enableLimitation: func() (el atomic.Bool) {
- el.Store(false)
- return
- }(),
- },
+ name: "returns nil after all goroutne returns",
+ fields: fields{},
beforeFunc: func(t *testing.T, g Group) {
t.Helper()
g.Go(func() error {
@@ -782,9 +751,8 @@ func Test_group_Wait(t *testing.T) {
checkFunc = defaultCheckFunc
}
g := &group{
- limitation: test.fields.limitation,
- errs: test.fields.errs,
- enableLimitation: test.fields.enableLimitation,
+ sem: test.fields.sem,
+ errs: test.fields.errs,
}
if test.beforeFunc != nil {
@@ -801,15 +769,524 @@ func Test_group_Wait(t *testing.T) {
// NOT IMPLEMENTED BELOW
//
-// func Test_group_closeLimitation(t *testing.T) {
+// func TestWithContext(t *testing.T) {
+// type args struct {
+// ctx context.Context
+// }
+// type want struct {
+// want Group
+// want1 context.Context
+// }
+// type test struct {
+// name string
+// args args
+// want want
+// checkFunc func(want, Group, context.Context) error
+// beforeFunc func(*testing.T, args)
+// afterFunc func(*testing.T, args)
+// }
+// defaultCheckFunc := func(w want, got Group, got1 context.Context) error {
+// if !reflect.DeepEqual(got, w.want) {
+// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", got, w.want)
+// }
+// if !reflect.DeepEqual(got1, w.want1) {
+// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", got1, w.want1)
+// }
+// return nil
+// }
+// tests := []test{
+// // TODO test cases
+// /*
+// {
+// name: "test_case_1",
+// args: args {
+// ctx:nil,
+// },
+// want: want{},
+// checkFunc: defaultCheckFunc,
+// beforeFunc: func(t *testing.T, args args) {
+// t.Helper()
+// },
+// afterFunc: func(t *testing.T, args args) {
+// t.Helper()
+// },
+// },
+// */
+//
+// // TODO test cases
+// /*
+// func() test {
+// return test {
+// name: "test_case_2",
+// args: args {
+// ctx:nil,
+// },
+// want: want{},
+// checkFunc: defaultCheckFunc,
+// beforeFunc: func(t *testing.T, args args) {
+// t.Helper()
+// },
+// afterFunc: func(t *testing.T, args args) {
+// t.Helper()
+// },
+// }
+// }(),
+// */
+// }
+//
+// for _, tc := range tests {
+// test := tc
+// t.Run(test.name, func(tt *testing.T) {
+// tt.Parallel()
+// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent())
+// if test.beforeFunc != nil {
+// test.beforeFunc(tt, test.args)
+// }
+// if test.afterFunc != nil {
+// defer test.afterFunc(tt, test.args)
+// }
+// checkFunc := test.checkFunc
+// if test.checkFunc == nil {
+// checkFunc = defaultCheckFunc
+// }
+//
+// got, got1 := WithContext(test.args.ctx)
+// if err := checkFunc(test.want, got, got1); err != nil {
+// tt.Errorf("error = %v", err)
+// }
+//
+// })
+// }
+// }
+//
+// func TestTryGo(t *testing.T) {
+// type args struct {
+// f func() error
+// }
+// type want struct {
+// want bool
+// }
+// type test struct {
+// name string
+// args args
+// want want
+// checkFunc func(want, bool) error
+// beforeFunc func(*testing.T, args)
+// afterFunc func(*testing.T, args)
+// }
+// defaultCheckFunc := func(w want, got bool) error {
+// if !reflect.DeepEqual(got, w.want) {
+// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", got, w.want)
+// }
+// return nil
+// }
+// tests := []test{
+// // TODO test cases
+// /*
+// {
+// name: "test_case_1",
+// args: args {
+// f:nil,
+// },
+// want: want{},
+// checkFunc: defaultCheckFunc,
+// beforeFunc: func(t *testing.T, args args) {
+// t.Helper()
+// },
+// afterFunc: func(t *testing.T, args args) {
+// t.Helper()
+// },
+// },
+// */
+//
+// // TODO test cases
+// /*
+// func() test {
+// return test {
+// name: "test_case_2",
+// args: args {
+// f:nil,
+// },
+// want: want{},
+// checkFunc: defaultCheckFunc,
+// beforeFunc: func(t *testing.T, args args) {
+// t.Helper()
+// },
+// afterFunc: func(t *testing.T, args args) {
+// t.Helper()
+// },
+// }
+// }(),
+// */
+// }
+//
+// for _, tc := range tests {
+// test := tc
+// t.Run(test.name, func(tt *testing.T) {
+// tt.Parallel()
+// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent())
+// if test.beforeFunc != nil {
+// test.beforeFunc(tt, test.args)
+// }
+// if test.afterFunc != nil {
+// defer test.afterFunc(tt, test.args)
+// }
+// checkFunc := test.checkFunc
+// if test.checkFunc == nil {
+// checkFunc = defaultCheckFunc
+// }
+//
+// got := TryGo(test.args.f)
+// if err := checkFunc(test.want, got); err != nil {
+// tt.Errorf("error = %v", err)
+// }
+//
+// })
+// }
+// }
+//
+// func Test_group_TryGo(t *testing.T) {
+// type args struct {
+// f func() error
+// }
+// type fields struct {
+// egctx context.Context
+// cancel context.CancelCauseFunc
+// sem *semaphore.Weighted
+// emap map[string]struct{}
+// errs []error
+// }
+// type want struct {
+// want bool
+// }
+// type test struct {
+// name string
+// args args
+// fields fields
+// want want
+// checkFunc func(want, bool) error
+// beforeFunc func(*testing.T, args)
+// afterFunc func(*testing.T, args)
+// }
+// defaultCheckFunc := func(w want, got bool) error {
+// if !reflect.DeepEqual(got, w.want) {
+// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", got, w.want)
+// }
+// return nil
+// }
+// tests := []test{
+// // TODO test cases
+// /*
+// {
+// name: "test_case_1",
+// args: args {
+// f:nil,
+// },
+// fields: fields {
+// egctx:nil,
+// cancel:nil,
+// sem:nil,
+// emap:nil,
+// errs:nil,
+// },
+// want: want{},
+// checkFunc: defaultCheckFunc,
+// beforeFunc: func(t *testing.T, args args) {
+// t.Helper()
+// },
+// afterFunc: func(t *testing.T, args args) {
+// t.Helper()
+// },
+// },
+// */
+//
+// // TODO test cases
+// /*
+// func() test {
+// return test {
+// name: "test_case_2",
+// args: args {
+// f:nil,
+// },
+// fields: fields {
+// egctx:nil,
+// cancel:nil,
+// sem:nil,
+// emap:nil,
+// errs:nil,
+// },
+// want: want{},
+// checkFunc: defaultCheckFunc,
+// beforeFunc: func(t *testing.T, args args) {
+// t.Helper()
+// },
+// afterFunc: func(t *testing.T, args args) {
+// t.Helper()
+// },
+// }
+// }(),
+// */
+// }
+//
+// for _, tc := range tests {
+// test := tc
+// t.Run(test.name, func(tt *testing.T) {
+// tt.Parallel()
+// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent())
+// if test.beforeFunc != nil {
+// test.beforeFunc(tt, test.args)
+// }
+// if test.afterFunc != nil {
+// defer test.afterFunc(tt, test.args)
+// }
+// checkFunc := test.checkFunc
+// if test.checkFunc == nil {
+// checkFunc = defaultCheckFunc
+// }
+// g := &group{
+// egctx: test.fields.egctx,
+// cancel: test.fields.cancel,
+// sem: test.fields.sem,
+// emap: test.fields.emap,
+// errs: test.fields.errs,
+// }
+//
+// got := g.TryGo(test.args.f)
+// if err := checkFunc(test.want, got); err != nil {
+// tt.Errorf("error = %v", err)
+// }
+//
+// })
+// }
+// }
+//
+// func Test_group_run(t *testing.T) {
+// type args struct {
+// f func() error
+// }
// type fields struct {
-// egctx context.Context
-// cancel context.CancelFunc
-// limitation chan struct{}
-// enableLimitation atomic.Bool
-// emap map[string]struct{}
-// errs []error
-// err error
+// egctx context.Context
+// cancel context.CancelCauseFunc
+// sem *semaphore.Weighted
+// emap map[string]struct{}
+// errs []error
+// }
+// type want struct {
+// }
+// type test struct {
+// name string
+// args args
+// fields fields
+// want want
+// checkFunc func(want) error
+// beforeFunc func(*testing.T, args)
+// afterFunc func(*testing.T, args)
+// }
+// defaultCheckFunc := func(w want) error {
+// return nil
+// }
+// tests := []test{
+// // TODO test cases
+// /*
+// {
+// name: "test_case_1",
+// args: args {
+// f:nil,
+// },
+// fields: fields {
+// egctx:nil,
+// cancel:nil,
+// sem:nil,
+// emap:nil,
+// errs:nil,
+// },
+// want: want{},
+// checkFunc: defaultCheckFunc,
+// beforeFunc: func(t *testing.T, args args) {
+// t.Helper()
+// },
+// afterFunc: func(t *testing.T, args args) {
+// t.Helper()
+// },
+// },
+// */
+//
+// // TODO test cases
+// /*
+// func() test {
+// return test {
+// name: "test_case_2",
+// args: args {
+// f:nil,
+// },
+// fields: fields {
+// egctx:nil,
+// cancel:nil,
+// sem:nil,
+// emap:nil,
+// errs:nil,
+// },
+// want: want{},
+// checkFunc: defaultCheckFunc,
+// beforeFunc: func(t *testing.T, args args) {
+// t.Helper()
+// },
+// afterFunc: func(t *testing.T, args args) {
+// t.Helper()
+// },
+// }
+// }(),
+// */
+// }
+//
+// for _, tc := range tests {
+// test := tc
+// t.Run(test.name, func(tt *testing.T) {
+// tt.Parallel()
+// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent())
+// if test.beforeFunc != nil {
+// test.beforeFunc(tt, test.args)
+// }
+// if test.afterFunc != nil {
+// defer test.afterFunc(tt, test.args)
+// }
+// checkFunc := test.checkFunc
+// if test.checkFunc == nil {
+// checkFunc = defaultCheckFunc
+// }
+// g := &group{
+// egctx: test.fields.egctx,
+// cancel: test.fields.cancel,
+// sem: test.fields.sem,
+// emap: test.fields.emap,
+// errs: test.fields.errs,
+// }
+//
+// g.run(test.args.f)
+// if err := checkFunc(test.want); err != nil {
+// tt.Errorf("error = %v", err)
+// }
+// })
+// }
+// }
+//
+// func Test_group_appendErr(t *testing.T) {
+// type args struct {
+// err error
+// }
+// type fields struct {
+// egctx context.Context
+// cancel context.CancelCauseFunc
+// sem *semaphore.Weighted
+// emap map[string]struct{}
+// errs []error
+// }
+// type want struct {
+// }
+// type test struct {
+// name string
+// args args
+// fields fields
+// want want
+// checkFunc func(want) error
+// beforeFunc func(*testing.T, args)
+// afterFunc func(*testing.T, args)
+// }
+// defaultCheckFunc := func(w want) error {
+// return nil
+// }
+// tests := []test{
+// // TODO test cases
+// /*
+// {
+// name: "test_case_1",
+// args: args {
+// err:nil,
+// },
+// fields: fields {
+// egctx:nil,
+// cancel:nil,
+// sem:nil,
+// emap:nil,
+// errs:nil,
+// },
+// want: want{},
+// checkFunc: defaultCheckFunc,
+// beforeFunc: func(t *testing.T, args args) {
+// t.Helper()
+// },
+// afterFunc: func(t *testing.T, args args) {
+// t.Helper()
+// },
+// },
+// */
+//
+// // TODO test cases
+// /*
+// func() test {
+// return test {
+// name: "test_case_2",
+// args: args {
+// err:nil,
+// },
+// fields: fields {
+// egctx:nil,
+// cancel:nil,
+// sem:nil,
+// emap:nil,
+// errs:nil,
+// },
+// want: want{},
+// checkFunc: defaultCheckFunc,
+// beforeFunc: func(t *testing.T, args args) {
+// t.Helper()
+// },
+// afterFunc: func(t *testing.T, args args) {
+// t.Helper()
+// },
+// }
+// }(),
+// */
+// }
+//
+// for _, tc := range tests {
+// test := tc
+// t.Run(test.name, func(tt *testing.T) {
+// tt.Parallel()
+// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent())
+// if test.beforeFunc != nil {
+// test.beforeFunc(tt, test.args)
+// }
+// if test.afterFunc != nil {
+// defer test.afterFunc(tt, test.args)
+// }
+// checkFunc := test.checkFunc
+// if test.checkFunc == nil {
+// checkFunc = defaultCheckFunc
+// }
+// g := &group{
+// egctx: test.fields.egctx,
+// cancel: test.fields.cancel,
+// sem: test.fields.sem,
+// emap: test.fields.emap,
+// errs: test.fields.errs,
+// }
+//
+// g.appendErr(test.args.err)
+// if err := checkFunc(test.want); err != nil {
+// tt.Errorf("error = %v", err)
+// }
+// })
+// }
+// }
+//
+// func Test_group_done(t *testing.T) {
+// type fields struct {
+// egctx context.Context
+// cancel context.CancelCauseFunc
+// sem *semaphore.Weighted
+// emap map[string]struct{}
+// errs []error
// }
// type want struct {
// }
@@ -832,11 +1309,9 @@ func Test_group_Wait(t *testing.T) {
// fields: fields {
// egctx:nil,
// cancel:nil,
-// limitation:nil,
-// enableLimitation:nil,
+// sem:nil,
// emap:nil,
// errs:nil,
-// err:nil,
// },
// want: want{},
// checkFunc: defaultCheckFunc,
@@ -857,11 +1332,9 @@ func Test_group_Wait(t *testing.T) {
// fields: fields {
// egctx:nil,
// cancel:nil,
-// limitation:nil,
-// enableLimitation:nil,
+// sem:nil,
// emap:nil,
// errs:nil,
-// err:nil,
// },
// want: want{},
// checkFunc: defaultCheckFunc,
@@ -892,16 +1365,14 @@ func Test_group_Wait(t *testing.T) {
// checkFunc = defaultCheckFunc
// }
// g := &group{
-// egctx: test.fields.egctx,
-// cancel: test.fields.cancel,
-// limitation: test.fields.limitation,
-// enableLimitation: test.fields.enableLimitation,
-// emap: test.fields.emap,
-// errs: test.fields.errs,
-// err: test.fields.err,
+// egctx: test.fields.egctx,
+// cancel: test.fields.cancel,
+// sem: test.fields.sem,
+// emap: test.fields.emap,
+// errs: test.fields.errs,
// }
//
-// g.closeLimitation()
+// g.done()
// if err := checkFunc(test.want); err != nil {
// tt.Errorf("error = %v", err)
// }
diff --git a/internal/sync/semaphore/semaphore.go b/internal/sync/semaphore/semaphore.go
new file mode 100644
index 00000000000..1ec284e1903
--- /dev/null
+++ b/internal/sync/semaphore/semaphore.go
@@ -0,0 +1,173 @@
+//
+// Copyright (c) 2009-2023 The Go Authors. All rights resered.
+// Modified by vdaas.org vald team
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+// * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+//
+
+// Package semaphore provides a weighted semaphore implementation.
+package semaphore
+
+import (
+ "container/list"
+ "context"
+
+ "github.com/vdaas/vald/internal/sync"
+)
+
+type waiter struct {
+ n int64
+ ready chan<- struct{} // Closed when semaphore acquired.
+}
+
+// NewWeighted creates a new weighted semaphore with the given
+// maximum combined weight for concurrent access.
+func NewWeighted(n int64) *Weighted {
+ w := &Weighted{size: n}
+ return w
+}
+
+// Weighted provides a way to bound concurrent access to a resource.
+// The callers can request access with a given weight.
+type Weighted struct {
+ size int64
+ cur int64
+ mu sync.Mutex
+ waiters list.List
+}
+
+// Acquire acquires the semaphore with a weight of n, blocking until resources
+// are available or ctx is done. On success, returns nil. On failure, returns
+// ctx.Err() and leaves the semaphore unchanged.
+//
+// If ctx is already done, Acquire may still succeed without blocking.
+func (s *Weighted) Acquire(ctx context.Context, n int64) error {
+ s.mu.Lock()
+ if s.size-s.cur >= n && s.waiters.Len() == 0 {
+ s.cur += n
+ s.mu.Unlock()
+ return nil
+ }
+
+ if n > s.size {
+ // Don't make other Acquire calls block on one that's doomed to fail.
+ s.mu.Unlock()
+ <-ctx.Done()
+ return ctx.Err()
+ }
+
+ ready := make(chan struct{})
+ w := waiter{n: n, ready: ready}
+ elem := s.waiters.PushBack(w)
+ s.mu.Unlock()
+
+ select {
+ case <-ctx.Done():
+ err := ctx.Err()
+ s.mu.Lock()
+ select {
+ case <-ready:
+ // Acquired the semaphore after we were canceled. Rather than trying to
+ // fix up the queue, just pretend we didn't notice the cancelation.
+ err = nil
+ default:
+ isFront := s.waiters.Front() == elem
+ s.waiters.Remove(elem)
+ // If we're at the front and there're extra tokens left, notify other waiters.
+ if isFront && s.size > s.cur {
+ s.notifyWaiters()
+ }
+ }
+ s.mu.Unlock()
+ return err
+
+ case <-ready:
+ return nil
+ }
+}
+
+// TryAcquire acquires the semaphore with a weight of n without blocking.
+// On success, returns true. On failure, returns false and leaves the semaphore unchanged.
+func (s *Weighted) TryAcquire(n int64) bool {
+ s.mu.Lock()
+ success := s.size-s.cur >= n && s.waiters.Len() == 0
+ if success {
+ s.cur += n
+ }
+ s.mu.Unlock()
+ return success
+}
+
+// Release releases the semaphore with a weight of n.
+func (s *Weighted) Release(n int64) {
+ s.mu.Lock()
+ s.cur -= n
+ if s.cur < 0 {
+ s.mu.Unlock()
+ panic("semaphore: released more than held")
+ }
+ s.notifyWaiters()
+ s.mu.Unlock()
+}
+
+func (s *Weighted) Resize(n int64) {
+ if n < 0 {
+ n = 0
+ }
+ s.mu.Lock()
+ s.size = n
+ s.mu.Unlock()
+}
+
+func (s *Weighted) notifyWaiters() {
+ for {
+ next := s.waiters.Front()
+ if next == nil {
+ break // No more waiters blocked.
+ }
+
+ w := next.Value.(waiter)
+ if s.size-s.cur < w.n {
+ // Not enough tokens for the next waiter. We could keep going (to try to
+ // find a waiter with a smaller request), but under load that could cause
+ // starvation for large requests; instead, we leave all remaining waiters
+ // blocked.
+ //
+ // Consider a semaphore used as a read-write lock, with N tokens, N
+ // readers, and one writer. Each reader can Acquire(1) to obtain a read
+ // lock. The writer can Acquire(N) to obtain a write lock, excluding all
+ // of the readers. If we allow the readers to jump ahead in the queue,
+ // the writer will starve — there is always one token available for every
+ // reader.
+ break
+ }
+
+ s.cur += w.n
+ s.waiters.Remove(next)
+ close(w.ready)
+ }
+}
diff --git a/internal/sync/semaphore/semaphore_bench_test.go b/internal/sync/semaphore/semaphore_bench_test.go
new file mode 100644
index 00000000000..3bab8eb893a
--- /dev/null
+++ b/internal/sync/semaphore/semaphore_bench_test.go
@@ -0,0 +1,159 @@
+//
+// Copyright (c) 2009-2023 The Go Authors. All rights resered.
+// Modified by vdaas.org vald team
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+// * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+//
+
+//go:build go1.7
+// +build go1.7
+
+package semaphore_test
+
+import (
+ "context"
+ "fmt"
+ "testing"
+
+ "github.com/vdaas/vald/internal/sync/semaphore"
+)
+
+// weighted is an interface matching a subset of *Weighted. It allows
+// alternate implementations for testing and benchmarking.
+type weighted interface {
+ Acquire(context.Context, int64) error
+ TryAcquire(int64) bool
+ Release(int64)
+}
+
+// semChan implements Weighted using a channel for
+// comparing against the condition variable-based implementation.
+type semChan chan struct{}
+
+func newSemChan(n int64) semChan {
+ return semChan(make(chan struct{}, n))
+}
+
+func (s semChan) Acquire(_ context.Context, n int64) error {
+ for i := int64(0); i < n; i++ {
+ s <- struct{}{}
+ }
+ return nil
+}
+
+func (s semChan) TryAcquire(n int64) bool {
+ if int64(len(s))+n > int64(cap(s)) {
+ return false
+ }
+
+ for i := int64(0); i < n; i++ {
+ s <- struct{}{}
+ }
+ return true
+}
+
+func (s semChan) Release(n int64) {
+ for i := int64(0); i < n; i++ {
+ <-s
+ }
+}
+
+// acquireN calls Acquire(size) on sem N times and then calls Release(size) N times.
+func acquireN(b *testing.B, sem weighted, size int64, N int) {
+ b.ResetTimer()
+ for i := 0; i < b.N; i++ {
+ for j := 0; j < N; j++ {
+ sem.Acquire(context.Background(), size)
+ }
+ for j := 0; j < N; j++ {
+ sem.Release(size)
+ }
+ }
+}
+
+// tryAcquireN calls TryAcquire(size) on sem N times and then calls Release(size) N times.
+func tryAcquireN(b *testing.B, sem weighted, size int64, N int) {
+ b.ResetTimer()
+ for i := 0; i < b.N; i++ {
+ for j := 0; j < N; j++ {
+ if !sem.TryAcquire(size) {
+ b.Fatalf("TryAcquire(%v) = false, want true", size)
+ }
+ }
+ for j := 0; j < N; j++ {
+ sem.Release(size)
+ }
+ }
+}
+
+func BenchmarkNewSeq(b *testing.B) {
+ for _, cap := range []int64{1, 128} {
+ b.Run(fmt.Sprintf("Weighted-%d", cap), func(b *testing.B) {
+ for i := 0; i < b.N; i++ {
+ _ = semaphore.NewWeighted(cap)
+ }
+ })
+ b.Run(fmt.Sprintf("semChan-%d", cap), func(b *testing.B) {
+ for i := 0; i < b.N; i++ {
+ _ = newSemChan(cap)
+ }
+ })
+ }
+}
+
+func BenchmarkAcquireSeq(b *testing.B) {
+ for _, c := range []struct {
+ cap, size int64
+ N int
+ }{
+ {1, 1, 1},
+ {2, 1, 1},
+ {16, 1, 1},
+ {128, 1, 1},
+ {2, 2, 1},
+ {16, 2, 8},
+ {128, 2, 64},
+ {2, 1, 2},
+ {16, 8, 2},
+ {128, 64, 2},
+ } {
+ for _, w := range []struct {
+ name string
+ w weighted
+ }{
+ {"Weighted", semaphore.NewWeighted(c.cap)},
+ {"semChan", newSemChan(c.cap)},
+ } {
+ b.Run(fmt.Sprintf("%s-acquire-%d-%d-%d", w.name, c.cap, c.size, c.N), func(b *testing.B) {
+ acquireN(b, w.w, c.size, c.N)
+ })
+ b.Run(fmt.Sprintf("%s-tryAcquire-%d-%d-%d", w.name, c.cap, c.size, c.N), func(b *testing.B) {
+ tryAcquireN(b, w.w, c.size, c.N)
+ })
+ }
+ }
+}
diff --git a/internal/sync/semaphore/semaphore_example_test.go b/internal/sync/semaphore/semaphore_example_test.go
new file mode 100644
index 00000000000..abc5798736b
--- /dev/null
+++ b/internal/sync/semaphore/semaphore_example_test.go
@@ -0,0 +1,111 @@
+//
+// Copyright (c) 2009-2023 The Go Authors. All rights resered.
+// Modified by vdaas.org vald team
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+// * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+//
+
+package semaphore_test
+
+import (
+ "context"
+ "fmt"
+ "log"
+ "runtime"
+
+ "github.com/vdaas/vald/internal/sync/semaphore"
+)
+
+// Example_workerPool demonstrates how to use a semaphore to limit the number of
+// goroutines working on parallel tasks.
+//
+// This use of a semaphore mimics a typical “worker pool” pattern, but without
+// the need to explicitly shut down idle workers when the work is done.
+func Example_workerPool() {
+ ctx := context.TODO()
+
+ var (
+ maxWorkers = runtime.GOMAXPROCS(0)
+ sem = semaphore.NewWeighted(int64(maxWorkers))
+ out = make([]int, 32)
+ )
+
+ // Compute the output using up to maxWorkers goroutines at a time.
+ for i := range out {
+ // When maxWorkers goroutines are in flight, Acquire blocks until one of the
+ // workers finishes.
+ if err := sem.Acquire(ctx, 1); err != nil {
+ log.Printf("Failed to acquire semaphore: %v", err)
+ break
+ }
+
+ go func(i int) {
+ defer sem.Release(1)
+ out[i] = collatzSteps(i + 1)
+ }(i)
+ }
+
+ // Acquire all of the tokens to wait for any remaining workers to finish.
+ //
+ // If you are already waiting for the workers by some other means (such as an
+ // errgroup.Group), you can omit this final Acquire call.
+ if err := sem.Acquire(ctx, int64(maxWorkers)); err != nil {
+ log.Printf("Failed to acquire semaphore: %v", err)
+ }
+
+ fmt.Println(out)
+
+ // Output:
+ // [0 1 7 2 5 8 16 3 19 6 14 9 9 17 17 4 12 20 20 7 7 15 15 10 23 10 111 18 18 18 106 5]
+}
+
+// collatzSteps computes the number of steps to reach 1 under the Collatz
+// conjecture. (See https://en.wikipedia.org/wiki/Collatz_conjecture.)
+func collatzSteps(n int) (steps int) {
+ if n <= 0 {
+ panic("nonpositive input")
+ }
+
+ for ; n > 1; steps++ {
+ if steps < 0 {
+ panic("too many steps")
+ }
+
+ if n%2 == 0 {
+ n /= 2
+ continue
+ }
+
+ const maxInt = int(^uint(0) >> 1)
+ if n > (maxInt-1)/3 {
+ panic("overflow")
+ }
+ n = 3*n + 1
+ }
+
+ return steps
+}
diff --git a/internal/sync/semaphore/semaphore_test.go b/internal/sync/semaphore/semaphore_test.go
new file mode 100644
index 00000000000..6290b6affa7
--- /dev/null
+++ b/internal/sync/semaphore/semaphore_test.go
@@ -0,0 +1,830 @@
+//
+// Copyright (c) 2009-2023 The Go Authors. All rights resered.
+// Modified by vdaas.org vald team
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+// * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+//
+
+package semaphore_test
+
+import (
+ "context"
+ "math/rand"
+ "runtime"
+ "testing"
+ "time"
+
+ "github.com/vdaas/vald/internal/sync"
+ "github.com/vdaas/vald/internal/sync/errgroup"
+ "github.com/vdaas/vald/internal/sync/semaphore"
+)
+
+const maxSleep = 1 * time.Millisecond
+
+func HammerWeighted(sem *semaphore.Weighted, n int64, loops int) {
+ for i := 0; i < loops; i++ {
+ sem.Acquire(context.Background(), n)
+ time.Sleep(time.Duration(rand.Int63n(int64(maxSleep/time.Nanosecond))) * time.Nanosecond)
+ sem.Release(n)
+ }
+}
+
+func TestWeighted(t *testing.T) {
+ t.Parallel()
+
+ n := runtime.GOMAXPROCS(0)
+ loops := 10000 / n
+ sem := semaphore.NewWeighted(int64(n))
+ var wg sync.WaitGroup
+ wg.Add(n)
+ for i := 0; i < n; i++ {
+ i := i
+ go func() {
+ defer wg.Done()
+ HammerWeighted(sem, int64(i), loops)
+ }()
+ }
+ wg.Wait()
+}
+
+func TestWeightedPanic(t *testing.T) {
+ t.Parallel()
+
+ defer func() {
+ if recover() == nil {
+ t.Fatal("release of an unacquired weighted semaphore did not panic")
+ }
+ }()
+ w := semaphore.NewWeighted(1)
+ w.Release(1)
+}
+
+func TestWeightedTryAcquire(t *testing.T) {
+ t.Parallel()
+
+ ctx := context.Background()
+ sem := semaphore.NewWeighted(2)
+ tries := []bool{}
+ sem.Acquire(ctx, 1)
+ tries = append(tries, sem.TryAcquire(1))
+ tries = append(tries, sem.TryAcquire(1))
+
+ sem.Release(2)
+
+ tries = append(tries, sem.TryAcquire(1))
+ sem.Acquire(ctx, 1)
+ tries = append(tries, sem.TryAcquire(1))
+
+ want := []bool{true, false, true, false}
+ for i := range tries {
+ if tries[i] != want[i] {
+ t.Errorf("tries[%d]: got %t, want %t", i, tries[i], want[i])
+ }
+ }
+}
+
+func TestWeightedAcquire(t *testing.T) {
+ t.Parallel()
+
+ ctx := context.Background()
+ sem := semaphore.NewWeighted(2)
+ tryAcquire := func(n int64) bool {
+ ctx, cancel := context.WithTimeout(ctx, 10*time.Millisecond)
+ defer cancel()
+ return sem.Acquire(ctx, n) == nil
+ }
+
+ tries := []bool{}
+ sem.Acquire(ctx, 1)
+ tries = append(tries, tryAcquire(1))
+ tries = append(tries, tryAcquire(1))
+
+ sem.Release(2)
+
+ tries = append(tries, tryAcquire(1))
+ sem.Acquire(ctx, 1)
+ tries = append(tries, tryAcquire(1))
+
+ want := []bool{true, false, true, false}
+ for i := range tries {
+ if tries[i] != want[i] {
+ t.Errorf("tries[%d]: got %t, want %t", i, tries[i], want[i])
+ }
+ }
+}
+
+func TestWeightedDoesntBlockIfTooBig(t *testing.T) {
+ t.Parallel()
+
+ const n = 2
+ sem := semaphore.NewWeighted(n)
+ {
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+ go sem.Acquire(ctx, n+1)
+ }
+
+ g, ctx := errgroup.WithContext(context.Background())
+ for i := n * 3; i > 0; i-- {
+ g.Go(func() error {
+ err := sem.Acquire(ctx, 1)
+ if err == nil {
+ time.Sleep(1 * time.Millisecond)
+ sem.Release(1)
+ }
+ return err
+ })
+ }
+ if err := g.Wait(); err != nil {
+ t.Errorf("semaphore.NewWeighted(%v) failed to AcquireCtx(_, 1) with AcquireCtx(_, %v) pending", n, n+1)
+ }
+}
+
+// TestLargeAcquireDoesntStarve times out if a large call to Acquire starves.
+// Merely returning from the test function indicates success.
+func TestLargeAcquireDoesntStarve(t *testing.T) {
+ t.Parallel()
+
+ ctx := context.Background()
+ n := int64(runtime.GOMAXPROCS(0))
+ sem := semaphore.NewWeighted(n)
+ running := true
+
+ var wg sync.WaitGroup
+ wg.Add(int(n))
+ for i := n; i > 0; i-- {
+ sem.Acquire(ctx, 1)
+ go func() {
+ defer func() {
+ sem.Release(1)
+ wg.Done()
+ }()
+ for running {
+ time.Sleep(1 * time.Millisecond)
+ sem.Release(1)
+ sem.Acquire(ctx, 1)
+ }
+ }()
+ }
+
+ sem.Acquire(ctx, n)
+ running = false
+ sem.Release(n)
+ wg.Wait()
+}
+
+// translated from https://github.com/zhiqiangxu/util/blob/master/mutex/crwmutex_test.go#L43
+func TestAllocCancelDoesntStarve(t *testing.T) {
+ sem := semaphore.NewWeighted(10)
+
+ // Block off a portion of the semaphore so that Acquire(_, 10) can eventually succeed.
+ sem.Acquire(context.Background(), 1)
+
+ // In the background, Acquire(_, 10).
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+ go func() {
+ sem.Acquire(ctx, 10)
+ }()
+
+ // Wait until the Acquire(_, 10) call blocks.
+ for sem.TryAcquire(1) {
+ sem.Release(1)
+ runtime.Gosched()
+ }
+
+ // Now try to grab a read lock, and simultaneously unblock the Acquire(_, 10) call.
+ // Both Acquire calls should unblock and return, in either order.
+ go cancel()
+
+ err := sem.Acquire(context.Background(), 1)
+ if err != nil {
+ t.Fatalf("Acquire(_, 1) failed unexpectedly: %v", err)
+ }
+ sem.Release(1)
+}
+
+// NOT IMPLEMENTED BELOW
+//
+// func TestNewWeighted(t *testing.T) {
+// type args struct {
+// n int64
+// }
+// type want struct {
+// want *Weighted
+// }
+// type test struct {
+// name string
+// args args
+// want want
+// checkFunc func(want, *Weighted) error
+// beforeFunc func(*testing.T, args)
+// afterFunc func(*testing.T, args)
+// }
+// defaultCheckFunc := func(w want, got *Weighted) error {
+// if !reflect.DeepEqual(got, w.want) {
+// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", got, w.want)
+// }
+// return nil
+// }
+// tests := []test{
+// // TODO test cases
+// /*
+// {
+// name: "test_case_1",
+// args: args {
+// n:0,
+// },
+// want: want{},
+// checkFunc: defaultCheckFunc,
+// beforeFunc: func(t *testing.T, args args) {
+// t.Helper()
+// },
+// afterFunc: func(t *testing.T, args args) {
+// t.Helper()
+// },
+// },
+// */
+//
+// // TODO test cases
+// /*
+// func() test {
+// return test {
+// name: "test_case_2",
+// args: args {
+// n:0,
+// },
+// want: want{},
+// checkFunc: defaultCheckFunc,
+// beforeFunc: func(t *testing.T, args args) {
+// t.Helper()
+// },
+// afterFunc: func(t *testing.T, args args) {
+// t.Helper()
+// },
+// }
+// }(),
+// */
+// }
+//
+// for _, tc := range tests {
+// test := tc
+// t.Run(test.name, func(tt *testing.T) {
+// tt.Parallel()
+// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent())
+// if test.beforeFunc != nil {
+// test.beforeFunc(tt, test.args)
+// }
+// if test.afterFunc != nil {
+// defer test.afterFunc(tt, test.args)
+// }
+// checkFunc := test.checkFunc
+// if test.checkFunc == nil {
+// checkFunc = defaultCheckFunc
+// }
+//
+// got := NewWeighted(test.args.n)
+// if err := checkFunc(test.want, got); err != nil {
+// tt.Errorf("error = %v", err)
+// }
+//
+// })
+// }
+// }
+//
+// func TestWeighted_Acquire(t *testing.T) {
+// type args struct {
+// ctx context.Context
+// n int64
+// }
+// type fields struct {
+// size int64
+// cur int64
+// waiters list.List
+// }
+// type want struct {
+// err error
+// }
+// type test struct {
+// name string
+// args args
+// fields fields
+// want want
+// checkFunc func(want, error) error
+// beforeFunc func(*testing.T, args)
+// afterFunc func(*testing.T, args)
+// }
+// defaultCheckFunc := func(w want, err error) error {
+// if !errors.Is(err, w.err) {
+// return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err)
+// }
+// return nil
+// }
+// tests := []test{
+// // TODO test cases
+// /*
+// {
+// name: "test_case_1",
+// args: args {
+// ctx:nil,
+// n:0,
+// },
+// fields: fields {
+// size:0,
+// cur:0,
+// waiters:nil,
+// },
+// want: want{},
+// checkFunc: defaultCheckFunc,
+// beforeFunc: func(t *testing.T, args args) {
+// t.Helper()
+// },
+// afterFunc: func(t *testing.T, args args) {
+// t.Helper()
+// },
+// },
+// */
+//
+// // TODO test cases
+// /*
+// func() test {
+// return test {
+// name: "test_case_2",
+// args: args {
+// ctx:nil,
+// n:0,
+// },
+// fields: fields {
+// size:0,
+// cur:0,
+// waiters:nil,
+// },
+// want: want{},
+// checkFunc: defaultCheckFunc,
+// beforeFunc: func(t *testing.T, args args) {
+// t.Helper()
+// },
+// afterFunc: func(t *testing.T, args args) {
+// t.Helper()
+// },
+// }
+// }(),
+// */
+// }
+//
+// for _, tc := range tests {
+// test := tc
+// t.Run(test.name, func(tt *testing.T) {
+// tt.Parallel()
+// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent())
+// if test.beforeFunc != nil {
+// test.beforeFunc(tt, test.args)
+// }
+// if test.afterFunc != nil {
+// defer test.afterFunc(tt, test.args)
+// }
+// checkFunc := test.checkFunc
+// if test.checkFunc == nil {
+// checkFunc = defaultCheckFunc
+// }
+// s := &Weighted{
+// size: test.fields.size,
+// cur: test.fields.cur,
+// waiters: test.fields.waiters,
+// }
+//
+// err := s.Acquire(test.args.ctx, test.args.n)
+// if err := checkFunc(test.want, err); err != nil {
+// tt.Errorf("error = %v", err)
+// }
+//
+// })
+// }
+// }
+//
+// func TestWeighted_TryAcquire(t *testing.T) {
+// type args struct {
+// n int64
+// }
+// type fields struct {
+// size int64
+// cur int64
+// waiters list.List
+// }
+// type want struct {
+// want bool
+// }
+// type test struct {
+// name string
+// args args
+// fields fields
+// want want
+// checkFunc func(want, bool) error
+// beforeFunc func(*testing.T, args)
+// afterFunc func(*testing.T, args)
+// }
+// defaultCheckFunc := func(w want, got bool) error {
+// if !reflect.DeepEqual(got, w.want) {
+// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", got, w.want)
+// }
+// return nil
+// }
+// tests := []test{
+// // TODO test cases
+// /*
+// {
+// name: "test_case_1",
+// args: args {
+// n:0,
+// },
+// fields: fields {
+// size:0,
+// cur:0,
+// waiters:nil,
+// },
+// want: want{},
+// checkFunc: defaultCheckFunc,
+// beforeFunc: func(t *testing.T, args args) {
+// t.Helper()
+// },
+// afterFunc: func(t *testing.T, args args) {
+// t.Helper()
+// },
+// },
+// */
+//
+// // TODO test cases
+// /*
+// func() test {
+// return test {
+// name: "test_case_2",
+// args: args {
+// n:0,
+// },
+// fields: fields {
+// size:0,
+// cur:0,
+// waiters:nil,
+// },
+// want: want{},
+// checkFunc: defaultCheckFunc,
+// beforeFunc: func(t *testing.T, args args) {
+// t.Helper()
+// },
+// afterFunc: func(t *testing.T, args args) {
+// t.Helper()
+// },
+// }
+// }(),
+// */
+// }
+//
+// for _, tc := range tests {
+// test := tc
+// t.Run(test.name, func(tt *testing.T) {
+// tt.Parallel()
+// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent())
+// if test.beforeFunc != nil {
+// test.beforeFunc(tt, test.args)
+// }
+// if test.afterFunc != nil {
+// defer test.afterFunc(tt, test.args)
+// }
+// checkFunc := test.checkFunc
+// if test.checkFunc == nil {
+// checkFunc = defaultCheckFunc
+// }
+// s := &Weighted{
+// size: test.fields.size,
+// cur: test.fields.cur,
+// waiters: test.fields.waiters,
+// }
+//
+// got := s.TryAcquire(test.args.n)
+// if err := checkFunc(test.want, got); err != nil {
+// tt.Errorf("error = %v", err)
+// }
+//
+// })
+// }
+// }
+//
+// func TestWeighted_Release(t *testing.T) {
+// type args struct {
+// n int64
+// }
+// type fields struct {
+// size int64
+// cur int64
+// waiters list.List
+// }
+// type want struct {
+// }
+// type test struct {
+// name string
+// args args
+// fields fields
+// want want
+// checkFunc func(want) error
+// beforeFunc func(*testing.T, args)
+// afterFunc func(*testing.T, args)
+// }
+// defaultCheckFunc := func(w want) error {
+// return nil
+// }
+// tests := []test{
+// // TODO test cases
+// /*
+// {
+// name: "test_case_1",
+// args: args {
+// n:0,
+// },
+// fields: fields {
+// size:0,
+// cur:0,
+// waiters:nil,
+// },
+// want: want{},
+// checkFunc: defaultCheckFunc,
+// beforeFunc: func(t *testing.T, args args) {
+// t.Helper()
+// },
+// afterFunc: func(t *testing.T, args args) {
+// t.Helper()
+// },
+// },
+// */
+//
+// // TODO test cases
+// /*
+// func() test {
+// return test {
+// name: "test_case_2",
+// args: args {
+// n:0,
+// },
+// fields: fields {
+// size:0,
+// cur:0,
+// waiters:nil,
+// },
+// want: want{},
+// checkFunc: defaultCheckFunc,
+// beforeFunc: func(t *testing.T, args args) {
+// t.Helper()
+// },
+// afterFunc: func(t *testing.T, args args) {
+// t.Helper()
+// },
+// }
+// }(),
+// */
+// }
+//
+// for _, tc := range tests {
+// test := tc
+// t.Run(test.name, func(tt *testing.T) {
+// tt.Parallel()
+// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent())
+// if test.beforeFunc != nil {
+// test.beforeFunc(tt, test.args)
+// }
+// if test.afterFunc != nil {
+// defer test.afterFunc(tt, test.args)
+// }
+// checkFunc := test.checkFunc
+// if test.checkFunc == nil {
+// checkFunc = defaultCheckFunc
+// }
+// s := &Weighted{
+// size: test.fields.size,
+// cur: test.fields.cur,
+// waiters: test.fields.waiters,
+// }
+//
+// s.Release(test.args.n)
+// if err := checkFunc(test.want); err != nil {
+// tt.Errorf("error = %v", err)
+// }
+// })
+// }
+// }
+//
+// func TestWeighted_Resize(t *testing.T) {
+// type args struct {
+// n int64
+// }
+// type fields struct {
+// size int64
+// cur int64
+// waiters list.List
+// }
+// type want struct {
+// }
+// type test struct {
+// name string
+// args args
+// fields fields
+// want want
+// checkFunc func(want) error
+// beforeFunc func(*testing.T, args)
+// afterFunc func(*testing.T, args)
+// }
+// defaultCheckFunc := func(w want) error {
+// return nil
+// }
+// tests := []test{
+// // TODO test cases
+// /*
+// {
+// name: "test_case_1",
+// args: args {
+// n:0,
+// },
+// fields: fields {
+// size:0,
+// cur:0,
+// waiters:nil,
+// },
+// want: want{},
+// checkFunc: defaultCheckFunc,
+// beforeFunc: func(t *testing.T, args args) {
+// t.Helper()
+// },
+// afterFunc: func(t *testing.T, args args) {
+// t.Helper()
+// },
+// },
+// */
+//
+// // TODO test cases
+// /*
+// func() test {
+// return test {
+// name: "test_case_2",
+// args: args {
+// n:0,
+// },
+// fields: fields {
+// size:0,
+// cur:0,
+// waiters:nil,
+// },
+// want: want{},
+// checkFunc: defaultCheckFunc,
+// beforeFunc: func(t *testing.T, args args) {
+// t.Helper()
+// },
+// afterFunc: func(t *testing.T, args args) {
+// t.Helper()
+// },
+// }
+// }(),
+// */
+// }
+//
+// for _, tc := range tests {
+// test := tc
+// t.Run(test.name, func(tt *testing.T) {
+// tt.Parallel()
+// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent())
+// if test.beforeFunc != nil {
+// test.beforeFunc(tt, test.args)
+// }
+// if test.afterFunc != nil {
+// defer test.afterFunc(tt, test.args)
+// }
+// checkFunc := test.checkFunc
+// if test.checkFunc == nil {
+// checkFunc = defaultCheckFunc
+// }
+// s := &Weighted{
+// size: test.fields.size,
+// cur: test.fields.cur,
+// waiters: test.fields.waiters,
+// }
+//
+// s.Resize(test.args.n)
+// if err := checkFunc(test.want); err != nil {
+// tt.Errorf("error = %v", err)
+// }
+// })
+// }
+// }
+//
+// func TestWeighted_notifyWaiters(t *testing.T) {
+// type fields struct {
+// size int64
+// cur int64
+// waiters list.List
+// }
+// type want struct {
+// }
+// type test struct {
+// name string
+// fields fields
+// want want
+// checkFunc func(want) error
+// beforeFunc func(*testing.T)
+// afterFunc func(*testing.T)
+// }
+// defaultCheckFunc := func(w want) error {
+// return nil
+// }
+// tests := []test{
+// // TODO test cases
+// /*
+// {
+// name: "test_case_1",
+// fields: fields {
+// size:0,
+// cur:0,
+// waiters:nil,
+// },
+// want: want{},
+// checkFunc: defaultCheckFunc,
+// beforeFunc: func(t *testing.T,) {
+// t.Helper()
+// },
+// afterFunc: func(t *testing.T,) {
+// t.Helper()
+// },
+// },
+// */
+//
+// // TODO test cases
+// /*
+// func() test {
+// return test {
+// name: "test_case_2",
+// fields: fields {
+// size:0,
+// cur:0,
+// waiters:nil,
+// },
+// want: want{},
+// checkFunc: defaultCheckFunc,
+// beforeFunc: func(t *testing.T,) {
+// t.Helper()
+// },
+// afterFunc: func(t *testing.T,) {
+// t.Helper()
+// },
+// }
+// }(),
+// */
+// }
+//
+// for _, tc := range tests {
+// test := tc
+// t.Run(test.name, func(tt *testing.T) {
+// tt.Parallel()
+// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent())
+// if test.beforeFunc != nil {
+// test.beforeFunc(tt)
+// }
+// if test.afterFunc != nil {
+// defer test.afterFunc(tt)
+// }
+// checkFunc := test.checkFunc
+// if test.checkFunc == nil {
+// checkFunc = defaultCheckFunc
+// }
+// s := &Weighted{
+// size: test.fields.size,
+// cur: test.fields.cur,
+// waiters: test.fields.waiters,
+// }
+//
+// s.notifyWaiters()
+// if err := checkFunc(test.want); err != nil {
+// tt.Errorf("error = %v", err)
+// }
+// })
+// }
+// }
diff --git a/internal/sync/singleflight/singleflight.go b/internal/sync/singleflight/singleflight.go
new file mode 100644
index 00000000000..c20730ab9f8
--- /dev/null
+++ b/internal/sync/singleflight/singleflight.go
@@ -0,0 +1,240 @@
+//
+// Copyright (c) 2009-2023 The Go Authors. All rights resered.
+// Modified by vdaas.org vald team
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+// * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+//
+
+// Package singleflight provides a duplicate function call suppression
+// mechanism.
+package singleflight
+
+import (
+ "bytes"
+ "context"
+ "errors"
+ "fmt"
+ "runtime/debug"
+ "sync/atomic"
+
+ "github.com/vdaas/vald/internal/sync"
+)
+
+// errGoexit indicates the runtime.Goexit was called in
+// the user given function.
+var errGoexit = errors.New("runtime.Goexit was called")
+
+// A panicError is an arbitrary value recovered from a panic
+// with the stack trace during the execution of given function.
+type panicError struct {
+ value interface{}
+ stack []byte
+}
+
+// Error implements error interface.
+func (p *panicError) Error() string {
+ return fmt.Sprintf("%v\n\n%s", p.value, p.stack)
+}
+
+func newPanicError(v interface{}) error {
+ stack := debug.Stack()
+
+ // The first line of the stack trace is of the form "goroutine N [status]:"
+ // but by the time the panic reaches Do the goroutine may no longer exist
+ // and its status will have changed. Trim out the misleading line.
+ if line := bytes.IndexByte(stack[:], '\n'); line >= 0 {
+ stack = stack[line+1:]
+ }
+ return &panicError{value: v, stack: stack}
+}
+
+// call is an in-flight or completed singleflight.Do call
+type call[V any] struct {
+ mu sync.Mutex
+
+ // These fields are written once before the WaitGroup is done
+ // and are only read after the WaitGroup is done.
+ val V
+ err error
+
+ // These fields are read and written with the singleflight
+ // mutex held before the WaitGroup is done, and are read but
+ // not written after the WaitGroup is done.
+ dups uint64
+ chans []chan<- Result[V]
+
+ chmu sync.RWMutex
+}
+
+func (c *call[V]) appendChans(chs ...chan<- Result[V]) {
+ c.chmu.Lock()
+ defer c.chmu.Unlock()
+ if len(c.chans) == 0 {
+ c.chans = make([]chan<- Result[V], 0, len(chs))
+ }
+ c.chans = append(c.chans, chs...)
+}
+
+// Group represents interface for zero time cache.
+type Group[V any] interface {
+ Do(ctx context.Context, key string, fn func(context.Context) (V, error)) (v V, shared bool, err error)
+ DoChan(ctx context.Context, key string, fn func(context.Context) (V, error)) <-chan Result[V]
+ Forget(key string)
+}
+
+// group represents a class of work and forms a namespace in
+// which units of work can be executed with duplicate suppression.
+type group[V any] struct {
+ m sync.Map[string, *call[V]]
+}
+
+// Result holds the results of Do, so they can be passed
+// on a channel.
+type Result[V any] struct {
+ Val V
+ Err error
+ Shared bool
+}
+
+// New returns Group implementation.
+func New[V any]() Group[V] {
+ return new(group[V])
+}
+
+// Do executes and returns the results of the given function, making
+// sure that only one execution is in-flight for a given key at a
+// time. If a duplicate comes in, the duplicate caller waits for the
+// original to complete and receives the same results.
+// The return value shared indicates whether v was given to multiple callers.
+func (g *group[V]) Do(ctx context.Context, key string, fn func(context.Context) (V, error)) (v V, shared bool, err error) {
+ c, loaded := g.m.LoadOrStore(key, new(call[V]))
+ if loaded {
+ atomic.AddUint64(&c.dups, 1)
+ c.mu.Lock()
+ c.mu.Unlock()
+ v, err = c.val, c.err
+ return v, true, err
+ }
+ c.mu.Lock()
+ g.doCall(ctx, c, key, fn)
+ c.mu.Unlock()
+ return c.val, atomic.LoadUint64(&c.dups) > 0, c.err
+}
+
+// DoChan is like Do but returns a channel that will receive the
+// results when they are ready.
+//
+// The returned channel will not be closed.
+func (g *group[V]) DoChan(ctx context.Context, key string, fn func(context.Context) (V, error)) <-chan Result[V] {
+ c, loaded := g.m.LoadOrStore(key, new(call[V]))
+ ch := make(chan Result[V], 1)
+ c.appendChans(ch)
+ if loaded {
+ atomic.AddUint64(&c.dups, 1)
+ return ch
+ }
+ go g.doCall(ctx, c, key, fn)
+
+ return ch
+}
+
+// doCall handles the single call for a key.
+func (g *group[V]) doCall(ctx context.Context, c *call[V], key string, fn func(ctx context.Context) (V, error)) {
+ normalReturn := false
+ recovered := false
+
+ // use double-defer to distinguish panic from runtime.Goexit,
+ // more details see https://golang.org/cl/134395
+ defer func() {
+ // the given function invoked runtime.Goexit
+ if !normalReturn && !recovered {
+ c.err = errGoexit
+ }
+
+ g.m.LoadAndDelete(key)
+ if e, ok := c.err.(*panicError); ok {
+ // In order to prevent the waiting channels from being blocked forever,
+ // needs to ensure that this panic cannot be recovered.
+ c.chmu.RLock()
+ defer c.chmu.RUnlock()
+ if len(c.chans) > 0 {
+ go panic(e)
+ select {} // Keep this goroutine around so that it will appear in the crash dump.
+ } else {
+ panic(e)
+ }
+ } else if c.err == errGoexit {
+ // Already in the process of goexit, no need to call again
+ } else {
+ // Normal return
+ c.chmu.RLock()
+ defer c.chmu.RUnlock()
+ for _, ch := range c.chans {
+ select {
+ case <-ctx.Done():
+ return
+ case ch <- Result[V]{
+ Val: c.val,
+ Err: c.err,
+ Shared: atomic.LoadUint64(&c.dups) > 0,
+ }:
+ }
+ }
+ }
+ }()
+
+ func() {
+ defer func() {
+ if !normalReturn {
+ // Ideally, we would wait to take a stack trace until we've determined
+ // whether this is a panic or a runtime.Goexit.
+ //
+ // Unfortunately, the only way we can distinguish the two is to see
+ // whether the recover stopped the goroutine from terminating, and by
+ // the time we know that, the part of the stack trace relevant to the
+ // panic has been discarded.
+ if r := recover(); r != nil {
+ c.err = newPanicError(r)
+ }
+ }
+ }()
+
+ c.val, c.err = fn(ctx)
+ normalReturn = true
+ }()
+
+ if !normalReturn {
+ recovered = true
+ }
+}
+
+// Forget tells the singleflight to forget about a key. Future calls
+// to Do for this key will call the function rather than waiting for
+// an earlier call to complete.
+func (g *group[V]) Forget(key string) {
+ g.m.LoadAndDelete(key)
+}
diff --git a/internal/sync/singleflight/singleflight_test.go b/internal/sync/singleflight/singleflight_test.go
new file mode 100644
index 00000000000..28ace0609f3
--- /dev/null
+++ b/internal/sync/singleflight/singleflight_test.go
@@ -0,0 +1,640 @@
+//
+// Copyright (C) 2019-2023 vdaas.org vald team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+// Package singleflight represents zero time caching
+package singleflight
+
+import (
+ "bytes"
+ "context"
+ "fmt"
+ "os"
+ "os/exec"
+ "reflect"
+ "runtime"
+ "runtime/debug"
+ "strings"
+ "sync/atomic"
+ "testing"
+ "time"
+
+ "github.com/vdaas/vald/internal/errors"
+ "github.com/vdaas/vald/internal/sync"
+ "github.com/vdaas/vald/internal/test/goleak"
+)
+
+func TestNew(t *testing.T) {
+ type want struct {
+ want Group[any]
+ }
+ type test struct {
+ name string
+ want want
+ checkFunc func(want, Group[any]) error
+ beforeFunc func()
+ afterFunc func()
+ }
+ defaultCheckFunc := func(w want, got Group[any]) error {
+ if !reflect.DeepEqual(got, w.want) {
+ return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", got, w.want)
+ }
+ return nil
+ }
+ tests := []test{
+ {
+ name: "returns Group implementation",
+ want: want{
+ want: &group[any]{},
+ },
+ },
+ }
+
+ for _, tc := range tests {
+ test := tc
+ t.Run(test.name, func(tt *testing.T) {
+ defer goleak.VerifyNone(tt)
+ if test.beforeFunc != nil {
+ test.beforeFunc()
+ }
+ if test.afterFunc != nil {
+ defer test.afterFunc()
+ }
+ checkFunc := test.checkFunc
+ if test.checkFunc == nil {
+ checkFunc = defaultCheckFunc
+ }
+
+ got := New[any]()
+ if err := checkFunc(test.want, got); err != nil {
+ tt.Errorf("error = %v", err)
+ }
+ })
+ }
+}
+
+func Test_group_Do(t *testing.T) {
+ type args[V any] struct {
+ ctx context.Context
+ key string
+ fn func(context.Context) (V, error)
+ }
+ type want[V any] struct {
+ wantV V
+ wantShared bool
+ err error
+ }
+ type test[V any] struct {
+ name string
+ args args[V]
+ want want[V]
+ beforeFunc func(args[V])
+ execFunc func(*testing.T, args[V]) (V, bool, error)
+ checkFunc func(want[V], V, bool, error) error
+ afterFunc func(args[V])
+ }
+ tests := []test[string]{
+ func() test[string] {
+ // routine1
+ key1 := "req_1"
+ var cnt1 uint32
+
+ // the unparam lint rule is disabled here because we need to match the interface to singleflight implementation.
+ // if this rule is not disabled, if will warns that the error will always return null.
+ //nolint:unparam
+ fn1 := func(context.Context) (string, error) {
+ atomic.AddUint32(&cnt1, 1)
+ return "res_1", nil
+ }
+
+ // routine 2
+ key2 := "req_2"
+ var cnt2 uint32
+
+ // the unparam lint rule is disabled here because we need to match the interface to singleflight implementation.
+ // if this rule is not disabled, if will warns that the error will always return null.
+ //nolint:unparam
+ fn2 := func(context.Context) (string, error) {
+ atomic.AddUint32(&cnt2, 1)
+ return "res_2", nil
+ }
+
+ return test[string]{
+ name: "returns (v, false, nil) when Do is called with another key",
+ args: args[string]{
+ key: key1,
+ ctx: context.Background(),
+ fn: fn1,
+ },
+ want: want[string]{
+ wantV: "res_1",
+ wantShared: false,
+ err: nil,
+ },
+ execFunc: func(t *testing.T, a args[string]) (got string, gotShared bool, err error) {
+ t.Helper()
+ g := New[string]()
+
+ wg := new(sync.WaitGroup)
+ wg.Add(1)
+ go func() {
+ got, gotShared, err = g.Do(a.ctx, a.key, a.fn)
+ wg.Done()
+ }()
+
+ wg.Add(1)
+ go func() {
+ _, _, _ = g.Do(a.ctx, key2, fn2)
+ wg.Done()
+ }()
+
+ wg.Wait()
+ return got, gotShared, err
+ },
+ checkFunc: func(w want[string], gotV string, gotShared bool, err error) error {
+ if got, want := int(atomic.LoadUint32(&cnt1)), 1; got != want {
+ return errors.Errorf("cnt got = %d, want = %d", got, want)
+ }
+ if got, want := int(atomic.LoadUint32(&cnt2)), 1; got != want {
+ return errors.Errorf("cnt got = %d, want = %d", got, want)
+ }
+ if !errors.Is(err, w.err) {
+ return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err)
+ }
+ if !reflect.DeepEqual(gotV, w.wantV) {
+ return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", gotV, w.wantV)
+ }
+ if !reflect.DeepEqual(gotShared, w.wantShared) {
+ return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", gotShared, w.wantShared)
+ }
+ return nil
+ },
+ }
+ }(),
+ func() test[string] {
+ // routine1
+ var cnt1 uint32
+
+ // the unparam lint rule is disabled here because we need to match the interface to singleflight implementation.
+ // if this rule is not disabled, if will warns that the error will always return null.
+ //nolint:unparam
+ fn1 := func(context.Context) (string, error) {
+ atomic.AddUint32(&cnt1, 1)
+ time.Sleep(time.Millisecond * 500)
+ return "res_1", nil
+ }
+
+ // routine 2
+ var cnt2 uint32
+
+ // the unparam lint rule is disabled here because we need to match the interface to singleflight implementation.
+ // if this rule is not disabled, if will warns that the error will always return null.
+ //nolint:unparam
+ fn2 := func(context.Context) (string, error) {
+ atomic.AddUint32(&cnt2, 1)
+ return "res_2", nil
+ }
+
+ w := want[string]{
+ wantV: "res_1",
+ wantShared: true,
+ err: nil,
+ }
+
+ checkFunc := func(w want[string], gotV string, gotShared bool, err error) error {
+ c1 := int(atomic.LoadUint32(&cnt1))
+ c2 := int(atomic.LoadUint32(&cnt2))
+ // since there is a chance that the go routine 2 is executed before routine 1, we need to check if either one is executed
+ if !((c1 == 1 && c2 == 0) || (c1 == 0 && c2 == 1)) {
+ return errors.Errorf("cnt1 and cnt2 is executed, %d, %d", c1, c2)
+ }
+ if !errors.Is(err, w.err) {
+ return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err)
+ }
+ if !reflect.DeepEqual(gotV, w.wantV) {
+ return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", gotV, w.wantV)
+ }
+ if !reflect.DeepEqual(gotShared, w.wantShared) {
+ return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", gotShared, w.wantShared)
+ }
+ return nil
+ }
+
+ return test[string]{
+ name: "returns (v, true, nil) when Do is called with the same key",
+ args: args[string]{
+ key: "req_1",
+ ctx: context.Background(),
+ fn: fn1,
+ },
+ want: w,
+ execFunc: func(t *testing.T, a args[string]) (string, bool, error) {
+ t.Helper()
+
+ g := New[string]()
+ wg := new(sync.WaitGroup)
+ var got, got1 string
+ var gotShared, gotShared1 bool
+ var err, err1 error
+
+ wg.Add(1)
+ go func() {
+ got, gotShared, err = g.Do(a.ctx, a.key, fn1)
+ wg.Done()
+ }()
+
+ // call with the same key but with another function
+ wg.Add(1)
+ time.Sleep(time.Millisecond * 100)
+ go func() {
+ got1, gotShared1, err1 = g.Do(a.ctx, a.key, fn2)
+ wg.Done()
+ }()
+
+ wg.Wait()
+
+ if err := checkFunc(w, got1, gotShared1, err1); err != nil {
+ t.Fatal(err)
+ }
+
+ return got, gotShared, err
+ },
+ checkFunc: checkFunc,
+ }
+ }(),
+ }
+
+ for _, tc := range tests {
+ test := tc
+ t.Run(test.name, func(tt *testing.T) {
+ defer goleak.VerifyNone(tt)
+ if test.afterFunc != nil {
+ defer test.afterFunc(test.args)
+ }
+ if test.beforeFunc != nil {
+ test.beforeFunc(test.args)
+ }
+
+ gotV, gotShared, err := test.execFunc(t, test.args)
+
+ if err := test.checkFunc(test.want, gotV, gotShared, err); err != nil {
+ tt.Errorf("error = %v", err)
+ }
+ })
+ }
+}
+
+func TestDo(t *testing.T) {
+ g := New[string]()
+ v, _, err := g.Do(context.Background(), "key", func(context.Context) (string, error) {
+ return "bar", nil
+ })
+ if got, want := fmt.Sprintf("%v (%T)", v, v), "bar (string)"; got != want {
+ t.Errorf("Do = %v; want %v", got, want)
+ }
+ if err != nil {
+ t.Errorf("Do error = %v", err)
+ }
+}
+
+func TestDoErr(t *testing.T) {
+ g := New[any]()
+ someErr := errors.New("Some error")
+ v, _, err := g.Do(context.Background(), "key", func(context.Context) (any, error) {
+ return nil, someErr
+ })
+ if err != someErr {
+ t.Errorf("Do error = %v; want someErr %v", err, someErr)
+ }
+ if v != nil {
+ t.Errorf("unexpected non-nil value %#v", v)
+ }
+}
+
+func TestDoDupSuppress(t *testing.T) {
+ g := New[string]()
+ var wg1, wg2 sync.WaitGroup
+ c := make(chan string, 1)
+ var calls int32
+ fn := func(ctx context.Context) (string, error) {
+ if atomic.AddInt32(&calls, 1) == 1 {
+ // First invocation.
+ wg1.Done()
+ }
+ v := <-c
+ c <- v // pump; make available for any future calls
+
+ time.Sleep(10 * time.Millisecond) // let more goroutines enter Do
+
+ return v, nil
+ }
+
+ const n = 10
+ wg1.Add(1)
+ for i := 0; i < n; i++ {
+ wg1.Add(1)
+ wg2.Add(1)
+ go func() {
+ defer wg2.Done()
+ wg1.Done()
+ s, _, err := g.Do(context.Background(), "key", fn)
+ if err != nil {
+ t.Errorf("Do error: %v", err)
+ return
+ }
+ if s != "bar" {
+ t.Errorf("Do = %T %v; want %q", s, s, "bar")
+ }
+ }()
+ }
+ wg1.Wait()
+ // At least one goroutine is in fn now and all of them have at
+ // least reached the line before the Do.
+ c <- "bar"
+ wg2.Wait()
+ if got := atomic.LoadInt32(&calls); got <= 0 || got >= n {
+ t.Errorf("number of calls = %d; want over 0 and less than %d", got, n)
+ }
+}
+
+// Test that singleflight behaves correctly after Forget called.
+// See https://github.com/golang/go/issues/31420
+func TestForget(t *testing.T) {
+ g := New[int]()
+
+ var (
+ firstStarted = make(chan struct{})
+ unblockFirst = make(chan struct{})
+ firstFinished = make(chan struct{})
+ )
+
+ go func() {
+ g.Do(context.Background(), "key", func(ctx context.Context) (i int, e error) {
+ close(firstStarted)
+ <-unblockFirst
+ close(firstFinished)
+ return
+ })
+ }()
+ <-firstStarted
+ g.Forget("key")
+
+ unblockSecond := make(chan struct{})
+ secondResult := g.DoChan(context.Background(), "key", func(ctx context.Context) (i int, e error) {
+ t.Log(2, "key")
+ <-unblockSecond
+ return 2, nil
+ })
+
+ close(unblockFirst)
+ <-firstFinished
+
+ thirdResult := g.DoChan(context.Background(), "key", func(ctx context.Context) (i int, e error) {
+ t.Log(3, "key")
+ return 3, nil
+ })
+
+ close(unblockSecond)
+ <-secondResult
+ r := <-thirdResult
+ if r.Val != 2 {
+ t.Errorf("We should receive result produced by second call, expected: 2, got %d", r.Val)
+ }
+}
+
+func TestDoChan(t *testing.T) {
+ g := New[string]()
+ ch := g.DoChan(context.Background(), "key", func(ctx context.Context) (string, error) {
+ return "bar", nil
+ })
+
+ res := <-ch
+ v := res.Val
+ err := res.Err
+ if got, want := fmt.Sprintf("%v (%T)", v, v), "bar (string)"; got != want {
+ t.Errorf("Do = %v; want %v", got, want)
+ }
+ if err != nil {
+ t.Errorf("Do error = %v", err)
+ }
+}
+
+// Test singleflight behaves correctly after Do panic.
+// See https://github.com/golang/go/issues/41133
+func TestPanicDo(t *testing.T) {
+ g := New[any]()
+ fn := func(ctx context.Context) (any, error) {
+ panic("invalid memory address or nil pointer dereference")
+ }
+
+ const n = 5
+ waited := int32(n)
+ panicCount := int32(0)
+ done := make(chan struct{})
+ for i := 0; i < n; i++ {
+ go func() {
+ defer func() {
+ if err := recover(); err != nil {
+ t.Logf("Got panic: %v\n%s", err, debug.Stack())
+ atomic.AddInt32(&panicCount, 1)
+ }
+
+ if atomic.AddInt32(&waited, -1) == 0 {
+ close(done)
+ }
+ }()
+
+ g.Do(context.Background(), "key", fn)
+ }()
+ }
+
+ select {
+ case <-done:
+ if panicCount != n {
+ t.Errorf("Expect %d panic, but got %d", n, panicCount)
+ }
+ case <-time.After(time.Second):
+ t.Fatalf("Do hangs")
+ }
+}
+
+func TestGoexitDo(t *testing.T) {
+ g := New[any]()
+ fn := func(context.Context) (interface{}, error) {
+ runtime.Goexit()
+ return nil, nil
+ }
+
+ const n = 5
+ waited := int32(n)
+ done := make(chan struct{})
+ for i := 0; i < n; i++ {
+ go func() {
+ var err error
+ defer func() {
+ if err != nil {
+ t.Errorf("Error should be nil, but got: %v", err)
+ }
+ if atomic.AddInt32(&waited, -1) == 0 {
+ close(done)
+ }
+ }()
+ _, _, err = g.Do(context.Background(), "key", fn)
+ }()
+ }
+
+ select {
+ case <-done:
+ case <-time.After(time.Second):
+ t.Fatalf("Do hangs")
+ }
+}
+
+func executable(t testing.TB) string {
+ exe, err := os.Executable()
+ if err != nil {
+ t.Skipf("skipping: test executable not found")
+ }
+
+ // Control case: check whether exec.Command works at all.
+ // (For example, it might fail with a permission error on iOS.)
+ cmd := exec.Command(exe, "-test.list=^$")
+ cmd.Env = []string{}
+ if err := cmd.Run(); err != nil {
+ t.Skipf("skipping: exec appears not to work on %s: %v", runtime.GOOS, err)
+ }
+
+ return exe
+}
+
+func TestPanicDoChan(t *testing.T) {
+ if os.Getenv("TEST_PANIC_DOCHAN") != "" {
+ defer func() {
+ recover()
+ }()
+
+ g := New[any]()
+ ch := g.DoChan(context.Background(), "", func(context.Context) (interface{}, error) {
+ panic("Panicking in DoChan")
+ })
+ <-ch
+ t.Fatalf("DoChan unexpectedly returned")
+ }
+
+ t.Parallel()
+
+ cmd := exec.Command(executable(t), "-test.run="+t.Name(), "-test.v")
+ cmd.Env = append(os.Environ(), "TEST_PANIC_DOCHAN=1")
+ out := new(bytes.Buffer)
+ cmd.Stdout = out
+ cmd.Stderr = out
+ if err := cmd.Start(); err != nil {
+ t.Fatal(err)
+ }
+
+ err := cmd.Wait()
+ t.Logf("%s:\n%s", strings.Join(cmd.Args, " "), out)
+ if err == nil {
+ t.Errorf("Test subprocess passed; want a crash due to panic in DoChan")
+ }
+ if bytes.Contains(out.Bytes(), []byte("DoChan unexpectedly")) {
+ t.Errorf("Test subprocess failed with an unexpected failure mode.")
+ }
+ if !bytes.Contains(out.Bytes(), []byte("Panicking in DoChan")) {
+ t.Errorf("Test subprocess failed, but the crash isn't caused by panicking in DoChan")
+ }
+}
+
+func TestPanicDoSharedByDoChan(t *testing.T) {
+ if os.Getenv("TEST_PANIC_DOCHAN") != "" {
+ blocked := make(chan struct{})
+ unblock := make(chan struct{})
+
+ g := New[any]()
+ go func() {
+ defer func() {
+ recover()
+ }()
+ g.Do(context.Background(), "", func(context.Context) (interface{}, error) {
+ close(blocked)
+ <-unblock
+ panic("Panicking in Do")
+ })
+ }()
+
+ <-blocked
+ ch := g.DoChan(context.Background(), "", func(context.Context) (interface{}, error) {
+ panic("DoChan unexpectedly executed callback")
+ })
+ close(unblock)
+ <-ch
+ t.Fatalf("DoChan unexpectedly returned")
+ }
+
+ t.Parallel()
+
+ cmd := exec.Command(executable(t), "-test.run="+t.Name(), "-test.v")
+ cmd.Env = append(os.Environ(), "TEST_PANIC_DOCHAN=1")
+ out := new(bytes.Buffer)
+ cmd.Stdout = out
+ cmd.Stderr = out
+ if err := cmd.Start(); err != nil {
+ t.Fatal(err)
+ }
+
+ err := cmd.Wait()
+ t.Logf("%s:\n%s", strings.Join(cmd.Args, " "), out)
+ if err == nil {
+ t.Errorf("Test subprocess passed; want a crash due to panic in Do shared by DoChan")
+ }
+ if bytes.Contains(out.Bytes(), []byte("DoChan unexpectedly")) {
+ t.Errorf("Test subprocess failed with an unexpected failure mode.")
+ }
+ if !bytes.Contains(out.Bytes(), []byte("Panicking in Do")) {
+ t.Errorf("Test subprocess failed, but the crash isn't caused by panicking in Do")
+ }
+}
+
+func ExampleGroup() {
+ g := New[string]()
+
+ block := make(chan struct{})
+ res1c := g.DoChan(context.Background(), "key", func(context.Context) (string, error) {
+ <-block
+ return "func 1", nil
+ })
+ res2c := g.DoChan(context.Background(), "key", func(context.Context) (string, error) {
+ <-block
+ return "func 2", nil
+ })
+ close(block)
+
+ res1 := <-res1c
+ res2 := <-res2c
+
+ // Results are shared by functions executed with duplicate keys.
+ fmt.Println("Shared:", res2.Shared)
+ // Only the first function is executed: it is registered and started with "key",
+ // and doesn't complete before the second funtion is registered with a duplicate key.
+ fmt.Println("Equal results:", res1.Val == res2.Val)
+ fmt.Println("Result:", res1.Val)
+
+ // Output:
+ // Shared: true
+ // Equal results: true
+ // Result: func 1
+}
+
+// NOT IMPLEMENTED BELOW
diff --git a/internal/test/comparator/comparators.go b/internal/test/comparator/comparators.go
index dcfcf7fb06d..085f7903261 100644
--- a/internal/test/comparator/comparators.go
+++ b/internal/test/comparator/comparators.go
@@ -15,9 +15,9 @@ package comparator
import (
"reflect"
- "sync"
"github.com/vdaas/vald/internal/errors"
+ "github.com/vdaas/vald/internal/sync"
)
var (
diff --git a/internal/timeutil/time_test.go b/internal/timeutil/time_test.go
index 577ba2a6a95..3f343cd46d0 100644
--- a/internal/timeutil/time_test.go
+++ b/internal/timeutil/time_test.go
@@ -80,6 +80,12 @@ func TestParse(t *testing.T) {
want: 0,
wantErr: true,
},
+ {
+ name: "returns 0 and incorrect string error when t is minus value",
+ t: "-1",
+ want: 0,
+ wantErr: true,
+ },
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
diff --git a/internal/worker/queue.go b/internal/worker/queue.go
index 12fb5fc7a1c..ccdc5ef02f5 100644
--- a/internal/worker/queue.go
+++ b/internal/worker/queue.go
@@ -23,9 +23,9 @@ import (
"sync/atomic"
"time"
- "github.com/vdaas/vald/internal/errgroup"
"github.com/vdaas/vald/internal/errors"
"github.com/vdaas/vald/internal/safety"
+ "github.com/vdaas/vald/internal/sync/errgroup"
)
// Queue represents the interface of queue.
diff --git a/internal/worker/queue_option.go b/internal/worker/queue_option.go
index 38c031bfc33..069c06e5ca9 100644
--- a/internal/worker/queue_option.go
+++ b/internal/worker/queue_option.go
@@ -18,7 +18,7 @@
package worker
import (
- "github.com/vdaas/vald/internal/errgroup"
+ "github.com/vdaas/vald/internal/sync/errgroup"
"github.com/vdaas/vald/internal/timeutil"
)
diff --git a/internal/worker/queue_option_test.go b/internal/worker/queue_option_test.go
index 52e82b0a32c..fb1e13aa403 100644
--- a/internal/worker/queue_option_test.go
+++ b/internal/worker/queue_option_test.go
@@ -22,8 +22,8 @@ import (
"testing"
"time"
- "github.com/vdaas/vald/internal/errgroup"
"github.com/vdaas/vald/internal/errors"
+ "github.com/vdaas/vald/internal/sync/errgroup"
"github.com/vdaas/vald/internal/test/goleak"
)
diff --git a/internal/worker/queue_test.go b/internal/worker/queue_test.go
index fe1c7e4ab06..87e0be11d47 100644
--- a/internal/worker/queue_test.go
+++ b/internal/worker/queue_test.go
@@ -24,8 +24,8 @@ import (
"testing"
"time"
- "github.com/vdaas/vald/internal/errgroup"
"github.com/vdaas/vald/internal/errors"
+ "github.com/vdaas/vald/internal/sync/errgroup"
"github.com/vdaas/vald/internal/test/comparator"
"github.com/vdaas/vald/internal/test/goleak"
)
diff --git a/internal/worker/worker.go b/internal/worker/worker.go
index 3118903071d..fad9060e895 100644
--- a/internal/worker/worker.go
+++ b/internal/worker/worker.go
@@ -22,11 +22,11 @@ import (
"reflect"
"sync/atomic"
- "github.com/vdaas/vald/internal/errgroup"
"github.com/vdaas/vald/internal/errors"
"github.com/vdaas/vald/internal/log"
"github.com/vdaas/vald/internal/observability/trace"
"github.com/vdaas/vald/internal/safety"
+ "github.com/vdaas/vald/internal/sync/errgroup"
)
// JobFunc represents the function of a job that works in the worker.
@@ -133,7 +133,7 @@ func (w *worker) startJobLoop(ctx context.Context) <-chan error {
w.eg.Go(safety.RecoverFunc(func() (err error) {
defer close(ech)
eg, ctx := errgroup.New(ctx)
- eg.Limitation(w.limitation)
+ eg.SetLimit(w.limitation)
limitation := make(chan struct{}, w.limitation)
defer close(limitation)
diff --git a/internal/worker/worker_option.go b/internal/worker/worker_option.go
index b105b82b094..3ec117f7e3b 100644
--- a/internal/worker/worker_option.go
+++ b/internal/worker/worker_option.go
@@ -18,7 +18,7 @@
package worker
import (
- "github.com/vdaas/vald/internal/errgroup"
+ "github.com/vdaas/vald/internal/sync/errgroup"
)
type WorkerOption func(w *worker) error
diff --git a/internal/worker/worker_test.go b/internal/worker/worker_test.go
index 0e0e9fe18ae..575e70aa21d 100644
--- a/internal/worker/worker_test.go
+++ b/internal/worker/worker_test.go
@@ -24,10 +24,10 @@ import (
"testing"
"time"
- "github.com/vdaas/vald/internal/errgroup"
"github.com/vdaas/vald/internal/errors"
"github.com/vdaas/vald/internal/log"
"github.com/vdaas/vald/internal/log/logger"
+ "github.com/vdaas/vald/internal/sync/errgroup"
"github.com/vdaas/vald/internal/test/comparator"
"github.com/vdaas/vald/internal/test/goleak"
)
diff --git a/k8s/agent/pdb.yaml b/k8s/agent/pdb.yaml
index c58182ba489..e44f015f5ee 100644
--- a/k8s/agent/pdb.yaml
+++ b/k8s/agent/pdb.yaml
@@ -14,7 +14,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
-apiVersion: policy/v1
+apiVersion: policy/v1beta1
kind: PodDisruptionBudget
metadata:
name: vald-agent-ngt
diff --git a/k8s/discoverer/pdb.yaml b/k8s/discoverer/pdb.yaml
index 28fda6d1b18..5dde0af42cf 100644
--- a/k8s/discoverer/pdb.yaml
+++ b/k8s/discoverer/pdb.yaml
@@ -14,7 +14,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
-apiVersion: policy/v1
+apiVersion: policy/v1beta1
kind: PodDisruptionBudget
metadata:
name: vald-discoverer
diff --git a/k8s/gateway/lb/hpa.yaml b/k8s/gateway/lb/hpa.yaml
index 39e47a14950..426b3254b43 100644
--- a/k8s/gateway/lb/hpa.yaml
+++ b/k8s/gateway/lb/hpa.yaml
@@ -14,7 +14,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
-apiVersion: autoscaling/v1
+apiVersion: autoscaling/v2
kind: HorizontalPodAutoscaler
metadata:
name: vald-lb-gateway
@@ -32,5 +32,11 @@ spec:
apiVersion: apps/v1
kind: Deployment
name: vald-lb-gateway
- targetCPUUtilizationPercentage: 80
+ metrics:
+ - type: Resource
+ resource:
+ name: cpu
+ target:
+ type: Utilization
+ averageUtilization: 80
status:
diff --git a/k8s/gateway/lb/pdb.yaml b/k8s/gateway/lb/pdb.yaml
index 3ccbd28e4c9..c5a3c864946 100644
--- a/k8s/gateway/lb/pdb.yaml
+++ b/k8s/gateway/lb/pdb.yaml
@@ -14,7 +14,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
-apiVersion: policy/v1
+apiVersion: policy/v1beta1
kind: PodDisruptionBudget
metadata:
name: vald-lb-gateway
diff --git a/k8s/manager/index/pdb.yaml b/k8s/manager/index/pdb.yaml
index a49e5c58278..294f05ef9f8 100644
--- a/k8s/manager/index/pdb.yaml
+++ b/k8s/manager/index/pdb.yaml
@@ -14,7 +14,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
-apiVersion: policy/v1
+apiVersion: policy/v1beta1
kind: PodDisruptionBudget
metadata:
name: vald-manager-index
diff --git a/pkg/agent/core/ngt/handler/grpc/handler.go b/pkg/agent/core/ngt/handler/grpc/handler.go
index ef041a3e251..f8b84a70bbb 100644
--- a/pkg/agent/core/ngt/handler/grpc/handler.go
+++ b/pkg/agent/core/ngt/handler/grpc/handler.go
@@ -23,9 +23,9 @@ import (
agent "github.com/vdaas/vald/apis/grpc/v1/agent/core"
"github.com/vdaas/vald/apis/grpc/v1/payload"
"github.com/vdaas/vald/apis/grpc/v1/vald"
- "github.com/vdaas/vald/internal/errgroup"
"github.com/vdaas/vald/internal/errors"
"github.com/vdaas/vald/internal/log"
+ "github.com/vdaas/vald/internal/sync/errgroup"
"github.com/vdaas/vald/pkg/agent/core/ngt/service"
)
diff --git a/pkg/agent/core/ngt/handler/grpc/handler_test.go b/pkg/agent/core/ngt/handler/grpc/handler_test.go
index 17e8b6de693..266d69f87f6 100644
--- a/pkg/agent/core/ngt/handler/grpc/handler_test.go
+++ b/pkg/agent/core/ngt/handler/grpc/handler_test.go
@@ -23,10 +23,10 @@ import (
"github.com/vdaas/vald/apis/grpc/v1/payload"
"github.com/vdaas/vald/internal/config"
- "github.com/vdaas/vald/internal/errgroup"
"github.com/vdaas/vald/internal/info"
"github.com/vdaas/vald/internal/log"
"github.com/vdaas/vald/internal/log/logger"
+ "github.com/vdaas/vald/internal/sync/errgroup"
"github.com/vdaas/vald/internal/test/data/request"
"github.com/vdaas/vald/internal/test/data/vector"
"github.com/vdaas/vald/internal/test/goleak"
diff --git a/pkg/agent/core/ngt/handler/grpc/index_test.go b/pkg/agent/core/ngt/handler/grpc/index_test.go
index aece9ec8d05..fdfb85d0c57 100644
--- a/pkg/agent/core/ngt/handler/grpc/index_test.go
+++ b/pkg/agent/core/ngt/handler/grpc/index_test.go
@@ -17,19 +17,19 @@ import (
"context"
"os"
"reflect"
- "sync"
"testing"
"time"
"github.com/vdaas/vald/apis/grpc/v1/payload"
"github.com/vdaas/vald/internal/config"
"github.com/vdaas/vald/internal/core/algorithm/ngt"
- "github.com/vdaas/vald/internal/errgroup"
"github.com/vdaas/vald/internal/errors"
"github.com/vdaas/vald/internal/file"
"github.com/vdaas/vald/internal/net"
"github.com/vdaas/vald/internal/net/grpc/codes"
"github.com/vdaas/vald/internal/net/grpc/status"
+ "github.com/vdaas/vald/internal/sync"
+ "github.com/vdaas/vald/internal/sync/errgroup"
"github.com/vdaas/vald/internal/test/comparator"
"github.com/vdaas/vald/internal/test/data/request"
"github.com/vdaas/vald/internal/test/data/vector"
diff --git a/pkg/agent/core/ngt/handler/grpc/insert_test.go b/pkg/agent/core/ngt/handler/grpc/insert_test.go
index 3bc376223af..757b03aac03 100644
--- a/pkg/agent/core/ngt/handler/grpc/insert_test.go
+++ b/pkg/agent/core/ngt/handler/grpc/insert_test.go
@@ -23,7 +23,6 @@ import (
"github.com/vdaas/vald/apis/grpc/v1/payload"
"github.com/vdaas/vald/internal/config"
"github.com/vdaas/vald/internal/core/algorithm/ngt"
- "github.com/vdaas/vald/internal/errgroup"
"github.com/vdaas/vald/internal/errors"
"github.com/vdaas/vald/internal/io"
"github.com/vdaas/vald/internal/net"
@@ -31,6 +30,7 @@ import (
"github.com/vdaas/vald/internal/net/grpc/errdetails"
"github.com/vdaas/vald/internal/net/grpc/status"
"github.com/vdaas/vald/internal/strings"
+ "github.com/vdaas/vald/internal/sync/errgroup"
"github.com/vdaas/vald/internal/test/data/request"
"github.com/vdaas/vald/internal/test/data/vector"
"github.com/vdaas/vald/internal/test/mock"
diff --git a/pkg/agent/core/ngt/handler/grpc/linear_search.go b/pkg/agent/core/ngt/handler/grpc/linear_search.go
index f26c48b5b35..554fb9d07fc 100644
--- a/pkg/agent/core/ngt/handler/grpc/linear_search.go
+++ b/pkg/agent/core/ngt/handler/grpc/linear_search.go
@@ -16,7 +16,6 @@ package grpc
import (
"context"
"fmt"
- "sync"
"github.com/vdaas/vald/apis/grpc/v1/payload"
"github.com/vdaas/vald/apis/grpc/v1/vald"
@@ -30,6 +29,7 @@ import (
"github.com/vdaas/vald/internal/observability/trace"
"github.com/vdaas/vald/internal/safety"
"github.com/vdaas/vald/internal/strings"
+ "github.com/vdaas/vald/internal/sync"
"go.opentelemetry.io/otel/attribute"
)
diff --git a/pkg/agent/core/ngt/handler/grpc/object_test.go b/pkg/agent/core/ngt/handler/grpc/object_test.go
index f44cdb8b18a..c4e4ed45661 100644
--- a/pkg/agent/core/ngt/handler/grpc/object_test.go
+++ b/pkg/agent/core/ngt/handler/grpc/object_test.go
@@ -22,11 +22,11 @@ import (
"github.com/vdaas/vald/internal/config"
"github.com/vdaas/vald/internal/conv"
"github.com/vdaas/vald/internal/core/algorithm/ngt"
- "github.com/vdaas/vald/internal/errgroup"
"github.com/vdaas/vald/internal/errors"
"github.com/vdaas/vald/internal/net"
"github.com/vdaas/vald/internal/net/grpc/codes"
"github.com/vdaas/vald/internal/net/grpc/status"
+ "github.com/vdaas/vald/internal/sync/errgroup"
"github.com/vdaas/vald/internal/test/data/request"
"github.com/vdaas/vald/internal/test/data/vector"
"github.com/vdaas/vald/pkg/agent/core/ngt/service"
diff --git a/pkg/agent/core/ngt/handler/grpc/option.go b/pkg/agent/core/ngt/handler/grpc/option.go
index 10f08130ad3..165a5433d24 100644
--- a/pkg/agent/core/ngt/handler/grpc/option.go
+++ b/pkg/agent/core/ngt/handler/grpc/option.go
@@ -21,10 +21,10 @@ import (
"os"
"runtime"
- "github.com/vdaas/vald/internal/errgroup"
"github.com/vdaas/vald/internal/errors"
"github.com/vdaas/vald/internal/log"
"github.com/vdaas/vald/internal/net"
+ "github.com/vdaas/vald/internal/sync/errgroup"
"github.com/vdaas/vald/pkg/agent/core/ngt/service"
)
diff --git a/pkg/agent/core/ngt/handler/grpc/option_test.go b/pkg/agent/core/ngt/handler/grpc/option_test.go
index 9a2e39f2b37..c6c4d8652a3 100644
--- a/pkg/agent/core/ngt/handler/grpc/option_test.go
+++ b/pkg/agent/core/ngt/handler/grpc/option_test.go
@@ -22,8 +22,8 @@ import (
"testing"
"github.com/vdaas/vald/internal/config"
- "github.com/vdaas/vald/internal/errgroup"
"github.com/vdaas/vald/internal/errors"
+ "github.com/vdaas/vald/internal/sync/errgroup"
"github.com/vdaas/vald/pkg/agent/core/ngt/service"
)
diff --git a/pkg/agent/core/ngt/handler/grpc/remove_test.go b/pkg/agent/core/ngt/handler/grpc/remove_test.go
index 308087420f7..2b924b3a00d 100644
--- a/pkg/agent/core/ngt/handler/grpc/remove_test.go
+++ b/pkg/agent/core/ngt/handler/grpc/remove_test.go
@@ -22,10 +22,10 @@ import (
"github.com/vdaas/vald/internal/config"
"github.com/vdaas/vald/internal/conv"
"github.com/vdaas/vald/internal/core/algorithm/ngt"
- "github.com/vdaas/vald/internal/errgroup"
"github.com/vdaas/vald/internal/errors"
"github.com/vdaas/vald/internal/net/grpc/codes"
"github.com/vdaas/vald/internal/net/grpc/status"
+ "github.com/vdaas/vald/internal/sync/errgroup"
"github.com/vdaas/vald/internal/test/data/request"
"github.com/vdaas/vald/internal/test/data/vector"
)
diff --git a/pkg/agent/core/ngt/handler/grpc/search.go b/pkg/agent/core/ngt/handler/grpc/search.go
index 2a9c9d6fdcd..c6941f15dcf 100644
--- a/pkg/agent/core/ngt/handler/grpc/search.go
+++ b/pkg/agent/core/ngt/handler/grpc/search.go
@@ -16,7 +16,6 @@ package grpc
import (
"context"
"fmt"
- "sync"
"github.com/vdaas/vald/apis/grpc/v1/payload"
"github.com/vdaas/vald/apis/grpc/v1/vald"
@@ -30,6 +29,7 @@ import (
"github.com/vdaas/vald/internal/observability/trace"
"github.com/vdaas/vald/internal/safety"
"github.com/vdaas/vald/internal/strings"
+ "github.com/vdaas/vald/internal/sync"
"github.com/vdaas/vald/pkg/agent/core/ngt/model"
"go.opentelemetry.io/otel/attribute"
)
diff --git a/pkg/agent/core/ngt/handler/grpc/search_test.go b/pkg/agent/core/ngt/handler/grpc/search_test.go
index d42cb2c754d..b70f123c319 100644
--- a/pkg/agent/core/ngt/handler/grpc/search_test.go
+++ b/pkg/agent/core/ngt/handler/grpc/search_test.go
@@ -23,10 +23,10 @@ import (
"github.com/vdaas/vald/internal/config"
"github.com/vdaas/vald/internal/conv"
"github.com/vdaas/vald/internal/core/algorithm/ngt"
- "github.com/vdaas/vald/internal/errgroup"
"github.com/vdaas/vald/internal/errors"
"github.com/vdaas/vald/internal/net/grpc/codes"
"github.com/vdaas/vald/internal/net/grpc/status"
+ "github.com/vdaas/vald/internal/sync/errgroup"
"github.com/vdaas/vald/internal/test/data/request"
"github.com/vdaas/vald/internal/test/data/vector"
"github.com/vdaas/vald/pkg/agent/core/ngt/service"
diff --git a/pkg/agent/core/ngt/handler/grpc/update_test.go b/pkg/agent/core/ngt/handler/grpc/update_test.go
index 11935a2e9be..c0967d23cd0 100644
--- a/pkg/agent/core/ngt/handler/grpc/update_test.go
+++ b/pkg/agent/core/ngt/handler/grpc/update_test.go
@@ -22,10 +22,10 @@ import (
"github.com/vdaas/vald/internal/config"
"github.com/vdaas/vald/internal/conv"
"github.com/vdaas/vald/internal/core/algorithm/ngt"
- "github.com/vdaas/vald/internal/errgroup"
"github.com/vdaas/vald/internal/errors"
"github.com/vdaas/vald/internal/net/grpc/codes"
"github.com/vdaas/vald/internal/net/grpc/status"
+ "github.com/vdaas/vald/internal/sync/errgroup"
"github.com/vdaas/vald/internal/test/data/request"
"github.com/vdaas/vald/internal/test/data/vector"
)
diff --git a/pkg/agent/core/ngt/handler/grpc/upsert.go b/pkg/agent/core/ngt/handler/grpc/upsert.go
index e94849d1ff7..41c6990186a 100644
--- a/pkg/agent/core/ngt/handler/grpc/upsert.go
+++ b/pkg/agent/core/ngt/handler/grpc/upsert.go
@@ -16,7 +16,6 @@ package grpc
import (
"context"
"fmt"
- "sync"
"github.com/vdaas/vald/apis/grpc/v1/payload"
"github.com/vdaas/vald/apis/grpc/v1/vald"
@@ -30,6 +29,7 @@ import (
"github.com/vdaas/vald/internal/observability/trace"
"github.com/vdaas/vald/internal/safety"
"github.com/vdaas/vald/internal/strings"
+ "github.com/vdaas/vald/internal/sync"
)
func (s *server) Upsert(ctx context.Context, req *payload.Upsert_Request) (loc *payload.Object_Location, err error) {
diff --git a/pkg/agent/core/ngt/handler/grpc/upsert_test.go b/pkg/agent/core/ngt/handler/grpc/upsert_test.go
index 8f067d86494..a9b2ce37bf4 100644
--- a/pkg/agent/core/ngt/handler/grpc/upsert_test.go
+++ b/pkg/agent/core/ngt/handler/grpc/upsert_test.go
@@ -22,10 +22,10 @@ import (
"github.com/vdaas/vald/internal/config"
"github.com/vdaas/vald/internal/conv"
"github.com/vdaas/vald/internal/core/algorithm/ngt"
- "github.com/vdaas/vald/internal/errgroup"
"github.com/vdaas/vald/internal/errors"
"github.com/vdaas/vald/internal/net/grpc/codes"
"github.com/vdaas/vald/internal/net/grpc/status"
+ "github.com/vdaas/vald/internal/sync/errgroup"
"github.com/vdaas/vald/internal/test/data/request"
"github.com/vdaas/vald/internal/test/data/vector"
)
diff --git a/pkg/agent/core/ngt/router/option.go b/pkg/agent/core/ngt/router/option.go
index 3b437c028b5..6b62d72e567 100644
--- a/pkg/agent/core/ngt/router/option.go
+++ b/pkg/agent/core/ngt/router/option.go
@@ -18,7 +18,7 @@
package router
import (
- "github.com/vdaas/vald/internal/errgroup"
+ "github.com/vdaas/vald/internal/sync/errgroup"
"github.com/vdaas/vald/pkg/agent/core/ngt/handler/rest"
)
diff --git a/pkg/agent/core/ngt/router/option_test.go b/pkg/agent/core/ngt/router/option_test.go
index 91076571be9..b48dc902b39 100644
--- a/pkg/agent/core/ngt/router/option_test.go
+++ b/pkg/agent/core/ngt/router/option_test.go
@@ -21,8 +21,8 @@ import (
"reflect"
"testing"
- "github.com/vdaas/vald/internal/errgroup"
"github.com/vdaas/vald/internal/errors"
+ "github.com/vdaas/vald/internal/sync/errgroup"
"github.com/vdaas/vald/internal/test/goleak"
"github.com/vdaas/vald/pkg/agent/core/ngt/handler/rest"
)
diff --git a/pkg/agent/core/ngt/router/router.go b/pkg/agent/core/ngt/router/router.go
index 582e6431ed3..7b10891e883 100644
--- a/pkg/agent/core/ngt/router/router.go
+++ b/pkg/agent/core/ngt/router/router.go
@@ -20,9 +20,9 @@ package router
import (
"net/http"
- "github.com/vdaas/vald/internal/errgroup"
"github.com/vdaas/vald/internal/net/http/middleware"
"github.com/vdaas/vald/internal/net/http/routing"
+ "github.com/vdaas/vald/internal/sync/errgroup"
"github.com/vdaas/vald/pkg/agent/core/ngt/handler/rest"
)
diff --git a/pkg/agent/core/ngt/router/router_test.go b/pkg/agent/core/ngt/router/router_test.go
index f892ee707de..d576da84cd3 100644
--- a/pkg/agent/core/ngt/router/router_test.go
+++ b/pkg/agent/core/ngt/router/router_test.go
@@ -22,9 +22,9 @@ import (
"testing"
"github.com/gorilla/mux"
- "github.com/vdaas/vald/internal/errgroup"
"github.com/vdaas/vald/internal/errors"
"github.com/vdaas/vald/internal/net/http/routing"
+ "github.com/vdaas/vald/internal/sync/errgroup"
"github.com/vdaas/vald/internal/test/goleak"
"github.com/vdaas/vald/pkg/agent/core/ngt/handler/rest"
)
diff --git a/pkg/agent/core/ngt/service/kvs/kvs.go b/pkg/agent/core/ngt/service/kvs/kvs.go
index 7524110ef0c..90fc2c4c5d7 100644
--- a/pkg/agent/core/ngt/service/kvs/kvs.go
+++ b/pkg/agent/core/ngt/service/kvs/kvs.go
@@ -18,12 +18,11 @@ package kvs
import (
"context"
- "sync"
"sync/atomic"
- "github.com/vdaas/vald/internal/errgroup"
"github.com/vdaas/vald/internal/safety"
- valdsync "github.com/vdaas/vald/internal/sync"
+ "github.com/vdaas/vald/internal/sync"
+ "github.com/vdaas/vald/internal/sync/errgroup"
"github.com/zeebo/xxh3"
)
@@ -52,8 +51,8 @@ type ValueStructUo struct {
type bidi struct {
concurrency int
l uint64
- ou [slen]*valdsync.Map[uint32, valueStructOu]
- uo [slen]*valdsync.Map[string, ValueStructUo]
+ ou [slen]*sync.Map[uint32, valueStructOu]
+ uo [slen]*sync.Map[string, ValueStructUo]
eg errgroup.Group
}
@@ -76,8 +75,8 @@ func New(opts ...Option) BidiMap {
opt(b)
}
for i := range b.ou {
- b.ou[i] = new(valdsync.Map[uint32, valueStructOu])
- b.uo[i] = new(valdsync.Map[string, ValueStructUo])
+ b.ou[i] = new(sync.Map[uint32, valueStructOu])
+ b.uo[i] = new(sync.Map[string, ValueStructUo])
}
if b.eg == nil {
@@ -85,7 +84,7 @@ func New(opts ...Option) BidiMap {
}
if b.concurrency > 0 {
- b.eg.Limitation(b.concurrency)
+ b.eg.SetLimit(b.concurrency)
}
return b
@@ -94,7 +93,7 @@ func New(opts ...Option) BidiMap {
// Get returns the value and boolean from the given key.
// If the value does not exist, it returns nil and false.
func (b *bidi) Get(key string) (oid uint32, timestamp int64, exists bool) {
- vs, ok := b.uo[xxh3.HashString(key)&mask].Load(key)
+ vs, ok := b.uo[getShardID(key)].Load(key)
if !ok {
return 0, 0, false
}
@@ -114,7 +113,7 @@ func (b *bidi) GetInverse(val uint32) (string, int64, bool) {
// Set sets the key and val to the bidi.
func (b *bidi) Set(key string, val uint32, ts int64) {
- id := xxh3.HashString(key) & mask
+ id := getShardID(key)
vs, loaded := b.uo[id].LoadOrStore(key, ValueStructUo{value: val, timestamp: ts})
old := vs.value
if !loaded { // increase the count only if the key is not exists before
@@ -129,7 +128,7 @@ func (b *bidi) Set(key string, val uint32, ts int64) {
// Delete deletes the key and the value from the bidi by the given key and returns val and true.
// If the value for the key does not exist, it returns nil and false.
func (b *bidi) Delete(key string) (val uint32, ok bool) {
- vs, ok := b.uo[xxh3.HashString(key)&mask].LoadAndDelete(key)
+ vs, ok := b.uo[getShardID(key)].LoadAndDelete(key)
val = vs.value
if ok {
b.ou[val&mask].Delete(val)
@@ -144,7 +143,7 @@ func (b *bidi) DeleteInverse(val uint32) (key string, ok bool) {
vs, ok := b.ou[val&mask].LoadAndDelete(val)
key = vs.value
if ok {
- b.uo[xxh3.HashString(key)&mask].Delete(key)
+ b.uo[getShardID(key)].LoadAndDelete(key)
atomic.AddUint64(&b.l, ^uint64(0))
}
return key, ok
@@ -158,12 +157,11 @@ func (b *bidi) Range(ctx context.Context, f func(string, uint32, int64) bool) {
wg.Add(1)
b.eg.Go(safety.RecoverFunc(func() (err error) {
b.uo[idx].Range(func(uuid string, val ValueStructUo) bool {
- f(uuid, val.value, val.timestamp)
select {
case <-ctx.Done():
return false
default:
- return true
+ return f(uuid, val.value, val.timestamp)
}
})
wg.Done()
@@ -187,3 +185,10 @@ func (b *bidi) Close() error {
}
return b.eg.Wait()
}
+
+func getShardID(key string) (id uint64) {
+ if len(key) > 128 {
+ return xxh3.HashString(key[:128]) & mask
+ }
+ return xxh3.HashString(key) & mask
+}
diff --git a/pkg/agent/core/ngt/service/kvs/kvs_test.go b/pkg/agent/core/ngt/service/kvs/kvs_test.go
index 60d811ac9f9..243b1c99b64 100644
--- a/pkg/agent/core/ngt/service/kvs/kvs_test.go
+++ b/pkg/agent/core/ngt/service/kvs/kvs_test.go
@@ -21,13 +21,12 @@ import (
"math"
"reflect"
"runtime"
- "sync"
"sync/atomic"
"testing"
- "github.com/vdaas/vald/internal/errgroup"
"github.com/vdaas/vald/internal/errors"
- valdsync "github.com/vdaas/vald/internal/sync"
+ "github.com/vdaas/vald/internal/sync"
+ "github.com/vdaas/vald/internal/sync/errgroup"
"github.com/vdaas/vald/internal/test/goleak"
)
@@ -52,12 +51,12 @@ func TestNew(t *testing.T) {
tests := []test{
func() test {
var (
- wantOu [slen]*valdsync.Map[uint32, valueStructOu]
- wantUo [slen]*valdsync.Map[string, ValueStructUo]
+ wantOu [slen]*sync.Map[uint32, valueStructOu]
+ wantUo [slen]*sync.Map[string, ValueStructUo]
)
for i := 0; i < slen; i++ {
- wantOu[i] = new(valdsync.Map[uint32, valueStructOu])
- wantUo[i] = new(valdsync.Map[string, ValueStructUo])
+ wantOu[i] = new(sync.Map[uint32, valueStructOu])
+ wantUo[i] = new(sync.Map[string, ValueStructUo])
}
return test{
name: "return the bidi struct",
@@ -104,8 +103,8 @@ func Test_bidi_Get(t *testing.T) {
key string
}
type fields struct {
- ou [slen]*valdsync.Map[uint32, valueStructOu]
- uo [slen]*valdsync.Map[string, ValueStructUo]
+ ou [slen]*sync.Map[uint32, valueStructOu]
+ uo [slen]*sync.Map[string, ValueStructUo]
l uint64
}
type want struct {
@@ -144,8 +143,8 @@ func Test_bidi_Get(t *testing.T) {
l: 0,
}
for i := 0; i < slen; i++ {
- fields.ou[i] = new(valdsync.Map[uint32, valueStructOu])
- fields.uo[i] = new(valdsync.Map[string, ValueStructUo])
+ fields.ou[i] = new(sync.Map[uint32, valueStructOu])
+ fields.uo[i] = new(sync.Map[string, ValueStructUo])
}
var (
@@ -177,8 +176,8 @@ func Test_bidi_Get(t *testing.T) {
l: 100,
}
for i := 0; i < slen; i++ {
- fields.ou[i] = new(valdsync.Map[uint32, valueStructOu])
- fields.uo[i] = new(valdsync.Map[string, ValueStructUo])
+ fields.ou[i] = new(sync.Map[uint32, valueStructOu])
+ fields.uo[i] = new(sync.Map[string, ValueStructUo])
}
var (
@@ -210,8 +209,8 @@ func Test_bidi_Get(t *testing.T) {
l: math.MaxUint64,
}
for i := 0; i < slen; i++ {
- fields.ou[i] = new(valdsync.Map[uint32, valueStructOu])
- fields.uo[i] = new(valdsync.Map[string, ValueStructUo])
+ fields.ou[i] = new(sync.Map[uint32, valueStructOu])
+ fields.uo[i] = new(sync.Map[string, ValueStructUo])
}
var (
@@ -243,8 +242,8 @@ func Test_bidi_Get(t *testing.T) {
l: 0,
}
for i := 0; i < slen; i++ {
- fields.ou[i] = new(valdsync.Map[uint32, valueStructOu])
- fields.uo[i] = new(valdsync.Map[string, ValueStructUo])
+ fields.ou[i] = new(sync.Map[uint32, valueStructOu])
+ fields.uo[i] = new(sync.Map[string, ValueStructUo])
}
var (
@@ -276,8 +275,8 @@ func Test_bidi_Get(t *testing.T) {
l: 0,
}
for i := 0; i < slen; i++ {
- fields.ou[i] = new(valdsync.Map[uint32, valueStructOu])
- fields.uo[i] = new(valdsync.Map[string, ValueStructUo])
+ fields.ou[i] = new(sync.Map[uint32, valueStructOu])
+ fields.uo[i] = new(sync.Map[string, ValueStructUo])
}
var (
@@ -307,8 +306,8 @@ func Test_bidi_Get(t *testing.T) {
l: 0,
}
for i := 0; i < slen; i++ {
- fields.ou[i] = new(valdsync.Map[uint32, valueStructOu])
- fields.uo[i] = new(valdsync.Map[string, ValueStructUo])
+ fields.ou[i] = new(sync.Map[uint32, valueStructOu])
+ fields.uo[i] = new(sync.Map[string, ValueStructUo])
}
var (
@@ -370,8 +369,8 @@ func Test_bidi_GetInverse(t *testing.T) {
val uint32
}
type fields struct {
- ou [slen]*valdsync.Map[uint32, valueStructOu]
- uo [slen]*valdsync.Map[string, ValueStructUo]
+ ou [slen]*sync.Map[uint32, valueStructOu]
+ uo [slen]*sync.Map[string, ValueStructUo]
l uint64
}
type want struct {
@@ -410,8 +409,8 @@ func Test_bidi_GetInverse(t *testing.T) {
l: 0,
}
for i := 0; i < slen; i++ {
- fields.ou[i] = new(valdsync.Map[uint32, valueStructOu])
- fields.uo[i] = new(valdsync.Map[string, ValueStructUo])
+ fields.ou[i] = new(sync.Map[uint32, valueStructOu])
+ fields.uo[i] = new(sync.Map[string, ValueStructUo])
}
var (
@@ -443,8 +442,8 @@ func Test_bidi_GetInverse(t *testing.T) {
l: 100,
}
for i := 0; i < slen; i++ {
- fields.ou[i] = new(valdsync.Map[uint32, valueStructOu])
- fields.uo[i] = new(valdsync.Map[string, ValueStructUo])
+ fields.ou[i] = new(sync.Map[uint32, valueStructOu])
+ fields.uo[i] = new(sync.Map[string, ValueStructUo])
}
var (
@@ -476,8 +475,8 @@ func Test_bidi_GetInverse(t *testing.T) {
l: math.MaxUint64,
}
for i := 0; i < slen; i++ {
- fields.ou[i] = new(valdsync.Map[uint32, valueStructOu])
- fields.uo[i] = new(valdsync.Map[string, ValueStructUo])
+ fields.ou[i] = new(sync.Map[uint32, valueStructOu])
+ fields.uo[i] = new(sync.Map[string, ValueStructUo])
}
var (
@@ -509,8 +508,8 @@ func Test_bidi_GetInverse(t *testing.T) {
l: 0,
}
for i := 0; i < slen; i++ {
- fields.ou[i] = new(valdsync.Map[uint32, valueStructOu])
- fields.uo[i] = new(valdsync.Map[string, ValueStructUo])
+ fields.ou[i] = new(sync.Map[uint32, valueStructOu])
+ fields.uo[i] = new(sync.Map[string, ValueStructUo])
}
var (
@@ -542,8 +541,8 @@ func Test_bidi_GetInverse(t *testing.T) {
l: 0,
}
for i := 0; i < slen; i++ {
- fields.ou[i] = new(valdsync.Map[uint32, valueStructOu])
- fields.uo[i] = new(valdsync.Map[string, ValueStructUo])
+ fields.ou[i] = new(sync.Map[uint32, valueStructOu])
+ fields.uo[i] = new(sync.Map[string, ValueStructUo])
}
var (
@@ -573,8 +572,8 @@ func Test_bidi_GetInverse(t *testing.T) {
l: 0,
}
for i := 0; i < slen; i++ {
- fields.ou[i] = new(valdsync.Map[uint32, valueStructOu])
- fields.uo[i] = new(valdsync.Map[string, ValueStructUo])
+ fields.ou[i] = new(sync.Map[uint32, valueStructOu])
+ fields.uo[i] = new(sync.Map[string, ValueStructUo])
}
var (
@@ -639,8 +638,8 @@ func Test_bidi_Set(t *testing.T) {
ts int64
}
type fields struct {
- ou [slen]*valdsync.Map[uint32, valueStructOu]
- uo [slen]*valdsync.Map[string, ValueStructUo]
+ ou [slen]*sync.Map[uint32, valueStructOu]
+ uo [slen]*sync.Map[string, ValueStructUo]
l uint64
}
type want struct {
@@ -687,8 +686,8 @@ func Test_bidi_Set(t *testing.T) {
l: 0,
}
for i := 0; i < slen; i++ {
- fields.ou[i] = new(valdsync.Map[uint32, valueStructOu])
- fields.uo[i] = new(valdsync.Map[string, ValueStructUo])
+ fields.ou[i] = new(sync.Map[uint32, valueStructOu])
+ fields.uo[i] = new(sync.Map[string, ValueStructUo])
}
var (
@@ -718,8 +717,8 @@ func Test_bidi_Set(t *testing.T) {
l: 100,
}
for i := 0; i < slen; i++ {
- fields.ou[i] = new(valdsync.Map[uint32, valueStructOu])
- fields.uo[i] = new(valdsync.Map[string, ValueStructUo])
+ fields.ou[i] = new(sync.Map[uint32, valueStructOu])
+ fields.uo[i] = new(sync.Map[string, ValueStructUo])
}
var (
@@ -749,8 +748,8 @@ func Test_bidi_Set(t *testing.T) {
l: math.MaxUint64,
}
for i := 0; i < slen; i++ {
- fields.ou[i] = new(valdsync.Map[uint32, valueStructOu])
- fields.uo[i] = new(valdsync.Map[string, ValueStructUo])
+ fields.ou[i] = new(sync.Map[uint32, valueStructOu])
+ fields.uo[i] = new(sync.Map[string, ValueStructUo])
}
var (
@@ -780,8 +779,8 @@ func Test_bidi_Set(t *testing.T) {
l: 0,
}
for i := 0; i < slen; i++ {
- fields.ou[i] = new(valdsync.Map[uint32, valueStructOu])
- fields.uo[i] = new(valdsync.Map[string, ValueStructUo])
+ fields.ou[i] = new(sync.Map[uint32, valueStructOu])
+ fields.uo[i] = new(sync.Map[string, ValueStructUo])
}
var (
@@ -820,8 +819,8 @@ func Test_bidi_Set(t *testing.T) {
l: 0,
}
for i := 0; i < slen; i++ {
- fields.ou[i] = new(valdsync.Map[uint32, valueStructOu])
- fields.uo[i] = new(valdsync.Map[string, ValueStructUo])
+ fields.ou[i] = new(sync.Map[uint32, valueStructOu])
+ fields.uo[i] = new(sync.Map[string, ValueStructUo])
}
key := "45637ec4-c85f-11ea-87d0"
@@ -845,8 +844,8 @@ func Test_bidi_Set(t *testing.T) {
l: 0,
}
for i := 0; i < slen; i++ {
- fields.ou[i] = new(valdsync.Map[uint32, valueStructOu])
- fields.uo[i] = new(valdsync.Map[string, ValueStructUo])
+ fields.ou[i] = new(sync.Map[uint32, valueStructOu])
+ fields.uo[i] = new(sync.Map[string, ValueStructUo])
}
var val uint32 = 14438
@@ -869,8 +868,8 @@ func Test_bidi_Set(t *testing.T) {
l: 0,
}
for i := 0; i < slen; i++ {
- fields.ou[i] = new(valdsync.Map[uint32, valueStructOu])
- fields.uo[i] = new(valdsync.Map[string, ValueStructUo])
+ fields.ou[i] = new(sync.Map[uint32, valueStructOu])
+ fields.uo[i] = new(sync.Map[string, ValueStructUo])
}
return test{
@@ -922,8 +921,8 @@ func Test_bidi_Delete(t *testing.T) {
key string
}
type fields struct {
- ou [slen]*valdsync.Map[uint32, valueStructOu]
- uo [slen]*valdsync.Map[string, ValueStructUo]
+ ou [slen]*sync.Map[uint32, valueStructOu]
+ uo [slen]*sync.Map[string, ValueStructUo]
l uint64
}
type want struct {
@@ -955,8 +954,8 @@ func Test_bidi_Delete(t *testing.T) {
l: 0,
}
for i := 0; i < slen; i++ {
- fields.ou[i] = new(valdsync.Map[uint32, valueStructOu])
- fields.uo[i] = new(valdsync.Map[string, ValueStructUo])
+ fields.ou[i] = new(sync.Map[uint32, valueStructOu])
+ fields.uo[i] = new(sync.Map[string, ValueStructUo])
}
var (
@@ -1002,8 +1001,8 @@ func Test_bidi_Delete(t *testing.T) {
l: 100,
}
for i := 0; i < slen; i++ {
- fields.ou[i] = new(valdsync.Map[uint32, valueStructOu])
- fields.uo[i] = new(valdsync.Map[string, ValueStructUo])
+ fields.ou[i] = new(sync.Map[uint32, valueStructOu])
+ fields.uo[i] = new(sync.Map[string, ValueStructUo])
}
var (
@@ -1049,8 +1048,8 @@ func Test_bidi_Delete(t *testing.T) {
l: math.MaxUint64,
}
for i := 0; i < slen; i++ {
- fields.ou[i] = new(valdsync.Map[uint32, valueStructOu])
- fields.uo[i] = new(valdsync.Map[string, ValueStructUo])
+ fields.ou[i] = new(sync.Map[uint32, valueStructOu])
+ fields.uo[i] = new(sync.Map[string, ValueStructUo])
}
var (
@@ -1096,8 +1095,8 @@ func Test_bidi_Delete(t *testing.T) {
l: 0,
}
for i := 0; i < slen; i++ {
- fields.ou[i] = new(valdsync.Map[uint32, valueStructOu])
- fields.uo[i] = new(valdsync.Map[string, ValueStructUo])
+ fields.ou[i] = new(sync.Map[uint32, valueStructOu])
+ fields.uo[i] = new(sync.Map[string, ValueStructUo])
}
var (
@@ -1140,8 +1139,8 @@ func Test_bidi_Delete(t *testing.T) {
l: 0,
}
for i := 0; i < slen; i++ {
- fields.ou[i] = new(valdsync.Map[uint32, valueStructOu])
- fields.uo[i] = new(valdsync.Map[string, ValueStructUo])
+ fields.ou[i] = new(sync.Map[uint32, valueStructOu])
+ fields.uo[i] = new(sync.Map[string, ValueStructUo])
}
var (
@@ -1203,8 +1202,8 @@ func Test_bidi_DeleteInverse(t *testing.T) {
val uint32
}
type fields struct {
- ou [slen]*valdsync.Map[uint32, valueStructOu]
- uo [slen]*valdsync.Map[string, ValueStructUo]
+ ou [slen]*sync.Map[uint32, valueStructOu]
+ uo [slen]*sync.Map[string, ValueStructUo]
l uint64
}
type want struct {
@@ -1236,8 +1235,8 @@ func Test_bidi_DeleteInverse(t *testing.T) {
l: 0,
}
for i := 0; i < slen; i++ {
- fields.ou[i] = new(valdsync.Map[uint32, valueStructOu])
- fields.uo[i] = new(valdsync.Map[string, ValueStructUo])
+ fields.ou[i] = new(sync.Map[uint32, valueStructOu])
+ fields.uo[i] = new(sync.Map[string, ValueStructUo])
}
var (
@@ -1283,8 +1282,8 @@ func Test_bidi_DeleteInverse(t *testing.T) {
l: 100,
}
for i := 0; i < slen; i++ {
- fields.ou[i] = new(valdsync.Map[uint32, valueStructOu])
- fields.uo[i] = new(valdsync.Map[string, ValueStructUo])
+ fields.ou[i] = new(sync.Map[uint32, valueStructOu])
+ fields.uo[i] = new(sync.Map[string, ValueStructUo])
}
var (
@@ -1330,8 +1329,8 @@ func Test_bidi_DeleteInverse(t *testing.T) {
l: math.MaxUint64,
}
for i := 0; i < slen; i++ {
- fields.ou[i] = new(valdsync.Map[uint32, valueStructOu])
- fields.uo[i] = new(valdsync.Map[string, ValueStructUo])
+ fields.ou[i] = new(sync.Map[uint32, valueStructOu])
+ fields.uo[i] = new(sync.Map[string, ValueStructUo])
}
var (
@@ -1377,8 +1376,8 @@ func Test_bidi_DeleteInverse(t *testing.T) {
l: 0,
}
for i := 0; i < slen; i++ {
- fields.ou[i] = new(valdsync.Map[uint32, valueStructOu])
- fields.uo[i] = new(valdsync.Map[string, ValueStructUo])
+ fields.ou[i] = new(sync.Map[uint32, valueStructOu])
+ fields.uo[i] = new(sync.Map[string, ValueStructUo])
}
var (
@@ -1421,8 +1420,8 @@ func Test_bidi_DeleteInverse(t *testing.T) {
l: 0,
}
for i := 0; i < slen; i++ {
- fields.ou[i] = new(valdsync.Map[uint32, valueStructOu])
- fields.uo[i] = new(valdsync.Map[string, ValueStructUo])
+ fields.ou[i] = new(sync.Map[uint32, valueStructOu])
+ fields.uo[i] = new(sync.Map[string, ValueStructUo])
}
var (
@@ -1485,8 +1484,8 @@ func Test_bidi_Range(t *testing.T) {
f func(string, uint32, int64) bool
}
type fields struct {
- ou [slen]*valdsync.Map[uint32, valueStructOu]
- uo [slen]*valdsync.Map[string, ValueStructUo]
+ ou [slen]*sync.Map[uint32, valueStructOu]
+ uo [slen]*sync.Map[string, ValueStructUo]
l uint64
}
type want struct {
@@ -1511,8 +1510,8 @@ func Test_bidi_Range(t *testing.T) {
l: 0,
}
for i := 0; i < slen; i++ {
- fields.ou[i] = new(valdsync.Map[uint32, valueStructOu])
- fields.uo[i] = new(valdsync.Map[string, ValueStructUo])
+ fields.ou[i] = new(sync.Map[uint32, valueStructOu])
+ fields.uo[i] = new(sync.Map[string, ValueStructUo])
}
var (
@@ -1563,8 +1562,8 @@ func Test_bidi_Range(t *testing.T) {
l: 100,
}
for i := 0; i < slen; i++ {
- fields.ou[i] = new(valdsync.Map[uint32, valueStructOu])
- fields.uo[i] = new(valdsync.Map[string, ValueStructUo])
+ fields.ou[i] = new(sync.Map[uint32, valueStructOu])
+ fields.uo[i] = new(sync.Map[string, ValueStructUo])
}
var (
@@ -1615,8 +1614,8 @@ func Test_bidi_Range(t *testing.T) {
l: math.MaxUint64,
}
for i := 0; i < slen; i++ {
- fields.ou[i] = new(valdsync.Map[uint32, valueStructOu])
- fields.uo[i] = new(valdsync.Map[string, ValueStructUo])
+ fields.ou[i] = new(sync.Map[uint32, valueStructOu])
+ fields.uo[i] = new(sync.Map[string, ValueStructUo])
}
var (
@@ -1700,8 +1699,8 @@ func Test_bidi_Range(t *testing.T) {
func Test_bidi_Len(t *testing.T) {
t.Parallel()
type fields struct {
- ou [slen]*valdsync.Map[uint32, valueStructOu]
- uo [slen]*valdsync.Map[string, ValueStructUo]
+ ou [slen]*sync.Map[uint32, valueStructOu]
+ uo [slen]*sync.Map[string, ValueStructUo]
l uint64
}
type want struct {
@@ -1784,8 +1783,8 @@ func Test_bidi_Len(t *testing.T) {
// type fields struct {
// concurrency int
// l uint64
-// ou [slen]*ou
-// uo [slen]*uo
+// ou [slen]*sync.Map[uint32, valueStructOu]
+// uo [slen]*sync.Map[string, ValueStructUo]
// eg errgroup.Group
// }
// type want struct {
diff --git a/pkg/agent/core/ngt/service/kvs/option.go b/pkg/agent/core/ngt/service/kvs/option.go
index 30eb68724de..b2a9d3dd473 100644
--- a/pkg/agent/core/ngt/service/kvs/option.go
+++ b/pkg/agent/core/ngt/service/kvs/option.go
@@ -19,7 +19,7 @@ package kvs
import (
"runtime"
- "github.com/vdaas/vald/internal/errgroup"
+ "github.com/vdaas/vald/internal/sync/errgroup"
)
// Option represents the functional option for bidi.
diff --git a/pkg/agent/core/ngt/service/ngt.go b/pkg/agent/core/ngt/service/ngt.go
index dc4f54515d5..45a19beaeff 100644
--- a/pkg/agent/core/ngt/service/ngt.go
+++ b/pkg/agent/core/ngt/service/ngt.go
@@ -27,14 +27,12 @@ import (
"path/filepath"
"reflect"
"runtime"
- "sync"
"sync/atomic"
"time"
"github.com/vdaas/vald/internal/config"
"github.com/vdaas/vald/internal/conv"
core "github.com/vdaas/vald/internal/core/algorithm/ngt"
- "github.com/vdaas/vald/internal/errgroup"
"github.com/vdaas/vald/internal/errors"
"github.com/vdaas/vald/internal/file"
"github.com/vdaas/vald/internal/log"
@@ -42,6 +40,8 @@ import (
"github.com/vdaas/vald/internal/safety"
"github.com/vdaas/vald/internal/slices"
"github.com/vdaas/vald/internal/strings"
+ "github.com/vdaas/vald/internal/sync"
+ "github.com/vdaas/vald/internal/sync/errgroup"
"github.com/vdaas/vald/pkg/agent/core/ngt/model"
"github.com/vdaas/vald/pkg/agent/core/ngt/service/kvs"
"github.com/vdaas/vald/pkg/agent/core/ngt/service/vqueue"
diff --git a/pkg/agent/core/ngt/service/ngt_test.go b/pkg/agent/core/ngt/service/ngt_test.go
index 5e9e066e5d9..1f72c919d59 100644
--- a/pkg/agent/core/ngt/service/ngt_test.go
+++ b/pkg/agent/core/ngt/service/ngt_test.go
@@ -24,7 +24,6 @@ import (
"math"
"os"
"path/filepath"
- "sync"
"sync/atomic"
"testing"
"time"
@@ -35,12 +34,13 @@ import (
"github.com/vdaas/vald/internal/client/v1/client/vald"
"github.com/vdaas/vald/internal/config"
core "github.com/vdaas/vald/internal/core/algorithm/ngt"
- "github.com/vdaas/vald/internal/errgroup"
"github.com/vdaas/vald/internal/errors"
"github.com/vdaas/vald/internal/file"
"github.com/vdaas/vald/internal/log"
"github.com/vdaas/vald/internal/safety"
"github.com/vdaas/vald/internal/strings"
+ "github.com/vdaas/vald/internal/sync"
+ "github.com/vdaas/vald/internal/sync/errgroup"
testdata "github.com/vdaas/vald/internal/test"
"github.com/vdaas/vald/internal/test/data/vector"
"github.com/vdaas/vald/internal/test/goleak"
@@ -1508,6 +1508,7 @@ func createRandomData(num int, cfg *createRandomDataConfig) []index {
//
// func Test_ngt_Search(t *testing.T) {
// type args struct {
+// ctx context.Context
// vec []float32
// size uint32
// epsilon float32
@@ -1579,6 +1580,7 @@ func createRandomData(num int, cfg *createRandomDataConfig) []index {
// {
// name: "test_case_1",
// args: args {
+// ctx:nil,
// vec:nil,
// size:0,
// epsilon:0,
@@ -1639,6 +1641,7 @@ func createRandomData(num int, cfg *createRandomDataConfig) []index {
// return test {
// name: "test_case_2",
// args: args {
+// ctx:nil,
// vec:nil,
// size:0,
// epsilon:0,
@@ -1749,7 +1752,7 @@ func createRandomData(num int, cfg *createRandomDataConfig) []index {
// historyLimit: test.fields.historyLimit,
// }
//
-// got, err := n.Search(test.args.vec, test.args.size, test.args.epsilon, test.args.radius)
+// got, err := n.Search(test.args.ctx, test.args.vec, test.args.size, test.args.epsilon, test.args.radius)
// if err := checkFunc(test.want, got, err); err != nil {
// tt.Errorf("error = %v", err)
// }
@@ -1760,6 +1763,7 @@ func createRandomData(num int, cfg *createRandomDataConfig) []index {
//
// func Test_ngt_SearchByID(t *testing.T) {
// type args struct {
+// ctx context.Context
// uuid string
// size uint32
// epsilon float32
@@ -1835,6 +1839,7 @@ func createRandomData(num int, cfg *createRandomDataConfig) []index {
// {
// name: "test_case_1",
// args: args {
+// ctx:nil,
// uuid:"",
// size:0,
// epsilon:0,
@@ -1895,6 +1900,7 @@ func createRandomData(num int, cfg *createRandomDataConfig) []index {
// return test {
// name: "test_case_2",
// args: args {
+// ctx:nil,
// uuid:"",
// size:0,
// epsilon:0,
@@ -2005,7 +2011,7 @@ func createRandomData(num int, cfg *createRandomDataConfig) []index {
// historyLimit: test.fields.historyLimit,
// }
//
-// gotVec, gotDst, err := n.SearchByID(test.args.uuid, test.args.size, test.args.epsilon, test.args.radius)
+// gotVec, gotDst, err := n.SearchByID(test.args.ctx, test.args.uuid, test.args.size, test.args.epsilon, test.args.radius)
// if err := checkFunc(test.want, gotVec, gotDst, err); err != nil {
// tt.Errorf("error = %v", err)
// }
diff --git a/pkg/agent/core/ngt/service/option.go b/pkg/agent/core/ngt/service/option.go
index 1d0fcfb3163..1dc4a8c574b 100644
--- a/pkg/agent/core/ngt/service/option.go
+++ b/pkg/agent/core/ngt/service/option.go
@@ -23,10 +23,10 @@ import (
"time"
core "github.com/vdaas/vald/internal/core/algorithm/ngt"
- "github.com/vdaas/vald/internal/errgroup"
"github.com/vdaas/vald/internal/file"
"github.com/vdaas/vald/internal/rand"
"github.com/vdaas/vald/internal/strings"
+ "github.com/vdaas/vald/internal/sync/errgroup"
"github.com/vdaas/vald/internal/timeutil"
)
diff --git a/pkg/agent/core/ngt/service/option_test.go b/pkg/agent/core/ngt/service/option_test.go
index 326264012d5..dbea1fff566 100644
--- a/pkg/agent/core/ngt/service/option_test.go
+++ b/pkg/agent/core/ngt/service/option_test.go
@@ -22,8 +22,8 @@ import (
"testing"
"time"
- "github.com/vdaas/vald/internal/errgroup"
"github.com/vdaas/vald/internal/errors"
+ "github.com/vdaas/vald/internal/sync/errgroup"
"github.com/vdaas/vald/internal/test/goleak"
)
diff --git a/pkg/agent/core/ngt/service/vqueue/queue.go b/pkg/agent/core/ngt/service/vqueue/queue.go
index e20c6d91fab..78d8837d96a 100644
--- a/pkg/agent/core/ngt/service/vqueue/queue.go
+++ b/pkg/agent/core/ngt/service/vqueue/queue.go
@@ -26,7 +26,7 @@ import (
"github.com/vdaas/vald/internal/errors"
"github.com/vdaas/vald/internal/log"
"github.com/vdaas/vald/internal/slices"
- valdsync "github.com/vdaas/vald/internal/sync"
+ "github.com/vdaas/vald/internal/sync"
)
// Queue represents vector queue cache interface
@@ -43,7 +43,7 @@ type Queue interface {
}
type vqueue struct {
- il, dl valdsync.Map[string, *index]
+ il, dl sync.Map[string, *index]
ic, dc uint64
}
diff --git a/pkg/agent/core/ngt/service/vqueue/queue_test.go b/pkg/agent/core/ngt/service/vqueue/queue_test.go
index 520a2437ba3..22703d396f0 100644
--- a/pkg/agent/core/ngt/service/vqueue/queue_test.go
+++ b/pkg/agent/core/ngt/service/vqueue/queue_test.go
@@ -178,8 +178,8 @@ func TestGetVector(t *testing.T) {
// date int64
// }
// type fields struct {
-// il indexMap
-// dl indexMap
+// il sync.Map[string, *index]
+// dl sync.Map[string, *index]
// ic uint64
// dc uint64
// }
@@ -212,8 +212,8 @@ func TestGetVector(t *testing.T) {
// date:0,
// },
// fields: fields {
-// il:indexMap{},
-// dl:indexMap{},
+// il:nil,
+// dl:nil,
// ic:0,
// dc:0,
// },
@@ -239,8 +239,8 @@ func TestGetVector(t *testing.T) {
// date:0,
// },
// fields: fields {
-// il:indexMap{},
-// dl:indexMap{},
+// il:nil,
+// dl:nil,
// ic:0,
// dc:0,
// },
@@ -294,8 +294,8 @@ func TestGetVector(t *testing.T) {
// date int64
// }
// type fields struct {
-// il indexMap
-// dl indexMap
+// il sync.Map[string, *index]
+// dl sync.Map[string, *index]
// ic uint64
// dc uint64
// }
@@ -327,8 +327,8 @@ func TestGetVector(t *testing.T) {
// date:0,
// },
// fields: fields {
-// il:indexMap{},
-// dl:indexMap{},
+// il:nil,
+// dl:nil,
// ic:0,
// dc:0,
// },
@@ -353,8 +353,8 @@ func TestGetVector(t *testing.T) {
// date:0,
// },
// fields: fields {
-// il:indexMap{},
-// dl:indexMap{},
+// il:nil,
+// dl:nil,
// ic:0,
// dc:0,
// },
@@ -407,8 +407,8 @@ func TestGetVector(t *testing.T) {
// uuid string
// }
// type fields struct {
-// il indexMap
-// dl indexMap
+// il sync.Map[string, *index]
+// dl sync.Map[string, *index]
// ic uint64
// dc uint64
// }
@@ -443,8 +443,8 @@ func TestGetVector(t *testing.T) {
// uuid:"",
// },
// fields: fields {
-// il:indexMap{},
-// dl:indexMap{},
+// il:nil,
+// dl:nil,
// ic:0,
// dc:0,
// },
@@ -468,8 +468,8 @@ func TestGetVector(t *testing.T) {
// uuid:"",
// },
// fields: fields {
-// il:indexMap{},
-// dl:indexMap{},
+// il:nil,
+// dl:nil,
// ic:0,
// dc:0,
// },
@@ -522,8 +522,8 @@ func TestGetVector(t *testing.T) {
// uuid string
// }
// type fields struct {
-// il indexMap
-// dl indexMap
+// il sync.Map[string, *index]
+// dl sync.Map[string, *index]
// ic uint64
// dc uint64
// }
@@ -554,8 +554,8 @@ func TestGetVector(t *testing.T) {
// uuid:"",
// },
// fields: fields {
-// il:indexMap{},
-// dl:indexMap{},
+// il:nil,
+// dl:nil,
// ic:0,
// dc:0,
// },
@@ -579,8 +579,8 @@ func TestGetVector(t *testing.T) {
// uuid:"",
// },
// fields: fields {
-// il:indexMap{},
-// dl:indexMap{},
+// il:nil,
+// dl:nil,
// ic:0,
// dc:0,
// },
@@ -633,8 +633,8 @@ func TestGetVector(t *testing.T) {
// uuid string
// }
// type fields struct {
-// il indexMap
-// dl indexMap
+// il sync.Map[string, *index]
+// dl sync.Map[string, *index]
// ic uint64
// dc uint64
// }
@@ -665,8 +665,8 @@ func TestGetVector(t *testing.T) {
// uuid:"",
// },
// fields: fields {
-// il:indexMap{},
-// dl:indexMap{},
+// il:nil,
+// dl:nil,
// ic:0,
// dc:0,
// },
@@ -690,8 +690,8 @@ func TestGetVector(t *testing.T) {
// uuid:"",
// },
// fields: fields {
-// il:indexMap{},
-// dl:indexMap{},
+// il:nil,
+// dl:nil,
// ic:0,
// dc:0,
// },
@@ -746,8 +746,8 @@ func TestGetVector(t *testing.T) {
// f func(uuid string, vector []float32, date int64) bool
// }
// type fields struct {
-// il indexMap
-// dl indexMap
+// il sync.Map[string, *index]
+// dl sync.Map[string, *index]
// ic uint64
// dc uint64
// }
@@ -776,8 +776,8 @@ func TestGetVector(t *testing.T) {
// f:nil,
// },
// fields: fields {
-// il:indexMap{},
-// dl:indexMap{},
+// il:nil,
+// dl:nil,
// ic:0,
// dc:0,
// },
@@ -803,8 +803,8 @@ func TestGetVector(t *testing.T) {
// f:nil,
// },
// fields: fields {
-// il:indexMap{},
-// dl:indexMap{},
+// il:nil,
+// dl:nil,
// ic:0,
// dc:0,
// },
@@ -858,8 +858,8 @@ func TestGetVector(t *testing.T) {
// f func(uuid string) bool
// }
// type fields struct {
-// il indexMap
-// dl indexMap
+// il sync.Map[string, *index]
+// dl sync.Map[string, *index]
// ic uint64
// dc uint64
// }
@@ -888,8 +888,8 @@ func TestGetVector(t *testing.T) {
// f:nil,
// },
// fields: fields {
-// il:indexMap{},
-// dl:indexMap{},
+// il:nil,
+// dl:nil,
// ic:0,
// dc:0,
// },
@@ -915,8 +915,8 @@ func TestGetVector(t *testing.T) {
// f:nil,
// },
// fields: fields {
-// il:indexMap{},
-// dl:indexMap{},
+// il:nil,
+// dl:nil,
// ic:0,
// dc:0,
// },
@@ -965,8 +965,8 @@ func TestGetVector(t *testing.T) {
//
// func Test_vqueue_IVQLen(t *testing.T) {
// type fields struct {
-// il indexMap
-// dl indexMap
+// il sync.Map[string, *index]
+// dl sync.Map[string, *index]
// ic uint64
// dc uint64
// }
@@ -993,8 +993,8 @@ func TestGetVector(t *testing.T) {
// {
// name: "test_case_1",
// fields: fields {
-// il:indexMap{},
-// dl:indexMap{},
+// il:nil,
+// dl:nil,
// ic:0,
// dc:0,
// },
@@ -1015,8 +1015,8 @@ func TestGetVector(t *testing.T) {
// return test {
// name: "test_case_2",
// fields: fields {
-// il:indexMap{},
-// dl:indexMap{},
+// il:nil,
+// dl:nil,
// ic:0,
// dc:0,
// },
@@ -1066,8 +1066,8 @@ func TestGetVector(t *testing.T) {
//
// func Test_vqueue_DVQLen(t *testing.T) {
// type fields struct {
-// il indexMap
-// dl indexMap
+// il sync.Map[string, *index]
+// dl sync.Map[string, *index]
// ic uint64
// dc uint64
// }
@@ -1094,8 +1094,8 @@ func TestGetVector(t *testing.T) {
// {
// name: "test_case_1",
// fields: fields {
-// il:indexMap{},
-// dl:indexMap{},
+// il:nil,
+// dl:nil,
// ic:0,
// dc:0,
// },
@@ -1116,8 +1116,8 @@ func TestGetVector(t *testing.T) {
// return test {
// name: "test_case_2",
// fields: fields {
-// il:indexMap{},
-// dl:indexMap{},
+// il:nil,
+// dl:nil,
// ic:0,
// dc:0,
// },
diff --git a/pkg/agent/core/ngt/usecase/agentd.go b/pkg/agent/core/ngt/usecase/agentd.go
index 2e198edea48..99a877d3182 100644
--- a/pkg/agent/core/ngt/usecase/agentd.go
+++ b/pkg/agent/core/ngt/usecase/agentd.go
@@ -22,7 +22,6 @@ import (
agent "github.com/vdaas/vald/apis/grpc/v1/agent/core"
vald "github.com/vdaas/vald/apis/grpc/v1/vald"
iconf "github.com/vdaas/vald/internal/config"
- "github.com/vdaas/vald/internal/errgroup"
"github.com/vdaas/vald/internal/net/grpc"
"github.com/vdaas/vald/internal/observability"
ngtmetrics "github.com/vdaas/vald/internal/observability/metrics/agent/core/ngt"
@@ -31,6 +30,7 @@ import (
"github.com/vdaas/vald/internal/safety"
"github.com/vdaas/vald/internal/servers/server"
"github.com/vdaas/vald/internal/servers/starter"
+ "github.com/vdaas/vald/internal/sync/errgroup"
"github.com/vdaas/vald/pkg/agent/core/ngt/config"
handler "github.com/vdaas/vald/pkg/agent/core/ngt/handler/grpc"
"github.com/vdaas/vald/pkg/agent/core/ngt/handler/rest"
diff --git a/pkg/agent/sidecar/service/observer/observer.go b/pkg/agent/sidecar/service/observer/observer.go
index d7d60e1df0c..301ba59af63 100644
--- a/pkg/agent/sidecar/service/observer/observer.go
+++ b/pkg/agent/sidecar/service/observer/observer.go
@@ -24,11 +24,9 @@ import (
"os"
"path/filepath"
"reflect"
- "sync"
"syscall"
"time"
- "github.com/vdaas/vald/internal/errgroup"
"github.com/vdaas/vald/internal/errors"
"github.com/vdaas/vald/internal/file"
"github.com/vdaas/vald/internal/file/watch"
@@ -37,6 +35,8 @@ import (
"github.com/vdaas/vald/internal/observability/attribute"
"github.com/vdaas/vald/internal/observability/trace"
"github.com/vdaas/vald/internal/safety"
+ "github.com/vdaas/vald/internal/sync"
+ "github.com/vdaas/vald/internal/sync/errgroup"
"github.com/vdaas/vald/pkg/agent/internal/metadata"
"github.com/vdaas/vald/pkg/agent/sidecar/service/storage"
)
diff --git a/pkg/agent/sidecar/service/observer/option.go b/pkg/agent/sidecar/service/observer/option.go
index 9c69d41a4f3..9e0a1cbd93c 100644
--- a/pkg/agent/sidecar/service/observer/option.go
+++ b/pkg/agent/sidecar/service/observer/option.go
@@ -18,8 +18,8 @@
package observer
import (
- "github.com/vdaas/vald/internal/errgroup"
"github.com/vdaas/vald/internal/file"
+ "github.com/vdaas/vald/internal/sync/errgroup"
"github.com/vdaas/vald/internal/timeutil"
"github.com/vdaas/vald/pkg/agent/internal/metadata"
"github.com/vdaas/vald/pkg/agent/sidecar/service/storage"
diff --git a/pkg/agent/sidecar/service/restorer/option.go b/pkg/agent/sidecar/service/restorer/option.go
index 5d6d8cb63de..7215d571198 100644
--- a/pkg/agent/sidecar/service/restorer/option.go
+++ b/pkg/agent/sidecar/service/restorer/option.go
@@ -19,7 +19,7 @@ package restorer
import (
"github.com/vdaas/vald/internal/backoff"
- "github.com/vdaas/vald/internal/errgroup"
+ "github.com/vdaas/vald/internal/sync/errgroup"
"github.com/vdaas/vald/pkg/agent/sidecar/service/storage"
)
diff --git a/pkg/agent/sidecar/service/restorer/restorer.go b/pkg/agent/sidecar/service/restorer/restorer.go
index ea2d6739c39..d3af05ad8a7 100644
--- a/pkg/agent/sidecar/service/restorer/restorer.go
+++ b/pkg/agent/sidecar/service/restorer/restorer.go
@@ -26,7 +26,6 @@ import (
"syscall"
"github.com/vdaas/vald/internal/backoff"
- "github.com/vdaas/vald/internal/errgroup"
"github.com/vdaas/vald/internal/errors"
"github.com/vdaas/vald/internal/file"
"github.com/vdaas/vald/internal/io"
@@ -34,6 +33,7 @@ import (
"github.com/vdaas/vald/internal/observability/trace"
"github.com/vdaas/vald/internal/safety"
"github.com/vdaas/vald/internal/strings"
+ "github.com/vdaas/vald/internal/sync/errgroup"
"github.com/vdaas/vald/pkg/agent/sidecar/service/storage"
)
diff --git a/pkg/agent/sidecar/service/storage/option.go b/pkg/agent/sidecar/service/storage/option.go
index 9e722a25164..d78e1e232db 100644
--- a/pkg/agent/sidecar/service/storage/option.go
+++ b/pkg/agent/sidecar/service/storage/option.go
@@ -22,7 +22,7 @@ import (
"github.com/vdaas/vald/internal/db/storage/blob/cloudstorage/urlopener"
"github.com/vdaas/vald/internal/db/storage/blob/s3"
"github.com/vdaas/vald/internal/db/storage/blob/s3/session"
- "github.com/vdaas/vald/internal/errgroup"
+ "github.com/vdaas/vald/internal/sync/errgroup"
)
type Option func(b *bs) error
diff --git a/pkg/agent/sidecar/service/storage/storage.go b/pkg/agent/sidecar/service/storage/storage.go
index 56b53031de5..5bcda32517f 100644
--- a/pkg/agent/sidecar/service/storage/storage.go
+++ b/pkg/agent/sidecar/service/storage/storage.go
@@ -28,9 +28,9 @@ import (
"github.com/vdaas/vald/internal/db/storage/blob/cloudstorage/urlopener"
"github.com/vdaas/vald/internal/db/storage/blob/s3"
"github.com/vdaas/vald/internal/db/storage/blob/s3/session"
- "github.com/vdaas/vald/internal/errgroup"
"github.com/vdaas/vald/internal/errors"
"github.com/vdaas/vald/internal/io"
+ "github.com/vdaas/vald/internal/sync/errgroup"
)
type Storage interface {
diff --git a/pkg/agent/sidecar/usecase/initcontainer/initcontainer.go b/pkg/agent/sidecar/usecase/initcontainer/initcontainer.go
index e670914921a..e153df6801a 100644
--- a/pkg/agent/sidecar/usecase/initcontainer/initcontainer.go
+++ b/pkg/agent/sidecar/usecase/initcontainer/initcontainer.go
@@ -25,7 +25,6 @@ import (
"github.com/vdaas/vald/internal/db/storage/blob/cloudstorage/urlopener"
"github.com/vdaas/vald/internal/db/storage/blob/s3"
"github.com/vdaas/vald/internal/db/storage/blob/s3/session"
- "github.com/vdaas/vald/internal/errgroup"
"github.com/vdaas/vald/internal/errors"
"github.com/vdaas/vald/internal/io"
"github.com/vdaas/vald/internal/log"
@@ -37,6 +36,7 @@ import (
"github.com/vdaas/vald/internal/safety"
"github.com/vdaas/vald/internal/servers/server"
"github.com/vdaas/vald/internal/servers/starter"
+ "github.com/vdaas/vald/internal/sync/errgroup"
"github.com/vdaas/vald/pkg/agent/sidecar/config"
handler "github.com/vdaas/vald/pkg/agent/sidecar/handler/grpc"
"github.com/vdaas/vald/pkg/agent/sidecar/handler/rest"
diff --git a/pkg/agent/sidecar/usecase/sidecar/sidecar.go b/pkg/agent/sidecar/usecase/sidecar/sidecar.go
index 2a4bb9b2bf5..9bcdfe9cee2 100644
--- a/pkg/agent/sidecar/usecase/sidecar/sidecar.go
+++ b/pkg/agent/sidecar/usecase/sidecar/sidecar.go
@@ -25,7 +25,6 @@ import (
"github.com/vdaas/vald/internal/db/storage/blob/cloudstorage/urlopener"
"github.com/vdaas/vald/internal/db/storage/blob/s3"
"github.com/vdaas/vald/internal/db/storage/blob/s3/session"
- "github.com/vdaas/vald/internal/errgroup"
"github.com/vdaas/vald/internal/log"
"github.com/vdaas/vald/internal/net"
"github.com/vdaas/vald/internal/net/grpc"
@@ -36,6 +35,7 @@ import (
"github.com/vdaas/vald/internal/safety"
"github.com/vdaas/vald/internal/servers/server"
"github.com/vdaas/vald/internal/servers/starter"
+ "github.com/vdaas/vald/internal/sync/errgroup"
"github.com/vdaas/vald/pkg/agent/sidecar/config"
handler "github.com/vdaas/vald/pkg/agent/sidecar/handler/grpc"
"github.com/vdaas/vald/pkg/agent/sidecar/handler/rest"
diff --git a/pkg/discoverer/k8s/handler/grpc/handler.go b/pkg/discoverer/k8s/handler/grpc/handler.go
index da5344bd892..6336ce9b5e3 100644
--- a/pkg/discoverer/k8s/handler/grpc/handler.go
+++ b/pkg/discoverer/k8s/handler/grpc/handler.go
@@ -28,8 +28,8 @@ import (
"github.com/vdaas/vald/internal/net/grpc/errdetails"
"github.com/vdaas/vald/internal/net/grpc/status"
"github.com/vdaas/vald/internal/observability/trace"
- "github.com/vdaas/vald/internal/singleflight"
"github.com/vdaas/vald/internal/strings"
+ "github.com/vdaas/vald/internal/sync/singleflight"
"github.com/vdaas/vald/pkg/discoverer/k8s/service"
)
@@ -81,7 +81,7 @@ func (s *server) Pods(ctx context.Context, req *payload.Discoverer_Request) (*pa
}
}()
key := singleflightKey(podPrefix, req)
- res, _, err := s.pgroup.Do(ctx, key, func() (*payload.Info_Pods, error) {
+ res, _, err := s.pgroup.Do(ctx, key, func(context.Context) (*payload.Info_Pods, error) {
return s.dsc.GetPods(req)
})
if err != nil {
@@ -154,7 +154,7 @@ func (s *server) Nodes(ctx context.Context, req *payload.Discoverer_Request) (*p
}()
key := singleflightKey(nodePrefix, req)
- res, _, err := s.ngroup.Do(ctx, key, func() (*payload.Info_Nodes, error) {
+ res, _, err := s.ngroup.Do(ctx, key, func(context.Context) (*payload.Info_Nodes, error) {
return s.dsc.GetNodes(req)
})
if err != nil {
diff --git a/pkg/discoverer/k8s/router/option.go b/pkg/discoverer/k8s/router/option.go
index 6565c6409b9..7eb7bfef693 100644
--- a/pkg/discoverer/k8s/router/option.go
+++ b/pkg/discoverer/k8s/router/option.go
@@ -18,7 +18,7 @@
package router
import (
- "github.com/vdaas/vald/internal/errgroup"
+ "github.com/vdaas/vald/internal/sync/errgroup"
"github.com/vdaas/vald/pkg/discoverer/k8s/handler/rest"
)
diff --git a/pkg/discoverer/k8s/router/router.go b/pkg/discoverer/k8s/router/router.go
index 8503a504dbb..c3f727be1cf 100644
--- a/pkg/discoverer/k8s/router/router.go
+++ b/pkg/discoverer/k8s/router/router.go
@@ -20,9 +20,9 @@ package router
import (
"net/http"
- "github.com/vdaas/vald/internal/errgroup"
"github.com/vdaas/vald/internal/net/http/middleware"
"github.com/vdaas/vald/internal/net/http/routing"
+ "github.com/vdaas/vald/internal/sync/errgroup"
"github.com/vdaas/vald/pkg/discoverer/k8s/handler/rest"
)
diff --git a/pkg/discoverer/k8s/service/discover.go b/pkg/discoverer/k8s/service/discover.go
index c6450f5e4a0..4f3a8b0eced 100644
--- a/pkg/discoverer/k8s/service/discover.go
+++ b/pkg/discoverer/k8s/service/discover.go
@@ -20,13 +20,11 @@ package service
import (
"context"
"reflect"
- "sync"
"sync/atomic"
"time"
"github.com/vdaas/vald/apis/grpc/v1/payload"
"github.com/vdaas/vald/internal/config"
- "github.com/vdaas/vald/internal/errgroup"
"github.com/vdaas/vald/internal/errors"
"github.com/vdaas/vald/internal/k8s"
mnode "github.com/vdaas/vald/internal/k8s/metrics/node"
@@ -37,7 +35,8 @@ import (
"github.com/vdaas/vald/internal/net"
"github.com/vdaas/vald/internal/safety"
"github.com/vdaas/vald/internal/slices"
- valdsync "github.com/vdaas/vald/internal/sync"
+ "github.com/vdaas/vald/internal/sync"
+ "github.com/vdaas/vald/internal/sync/errgroup"
)
type Discoverer interface {
@@ -48,10 +47,10 @@ type Discoverer interface {
type discoverer struct {
maxPods int
- nodes valdsync.Map[string, *node.Node]
- nodeMetrics valdsync.Map[string, mnode.Node]
- pods valdsync.Map[string, *[]pod.Pod]
- podMetrics valdsync.Map[string, mpod.Pod]
+ nodes sync.Map[string, *node.Node]
+ nodeMetrics sync.Map[string, mnode.Node]
+ pods sync.Map[string, *[]pod.Pod]
+ podMetrics sync.Map[string, mpod.Pod]
podsByNode atomic.Value
podsByNamespace atomic.Value
podsByName atomic.Value
diff --git a/pkg/discoverer/k8s/service/discover_test.go b/pkg/discoverer/k8s/service/discover_test.go
index f7f68f5e8e3..4e8659477ab 100644
--- a/pkg/discoverer/k8s/service/discover_test.go
+++ b/pkg/discoverer/k8s/service/discover_test.go
@@ -118,10 +118,10 @@ package service
// }
// type fields struct {
// maxPods int
-// nodes nodeMap
-// nodeMetrics nodeMetricsMap
-// pods podsMap
-// podMetrics podMetricsMap
+// nodes sync.Map[string, *node.Node]
+// nodeMetrics sync.Map[string, mnode.Node]
+// pods sync.Map[string, *[]pod.Pod]
+// podMetrics sync.Map[string, mpod.Pod]
// podsByNode atomic.Value
// podsByNamespace atomic.Value
// podsByName atomic.Value
@@ -165,10 +165,10 @@ package service
// },
// fields: fields {
// maxPods:0,
-// nodes:nodeMap{},
-// nodeMetrics:nodeMetricsMap{},
-// pods:podsMap{},
-// podMetrics:podMetricsMap{},
+// nodes:nil,
+// nodeMetrics:nil,
+// pods:nil,
+// podMetrics:nil,
// podsByNode:nil,
// podsByNamespace:nil,
// podsByName:nil,
@@ -201,10 +201,10 @@ package service
// },
// fields: fields {
// maxPods:0,
-// nodes:nodeMap{},
-// nodeMetrics:nodeMetricsMap{},
-// pods:podsMap{},
-// podMetrics:podMetricsMap{},
+// nodes:nil,
+// nodeMetrics:nil,
+// pods:nil,
+// podMetrics:nil,
// podsByNode:nil,
// podsByNamespace:nil,
// podsByName:nil,
@@ -277,10 +277,10 @@ package service
// }
// type fields struct {
// maxPods int
-// nodes nodeMap
-// nodeMetrics nodeMetricsMap
-// pods podsMap
-// podMetrics podMetricsMap
+// nodes sync.Map[string, *node.Node]
+// nodeMetrics sync.Map[string, mnode.Node]
+// pods sync.Map[string, *[]pod.Pod]
+// podMetrics sync.Map[string, mpod.Pod]
// podsByNode atomic.Value
// podsByNamespace atomic.Value
// podsByName atomic.Value
@@ -324,10 +324,10 @@ package service
// },
// fields: fields {
// maxPods:0,
-// nodes:nodeMap{},
-// nodeMetrics:nodeMetricsMap{},
-// pods:podsMap{},
-// podMetrics:podMetricsMap{},
+// nodes:nil,
+// nodeMetrics:nil,
+// pods:nil,
+// podMetrics:nil,
// podsByNode:nil,
// podsByNamespace:nil,
// podsByName:nil,
@@ -360,10 +360,10 @@ package service
// },
// fields: fields {
// maxPods:0,
-// nodes:nodeMap{},
-// nodeMetrics:nodeMetricsMap{},
-// pods:podsMap{},
-// podMetrics:podMetricsMap{},
+// nodes:nil,
+// nodeMetrics:nil,
+// pods:nil,
+// podMetrics:nil,
// podsByNode:nil,
// podsByNamespace:nil,
// podsByName:nil,
@@ -436,10 +436,10 @@ package service
// }
// type fields struct {
// maxPods int
-// nodes nodeMap
-// nodeMetrics nodeMetricsMap
-// pods podsMap
-// podMetrics podMetricsMap
+// nodes sync.Map[string, *node.Node]
+// nodeMetrics sync.Map[string, mnode.Node]
+// pods sync.Map[string, *[]pod.Pod]
+// podMetrics sync.Map[string, mpod.Pod]
// podsByNode atomic.Value
// podsByNamespace atomic.Value
// podsByName atomic.Value
@@ -483,10 +483,10 @@ package service
// },
// fields: fields {
// maxPods:0,
-// nodes:nodeMap{},
-// nodeMetrics:nodeMetricsMap{},
-// pods:podsMap{},
-// podMetrics:podMetricsMap{},
+// nodes:nil,
+// nodeMetrics:nil,
+// pods:nil,
+// podMetrics:nil,
// podsByNode:nil,
// podsByNamespace:nil,
// podsByName:nil,
@@ -519,10 +519,10 @@ package service
// },
// fields: fields {
// maxPods:0,
-// nodes:nodeMap{},
-// nodeMetrics:nodeMetricsMap{},
-// pods:podsMap{},
-// podMetrics:podMetricsMap{},
+// nodes:nil,
+// nodeMetrics:nil,
+// pods:nil,
+// podMetrics:nil,
// podsByNode:nil,
// podsByNamespace:nil,
// podsByName:nil,
diff --git a/pkg/discoverer/k8s/service/option.go b/pkg/discoverer/k8s/service/option.go
index 635bdd2f544..8174679d297 100644
--- a/pkg/discoverer/k8s/service/option.go
+++ b/pkg/discoverer/k8s/service/option.go
@@ -20,8 +20,8 @@ package service
import (
"time"
- "github.com/vdaas/vald/internal/errgroup"
"github.com/vdaas/vald/internal/net"
+ "github.com/vdaas/vald/internal/sync/errgroup"
"github.com/vdaas/vald/internal/timeutil"
)
diff --git a/pkg/discoverer/k8s/usecase/discovered.go b/pkg/discoverer/k8s/usecase/discovered.go
index fa0f0326389..9d564fa17dd 100644
--- a/pkg/discoverer/k8s/usecase/discovered.go
+++ b/pkg/discoverer/k8s/usecase/discovered.go
@@ -21,7 +21,6 @@ import (
"github.com/vdaas/vald/apis/grpc/v1/discoverer"
iconf "github.com/vdaas/vald/internal/config"
- "github.com/vdaas/vald/internal/errgroup"
"github.com/vdaas/vald/internal/log"
"github.com/vdaas/vald/internal/net"
"github.com/vdaas/vald/internal/net/grpc"
@@ -32,6 +31,7 @@ import (
"github.com/vdaas/vald/internal/safety"
"github.com/vdaas/vald/internal/servers/server"
"github.com/vdaas/vald/internal/servers/starter"
+ "github.com/vdaas/vald/internal/sync/errgroup"
"github.com/vdaas/vald/pkg/discoverer/k8s/config"
handler "github.com/vdaas/vald/pkg/discoverer/k8s/handler/grpc"
"github.com/vdaas/vald/pkg/discoverer/k8s/handler/rest"
diff --git a/pkg/gateway/filter/handler/grpc/handler.go b/pkg/gateway/filter/handler/grpc/handler.go
index 1edba3227dc..45ccf9da5a4 100644
--- a/pkg/gateway/filter/handler/grpc/handler.go
+++ b/pkg/gateway/filter/handler/grpc/handler.go
@@ -21,7 +21,6 @@ import (
"context"
"fmt"
"strconv"
- "sync"
"github.com/vdaas/vald/apis/grpc/v1/payload"
"github.com/vdaas/vald/apis/grpc/v1/vald"
@@ -29,7 +28,6 @@ import (
"github.com/vdaas/vald/internal/client/v1/client/filter/ingress"
client "github.com/vdaas/vald/internal/client/v1/client/vald"
"github.com/vdaas/vald/internal/core/algorithm"
- "github.com/vdaas/vald/internal/errgroup"
"github.com/vdaas/vald/internal/errors"
"github.com/vdaas/vald/internal/info"
"github.com/vdaas/vald/internal/log"
@@ -39,6 +37,8 @@ import (
"github.com/vdaas/vald/internal/net/grpc/status"
"github.com/vdaas/vald/internal/observability/trace"
"github.com/vdaas/vald/internal/safety"
+ "github.com/vdaas/vald/internal/sync"
+ "github.com/vdaas/vald/internal/sync/errgroup"
)
type server struct {
diff --git a/pkg/gateway/filter/handler/grpc/option.go b/pkg/gateway/filter/handler/grpc/option.go
index a0bd58a4e3b..f2694c09666 100644
--- a/pkg/gateway/filter/handler/grpc/option.go
+++ b/pkg/gateway/filter/handler/grpc/option.go
@@ -24,9 +24,9 @@ import (
"github.com/vdaas/vald/internal/client/v1/client/filter/egress"
"github.com/vdaas/vald/internal/client/v1/client/filter/ingress"
"github.com/vdaas/vald/internal/client/v1/client/vald"
- "github.com/vdaas/vald/internal/errgroup"
"github.com/vdaas/vald/internal/log"
"github.com/vdaas/vald/internal/net"
+ "github.com/vdaas/vald/internal/sync/errgroup"
)
type Option func(*server)
diff --git a/pkg/gateway/filter/usecase/vald.go b/pkg/gateway/filter/usecase/vald.go
index 447671b99bf..88fa49372e9 100644
--- a/pkg/gateway/filter/usecase/vald.go
+++ b/pkg/gateway/filter/usecase/vald.go
@@ -23,7 +23,6 @@ import (
"github.com/vdaas/vald/internal/client/v1/client/filter/egress"
"github.com/vdaas/vald/internal/client/v1/client/filter/ingress"
client "github.com/vdaas/vald/internal/client/v1/client/vald"
- "github.com/vdaas/vald/internal/errgroup"
"github.com/vdaas/vald/internal/errors"
"github.com/vdaas/vald/internal/net/grpc"
"github.com/vdaas/vald/internal/observability"
@@ -34,6 +33,7 @@ import (
"github.com/vdaas/vald/internal/servers/server"
"github.com/vdaas/vald/internal/servers/starter"
"github.com/vdaas/vald/internal/slices"
+ "github.com/vdaas/vald/internal/sync/errgroup"
"github.com/vdaas/vald/pkg/gateway/filter/config"
handler "github.com/vdaas/vald/pkg/gateway/filter/handler/grpc"
"github.com/vdaas/vald/pkg/gateway/filter/handler/rest"
diff --git a/pkg/gateway/lb/handler/grpc/aggregation.go b/pkg/gateway/lb/handler/grpc/aggregation.go
index 3779201327b..702abc34e67 100644
--- a/pkg/gateway/lb/handler/grpc/aggregation.go
+++ b/pkg/gateway/lb/handler/grpc/aggregation.go
@@ -18,13 +18,11 @@ import (
"fmt"
"math"
"math/big"
- "sync"
"sync/atomic"
"time"
"github.com/vdaas/vald/apis/grpc/v1/payload"
"github.com/vdaas/vald/apis/grpc/v1/vald"
- "github.com/vdaas/vald/internal/errgroup"
"github.com/vdaas/vald/internal/errors"
"github.com/vdaas/vald/internal/info"
"github.com/vdaas/vald/internal/log"
@@ -35,7 +33,8 @@ import (
"github.com/vdaas/vald/internal/observability/trace"
"github.com/vdaas/vald/internal/safety"
"github.com/vdaas/vald/internal/slices"
- valdsync "github.com/vdaas/vald/internal/sync"
+ "github.com/vdaas/vald/internal/sync"
+ "github.com/vdaas/vald/internal/sync/errgroup"
)
type Aggregator interface {
@@ -366,7 +365,7 @@ type valdStdAggr struct {
dch chan DistPayload
closed atomic.Bool
maxDist atomic.Value
- visited valdsync.Map[string, any]
+ visited sync.Map[string, any]
result []*payload.Object_Distance
cancel context.CancelFunc
}
@@ -496,7 +495,7 @@ type valdPairingHeapAggr struct {
num int
ph *PairingHeap
mu sync.Mutex
- visited valdsync.Map[string, any]
+ visited sync.Map[string, any]
result []*payload.Object_Distance
}
diff --git a/pkg/gateway/lb/handler/grpc/aggregation_test.go b/pkg/gateway/lb/handler/grpc/aggregation_test.go
index c7a1ef1e8a3..d36e1c4de42 100644
--- a/pkg/gateway/lb/handler/grpc/aggregation_test.go
+++ b/pkg/gateway/lb/handler/grpc/aggregation_test.go
@@ -24,6 +24,7 @@ package grpc
// dch chan DistPayload
// closed atomic.Bool
// maxDist atomic.Value
+// visited sync.Map[string, any]
// result []*payload.Object_Distance
// cancel context.CancelFunc
// }
@@ -54,6 +55,7 @@ package grpc
// dch:nil,
// closed:nil,
// maxDist:nil,
+// visited:nil,
// result:nil,
// cancel:nil,
// },
@@ -81,6 +83,7 @@ package grpc
// dch:nil,
// closed:nil,
// maxDist:nil,
+// visited:nil,
// result:nil,
// cancel:nil,
// },
@@ -117,6 +120,7 @@ package grpc
// dch: test.fields.dch,
// closed: test.fields.closed,
// maxDist: test.fields.maxDist,
+// visited: test.fields.visited,
// result: test.fields.result,
// cancel: test.fields.cancel,
// }
@@ -139,6 +143,7 @@ package grpc
// dch chan DistPayload
// closed atomic.Bool
// maxDist atomic.Value
+// visited sync.Map[string, any]
// result []*payload.Object_Distance
// cancel context.CancelFunc
// }
@@ -170,6 +175,7 @@ package grpc
// dch:nil,
// closed:nil,
// maxDist:nil,
+// visited:nil,
// result:nil,
// cancel:nil,
// },
@@ -198,6 +204,7 @@ package grpc
// dch:nil,
// closed:nil,
// maxDist:nil,
+// visited:nil,
// result:nil,
// cancel:nil,
// },
@@ -234,6 +241,7 @@ package grpc
// dch: test.fields.dch,
// closed: test.fields.closed,
// maxDist: test.fields.maxDist,
+// visited: test.fields.visited,
// result: test.fields.result,
// cancel: test.fields.cancel,
// }
@@ -252,6 +260,7 @@ package grpc
// dch chan DistPayload
// closed atomic.Bool
// maxDist atomic.Value
+// visited sync.Map[string, any]
// result []*payload.Object_Distance
// cancel context.CancelFunc
// }
@@ -282,6 +291,7 @@ package grpc
// dch:nil,
// closed:nil,
// maxDist:nil,
+// visited:nil,
// result:nil,
// cancel:nil,
// },
@@ -306,6 +316,7 @@ package grpc
// dch:nil,
// closed:nil,
// maxDist:nil,
+// visited:nil,
// result:nil,
// cancel:nil,
// },
@@ -342,6 +353,7 @@ package grpc
// dch: test.fields.dch,
// closed: test.fields.closed,
// maxDist: test.fields.maxDist,
+// visited: test.fields.visited,
// result: test.fields.result,
// cancel: test.fields.cancel,
// }
@@ -360,9 +372,10 @@ package grpc
// in0 context.Context
// }
// type fields struct {
-// num int
-// ph *PairingHeap
-// result []*payload.Object_Distance
+// num int
+// ph *PairingHeap
+// visited sync.Map[string, any]
+// result []*payload.Object_Distance
// }
// type want struct {
// }
@@ -389,6 +402,7 @@ package grpc
// fields: fields {
// num:0,
// ph:PairingHeap{},
+// visited:nil,
// result:nil,
// },
// want: want{},
@@ -413,6 +427,7 @@ package grpc
// fields: fields {
// num:0,
// ph:PairingHeap{},
+// visited:nil,
// result:nil,
// },
// want: want{},
@@ -444,9 +459,10 @@ package grpc
// checkFunc = defaultCheckFunc
// }
// v := &valdPairingHeapAggr{
-// num: test.fields.num,
-// ph: test.fields.ph,
-// result: test.fields.result,
+// num: test.fields.num,
+// ph: test.fields.ph,
+// visited: test.fields.visited,
+// result: test.fields.result,
// }
//
// v.Start(test.args.in0)
@@ -463,9 +479,10 @@ package grpc
// data *payload.Search_Response
// }
// type fields struct {
-// num int
-// ph *PairingHeap
-// result []*payload.Object_Distance
+// num int
+// ph *PairingHeap
+// visited sync.Map[string, any]
+// result []*payload.Object_Distance
// }
// type want struct {
// }
@@ -493,6 +510,7 @@ package grpc
// fields: fields {
// num:0,
// ph:PairingHeap{},
+// visited:nil,
// result:nil,
// },
// want: want{},
@@ -518,6 +536,7 @@ package grpc
// fields: fields {
// num:0,
// ph:PairingHeap{},
+// visited:nil,
// result:nil,
// },
// want: want{},
@@ -549,9 +568,10 @@ package grpc
// checkFunc = defaultCheckFunc
// }
// v := &valdPairingHeapAggr{
-// num: test.fields.num,
-// ph: test.fields.ph,
-// result: test.fields.result,
+// num: test.fields.num,
+// ph: test.fields.ph,
+// visited: test.fields.visited,
+// result: test.fields.result,
// }
//
// v.Send(test.args.ctx, test.args.data)
@@ -564,9 +584,10 @@ package grpc
//
// func Test_valdPairingHeapAggr_Result(t *testing.T) {
// type fields struct {
-// num int
-// ph *PairingHeap
-// result []*payload.Object_Distance
+// num int
+// ph *PairingHeap
+// visited sync.Map[string, any]
+// result []*payload.Object_Distance
// }
// type want struct {
// want *payload.Search_Response
@@ -593,6 +614,7 @@ package grpc
// fields: fields {
// num:0,
// ph:PairingHeap{},
+// visited:nil,
// result:nil,
// },
// want: want{},
@@ -614,6 +636,7 @@ package grpc
// fields: fields {
// num:0,
// ph:PairingHeap{},
+// visited:nil,
// result:nil,
// },
// want: want{},
@@ -645,9 +668,10 @@ package grpc
// checkFunc = defaultCheckFunc
// }
// v := &valdPairingHeapAggr{
-// num: test.fields.num,
-// ph: test.fields.ph,
-// result: test.fields.result,
+// num: test.fields.num,
+// ph: test.fields.ph,
+// visited: test.fields.visited,
+// result: test.fields.result,
// }
//
// got := v.Result()
diff --git a/pkg/gateway/lb/handler/grpc/handler.go b/pkg/gateway/lb/handler/grpc/handler.go
index 30f14f00052..2f25951b132 100644
--- a/pkg/gateway/lb/handler/grpc/handler.go
+++ b/pkg/gateway/lb/handler/grpc/handler.go
@@ -21,7 +21,6 @@ import (
"context"
"fmt"
"strconv"
- "sync"
"sync/atomic"
"time"
@@ -29,7 +28,6 @@ import (
"github.com/vdaas/vald/apis/grpc/v1/vald"
"github.com/vdaas/vald/internal/conv"
"github.com/vdaas/vald/internal/core/algorithm"
- "github.com/vdaas/vald/internal/errgroup"
"github.com/vdaas/vald/internal/errors"
"github.com/vdaas/vald/internal/info"
"github.com/vdaas/vald/internal/log"
@@ -42,6 +40,8 @@ import (
"github.com/vdaas/vald/internal/safety"
"github.com/vdaas/vald/internal/slices"
"github.com/vdaas/vald/internal/strings"
+ "github.com/vdaas/vald/internal/sync"
+ "github.com/vdaas/vald/internal/sync/errgroup"
"github.com/vdaas/vald/pkg/gateway/lb/service"
)
@@ -1484,7 +1484,7 @@ func (s *server) MultiInsert(ctx context.Context, reqs *payload.Insert_MultiRequ
lmu sync.Mutex
)
eg, ectx := errgroup.New(ctx)
- eg.Limitation(s.multiConcurrency)
+ eg.SetLimit(s.multiConcurrency)
locs = &payload.Object_Locations{
Locations: make([]*payload.Object_Location, len(reqs.GetRequests())),
}
@@ -2021,7 +2021,7 @@ func (s *server) MultiUpdate(ctx context.Context, reqs *payload.Update_MultiRequ
lmu sync.Mutex
)
eg, ectx := errgroup.New(ctx)
- eg.Limitation(s.multiConcurrency)
+ eg.SetLimit(s.multiConcurrency)
locs = &payload.Object_Locations{
Locations: make([]*payload.Object_Location, len(reqs.GetRequests())),
}
@@ -2370,7 +2370,7 @@ func (s *server) MultiUpsert(ctx context.Context, reqs *payload.Upsert_MultiRequ
lmu sync.Mutex
)
eg, ectx := errgroup.New(ctx)
- eg.Limitation(s.multiConcurrency)
+ eg.SetLimit(s.multiConcurrency)
locs = &payload.Object_Locations{
Locations: make([]*payload.Object_Location, len(reqs.GetRequests())),
}
@@ -2674,7 +2674,7 @@ func (s *server) MultiRemove(ctx context.Context, reqs *payload.Remove_MultiRequ
lmu sync.Mutex
)
eg, ectx := errgroup.New(ctx)
- eg.Limitation(s.multiConcurrency)
+ eg.SetLimit(s.multiConcurrency)
locs = &payload.Object_Locations{
Locations: make([]*payload.Object_Location, len(reqs.GetRequests())),
}
diff --git a/pkg/gateway/lb/handler/grpc/option.go b/pkg/gateway/lb/handler/grpc/option.go
index d1a5d01e930..cbb0ef7f6cb 100644
--- a/pkg/gateway/lb/handler/grpc/option.go
+++ b/pkg/gateway/lb/handler/grpc/option.go
@@ -22,9 +22,9 @@ import (
"runtime"
"time"
- "github.com/vdaas/vald/internal/errgroup"
"github.com/vdaas/vald/internal/log"
"github.com/vdaas/vald/internal/net"
+ "github.com/vdaas/vald/internal/sync/errgroup"
"github.com/vdaas/vald/internal/timeutil"
"github.com/vdaas/vald/pkg/gateway/lb/service"
)
diff --git a/pkg/gateway/lb/handler/grpc/search_benchmark_test.go b/pkg/gateway/lb/handler/grpc/search_benchmark_test.go
index 91029300878..834d9827192 100644
--- a/pkg/gateway/lb/handler/grpc/search_benchmark_test.go
+++ b/pkg/gateway/lb/handler/grpc/search_benchmark_test.go
@@ -18,15 +18,15 @@ import (
"flag"
"fmt"
"runtime"
- "sync"
"sync/atomic"
"testing"
"time"
"github.com/vdaas/vald/apis/grpc/v1/payload"
- "github.com/vdaas/vald/internal/errgroup"
"github.com/vdaas/vald/internal/rand"
"github.com/vdaas/vald/internal/slices"
+ "github.com/vdaas/vald/internal/sync"
+ "github.com/vdaas/vald/internal/sync/errgroup"
"github.com/vdaas/vald/internal/test/data/strings"
)
@@ -116,7 +116,7 @@ func doSearchWithAggregator(ctx context.Context, k, concurrency int, anew func(n
f func(ctx context.Context) *payload.Search_Response,
) (res *payload.Search_Response, err error) {
eg, ectx := errgroup.New(ctx)
- eg.Limitation(concurrency)
+ eg.SetLimit(concurrency)
aggr := anew(k, concurrency)
aggr.Start(ectx)
for i := 0; i < concurrency; i++ {
diff --git a/pkg/gateway/lb/service/gateway.go b/pkg/gateway/lb/service/gateway.go
index 1b046637e6d..47ea5c75ffc 100644
--- a/pkg/gateway/lb/service/gateway.go
+++ b/pkg/gateway/lb/service/gateway.go
@@ -24,11 +24,11 @@ import (
"github.com/vdaas/vald/apis/grpc/v1/vald"
"github.com/vdaas/vald/internal/client/v1/client/discoverer"
- "github.com/vdaas/vald/internal/errgroup"
"github.com/vdaas/vald/internal/errors"
"github.com/vdaas/vald/internal/net/grpc"
"github.com/vdaas/vald/internal/observability/trace"
- valdsync "github.com/vdaas/vald/internal/sync"
+ "github.com/vdaas/vald/internal/sync"
+ "github.com/vdaas/vald/internal/sync/errgroup"
)
type Gateway interface {
@@ -102,7 +102,7 @@ func (g *gateway) DoMulti(ctx context.Context, num int,
} else {
limit = uint32(num)
}
- var visited valdsync.Map[string, any]
+ var visited sync.Map[string, any]
err = g.client.GetClient().OrderedRange(sctx, addrs, func(ictx context.Context,
addr string,
conn *grpc.ClientConn,
diff --git a/pkg/gateway/lb/service/option.go b/pkg/gateway/lb/service/option.go
index c88d2107d95..988e7fa7322 100644
--- a/pkg/gateway/lb/service/option.go
+++ b/pkg/gateway/lb/service/option.go
@@ -19,7 +19,7 @@ package service
import (
"github.com/vdaas/vald/internal/client/v1/client/discoverer"
- "github.com/vdaas/vald/internal/errgroup"
+ "github.com/vdaas/vald/internal/sync/errgroup"
)
type Option func(g *gateway) error
diff --git a/pkg/gateway/lb/usecase/vald.go b/pkg/gateway/lb/usecase/vald.go
index baa08e94d93..e5b8abce48d 100644
--- a/pkg/gateway/lb/usecase/vald.go
+++ b/pkg/gateway/lb/usecase/vald.go
@@ -22,7 +22,6 @@ import (
"github.com/vdaas/vald/apis/grpc/v1/vald"
"github.com/vdaas/vald/internal/client/v1/client/discoverer"
- "github.com/vdaas/vald/internal/errgroup"
"github.com/vdaas/vald/internal/net/grpc"
"github.com/vdaas/vald/internal/observability"
backoffmetrics "github.com/vdaas/vald/internal/observability/metrics/backoff"
@@ -31,6 +30,7 @@ import (
"github.com/vdaas/vald/internal/safety"
"github.com/vdaas/vald/internal/servers/server"
"github.com/vdaas/vald/internal/servers/starter"
+ "github.com/vdaas/vald/internal/sync/errgroup"
"github.com/vdaas/vald/pkg/gateway/lb/config"
handler "github.com/vdaas/vald/pkg/gateway/lb/handler/grpc"
"github.com/vdaas/vald/pkg/gateway/lb/handler/rest"
diff --git a/pkg/manager/index/handler/grpc/handler.go b/pkg/manager/index/handler/grpc/handler.go
index a58fd80fdfe..fbddf3f6914 100644
--- a/pkg/manager/index/handler/grpc/handler.go
+++ b/pkg/manager/index/handler/grpc/handler.go
@@ -53,3 +53,13 @@ func (s *server) IndexInfo(ctx context.Context, _ *payload.Empty) (res *payload.
Indexing: s.indexer.IsIndexing(),
}, nil
}
+
+func (s *server) IndexDetail(ctx context.Context, _ *payload.Empty) (res *payload.Info_Index_Detail, err error) {
+ ctx, span := trace.StartSpan(ctx, "vald/manager-index.IndexDetail")
+ defer func() {
+ if span != nil {
+ span.End()
+ }
+ }()
+ return s.indexer.LoadIndexDetail(), nil
+}
diff --git a/pkg/manager/index/handler/grpc/handler_test.go b/pkg/manager/index/handler/grpc/handler_test.go
index 4fc33375179..f85fbe68d42 100644
--- a/pkg/manager/index/handler/grpc/handler_test.go
+++ b/pkg/manager/index/handler/grpc/handler_test.go
@@ -214,3 +214,113 @@ package grpc
// })
// }
// }
+//
+// func Test_server_IndexDetail(t *testing.T) {
+// type args struct {
+// ctx context.Context
+// in1 *payload.Empty
+// }
+// type fields struct {
+// indexer service.Indexer
+// UnimplementedIndexServer index.UnimplementedIndexServer
+// }
+// type want struct {
+// wantRes *payload.Info_Index_Detail
+// err error
+// }
+// type test struct {
+// name string
+// args args
+// fields fields
+// want want
+// checkFunc func(want, *payload.Info_Index_Detail, error) error
+// beforeFunc func(*testing.T, args)
+// afterFunc func(*testing.T, args)
+// }
+// defaultCheckFunc := func(w want, gotRes *payload.Info_Index_Detail, err error) error {
+// if !errors.Is(err, w.err) {
+// return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err)
+// }
+// if !reflect.DeepEqual(gotRes, w.wantRes) {
+// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", gotRes, w.wantRes)
+// }
+// return nil
+// }
+// tests := []test{
+// // TODO test cases
+// /*
+// {
+// name: "test_case_1",
+// args: args {
+// ctx:nil,
+// in1:nil,
+// },
+// fields: fields {
+// indexer:nil,
+// UnimplementedIndexServer:nil,
+// },
+// want: want{},
+// checkFunc: defaultCheckFunc,
+// beforeFunc: func(t *testing.T, args args) {
+// t.Helper()
+// },
+// afterFunc: func(t *testing.T, args args) {
+// t.Helper()
+// },
+// },
+// */
+//
+// // TODO test cases
+// /*
+// func() test {
+// return test {
+// name: "test_case_2",
+// args: args {
+// ctx:nil,
+// in1:nil,
+// },
+// fields: fields {
+// indexer:nil,
+// UnimplementedIndexServer:nil,
+// },
+// want: want{},
+// checkFunc: defaultCheckFunc,
+// beforeFunc: func(t *testing.T, args args) {
+// t.Helper()
+// },
+// afterFunc: func(t *testing.T, args args) {
+// t.Helper()
+// },
+// }
+// }(),
+// */
+// }
+//
+// for _, tc := range tests {
+// test := tc
+// t.Run(test.name, func(tt *testing.T) {
+// tt.Parallel()
+// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent())
+// if test.beforeFunc != nil {
+// test.beforeFunc(tt, test.args)
+// }
+// if test.afterFunc != nil {
+// defer test.afterFunc(tt, test.args)
+// }
+// checkFunc := test.checkFunc
+// if test.checkFunc == nil {
+// checkFunc = defaultCheckFunc
+// }
+// s := &server{
+// indexer: test.fields.indexer,
+// UnimplementedIndexServer: test.fields.UnimplementedIndexServer,
+// }
+//
+// gotRes, err := s.IndexDetail(test.args.ctx, test.args.in1)
+// if err := checkFunc(test.want, gotRes, err); err != nil {
+// tt.Errorf("error = %v", err)
+// }
+//
+// })
+// }
+// }
diff --git a/pkg/manager/index/service/indexer.go b/pkg/manager/index/service/indexer.go
index 365db870c6f..d3ec90bea94 100644
--- a/pkg/manager/index/service/indexer.go
+++ b/pkg/manager/index/service/indexer.go
@@ -27,7 +27,6 @@ import (
agent "github.com/vdaas/vald/apis/grpc/v1/agent/core"
"github.com/vdaas/vald/apis/grpc/v1/payload"
"github.com/vdaas/vald/internal/client/v1/client/discoverer"
- "github.com/vdaas/vald/internal/errgroup"
"github.com/vdaas/vald/internal/errors"
"github.com/vdaas/vald/internal/log"
"github.com/vdaas/vald/internal/net/grpc"
@@ -35,7 +34,8 @@ import (
"github.com/vdaas/vald/internal/net/grpc/status"
"github.com/vdaas/vald/internal/observability/trace"
"github.com/vdaas/vald/internal/safety"
- valdsync "github.com/vdaas/vald/internal/sync"
+ "github.com/vdaas/vald/internal/sync"
+ "github.com/vdaas/vald/internal/sync/errgroup"
)
type Indexer interface {
@@ -43,6 +43,7 @@ type Indexer interface {
NumberOfUUIDs() uint32
NumberOfUncommittedUUIDs() uint32
IsIndexing() bool
+ LoadIndexDetail() *payload.Info_Index_Detail
}
type index struct {
@@ -54,9 +55,9 @@ type index struct {
saveIndexDurationLimit time.Duration
saveIndexWaitDuration time.Duration
saveIndexTargetAddrCh chan string
- schMap valdsync.Map[string, any]
+ schMap sync.Map[string, any]
concurrency int
- indexInfos valdsync.Map[string, *payload.Info_Index_Count]
+ indexInfos sync.Map[string, *payload.Info_Index_Count]
indexing atomic.Value // bool
minUncommitted uint32
uuidsCount uint32
@@ -121,7 +122,9 @@ func (idx *index) Start(ctx context.Context) (<-chan error, error) {
ech <- err
case err = <-sech:
ech <- err
- case <-it.C:
+ case <-it.C: // index duration ticker
+ // execute CreateIndex. This execution ignores low index agent,
+ // and does not immediately execute SaveIndex.
err = idx.execute(grpc.WithGRPCMethod(ctx, "core.v1.Agent/CreateIndex"), true, false)
if err != nil {
ech <- err
@@ -129,7 +132,9 @@ func (idx *index) Start(ctx context.Context) (<-chan error, error) {
err = nil
}
it.Reset(idx.indexDuration)
- case <-itl.C:
+ case <-itl.C: // index duration limit ticker
+ // execute CreateIndex. This execution always executes CreateIndex regardless of the state of the uncommitted index,
+ // but does not immediately execute SaveIndex.
err = idx.execute(grpc.WithGRPCMethod(ctx, "core.v1.Agent/CreateIndex"), false, false)
if err != nil {
ech <- err
@@ -137,7 +142,9 @@ func (idx *index) Start(ctx context.Context) (<-chan error, error) {
err = nil
}
itl.Reset(idx.indexDurationLimit)
- case <-stl.C:
+ case <-stl.C: // save index duration limit ticker
+ // execute CreateIndex. This execution always executes CreateIndex regardless of the state of the uncommitted index,
+ // and immediately execute SaveIndex using CreateAndSaveIndex operation.
err = idx.execute(grpc.WithGRPCMethod(ctx, "core.v1.Agent/CreateAndSaveIndex"), false, true)
if err != nil {
ech <- err
@@ -162,12 +169,12 @@ func (idx *index) Start(ctx context.Context) (<-chan error, error) {
select {
case <-ctx.Done():
return
- case addr := <-idx.saveIndexTargetAddrCh:
- idx.schMap.Delete(addr)
+ case addr := <-idx.saveIndexTargetAddrCh: // this channel value send from execute func after thier CreateIndex operation when argument immediateSaving=false.
_, err := idx.client.GetClient().
Do(grpc.WithGRPCMethod(ctx, "core.v1.Agent/SaveIndex"), addr, func(ctx context.Context, conn *grpc.ClientConn, copts ...grpc.CallOption) (interface{}, error) {
return agent.NewAgentClient(conn).SaveIndex(ctx, &payload.Empty{}, copts...)
})
+ idx.schMap.Delete(addr) // unlock duplicate signal sending.
if err != nil {
log.Warnf("an error occurred while calling SaveIndex of %s: %s", addr, err)
select {
@@ -198,7 +205,7 @@ func (idx *index) execute(ctx context.Context, enableLowIndexSkip, immediateSavi
idx.indexing.Store(true)
defer idx.indexing.Store(false)
addrs := idx.client.GetAddrs(ctx)
- err = idx.client.GetClient().OrderedRangeConcurrent(ctx, addrs,
+ return errors.Join(idx.client.GetClient().OrderedRangeConcurrent(ctx, addrs,
idx.concurrency,
func(ctx context.Context,
addr string, conn *grpc.ClientConn, copts ...grpc.CallOption,
@@ -227,7 +234,7 @@ func (idx *index) execute(ctx context.Context, enableLowIndexSkip, immediateSavi
log.Warnf("an error occurred while calling CreateIndex of %s: %s", addr, err)
return err
}
- _, ok := idx.schMap.Load(addr)
+ _, ok := idx.schMap.Load(addr) // prevent duplicate save index signal.
if !ok {
select {
case <-ctx.Done():
@@ -249,11 +256,7 @@ func (idx *index) execute(ctx context.Context, enableLowIndexSkip, immediateSavi
}
idx.waitForNextSaving(ctx)
return nil
- })
- if err != nil {
- return err
- }
- return idx.loadInfos(ctx)
+ }), idx.loadInfos(ctx))
}
func (idx *index) waitForNextSaving(ctx context.Context) {
@@ -276,7 +279,7 @@ func (idx *index) loadInfos(ctx context.Context) (err error) {
}()
var u, ucu uint32
- var infoMap valdsync.Map[string, *payload.Info_Index_Count]
+ var infoMap sync.Map[string, *payload.Info_Index_Count]
err = idx.client.GetClient().RangeConcurrent(ctx, len(idx.client.GetAddrs(ctx)),
func(ctx context.Context,
addr string, conn *grpc.ClientConn, copts ...grpc.CallOption,
@@ -328,3 +331,12 @@ func (idx *index) NumberOfUUIDs() uint32 {
func (idx *index) NumberOfUncommittedUUIDs() uint32 {
return atomic.LoadUint32(&idx.uncommittedUUIDsCount)
}
+
+func (idx *index) LoadIndexDetail() (detail *payload.Info_Index_Detail) {
+ detail = new(payload.Info_Index_Detail)
+ idx.indexInfos.Range(func(addr string, info *payload.Info_Index_Count) bool {
+ detail.Counts[addr] = info
+ return true
+ })
+ return detail
+}
diff --git a/pkg/manager/index/service/indexer_test.go b/pkg/manager/index/service/indexer_test.go
index 226e926c072..ff644fbe21e 100644
--- a/pkg/manager/index/service/indexer_test.go
+++ b/pkg/manager/index/service/indexer_test.go
@@ -122,8 +122,9 @@ package service
// saveIndexDurationLimit time.Duration
// saveIndexWaitDuration time.Duration
// saveIndexTargetAddrCh chan string
+// schMap sync.Map[string, any]
// concurrency int
-// indexInfos indexInfos
+// indexInfos sync.Map[string, *payload.Info_Index_Count]
// indexing atomic.Value
// minUncommitted uint32
// uuidsCount uint32
@@ -168,8 +169,9 @@ package service
// saveIndexDurationLimit:nil,
// saveIndexWaitDuration:nil,
// saveIndexTargetAddrCh:nil,
+// schMap:nil,
// concurrency:0,
-// indexInfos:indexInfos{},
+// indexInfos:nil,
// indexing:nil,
// minUncommitted:0,
// uuidsCount:0,
@@ -203,8 +205,9 @@ package service
// saveIndexDurationLimit:nil,
// saveIndexWaitDuration:nil,
// saveIndexTargetAddrCh:nil,
+// schMap:nil,
// concurrency:0,
-// indexInfos:indexInfos{},
+// indexInfos:nil,
// indexing:nil,
// minUncommitted:0,
// uuidsCount:0,
@@ -247,6 +250,7 @@ package service
// saveIndexDurationLimit: test.fields.saveIndexDurationLimit,
// saveIndexWaitDuration: test.fields.saveIndexWaitDuration,
// saveIndexTargetAddrCh: test.fields.saveIndexTargetAddrCh,
+// schMap: test.fields.schMap,
// concurrency: test.fields.concurrency,
// indexInfos: test.fields.indexInfos,
// indexing: test.fields.indexing,
@@ -274,8 +278,9 @@ package service
// saveIndexDurationLimit time.Duration
// saveIndexWaitDuration time.Duration
// saveIndexTargetAddrCh chan string
+// schMap sync.Map[string, any]
// concurrency int
-// indexInfos indexInfos
+// indexInfos sync.Map[string, *payload.Info_Index_Count]
// indexing atomic.Value
// minUncommitted uint32
// uuidsCount uint32
@@ -312,8 +317,9 @@ package service
// saveIndexDurationLimit:nil,
// saveIndexWaitDuration:nil,
// saveIndexTargetAddrCh:nil,
+// schMap:nil,
// concurrency:0,
-// indexInfos:indexInfos{},
+// indexInfos:nil,
// indexing:nil,
// minUncommitted:0,
// uuidsCount:0,
@@ -344,8 +350,9 @@ package service
// saveIndexDurationLimit:nil,
// saveIndexWaitDuration:nil,
// saveIndexTargetAddrCh:nil,
+// schMap:nil,
// concurrency:0,
-// indexInfos:indexInfos{},
+// indexInfos:nil,
// indexing:nil,
// minUncommitted:0,
// uuidsCount:0,
@@ -388,6 +395,7 @@ package service
// saveIndexDurationLimit: test.fields.saveIndexDurationLimit,
// saveIndexWaitDuration: test.fields.saveIndexWaitDuration,
// saveIndexTargetAddrCh: test.fields.saveIndexTargetAddrCh,
+// schMap: test.fields.schMap,
// concurrency: test.fields.concurrency,
// indexInfos: test.fields.indexInfos,
// indexing: test.fields.indexing,
@@ -415,8 +423,9 @@ package service
// saveIndexDurationLimit time.Duration
// saveIndexWaitDuration time.Duration
// saveIndexTargetAddrCh chan string
+// schMap sync.Map[string, any]
// concurrency int
-// indexInfos indexInfos
+// indexInfos sync.Map[string, *payload.Info_Index_Count]
// indexing atomic.Value
// minUncommitted uint32
// uuidsCount uint32
@@ -453,8 +462,9 @@ package service
// saveIndexDurationLimit:nil,
// saveIndexWaitDuration:nil,
// saveIndexTargetAddrCh:nil,
+// schMap:nil,
// concurrency:0,
-// indexInfos:indexInfos{},
+// indexInfos:nil,
// indexing:nil,
// minUncommitted:0,
// uuidsCount:0,
@@ -485,8 +495,9 @@ package service
// saveIndexDurationLimit:nil,
// saveIndexWaitDuration:nil,
// saveIndexTargetAddrCh:nil,
+// schMap:nil,
// concurrency:0,
-// indexInfos:indexInfos{},
+// indexInfos:nil,
// indexing:nil,
// minUncommitted:0,
// uuidsCount:0,
@@ -529,6 +540,7 @@ package service
// saveIndexDurationLimit: test.fields.saveIndexDurationLimit,
// saveIndexWaitDuration: test.fields.saveIndexWaitDuration,
// saveIndexTargetAddrCh: test.fields.saveIndexTargetAddrCh,
+// schMap: test.fields.schMap,
// concurrency: test.fields.concurrency,
// indexInfos: test.fields.indexInfos,
// indexing: test.fields.indexing,
@@ -556,8 +568,9 @@ package service
// saveIndexDurationLimit time.Duration
// saveIndexWaitDuration time.Duration
// saveIndexTargetAddrCh chan string
+// schMap sync.Map[string, any]
// concurrency int
-// indexInfos indexInfos
+// indexInfos sync.Map[string, *payload.Info_Index_Count]
// indexing atomic.Value
// minUncommitted uint32
// uuidsCount uint32
@@ -594,8 +607,9 @@ package service
// saveIndexDurationLimit:nil,
// saveIndexWaitDuration:nil,
// saveIndexTargetAddrCh:nil,
+// schMap:nil,
// concurrency:0,
-// indexInfos:indexInfos{},
+// indexInfos:nil,
// indexing:nil,
// minUncommitted:0,
// uuidsCount:0,
@@ -626,8 +640,9 @@ package service
// saveIndexDurationLimit:nil,
// saveIndexWaitDuration:nil,
// saveIndexTargetAddrCh:nil,
+// schMap:nil,
// concurrency:0,
-// indexInfos:indexInfos{},
+// indexInfos:nil,
// indexing:nil,
// minUncommitted:0,
// uuidsCount:0,
@@ -670,6 +685,7 @@ package service
// saveIndexDurationLimit: test.fields.saveIndexDurationLimit,
// saveIndexWaitDuration: test.fields.saveIndexWaitDuration,
// saveIndexTargetAddrCh: test.fields.saveIndexTargetAddrCh,
+// schMap: test.fields.schMap,
// concurrency: test.fields.concurrency,
// indexInfos: test.fields.indexInfos,
// indexing: test.fields.indexing,
@@ -686,3 +702,148 @@ package service
// })
// }
// }
+//
+// func Test_index_LoadIndexDetail(t *testing.T) {
+// type fields struct {
+// client discoverer.Client
+// eg errgroup.Group
+// creationPoolSize uint32
+// indexDuration time.Duration
+// indexDurationLimit time.Duration
+// saveIndexDurationLimit time.Duration
+// saveIndexWaitDuration time.Duration
+// saveIndexTargetAddrCh chan string
+// schMap sync.Map[string, any]
+// concurrency int
+// indexInfos sync.Map[string, *payload.Info_Index_Count]
+// indexing atomic.Value
+// minUncommitted uint32
+// uuidsCount uint32
+// uncommittedUUIDsCount uint32
+// }
+// type want struct {
+// wantDetail *payload.Info_Index_Detail
+// }
+// type test struct {
+// name string
+// fields fields
+// want want
+// checkFunc func(want, *payload.Info_Index_Detail) error
+// beforeFunc func(*testing.T)
+// afterFunc func(*testing.T)
+// }
+// defaultCheckFunc := func(w want, gotDetail *payload.Info_Index_Detail) error {
+// if !reflect.DeepEqual(gotDetail, w.wantDetail) {
+// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", gotDetail, w.wantDetail)
+// }
+// return nil
+// }
+// tests := []test{
+// // TODO test cases
+// /*
+// {
+// name: "test_case_1",
+// fields: fields {
+// client:nil,
+// eg:nil,
+// creationPoolSize:0,
+// indexDuration:nil,
+// indexDurationLimit:nil,
+// saveIndexDurationLimit:nil,
+// saveIndexWaitDuration:nil,
+// saveIndexTargetAddrCh:nil,
+// schMap:nil,
+// concurrency:0,
+// indexInfos:nil,
+// indexing:nil,
+// minUncommitted:0,
+// uuidsCount:0,
+// uncommittedUUIDsCount:0,
+// },
+// want: want{},
+// checkFunc: defaultCheckFunc,
+// beforeFunc: func(t *testing.T,) {
+// t.Helper()
+// },
+// afterFunc: func(t *testing.T,) {
+// t.Helper()
+// },
+// },
+// */
+//
+// // TODO test cases
+// /*
+// func() test {
+// return test {
+// name: "test_case_2",
+// fields: fields {
+// client:nil,
+// eg:nil,
+// creationPoolSize:0,
+// indexDuration:nil,
+// indexDurationLimit:nil,
+// saveIndexDurationLimit:nil,
+// saveIndexWaitDuration:nil,
+// saveIndexTargetAddrCh:nil,
+// schMap:nil,
+// concurrency:0,
+// indexInfos:nil,
+// indexing:nil,
+// minUncommitted:0,
+// uuidsCount:0,
+// uncommittedUUIDsCount:0,
+// },
+// want: want{},
+// checkFunc: defaultCheckFunc,
+// beforeFunc: func(t *testing.T,) {
+// t.Helper()
+// },
+// afterFunc: func(t *testing.T,) {
+// t.Helper()
+// },
+// }
+// }(),
+// */
+// }
+//
+// for _, tc := range tests {
+// test := tc
+// t.Run(test.name, func(tt *testing.T) {
+// tt.Parallel()
+// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent())
+// if test.beforeFunc != nil {
+// test.beforeFunc(tt)
+// }
+// if test.afterFunc != nil {
+// defer test.afterFunc(tt)
+// }
+// checkFunc := test.checkFunc
+// if test.checkFunc == nil {
+// checkFunc = defaultCheckFunc
+// }
+// idx := &index{
+// client: test.fields.client,
+// eg: test.fields.eg,
+// creationPoolSize: test.fields.creationPoolSize,
+// indexDuration: test.fields.indexDuration,
+// indexDurationLimit: test.fields.indexDurationLimit,
+// saveIndexDurationLimit: test.fields.saveIndexDurationLimit,
+// saveIndexWaitDuration: test.fields.saveIndexWaitDuration,
+// saveIndexTargetAddrCh: test.fields.saveIndexTargetAddrCh,
+// schMap: test.fields.schMap,
+// concurrency: test.fields.concurrency,
+// indexInfos: test.fields.indexInfos,
+// indexing: test.fields.indexing,
+// minUncommitted: test.fields.minUncommitted,
+// uuidsCount: test.fields.uuidsCount,
+// uncommittedUUIDsCount: test.fields.uncommittedUUIDsCount,
+// }
+//
+// gotDetail := idx.LoadIndexDetail()
+// if err := checkFunc(test.want, gotDetail); err != nil {
+// tt.Errorf("error = %v", err)
+// }
+//
+// })
+// }
+// }
diff --git a/pkg/manager/index/service/option.go b/pkg/manager/index/service/option.go
index a675c656efa..d10f86dbfed 100644
--- a/pkg/manager/index/service/option.go
+++ b/pkg/manager/index/service/option.go
@@ -19,7 +19,7 @@ package service
import (
"github.com/vdaas/vald/internal/client/v1/client/discoverer"
- "github.com/vdaas/vald/internal/errgroup"
+ "github.com/vdaas/vald/internal/sync/errgroup"
"github.com/vdaas/vald/internal/timeutil"
)
diff --git a/pkg/manager/index/usecase/indexer.go b/pkg/manager/index/usecase/indexer.go
index 11aa90054d4..9619bddec19 100644
--- a/pkg/manager/index/usecase/indexer.go
+++ b/pkg/manager/index/usecase/indexer.go
@@ -22,7 +22,6 @@ import (
"github.com/vdaas/vald/apis/grpc/v1/manager/index"
"github.com/vdaas/vald/internal/client/v1/client/discoverer"
iconf "github.com/vdaas/vald/internal/config"
- "github.com/vdaas/vald/internal/errgroup"
"github.com/vdaas/vald/internal/net/grpc"
"github.com/vdaas/vald/internal/observability"
backoffmetrics "github.com/vdaas/vald/internal/observability/metrics/backoff"
@@ -32,6 +31,7 @@ import (
"github.com/vdaas/vald/internal/safety"
"github.com/vdaas/vald/internal/servers/server"
"github.com/vdaas/vald/internal/servers/starter"
+ "github.com/vdaas/vald/internal/sync/errgroup"
"github.com/vdaas/vald/pkg/manager/index/config"
handler "github.com/vdaas/vald/pkg/manager/index/handler/grpc"
"github.com/vdaas/vald/pkg/manager/index/handler/rest"
diff --git a/pkg/tools/cli/loadtest/service/loader.go b/pkg/tools/cli/loadtest/service/loader.go
index 216aa72a47a..be1adf71974 100644
--- a/pkg/tools/cli/loadtest/service/loader.go
+++ b/pkg/tools/cli/loadtest/service/loader.go
@@ -22,11 +22,11 @@ import (
"time"
"github.com/vdaas/vald/apis/grpc/v1/payload"
- "github.com/vdaas/vald/internal/errgroup"
"github.com/vdaas/vald/internal/errors"
"github.com/vdaas/vald/internal/log"
"github.com/vdaas/vald/internal/net/grpc"
"github.com/vdaas/vald/internal/safety"
+ "github.com/vdaas/vald/internal/sync/errgroup"
"github.com/vdaas/vald/pkg/tools/cli/loadtest/assets"
"github.com/vdaas/vald/pkg/tools/cli/loadtest/config"
)
@@ -217,7 +217,7 @@ func (l *loader) do(ctx context.Context, f func(interface{}, error), notify func
}))
err = eg.Wait()
case config.Insert, config.Search:
- eg.Limitation(l.concurrency)
+ eg.SetLimit(l.concurrency)
for {
r := l.dataProvider()
diff --git a/pkg/tools/cli/loadtest/service/loader_option.go b/pkg/tools/cli/loadtest/service/loader_option.go
index e96fc944e41..0ac342d2ad3 100644
--- a/pkg/tools/cli/loadtest/service/loader_option.go
+++ b/pkg/tools/cli/loadtest/service/loader_option.go
@@ -14,9 +14,9 @@
package service
import (
- "github.com/vdaas/vald/internal/errgroup"
"github.com/vdaas/vald/internal/errors"
"github.com/vdaas/vald/internal/net/grpc"
+ "github.com/vdaas/vald/internal/sync/errgroup"
"github.com/vdaas/vald/internal/timeutil"
"github.com/vdaas/vald/pkg/tools/cli/loadtest/config"
)
diff --git a/pkg/tools/cli/loadtest/usecase/load.go b/pkg/tools/cli/loadtest/usecase/load.go
index dc67c02c0d3..501a45f0bb6 100644
--- a/pkg/tools/cli/loadtest/usecase/load.go
+++ b/pkg/tools/cli/loadtest/usecase/load.go
@@ -19,12 +19,12 @@ package usecase
import (
"context"
- "github.com/vdaas/vald/internal/errgroup"
"github.com/vdaas/vald/internal/errors"
"github.com/vdaas/vald/internal/log"
"github.com/vdaas/vald/internal/net/grpc"
"github.com/vdaas/vald/internal/runner"
"github.com/vdaas/vald/internal/safety"
+ "github.com/vdaas/vald/internal/sync/errgroup"
"github.com/vdaas/vald/pkg/tools/cli/loadtest/config"
"github.com/vdaas/vald/pkg/tools/cli/loadtest/service"
)
diff --git a/tests/e2e/operation/stream.go b/tests/e2e/operation/stream.go
index 221b41aa62d..143399191a7 100644
--- a/tests/e2e/operation/stream.go
+++ b/tests/e2e/operation/stream.go
@@ -19,7 +19,6 @@ import (
"context"
"reflect"
"strconv"
- "sync"
"testing"
"github.com/vdaas/vald/apis/grpc/v1/payload"
@@ -29,6 +28,7 @@ import (
"github.com/vdaas/vald/internal/net/grpc/errdetails"
"github.com/vdaas/vald/internal/net/grpc/status"
"github.com/vdaas/vald/internal/strings"
+ "github.com/vdaas/vald/internal/sync"
)
type (
diff --git a/tests/e2e/pkg/agent/core/ngt/service/ngt_e2s_test.go b/tests/e2e/pkg/agent/core/ngt/service/ngt_e2s_test.go
index e4c5e7d8324..d16f47bcdad 100644
--- a/tests/e2e/pkg/agent/core/ngt/service/ngt_e2s_test.go
+++ b/tests/e2e/pkg/agent/core/ngt/service/ngt_e2s_test.go
@@ -20,12 +20,12 @@ package service
import (
"context"
"strconv"
- "sync"
"testing"
"time"
"github.com/vdaas/vald/internal/config"
"github.com/vdaas/vald/internal/errors"
+ "github.com/vdaas/vald/internal/sync"
"github.com/vdaas/vald/pkg/agent/core/ngt/service"
)
diff --git a/tests/performance/max_vector_dim_test.go b/tests/performance/max_vector_dim_test.go
index 03c6e72a587..93bd3bb9c63 100644
--- a/tests/performance/max_vector_dim_test.go
+++ b/tests/performance/max_vector_dim_test.go
@@ -18,7 +18,6 @@ import (
"context"
"os"
"strconv"
- "sync"
"testing"
"time"
@@ -26,10 +25,11 @@ import (
"github.com/vdaas/vald/internal/config"
"github.com/vdaas/vald/internal/core/algorithm"
"github.com/vdaas/vald/internal/core/algorithm/ngt"
- "github.com/vdaas/vald/internal/errgroup"
"github.com/vdaas/vald/internal/errors"
"github.com/vdaas/vald/internal/safety"
"github.com/vdaas/vald/internal/strings"
+ "github.com/vdaas/vald/internal/sync"
+ "github.com/vdaas/vald/internal/sync/errgroup"
"github.com/vdaas/vald/internal/test/data/vector"
"github.com/vdaas/vald/pkg/agent/core/ngt/handler/grpc"
"github.com/vdaas/vald/pkg/agent/core/ngt/service"
diff --git a/versions/HELM_VERSION b/versions/HELM_VERSION
index 10d3e4ccb11..2ffba3ce657 100644
--- a/versions/HELM_VERSION
+++ b/versions/HELM_VERSION
@@ -1 +1 @@
-v3.12.1
+v3.12.2
diff --git a/versions/JAEGER_OPERATOR_VERSION b/versions/JAEGER_OPERATOR_VERSION
index b8c7fd6aca7..64f878193d4 100644
--- a/versions/JAEGER_OPERATOR_VERSION
+++ b/versions/JAEGER_OPERATOR_VERSION
@@ -1 +1 @@
-2.46.1
+2.46.2
diff --git a/versions/KUBECTL_VERSION b/versions/KUBECTL_VERSION
index 6c6ba5f810b..6f6fa3c2702 100644
--- a/versions/KUBECTL_VERSION
+++ b/versions/KUBECTL_VERSION
@@ -1 +1 @@
-v1.27.3
\ No newline at end of file
+v1.27.4
\ No newline at end of file
diff --git a/versions/NGT_VERSION b/versions/NGT_VERSION
index a14da2902b1..3e3c2f1e5ed 100644
--- a/versions/NGT_VERSION
+++ b/versions/NGT_VERSION
@@ -1 +1 @@
-2.0.16
+2.1.1
diff --git a/versions/OPERATOR_SDK_VERSION b/versions/OPERATOR_SDK_VERSION
index 17e22489a72..f3f644dcf8d 100644
--- a/versions/OPERATOR_SDK_VERSION
+++ b/versions/OPERATOR_SDK_VERSION
@@ -1 +1 @@
-v1.30
+v1.31
diff --git a/versions/PROMETHEUS_STACK_VERSION b/versions/PROMETHEUS_STACK_VERSION
index b48484e16a4..0f0e20288ad 100644
--- a/versions/PROMETHEUS_STACK_VERSION
+++ b/versions/PROMETHEUS_STACK_VERSION
@@ -1 +1 @@
-47.6.1
+48.2.2
diff --git a/versions/TELEPRESENCE_VERSION b/versions/TELEPRESENCE_VERSION
index 8b5925249b1..7243b12cf41 100644
--- a/versions/TELEPRESENCE_VERSION
+++ b/versions/TELEPRESENCE_VERSION
@@ -1 +1 @@
-2.14.1-rc.3
+2.14.2
diff --git a/versions/YQ_VERSION b/versions/YQ_VERSION
index d8b501ab260..7606f2caad4 100644
--- a/versions/YQ_VERSION
+++ b/versions/YQ_VERSION
@@ -1 +1 @@
-v4.34.1
+v4.34.2