From f32949eee4bf250e0f2d2e6f92ef608ea8002ce3 Mon Sep 17 00:00:00 2001 From: vdaas-ci <57942646+vdaas-ci@users.noreply.github.com> Date: Fri, 11 Oct 2024 14:56:16 +0900 Subject: [PATCH] Add String sorted topologicalSort (#2696) (#2698) Co-authored-by: Yusuke Kato --- dockers/agent/core/agent/Dockerfile | 2 +- dockers/agent/core/faiss/Dockerfile | 2 +- dockers/agent/core/ngt/Dockerfile | 2 +- dockers/agent/sidecar/Dockerfile | 2 +- dockers/binfmt/Dockerfile | 2 +- dockers/buildbase/Dockerfile | 2 +- dockers/buildkit/Dockerfile | 2 +- dockers/buildkit/syft/scanner/Dockerfile | 2 +- dockers/ci/base/Dockerfile | 2 +- dockers/dev/Dockerfile | 2 +- dockers/discoverer/k8s/Dockerfile | 2 +- dockers/gateway/filter/Dockerfile | 2 +- dockers/gateway/lb/Dockerfile | 2 +- dockers/gateway/mirror/Dockerfile | 2 +- dockers/index/job/correction/Dockerfile | 2 +- dockers/index/job/creation/Dockerfile | 2 +- .../index/job/readreplica/rotate/Dockerfile | 2 +- dockers/index/job/save/Dockerfile | 2 +- dockers/index/operator/Dockerfile | 2 +- dockers/manager/index/Dockerfile | 2 +- dockers/operator/helm/Dockerfile | 2 +- dockers/tools/benchmark/job/Dockerfile | 2 +- dockers/tools/benchmark/operator/Dockerfile | 2 +- dockers/tools/cli/loadtest/Dockerfile | 2 +- hack/docker/gen/main.go | 60 ++++++++++++++----- 25 files changed, 69 insertions(+), 39 deletions(-) diff --git a/dockers/agent/core/agent/Dockerfile b/dockers/agent/core/agent/Dockerfile index 1a87ca7bec..5917842725 100644 --- a/dockers/agent/core/agent/Dockerfile +++ b/dockers/agent/core/agent/Dockerfile @@ -93,4 +93,4 @@ LABEL maintainer="vdaas.org vald team " COPY --from=builder /usr/bin/agent /usr/bin/agent # skipcq: DOK-DL3002 USER nonroot:nonroot -ENTRYPOINT ["/usr/bin/agent"] +ENTRYPOINT ["/usr/bin/agent"] \ No newline at end of file diff --git a/dockers/agent/core/faiss/Dockerfile b/dockers/agent/core/faiss/Dockerfile index f466a02fec..bfd27599f6 100644 --- a/dockers/agent/core/faiss/Dockerfile +++ b/dockers/agent/core/faiss/Dockerfile @@ -95,4 +95,4 @@ COPY --from=builder /usr/bin/faiss /usr/bin/faiss COPY cmd/agent/core/faiss/sample.yaml /etc/server/config.yaml # skipcq: DOK-DL3002 USER nonroot:nonroot -ENTRYPOINT ["/usr/bin/faiss"] +ENTRYPOINT ["/usr/bin/faiss"] \ No newline at end of file diff --git a/dockers/agent/core/ngt/Dockerfile b/dockers/agent/core/ngt/Dockerfile index 076dc3b695..51e564c224 100644 --- a/dockers/agent/core/ngt/Dockerfile +++ b/dockers/agent/core/ngt/Dockerfile @@ -94,4 +94,4 @@ COPY --from=builder /usr/bin/ngt /usr/bin/ngt COPY cmd/agent/core/ngt/sample.yaml /etc/server/config.yaml # skipcq: DOK-DL3002 USER nonroot:nonroot -ENTRYPOINT ["/usr/bin/ngt"] +ENTRYPOINT ["/usr/bin/ngt"] \ No newline at end of file diff --git a/dockers/agent/sidecar/Dockerfile b/dockers/agent/sidecar/Dockerfile index b4a1d5ed90..5c19e85c8e 100644 --- a/dockers/agent/sidecar/Dockerfile +++ b/dockers/agent/sidecar/Dockerfile @@ -84,4 +84,4 @@ LABEL maintainer="vdaas.org vald team " COPY --from=builder /usr/bin/sidecar /usr/bin/sidecar # skipcq: DOK-DL3002 USER nonroot:nonroot -ENTRYPOINT ["/usr/bin/sidecar"] +ENTRYPOINT ["/usr/bin/sidecar"] \ No newline at end of file diff --git a/dockers/binfmt/Dockerfile b/dockers/binfmt/Dockerfile index f2d73909bd..47284d17e2 100644 --- a/dockers/binfmt/Dockerfile +++ b/dockers/binfmt/Dockerfile @@ -16,4 +16,4 @@ # # DO_NOT_EDIT this Dockerfile is generated by https://github.com/vdaas/vald/blob/main/hack/docker/gen/main.go -FROM tonistiigi/binfmt:master AS builder +FROM tonistiigi/binfmt:master AS builder \ No newline at end of file diff --git a/dockers/buildbase/Dockerfile b/dockers/buildbase/Dockerfile index 301a31e010..6457b01457 100644 --- a/dockers/buildbase/Dockerfile +++ b/dockers/buildbase/Dockerfile @@ -16,4 +16,4 @@ # # DO_NOT_EDIT this Dockerfile is generated by https://github.com/vdaas/vald/blob/main/hack/docker/gen/main.go -FROM ubuntu:devel AS builder +FROM ubuntu:devel AS builder \ No newline at end of file diff --git a/dockers/buildkit/Dockerfile b/dockers/buildkit/Dockerfile index 99c71c61e2..9dd722ea25 100644 --- a/dockers/buildkit/Dockerfile +++ b/dockers/buildkit/Dockerfile @@ -16,4 +16,4 @@ # # DO_NOT_EDIT this Dockerfile is generated by https://github.com/vdaas/vald/blob/main/hack/docker/gen/main.go -FROM moby/buildkit:master AS builder +FROM moby/buildkit:master AS builder \ No newline at end of file diff --git a/dockers/buildkit/syft/scanner/Dockerfile b/dockers/buildkit/syft/scanner/Dockerfile index 4a23207c47..df32fe5705 100644 --- a/dockers/buildkit/syft/scanner/Dockerfile +++ b/dockers/buildkit/syft/scanner/Dockerfile @@ -16,4 +16,4 @@ # # DO_NOT_EDIT this Dockerfile is generated by https://github.com/vdaas/vald/blob/main/hack/docker/gen/main.go -FROM docker/buildkit-syft-scanner:edge AS scanner +FROM docker/buildkit-syft-scanner:edge AS scanner \ No newline at end of file diff --git a/dockers/ci/base/Dockerfile b/dockers/ci/base/Dockerfile index 6c73efea03..e39363d3d1 100644 --- a/dockers/ci/base/Dockerfile +++ b/dockers/ci/base/Dockerfile @@ -127,4 +127,4 @@ RUN --mount=type=bind,target=.,rw \ && rm -rf ${GOPATH}/src/github.com/${ORG}/${REPO}/* # skipcq: DOK-DL3002 USER root:root -ENTRYPOINT ["/bin/bash"] +ENTRYPOINT ["/bin/bash"] \ No newline at end of file diff --git a/dockers/dev/Dockerfile b/dockers/dev/Dockerfile index 0e0996b5e7..3e80596341 100644 --- a/dockers/dev/Dockerfile +++ b/dockers/dev/Dockerfile @@ -145,4 +145,4 @@ RUN --mount=type=bind,target=.,rw \ && make faiss/install \ && rm -rf ${GOPATH}/src/github.com/${ORG}/${REPO}/* # skipcq: DOK-DL3002 -USER root:root +USER root:root \ No newline at end of file diff --git a/dockers/discoverer/k8s/Dockerfile b/dockers/discoverer/k8s/Dockerfile index a9bf1e5781..84302fe9f6 100644 --- a/dockers/discoverer/k8s/Dockerfile +++ b/dockers/discoverer/k8s/Dockerfile @@ -85,4 +85,4 @@ COPY --from=builder /usr/bin/discoverer /usr/bin/discoverer COPY cmd/discoverer/k8s/sample.yaml /etc/server/config.yaml # skipcq: DOK-DL3002 USER nonroot:nonroot -ENTRYPOINT ["/usr/bin/discoverer"] +ENTRYPOINT ["/usr/bin/discoverer"] \ No newline at end of file diff --git a/dockers/gateway/filter/Dockerfile b/dockers/gateway/filter/Dockerfile index 4ddc9858e9..c4b3c751aa 100644 --- a/dockers/gateway/filter/Dockerfile +++ b/dockers/gateway/filter/Dockerfile @@ -85,4 +85,4 @@ COPY --from=builder /usr/bin/filter /usr/bin/filter COPY cmd/gateway/filter/sample.yaml /etc/server/config.yaml # skipcq: DOK-DL3002 USER nonroot:nonroot -ENTRYPOINT ["/usr/bin/filter"] +ENTRYPOINT ["/usr/bin/filter"] \ No newline at end of file diff --git a/dockers/gateway/lb/Dockerfile b/dockers/gateway/lb/Dockerfile index c3bd773deb..8d666ebfa4 100644 --- a/dockers/gateway/lb/Dockerfile +++ b/dockers/gateway/lb/Dockerfile @@ -85,4 +85,4 @@ COPY --from=builder /usr/bin/lb /usr/bin/lb COPY cmd/gateway/lb/sample.yaml /etc/server/config.yaml # skipcq: DOK-DL3002 USER nonroot:nonroot -ENTRYPOINT ["/usr/bin/lb"] +ENTRYPOINT ["/usr/bin/lb"] \ No newline at end of file diff --git a/dockers/gateway/mirror/Dockerfile b/dockers/gateway/mirror/Dockerfile index 693d891601..eb136bd4ac 100644 --- a/dockers/gateway/mirror/Dockerfile +++ b/dockers/gateway/mirror/Dockerfile @@ -85,4 +85,4 @@ COPY --from=builder /usr/bin/mirror /usr/bin/mirror COPY cmd/gateway/mirror/sample.yaml /etc/server/config.yaml # skipcq: DOK-DL3002 USER nonroot:nonroot -ENTRYPOINT ["/usr/bin/mirror"] +ENTRYPOINT ["/usr/bin/mirror"] \ No newline at end of file diff --git a/dockers/index/job/correction/Dockerfile b/dockers/index/job/correction/Dockerfile index 1dbefb3c48..96152e486e 100644 --- a/dockers/index/job/correction/Dockerfile +++ b/dockers/index/job/correction/Dockerfile @@ -85,4 +85,4 @@ COPY --from=builder /usr/bin/index-correction /usr/bin/index-correction COPY cmd/index/job/correction/sample.yaml /etc/server/config.yaml # skipcq: DOK-DL3002 USER nonroot:nonroot -ENTRYPOINT ["/usr/bin/index-correction"] +ENTRYPOINT ["/usr/bin/index-correction"] \ No newline at end of file diff --git a/dockers/index/job/creation/Dockerfile b/dockers/index/job/creation/Dockerfile index 13576d6dc6..3d89245d18 100644 --- a/dockers/index/job/creation/Dockerfile +++ b/dockers/index/job/creation/Dockerfile @@ -85,4 +85,4 @@ COPY --from=builder /usr/bin/index-creation /usr/bin/index-creation COPY cmd/index/job/creation/sample.yaml /etc/server/config.yaml # skipcq: DOK-DL3002 USER nonroot:nonroot -ENTRYPOINT ["/usr/bin/index-creation"] +ENTRYPOINT ["/usr/bin/index-creation"] \ No newline at end of file diff --git a/dockers/index/job/readreplica/rotate/Dockerfile b/dockers/index/job/readreplica/rotate/Dockerfile index d443e0fcc0..2c5f7cfba9 100644 --- a/dockers/index/job/readreplica/rotate/Dockerfile +++ b/dockers/index/job/readreplica/rotate/Dockerfile @@ -85,4 +85,4 @@ COPY --from=builder /usr/bin/readreplica-rotate /usr/bin/readreplica-rotate COPY cmd/index/job/readreplica/rotate/sample.yaml /etc/server/config.yaml # skipcq: DOK-DL3002 USER nonroot:nonroot -ENTRYPOINT ["/usr/bin/readreplica-rotate"] +ENTRYPOINT ["/usr/bin/readreplica-rotate"] \ No newline at end of file diff --git a/dockers/index/job/save/Dockerfile b/dockers/index/job/save/Dockerfile index 9ef2ef629c..5c4b9ccc45 100644 --- a/dockers/index/job/save/Dockerfile +++ b/dockers/index/job/save/Dockerfile @@ -85,4 +85,4 @@ COPY --from=builder /usr/bin/index-save /usr/bin/index-save COPY cmd/index/job/save/sample.yaml /etc/server/config.yaml # skipcq: DOK-DL3002 USER nonroot:nonroot -ENTRYPOINT ["/usr/bin/index-save"] +ENTRYPOINT ["/usr/bin/index-save"] \ No newline at end of file diff --git a/dockers/index/operator/Dockerfile b/dockers/index/operator/Dockerfile index d1ff09eec6..fa713abbd1 100644 --- a/dockers/index/operator/Dockerfile +++ b/dockers/index/operator/Dockerfile @@ -85,4 +85,4 @@ COPY --from=builder /usr/bin/index-operator /usr/bin/index-operator COPY cmd/index/operator/sample.yaml /etc/server/config.yaml # skipcq: DOK-DL3002 USER nonroot:nonroot -ENTRYPOINT ["/usr/bin/index-operator"] +ENTRYPOINT ["/usr/bin/index-operator"] \ No newline at end of file diff --git a/dockers/manager/index/Dockerfile b/dockers/manager/index/Dockerfile index 2fadc08ebb..5fe2b80e92 100644 --- a/dockers/manager/index/Dockerfile +++ b/dockers/manager/index/Dockerfile @@ -85,4 +85,4 @@ COPY --from=builder /usr/bin/index /usr/bin/index COPY cmd/manager/index/sample.yaml /etc/server/config.yaml # skipcq: DOK-DL3002 USER nonroot:nonroot -ENTRYPOINT ["/usr/bin/index"] +ENTRYPOINT ["/usr/bin/index"] \ No newline at end of file diff --git a/dockers/operator/helm/Dockerfile b/dockers/operator/helm/Dockerfile index 049914cba3..5e06bc5f31 100644 --- a/dockers/operator/helm/Dockerfile +++ b/dockers/operator/helm/Dockerfile @@ -106,4 +106,4 @@ COPY --from=builder /opt/helm/charts/vald /opt/helm/charts/vald COPY --from=builder /opt/helm/charts/vald-helm-operator /opt/helm/charts/vald-helm-operator # skipcq: DOK-DL3002 USER nonroot:nonroot -ENTRYPOINT ["/usr/bin/helm-operator", "run", "--watches-file=/opt/helm/watches.yaml"] +ENTRYPOINT ["/usr/bin/helm-operator", "run", "--watches-file=/opt/helm/watches.yaml"] \ No newline at end of file diff --git a/dockers/tools/benchmark/job/Dockerfile b/dockers/tools/benchmark/job/Dockerfile index c4374b6d7b..92440346c7 100644 --- a/dockers/tools/benchmark/job/Dockerfile +++ b/dockers/tools/benchmark/job/Dockerfile @@ -93,4 +93,4 @@ COPY --from=builder /usr/bin/job /usr/bin/job COPY cmd/tools/benchmark/job/sample.yaml /etc/server/config.yaml # skipcq: DOK-DL3002 USER nonroot:nonroot -ENTRYPOINT ["/usr/bin/job"] +ENTRYPOINT ["/usr/bin/job"] \ No newline at end of file diff --git a/dockers/tools/benchmark/operator/Dockerfile b/dockers/tools/benchmark/operator/Dockerfile index f99fa41491..580eca69f1 100644 --- a/dockers/tools/benchmark/operator/Dockerfile +++ b/dockers/tools/benchmark/operator/Dockerfile @@ -85,4 +85,4 @@ COPY --from=builder /usr/bin/operator /usr/bin/operator COPY cmd/tools/benchmark/operator/sample.yaml /etc/server/config.yaml # skipcq: DOK-DL3002 USER nonroot:nonroot -ENTRYPOINT ["/usr/bin/operator"] +ENTRYPOINT ["/usr/bin/operator"] \ No newline at end of file diff --git a/dockers/tools/cli/loadtest/Dockerfile b/dockers/tools/cli/loadtest/Dockerfile index c75c5a339d..282806ad4c 100644 --- a/dockers/tools/cli/loadtest/Dockerfile +++ b/dockers/tools/cli/loadtest/Dockerfile @@ -93,4 +93,4 @@ COPY --from=builder /usr/bin/loadtest /usr/bin/loadtest COPY cmd/tools/cli/loadtest/sample.yaml /etc/server/config.yaml # skipcq: DOK-DL3002 USER nonroot:nonroot -ENTRYPOINT ["/usr/bin/loadtest"] +ENTRYPOINT ["/usr/bin/loadtest"] \ No newline at end of file diff --git a/hack/docker/gen/main.go b/hack/docker/gen/main.go index df6d58ec62..0e0bb98d57 100644 --- a/hack/docker/gen/main.go +++ b/hack/docker/gen/main.go @@ -23,7 +23,6 @@ import ( "io/fs" "os" "os/signal" - "regexp" "slices" "syscall" "text/template" @@ -437,33 +436,51 @@ func appendM[K comparable](maps ...map[K]string) map[K]string { return result } -var re = regexp.MustCompile(`\$\{?(\w+)\}?`) - +// extractVariables efficiently extracts variables from strings func extractVariables(value string) []string { - matches := re.FindAllStringSubmatch(value, -1) - vars := make([]string, 0, len(matches)) - for _, match := range matches { - vars = append(vars, match[1]) + var vars []string + start := -1 + for i := 0; i < len(value); i++ { + if value[i] == '$' && i+1 < len(value) && value[i+1] == '{' { + start = i + 2 + } else if start != -1 && value[i] == '}' { + vars = append(vars, value[start:i]) + start = -1 + } else if value[i] == '$' && start == -1 { + start = i + 1 + for start < len(value) && (('a' <= value[start] && value[start] <= 'z') || ('A' <= value[start] && value[start] <= 'Z') || ('0' <= value[start] && value[start] <= '9') || value[start] == '_') { + start++ + } + vars = append(vars, value[i+1:start]) + i = start - 1 + start = -1 + } } return vars } +// topologicalSort sorts the elements topologically and ensures that equal-level nodes are sorted by name func topologicalSort(envMap map[string]string) []string { - // Graph structures - inDegree := make(map[string]int) - graph := make(map[string][]string) + inDegree := make(map[string]int) // Tracks the in-degree of each node + graph := make(map[string][]string) // Tracks the edges between nodes + result := make([]string, 0, len(envMap)) // Result slice pre-allocated for efficiency - // Initialize the graph + gl := 0 + // Initialize the graph structure and in-degrees for key, value := range envMap { vars := extractVariables(value) for _, refKey := range vars { - if refKey != key { + if refKey != key { // Prevent self-dependency graph[refKey] = append(graph[refKey], key) + if len(graph[refKey]) > gl { + gl = len(graph[refKey]) + } inDegree[key]++ } } } + // Initialize the queue with nodes having in-degree 0 (no dependencies) queue := make([]string, 0, len(envMap)-len(graph)) for key := range envMap { if inDegree[key] == 0 { @@ -471,22 +488,35 @@ func topologicalSort(envMap map[string]string) []string { } } + // Sort the initial queue to maintain lexicographical order for nodes with no dependencies slices.Sort(queue) - // Topological sort - result := make([]string, 0, len(envMap)) + // Preallocate a reusable slice for collecting new nodes + newNodes := make([]string, 0, gl) + // Topological sort process for len(queue) > 0 { node := queue[0] queue = queue[1:] + + // Append the result as `node=value` if value, exists := envMap[node]; exists { result = append(result, node+"="+value) } + + // Process all neighbors and decrement their in-degrees for _, neighbor := range graph[node] { inDegree[neighbor]-- if inDegree[neighbor] == 0 { - queue = append(queue, neighbor) + newNodes = append(newNodes, neighbor) } } + + // If new nodes were found, sort them and append to the queue + if len(newNodes) > 0 { + slices.Sort(newNodes) // Sort new nodes only once + queue = append(queue, newNodes...) + newNodes = newNodes[:0] // Reuse the slice by resetting it + } } return result