Implement continuous benchmark tool

.github/workflows/dockers-benchmark-job-image.yml

@@ -0,0 +1,82 @@
+#
+# Copyright (C) 2019-2023 vdaas.org vald team <[email protected]>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# You may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+name: "Build docker image: benchmark-job"
+on:
+  push:
+    branches:
+      - main
+    tags:
+      - "*.*.*"
+      - "v*.*.*"
+      - "*.*.*-*"
+      - "v*.*.*-*"
+    paths:
+      - ".github/actions/docker-build/actions.yaml"
+      - ".github/workflows/_docker-image.yaml"
+      - ".github/workflows/dockers-benchmak-job-image.yml"
+      - "go.mod"
+      - "go.sum"
+      - "internal/**"
+      - "!internal/**/*_test.go"
+      - "!internal/db/**"
+      - "apis/grpc/**"
+      - "pkg/benchmark/operator/**"
+      - "cmd/benchmark/operator/**"
+      - "pkg/benchmark/job/**"
+      - "cmd/benchmark/job/**"
+      - "dockers/tools/benchmark/job/Dockerfile"
+      - "versions/GO_VERSION"
+  pull_request:
+    paths:
+      - ".github/actions/docker-build/actions.yaml"
+      - ".github/workflows/_docker-image.yaml"
+      - ".github/workflows/dockers-benchmak-job-image.yml"
+      - "go.mod"
+      - "go.sum"
+      - "internal/**"
+      - "!internal/**/*_test.go"
+      - "!internal/db/**"
+      - "apis/grpc/**"
+      - "pkg/benchmark/operator/**"
+      - "cmd/benchmark/operator/**"
+      - "pkg/benchmark/job/**"
+      - "cmd/benchmark/job/**"
+      - "dockers/tools/benchmark/job/Dockerfile"
+      - "versions/GO_VERSION"
+  pull_request_target:
+    paths:
+      - ".github/actions/docker-build/actions.yaml"
+      - ".github/workflows/_docker-image.yaml"
+      - ".github/workflows/dockers-benchmak-job-image.yml"
+      - "go.mod"
+      - "go.sum"
+      - "internal/**"
+      - "!internal/**/*_test.go"
+      - "!internal/db/**"
+      - "apis/grpc/**"
+      - "pkg/benchmark/operator/**"
+      - "cmd/benchmark/operator/**"
+      - "pkg/benchmark/job/**"
+      - "cmd/benchmark/job/**"
+      - "dockers/tools/benchmark/job/Dockerfile"
+      - "versions/GO_VERSION"
+
+jobs:
+  build:
+    uses: ./.github/workflows/_docker-image.yaml
+    with:
+      target: benchmark-job
+    secrets: inherit

.github/workflows/dockers-benchmark-operator-image.yaml

@@ -0,0 +1,82 @@
+#
+# Copyright (C) 2019-2023 vdaas.org vald team <[email protected]>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# You may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+name: "Build docker image: benchmark-operator"
+on:
+  push:
+    branches:
+      - main
+    tags:
+      - "*.*.*"
+      - "v*.*.*"
+      - "*.*.*-*"
+      - "v*.*.*-*"
+    paths:
+      - ".github/actions/docker-build/actions.yaml"
+      - ".github/workflows/_docker-image.yaml"
+      - ".github/workflows/dockers-benchmak-operator-image.yml"
+      - "go.mod"
+      - "go.sum"
+      - "internal/**"
+      - "!internal/**/*_test.go"
+      - "!internal/db/**"
+      - "apis/grpc/**"
+      - "pkg/benchmark/operator/**"
+      - "cmd/benchmark/operator/**"
+      - "pkg/benchmark/job/**"
+      - "cmd/benchmark/job/**"
+      - "dockers/tools/benchmark/operator/Dockerfile"
+      - "versions/GO_VERSION"
+  pull_request:
+    paths:
+      - ".github/actions/docker-build/actions.yaml"
+      - ".github/workflows/_docker-image.yaml"
+      - ".github/workflows/dockers-benchmak-operator-image.yml"
+      - "go.mod"
+      - "go.sum"
+      - "internal/**"
+      - "!internal/**/*_test.go"
+      - "!internal/db/**"
+      - "apis/grpc/**"
+      - "pkg/benchmark/operator/**"
+      - "cmd/benchmark/operator/**"
+      - "pkg/benchmark/job/**"
+      - "cmd/benchmark/job/**"
+      - "dockers/tools/benchmark/operator/Dockerfile"
+      - "versions/GO_VERSION"
+  pull_request_target:
+    paths:
+      - ".github/actions/docker-build/actions.yaml"
+      - ".github/workflows/_docker-image.yaml"
+      - ".github/workflows/dockers-benchmak-operator-image.yml"
+      - "go.mod"
+      - "go.sum"
+      - "internal/**"
+      - "!internal/**/*_test.go"
+      - "!internal/db/**"
+      - "apis/grpc/**"
+      - "pkg/benchmark/operator/**"
+      - "cmd/benchmark/operator/**"
+      - "pkg/benchmark/job/**"
+      - "cmd/benchmark/job/**"
+      - "dockers/tools/benchmark/operator/Dockerfile"
+      - "versions/GO_VERSION"
+
+jobs:
+  build:
+    uses: ./.github/workflows/_docker-image.yaml
+    with:
+      target: benchmark-operator
+    secrets: inherit

Makefile

@@ -32,6 +32,8 @@ LB_GATEWAY_IMAGE                = $(NAME)-lb-gateway
 LOADTEST_IMAGE                  = $(NAME)-loadtest
 INDEX_CORRECTION_IMAGE      	= $(NAME)-index-correction
 MANAGER_INDEX_IMAGE             = $(NAME)-manager-index
+BENCHMARK_JOB_IMAGE             = $(NAME)-benchmark-job
+BENCHMARK_OPERATOR_IMAGE        = $(NAME)-benchmark-operator
 MAINTAINER                      = "$(ORG).org $(NAME) team <$(NAME)@$(ORG).org>"
 
 VERSION ?= $(eval VERSION := $(shell cat versions/VALD_VERSION))$(VERSION)

Makefile.d/build.mk

@@ -22,7 +22,9 @@ binary/build: \
 	cmd/discoverer/k8s/discoverer \
 	cmd/gateway/lb/lb \
 	cmd/gateway/filter/filter \
-	cmd/manager/index/index
+	cmd/manager/index/index \
+	cmd/tools/benchmark/job/job \
+	cmd/tools/benchmark/operator/operator
 
 cmd/agent/core/ngt/ngt: \
 	ngt/install \
@@ -235,6 +237,68 @@ cmd/index/job/correction/index-correction: \
 		$(dir $@)main.go
 	$@ -version
 
+cmd/tools/benchmark/job/job: \
+	$(GO_SOURCES_INTERNAL) \
+	$(PBGOS) \
+	$(shell find ./cmd/tools/benchmark/job -type f -name '*.go' -not -name '*_test.go' -not -name 'doc.go') \
+	$(shell find ./pkg/tools/benchmark/job -type f -name '*.go' -not -name '*_test.go' -not -name 'doc.go')
+	CFLAGS="$(CFLAGS)" \
+	CXXFLAGS="$(CXXFLAGS)" \
+	CGO_ENABLED=1 \
+	CGO_CXXFLAGS="-g -Ofast -march=native" \
+	CGO_FFLAGS="-g -Ofast -march=native" \
+	CGO_LDFLAGS="-g -Ofast -march=native" \
+	GO111MODULE=on \
+	GOPRIVATE=$(GOPRIVATE) \
+	go build \
+		--ldflags "-s -w \
+		-X '$(GOPKG)/internal/info.Version=$(VERSION)' \
+		-X '$(GOPKG)/internal/info.GitCommit=$(GIT_COMMIT)' \
+		-X '$(GOPKG)/internal/info.BuildTime=$(DATETIME)' \
+		-X '$(GOPKG)/internal/info.GoVersion=$(GO_VERSION)' \
+		-X '$(GOPKG)/internal/info.GoOS=$(GOOS)' \
+		-X '$(GOPKG)/internal/info.GoArch=$(GOARCH)' \
+		-X '$(GOPKG)/internal/info.CGOEnabled=$${CGO_ENABLED}' \
+		-X '$(GOPKG)/internal/info.NGTVersion=$(NGT_VERSION)' \
+		-X '$(GOPKG)/internal/info.BuildCPUInfoFlags=$(CPU_INFO_FLAGS)' \
+		-buildid=" \
+		-mod=readonly \
+		-modcacherw \
+		-a \
+		-tags "cgo osusergo netgo" \
+		-trimpath \
+		-o $@ \
+		$(dir $@)main.go
+	$@ -version
+
+cmd/tools/benchmark/operator/operator: \
+	$(GO_SOURCES_INTERNAL) \
+	$(PBGOS) \
+	$(shell find ./cmd/tools/benchmark/operator -type f -name '*.go' -not -name '*_test.go' -not -name 'doc.go') \
+	$(shell find ./pkg/tools/benchmark/operator -type f -name '*.go' -not -name '*_test.go' -not -name 'doc.go')
+	CFLAGS="$(CFLAGS)" \
+	CXXFLAGS="$(CXXFLAGS)" \
+	CGO_ENABLED=1 \
+	CGO_CXXFLAGS="-g -Ofast -march=native" \
+	CGO_FFLAGS="-g -Ofast -march=native" \
+	CGO_LDFLAGS="-g -Ofast -march=native" \
+	GO111MODULE=on \
+	GOPRIVATE=$(GOPRIVATE) \
+	go build \
+		--ldflags "-s -w \
+		-X '$(GOPKG)/internal/info.CGOEnabled=$${CGO_ENABLED}' \
+		-X '$(GOPKG)/internal/info.NGTVersion=$(NGT_VERSION)' \
+		-X '$(GOPKG)/internal/info.BuildCPUInfoFlags=$(CPU_INFO_FLAGS)' \
+		-buildid=" \
+		-mod=readonly \
+		-modcacherw \
+		-a \
+		-tags "cgo osusergo netgo" \
+		-trimpath \
+		-o $@ \
+		$(dir $@)main.go
+	$@ -version
+
 .PHONY: binary/build/zip
 ## build all binaries and zip them
 binary/build/zip: \
@@ -269,3 +333,10 @@ artifacts/vald-manager-index-$(GOOS)-$(GOARCH).zip: cmd/manager/index/index
 	$(call mkdir, $(dir $@))
 	zip --junk-paths $@ $<
 
+artifacts/vald-benchmark-job-$(GOOS)-$(GOARCH).zip: cmd/tools/benchmark/job/job
+	$(call mkdir, $(dir $@))
+	zip --junk-paths $@ $<
+
+artifacts/vald-benchmark-operator-$(GOOS)-$(GOARCH).zip: cmd/tools/benchmark/operator/operator
+	$(call mkdir, $(dir $@))
+	zip --junk-paths $@ $<

Makefile.d/docker.mk

@@ -22,7 +22,9 @@ docker/build: \
 	docker/build/gateway-lb \
 	docker/build/gateway-filter \
 	docker/build/manager-index \
-	docker/build/operator/helm
+	docker/build/benchmark-job \
+	docker/build/benchmark-operator \
+	docker/build/helm-operator
 
 .PHONY: docker/name/org
 docker/name/org:
@@ -202,3 +204,33 @@ docker/build/index-correction:
 	    -t $(ORG)/$(INDEX_CORRECTION_IMAGE):$(TAG) . \
 	    --build-arg MAINTAINER=$(MAINTAINER) \
 	    --build-arg GO_VERSION=$(GO_VERSION)
+
+.PHONY: docker/name/benchmark-job
+docker/name/benchmark-job:
+	@echo "$(ORG)/$(BENCHMARK_JOB_IMAGE)"
+
+.PHONY: docker/build/benchmark-job
+## build benchmark job
+docker/build/benchmark-job:
+	$(DOCKER) build \
+	    $(DOCKER_OPTS) \
+	    -f dockers/tools/benchmark/job/Dockerfile \
+	    -t $(ORG)/$(BENCHMARK_JOB_IMAGE):$(TAG) . \
+	    --build-arg GO_VERSION=$(GO_VERSION) \
+	    --build-arg DISTROLESS_IMAGE=$(DISTROLESS_IMAGE) \
+	    --build-arg DISTROLESS_IMAGE_TAG=$(DISTROLESS_IMAGE_TAG)
+
+.PHONY: docker/name/benchmark-operator
+docker/name/benchmark-operator:
+	@echo "$(ORG)/$(BENCHMARK_OPERATOR_IMAGE)"
+
+.PHONY: docker/build/benchmark-operator
+## build benchmark operator
+docker/build/benchmark-operator:
+	$(DOCKER) build \
+	    $(DOCKER_OPTS) \
+	    -f dockers/tools/benchmark/operator/Dockerfile \
+	    -t $(ORG)/$(BENCHMARK_OPERATOR_IMAGE):$(TAG) . \
+	    --build-arg GO_VERSION=$(GO_VERSION) \
+	    --build-arg DISTROLESS_IMAGE=$(DISTROLESS_IMAGE) \
+	    --build-arg DISTROLESS_IMAGE_TAG=$(DISTROLESS_IMAGE_TAG)

Makefile.d/helm.mk

@@ -90,6 +90,37 @@ charts/vald-helm-operator/values.schema.json: \
 	GOPRIVATE=$(GOPRIVATE) \
 	go run -mod=readonly hack/helm/schema/gen/main.go charts/vald-helm-operator/values.yaml > charts/vald-helm-operator/values.schema.json
 
+.PHONY: helm/schema/vald-benchmark-job
+## generate json schema for Vald Benchmark Job Chart
+helm/schema/vald-benchmark-job: charts/vald-benchmark-operator/job-values.schema.json
+
+charts/vald-benchmark-operator/job-values.schema.json: \
+	charts/vald-benchmark-operator/schemas/job-values.yaml \
+	hack/helm/schema/gen/main.go
+	GOPRIVATE=$(GOPRIVATE) \
+	go run -mod=readonly hack/helm/schema/gen/main.go charts/vald-benchmark-operator/schemas/job-values.yaml > charts/vald-benchmark-operator/job-values.schema.json
+
+.PHONY: helm/schema/vald-benchmark-job
+## generate json schema for Vald Benchmark Job Chart
+helm/schema/vald-benchmark-scenario: charts/vald-benchmark-operator/scenario-values.schema.json
+
+charts/vald-benchmark-operator/scenario-values.schema.json: \
+	charts/vald-benchmark-operator/schemas/scenario-values.yaml \
+	hack/helm/schema/gen/main.go
+	GOPRIVATE=$(GOPRIVATE) \
+	go run -mod=readonly hack/helm/schema/gen/main.go charts/vald-benchmark-operator/schemas/scenario-values.yaml > charts/vald-benchmark-operator/scenario-values.schema.json
+
+.PHONY: helm/schema/vald-benchmark-operator
+## generate json schema for Vald Benchmark Operator Chart
+helm/schema/vald-benchmark-operator: charts/vald-benchmark-operator/values.schema.json
+
+charts/vald-benchmark-operator/values.schema.json: \
+	charts/vald-benchmark-operator/values.yaml \
+	hack/helm/schema/gen/main.go
+	GOPRIVATE=$(GOPRIVATE) \
+	go run -mod=readonly hack/helm/schema/gen/main.go charts/vald-benchmark-operator/values.yaml > charts/vald-benchmark-operator/values.schema.json
+
+
 .PHONY: yq/install
 ## install yq
 yq/install: $(BINDIR)/yq
@@ -121,3 +152,36 @@ helm/schema/crd/vald-helm-operator: \
 	charts/vald-helm-operator/values.yaml > $(TEMP_DIR)/valdhelmoperatorrelease-spec.yaml
 	$(BINDIR)/yq eval-all 'select(fileIndex==0).spec.versions[0].schema.openAPIV3Schema.properties.spec = select(fileIndex==1).spec | select(fileIndex==0)' \
 	$(TEMP_DIR)/valdhelmoperatorrelease.yaml $(TEMP_DIR)/valdhelmoperatorrelease-spec.yaml > charts/vald-helm-operator/crds/valdhelmoperatorrelease.yaml
+
+.PHONY: helm/schema/crd/vald-benchmark-job
+## generate OpenAPI v3 schema for ValdBenchmarkJobRelease
+helm/schema/crd/vald-benchmark-job: \
+	yq/install
+	mv charts/vald-benchmark-operator/crds/valdbenchmarkjob.yaml $(TEMP_DIR)/valdbenchmarkjob.yaml
+	GOPRIVATE=$(GOPRIVATE) \
+	go run -mod=readonly hack/helm/schema/crd/main.go \
+	charts/vald-benchmark-operator/schemas/job-values.yaml > $(TEMP_DIR)/valdbenchmarkjob-spec.yaml
+	$(BINDIR)/yq eval-all 'select(fileIndex==0).spec.versions[0].schema.openAPIV3Schema.properties.spec = select(fileIndex==1).spec | select(fileIndex==0)' \
+	$(TEMP_DIR)/valdbenchmarkjob.yaml $(TEMP_DIR)/valdbenchmarkjob-spec.yaml > charts/vald-benchmark-operator/crds/valdbenchmarkjob.yaml
+
+.PHONY: helm/schema/crd/vald-benchmark-scenario
+## generate OpenAPI v3 schema for ValdBenchmarkScenarioRelease
+helm/schema/crd/vald-benchmark-scenario: \
+	yq/install
+	mv charts/vald-benchmark-operator/crds/valdbenchmarkscenario.yaml $(TEMP_DIR)/valdbenchmarkscenario.yaml
+	GOPRIVATE=$(GOPRIVATE) \
+	go run -mod=readonly hack/helm/schema/crd/main.go \
+	charts/vald-benchmark-operator/schemas/scenario-values.yaml > $(TEMP_DIR)/valdbenchmarkscenario-spec.yaml
+	$(BINDIR)/yq eval-all 'select(fileIndex==0).spec.versions[0].schema.openAPIV3Schema.properties.spec = select(fileIndex==1).spec | select(fileIndex==0)' \
+	$(TEMP_DIR)/valdbenchmarkscenario.yaml $(TEMP_DIR)/valdbenchmarkscenario-spec.yaml > charts/vald-benchmark-operator/crds/valdbenchmarkscenario.yaml
+
+.PHONY: helm/schema/crd/vald-benchmark-operator
+## generate OpenAPI v3 schema for ValdBenchmarkOperatorRelease
+helm/schema/crd/vald-benchmark-operator: \
+	yq/install
+	mv charts/vald-benchmark-operator/crds/valdbenchmarkoperatorrelease.yaml $(TEMP_DIR)/valdbenchmarkoperatorrelease.yaml
+	GOPRIVATE=$(GOPRIVATE) \
+	go run -mod=readonly hack/helm/schema/crd/main.go \
+	charts/vald-benchmark-operator/values.yaml > $(TEMP_DIR)/valdbenchmarkoperatorrelease-spec.yaml
+	$(BINDIR)/yq eval-all 'select(fileIndex==0).spec.versions[0].schema.openAPIV3Schema.properties.spec = select(fileIndex==1).spec | select(fileIndex==0)' \
+	$(TEMP_DIR)/valdbenchmarkoperatorrelease.yaml $(TEMP_DIR)/valdbenchmarkoperatorrelease-spec.yaml > charts/vald-benchmark-operator/crds/valdbenchmarkoperatorrelease.yaml