Add index correction helm templates and E2E (#2200)

* implement the initail framework

* add corrector configuration

* add corrector logic

* add build make command for index correction binary

* add Dockerfile for index correction

* add Docker image for index job correction

* add timer

* fix tag align

* tmp

* fix log

* temporally implement two versions of correct function

* set eg limit from config

* add stream list concurrency config

* implement index id caching

* add config to use cache or not

* style: Format code with prettier and gofumpt

* refactor availableAddrs

* add kvs range duration

* add leftAgentAddrs for performance

* Revert "add kvs range duration"

This reverts commit 5b647be.

* refactor

* fix without cache bug

* enable observability

* refactor

* SIGTERM after complete

* add metrics server

* add pcache

* remove comment

* [TEMP] use pcache

* [TMP] use pcache

* fix empty shard returns error

* fix to use local map

* [TMP] add prestop for pcache

* [TEMP] add pcache config

* style: Format code with prettier and gofumpt

* [TEMP] add pcache log

* fix map alloc size

* [TMP] Add bbolt cache

* update bbolt

* fix bbolt bug

* add bbolt test

* [TEMP] use bbolt as persistent cache

* style: Format code with prettier and gofumpt

* add SetBatch to bbolt

* use batch to write map to disk

* style: Format code with prettier and gofumpt

* delete the map elements on finalize

* manually call GC after the map shrink

* add limit to SetBatch goroutine number

* stop unnecesarry GC

* increase eg limit to the MaxBatchSize

* use ch to set batch bbolt

* fix servers shutdown properly

* use internal/kvs/bbolt

* refactor

* always use bbolt cache for correction

* update sample.yaml for correction

* style: format code with Prettier and Gofumpt

This commit fixes the style issues introduced in 319ec8b according to the output
from Prettier and Gofumpt.

Details: #2152

* use go std slices pkg

* refactor

* add comment

* remove valdsync

* use vald errgroup

* refactor

* Define ErrNoAvailableAgentToInsert

* update comment in English

* Apply new actions yaml format

* Disable godox

* style: format code with Prettier and Gofumpt

This commit fixes the style issues introduced in c860ddc according to the output
from Prettier and Gofumpt.

Details: #2194

* remove comment

* Apply format

* Add type check for type assertion

* use const to specify filemode

* Add bbolt concurrency as config

* fix var style

* Suppress linter

* fix comment

* add test template

* Refactor parameters for index correction

* Refactor config

* Add corrector test

* style: format code with Prettier and Gofumpt

This commit fixes the style issues introduced in 004bf81 according to the output
from Prettier and Gofumpt.

Details: #2194

* Add timestamp check

* Apply format

* fix schema type

* Fix DeepSource errors

* Fix misspell

* Add type check

* Remove unused config

* Fix DeepSource error

* Add required go:build e2e tag

* Remove memo

* Refactor comment

* Add index job correction helm templates

* Add more fields

* Add index correction job E2E test

* Add e2e action for job

* [REVERT THIS] Temporally change version

* Fix name and command

* Apply format

* update crd

* Revert "[REVERT THIS] Temporally change version"

This reverts commit 1801a63.

* Remove unused pkg

* Remove experimental file

* remove old workflow

* Fix cron job name to new one

* Update sample.yaml

* fix build path

* Fix corrector name

* add e2e-jobs to slack notification

* Update crds

---------

Co-authored-by: deepsource-autofix[bot] <62050782+deepsource-autofix[bot]@users.noreply.github.com>

.github/helm/values/values-lb.yaml

@@ -69,3 +69,8 @@ manager:
       auto_index_duration_limit: 2m
       auto_index_check_duration: 30s
       auto_index_length: 1000
+    corrector:
+      enabled: true
+      # suspend because you do not want corrector to start automatically in CI
+      # instead run it manually
+      suspend: true

.github/workflows/e2e.yml

@@ -227,6 +227,42 @@ jobs:
         env:
           POD_NAME: ${{ steps.deploy_vald.outputs.POD_NAME }}
 
+  e2e-jobs:
+    name: "E2E test (Jobs)"
+    needs: [dump-contexts-to-log]
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set Git config
+        run: |
+          git config --global --add safe.directory ${GITHUB_WORKSPACE}
+
+      - name: Setup E2E environment
+        id: setup_e2e
+        uses: ./.github/actions/setup-e2e
+
+      - name: Deploy Vald
+        id: deploy_vald
+        uses: ./.github/actions/e2e-deploy-vald
+        with:
+          helm_extra_options: ${{ steps.setup_e2e.outputs.HELM_EXTRA_OPTIONS }}
+          values: .github/helm/values/values-lb.yaml
+          wait_for_selector: app=vald-lb-gateway
+
+      - name: Run E2E Jobs
+        run: |
+          make hack/benchmark/assets/dataset/${{ env.DATASET }}
+          make E2E_BIND_PORT=8081 \
+            E2E_INSERT_COUNT=10000\
+            E2E_WAIT_FOR_CREATE_INDEX_DURATION=3m \
+            E2E_TARGET_POD_NAME=${POD_NAME} \
+            E2E_TARGET_NAMESPACE=default \
+            e2e/index/job/correction
+        env:
+          POD_NAME: ${{ steps.deploy_vald.outputs.POD_NAME }}
+
   e2e-agent-and-sidecar:
     name: "E2E Agent & Sidecar test"
     needs: [dump-contexts-to-log]
@@ -278,6 +314,7 @@ jobs:
       - e2e-stream-crud
       - e2e-stream-crud-for-operator
       - e2e-stream-crud-skip-exist-check
+      - e2e-jobs
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3

Makefile.d/build.mk

@@ -209,7 +209,7 @@ cmd/manager/index/index: \
 cmd/index/job/correction/index-correction: \
 	$(GO_SOURCES_INTERNAL) \
 	$(PBGOS) \
-	$(shell find $(ROOTDIR)/cmd/index/job/correction/correction -type f -name '*.go' -not -name '*_test.go' -not -name 'doc.go') \
+	$(shell find $(ROOTDIR)/cmd/index/job/correction -type f -name '*.go' -not -name '*_test.go' -not -name 'doc.go') \
 	$(shell find $(ROOTDIR)/pkg/index/job/correction -type f -name '*.go' -not -name '*_test.go' -not -name 'doc.go')
 	$(eval CGO_ENABLED = 0)
 	CGO_ENABLED=$(CGO_ENABLED) \

Makefile.d/e2e.mk

@@ -69,6 +69,11 @@ e2e/remove/timestamp:
 e2e/insert/search:
 	$(call run-e2e-crud-test,-run TestE2EInsertAndSearch)
 
+.PHONY: e2e/index/job/correction
+## run index correction job e2e
+e2e/index/job/correction:
+	$(call run-e2e-crud-test,-run TestE2EIndexJobCorrection)
+
 .PHONY: e2e/maxdim
 ## run e2e/maxdim
 e2e/maxdim:
@@ -79,4 +84,3 @@ e2e/maxdim:
 e2e/sidecar:
 	$(call run-e2e-sidecar-test,-run TestE2EForSidecar)
 
-

Makefile.d/k8s.mk

@@ -23,22 +23,26 @@ k8s/manifest/clean:
 		k8s/agent \
 		k8s/discoverer \
 		k8s/gateway \
-		k8s/manager
+		k8s/manager \
+		k8s/index
 
 .PHONY: k8s/manifest/update
 ## update k8s manifests using helm templates
 k8s/manifest/update: \
 	k8s/manifest/clean
 	helm template \
-	    --values $(HELM_VALUES) \
-	    --output-dir $(TEMP_DIR) \
-	    charts/vald
+		--values $(HELM_VALUES) \
+		--set defaults.image.tag=$(VERSION) \
+		--output-dir $(TEMP_DIR) \
+		charts/vald
 	mkdir -p k8s/gateway
 	mkdir -p k8s/manager
+	mkdir -p k8s/index/job
 	mv $(TEMP_DIR)/vald/templates/agent k8s/agent
 	mv $(TEMP_DIR)/vald/templates/discoverer k8s/discoverer
 	mv $(TEMP_DIR)/vald/templates/gateway/lb k8s/gateway/lb
 	mv $(TEMP_DIR)/vald/templates/manager/index k8s/manager/index
+	mv $(TEMP_DIR)/vald/templates/index/job/correction k8s/index/job/correction
 	rm -rf $(TEMP_DIR)
 
 .PHONY: k8s/manifest/helm-operator/clean
@@ -80,6 +84,7 @@ k8s/vald/deploy:
 	kubectl apply -f $(TEMP_DIR)/vald/templates/agent || true
 	kubectl apply -f $(TEMP_DIR)/vald/templates/discoverer || true
 	kubectl apply -f $(TEMP_DIR)/vald/templates/gateway/lb || true
+	kubectl apply -f $(TEMP_DIR)/vald/templates/index/job/correction || true
 	rm -rf $(TEMP_DIR)
 	kubectl get pods -o jsonpath="{.items[*].spec.containers[*].image}" | tr " " "\n"
 
@@ -97,6 +102,7 @@ k8s/vald/delete:
 	    --set manager.index.image.repository=$(CRORG)/$(MANAGER_INDEX_IMAGE) \
 	    --output-dir $(TEMP_DIR) \
 	    charts/vald
+	kubectl delete -f $(TEMP_DIR)/vald/templates/index/job/correction
 	kubectl delete -f $(TEMP_DIR)/vald/templates/gateway/lb
 	kubectl delete -f $(TEMP_DIR)/vald/templates/manager/index
 	kubectl delete -f $(TEMP_DIR)/vald/templates/discoverer