Merge pull request #4 from mbaldessari/make-common

Remake common

common/.ansible-lint

@@ -14,4 +14,8 @@ skip_list:
 exclude_paths:
   - ./ansible/playbooks/vault/vault.yaml
   - ./ansible/playbooks/iib-ci/iib-ci.yaml
+  - ./ansible/playbooks/k8s_secrets/k8s_secrets.yml
+  - ./ansible/playbooks/process_secrets/process_secrets.yml
+  - ./ansible/playbooks/write-token-kubeconfig/write-token-kubeconfig.yml
+  - ./ansible/playbooks/process_secrets/display_secrets_info.yml
   - ./ansible/roles/vault_utils/tests/test.yml

common/.github/workflows/chart-branches.yml

@@ -32,7 +32,7 @@ jobs:
       - name: Checkout Code
         uses: actions/checkout@v4
 
-      - uses: dorny/paths-filter@v2
+      - uses: dorny/paths-filter@v3
         id: filter
         with:
           filters: |
@@ -49,9 +49,7 @@ jobs:
 
   acm:
     needs: changes
-    if: |
-      ${{ needs.changes.outputs.acm == 'true' }} &&
-      github.repository == 'validatedpatterns/common'
+    if: ${{ (needs.changes.outputs.acm == 'true') && (github.repository == 'validatedpatterns/common') }}
     uses: validatedpatterns/common/.github/workflows/chart-split.yml@main
     permissions:
       actions: write
@@ -63,9 +61,7 @@ jobs:
 
   golang-external-secrets:
     needs: changes
-    if: |
-      ${{ needs.changes.outputs.golang-external-secrets == 'true' }} &&
-      github.repository == 'validatedpatterns/common'
+    if: ${{ (needs.changes.outputs.golang-external-secrets == 'true') && (github.repository == 'validatedpatterns/common') }}
     uses: validatedpatterns/common/.github/workflows/chart-split.yml@main
     permissions:
       actions: write
@@ -77,9 +73,7 @@ jobs:
 
   hashicorp-vault:
     needs: changes
-    if: |
-      ${{ needs.changes.outputs.hashicorp-vault == 'true' }} &&
-      github.repository == 'validatedpatterns/common'
+    if: ${{ (needs.changes.outputs.hashicorp-vault == 'true') && (github.repository == 'validatedpatterns/common') }}
     uses: validatedpatterns/common/.github/workflows/chart-split.yml@main
     permissions:
       actions: write
@@ -91,9 +85,7 @@ jobs:
 
   letsencrypt:
     needs: changes
-    if: |
-      ${{ needs.changes.outputs.letsencrypt == 'true' }} &&
-      github.repository == 'validatedpatterns/common'
+    if: ${{ (needs.changes.outputs.letsencrypt == 'true') && (github.repository == 'validatedpatterns/common') }}
     uses: validatedpatterns/common/.github/workflows/chart-split.yml@main
     permissions:
       actions: write
@@ -105,9 +97,7 @@ jobs:
 
   clustergroup:
     needs: changes
-    if: |
-      ${{ needs.changes.outputs.clustergroup == 'true' }} &&
-      github.repository == 'validatedpatterns/common'
+    if: ${{ (needs.changes.outputs.clustergroup == 'true') && (github.repository == 'validatedpatterns/common') }}
     uses: validatedpatterns/common/.github/workflows/chart-split.yml@main
     permissions:
       actions: write

common/.github/workflows/chart-split.yml

@@ -31,8 +31,12 @@ jobs:
           set -e
           N="${{ inputs.chart_name }}"
           B="${N}-main-single-chart"
+          GITIMG="quay.io/hybridcloudpatterns/gitsubtree-container:2.40.1"
+          sudo apt-get update -y && sudo apt-get install -y podman
+          echo "Running subtree split for ${B}"
+          podman pull "${GITIMG}"
           git push origin -d "${B}" || /bin/true
-          git subtree split -P "${N}" -b "${B}"
-          git push -f -u origin "${B}"
+          # Git subtree got broken on recent versions of git hence this container
+          podman run --net=host --rm -t -v .:/git "${GITIMG}" subtree split -P "${N}" -b "${B}"
           #git clone https://validatedpatterns:${GITHUB_TOKEN}@github.com/validatedpatterns/common.git -b "acm-main-single-chart" --single-branch
           git push --force https://validatedpatterns:"${GITHUB_TOKEN}"@github.com/${{ inputs.target_repository }}.git "${B}:main"

common/.github/workflows/linter.yml

@@ -34,9 +34,9 @@ jobs:
           # Full git history is needed to get a proper list of changed files within `super-linter`
           fetch-depth: 0
       - name: Setup helm
-        uses: azure/setup-helm@v3
+        uses: azure/setup-helm@v4
         with:
-          version: 'v3.12.3'
+          version: 'v3.14.0'
 
 
       ################################

common/.github/workflows/superlinter.yml

@@ -21,17 +21,23 @@ jobs:
       # Run Linter against code base #
       ################################
       - name: Lint Code Base
-        uses: github/super-linter/slim@v5
+        uses: super-linter/super-linter/slim@v7
         env:
           VALIDATE_ALL_CODEBASE: true
           DEFAULT_BRANCH: main
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           # These are the validation we disable atm
           VALIDATE_ANSIBLE: false
           VALIDATE_BASH: false
+          VALIDATE_CHECKOV: false
           VALIDATE_JSCPD: false
+          VALIDATE_JSON_PRETTIER: false
+          VALIDATE_MARKDOWN_PRETTIER: false
           VALIDATE_KUBERNETES_KUBECONFORM: false
+          VALIDATE_PYTHON_PYLINT: false
+          VALIDATE_SHELL_SHFMT: false
           VALIDATE_YAML: false
+          VALIDATE_YAML_PRETTIER: false
           # VALIDATE_DOCKERFILE_HADOLINT: false
           # VALIDATE_MARKDOWN: false
           # VALIDATE_NATURAL_LANGUAGE: false

common/.gitignore

@@ -5,6 +5,7 @@ __pycache__/
 *.swo
 values-secret.yaml
 .*.expected.yaml
+.vscode
 pattern-vault.init
 pattern-vault.init.bak
 super-linter.log

common/Changes.md

@@ -124,7 +124,7 @@
 ## October 3, 2022
 
 * Restore the ability to install a non-default site: `make TARGET_SITE=mysite install`
-* Revised tests (new output and filenames, requires adding new result files to git)
+* Revised tests (new output and filenames, requires adding new result files to Git)
 * ACM 2.6 required for ACM-based managed sites
 * Introduced global.clusterDomain template variable (without the `apps.` prefix)
 * Removed the ability to send specific charts to another cluster, use hosted argo sites instead

common/Makefile

@@ -1,4 +1,5 @@
 NAME ?= $(shell basename "`pwd`")
+
 ifneq ($(origin TARGET_SITE), undefined)
   TARGET_SITE_OPT=--set main.clusterGroupName=$(TARGET_SITE)
 endif
@@ -55,11 +56,12 @@ help: ## This help message
 show: ## show the starting template without installing it
 	helm template common/operator-install/ --name-template $(NAME) $(HELM_OPTS)
 
-preview-all:
+preview-all: ## (EXPERIMENTAL) Previews all applications on hub and managed clusters
+	@echo "NOTE: This is just a tentative approximation of rendering all hub and managed clusters templates"
 	@common/scripts/preview-all.sh $(TARGET_REPO) $(TARGET_BRANCH)
 
 preview-%:
-	CLUSTERGROUP?=$(shell yq ".main.clusterGroupName" values-global.yaml)
+	$(eval CLUSTERGROUP ?= $(shell yq ".main.clusterGroupName" values-global.yaml))
 	@common/scripts/preview.sh $(CLUSTERGROUP) $* $(TARGET_REPO) $(TARGET_BRANCH)
 
 .PHONY: operator-deploy
@@ -77,20 +79,49 @@ uninstall: ## runs helm uninstall
 	@oc delete csv -n openshift-operators $(CSV)
 
 .PHONY: load-secrets
-load-secrets: ## loads the secrets into the vault
+load-secrets: ## loads the secrets into the backend determined by values-global setting
+	common/scripts/process-secrets.sh $(NAME)
+
+.PHONY: legacy-load-secrets
+legacy-load-secrets: ## loads the secrets into vault (only)
 	common/scripts/vault-utils.sh push_secrets $(NAME)
 
+.PHONY: secrets-backend-vault
+secrets-backend-vault: ## Edits values files to use default Vault+ESO secrets config
+	common/scripts/set-secret-backend.sh vault
+	common/scripts/manage-secret-app.sh vault present
+	common/scripts/manage-secret-app.sh golang-external-secrets present
+	common/scripts/manage-secret-namespace.sh validated-patterns-secrets absent
+	@git diff --exit-code || echo "Secrets backend set to vault, please review changes, commit, and push to activate in the pattern"
+
+.PHONY: secrets-backend-kubernetes
+secrets-backend-kubernetes: ## Edits values file to use Kubernetes+ESO secrets config
+	common/scripts/set-secret-backend.sh kubernetes
+	common/scripts/manage-secret-namespace.sh validated-patterns-secrets present
+	common/scripts/manage-secret-app.sh vault absent
+	common/scripts/manage-secret-app.sh golang-external-secrets present
+	@git diff --exit-code || echo "Secrets backend set to kubernetes, please review changes, commit, and push to activate in the pattern"
+
+.PHONY: secrets-backend-none
+secrets-backend-none: ## Edits values files to remove secrets manager + ESO
+	common/scripts/set-secret-backend.sh none
+	common/scripts/manage-secret-app.sh vault absent
+	common/scripts/manage-secret-app.sh golang-external-secrets absent
+	common/scripts/manage-secret-namespace.sh validated-patterns-secrets absent
+	@git diff --exit-code || echo "Secrets backend set to none, please review changes, commit, and push to activate in the pattern"
+
 .PHONY: load-iib
 load-iib: ## CI target to install Index Image Bundles
 	@set -e; if [ x$(INDEX_IMAGES) != x ]; then \
-		for IIB in $(shell echo $(INDEX_IMAGES) | tr ',' '\n'); do \
-			INDEX_IMAGE="$${IIB}" ansible-playbook common/ansible/playbooks/iib-ci/iib-ci.yaml; \
-		done; \
+		ansible-playbook common/ansible/playbooks/iib-ci/iib-ci.yaml; \
 	else \
 		echo "No INDEX_IMAGES defined. Bailing out"; \
 		exit 1; \
 	fi
 
+.PHONY: token-kubeconfig
+token-kubeconfig: ## Create a local ~/.kube/config with password (not usually needed)
+	common/scripts/write-token-kubeconfig.sh
 
 ##@ Validation Tasks
 
@@ -99,14 +130,9 @@ load-iib: ## CI target to install Index Image Bundles
 .PHONY: validate-origin
 validate-origin: ## verify the git origin is available
 	@echo "Checking repository:"
-	@echo -n "  $(TARGET_REPO) - branch $(TARGET_BRANCH): "
-	@if [ ! -f /run/.containerenv ]; then\
-		git ls-remote --exit-code --heads $(TARGET_REPO) $(TARGET_BRANCH) >/dev/null &&\
-				echo "OK" ||\
-				(echo "NOT FOUND"; exit 1);\
-	else\
-		echo "Running inside a container: Skipping git ssh checks";\
-	fi
+	@echo -n "  $(TARGET_REPO) - branch '$(TARGET_BRANCH)': "
+	@git ls-remote --exit-code --heads $(TARGET_REPO) $(TARGET_BRANCH) >/dev/null &&\
+		echo "OK" || (echo "NOT FOUND"; exit 1)
 
 .PHONY: validate-cluster
 validate-cluster: ## Do some cluster validations before installing
@@ -130,15 +156,19 @@ validate-schema: ## validates values files against schema in common/clustergroup
 
 .PHONY: validate-prereq
 validate-prereq: ## verify pre-requisites
-	@echo "Checking prerequisites:"
-	@for t in $(EXECUTABLES); do if ! which $$t > /dev/null 2>&1; then echo "No $$t in PATH"; exit 1; fi; done
-	@echo "  Check for '$(EXECUTABLES)': OK"
-	@echo -n "  Check for python-kubernetes: "
-	@if ! ansible -m ansible.builtin.command -a "{{ ansible_python_interpreter }} -c 'import kubernetes'" localhost > /dev/null 2>&1; then echo "Not found"; exit 1; fi
-	@echo "OK"
-	@echo -n "  Check for kubernetes.core collection: "
-	@if ! ansible-galaxy collection list | grep kubernetes.core > /dev/null 2>&1; then echo "Not found"; exit 1; fi
-	@echo "OK"
+	@if [ ! -f /run/.containerenv ]; then\
+	  echo "Checking prerequisites:";\
+	  for t in $(EXECUTABLES); do if ! which $$t > /dev/null 2>&1; then echo "No $$t in PATH"; exit 1; fi; done;\
+	  echo "  Check for '$(EXECUTABLES)': OK";\
+	  echo -n "  Check for python-kubernetes: ";\
+	  if ! ansible -m ansible.builtin.command -a "{{ ansible_python_interpreter }} -c 'import kubernetes'" localhost > /dev/null 2>&1; then echo "Not found"; exit 1; fi;\
+	  echo "OK";\
+	  echo -n "  Check for kubernetes.core collection: ";\
+	  if ! ansible-galaxy collection list | grep kubernetes.core > /dev/null 2>&1; then echo "Not found"; exit 1; fi;\
+	  echo "OK";\
+	else\
+	  echo "Skipping prerequisites check as we're running inside a container";\
+	fi
 
 .PHONY: argo-healthcheck
 argo-healthcheck: ## Checks if all argo applications are synced
@@ -203,17 +233,23 @@ kubeconform: ## run helm kubeconform
 super-linter: ## Runs super linter locally
 	rm -rf .mypy_cache
 	podman run -e RUN_LOCAL=true -e USE_FIND_ALGORITHM=true	\
+					-e VALIDATE_ANSIBLE=false \
 					-e VALIDATE_BASH=false \
+					-e VALIDATE_CHECKOV=false \
+					-e VALIDATE_DOCKERFILE_HADOLINT=false \
 					-e VALIDATE_JSCPD=false \
+					-e VALIDATE_JSON_PRETTIER=false \
+					-e VALIDATE_MARKDOWN_PRETTIER=false \
 					-e VALIDATE_KUBERNETES_KUBECONFORM=false \
-					-e VALIDATE_YAML=false \
-					-e VALIDATE_ANSIBLE=false \
-					-e VALIDATE_DOCKERFILE_HADOLINT=false \
+					-e VALIDATE_PYTHON_PYLINT=false \
+					-e VALIDATE_SHELL_SHFMT=false \
 					-e VALIDATE_TEKTON=false \
+					-e VALIDATE_YAML=false \
+					-e VALIDATE_YAML_PRETTIER=false \
 					$(DISABLE_LINTERS) \
 					-v $(PWD):/tmp/lint:rw,z \
 					-w /tmp/lint \
-					docker.io/github/super-linter:slim-v5
+					ghcr.io/super-linter/super-linter:slim-v7
 
 .PHONY: ansible-lint
 ansible-lint: ## run ansible lint on ansible/ folder

common/acm/Chart.yaml

@@ -3,4 +3,4 @@ description: A Helm chart to configure Advanced Cluster Manager for OpenShift.
 keywords:
 - pattern
 name: acm
-version: 0.0.1
+version: 0.1.1

common/acm/README.md

@@ -0,0 +1,5 @@
+# Validated Patterns ACM chart
+
+This chart is used to set up ACM in [Validated Patterns](https://validatedpatterns.io)
+
+Please send PRs [here](https://github.com/validatedpatterns/common)

common/acm/templates/_helpers.tpl

@@ -5,9 +5,53 @@ Default always defined valueFiles to be included when pushing the cluster wide a
 - "/values-global.yaml"
 - "/values-{{ .name }}.yaml"
 - '/values-{{ `{{ (lookup "config.openshift.io/v1" "Infrastructure" "" "cluster").spec.platformSpec.type }}` }}.yaml'
-- '/values-{{ `{{ (lookup "config.openshift.io/v1" "Infrastructure" "" "cluster").spec.platformSpec.type }}` }}-{{ `{{ printf "%d.%d" ((semver (lookup "operator.openshift.io/v1" "OpenShiftControllerManager" "" "cluster").status.version).Major) ((semver (lookup "operator.openshift.io/v1" "OpenShiftControllerManager" "" "cluster").status.version).Minor) }}` }}.yaml'
+- '/values-{{ `{{ (lookup "config.openshift.io/v1" "Infrastructure" "" "cluster").spec.platformSpec.type }}` }}-{{ `{{ printf "%d.%d" ((semver (index (lookup "config.openshift.io/v1" "ClusterVersion" "" "version").status.history 0).version).Major) ((semver (index (lookup "config.openshift.io/v1" "ClusterVersion" "" "version").status.history 0).version).Minor) }}` }}.yaml'
 - '/values-{{ `{{ (lookup "config.openshift.io/v1" "Infrastructure" "" "cluster").spec.platformSpec.type }}` }}-{{ .name }}.yaml'
 # We cannot use $.Values.global.clusterVersion because that gets resolved to the
 # hub's cluster version, whereas we want to include the spoke cluster version
-- '/values-{{ `{{ printf "%d.%d" ((semver (lookup "operator.openshift.io/v1" "OpenShiftControllerManager" "" "cluster").status.version).Major) ((semver (lookup "operator.openshift.io/v1" "OpenShiftControllerManager" "" "cluster").status.version).Minor) }}` }}-{{ .name }}.yaml'
+- '/values-{{ `{{ printf "%d.%d" ((semver (index (lookup "config.openshift.io/v1" "ClusterVersion" "" "version").status.history 0).version).Major) ((semver (index (lookup "config.openshift.io/v1" "ClusterVersion" "" "version").status.history 0).version).Minor) }}` }}.yaml'
 {{- end }} {{- /*acm.app.policies.valuefiles */}}
+
+{{- define "acm.app.policies.multisourcevaluefiles" -}}
+- "$patternref/values-global.yaml"
+- "$patternref/values-{{ .name }}.yaml"
+- '$patternref/values-{{ `{{ (lookup "config.openshift.io/v1" "Infrastructure" "" "cluster").spec.platformSpec.type }}` }}.yaml'
+- '$patternref/values-{{ `{{ (lookup "config.openshift.io/v1" "Infrastructure" "" "cluster").spec.platformSpec.type }}` }}-{{ `{{ printf "%d.%d" ((semver (index (lookup "config.openshift.io/v1" "ClusterVersion" "" "version").status.history 0).version).Major) ((semver (index (lookup "config.openshift.io/v1" "ClusterVersion" "" "version").status.history 0).version).Minor) }}` }}.yaml'
+- '$patternref/values-{{ `{{ (lookup "config.openshift.io/v1" "Infrastructure" "" "cluster").spec.platformSpec.type }}` }}-{{ .name }}.yaml'
+# We cannot use $.Values.global.clusterVersion because that gets resolved to the
+# hub's cluster version, whereas we want to include the spoke cluster version
+- '$patternref/values-{{ `{{ printf "%d.%d" ((semver (index (lookup "config.openshift.io/v1" "ClusterVersion" "" "version").status.history 0).version).Major) ((semver (index (lookup "config.openshift.io/v1" "ClusterVersion" "" "version").status.history 0).version).Minor) }}` }}.yaml'
+{{- end }} {{- /*acm.app.policies.multisourcevaluefiles */}}
+
+{{- define "acm.app.policies.helmparameters" -}}
+- name: global.repoURL
+  value: {{ $.Values.global.repoURL }}
+- name: global.targetRevision
+  value: {{ $.Values.global.targetRevision }}
+- name: global.namespace
+  value: $ARGOCD_APP_NAMESPACE
+- name: global.pattern
+  value: {{ $.Values.global.pattern }}
+- name: global.hubClusterDomain
+  value: {{ $.Values.global.hubClusterDomain }}
+- name: global.localClusterDomain
+  value: '{{ `{{ (lookup "config.openshift.io/v1" "Ingress" "" "cluster").spec.domain }}` }}'
+- name: global.clusterDomain
+  value: '{{ `{{ (lookup "config.openshift.io/v1" "Ingress" "" "cluster").spec.domain | replace "apps." "" }}` }}'
+- name: global.clusterVersion
+  value: '{{ `{{ printf "%d.%d" ((semver (index (lookup "config.openshift.io/v1" "ClusterVersion" "" "version").status.history 0).version).Major) ((semver (index (lookup "config.openshift.io/v1" "ClusterVersion" "" "version").status.history 0).version).Minor) }}` }}'
+- name: global.localClusterName
+  value: '{{ `{{ (split "." (lookup "config.openshift.io/v1" "Ingress" "" "cluster").spec.domain)._1 }}` }}'
+- name: global.clusterPlatform
+  value: {{ $.Values.global.clusterPlatform }}
+- name: global.multiSourceSupport
+  value: {{ $.Values.global.multiSourceSupport | quote }}
+- name: global.multiSourceRepoUrl
+  value: {{ $.Values.global.multiSourceRepoUrl }}
+- name: global.multiSourceTargetRevision
+  value: {{ $.Values.global.multiSourceTargetRevision }}
+- name: global.privateRepo
+  value: {{ $.Values.global.privateRepo | quote }}
+- name: global.experimentalCapabilities
+  value: {{ $.Values.global.experimentalCapabilities }}
+{{- end }} {{- /*acm.app.policies.helmparameters */}}

common/acm/templates/multiclusterhub.yaml

@@ -1,11 +1,13 @@
+{{- $channel := "" }}
+{{- if .Values.acm.mce_operator.channel }}
+{{- $channel = printf ",\"channel\": \"%s\"" .Values.acm.mce_operator.channel }}
+{{- end }}
 apiVersion: operator.open-cluster-management.io/v1
 kind: MultiClusterHub
 metadata:
   name: multiclusterhub
   namespace: open-cluster-management
   annotations:
     argocd.argoproj.io/sync-wave: "-1"
-    {{- if kindIs "map" .Values.clusterGroup.subscriptions }}
-    installer.open-cluster-management.io/mce-subscription-spec: '{"source": "{{ default "redhat-operators" .Values.clusterGroup.subscriptions.acm.source }}" }'
-    {{- end }}
+    installer.open-cluster-management.io/mce-subscription-spec: '{"source": "{{ default "redhat-operators" .Values.acm.mce_operator.source }}" {{- $channel }} }'
 spec: {}