From b766e0547d6790aa400b253a060d37249f163729 Mon Sep 17 00:00:00 2001 From: Gerd Oberlechner Date: Tue, 7 Jan 2025 23:00:12 +0100 Subject: [PATCH] provision shard config generation in helm chart moving the provision shard config generation is required mostly due to a technical limitation of templatize right now. short-term the provision shard config will vanish for in-cluster deployments as shards will need to be brought in by a registration API. Signed-off-by: Gerd Oberlechner --- .github/workflows/environment-infra-cd.yml | 2 +- cluster-service/Makefile | 21 ++++++++-------- .../templates/provisioning-shards.secret.yaml | 25 +++++++++++++++++-- cluster-service/deploy/helm/values.yaml | 10 ++++++-- 4 files changed, 43 insertions(+), 15 deletions(-) diff --git a/.github/workflows/environment-infra-cd.yml b/.github/workflows/environment-infra-cd.yml index 4ad3838b2..65c365ce4 100644 --- a/.github/workflows/environment-infra-cd.yml +++ b/.github/workflows/environment-infra-cd.yml @@ -150,4 +150,4 @@ - name: 'Deploy rest' run: | cd dev-infrastructure/ - PRINCIPAL_ID=${{ secrets.GHA_PRINCIPAL_ID }} mgmt.aks.admin-access mgmt.enable-aks-metrics + PRINCIPAL_ID=${{ secrets.GHA_PRINCIPAL_ID }} make mgmt.aks.admin-access mgmt.enable-aks-metrics diff --git a/cluster-service/Makefile b/cluster-service/Makefile index 9b5669182..4e9d73bc9 100644 --- a/cluster-service/Makefile +++ b/cluster-service/Makefile @@ -5,7 +5,7 @@ HELM_CMD ?= helm upgrade --install ZONE_NAME ?= "${REGIONAL_DNS_SUBDOMAIN}.${BASE_DNS_ZONE_NAME}" -deploy: provision-shard +deploy: @kubectl create namespace cluster-service --dry-run=client -o json | kubectl apply -f - && \ kubectl label namespace cluster-service "istio.io/rev=${ISTO_TAG}" --overwrite=true && \ AZURE_CS_MI_CLIENT_ID=$(shell az identity show -g ${RESOURCEGROUP} -n clusters-service --query clientId -o tsv) && \ @@ -23,6 +23,9 @@ deploy: provision-shard OP_FILE_CSI_DRIVER_ROLE_ID=$(shell az role definition list --name "${OP_FILE_CSI_DRIVER_ROLE_NAME}" --query "[].name" -o tsv) && \ OP_IMAGE_REGISTRY_DRIVER_ROLE_ID=$(shell az role definition list --name "${OP_IMAGE_REGISTRY_DRIVER_ROLE_NAME}" --query "[].name" -o tsv) && \ OP_CLOUD_NETWORK_CONFIG_ROLE_ID=$(shell az role definition list --name "${OP_CLOUD_NETWORK_CONFIG_ROLE_NAME}" --query "[].name" -o tsv) && \ + ZONE_RESOURCE_ID=$(shell az network dns zone show -n ${ZONE_NAME} -g ${REGIONAL_RESOURCEGROUP} --query id -o tsv) && \ + CX_SECRETS_KV_URL=$(shell az keyvault show -n ${CX_SECRETS_KV_NAME} -g ${MGMT_RESOURCEGROUP} --query properties.vaultUri -o tsv) && \ + CX_MI_KV_URL=$(shell az keyvault show -n ${CX_MI_KV_NAME} -g ${MGMT_RESOURCEGROUP} --query properties.vaultUri -o tsv) && \ ${HELM_CMD} cluster-service deploy/helm \ --namespace cluster-service \ -f deploy/helm/$${OVERRIDES} \ @@ -40,7 +43,12 @@ deploy: provision-shard --set fpaCertName=${FPA_CERT_NAME} \ --set ocpAcrResourceId=$${OCP_ACR_RESOURCE_ID} \ --set ocpAcrUrl=$${OCP_ACR_URL} \ - --set provisionShardsConfig="$(shell base64 -i deploy/provisioning-shards.yml | tr -d '\n')" \ + --set shard.consumerName="${CONSUMER_NAME}" \ + --set shard.zoneResourceId="$${ZONE_RESOURCE_ID}" \ + --set shard.cxSecretsKeyVaultUrl="$${CX_SECRETS_KV_URL}" \ + --set shard.cxMiKeyVaultUrl="$${CX_MI_KV_URL}" \ + --set shard.maestroRestUrl="http://maestro.maestro.svc.cluster.local:8000" \ + --set shard.maestroGrpUrl="maestro-grpc.maestro.svc.cluster.local:8090" \ --set databaseHost=$${DB_HOST} \ --set azureMiMockServicePrincipalPrincipalId=${AZURE_MI_MOCK_SERVICE_PRINCIPAL_PRINCIPAL_ID} \ --set azureMiMockServicePrincipalClientId=${AZURE_MI_MOCK_SERVICE_PRINCIPAL_CLIENT_ID} \ @@ -73,13 +81,6 @@ create-pr-env-sp: --role 'Azure Kubernetes Service RBAC Admin' \ --scopes "$${CLUSTER_ID}" -provision-shard: - @ZONE_RESOURCE_ID=$(shell az network dns zone show -n ${ZONE_NAME} -g ${REGIONAL_RESOURCEGROUP} --query id -o tsv) && \ - CX_SECRETS_KV_URL=$(shell az keyvault show -n ${CX_SECRETS_KV_NAME} -g ${MGMT_RESOURCEGROUP} --query properties.vaultUri -o tsv) && \ - CX_MI_KV_URL=$(shell az keyvault show -n ${CX_MI_KV_NAME} -g ${MGMT_RESOURCEGROUP} --query properties.vaultUri -o tsv) && \ - ../templatize.sh $(DEPLOY_ENV) deploy/provisioning-shards.tmpl.yml deploy/provisioning-shards.yml -e zoneResourceId=$${ZONE_RESOURCE_ID},cxSecretsKeyVaultUrl=$${CX_SECRETS_KV_URL},cxMiKeyVaultUrl=$${CX_MI_KV_URL},maestroRestUrl=http://maestro.maestro.svc.cluster.local:8000,maestroGrpUrl=maestro-grpc.maestro.svc.cluster.local:8090 - @cat deploy/provisioning-shards.yml - local-deploy-provision-shard: @ZONE_RESOURCE_ID=$(shell az network dns zone show -n ${ZONE_NAME} -g ${REGIONAL_RESOURCEGROUP} --query id -o tsv) && \ CX_SECRETS_KV_URL=$(shell az keyvault show -n ${CX_SECRETS_KV_NAME} -g ${MGMT_RESOURCEGROUP} --query properties.vaultUri -o tsv) && \ @@ -159,4 +160,4 @@ shared-storage-runtime-config: @cat deploy/azure-runtime-config.yaml .PHONY: shared-storage-runtime-config -.PHONY: create-pr-env-sp deploy deploy-using-azure-db deploy-integ provision-shard configure-tmp-provision-shard deploy-secrets-template deploy-secrets-template-using-azure-db deploy-istio-configurations-template deploy-namespace-template runtime-config +.PHONY: create-pr-env-sp deploy deploy-using-azure-db deploy-integ configure-tmp-provision-shard deploy-secrets-template deploy-secrets-template-using-azure-db deploy-istio-configurations-template deploy-namespace-template runtime-config diff --git a/cluster-service/deploy/helm/templates/provisioning-shards.secret.yaml b/cluster-service/deploy/helm/templates/provisioning-shards.secret.yaml index 4e6692eee..78dcd4e1c 100644 --- a/cluster-service/deploy/helm/templates/provisioning-shards.secret.yaml +++ b/cluster-service/deploy/helm/templates/provisioning-shards.secret.yaml @@ -4,5 +4,26 @@ kind: Secret metadata: name: provision-shards namespace: {{ .Release.Namespace }} -data: - config: '{{ .Values.provisionShardsConfig }}' +stringData: + config: | + provision_shards: + - id: 1 + maestro_config: | + { + "rest_api_config": { + "url": "{{ .Values.shard.maestroRestUrl }}" + }, + "grpc_api_config": { + "url": "{{ .Values.shard.maestroGrpUrl }}" + }, + "consumer_name": "{{ .Values.shard.consumerName }}" + } + status: active + management_cluster_id: local-cluster + region: {{ .Values.region }} + cloud_provider: azure + topology: dedicated + azure_shard: + public_dns_zone_resource_id: "{{ .Values.shard.zoneResourceId }}" + cx_secrets_key_vault_url: "{{ .Values.shard.cxSecretsKeyVaultUrl }}" + cx_managed_identities_key_vault_url: "{{ .Values.shard.cxMiKeyVaultUrl }}" diff --git a/cluster-service/deploy/helm/values.yaml b/cluster-service/deploy/helm/values.yaml index 3ffaeaaad..17769a968 100644 --- a/cluster-service/deploy/helm/values.yaml +++ b/cluster-service/deploy/helm/values.yaml @@ -228,8 +228,14 @@ azureMiMockServicePrincipalClientId: "" # The principal id of the mock managed identities service principal. azureMiMockServicePrincipalPrincipalId: "" -# Base64 encoded provision shards config. -provisionShardsConfig: "" +# Details for a provisioning shard +shard: + consumerName: "" + zoneResourceId: "" + cxSecretsKeyVaultUrl: "" + cxMiKeyVaultUrl: "" + maestroRestUrl: "" + maestroGrpUrl: "" # ocm client id clientId: "foo"