Azure · geoberle · Oct 30, 2024
@@ -4,7 +4,7 @@ $(shell ../templatize.sh $(DEPLOY_ENV) config.tmpl.mk config.mk)
 include config.mk
 
 deploy: deploy-namespace-template deploy-istio-configurations-template ${DB_SECRET_TARGET}
-	AZURE_CS_MI_CLIENT_ID=$(shell az identity show -g ${RESOURCEGROUP} -n clusters-service --query clientId -o tsv) && \
+	AZURE_CS_MI_CLIENT_ID=$(shell az identity show -g ${RESOURCEGROUP} -n ${CS_MI_NAME} --query clientId -o tsv) && \
 	CS_SERVICE_PRINCIPAL_CREDS_BASE64='$(shell az keyvault secret show --vault-name "${SERVICE_KV}" --name "aro-hcp-dev-sp-cs" | jq .value -r | base64 | tr -d '\n')' && \
 	TENANT_ID=$(shell az account show --query tenantId --output tsv) && \
 	OIDC_BLOB_SERVICE_ENDPOINT=$(shell az storage account show -n ${OIDC_STORAGE_ACCOUNT} -g ${RESOURCEGROUP} --query primaryEndpoints.blob -o tsv) && \
@@ -13,38 +13,44 @@ deploy: deploy-namespace-template deploy-istio-configurations-template ${DB_SECR
 	OCP_ACR_URL=$(shell az acr show -n ${OCP_ACR_NAME} --query loginServer -o tsv) && \
 	OCP_ACR_RESOURCE_ID=$(shell az acr show -n ${OCP_ACR_NAME} --query id -o tsv) && \
 	oc process --local -f deploy/openshift-templates/arohcp-service-template.yml \
-	  -p AZURE_CS_MI_CLIENT_ID=$${AZURE_CS_MI_CLIENT_ID} \
-	  -p TENANT_ID=$${TENANT_ID} \
-	  -p REGION=${REGION} \
-	  -p SERVICE_KEYVAULT_NAME=${SERVICE_KV} \
-	  -p CS_SERVICE_PRINCIPAL_CREDS_BASE64=$${CS_SERVICE_PRINCIPAL_CREDS_BASE64} \
-	  -p IMAGE_REGISTRY=${ACR_NAME}.azurecr.io \
-	  -p IMAGE_REPOSITORY=${IMAGE_REPO} \
-	  -p AZURE_FIRST_PARTY_APPLICATION_CLIENT_ID=${AZURE_FIRST_PARTY_APPLICATION_CLIENT_ID} \
-	  -p FPA_CERT_NAME=${FPA_CERT_NAME} \
-	  -p IMAGE_TAG=${IMAGE_TAG} \
-	  -p OCP_ACR_RESOURCE_ID=$${OCP_ACR_RESOURCE_ID} \
-	  -p OCP_ACR_URL=$${OCP_ACR_URL} \
-	  -p DATABASE_DISABLE_TLS=${DATABASE_DISABLE_TLS} \
-	  -p DATABASE_AUTH_METHOD=${DATABASE_AUTH_METHOD} | oc apply -f -
+		-p NAMESPACE="${CS_NAMESPACE_NAME}" \
+		-p AZURE_CS_MI_CLIENT_ID=$${AZURE_CS_MI_CLIENT_ID} \
+		-p TENANT_ID=$${TENANT_ID} \
+		-p REGION=${REGION} \
+		-p SERVICE_KEYVAULT_NAME=${SERVICE_KV} \
+		-p CS_SERVICE_PRINCIPAL_CREDS_BASE64=$${CS_SERVICE_PRINCIPAL_CREDS_BASE64} \
+		-p IMAGE_REGISTRY=${ACR_NAME}.azurecr.io \
+		-p IMAGE_REPOSITORY=${IMAGE_REPO} \
+		-p AZURE_FIRST_PARTY_APPLICATION_CLIENT_ID=${AZURE_FIRST_PARTY_APPLICATION_CLIENT_ID} \
+		-p FPA_CERT_NAME=${FPA_CERT_NAME} \
+		-p IMAGE_TAG=${IMAGE_TAG} \
+		-p OCP_ACR_RESOURCE_ID=$${OCP_ACR_RESOURCE_ID} \
+		-p OCP_ACR_URL=$${OCP_ACR_URL} \
+		-p SERVICE_ACCOUNT_NAME=${CS_SA_NAME} \
+		-p DATABASE_DISABLE_TLS=${DATABASE_DISABLE_TLS} \
+		-p DATABASE_AUTH_METHOD=${DATABASE_AUTH_METHOD} | oc apply -f -
 
 deploy-namespace-template:
 	ISTO_VERSION=$(shell az aks show -n ${AKS_NAME} -g ${RESOURCEGROUP} --query serviceMeshProfile.istio.revisions[-1] -o tsv) && \
 	oc process --local -f deploy/openshift-templates/arohcp-namespace-template.yml \
-		-p ISTIO_VERSION=$${ISTO_VERSION} | oc apply -f -
+		-p ISTIO_VERSION=$${ISTO_VERSION} \
+		-p NAMESPACE="${CS_NAMESPACE_NAME}" | oc apply -f -
 
 deploy-istio-configurations-template:
 	kubectl apply -f deploy/istio.yml
 
 deploy-local-db-secret: provision-shard
 	oc process --local -f deploy/openshift-templates/arohcp-secrets-template.yml \
-		-p PROVISION_SHARDS_CONFIG="$$( base64 -i deploy/provisioning-shards.yml)" | oc apply -f -
-	oc process --local -f deploy/openshift-templates/arohcp-db-template.yml | oc apply -f -
+		-p PROVISION_SHARDS_CONFIG="$$( base64 -i deploy/provisioning-shards.yml)" \
+		-p NAMESPACE="${CS_NAMESPACE_NAME}" | oc apply -f -
+	oc process --local -f deploy/openshift-templates/arohcp-db-template.yml \
+		-p NAMESPACE="${CS_NAMESPACE_NAME}" | oc apply -f -
 
 deploy-azure-db-secret: provision-shard
 	oc process --local -f deploy/openshift-templates/arohcp-secrets-template.yml \
-		-p DATABASE_USER=clusters-service \
-		-p DATABASE_NAME=clusters-service \
+		-p NAMESPACE="${CS_NAMESPACE_NAME}" \
+		-p DATABASE_USER=${CS_MI_NAME} \
+		-p DATABASE_NAME=${CS_DB_NAME} \
 		-p DATABASE_PASSWORD="" \
 		-p DATABASE_HOST=$(shell az postgres flexible-server show --resource-group ${RESOURCEGROUP} -n ${DATABASE_SERVER_NAME} --query fullyQualifiedDomainName -o tsv) \
 		-p PROVISION_SHARDS_CONFIG="$$( base64 -i deploy/provisioning-shards.yml)" | oc apply -f -

@@ -13,6 +13,11 @@ AZURE_FIRST_PARTY_APPLICATION_CLIENT_ID ?= {{ .firstPartyAppClientId }}
 FPA_CERT_NAME ?= firstPartyCert
 ZONE_NAME ?= {{ .regionalDNSSubdomain }}.{{ .baseDnsZoneName }}
 
+CS_MI_NAME ?= {{ .clusterServiceManagedIdentityName }}
+CS_DB_NAME ?= {{ .clusterServiceDatabaseName }}
+CS_SA_NAME ?= {{ .clusterServiceServiceAccountName }}
+CS_NAMESPACE_NAME ?= {{ .clusterServiceNamespace  }}
+
 DATABASE_DISABLE_TLS ?= {{ not .clusterServicePostgresDeploy }}
 DATABASE_AUTH_METHOD ?= {{ ternary "az-entra" "postgres" .clusterServicePostgresDeploy }}
 DATABASE_SERVER_NAME ?= {{ .clusterServicePostgresName }}

@@ -138,6 +138,11 @@ parameters:
   description: The resource ID of the Azure Container Registry where the OpenShift images are stored.
   required: true
 
+- name: SERVICE_ACCOUNT_NAME
+  description: The name of the service account used to run CS.
+  required: true
+  value: "clusters-service"
+
 
 # These limits are based on the metrics collected in the production environment
 # over the last year. In particular the following Prometheus queries were used
@@ -472,7 +477,7 @@ objects:
 - apiVersion: v1
   kind: ServiceAccount
   metadata:
-    name: clusters-service
+    name: ${SERVICE_ACCOUNT_NAME}
     namespace: ${NAMESPACE}
     labels:
       app: clusters-service
@@ -506,8 +511,8 @@ objects:
           app: clusters-service
           azure.workload.identity/use: "true"
       spec:
-        serviceAccount: clusters-service
-        serviceAccountName: clusters-service
+        serviceAccount: ${SERVICE_ACCOUNT_NAME}
+        serviceAccountName: ${SERVICE_ACCOUNT_NAME}
         volumes:
         - name: service
           secret:

@@ -0,0 +1,143 @@
+defaults:
+  region: {{ .ctx.region }}
+
+  # Resourcegroups
+  globalRG: global-shared-resources
+  regionRG: '{{ .ctx.region }}-shared-resources'
+  serviceClusterRG: hcp-underlay-{{ .ctx.region }}-svc
+  managementClusterRG: hcp-underlay-{{ .ctx.region }}-mgmt-{{ .ctx.stamp }}
+  imageSyncRG: hcp-underlay-imagesync
+
+  # General AKS config
+  kubernetesVersion: 1.30.5
+  istioVersion: "['asm-1-22']"
+  vnetAddressPrefix: "10.128.0.0/14"
+  subnetPrefix: "10.128.8.0/21"
+  podSubnetPrefix: "10.128.64.0/18"
+  aksName: aro-hcp-aks
+
+  # ACR
+  serviceComponentAcrResourceGroups: global-shared-resources
+
+  # SVC cluster specifics
+  svcEtcdKVName: arohcp-etcd-{{ .ctx.regionShort }}
+  svcEtcdKVSoftDelete: true
+
+  # MGMT cluster specifics
+  mgmtEtcdKVName: arohcp-etcd-{{ .ctx.regionShort }}-{{ .ctx.stamp }}
+  mgmtEtcdKVSoftDelete: true
+
+  # Frontend
+  frontendCosmosDBDeploy: true
+  frontendCosmosDBDisableLocalAuth: true
+  frontendCosmosDBName: arohcp-rp-{{ .ctx.regionShort }}
+
+  # Maestro
+  maestroKeyVaultName: arohcp-maestro-{{ .ctx.regionShort }}
+  maestroEventgridName: arohcp-maestro-{{ .ctx.regionShort }}
+  maestroEventGridMaxClientSessionsPerAuthName: '4'
+  maestroCertDomain: 'selfsigned.maestro.keyvault.azure.com'
+  maestroPostgresName: arohcp-maestro-{{ .ctx.regionShort }}
+  maestroPostgresServerVersion: '15'
+  maestroPostgresServerStorageSizeGB: '32'
+  maestroPostgresDeploy: true
+  maestroPostgresPrivate: false
+  maestroRestrictIstioIngress: true
+  maestroConsumerName: hcp-underlay-{{ .ctx.regionShort }}-mgmt-{{ .ctx.stamp }}
+  maestroImageBase: quay.io/redhat-user-workloads/maestro-rhtap-tenant/maestro/maestro
+
+  # Cluster Service
+  clusterServicePostgresName: arohcp-cs-{{ .ctx.regionShort }}
+  clusterServicePostgresDeploy: true
+  clusterServicePostgresPrivate: false
+  clusterServiceAcrRG: global-shared-resources
+
+  # Image Sync
+  imageSyncAcrRG: global-shared-resources
+  imageSyncEnvironmentName: aro-hcp-image-sync
+  imageSyncRepositories: registry.k8s.io/external-dns/external-dns,quay.io/acm-d/rhtap-hypershift-operator,quay.io/app-sre/uhc-clusters-service,quay.io/package-operator/package-operator-package
+  imageSyncImageRepo: image-sync/component-sync
+  imageSyncImageTag: latest
+  ocMirrorImageRepo: image-sync/oc-mirror
+  ocMirrorImageTag: 7abc8af
+
+  # Service KeyVault
+  serviceKeyVaultName: arohcp-svc-{{ .ctx.regionShort }}
+  serviceKeyVaultRG: hcp-underlay-{{ .ctx.regionShort }}
+  serviceKeyVaultRegion: {{ .ctx.region }}
+  serviceKeyVaultSoftDelete: true
+  serviceKeyVaultPrivate: true
+
+  # Management Cluster KV
+  cxKeyVaultName: arohcp-cx-{{ .ctx.regionShort }}-{{ .ctx.stamp }}
+  cxKeyVaultSoftDelete: true
+  cxKeyVaultPrivate: true
+  msiKeyVaultName: arohcp-msi-{{ .ctx.regionShort }}-{{ .ctx.stamp }}
+  msiKeyVaultSoftDelete: true
+  msiKeyVaultPrivate: true
+  mgmtKeyVaultName: arohcp-mgmt-{{ .ctx.regionShort }}-{{ .ctx.stamp }}
+  mgmtKeyVaultSoftDelete: true
+  mgmtKeyVaultPrivate: true
+
+  # OIDC
+  oidcStorageAccountName: arohcpoidc{{ .ctx.regionShort }}
+
+  # DNS
+  baseDnsZoneRG: 'global-shared-resources'
+  regionalDNSSubdomain: '{{ .ctx.region }}'
+
+  # Metrics
+  monitoringWorkspaceName: 'arohcp-{{ .ctx.regionShort }}'
+  grafanaName: 'arohcp-{{ .ctx.regionShort }}'
+  monitoringMsiName: 'aro-hcp-metrics-msi-{{ .ctx.regionShort }}'
+clouds:
+  public:
+    # this configuration serves as a template for for all RH DEV subscription deployments
+    # the following vars need approprivate overrides:
+    defaults:
+      # Maestro
+      maestroPostgresDeploy: false
+      maestroImageTag: ea066c250a002f0cc458711945165591bc9f6d3f
+      # Cluster Service
+      clusterServiceImageTag: a23276d
+      clusterServiceImageRepo: app-sre/uhc-clusters-service
+      # Hypershift Operator
+      hypershiftOperatorImageTag: 99a256f
+      externalDNSImageTag: v0.14.2
+    environments:
+      int:
+        # this is the MSFT INT environment
+        defaults:
+          # Subscriptions
+          serviceClusterSubscription: hcp-{{ ctx.region }}
+          managementClusterSubscription: hcp-{{ ctx.region }}
+          # DNS
+          baseDnsZoneName: aro-hcp.azure-test.net'
+          regionalDNSSubdomain: '{{ .ctx.region }}'
+          # 1P app
+          firstPartyAppClientId: '??? the one used by CS to do first party stuff ???'
+          # disable KV softdelete for easy cleanup and recreate in INT
+          cxKeyVaultSoftDelete: false
+          cxKeyVaultPrivate: false
+          msiKeyVaultSoftDelete: false
+          msiKeyVaultPrivate: false
+          mgmtKeyVaultSoftDelete: false
+          mgmtKeyVaultPrivate: false
+          svcEtcdKVSoftDelete: false
+          mgmtEtcdKVSoftDelete: false
+          # Grafana
+          grafanaAdminGroupPrincipalId: '??? the one to be used as Grafana Admin in grafana.bicep ???'
+          # DEVOPS MSI
+          # lets create this MSI manually for the time being and automate soon
+          # but we should use the MSI name as an input and not the resource ID of the MSI
+          aroDevopsMsiId: '??? the one for OIDC deployment script / lives in the global RG / needs to be created first thing on regional buildout ???'
+          # MGMTM AKS nodepools - big enough for 2 HCPs
+          mgmtSystemAgentPoolMinCount: 1
+          mgmtSystemAgentPoolMaxCount: 4
+          mgmtSystemAgentPoolVmSize: 'Standard_D2s_v3'
+          mgmtSystemAgentPoolOsDiskSizeGB: 32
+          mgmtUserAgentPoolMinCount: 2
+          mgmtUserAgentPoolMaxCount: 12
+          mgmtUserAgentPoolVmSize: 'Standard_D4s_v3'
+          mgmtUserAgentPoolOsDiskSizeGB: 100
+          mgmtUserAgentPoolAzCount: 3
@@ -40,6 +40,10 @@ defaults:
   maestroEventGridMaxClientSessionsPerAuthName: '4'
   maestroCertDomain: 'selfsigned.maestro.keyvault.azure.com'
   maestroPostgresName: arohcp-maestro-{{ .ctx.regionShort }}
+  maestroServerNamespace: maestro
+  maestroServerManagedIdentityName: maestro-server
+  maestroServerServiceAccountName: maestro
+  maestroDatabaseName: maestro
   maestroPostgresServerVersion: '15'
   maestroPostgresServerStorageSizeGB: '32'
   maestroPostgresDeploy: true
@@ -58,6 +62,10 @@ defaults:
   clusterServicePostgresDeploy: true
   clusterServicePostgresPrivate: false
   clusterServiceAcrRG: global
+  clusterServiceNamespace: cluster-service
+  clusterServiceManagedIdentityName: clusters-service
+  clusterServiceServiceAccountName: clusters-service
+  clusterServiceDatabaseName: clusters-service
 
   # Image Sync
   imageSyncAcrRG: global

@@ -5,17 +5,22 @@
   "baseDnsZoneName": "hcp.osadev.cloud",
   "baseDnsZoneRG": "global",
   "clusterServiceAcrRG": "global",
+  "clusterServiceDatabaseName": "clusters-service",
   "clusterServiceImageRepo": "app-sre/uhc-clusters-service",
   "clusterServiceImageTag": "a23276d",
+  "clusterServiceManagedIdentityName": "clusters-service",
+  "clusterServiceNamespace": "cluster-service",
   "clusterServicePostgresDeploy": true,
   "clusterServicePostgresName": "arohcp-cs-cspr",
   "clusterServicePostgresPrivate": false,
+  "clusterServiceServiceAccountName": "clusters-service",
   "cxKeyVaultName": "arohcp-cx-cspr-1",
   "cxKeyVaultPrivate": false,
   "cxKeyVaultSoftDelete": false,
   "externalDNSImageTag": "v0.14.2",
   "externalDNSManagedIdentityName": "external-dns",
   "externalDNSServiceAccountName": "external-dns",
+  "extraVars": {},
   "firstPartyAppClientId": "57e54810-3138-4f38-bd3b-29cb33f4c358",
   "frontendCosmosDBDeploy": true,
   "frontendCosmosDBDisableLocalAuth": true,
@@ -36,6 +41,7 @@
   "kubernetesVersion": "1.30.5",
   "maestroCertDomain": "selfsigned.maestro.keyvault.azure.com",
   "maestroConsumerName": "hcp-underlay-cspr-mgmt-1",
+  "maestroDatabaseName": "maestro",
   "maestroEventGridMaxClientSessionsPerAuthName": "4",
   "maestroEventgridName": "arohcp-maestro-cspr",
   "maestroImageBase": "quay.io/redhat-user-workloads/maestro-rhtap-tenant/maestro/maestro",
@@ -47,6 +53,9 @@
   "maestroPostgresServerStorageSizeGB": "32",
   "maestroPostgresServerVersion": "15",
   "maestroRestrictIstioIngress": false,
+  "maestroServerManagedIdentityName": "maestro-server",
+  "maestroServerNamespace": "maestro",
+  "maestroServerServiceAccountName": "maestro",
   "managementClusterRG": "hcp-underlay-cspr-mgmt-1",
   "managementClusterSubscription": "ARO Hosted Control Planes (EA Subscription 1)",
   "mgmtEtcdKVName": "arohcp-etcd-cspr-1",

@@ -5,17 +5,22 @@
   "baseDnsZoneName": "hcp.osadev.cloud",
   "baseDnsZoneRG": "global",
   "clusterServiceAcrRG": "global",
+  "clusterServiceDatabaseName": "clusters-service",
   "clusterServiceImageRepo": "app-sre/uhc-clusters-service",
   "clusterServiceImageTag": "a23276d",
+  "clusterServiceManagedIdentityName": "clusters-service",
+  "clusterServiceNamespace": "cluster-service",
   "clusterServicePostgresDeploy": true,
   "clusterServicePostgresName": "arohcp-cs-dev",
   "clusterServicePostgresPrivate": false,
+  "clusterServiceServiceAccountName": "clusters-service",
   "cxKeyVaultName": "arohcp-cx-dev-1",
   "cxKeyVaultPrivate": false,
   "cxKeyVaultSoftDelete": false,
   "externalDNSImageTag": "v0.14.2",
   "externalDNSManagedIdentityName": "external-dns",
   "externalDNSServiceAccountName": "external-dns",
+  "extraVars": {},
   "firstPartyAppClientId": "57e54810-3138-4f38-bd3b-29cb33f4c358",
   "frontendCosmosDBDeploy": true,
   "frontendCosmosDBDisableLocalAuth": true,
@@ -36,6 +41,7 @@
   "kubernetesVersion": "1.30.5",
   "maestroCertDomain": "selfsigned.maestro.keyvault.azure.com",
   "maestroConsumerName": "hcp-underlay-dev-mgmt-1",
+  "maestroDatabaseName": "maestro",
   "maestroEventGridMaxClientSessionsPerAuthName": "4",
   "maestroEventgridName": "arohcp-maestro-dev",
   "maestroImageBase": "quay.io/redhat-user-workloads/maestro-rhtap-tenant/maestro/maestro",
@@ -47,6 +53,9 @@
   "maestroPostgresServerStorageSizeGB": "32",
   "maestroPostgresServerVersion": "15",
   "maestroRestrictIstioIngress": true,
+  "maestroServerManagedIdentityName": "maestro-server",
+  "maestroServerNamespace": "maestro",
+  "maestroServerServiceAccountName": "maestro",
   "managementClusterRG": "hcp-underlay-dev-mgmt-1",
   "managementClusterSubscription": "ARO Hosted Control Planes (EA Subscription 1)",
   "mgmtEtcdKVName": "arohcp-etcd-dev-1",