diff --git a/cluster-service/Makefile b/cluster-service/Makefile index 431ae21b4..b08a129e7 100644 --- a/cluster-service/Makefile +++ b/cluster-service/Makefile @@ -4,7 +4,7 @@ $(shell ../templatize.sh $(DEPLOY_ENV) config.tmpl.mk config.mk) include config.mk deploy: deploy-namespace-template deploy-istio-configurations-template ${DB_SECRET_TARGET} - AZURE_CS_MI_CLIENT_ID=$(shell az identity show -g ${RESOURCEGROUP} -n clusters-service --query clientId -o tsv) && \ + AZURE_CS_MI_CLIENT_ID=$(shell az identity show -g ${RESOURCEGROUP} -n ${CS_MI_NAME} --query clientId -o tsv) && \ CS_SERVICE_PRINCIPAL_CREDS_BASE64='$(shell az keyvault secret show --vault-name "${SERVICE_KV}" --name "aro-hcp-dev-sp-cs" | jq .value -r | base64 | tr -d '\n')' && \ TENANT_ID=$(shell az account show --query tenantId --output tsv) && \ OIDC_BLOB_SERVICE_ENDPOINT=$(shell az storage account show -n ${OIDC_STORAGE_ACCOUNT} -g ${RESOURCEGROUP} --query primaryEndpoints.blob -o tsv) && \ @@ -13,38 +13,44 @@ deploy: deploy-namespace-template deploy-istio-configurations-template ${DB_SECR OCP_ACR_URL=$(shell az acr show -n ${OCP_ACR_NAME} --query loginServer -o tsv) && \ OCP_ACR_RESOURCE_ID=$(shell az acr show -n ${OCP_ACR_NAME} --query id -o tsv) && \ oc process --local -f deploy/openshift-templates/arohcp-service-template.yml \ - -p AZURE_CS_MI_CLIENT_ID=$${AZURE_CS_MI_CLIENT_ID} \ - -p TENANT_ID=$${TENANT_ID} \ - -p REGION=${REGION} \ - -p SERVICE_KEYVAULT_NAME=${SERVICE_KV} \ - -p CS_SERVICE_PRINCIPAL_CREDS_BASE64=$${CS_SERVICE_PRINCIPAL_CREDS_BASE64} \ - -p IMAGE_REGISTRY=${ACR_NAME}.azurecr.io \ - -p IMAGE_REPOSITORY=${IMAGE_REPO} \ - -p AZURE_FIRST_PARTY_APPLICATION_CLIENT_ID=${AZURE_FIRST_PARTY_APPLICATION_CLIENT_ID} \ - -p FPA_CERT_NAME=${FPA_CERT_NAME} \ - -p IMAGE_TAG=${IMAGE_TAG} \ - -p OCP_ACR_RESOURCE_ID=$${OCP_ACR_RESOURCE_ID} \ - -p OCP_ACR_URL=$${OCP_ACR_URL} \ - -p DATABASE_DISABLE_TLS=${DATABASE_DISABLE_TLS} \ - -p DATABASE_AUTH_METHOD=${DATABASE_AUTH_METHOD} | oc apply -f - + -p NAMESPACE="${CS_NAMESPACE_NAME}" \ + -p AZURE_CS_MI_CLIENT_ID=$${AZURE_CS_MI_CLIENT_ID} \ + -p TENANT_ID=$${TENANT_ID} \ + -p REGION=${REGION} \ + -p SERVICE_KEYVAULT_NAME=${SERVICE_KV} \ + -p CS_SERVICE_PRINCIPAL_CREDS_BASE64=$${CS_SERVICE_PRINCIPAL_CREDS_BASE64} \ + -p IMAGE_REGISTRY=${ACR_NAME}.azurecr.io \ + -p IMAGE_REPOSITORY=${IMAGE_REPO} \ + -p AZURE_FIRST_PARTY_APPLICATION_CLIENT_ID=${AZURE_FIRST_PARTY_APPLICATION_CLIENT_ID} \ + -p FPA_CERT_NAME=${FPA_CERT_NAME} \ + -p IMAGE_TAG=${IMAGE_TAG} \ + -p OCP_ACR_RESOURCE_ID=$${OCP_ACR_RESOURCE_ID} \ + -p OCP_ACR_URL=$${OCP_ACR_URL} \ + -p SERVICE_ACCOUNT_NAME=${CS_SA_NAME} \ + -p DATABASE_DISABLE_TLS=${DATABASE_DISABLE_TLS} \ + -p DATABASE_AUTH_METHOD=${DATABASE_AUTH_METHOD} | oc apply -f - deploy-namespace-template: ISTO_VERSION=$(shell az aks show -n ${AKS_NAME} -g ${RESOURCEGROUP} --query serviceMeshProfile.istio.revisions[-1] -o tsv) && \ oc process --local -f deploy/openshift-templates/arohcp-namespace-template.yml \ - -p ISTIO_VERSION=$${ISTO_VERSION} | oc apply -f - + -p ISTIO_VERSION=$${ISTO_VERSION} \ + -p NAMESPACE="${CS_NAMESPACE_NAME}" | oc apply -f - deploy-istio-configurations-template: kubectl apply -f deploy/istio.yml deploy-local-db-secret: provision-shard oc process --local -f deploy/openshift-templates/arohcp-secrets-template.yml \ - -p PROVISION_SHARDS_CONFIG="$$( base64 -i deploy/provisioning-shards.yml)" | oc apply -f - - oc process --local -f deploy/openshift-templates/arohcp-db-template.yml | oc apply -f - + -p PROVISION_SHARDS_CONFIG="$$( base64 -i deploy/provisioning-shards.yml)" \ + -p NAMESPACE="${CS_NAMESPACE_NAME}" | oc apply -f - + oc process --local -f deploy/openshift-templates/arohcp-db-template.yml \ + -p NAMESPACE="${CS_NAMESPACE_NAME}" | oc apply -f - deploy-azure-db-secret: provision-shard oc process --local -f deploy/openshift-templates/arohcp-secrets-template.yml \ - -p DATABASE_USER=clusters-service \ - -p DATABASE_NAME=clusters-service \ + -p NAMESPACE="${CS_NAMESPACE_NAME}" \ + -p DATABASE_USER=${CS_MI_NAME} \ + -p DATABASE_NAME=${CS_DB_NAME} \ -p DATABASE_PASSWORD="" \ -p DATABASE_HOST=$(shell az postgres flexible-server show --resource-group ${RESOURCEGROUP} -n ${DATABASE_SERVER_NAME} --query fullyQualifiedDomainName -o tsv) \ -p PROVISION_SHARDS_CONFIG="$$( base64 -i deploy/provisioning-shards.yml)" | oc apply -f - diff --git a/cluster-service/config.tmpl.mk b/cluster-service/config.tmpl.mk index 8628c44aa..8f7bfe4e5 100644 --- a/cluster-service/config.tmpl.mk +++ b/cluster-service/config.tmpl.mk @@ -13,6 +13,11 @@ AZURE_FIRST_PARTY_APPLICATION_CLIENT_ID ?= {{ .firstPartyAppClientId }} FPA_CERT_NAME ?= firstPartyCert ZONE_NAME ?= {{ .regionalDNSSubdomain }}.{{ .baseDnsZoneName }} +CS_MI_NAME ?= {{ .clusterServiceManagedIdentityName }} +CS_DB_NAME ?= {{ .clusterServiceDatabaseName }} +CS_SA_NAME ?= {{ .clusterServiceServiceAccountName }} +CS_NAMESPACE_NAME ?= {{ .clusterServiceNamespace }} + DATABASE_DISABLE_TLS ?= {{ not .clusterServicePostgresDeploy }} DATABASE_AUTH_METHOD ?= {{ ternary "az-entra" "postgres" .clusterServicePostgresDeploy }} DATABASE_SERVER_NAME ?= {{ .clusterServicePostgresName }} diff --git a/cluster-service/deploy/openshift-templates/arohcp-service-template.yml b/cluster-service/deploy/openshift-templates/arohcp-service-template.yml index 3ea72a690..e4a4369ed 100644 --- a/cluster-service/deploy/openshift-templates/arohcp-service-template.yml +++ b/cluster-service/deploy/openshift-templates/arohcp-service-template.yml @@ -138,6 +138,11 @@ parameters: description: The resource ID of the Azure Container Registry where the OpenShift images are stored. required: true +- name: SERVICE_ACCOUNT_NAME + description: The name of the service account used to run CS. + required: true + value: "clusters-service" + # These limits are based on the metrics collected in the production environment # over the last year. In particular the following Prometheus queries were used @@ -472,7 +477,7 @@ objects: - apiVersion: v1 kind: ServiceAccount metadata: - name: clusters-service + name: ${SERVICE_ACCOUNT_NAME} namespace: ${NAMESPACE} labels: app: clusters-service @@ -506,8 +511,8 @@ objects: app: clusters-service azure.workload.identity/use: "true" spec: - serviceAccount: clusters-service - serviceAccountName: clusters-service + serviceAccount: ${SERVICE_ACCOUNT_NAME} + serviceAccountName: ${SERVICE_ACCOUNT_NAME} volumes: - name: service secret: diff --git a/config/config.msft.yaml b/config/config.msft.yaml new file mode 100644 index 000000000..ae9b9463d --- /dev/null +++ b/config/config.msft.yaml @@ -0,0 +1,143 @@ +defaults: + region: {{ .ctx.region }} + + # Resourcegroups + globalRG: global-shared-resources + regionRG: '{{ .ctx.region }}-shared-resources' + serviceClusterRG: hcp-underlay-{{ .ctx.region }}-svc + managementClusterRG: hcp-underlay-{{ .ctx.region }}-mgmt-{{ .ctx.stamp }} + imageSyncRG: hcp-underlay-imagesync + + # General AKS config + kubernetesVersion: 1.30.5 + istioVersion: "['asm-1-22']" + vnetAddressPrefix: "10.128.0.0/14" + subnetPrefix: "10.128.8.0/21" + podSubnetPrefix: "10.128.64.0/18" + aksName: aro-hcp-aks + + # ACR + serviceComponentAcrResourceGroups: global-shared-resources + + # SVC cluster specifics + svcEtcdKVName: arohcp-etcd-{{ .ctx.regionShort }} + svcEtcdKVSoftDelete: true + + # MGMT cluster specifics + mgmtEtcdKVName: arohcp-etcd-{{ .ctx.regionShort }}-{{ .ctx.stamp }} + mgmtEtcdKVSoftDelete: true + + # Frontend + frontendCosmosDBDeploy: true + frontendCosmosDBDisableLocalAuth: true + frontendCosmosDBName: arohcp-rp-{{ .ctx.regionShort }} + + # Maestro + maestroKeyVaultName: arohcp-maestro-{{ .ctx.regionShort }} + maestroEventgridName: arohcp-maestro-{{ .ctx.regionShort }} + maestroEventGridMaxClientSessionsPerAuthName: '4' + maestroCertDomain: 'selfsigned.maestro.keyvault.azure.com' + maestroPostgresName: arohcp-maestro-{{ .ctx.regionShort }} + maestroPostgresServerVersion: '15' + maestroPostgresServerStorageSizeGB: '32' + maestroPostgresDeploy: true + maestroPostgresPrivate: false + maestroRestrictIstioIngress: true + maestroConsumerName: hcp-underlay-{{ .ctx.regionShort }}-mgmt-{{ .ctx.stamp }} + maestroImageBase: quay.io/redhat-user-workloads/maestro-rhtap-tenant/maestro/maestro + + # Cluster Service + clusterServicePostgresName: arohcp-cs-{{ .ctx.regionShort }} + clusterServicePostgresDeploy: true + clusterServicePostgresPrivate: false + clusterServiceAcrRG: global-shared-resources + + # Image Sync + imageSyncAcrRG: global-shared-resources + imageSyncEnvironmentName: aro-hcp-image-sync + imageSyncRepositories: registry.k8s.io/external-dns/external-dns,quay.io/acm-d/rhtap-hypershift-operator,quay.io/app-sre/uhc-clusters-service,quay.io/package-operator/package-operator-package + imageSyncImageRepo: image-sync/component-sync + imageSyncImageTag: latest + ocMirrorImageRepo: image-sync/oc-mirror + ocMirrorImageTag: 7abc8af + + # Service KeyVault + serviceKeyVaultName: arohcp-svc-{{ .ctx.regionShort }} + serviceKeyVaultRG: hcp-underlay-{{ .ctx.regionShort }} + serviceKeyVaultRegion: {{ .ctx.region }} + serviceKeyVaultSoftDelete: true + serviceKeyVaultPrivate: true + + # Management Cluster KV + cxKeyVaultName: arohcp-cx-{{ .ctx.regionShort }}-{{ .ctx.stamp }} + cxKeyVaultSoftDelete: true + cxKeyVaultPrivate: true + msiKeyVaultName: arohcp-msi-{{ .ctx.regionShort }}-{{ .ctx.stamp }} + msiKeyVaultSoftDelete: true + msiKeyVaultPrivate: true + mgmtKeyVaultName: arohcp-mgmt-{{ .ctx.regionShort }}-{{ .ctx.stamp }} + mgmtKeyVaultSoftDelete: true + mgmtKeyVaultPrivate: true + + # OIDC + oidcStorageAccountName: arohcpoidc{{ .ctx.regionShort }} + + # DNS + baseDnsZoneRG: 'global-shared-resources' + regionalDNSSubdomain: '{{ .ctx.region }}' + + # Metrics + monitoringWorkspaceName: 'arohcp-{{ .ctx.regionShort }}' + grafanaName: 'arohcp-{{ .ctx.regionShort }}' + monitoringMsiName: 'aro-hcp-metrics-msi-{{ .ctx.regionShort }}' +clouds: + public: + # this configuration serves as a template for for all RH DEV subscription deployments + # the following vars need approprivate overrides: + defaults: + # Maestro + maestroPostgresDeploy: false + maestroImageTag: ea066c250a002f0cc458711945165591bc9f6d3f + # Cluster Service + clusterServiceImageTag: a23276d + clusterServiceImageRepo: app-sre/uhc-clusters-service + # Hypershift Operator + hypershiftOperatorImageTag: 99a256f + externalDNSImageTag: v0.14.2 + environments: + int: + # this is the MSFT INT environment + defaults: + # Subscriptions + serviceClusterSubscription: hcp-{{ ctx.region }} + managementClusterSubscription: hcp-{{ ctx.region }} + # DNS + baseDnsZoneName: aro-hcp.azure-test.net' + regionalDNSSubdomain: '{{ .ctx.region }}' + # 1P app + firstPartyAppClientId: '??? the one used by CS to do first party stuff ???' + # disable KV softdelete for easy cleanup and recreate in INT + cxKeyVaultSoftDelete: false + cxKeyVaultPrivate: false + msiKeyVaultSoftDelete: false + msiKeyVaultPrivate: false + mgmtKeyVaultSoftDelete: false + mgmtKeyVaultPrivate: false + svcEtcdKVSoftDelete: false + mgmtEtcdKVSoftDelete: false + # Grafana + grafanaAdminGroupPrincipalId: '??? the one to be used as Grafana Admin in grafana.bicep ???' + # DEVOPS MSI + # lets create this MSI manually for the time being and automate soon + # but we should use the MSI name as an input and not the resource ID of the MSI + aroDevopsMsiId: '??? the one for OIDC deployment script / lives in the global RG / needs to be created first thing on regional buildout ???' + # MGMTM AKS nodepools - big enough for 2 HCPs + mgmtSystemAgentPoolMinCount: 1 + mgmtSystemAgentPoolMaxCount: 4 + mgmtSystemAgentPoolVmSize: 'Standard_D2s_v3' + mgmtSystemAgentPoolOsDiskSizeGB: 32 + mgmtUserAgentPoolMinCount: 2 + mgmtUserAgentPoolMaxCount: 12 + mgmtUserAgentPoolVmSize: 'Standard_D4s_v3' + mgmtUserAgentPoolOsDiskSizeGB: 100 + mgmtUserAgentPoolAzCount: 3 diff --git a/config/config.yaml b/config/config.yaml index 5eba0d0a3..83dd32251 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -40,6 +40,10 @@ defaults: maestroEventGridMaxClientSessionsPerAuthName: '4' maestroCertDomain: 'selfsigned.maestro.keyvault.azure.com' maestroPostgresName: arohcp-maestro-{{ .ctx.regionShort }} + maestroServerNamespace: maestro + maestroServerManagedIdentityName: maestro-server + maestroServerServiceAccountName: maestro + maestroDatabaseName: maestro maestroPostgresServerVersion: '15' maestroPostgresServerStorageSizeGB: '32' maestroPostgresDeploy: true @@ -58,6 +62,10 @@ defaults: clusterServicePostgresDeploy: true clusterServicePostgresPrivate: false clusterServiceAcrRG: global + clusterServiceNamespace: cluster-service + clusterServiceManagedIdentityName: clusters-service + clusterServiceServiceAccountName: clusters-service + clusterServiceDatabaseName: clusters-service # Image Sync imageSyncAcrRG: global diff --git a/config/public-cloud-cs-pr.json b/config/public-cloud-cs-pr.json index 9545a84ff..4d7525da9 100644 --- a/config/public-cloud-cs-pr.json +++ b/config/public-cloud-cs-pr.json @@ -5,17 +5,22 @@ "baseDnsZoneName": "hcp.osadev.cloud", "baseDnsZoneRG": "global", "clusterServiceAcrRG": "global", + "clusterServiceDatabaseName": "clusters-service", "clusterServiceImageRepo": "app-sre/uhc-clusters-service", "clusterServiceImageTag": "a23276d", + "clusterServiceManagedIdentityName": "clusters-service", + "clusterServiceNamespace": "cluster-service", "clusterServicePostgresDeploy": true, "clusterServicePostgresName": "arohcp-cs-cspr", "clusterServicePostgresPrivate": false, + "clusterServiceServiceAccountName": "clusters-service", "cxKeyVaultName": "arohcp-cx-cspr-1", "cxKeyVaultPrivate": false, "cxKeyVaultSoftDelete": false, "externalDNSImageTag": "v0.14.2", "externalDNSManagedIdentityName": "external-dns", "externalDNSServiceAccountName": "external-dns", + "extraVars": {}, "firstPartyAppClientId": "57e54810-3138-4f38-bd3b-29cb33f4c358", "frontendCosmosDBDeploy": true, "frontendCosmosDBDisableLocalAuth": true, @@ -36,6 +41,7 @@ "kubernetesVersion": "1.30.5", "maestroCertDomain": "selfsigned.maestro.keyvault.azure.com", "maestroConsumerName": "hcp-underlay-cspr-mgmt-1", + "maestroDatabaseName": "maestro", "maestroEventGridMaxClientSessionsPerAuthName": "4", "maestroEventgridName": "arohcp-maestro-cspr", "maestroImageBase": "quay.io/redhat-user-workloads/maestro-rhtap-tenant/maestro/maestro", @@ -47,6 +53,9 @@ "maestroPostgresServerStorageSizeGB": "32", "maestroPostgresServerVersion": "15", "maestroRestrictIstioIngress": false, + "maestroServerManagedIdentityName": "maestro-server", + "maestroServerNamespace": "maestro", + "maestroServerServiceAccountName": "maestro", "managementClusterRG": "hcp-underlay-cspr-mgmt-1", "managementClusterSubscription": "ARO Hosted Control Planes (EA Subscription 1)", "mgmtEtcdKVName": "arohcp-etcd-cspr-1", diff --git a/config/public-cloud-dev.json b/config/public-cloud-dev.json index b642989bc..db4b0042d 100644 --- a/config/public-cloud-dev.json +++ b/config/public-cloud-dev.json @@ -5,17 +5,22 @@ "baseDnsZoneName": "hcp.osadev.cloud", "baseDnsZoneRG": "global", "clusterServiceAcrRG": "global", + "clusterServiceDatabaseName": "clusters-service", "clusterServiceImageRepo": "app-sre/uhc-clusters-service", "clusterServiceImageTag": "a23276d", + "clusterServiceManagedIdentityName": "clusters-service", + "clusterServiceNamespace": "cluster-service", "clusterServicePostgresDeploy": true, "clusterServicePostgresName": "arohcp-cs-dev", "clusterServicePostgresPrivate": false, + "clusterServiceServiceAccountName": "clusters-service", "cxKeyVaultName": "arohcp-cx-dev-1", "cxKeyVaultPrivate": false, "cxKeyVaultSoftDelete": false, "externalDNSImageTag": "v0.14.2", "externalDNSManagedIdentityName": "external-dns", "externalDNSServiceAccountName": "external-dns", + "extraVars": {}, "firstPartyAppClientId": "57e54810-3138-4f38-bd3b-29cb33f4c358", "frontendCosmosDBDeploy": true, "frontendCosmosDBDisableLocalAuth": true, @@ -36,6 +41,7 @@ "kubernetesVersion": "1.30.5", "maestroCertDomain": "selfsigned.maestro.keyvault.azure.com", "maestroConsumerName": "hcp-underlay-dev-mgmt-1", + "maestroDatabaseName": "maestro", "maestroEventGridMaxClientSessionsPerAuthName": "4", "maestroEventgridName": "arohcp-maestro-dev", "maestroImageBase": "quay.io/redhat-user-workloads/maestro-rhtap-tenant/maestro/maestro", @@ -47,6 +53,9 @@ "maestroPostgresServerStorageSizeGB": "32", "maestroPostgresServerVersion": "15", "maestroRestrictIstioIngress": true, + "maestroServerManagedIdentityName": "maestro-server", + "maestroServerNamespace": "maestro", + "maestroServerServiceAccountName": "maestro", "managementClusterRG": "hcp-underlay-dev-mgmt-1", "managementClusterSubscription": "ARO Hosted Control Planes (EA Subscription 1)", "mgmtEtcdKVName": "arohcp-etcd-dev-1", diff --git a/config/public-cloud-personal-dev.json b/config/public-cloud-personal-dev.json index a5845caba..4b28d154e 100644 --- a/config/public-cloud-personal-dev.json +++ b/config/public-cloud-personal-dev.json @@ -5,17 +5,22 @@ "baseDnsZoneName": "hcp.osadev.cloud", "baseDnsZoneRG": "global", "clusterServiceAcrRG": "global", + "clusterServiceDatabaseName": "clusters-service", "clusterServiceImageRepo": "app-sre/uhc-clusters-service", "clusterServiceImageTag": "a23276d", + "clusterServiceManagedIdentityName": "clusters-service", + "clusterServiceNamespace": "cluster-service", "clusterServicePostgresDeploy": false, "clusterServicePostgresName": "arohcp-cs-usw3tst", "clusterServicePostgresPrivate": false, + "clusterServiceServiceAccountName": "clusters-service", "cxKeyVaultName": "arohcp-cx-usw3tst-1", "cxKeyVaultPrivate": false, "cxKeyVaultSoftDelete": false, "externalDNSImageTag": "v0.14.2", "externalDNSManagedIdentityName": "external-dns", "externalDNSServiceAccountName": "external-dns", + "extraVars": {}, "firstPartyAppClientId": "57e54810-3138-4f38-bd3b-29cb33f4c358", "frontendCosmosDBDeploy": true, "frontendCosmosDBDisableLocalAuth": true, @@ -36,6 +41,7 @@ "kubernetesVersion": "1.30.5", "maestroCertDomain": "selfsigned.maestro.keyvault.azure.com", "maestroConsumerName": "hcp-underlay-usw3tst-mgmt-1", + "maestroDatabaseName": "maestro", "maestroEventGridMaxClientSessionsPerAuthName": "4", "maestroEventgridName": "arohcp-maestro-usw3tst", "maestroImageBase": "quay.io/redhat-user-workloads/maestro-rhtap-tenant/maestro/maestro", @@ -47,6 +53,9 @@ "maestroPostgresServerStorageSizeGB": "32", "maestroPostgresServerVersion": "15", "maestroRestrictIstioIngress": true, + "maestroServerManagedIdentityName": "maestro-server", + "maestroServerNamespace": "maestro", + "maestroServerServiceAccountName": "maestro", "managementClusterRG": "hcp-underlay-usw3tst-mgmt-1", "managementClusterSubscription": "ARO Hosted Control Planes (EA Subscription 1)", "mgmtEtcdKVName": "arohcp-etcd-usw3tst-1", diff --git a/dev-infrastructure/Makefile b/dev-infrastructure/Makefile index b21c95600..5a3bef22b 100644 --- a/dev-infrastructure/Makefile +++ b/dev-infrastructure/Makefile @@ -403,20 +403,14 @@ acr-ocp.what-if: global.rg # Postgres Authentication Helpers # -cs-current-user-pg-connect: - @scripts/cs-current-user-pg-connect.sh $(SVC_RESOURCEGROUP) $(CS_PG_NAME) -.PHONY: cs-current-user-pg-connect - cs-miwi-pg-connect: - @scripts/cs-miwi-pg-connect.sh $(SVC_RESOURCEGROUP) $(CS_PG_NAME) "clusters-service" "cluster-service" "clusters-service" + @KUBECONFIG=${SVC_KUBECONFIG_FILE} && \ + scripts/miwi-pg-connect.sh $(SVC_RESOURCEGROUP) $(CS_PG_NAME) $(CS_MI_NAME) $(CS_NAMESPACE_NAME) $(CS_SA_NAME) $(CS_DB_NAME) .PHONY: cs-miwi-pg-connect -maestro-current-user-pg-connect: - @scripts/cs-current-user-pg-connect.sh $(SVC_RESOURCEGROUP) $(MAESTRO_PG_NAME) -.PHONY: maestro-current-user-pg-connect - maestro-miwi-pg-connect: - @scripts/cs-miwi-pg-connect.sh $(SVC_RESOURCEGROUP) $(MAESTRO_PG_NAME) "maestro-server" "maestro" "maestro" + @KUBECONFIG=${SVC_KUBECONFIG_FILE} && \ + scripts/miwi-pg-connect.sh $(SVC_RESOURCEGROUP) $(MAESTRO_PG_NAME) $(MAESTRO_MI_NAME) $(MAESTRO_NAMESPACE_NAME) $(MAESTRO_SA_NAME) $(MAESTRO_DB_NAME) .PHONY: maestro-miwi-pg-connect # diff --git a/dev-infrastructure/config.tmpl.mk b/dev-infrastructure/config.tmpl.mk index bebf2beea..a05d179d4 100644 --- a/dev-infrastructure/config.tmpl.mk +++ b/dev-infrastructure/config.tmpl.mk @@ -9,5 +9,15 @@ IMAGE_SYNC_ENVIRONMENT ?= {{ .imageSyncEnvironmentName }} ARO_HCP_IMAGE_ACR ?= {{ .svcAcrName }} REPOSITORIES_TO_SYNC ?= '{{ .imageSyncRepositories }}' AKS_NAME ?= {{ .aksName }} + CS_PG_NAME ?= {{ .clusterServicePostgresName }} +CS_MI_NAME ?= {{ .clusterServiceManagedIdentityName }} +CS_NAMESPACE_NAME ?= {{ .clusterServiceNamespace }} +CS_SA_NAME = {{ .clusterServiceServiceAccountName }} +CS_DB_NAME ?= {{ .clusterServiceDatabaseName }} + MAESTRO_PG_NAME ?= {{ .maestroPostgresName }} +MAESTRO_MI_NAME ?= {{ .maestroServerManagedIdentityName }} +MAESTRO_NAMESPACE_NAME ?= {{ .maestroServerNamespace }} +MAESTRO_SA_NAME = {{ .maestroServerServiceAccountName }} +MAESTRO_DB_NAME ?= {{ .maestroDatabaseName }} diff --git a/dev-infrastructure/configurations/svc-cluster.tmpl.bicepparam b/dev-infrastructure/configurations/svc-cluster.tmpl.bicepparam index 6a3ac77fe..e2d68b1fe 100644 --- a/dev-infrastructure/configurations/svc-cluster.tmpl.bicepparam +++ b/dev-infrastructure/configurations/svc-cluster.tmpl.bicepparam @@ -17,14 +17,22 @@ param maestroKeyVaultName = '{{ .maestroKeyVaultName }}' param maestroEventGridNamespacesName = '{{ .maestroEventgridName }}' param maestroCertDomain = '{{ .maestroCertDomain}}' param maestroPostgresServerName = '{{ .maestroPostgresName }}' +param maestroDatabaseName = '{{ .maestroDatabaseName }}' param maestroPostgresServerVersion = '{{ .maestroPostgresServerVersion }}' param maestroPostgresServerStorageSizeGB = {{ .maestroPostgresServerStorageSizeGB }} param deployMaestroPostgres = {{ .maestroPostgresDeploy }} param maestroPostgresPrivate = {{ .maestroPostgresPrivate }} +param maestroMIName = '{{ .maestroServerManagedIdentityName }}' +param maestroNamespace = '{{ .maestroServerNamespace }}' +param maestroServiceAccountName = '{{ .maestroServerServiceAccountName }}' param deployCsInfra = {{ .clusterServicePostgresDeploy }} param csPostgresServerName = '{{ .clusterServicePostgresName }}' -param clusterServicePostgresPrivate = {{ .clusterServicePostgresPrivate }} +param csDatabaseName = '{{ .clusterServiceDatabaseName }}' +param csPostgresPrivate = {{ .clusterServicePostgresPrivate }} +param csMIName = '{{ .clusterServiceManagedIdentityName }}' +param csNamespace = '{{ .clusterServiceNamespace }}' +param csServiceAccountName = '{{ .clusterServiceServiceAccountName }}' param serviceKeyVaultName = '{{ .serviceKeyVaultName }}' param serviceKeyVaultResourceGroup = '{{ .serviceKeyVaultRG }}' diff --git a/dev-infrastructure/docs/development-setup.md b/dev-infrastructure/docs/development-setup.md index 662a3ef49..c42e41e48 100644 --- a/dev-infrastructure/docs/development-setup.md +++ b/dev-infrastructure/docs/development-setup.md @@ -448,20 +448,10 @@ To create a cluster in CS using a locally running Frontend, see the frontend [RE ### Access Maestro Postgres from outside of the AKS cluster -To connect to the database as current user run +To connect to the database with the managed identity of Maestro, run the following command ```sh - eval $(AKSCONFIG=svc-cluster make maestro-current-user-pg-connect) - psql -d maestro - ``` - -The output of the make target is in ENV var format for the `psql` tool, so this works to get a connection into the DB. - -To connect to the database with the managed identity of Maestro, make sure to have a KUBECONFIG for the cluster that runs Maestro Server and run - - ```sh - eval $(AKSCONFIG=svc-cluster make maestro-miwi-pg-connect) - psql -d maestro + make maestro-miwi-pg-connect ``` Once logged in, verify the connection with `\conninfo` @@ -474,20 +464,10 @@ To create a Postgres DB on Azure enabled for Entra authentication, a svc cluster ### Access the database from outside of the AKS cluster -To connect to the database as current user run - - ```sh - eval $(make cs-current-user-pg-connect) - psql -d clusters-service - ``` - -The output of the make target is in ENV var format for the `psql` tool, so this works to get a connection into the DB. - -To connect to the database with the managed identity of CS, make sure to have a KUBECONFIG for the cluster that runs CS and run +To connect to the database with the managed identity of CS, run the following command ```sh - eval $(make cs-miwi-pg-connect) - psql -d clusters-service + make cs-miwi-pg-connect ``` Once logged in, verify the connection with `\conninfo` diff --git a/dev-infrastructure/modules/cluster-service.bicep b/dev-infrastructure/modules/cluster-service.bicep index ed871f47e..66d8b6f98 100644 --- a/dev-infrastructure/modules/cluster-service.bicep +++ b/dev-infrastructure/modules/cluster-service.bicep @@ -8,7 +8,7 @@ param clusterServiceManagedIdentityName string param clusterServiceManagedIdentityPrincipalId string @description('The name of the database to create for CS') -param csDatabaseName string = 'clusters-service' +param csDatabaseName string @description('The name of the Postgres server for CS') param postgresServerName string diff --git a/dev-infrastructure/modules/maestro/maestro-server.bicep b/dev-infrastructure/modules/maestro/maestro-server.bicep index 1cb940821..1012d3a65 100644 --- a/dev-infrastructure/modules/maestro/maestro-server.bicep +++ b/dev-infrastructure/modules/maestro/maestro-server.bicep @@ -48,7 +48,7 @@ param privateEndpointSubnetId string = '' param privateEndpointVnetId string = '' @description('The name of the database to create for Maestro') -param maestroDatabaseName string = 'maestro' +param maestroDatabaseName string @description('The name of the Managed Identity for the Maestro cluster service') param maestroServerManagedIdentityName string diff --git a/dev-infrastructure/scripts/cs-current-user-pg-connect.sh b/dev-infrastructure/scripts/cs-current-user-pg-connect.sh deleted file mode 100755 index da42c6ee2..000000000 --- a/dev-infrastructure/scripts/cs-current-user-pg-connect.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/sh - -RESOURCEGROUP=$1 -DB_SERVER_NAME_PREFIX=$2 - -CURRENTUSER=$(az ad signed-in-user show -o json | jq -r '.id') -CURRENTUSER_NAME=$(az ad signed-in-user show -o json | jq -r '.userPrincipalName') - -CS_DB=$(az postgres flexible-server list -g ${RESOURCEGROUP} -o json | jq --arg prefix "${DB_SERVER_NAME_PREFIX}" '.[] | select(.name | startswith($prefix))') -CS_DB_NAME=$(echo ${CS_DB} | jq -r .name) - -ALREADY_ADMIN=$(az postgres flexible-server ad-admin list -g ${RESOURCEGROUP} -s ${CS_DB_NAME} -o json | jq --arg principalname "${CURRENTUSER_NAME}" '[.[] | select(.principalName == $principalname)] | length') -if [ $ALREADY_ADMIN -eq 0 ]; then - az postgres flexible-server ad-admin create --server-name ${CS_DB_NAME} --resource-group ${RESOURCEGROUP} --object-id ${CURRENTUSER} --display-name ${CURRENTUSER_NAME} -fi - -echo export PGHOST=$(echo ${CS_DB} | jq -r .fullyQualifiedDomainName) -echo export PGUSER=$CURRENTUSER_NAME -echo export PGPASSWORD=$(az account get-access-token --resource='https://ossrdbms-aad.database.windows.net' -o json | jq .accessToken -r) diff --git a/dev-infrastructure/scripts/cs-miwi-pg-connect.sh b/dev-infrastructure/scripts/cs-miwi-pg-connect.sh deleted file mode 100755 index 507f45f42..000000000 --- a/dev-infrastructure/scripts/cs-miwi-pg-connect.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash - -RESOURCEGROUP=$1 -DB_SERVER_NAME_PREFIX=$2 -MANAGED_IDENTITY_NAME=$3 -NAMESPACE=$4 -SA_NAME=$5 - -# prep creds and configs -PGHOST=$(az postgres flexible-server list --resource-group ${RESOURCEGROUP} --query "[?starts_with(name, '${DB_SERVER_NAME_PREFIX}')].fullyQualifiedDomainName" -o tsv) -AZURE_TENANT_ID=$(az account show -o json | jq .homeTenantId -r) -AZURE_CLIENT_ID=$(az identity show -g ${RESOURCEGROUP} -n ${MANAGED_IDENTITY_NAME} --query clientId -o tsv) -SA_TOKEN=$(kubectl create token ${SA_NAME} --namespace=${NAMESPACE} --audience api://AzureADTokenExchange) - -# az login with managed identity via SA token -export AZURE_CONFIG_DIR="${HOME}/.azure-profile-cs-${RESOURCEGROUP}" -rm -rf $AZURE_CONFIG_DIR -az login --federated-token ${SA_TOKEN} --service-principal -u $AZURE_CLIENT_ID -t $AZURE_TENANT_ID > /dev/null 2>&1 - -# get tmp DB password -PGPASSWORD=$(az account get-access-token --resource='https://ossrdbms-aad.database.windows.net' -o json | jq .accessToken -r) -rm -rf $AZURE_CONFIG_DIR - -echo export PGHOST=${PGHOST} -echo export PGUSER=${MANAGED_IDENTITY_NAME} -echo export PGPASSWORD=${PGPASSWORD} diff --git a/dev-infrastructure/scripts/miwi-pg-connect.sh b/dev-infrastructure/scripts/miwi-pg-connect.sh new file mode 100755 index 000000000..8f1b147ad --- /dev/null +++ b/dev-infrastructure/scripts/miwi-pg-connect.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +RESOURCEGROUP=$1 +DB_SERVER_NAME_PREFIX=$2 +MANAGED_IDENTITY_NAME=$3 +NAMESPACE=$4 +SA_NAME=$5 +DB_NAME=$6 + +# prep creds and configs +export PGHOST=$(az postgres flexible-server list --resource-group ${RESOURCEGROUP} --query "[?starts_with(name, '${DB_SERVER_NAME_PREFIX}')].fullyQualifiedDomainName" -o tsv) +AZURE_TENANT_ID=$(az account show -o json | jq .homeTenantId -r) +AZURE_CLIENT_ID=$(az identity show -g ${RESOURCEGROUP} -n ${MANAGED_IDENTITY_NAME} --query clientId -o tsv) +SA_TOKEN=$(kubectl create token ${SA_NAME} --duration=1h --namespace=${NAMESPACE} --audience api://AzureADTokenExchange) + +# az login with managed identity via SA token +export AZURE_CONFIG_DIR="${HOME}/.azure-profile-cs-${RESOURCEGROUP}" +rm -rf $AZURE_CONFIG_DIR +az login --federated-token ${SA_TOKEN} --service-principal -u $AZURE_CLIENT_ID -t $AZURE_TENANT_ID > /dev/null 2>&1 + +# get tmp DB password +export PGPASSWORD=$(az account get-access-token --resource='https://ossrdbms-aad.database.windows.net' -o json | jq .accessToken -r) +rm -rf $AZURE_CONFIG_DIR + +export PGUSER=${MANAGED_IDENTITY_NAME} +podman run -it --rm \ + --env PGHOST \ + --env PGUSER \ + --env PGPASSWORD \ + postgres \ + psql \ + -d $DB_NAME -p 5432 diff --git a/dev-infrastructure/templates/svc-cluster.bicep b/dev-infrastructure/templates/svc-cluster.bicep index a21a90bc0..a736afa06 100644 --- a/dev-infrastructure/templates/svc-cluster.bicep +++ b/dev-infrastructure/templates/svc-cluster.bicep @@ -73,8 +73,20 @@ param deployCsInfra bool @maxLength(60) param csPostgresServerName string +@description('The name of the database to create for CS') +param csDatabaseName string + @description('If true, make the CS Postgres instance private') -param clusterServicePostgresPrivate bool = true +param csPostgresPrivate bool = true + +@description('The name of the cluster service managed identity') +param csMIName string + +@description('The name of the cluster service namespace') +param csNamespace string + +@description('The name of the cluster service service account') +param csServiceAccountName string @description('Deploy ARO HCP Maestro Postgres if true') param deployMaestroPostgres bool = true @@ -86,12 +98,24 @@ param maestroPostgresPrivate bool = true @maxLength(60) param maestroPostgresServerName string +@description('The name of the database to create for Maestro') +param maestroDatabaseName string + @description('The version of the Postgres server for Maestro') param maestroPostgresServerVersion string @description('The size of the Postgres server for Maestro') param maestroPostgresServerStorageSizeGB int +@description('The name of the maestro managed identity') +param maestroMIName string + +@description('The name of themaestro namespace') +param maestroNamespace string + +@description('The name of the maestro service account') +param maestroServiceAccountName string + @description('The name of the service keyvault') param serviceKeyVaultName string @@ -125,8 +149,6 @@ param aroDevopsMsiId string @description('This is a regional DNS zone') param regionalDNSZoneName string -var clusterServiceMIName = 'clusters-service' - // Tags the resource group resource subscriptionTags 'Microsoft.Resources/tags@2024-03-01' = { name: 'default' @@ -168,14 +190,14 @@ module svcCluster '../modules/aks-cluster-base.bicep' = { serviceAccountName: 'backend' } maestro_wi: { - uamiName: 'maestro-server' - namespace: 'maestro' - serviceAccountName: 'maestro' + uamiName: maestroMIName + namespace: maestroNamespace + serviceAccountName: maestroServiceAccountName } cs_wi: { - uamiName: clusterServiceMIName - namespace: 'cluster-service' - serviceAccountName: 'clusters-service' + uamiName: csMIName + namespace: csNamespace + serviceAccountName: csServiceAccountName } image_sync_wi: { uamiName: 'image-sync' @@ -224,6 +246,7 @@ module maestroServer '../modules/maestro/maestro-server.bicep' = { postgresServerName: maestroPostgresServerName postgresServerVersion: maestroPostgresServerVersion postgresServerStorageSizeGB: maestroPostgresServerStorageSizeGB + maestroDatabaseName: maestroDatabaseName privateEndpointSubnetId: svcCluster.outputs.aksNodeSubnetId privateEndpointVnetId: svcCluster.outputs.aksVnetId postgresServerPrivate: maestroPostgresPrivate @@ -273,21 +296,19 @@ module serviceKeyVaultPrivateEndpoint '../modules/private-endpoint.bicep' = { // C L U S T E R S E R V I C E // -var csManagedIdentityPrincipalId = filter( - svcCluster.outputs.userAssignedIdentities, - id => id.uamiName == clusterServiceMIName -)[0].uamiPrincipalID +var csManagedIdentityPrincipalId = filter(svcCluster.outputs.userAssignedIdentities, id => id.uamiName == csMIName)[0].uamiPrincipalID module cs '../modules/cluster-service.bicep' = if (deployCsInfra) { name: 'cluster-service' params: { location: location postgresServerName: csPostgresServerName + csDatabaseName: csDatabaseName privateEndpointSubnetId: svcCluster.outputs.aksNodeSubnetId privateEndpointVnetId: svcCluster.outputs.aksVnetId - postgresServerPrivate: clusterServicePostgresPrivate + postgresServerPrivate: csPostgresPrivate clusterServiceManagedIdentityPrincipalId: csManagedIdentityPrincipalId - clusterServiceManagedIdentityName: clusterServiceMIName + clusterServiceManagedIdentityName: csMIName } dependsOn: [ maestroServer @@ -379,14 +400,14 @@ module acrManageTokenRole '../modules/acr-permissions.bicep' = [ } ] -// oidc +// O I D C module oidc '../modules/oidc/main.bicep' = { name: '${deployment().name}-oidc' params: { location: location storageAccountName: oidcStorageAccountName - rpMsiName: clusterServiceMIName + rpMsiName: csMIName skuName: oidcStorageAccountSku aroDevopsMsiId: aroDevopsMsiId deploymentScriptLocation: location diff --git a/maestro/registration/Makefile b/maestro/registration/Makefile index d62b33184..700006415 100644 --- a/maestro/registration/Makefile +++ b/maestro/registration/Makefile @@ -4,11 +4,11 @@ $(shell ../../templatize.sh $(DEPLOY_ENV) config.tmpl.mk config.mk) include config.mk deploy: - @if ! kubectl get service maestro -n maestro > /dev/null 2>&1; then \ - echo "Error: Service 'maestro' not found in namespace 'maestro'"; \ + @if ! kubectl get service maestro -n ${MAESTRO_NAMESPACE_NAME} > /dev/null 2>&1; then \ + echo "Error: Service 'maestro' not found in namespace '${CONSUMER_NAME}'"; \ exit 1; \ fi helm upgrade --install ${CONSUMER_NAME} ./helm \ - --namespace maestro \ + --namespace ${MAESTRO_NAMESPACE_NAME} \ --set consumerName=${CONSUMER_NAME} .PHONY: deploy diff --git a/maestro/registration/config.tmpl.mk b/maestro/registration/config.tmpl.mk index 8918876a7..020b96006 100644 --- a/maestro/registration/config.tmpl.mk +++ b/maestro/registration/config.tmpl.mk @@ -1 +1,2 @@ CONSUMER_NAME ?= {{ .maestroConsumerName }} +MAESTRO_NAMESPACE_NAME ?= {{ .maestroServerNamespace }} diff --git a/maestro/server/Makefile b/maestro/server/Makefile index 2f97063fb..14a8fc501 100644 --- a/maestro/server/Makefile +++ b/maestro/server/Makefile @@ -4,15 +4,18 @@ $(shell ../../templatize.sh $(DEPLOY_ENV) config.tmpl.mk config.mk) include config.mk deploy: - kubectl create namespace maestro --dry-run=client -o json | kubectl apply -f - + kubectl create namespace ${MAESTRO_NAMESPACE_NAME} --dry-run=client -o json | kubectl apply -f - ISTO_VERSION=$(shell az aks show -n ${AKS_NAME} -g ${SVC_RG} --query serviceMeshProfile.istio.revisions[-1] -o tsv) && \ kubectl label namespace maestro "istio.io/rev=$${ISTO_VERSION}" --overwrite=true EVENTGRID_HOSTNAME=$(shell az resource show -n ${EVENTGRID_NAME} -g ${REGION_RG} --resource-type "Microsoft.EventGrid/namespaces" --query properties.topicSpacesConfiguration.hostname -o tsv) && \ TENANT_ID=$(shell az account show --query tenantId --output tsv) && \ - MAESTRO_MI_CLIENT_ID=$(shell az identity show -g "${SVC_RG}" -n maestro-server --query clientId -o tsv) && \ + MAESTRO_MI_CLIENT_ID=$(shell az identity show -g "${SVC_RG}" -n ${MAESTRO_MI_NAME} --query clientId -o tsv) && \ helm upgrade --install maestro-server ./helm \ - --namespace maestro \ + --namespace ${MAESTRO_NAMESPACE_NAME} \ --set broker.host=$${EVENTGRID_HOSTNAME} \ + --set maestro.serviceAccount=${MAESTRO_SA_NAME} \ + --set clusterService.namespace=${CS_NAMESPACE_NAME} \ + --set clusterService.serviceAccount=${CS_SA_NAME} \ --set credsKeyVault.name=${KEYVAULT_NAME} \ --set azure.clientId=$${MAESTRO_MI_CLIENT_ID} \ --set azure.tenantId=$${TENANT_ID} \ diff --git a/maestro/server/config.tmpl.mk b/maestro/server/config.tmpl.mk index 31acff9d0..778e8bee0 100644 --- a/maestro/server/config.tmpl.mk +++ b/maestro/server/config.tmpl.mk @@ -8,3 +8,10 @@ USE_CONTAINERIZED_DB ?= {{ not .maestroPostgresDeploy }} USE_DATABASE_SSL ?= {{ ternary "enable" "disable" .maestroPostgresDeploy }} ISTIO_RESTRICT_INGRESS ?= {{ .maestroRestrictIstioIngress }} KEYVAULT_NAME ?= {{ .maestroKeyVaultName }} + +MAESTRO_NAMESPACE_NAME ?= {{ .maestroServerNamespace }} +MAESTRO_SA_NAME = {{ .maestroServerServiceAccountName }} +MAESTRO_MI_NAME ?= {{ .maestroServerManagedIdentityName }} + +CS_NAMESPACE_NAME ?= {{ .clusterServiceNamespace }} +CS_SA_NAME = {{ .clusterServiceServiceAccountName }}