From 1ffd928fe8fe77460b1269679cb417434c095e9f Mon Sep 17 00:00:00 2001 From: Gerd Oberlechner Date: Wed, 30 Oct 2024 09:42:47 +0100 Subject: [PATCH] use config templates for maestro and CS DB configuration * installing the psql client tool locally brings a log of dependencies on most systems. by using a container we avoid having to install too many things on developer machines * parameterize all fields required for CS and Maestro DB setup so we can have consistency between infra RBAC deployment, service deployment and DB deployment Signed-off-by: Gerd Oberlechner --- cluster-service/Makefile | 46 +++++++++------- cluster-service/config.tmpl.mk | 5 ++ .../arohcp-service-template.yml | 11 +++- config/config.yaml | 8 +++ config/public-cloud-cs-pr.json | 8 +++ config/public-cloud-dev.json | 8 +++ config/public-cloud-personal-dev.json | 8 +++ dev-infrastructure/Makefile | 14 ++--- dev-infrastructure/config.tmpl.mk | 10 ++++ .../svc-cluster.tmpl.bicepparam | 10 +++- dev-infrastructure/docs/development-setup.md | 28 ++-------- .../modules/cluster-service.bicep | 2 +- .../modules/maestro/maestro-server.bicep | 2 +- .../scripts/cs-current-user-pg-connect.sh | 19 ------- .../scripts/cs-miwi-pg-connect.sh | 26 --------- dev-infrastructure/scripts/miwi-pg-connect.sh | 32 +++++++++++ .../templates/svc-cluster.bicep | 55 +++++++++++++------ maestro/registration/Makefile | 6 +- maestro/registration/config.tmpl.mk | 1 + maestro/server/Makefile | 9 ++- maestro/server/config.tmpl.mk | 7 +++ 21 files changed, 187 insertions(+), 128 deletions(-) delete mode 100755 dev-infrastructure/scripts/cs-current-user-pg-connect.sh delete mode 100755 dev-infrastructure/scripts/cs-miwi-pg-connect.sh create mode 100755 dev-infrastructure/scripts/miwi-pg-connect.sh diff --git a/cluster-service/Makefile b/cluster-service/Makefile index 570ac3d82..954b9e413 100644 --- a/cluster-service/Makefile +++ b/cluster-service/Makefile @@ -4,7 +4,7 @@ $(shell ../templatize.sh $(DEPLOY_ENV) config.tmpl.mk config.mk) include config.mk deploy: deploy-namespace-template deploy-istio-configurations-template ${DB_SECRET_TARGET} - AZURE_CS_MI_CLIENT_ID=$(shell az identity show -g ${RESOURCEGROUP} -n clusters-service --query clientId -o tsv) && \ + AZURE_CS_MI_CLIENT_ID=$(shell az identity show -g ${RESOURCEGROUP} -n ${CS_MI_NAME} --query clientId -o tsv) && \ CS_SERVICE_PRINCIPAL_CREDS_BASE64='$(shell az keyvault secret show --vault-name "${SERVICE_KV}" --name "aro-hcp-dev-sp-cs" | jq .value -r | base64 | tr -d '\n')' && \ TENANT_ID=$(shell az account show --query tenantId --output tsv) && \ OIDC_BLOB_SERVICE_ENDPOINT=$(shell az storage account show -n ${OIDC_STORAGE_ACCOUNT} -g ${RESOURCEGROUP} --query primaryEndpoints.blob -o tsv) && \ @@ -13,38 +13,44 @@ deploy: deploy-namespace-template deploy-istio-configurations-template ${DB_SECR OCP_ACR_URL=$(shell az acr show -n ${OCP_ACR_NAME} --query loginServer -o tsv) && \ OCP_ACR_RESOURCE_ID=$(shell az acr show -n ${OCP_ACR_NAME} --query id -o tsv) && \ oc process --local -f deploy/openshift-templates/arohcp-service-template.yml \ - -p AZURE_CS_MI_CLIENT_ID=$${AZURE_CS_MI_CLIENT_ID} \ - -p TENANT_ID=$${TENANT_ID} \ - -p REGION=${REGION} \ - -p SERVICE_KEYVAULT_NAME=${SERVICE_KV} \ - -p CS_SERVICE_PRINCIPAL_CREDS_BASE64=$${CS_SERVICE_PRINCIPAL_CREDS_BASE64} \ - -p IMAGE_REGISTRY=${ACR_NAME}.azurecr.io \ - -p IMAGE_REPOSITORY=${IMAGE_REPO} \ - -p AZURE_FIRST_PARTY_APPLICATION_CLIENT_ID=${AZURE_FIRST_PARTY_APPLICATION_CLIENT_ID} \ - -p FPA_CERT_NAME=${FPA_CERT_NAME} \ - -p IMAGE_TAG=${IMAGE_TAG} \ - -p OCP_ACR_RESOURCE_ID=$${OCP_ACR_RESOURCE_ID} \ - -p OCP_ACR_URL=$${OCP_ACR_URL} \ - -p DATABASE_DISABLE_TLS=${DATABASE_DISABLE_TLS} \ - -p DATABASE_AUTH_METHOD=${DATABASE_AUTH_METHOD} | oc apply -f - + -p NAMESPACE="${CS_NAMESPACE_NAME}" \ + -p AZURE_CS_MI_CLIENT_ID=$${AZURE_CS_MI_CLIENT_ID} \ + -p TENANT_ID=$${TENANT_ID} \ + -p REGION=${REGION} \ + -p SERVICE_KEYVAULT_NAME=${SERVICE_KV} \ + -p CS_SERVICE_PRINCIPAL_CREDS_BASE64=$${CS_SERVICE_PRINCIPAL_CREDS_BASE64} \ + -p IMAGE_REGISTRY=${ACR_NAME}.azurecr.io \ + -p IMAGE_REPOSITORY=${IMAGE_REPO} \ + -p AZURE_FIRST_PARTY_APPLICATION_CLIENT_ID=${AZURE_FIRST_PARTY_APPLICATION_CLIENT_ID} \ + -p FPA_CERT_NAME=${FPA_CERT_NAME} \ + -p IMAGE_TAG=${IMAGE_TAG} \ + -p OCP_ACR_RESOURCE_ID=$${OCP_ACR_RESOURCE_ID} \ + -p OCP_ACR_URL=$${OCP_ACR_URL} \ + -p SERVICE_ACCOUNT_NAME=${CS_SA_NAME} \ + -p DATABASE_DISABLE_TLS=${DATABASE_DISABLE_TLS} \ + -p DATABASE_AUTH_METHOD=${DATABASE_AUTH_METHOD} | oc apply -f - deploy-namespace-template: ISTO_VERSION=$(shell az aks show -n ${AKS_NAME} -g ${RESOURCEGROUP} --query serviceMeshProfile.istio.revisions[-1] -o tsv) && \ oc process --local -f deploy/openshift-templates/arohcp-namespace-template.yml \ - -p ISTIO_VERSION=$${ISTO_VERSION} | oc apply -f - + -p ISTIO_VERSION=$${ISTO_VERSION} \ + -p NAMESPACE="${CS_NAMESPACE_NAME}" | oc apply -f - deploy-istio-configurations-template: kubectl apply -f deploy/istio.yml deploy-local-db-secret: configure-tmp-provision-shard oc process --local -f deploy/openshift-templates/arohcp-secrets-template.yml \ - -p PROVISION_SHARDS_CONFIG="$$( base64 -i deploy/provisioning-shards.yml)" | oc apply -f - - oc process --local -f deploy/openshift-templates/arohcp-db-template.yml | oc apply -f - + -p PROVISION_SHARDS_CONFIG="$$( base64 -i deploy/provisioning-shards.yml)" \ + -p NAMESPACE="${CS_NAMESPACE_NAME}" | oc apply -f - + oc process --local -f deploy/openshift-templates/arohcp-db-template.yml \ + -p NAMESPACE="${CS_NAMESPACE_NAME}" | oc apply -f - deploy-azure-db-secret: configure-tmp-provision-shard oc process --local -f deploy/openshift-templates/arohcp-secrets-template.yml \ - -p DATABASE_USER=clusters-service \ - -p DATABASE_NAME=clusters-service \ + -p NAMESPACE="${CS_NAMESPACE_NAME}" \ + -p DATABASE_USER=${CS_MI_NAME} \ + -p DATABASE_NAME=${CS_DB_NAME} \ -p DATABASE_PASSWORD="" \ -p DATABASE_HOST=$(shell az postgres flexible-server show --resource-group ${RESOURCEGROUP} -n ${DATABASE_SERVER_NAME} --query fullyQualifiedDomainName -o tsv) \ -p PROVISION_SHARDS_CONFIG="$$( base64 -i deploy/provisioning-shards.yml)" | oc apply -f - diff --git a/cluster-service/config.tmpl.mk b/cluster-service/config.tmpl.mk index 6b3861535..274153e19 100644 --- a/cluster-service/config.tmpl.mk +++ b/cluster-service/config.tmpl.mk @@ -13,6 +13,11 @@ AZURE_FIRST_PARTY_APPLICATION_CLIENT_ID ?= {{ .firstPartyAppClientId }} FPA_CERT_NAME ?= firstPartyCert ZONE_NAME ?= {{ .regionalDNSSubdomain }}.{{ .baseDnsZoneName }} +CS_MI_NAME ?= {{ .clusterServiceManagedIdentityName }} +CS_DB_NAME ?= {{ .clusterServiceDatabaseName }} +CS_SA_NAME ?= {{ .clusterServiceServiceAccountName }} +CS_NAMESPACE_NAME ?= {{ .clusterServiceNamespace }} + DATABASE_DISABLE_TLS ?= {{ not .clusterServicePostgresDeploy }} DATABASE_AUTH_METHOD ?= {{ ternary "az-entra" "postgres" .clusterServicePostgresDeploy }} DATABASE_SERVER_NAME ?= {{ .clusterServicePostgresName }} diff --git a/cluster-service/deploy/openshift-templates/arohcp-service-template.yml b/cluster-service/deploy/openshift-templates/arohcp-service-template.yml index 552d4f8c7..d01d734d4 100644 --- a/cluster-service/deploy/openshift-templates/arohcp-service-template.yml +++ b/cluster-service/deploy/openshift-templates/arohcp-service-template.yml @@ -138,6 +138,11 @@ parameters: description: The resource ID of the Azure Container Registry where the OpenShift images are stored. required: true +- name: SERVICE_ACCOUNT_NAME + description: The name of the service account used to run CS. + required: true + value: "clusters-service" + # These limits are based on the metrics collected in the production environment # over the last year. In particular the following Prometheus queries were used @@ -371,7 +376,7 @@ objects: - apiVersion: v1 kind: ServiceAccount metadata: - name: clusters-service + name: ${SERVICE_ACCOUNT_NAME} namespace: ${NAMESPACE} labels: app: clusters-service @@ -405,8 +410,8 @@ objects: app: clusters-service azure.workload.identity/use: "true" spec: - serviceAccount: clusters-service - serviceAccountName: clusters-service + serviceAccount: ${SERVICE_ACCOUNT_NAME} + serviceAccountName: ${SERVICE_ACCOUNT_NAME} volumes: - name: service secret: diff --git a/config/config.yaml b/config/config.yaml index 7c6c28980..f4a560ccd 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -37,6 +37,10 @@ defaults: maestroEventGridMaxClientSessionsPerAuthName: '4' maestroCertDomain: 'selfsigned.maestro.keyvault.azure.com' maestroPostgresName: {{ azurePostgresName "maestro" 5 .ctx.region .ctx.regionStamp }} + maestroServerNamespace: maestro + maestroServerManagedIdentityName: maestro-server + maestroServerServiceAccountName: maestro + maestroDatabaseName: maestro maestroPostgresServerVersion: '15' maestroPostgresServerStorageSizeGB: '32' maestroPostgresDeploy: true @@ -49,6 +53,10 @@ defaults: clusterServicePostgresDeploy: true clusterServicePostgresPrivate: false clusterServiceAcrRG: global + clusterServiceNamespace: cluster-service + clusterServiceManagedIdentityName: clusters-service + clusterServiceServiceAccountName: clusters-service + clusterServiceDatabaseName: clusters-service # Image Sync imageSyncAcrRG: global diff --git a/config/public-cloud-cs-pr.json b/config/public-cloud-cs-pr.json index d53a3f032..b161563c5 100644 --- a/config/public-cloud-cs-pr.json +++ b/config/public-cloud-cs-pr.json @@ -5,11 +5,15 @@ "baseDnsZoneName": "hcp.osadev.cloud", "baseDnsZoneRG": "global", "clusterServiceAcrRG": "global", + "clusterServiceDatabaseName": "clusters-service", "clusterServiceImageRepo": "app-sre/uhc-clusters-service", "clusterServiceImageTag": "a23276d", + "clusterServiceManagedIdentityName": "clusters-service", + "clusterServiceNamespace": "cluster-service", "clusterServicePostgresDeploy": true, "clusterServicePostgresName": "cs-9c782", "clusterServicePostgresPrivate": false, + "clusterServiceServiceAccountName": "clusters-service", "cxKeyVaultName": "aro-hcp-cx-1abb8", "cxKeyVaultPrivate": false, "cxKeyVaultSoftDelete": false, @@ -32,6 +36,7 @@ "kubernetesVersion": "1.30.5", "maestroCertDomain": "selfsigned.maestro.keyvault.azure.com", "maestroConsumerName": "hcp-underlay-westus3-cs-pr-mgmt-1", + "maestroDatabaseName": "maestro", "maestroEventGridMaxClientSessionsPerAuthName": "4", "maestroEventgridName": "maestro-9c782", "maestroImageBase": "quay.io/redhat-user-workloads/maestro-rhtap-tenant/maestro/maestro", @@ -43,6 +48,9 @@ "maestroPostgresServerStorageSizeGB": "32", "maestroPostgresServerVersion": "15", "maestroRestrictIstioIngress": false, + "maestroServerManagedIdentityName": "maestro-server", + "maestroServerNamespace": "maestro", + "maestroServerServiceAccountName": "maestro", "managementClusterRG": "hcp-underlay-westus3-cs-pr-mgmt-1", "mgmtEtcdKVName": "aro-hcp-etcd-1abb8", "mgmtEtcdKVSoftDelete": false, diff --git a/config/public-cloud-dev.json b/config/public-cloud-dev.json index c318621ba..64fc455d2 100644 --- a/config/public-cloud-dev.json +++ b/config/public-cloud-dev.json @@ -5,11 +5,15 @@ "baseDnsZoneName": "hcp.osadev.cloud", "baseDnsZoneRG": "global", "clusterServiceAcrRG": "global", + "clusterServiceDatabaseName": "clusters-service", "clusterServiceImageRepo": "app-sre/uhc-clusters-service", "clusterServiceImageTag": "a23276d", + "clusterServiceManagedIdentityName": "clusters-service", + "clusterServiceNamespace": "cluster-service", "clusterServicePostgresDeploy": true, "clusterServicePostgresName": "cs-157ff", "clusterServicePostgresPrivate": false, + "clusterServiceServiceAccountName": "clusters-service", "cxKeyVaultName": "aro-hcp-cx-08101", "cxKeyVaultPrivate": false, "cxKeyVaultSoftDelete": false, @@ -32,6 +36,7 @@ "kubernetesVersion": "1.30.5", "maestroCertDomain": "selfsigned.maestro.keyvault.azure.com", "maestroConsumerName": "hcp-underlay-westus3-dev-mgmt-1", + "maestroDatabaseName": "maestro", "maestroEventGridMaxClientSessionsPerAuthName": "4", "maestroEventgridName": "maestro-157ff", "maestroImageBase": "quay.io/redhat-user-workloads/maestro-rhtap-tenant/maestro/maestro", @@ -43,6 +48,9 @@ "maestroPostgresServerStorageSizeGB": "32", "maestroPostgresServerVersion": "15", "maestroRestrictIstioIngress": true, + "maestroServerManagedIdentityName": "maestro-server", + "maestroServerNamespace": "maestro", + "maestroServerServiceAccountName": "maestro", "managementClusterRG": "hcp-underlay-westus3-dev-mgmt-1", "mgmtEtcdKVName": "aro-hcp-etcd-08101", "mgmtEtcdKVSoftDelete": false, diff --git a/config/public-cloud-personal-dev.json b/config/public-cloud-personal-dev.json index fd89f0cf1..9591e54ae 100644 --- a/config/public-cloud-personal-dev.json +++ b/config/public-cloud-personal-dev.json @@ -5,11 +5,15 @@ "baseDnsZoneName": "hcp.osadev.cloud", "baseDnsZoneRG": "global", "clusterServiceAcrRG": "global", + "clusterServiceDatabaseName": "clusters-service", "clusterServiceImageRepo": "app-sre/uhc-clusters-service", "clusterServiceImageTag": "a23276d", + "clusterServiceManagedIdentityName": "clusters-service", + "clusterServiceNamespace": "cluster-service", "clusterServicePostgresDeploy": false, "clusterServicePostgresName": "cs-76fc6", "clusterServicePostgresPrivate": false, + "clusterServiceServiceAccountName": "clusters-service", "cxKeyVaultName": "aro-hcp-cx-85fcc", "cxKeyVaultPrivate": false, "cxKeyVaultSoftDelete": false, @@ -32,6 +36,7 @@ "kubernetesVersion": "1.30.5", "maestroCertDomain": "selfsigned.maestro.keyvault.azure.com", "maestroConsumerName": "hcp-underlay-westus3-tst-mgmt-1", + "maestroDatabaseName": "maestro", "maestroEventGridMaxClientSessionsPerAuthName": "4", "maestroEventgridName": "maestro-76fc6", "maestroImageBase": "quay.io/redhat-user-workloads/maestro-rhtap-tenant/maestro/maestro", @@ -43,6 +48,9 @@ "maestroPostgresServerStorageSizeGB": "32", "maestroPostgresServerVersion": "15", "maestroRestrictIstioIngress": true, + "maestroServerManagedIdentityName": "maestro-server", + "maestroServerNamespace": "maestro", + "maestroServerServiceAccountName": "maestro", "managementClusterRG": "hcp-underlay-westus3-tst-mgmt-1", "mgmtEtcdKVName": "aro-hcp-etcd-85fcc", "mgmtEtcdKVSoftDelete": false, diff --git a/dev-infrastructure/Makefile b/dev-infrastructure/Makefile index 29a5adda5..70d9ddf99 100644 --- a/dev-infrastructure/Makefile +++ b/dev-infrastructure/Makefile @@ -402,20 +402,14 @@ acr-ocp.what-if: global.rg # Postgres Authentication Helpers # -cs-current-user-pg-connect: - @scripts/cs-current-user-pg-connect.sh $(SVC_RESOURCEGROUP) $(CS_PG_NAME) -.PHONY: cs-current-user-pg-connect - cs-miwi-pg-connect: - @scripts/cs-miwi-pg-connect.sh $(SVC_RESOURCEGROUP) $(CS_PG_NAME) "clusters-service" "cluster-service" "clusters-service" + @KUBECONFIG=${SVC_KUBECONFIG_FILE} && \ + scripts/miwi-pg-connect.sh $(SVC_RESOURCEGROUP) $(CS_PG_NAME) $(CS_MI_NAME) $(CS_NAMESPACE_NAME) $(CS_SA_NAME) $(CS_DB_NAME) .PHONY: cs-miwi-pg-connect -maestro-current-user-pg-connect: - @scripts/cs-current-user-pg-connect.sh $(SVC_RESOURCEGROUP) $(MAESTRO_PG_NAME) -.PHONY: maestro-current-user-pg-connect - maestro-miwi-pg-connect: - @scripts/cs-miwi-pg-connect.sh $(SVC_RESOURCEGROUP) $(MAESTRO_PG_NAME) "maestro-server" "maestro" "maestro" + @KUBECONFIG=${SVC_KUBECONFIG_FILE} && \ + scripts/miwi-pg-connect.sh $(SVC_RESOURCEGROUP) $(MAESTRO_PG_NAME) $(MAESTRO_MI_NAME) $(MAESTRO_NAMESPACE_NAME) $(MAESTRO_SA_NAME) $(MAESTRO_DB_NAME) .PHONY: maestro-miwi-pg-connect # diff --git a/dev-infrastructure/config.tmpl.mk b/dev-infrastructure/config.tmpl.mk index bebf2beea..a05d179d4 100644 --- a/dev-infrastructure/config.tmpl.mk +++ b/dev-infrastructure/config.tmpl.mk @@ -9,5 +9,15 @@ IMAGE_SYNC_ENVIRONMENT ?= {{ .imageSyncEnvironmentName }} ARO_HCP_IMAGE_ACR ?= {{ .svcAcrName }} REPOSITORIES_TO_SYNC ?= '{{ .imageSyncRepositories }}' AKS_NAME ?= {{ .aksName }} + CS_PG_NAME ?= {{ .clusterServicePostgresName }} +CS_MI_NAME ?= {{ .clusterServiceManagedIdentityName }} +CS_NAMESPACE_NAME ?= {{ .clusterServiceNamespace }} +CS_SA_NAME = {{ .clusterServiceServiceAccountName }} +CS_DB_NAME ?= {{ .clusterServiceDatabaseName }} + MAESTRO_PG_NAME ?= {{ .maestroPostgresName }} +MAESTRO_MI_NAME ?= {{ .maestroServerManagedIdentityName }} +MAESTRO_NAMESPACE_NAME ?= {{ .maestroServerNamespace }} +MAESTRO_SA_NAME = {{ .maestroServerServiceAccountName }} +MAESTRO_DB_NAME ?= {{ .maestroDatabaseName }} diff --git a/dev-infrastructure/configurations/svc-cluster.tmpl.bicepparam b/dev-infrastructure/configurations/svc-cluster.tmpl.bicepparam index 6a3ac77fe..e2d68b1fe 100644 --- a/dev-infrastructure/configurations/svc-cluster.tmpl.bicepparam +++ b/dev-infrastructure/configurations/svc-cluster.tmpl.bicepparam @@ -17,14 +17,22 @@ param maestroKeyVaultName = '{{ .maestroKeyVaultName }}' param maestroEventGridNamespacesName = '{{ .maestroEventgridName }}' param maestroCertDomain = '{{ .maestroCertDomain}}' param maestroPostgresServerName = '{{ .maestroPostgresName }}' +param maestroDatabaseName = '{{ .maestroDatabaseName }}' param maestroPostgresServerVersion = '{{ .maestroPostgresServerVersion }}' param maestroPostgresServerStorageSizeGB = {{ .maestroPostgresServerStorageSizeGB }} param deployMaestroPostgres = {{ .maestroPostgresDeploy }} param maestroPostgresPrivate = {{ .maestroPostgresPrivate }} +param maestroMIName = '{{ .maestroServerManagedIdentityName }}' +param maestroNamespace = '{{ .maestroServerNamespace }}' +param maestroServiceAccountName = '{{ .maestroServerServiceAccountName }}' param deployCsInfra = {{ .clusterServicePostgresDeploy }} param csPostgresServerName = '{{ .clusterServicePostgresName }}' -param clusterServicePostgresPrivate = {{ .clusterServicePostgresPrivate }} +param csDatabaseName = '{{ .clusterServiceDatabaseName }}' +param csPostgresPrivate = {{ .clusterServicePostgresPrivate }} +param csMIName = '{{ .clusterServiceManagedIdentityName }}' +param csNamespace = '{{ .clusterServiceNamespace }}' +param csServiceAccountName = '{{ .clusterServiceServiceAccountName }}' param serviceKeyVaultName = '{{ .serviceKeyVaultName }}' param serviceKeyVaultResourceGroup = '{{ .serviceKeyVaultRG }}' diff --git a/dev-infrastructure/docs/development-setup.md b/dev-infrastructure/docs/development-setup.md index 7dc02f68b..1e22bded8 100644 --- a/dev-infrastructure/docs/development-setup.md +++ b/dev-infrastructure/docs/development-setup.md @@ -558,20 +558,10 @@ To tear down your CS setup: ### Access Maestro Postgres from outside of the AKS cluster -To connect to the database as current user run +To connect to the database with the managed identity of Maestro, run the following command ```sh - eval $(AKSCONFIG=svc-cluster make maestro-current-user-pg-connect) - psql -d maestro - ``` - -The output of the make target is in ENV var format for the `psql` tool, so this works to get a connection into the DB. - -To connect to the database with the managed identity of Maestro, make sure to have a KUBECONFIG for the cluster that runs Maestro Server and run - - ```sh - eval $(AKSCONFIG=svc-cluster make maestro-miwi-pg-connect) - psql -d maestro + make maestro-miwi-pg-connect ``` Once logged in, verify the connection with `\conninfo` @@ -584,20 +574,10 @@ To create a Postgres DB on Azure enabled for Entra authentication, a svc cluster ### Access the database from outside of the AKS cluster -To connect to the database as current user run - - ```sh - eval $(make cs-current-user-pg-connect) - psql -d clusters-service - ``` - -The output of the make target is in ENV var format for the `psql` tool, so this works to get a connection into the DB. - -To connect to the database with the managed identity of CS, make sure to have a KUBECONFIG for the cluster that runs CS and run +To connect to the database with the managed identity of CS, run the following command ```sh - eval $(make cs-miwi-pg-connect) - psql -d clusters-service + make cs-miwi-pg-connect ``` Once logged in, verify the connection with `\conninfo` diff --git a/dev-infrastructure/modules/cluster-service.bicep b/dev-infrastructure/modules/cluster-service.bicep index ed871f47e..66d8b6f98 100644 --- a/dev-infrastructure/modules/cluster-service.bicep +++ b/dev-infrastructure/modules/cluster-service.bicep @@ -8,7 +8,7 @@ param clusterServiceManagedIdentityName string param clusterServiceManagedIdentityPrincipalId string @description('The name of the database to create for CS') -param csDatabaseName string = 'clusters-service' +param csDatabaseName string @description('The name of the Postgres server for CS') param postgresServerName string diff --git a/dev-infrastructure/modules/maestro/maestro-server.bicep b/dev-infrastructure/modules/maestro/maestro-server.bicep index 1cb940821..1012d3a65 100644 --- a/dev-infrastructure/modules/maestro/maestro-server.bicep +++ b/dev-infrastructure/modules/maestro/maestro-server.bicep @@ -48,7 +48,7 @@ param privateEndpointSubnetId string = '' param privateEndpointVnetId string = '' @description('The name of the database to create for Maestro') -param maestroDatabaseName string = 'maestro' +param maestroDatabaseName string @description('The name of the Managed Identity for the Maestro cluster service') param maestroServerManagedIdentityName string diff --git a/dev-infrastructure/scripts/cs-current-user-pg-connect.sh b/dev-infrastructure/scripts/cs-current-user-pg-connect.sh deleted file mode 100755 index da42c6ee2..000000000 --- a/dev-infrastructure/scripts/cs-current-user-pg-connect.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/sh - -RESOURCEGROUP=$1 -DB_SERVER_NAME_PREFIX=$2 - -CURRENTUSER=$(az ad signed-in-user show -o json | jq -r '.id') -CURRENTUSER_NAME=$(az ad signed-in-user show -o json | jq -r '.userPrincipalName') - -CS_DB=$(az postgres flexible-server list -g ${RESOURCEGROUP} -o json | jq --arg prefix "${DB_SERVER_NAME_PREFIX}" '.[] | select(.name | startswith($prefix))') -CS_DB_NAME=$(echo ${CS_DB} | jq -r .name) - -ALREADY_ADMIN=$(az postgres flexible-server ad-admin list -g ${RESOURCEGROUP} -s ${CS_DB_NAME} -o json | jq --arg principalname "${CURRENTUSER_NAME}" '[.[] | select(.principalName == $principalname)] | length') -if [ $ALREADY_ADMIN -eq 0 ]; then - az postgres flexible-server ad-admin create --server-name ${CS_DB_NAME} --resource-group ${RESOURCEGROUP} --object-id ${CURRENTUSER} --display-name ${CURRENTUSER_NAME} -fi - -echo export PGHOST=$(echo ${CS_DB} | jq -r .fullyQualifiedDomainName) -echo export PGUSER=$CURRENTUSER_NAME -echo export PGPASSWORD=$(az account get-access-token --resource='https://ossrdbms-aad.database.windows.net' -o json | jq .accessToken -r) diff --git a/dev-infrastructure/scripts/cs-miwi-pg-connect.sh b/dev-infrastructure/scripts/cs-miwi-pg-connect.sh deleted file mode 100755 index 507f45f42..000000000 --- a/dev-infrastructure/scripts/cs-miwi-pg-connect.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash - -RESOURCEGROUP=$1 -DB_SERVER_NAME_PREFIX=$2 -MANAGED_IDENTITY_NAME=$3 -NAMESPACE=$4 -SA_NAME=$5 - -# prep creds and configs -PGHOST=$(az postgres flexible-server list --resource-group ${RESOURCEGROUP} --query "[?starts_with(name, '${DB_SERVER_NAME_PREFIX}')].fullyQualifiedDomainName" -o tsv) -AZURE_TENANT_ID=$(az account show -o json | jq .homeTenantId -r) -AZURE_CLIENT_ID=$(az identity show -g ${RESOURCEGROUP} -n ${MANAGED_IDENTITY_NAME} --query clientId -o tsv) -SA_TOKEN=$(kubectl create token ${SA_NAME} --namespace=${NAMESPACE} --audience api://AzureADTokenExchange) - -# az login with managed identity via SA token -export AZURE_CONFIG_DIR="${HOME}/.azure-profile-cs-${RESOURCEGROUP}" -rm -rf $AZURE_CONFIG_DIR -az login --federated-token ${SA_TOKEN} --service-principal -u $AZURE_CLIENT_ID -t $AZURE_TENANT_ID > /dev/null 2>&1 - -# get tmp DB password -PGPASSWORD=$(az account get-access-token --resource='https://ossrdbms-aad.database.windows.net' -o json | jq .accessToken -r) -rm -rf $AZURE_CONFIG_DIR - -echo export PGHOST=${PGHOST} -echo export PGUSER=${MANAGED_IDENTITY_NAME} -echo export PGPASSWORD=${PGPASSWORD} diff --git a/dev-infrastructure/scripts/miwi-pg-connect.sh b/dev-infrastructure/scripts/miwi-pg-connect.sh new file mode 100755 index 000000000..8f1b147ad --- /dev/null +++ b/dev-infrastructure/scripts/miwi-pg-connect.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +RESOURCEGROUP=$1 +DB_SERVER_NAME_PREFIX=$2 +MANAGED_IDENTITY_NAME=$3 +NAMESPACE=$4 +SA_NAME=$5 +DB_NAME=$6 + +# prep creds and configs +export PGHOST=$(az postgres flexible-server list --resource-group ${RESOURCEGROUP} --query "[?starts_with(name, '${DB_SERVER_NAME_PREFIX}')].fullyQualifiedDomainName" -o tsv) +AZURE_TENANT_ID=$(az account show -o json | jq .homeTenantId -r) +AZURE_CLIENT_ID=$(az identity show -g ${RESOURCEGROUP} -n ${MANAGED_IDENTITY_NAME} --query clientId -o tsv) +SA_TOKEN=$(kubectl create token ${SA_NAME} --duration=1h --namespace=${NAMESPACE} --audience api://AzureADTokenExchange) + +# az login with managed identity via SA token +export AZURE_CONFIG_DIR="${HOME}/.azure-profile-cs-${RESOURCEGROUP}" +rm -rf $AZURE_CONFIG_DIR +az login --federated-token ${SA_TOKEN} --service-principal -u $AZURE_CLIENT_ID -t $AZURE_TENANT_ID > /dev/null 2>&1 + +# get tmp DB password +export PGPASSWORD=$(az account get-access-token --resource='https://ossrdbms-aad.database.windows.net' -o json | jq .accessToken -r) +rm -rf $AZURE_CONFIG_DIR + +export PGUSER=${MANAGED_IDENTITY_NAME} +podman run -it --rm \ + --env PGHOST \ + --env PGUSER \ + --env PGPASSWORD \ + postgres \ + psql \ + -d $DB_NAME -p 5432 diff --git a/dev-infrastructure/templates/svc-cluster.bicep b/dev-infrastructure/templates/svc-cluster.bicep index 86a246e54..b111aee86 100644 --- a/dev-infrastructure/templates/svc-cluster.bicep +++ b/dev-infrastructure/templates/svc-cluster.bicep @@ -73,8 +73,20 @@ param deployCsInfra bool @maxLength(60) param csPostgresServerName string +@description('The name of the database to create for CS') +param csDatabaseName string + @description('If true, make the CS Postgres instance private') -param clusterServicePostgresPrivate bool = true +param csPostgresPrivate bool = true + +@description('The name of the cluster service managed identity') +param csMIName string + +@description('The name of the cluster service namespace') +param csNamespace string + +@description('The name of the cluster service service account') +param csServiceAccountName string @description('Deploy ARO HCP Maestro Postgres if true') param deployMaestroPostgres bool = true @@ -86,12 +98,24 @@ param maestroPostgresPrivate bool = true @maxLength(60) param maestroPostgresServerName string +@description('The name of the database to create for Maestro') +param maestroDatabaseName string + @description('The version of the Postgres server for Maestro') param maestroPostgresServerVersion string @description('The size of the Postgres server for Maestro') param maestroPostgresServerStorageSizeGB int +@description('The name of the maestro managed identity') +param maestroMIName string + +@description('The name of themaestro namespace') +param maestroNamespace string + +@description('The name of the maestro service account') +param maestroServiceAccountName string + @description('The name of the service keyvault') param serviceKeyVaultName string @@ -125,8 +149,6 @@ param aroDevopsMsiId string @description('This is a regional DNS zone') param regionalDNSZoneName string -var clusterServiceMIName = 'clusters-service' - // Tags the resource group resource subscriptionTags 'Microsoft.Resources/tags@2024-03-01' = { name: 'default' @@ -168,14 +190,14 @@ module svcCluster '../modules/aks-cluster-base.bicep' = { serviceAccountName: 'backend' } maestro_wi: { - uamiName: 'maestro-server' - namespace: 'maestro' - serviceAccountName: 'maestro' + uamiName: maestroMIName + namespace: maestroNamespace + serviceAccountName: maestroServiceAccountName } cs_wi: { - uamiName: clusterServiceMIName - namespace: 'cluster-service' - serviceAccountName: 'clusters-service' + uamiName: csMIName + namespace: csNamespace + serviceAccountName: csServiceAccountName } image_sync_wi: { uamiName: 'image-sync' @@ -224,6 +246,7 @@ module maestroServer '../modules/maestro/maestro-server.bicep' = { postgresServerName: maestroPostgresServerName postgresServerVersion: maestroPostgresServerVersion postgresServerStorageSizeGB: maestroPostgresServerStorageSizeGB + maestroDatabaseName: maestroDatabaseName privateEndpointSubnetId: svcCluster.outputs.aksNodeSubnetId privateEndpointVnetId: svcCluster.outputs.aksVnetId postgresServerPrivate: maestroPostgresPrivate @@ -272,21 +295,19 @@ module serviceKeyVaultPrivateEndpoint '../modules/keyvault/keyvault-private-endp // C L U S T E R S E R V I C E // -var csManagedIdentityPrincipalId = filter( - svcCluster.outputs.userAssignedIdentities, - id => id.uamiName == clusterServiceMIName -)[0].uamiPrincipalID +var csManagedIdentityPrincipalId = filter(svcCluster.outputs.userAssignedIdentities, id => id.uamiName == csMIName)[0].uamiPrincipalID module cs '../modules/cluster-service.bicep' = if (deployCsInfra) { name: 'cluster-service' params: { location: location postgresServerName: csPostgresServerName + csDatabaseName: csDatabaseName privateEndpointSubnetId: svcCluster.outputs.aksNodeSubnetId privateEndpointVnetId: svcCluster.outputs.aksVnetId - postgresServerPrivate: clusterServicePostgresPrivate + postgresServerPrivate: csPostgresPrivate clusterServiceManagedIdentityPrincipalId: csManagedIdentityPrincipalId - clusterServiceManagedIdentityName: clusterServiceMIName + clusterServiceManagedIdentityName: csMIName } dependsOn: [ maestroServer @@ -378,14 +399,14 @@ module acrContributorRole '../modules/acr-permissions.bicep' = [ } ] -// oidc +// O I D C module oidc '../modules/oidc/main.bicep' = { name: '${deployment().name}-oidc' params: { location: location storageAccountName: oidcStorageAccountName - rpMsiName: clusterServiceMIName + rpMsiName: csMIName skuName: oidcStorageAccountSku aroDevopsMsiId: aroDevopsMsiId deploymentScriptLocation: location diff --git a/maestro/registration/Makefile b/maestro/registration/Makefile index d62b33184..700006415 100644 --- a/maestro/registration/Makefile +++ b/maestro/registration/Makefile @@ -4,11 +4,11 @@ $(shell ../../templatize.sh $(DEPLOY_ENV) config.tmpl.mk config.mk) include config.mk deploy: - @if ! kubectl get service maestro -n maestro > /dev/null 2>&1; then \ - echo "Error: Service 'maestro' not found in namespace 'maestro'"; \ + @if ! kubectl get service maestro -n ${MAESTRO_NAMESPACE_NAME} > /dev/null 2>&1; then \ + echo "Error: Service 'maestro' not found in namespace '${CONSUMER_NAME}'"; \ exit 1; \ fi helm upgrade --install ${CONSUMER_NAME} ./helm \ - --namespace maestro \ + --namespace ${MAESTRO_NAMESPACE_NAME} \ --set consumerName=${CONSUMER_NAME} .PHONY: deploy diff --git a/maestro/registration/config.tmpl.mk b/maestro/registration/config.tmpl.mk index 8918876a7..020b96006 100644 --- a/maestro/registration/config.tmpl.mk +++ b/maestro/registration/config.tmpl.mk @@ -1 +1,2 @@ CONSUMER_NAME ?= {{ .maestroConsumerName }} +MAESTRO_NAMESPACE_NAME ?= {{ .maestroServerNamespace }} diff --git a/maestro/server/Makefile b/maestro/server/Makefile index 2f97063fb..14a8fc501 100644 --- a/maestro/server/Makefile +++ b/maestro/server/Makefile @@ -4,15 +4,18 @@ $(shell ../../templatize.sh $(DEPLOY_ENV) config.tmpl.mk config.mk) include config.mk deploy: - kubectl create namespace maestro --dry-run=client -o json | kubectl apply -f - + kubectl create namespace ${MAESTRO_NAMESPACE_NAME} --dry-run=client -o json | kubectl apply -f - ISTO_VERSION=$(shell az aks show -n ${AKS_NAME} -g ${SVC_RG} --query serviceMeshProfile.istio.revisions[-1] -o tsv) && \ kubectl label namespace maestro "istio.io/rev=$${ISTO_VERSION}" --overwrite=true EVENTGRID_HOSTNAME=$(shell az resource show -n ${EVENTGRID_NAME} -g ${REGION_RG} --resource-type "Microsoft.EventGrid/namespaces" --query properties.topicSpacesConfiguration.hostname -o tsv) && \ TENANT_ID=$(shell az account show --query tenantId --output tsv) && \ - MAESTRO_MI_CLIENT_ID=$(shell az identity show -g "${SVC_RG}" -n maestro-server --query clientId -o tsv) && \ + MAESTRO_MI_CLIENT_ID=$(shell az identity show -g "${SVC_RG}" -n ${MAESTRO_MI_NAME} --query clientId -o tsv) && \ helm upgrade --install maestro-server ./helm \ - --namespace maestro \ + --namespace ${MAESTRO_NAMESPACE_NAME} \ --set broker.host=$${EVENTGRID_HOSTNAME} \ + --set maestro.serviceAccount=${MAESTRO_SA_NAME} \ + --set clusterService.namespace=${CS_NAMESPACE_NAME} \ + --set clusterService.serviceAccount=${CS_SA_NAME} \ --set credsKeyVault.name=${KEYVAULT_NAME} \ --set azure.clientId=$${MAESTRO_MI_CLIENT_ID} \ --set azure.tenantId=$${TENANT_ID} \ diff --git a/maestro/server/config.tmpl.mk b/maestro/server/config.tmpl.mk index 31acff9d0..778e8bee0 100644 --- a/maestro/server/config.tmpl.mk +++ b/maestro/server/config.tmpl.mk @@ -8,3 +8,10 @@ USE_CONTAINERIZED_DB ?= {{ not .maestroPostgresDeploy }} USE_DATABASE_SSL ?= {{ ternary "enable" "disable" .maestroPostgresDeploy }} ISTIO_RESTRICT_INGRESS ?= {{ .maestroRestrictIstioIngress }} KEYVAULT_NAME ?= {{ .maestroKeyVaultName }} + +MAESTRO_NAMESPACE_NAME ?= {{ .maestroServerNamespace }} +MAESTRO_SA_NAME = {{ .maestroServerServiceAccountName }} +MAESTRO_MI_NAME ?= {{ .maestroServerManagedIdentityName }} + +CS_NAMESPACE_NAME ?= {{ .clusterServiceNamespace }} +CS_SA_NAME = {{ .clusterServiceServiceAccountName }}