Skip to content

Commit 3ca4e29

Browse files
trevorwilliams2025trevorwilliams2025
committed
Add SRE Tooling AKS Cluster deployment
1 parent d2cd6af commit 3ca4e29

11 files changed

+941
-0
lines changed

‎dev-infrastructure/Makefile.sre-tooling‎

Lines changed: 142 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,142 @@
1+
#
2+
# SRE Tooling AKS Cluster
3+
# Standalone Makefile - does not require main Makefile
4+
#
5+
# Environment variables required:
6+
# SRE_TOOLING_ENV: dev or pers
7+
# SRE_TOOLING_RG: Resource group name (e.g., hcp-dev-sre-tooling or hcp-pers-sre-tooling)
8+
# SRE_TOOLING_SUBSCRIPTION_ID: Subscription ID
9+
# SERVICE_KEYVAULT_NAME: Name of existing service key vault
10+
# SERVICE_KEYVAULT_RG: Resource group of service key vault
11+
# REGIONAL_RG: Regional resource group name
12+
# SVC_ACR_RESOURCE_ID: Resource ID of SVC ACR
13+
# GLOBAL_MSI_ID: Resource ID of global MSI
14+
# KV_CERT_OFFICER_PRINCIPAL_ID: Principal ID for KV certificate officer
15+
# AZURE_MONITORING_WORKSPACE_ID: Resource ID of Azure Monitor Workspace (optional)
16+
# ADMIN_API_MI_NAME: Name of Admin API managed identity
17+
#
18+
# Usage: make -f Makefile.sre-tooling <target>
19+
#
20+
21+
# Set SKIP_CONFIRM to a non-empty value to skip "what-if" confirmation prompts.
22+
ifndef SKIP_CONFIRM
23+
PROMPT_TO_CONFIRM = "--confirm-with-what-if"
24+
endif
25+
26+
SRE_TOOLING_ENVS = dev pers
27+
28+
sre-tooling-infra:
29+
@[ "${SRE_TOOLING_ENV}" ] || ( echo ">> SRE_TOOLING_ENV is not set (dev or pers)"; exit 1 )
30+
@[ "${SRE_TOOLING_ENV}" = "dev" ] || [ "${SRE_TOOLING_ENV}" = "pers" ] || ( echo ">> SRE_TOOLING_ENV must be 'dev' or 'pers', got: ${SRE_TOOLING_ENV}"; exit 1 )
31+
@[ "${SRE_TOOLING_RG}" ] || ( echo ">> SRE_TOOLING_RG is not set"; exit 1 )
32+
@[ "${SRE_TOOLING_SUBSCRIPTION_ID}" ] || ( echo ">> SRE_TOOLING_SUBSCRIPTION_ID is not set"; exit 1 )
33+
@[ "${SERVICE_KEYVAULT_NAME}" ] || ( echo ">> SERVICE_KEYVAULT_NAME is not set"; exit 1 )
34+
@[ "${SERVICE_KEYVAULT_RG}" ] || ( echo ">> SERVICE_KEYVAULT_RG is not set"; exit 1 )
35+
@[ "${GLOBAL_MSI_ID}" ] || ( echo ">> GLOBAL_MSI_ID is not set"; exit 1 )
36+
@[ "${KV_CERT_OFFICER_PRINCIPAL_ID}" ] || ( echo ">> KV_CERT_OFFICER_PRINCIPAL_ID is not set"; exit 1 )
37+
az group create \
38+
--resource-group ${SRE_TOOLING_RG} --subscription ${SRE_TOOLING_SUBSCRIPTION_ID} \
39+
--location westus3 --tags persist=true environment=${SRE_TOOLING_ENV} || true
40+
az deployment group create \
41+
--name sre-tooling-infra-${SRE_TOOLING_ENV} \
42+
--resource-group ${SRE_TOOLING_RG} \
43+
--mode complete \
44+
--subscription ${SRE_TOOLING_SUBSCRIPTION_ID} \
45+
--template-file templates/sre-tooling-infra.bicep \
46+
$(PROMPT_TO_CONFIRM) \
47+
--parameters configurations/sre-tooling-infra.bicepparam \
48+
--parameters serviceKeyVaultName=${SERVICE_KEYVAULT_NAME} \
49+
--parameters serviceKeyVaultResourceGroup=${SERVICE_KEYVAULT_RG} \
50+
--parameters globalMSIId=${GLOBAL_MSI_ID} \
51+
--parameters kvCertOfficerPrincipalId=${KV_CERT_OFFICER_PRINCIPAL_ID} \
52+
--parameters serviceKeyVaultTagValue=${SRE_TOOLING_ENV}
53+
.PHONY: sre-tooling-infra
54+
55+
sre-tooling-infra.what-if:
56+
@[ "${SRE_TOOLING_ENV}" ] || ( echo ">> SRE_TOOLING_ENV is not set (dev or pers)"; exit 1 )
57+
@[ "${SRE_TOOLING_ENV}" = "dev" ] || [ "${SRE_TOOLING_ENV}" = "pers" ] || ( echo ">> SRE_TOOLING_ENV must be 'dev' or 'pers', got: ${SRE_TOOLING_ENV}"; exit 1 )
58+
@[ "${SRE_TOOLING_RG}" ] || ( echo ">> SRE_TOOLING_RG is not set"; exit 1 )
59+
@[ "${SRE_TOOLING_SUBSCRIPTION_ID}" ] || ( echo ">> SRE_TOOLING_SUBSCRIPTION_ID is not set"; exit 1 )
60+
@[ "${SERVICE_KEYVAULT_NAME}" ] || ( echo ">> SERVICE_KEYVAULT_NAME is not set"; exit 1 )
61+
@[ "${SERVICE_KEYVAULT_RG}" ] || ( echo ">> SERVICE_KEYVAULT_RG is not set"; exit 1 )
62+
@[ "${GLOBAL_MSI_ID}" ] || ( echo ">> GLOBAL_MSI_ID is not set"; exit 1 )
63+
@[ "${KV_CERT_OFFICER_PRINCIPAL_ID}" ] || ( echo ">> KV_CERT_OFFICER_PRINCIPAL_ID is not set"; exit 1 )
64+
az deployment group what-if \
65+
--name sre-tooling-infra-${SRE_TOOLING_ENV} \
66+
--resource-group ${SRE_TOOLING_RG} \
67+
--subscription ${SRE_TOOLING_SUBSCRIPTION_ID} \
68+
--template-file templates/sre-tooling-infra.bicep \
69+
--parameters configurations/sre-tooling-infra.bicepparam \
70+
--parameters serviceKeyVaultName=${SERVICE_KEYVAULT_NAME} \
71+
--parameters serviceKeyVaultResourceGroup=${SERVICE_KEYVAULT_RG} \
72+
--parameters globalMSIId=${GLOBAL_MSI_ID} \
73+
--parameters kvCertOfficerPrincipalId=${KV_CERT_OFFICER_PRINCIPAL_ID} \
74+
--parameters serviceKeyVaultTagValue=${SRE_TOOLING_ENV}
75+
.PHONY: sre-tooling-infra.what-if
76+
77+
sre-tooling-cluster:
78+
@[ "${SRE_TOOLING_ENV}" ] || ( echo ">> SRE_TOOLING_ENV is not set (dev or pers)"; exit 1 )
79+
@[ "${SRE_TOOLING_ENV}" = "dev" ] || [ "${SRE_TOOLING_ENV}" = "pers" ] || ( echo ">> SRE_TOOLING_ENV must be 'dev' or 'pers', got: ${SRE_TOOLING_ENV}"; exit 1 )
80+
@[ "${SRE_TOOLING_RG}" ] || ( echo ">> SRE_TOOLING_RG is not set"; exit 1 )
81+
@[ "${SRE_TOOLING_SUBSCRIPTION_ID}" ] || ( echo ">> SRE_TOOLING_SUBSCRIPTION_ID is not set"; exit 1 )
82+
@[ "${SERVICE_KEYVAULT_NAME}" ] || ( echo ">> SERVICE_KEYVAULT_NAME is not set"; exit 1 )
83+
@[ "${SERVICE_KEYVAULT_RG}" ] || ( echo ">> SERVICE_KEYVAULT_RG is not set"; exit 1 )
84+
@[ "${REGIONAL_RG}" ] || ( echo ">> REGIONAL_RG is not set"; exit 1 )
85+
@[ "${SVC_ACR_RESOURCE_ID}" ] || ( echo ">> SVC_ACR_RESOURCE_ID is not set"; exit 1 )
86+
@[ "${GLOBAL_MSI_ID}" ] || ( echo ">> GLOBAL_MSI_ID is not set"; exit 1 )
87+
@[ "${ADMIN_API_MI_NAME}" ] || ( echo ">> ADMIN_API_MI_NAME is not set"; exit 1 )
88+
@$(eval DEFAULT_CLUSTER_NAME = $(if $(filter pers,${SRE_TOOLING_ENV}),pers-westus3-sre-tooling,sre-tooling-aks))
89+
@$(eval AKS_CLUSTER_NAME = $(or ${AKS_CLUSTER_NAME},${DEFAULT_CLUSTER_NAME}))
90+
@echo "Using cluster name: ${AKS_CLUSTER_NAME}"
91+
az deployment group create \
92+
--name sre-tooling-cluster-${SRE_TOOLING_ENV} \
93+
--resource-group ${SRE_TOOLING_RG} \
94+
--mode complete \
95+
--subscription ${SRE_TOOLING_SUBSCRIPTION_ID} \
96+
--template-file templates/sre-tooling-cluster.bicep \
97+
$(PROMPT_TO_CONFIRM) \
98+
--parameters configurations/sre-tooling-cluster.bicepparam \
99+
--parameters serviceKeyVaultName=${SERVICE_KEYVAULT_NAME} \
100+
--parameters serviceKeyVaultResourceGroup=${SERVICE_KEYVAULT_RG} \
101+
--parameters regionalResourceGroup=${REGIONAL_RG} \
102+
--parameters svcAcrResourceId=${SVC_ACR_RESOURCE_ID} \
103+
--parameters globalMSIId=${GLOBAL_MSI_ID} \
104+
--parameters adminApiMIName=${ADMIN_API_MI_NAME} \
105+
--parameters aksKeyVaultName=sre-tooling-${SRE_TOOLING_ENV}-etcd-kv \
106+
--parameters aksKeyVaultTagValue=${SRE_TOOLING_ENV} \
107+
--parameters aksClusterName=${AKS_CLUSTER_NAME} \
108+
$(if $(AZURE_MONITORING_WORKSPACE_ID),--parameters azureMonitoringWorkspaceId=${AZURE_MONITORING_WORKSPACE_ID})
109+
.PHONY: sre-tooling-cluster
110+
111+
sre-tooling-cluster.what-if:
112+
@[ "${SRE_TOOLING_ENV}" ] || ( echo ">> SRE_TOOLING_ENV is not set (dev or pers)"; exit 1 )
113+
@[ "${SRE_TOOLING_ENV}" = "dev" ] || [ "${SRE_TOOLING_ENV}" = "pers" ] || ( echo ">> SRE_TOOLING_ENV must be 'dev' or 'pers', got: ${SRE_TOOLING_ENV}"; exit 1 )
114+
@[ "${SRE_TOOLING_RG}" ] || ( echo ">> SRE_TOOLING_RG is not set"; exit 1 )
115+
@[ "${SRE_TOOLING_SUBSCRIPTION_ID}" ] || ( echo ">> SRE_TOOLING_SUBSCRIPTION_ID is not set"; exit 1 )
116+
@[ "${SERVICE_KEYVAULT_NAME}" ] || ( echo ">> SERVICE_KEYVAULT_NAME is not set"; exit 1 )
117+
@[ "${SERVICE_KEYVAULT_RG}" ] || ( echo ">> SERVICE_KEYVAULT_RG is not set"; exit 1 )
118+
@[ "${REGIONAL_RG}" ] || ( echo ">> REGIONAL_RG is not set"; exit 1 )
119+
@[ "${SVC_ACR_RESOURCE_ID}" ] || ( echo ">> SVC_ACR_RESOURCE_ID is not set"; exit 1 )
120+
@[ "${GLOBAL_MSI_ID}" ] || ( echo ">> GLOBAL_MSI_ID is not set"; exit 1 )
121+
@[ "${ADMIN_API_MI_NAME}" ] || ( echo ">> ADMIN_API_MI_NAME is not set"; exit 1 )
122+
@$(eval DEFAULT_CLUSTER_NAME = $(if $(filter pers,${SRE_TOOLING_ENV}),pers-westus3-sre-tooling,sre-tooling-aks))
123+
@$(eval AKS_CLUSTER_NAME = $(or ${AKS_CLUSTER_NAME},${DEFAULT_CLUSTER_NAME}))
124+
@echo "Using cluster name: ${AKS_CLUSTER_NAME}"
125+
az deployment group what-if \
126+
--name sre-tooling-cluster-${SRE_TOOLING_ENV} \
127+
--resource-group ${SRE_TOOLING_RG} \
128+
--subscription ${SRE_TOOLING_SUBSCRIPTION_ID} \
129+
--template-file templates/sre-tooling-cluster.bicep \
130+
--parameters configurations/sre-tooling-cluster.bicepparam \
131+
--parameters serviceKeyVaultName=${SERVICE_KEYVAULT_NAME} \
132+
--parameters serviceKeyVaultResourceGroup=${SERVICE_KEYVAULT_RG} \
133+
--parameters regionalResourceGroup=${REGIONAL_RG} \
134+
--parameters svcAcrResourceId=${SVC_ACR_RESOURCE_ID} \
135+
--parameters globalMSIId=${GLOBAL_MSI_ID} \
136+
--parameters adminApiMIName=${ADMIN_API_MI_NAME} \
137+
--parameters aksKeyVaultName=sre-tooling-${SRE_TOOLING_ENV}-etcd-kv \
138+
--parameters aksKeyVaultTagValue=${SRE_TOOLING_ENV} \
139+
--parameters aksClusterName=${AKS_CLUSTER_NAME} \
140+
$(if $(AZURE_MONITORING_WORKSPACE_ID),--parameters azureMonitoringWorkspaceId=${AZURE_MONITORING_WORKSPACE_ID})
141+
.PHONY: sre-tooling-cluster.what-if
142+

‎dev-infrastructure/configurations/sre-tooling-cluster.bicepparam‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
using '../templates/sre-tooling-cluster.bicep'
2+
3+
// Location
4+
param location = 'westus3'
5+
6+
// AKS Cluster
7+
// Note: This will be overridden by Makefile based on SRE_TOOLING_ENV
8+
// Default: 'sre-tooling-aks' for dev, 'pers-westus3-sre-tooling' for pers
9+
param aksClusterName = 'sre-tooling-aks'
10+
param kubernetesVersion = '1.32'
11+
param vnetAddressPrefix = '10.0.0.0/16'
12+
param subnetPrefix = '10.0.0.0/24'
13+
param podSubnetPrefix = '10.0.1.0/24'
14+
15+
// System Agent Pool
16+
param systemAgentMinCount = 2
17+
param systemAgentMaxCount = 3
18+
param systemAgentPoolCount = 1
19+
param systemAgentPoolZones = '1,2,3'
20+
param systemAgentVMSize = 'Standard_D2s_v3'
21+
param systemZoneRedundantMode = 'Zone'
22+
param aksSystemOsDiskSizeGB = 32
23+
24+
// User Agent Pool
25+
param userAgentMinCount = 1
26+
param userAgentMaxCount = 3
27+
param userAgentVMSize = 'Standard_D2s_v3'
28+
param userAgentPoolCount = 1
29+
param userAgentPoolZones = '1,2,3'
30+
param userZoneRedundantMode = 'Zone'
31+
param userOsDiskSizeGB = 32
32+
33+
// Infra Agent Pool (for Prometheus)
34+
param infraAgentMinCount = 1
35+
param infraAgentMaxCount = 2
36+
param infraAgentVMSize = 'Standard_D4s_v3'
37+
param infraAgentPoolCount = 1
38+
param infraAgentPoolZones = '1,2,3'
39+
param infraZoneRedundantMode = 'Zone'
40+
param infraOsDiskSizeGB = 64
41+
42+
// Network
43+
param aksNetworkDataplane = 'azure'
44+
param aksNetworkPolicy = 'azure'
45+
46+
// Key Vault for AKS etcd
47+
param aksKeyVaultName = ''
48+
param aksKeyVaultTagName = 'aro-hcp-environment'
49+
param aksKeyVaultTagValue = 'dev'
50+
param aksEtcdKVEnableSoftDelete = true
51+
param aksClusterOutboundIPAddressIPTags = ''
52+
53+
// These will be overridden via command line
54+
param svcAcrResourceId = ''
55+
param serviceKeyVaultName = ''
56+
param serviceKeyVaultResourceGroup = ''
57+
param regionalResourceGroup = ''
58+
param globalMSIId = ''
59+
param azureMonitoringWorkspaceId = ''
60+
param logsNamespace = 'logs'
61+
param logsMSI = 'logs-msi'
62+
param logsServiceAccount = 'logs-service-account'
63+
param adminApiMIName = ''
64+
param adminApiNamespace = 'admin-api'
65+
param adminApiServiceAccountName = 'admin-api-service-account'
66+

‎dev-infrastructure/configurations/sre-tooling-infra.bicepparam‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
using '../templates/sre-tooling-infra.bicep'
2+
3+
// These will be overridden via command line in Makefile
4+
param serviceKeyVaultName = ''
5+
param serviceKeyVaultResourceGroup = ''
6+
param serviceKeyVaultLocation = 'westus3'
7+
param serviceKeyVaultSoftDelete = true
8+
param serviceKeyVaultPrivate = true
9+
param serviceKeyVaultTagName = 'aro-hcp-environment'
10+
param serviceKeyVaultTagValue = 'dev'
11+
param globalMSIId = ''
12+
param kvCertOfficerPrincipalId = ''
13+

‎dev-infrastructure/templates/output-sre-tooling-cluster.bicep‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
import { safeTake } from '../modules/common.bicep'
2+
3+
@description('Azure Region Location')
4+
param location string = resourceGroup().location
5+
6+
@description('AKS cluster name')
7+
param aksClusterName string
8+
9+
@description('The managed identity name of the logs')
10+
param logsMSI string
11+
12+
@description('The name of the Admin API managed identity')
13+
param adminApiMIName string
14+
15+
var dceName = safeTake('MSProm-${location}-${aksClusterName}', 44)
16+
var dcrName = safeTake('MSProm-${location}-${aksClusterName}', 44)
17+
18+
resource dce 'Microsoft.Insights/dataCollectionEndpoints@2022-06-01' existing = {
19+
name: dceName
20+
}
21+
22+
resource dcr 'Microsoft.Insights/dataCollectionRules@2022-06-01' existing = {
23+
name: dcrName
24+
}
25+
26+
resource prometheusUAMI 'Microsoft.ManagedIdentity/userAssignedIdentities@2024-11-30' existing = {
27+
name: 'prometheus'
28+
}
29+
30+
resource logsUAMI 'Microsoft.ManagedIdentity/userAssignedIdentities@2024-11-30' existing = {
31+
name: logsMSI
32+
}
33+
34+
resource adminApiUAMI 'Microsoft.ManagedIdentity/userAssignedIdentities@2024-11-30' existing = {
35+
name: adminApiMIName
36+
}
37+
38+
output dcrRemoteWriteUrl string = '${dce.properties.metricsIngestion.endpoint}/dataCollectionRules/${dcr.properties.immutableId}/streams/Microsoft-PrometheusMetrics/api/v1/write?api-version=2023-04-24'
39+
output hcpDcrRemoteWriteUrl string = 'NONE'
40+
output prometheusUAMIClientId string = prometheusUAMI.properties.clientId
41+
output clusterLogPrincipalId string = logsUAMI.properties.principalId
42+
output adminApiPrincipalId string = adminApiUAMI.properties.principalId

0 commit comments

Comments
 (0)