oc-mirror deployment on container app

Author: geoberle

.github/workflows/services-cd.yml

@@ -96,10 +96,6 @@
           run: |
             make maestro.server.deploy maestro.registration.deploy
 
-        - name: 'Deploy Image Sync'
-          run: |
-            make imagesync.deploy
-
         - name: 'Deploy Prometheus Config'
           run: |
             make metrics.deploy

Makefile

@@ -41,10 +41,18 @@ infra.svc:
 	@cd dev-infrastructure && DEPLOY_ENV=$(DEPLOY_ENV) make svc.init
 .PHONY: infra.svc
 
+infra.svc.aks.kubeconfigfile:
+	@cd dev-infrastructure && DEPLOY_ENV=$(DEPLOY_ENV) make svc.aks.kubeconfigfile
+.PHONY: infra.svc.aks.kubeconfigfile
+
 infra.mgmt:
 	@cd dev-infrastructure && DEPLOY_ENV=$(DEPLOY_ENV) make mgmt.init
 .PHONY: infra.mgmt
 
+infra.mgmt.aks.kubeconfigfile:
+	@cd dev-infrastructure && DEPLOY_ENV=$(DEPLOY_ENV) make mgmt.aks.kubeconfigfile
+.PHONY: infra.mgmt.aks.kubeconfigfile
+
 infra.imagesync:
 	@cd dev-infrastructure && DEPLOY_ENV=$(DEPLOY_ENV) make imagesync
 .PHONY: infra.imagesync
@@ -152,14 +160,6 @@ hypershift.deploy:
 	@./svc-deploy.sh $(DEPLOY_ENV) hypershiftoperator mgmt
 .PHONY: hypershift.deploy
 
-#
-# Image Sync
-#
-
-imagesync.deploy:
-	@./svc-deploy.sh $(DEPLOY_ENV) image-sync/deployment svc
-.PHONY: imagesync.deploy
-
 #
 # Deploy ALL components
 #

config/config.yaml

@@ -56,6 +56,8 @@ defaults:
   imageSyncRepositories: registry.k8s.io/external-dns/external-dns,quay.io/acm-d/rhtap-hypershift-operator,quay.io/app-sre/uhc-clusters-service
   imageSyncImageRepo: image-sync/component-sync
   imageSyncImageTag: latest
+  ocMirrorImageRepo: image-sync/oc-mirror
+  ocMirrorImageTag: 7abc8af
 
   # Service KeyVault
   serviceKeyVaultName: {{ azureKeyVaultName "svc-kv" 5 .ctx.region .ctx.regionStamp }}

dev-infrastructure/configurations/image-sync.tmpl.bicepparam

@@ -7,9 +7,12 @@ param keyVaultName = '{{ .serviceKeyVaultName}}'
 param keyVaultResourceGroup = '{{ .serviceKeyVaultRG }}'
 
 param bearerSecretName = 'bearer-secret'
-param pullSecretName = 'component-sync-pull-secret'
-
+param componentSyncPullSecretName = 'component-sync-pull-secret'
 param componentSyncImage = '{{ .svcAcrName }}.azurecr.io/{{ .imageSyncImageRepo }}:{{ .imageSyncImageTag }}'
 param svcAcrName = '{{ .svcAcrName }}'
+
+param ocpAcrName = '{{ .ocpAcrName }}'
+param ocpPullSecretName = 'pull-secret'
 param repositoriesToSync = '{{ .imageSyncRepositories }}'
+param ocMirrorImage = '{{ .svcAcrName }}.azurecr.io/{{ .ocMirrorImageRepo }}:{{ .ocMirrorImageTag }}'
 param numberOfTags = 10

dev-infrastructure/docs/development-setup.md

@@ -117,7 +117,12 @@ To create the service cluster, management cluster and supporting infrastructure
   SKIP_CONFIRM=1 make infra.all
   ```
 
-Running this the first time takes around 60 minutes.
+Running this the first time takes around 60 minutes. Afterwards you can access your clusters with
+
+  ```bash
+  export KUBECONFIG=$(make infra.svc.aks.kubeconfigfile)
+  export KUBECONFIG=$(make infra.mgmt.aks.kubeconfigfile)
+  ```
 
 If you only need a management cluster or service cluster for development work, consider using one of the following commands. They take less time and the resulting infrastructure costs less money
 

dev-infrastructure/templates/image-sync.bicep

@@ -16,14 +16,17 @@ param acrResourceGroup string
 @description('Name of the service component ACR registry')
 param svcAcrName string
 
+@description('Name of the OCP ACR registry')
+param ocpAcrName string
+
 @description('Name of the keyvault where the pull secret is stored')
 param keyVaultName string
 
 @description('Name of the KeyVault RG')
 param keyVaultResourceGroup string
 
-@description('The name of the pull secret')
-param pullSecretName string
+@description('The name of the pull secret for the component sync job')
+param componentSyncPullSecretName string
 
 @description('The name of the Quay API bearer token secret')
 param bearerSecretName string
@@ -37,6 +40,12 @@ param repositoriesToSync string
 @description('The number of tags to sync per image in the repo list')
 param numberOfTags int = 10
 
+@description('The image to use for the oc-mirror job')
+param ocMirrorImage string
+
+@description('The name of the pull secret for the oc-mirror job')
+param ocpPullSecretName string
+
 //
 // Container App Infra
 //
@@ -94,7 +103,7 @@ module acrPullRole '../modules/acr-permissions.bicep' = {
 }
 
 module pullSecretPermission '../modules/keyvault/keyvault-secret-access.bicep' = [
-  for secretName in [pullSecretName, bearerSecretName]: {
+  for secretName in [componentSyncPullSecretName, bearerSecretName, ocpPullSecretName]: {
     name: guid(imageSyncManagedIdentity, location, keyVaultName, secretName, 'secret-user')
     scope: resourceGroup(keyVaultResourceGroup)
     params: {
@@ -110,11 +119,11 @@ module pullSecretPermission '../modules/keyvault/keyvault-secret-access.bicep' =
 // Component sync job
 //
 
-var jobName = 'component-sync'
+var componentSyncJobName = 'component-sync'
 var pullSecretFile = 'quayio-auth.json'
 
 resource componentSyncJob 'Microsoft.App/jobs@2024-03-01' = {
-  name: jobName
+  name: componentSyncJobName
   location: location
 
   identity: {
@@ -143,7 +152,7 @@ resource componentSyncJob 'Microsoft.App/jobs@2024-03-01' = {
       secrets: [
         {
           name: 'pull-secrets'
-          keyVaultUrl: 'https://${keyVaultName}${environment().suffixes.keyvaultDns}/secrets/${pullSecretName}'
+          keyVaultUrl: 'https://${keyVaultName}${environment().suffixes.keyvaultDns}/secrets/${componentSyncPullSecretName}'
           identity: uami.id
         }
         {
@@ -156,7 +165,7 @@ resource componentSyncJob 'Microsoft.App/jobs@2024-03-01' = {
     template: {
       containers: [
         {
-          name: jobName
+          name: componentSyncJobName
           image: componentSyncImage
           volumeMounts: [
             { volumeName: 'pull-secrets-updated', mountPath: '/auth' }
@@ -213,3 +222,142 @@ resource componentSyncJob 'Microsoft.App/jobs@2024-03-01' = {
     }
   }
 }
+
+// oc-mirror job
+
+var ocMirrorJobName = 'oc-mirror'
+
+var ocMirrorConfig = {
+  kind: 'ImageSetConfiguration'
+  apiVersion: 'mirror.openshift.io/v1alpha2'
+  storageConfig: {
+    registry: {
+      imageURL: '${ocpAcrName}${environment().suffixes.acrLoginServer}/mirror/oc-mirror-metadata'
+      skipTLS: false
+    }
+  }
+  mirror: {
+    platform: {
+      architectures: ['multi']
+      channels: [
+        {
+          name: 'stable-4.16'
+          type: 'ocp'
+          full: true
+        }
+        {
+          name: 'stable-4.17'
+          type: 'ocp'
+          full: true
+        }
+      ]
+      graph: true
+    }
+    additionalImages: [
+      { name: 'registry.redhat.io/redhat/redhat-operator-index:v4.16' }
+      { name: 'registry.redhat.io/redhat/certified-operator-index:v4.16' }
+      { name: 'registry.redhat.io/redhat/community-operator-index:v4.16' }
+      { name: 'registry.redhat.io/redhat/redhat-marketplace-index:v4.16' }
+      { name: 'registry.redhat.io/redhat/redhat-operator-index:v4.17' }
+      { name: 'registry.redhat.io/redhat/certified-operator-index:v4.17' }
+      { name: 'registry.redhat.io/redhat/community-operator-index:v4.17' }
+      { name: 'registry.redhat.io/redhat/redhat-marketplace-index:v4.17' }
+    ]
+  }
+}
+
+resource ocMirrorJob 'Microsoft.App/jobs@2024-03-01' = {
+  name: ocMirrorJobName
+  location: location
+  identity: {
+    type: 'UserAssigned'
+    userAssignedIdentities: {
+      '${uami.id}': {}
+    }
+  }
+
+  properties: {
+    environmentId: containerAppEnvironment.id
+    configuration: {
+      eventTriggerConfig: {}
+      triggerType: 'Schedule'
+      manualTriggerConfig: {
+        parallelism: 1
+      }
+      scheduleTriggerConfig: {
+        cronExpression: '0 * * * *'
+        parallelism: 1
+      }
+      replicaTimeout: 4 * 60 * 60
+      registries: [
+        {
+          identity: uami.id
+          server: '${svcAcrName}${environment().suffixes.acrLoginServer}'
+        }
+      ]
+      secrets: [
+        {
+          name: 'pull-secrets'
+          keyVaultUrl: 'https://${keyVaultName}${environment().suffixes.keyvaultDns}/secrets/${ocpPullSecretName}'
+          identity: uami.id
+        }
+      ]
+    }
+    template: {
+      containers: [
+        {
+          name: ocMirrorJobName
+          image: ocMirrorImage
+          volumeMounts: [
+            { volumeName: 'pull-secrets-updated', mountPath: '/etc/containers' }
+          ]
+          env: [
+            { name: 'IMAGE_SET_CONFIG', value: base64(string(ocMirrorConfig)) }
+            { name: 'REGISTRY', value: ocpAcrName }
+            { name: 'REGISTRY_URL', value: '${ocpAcrName}${environment().suffixes.acrLoginServer}' }
+            { name: 'XDG_RUNTIME_DIR', value: '/etc' }
+            { name: 'AZURE_CLIENT_ID', value: uami.properties.clientId }
+            {
+              name: 'APPSETTING_WEBSITE_SITE_NAME'
+              value: 'workaround - https://github.com/microsoft/azure-container-apps/issues/502'
+            }
+          ]
+          resources: {
+            cpu: 2
+            memory: '4Gi'
+          }
+        }
+      ]
+      initContainers: [
+        {
+          name: 'decodesecrets'
+          image: 'mcr.microsoft.com/azure-cli:cbl-mariner2.0'
+          command: [
+            '/bin/sh'
+          ]
+          args: [
+            '-c'
+            'cat /tmp/secret-orig/pull-secrets | base64 -d > /etc/containers/auth.json'
+          ]
+          volumeMounts: [
+            { volumeName: 'pull-secrets-updated', mountPath: '/etc/containers' }
+            { volumeName: 'pull-secrets', mountPath: '/tmp/secret-orig' }
+          ]
+        }
+      ]
+      volumes: [
+        {
+          name: 'pull-secrets-updated'
+          storageType: 'EmptyDir'
+        }
+        {
+          name: 'pull-secrets'
+          storageType: 'Secret'
+          secrets: [
+            { secretRef: 'pull-secrets' }
+          ]
+        }
+      ]
+    }
+  }
+}