AGENT-329: add mce deployment (#1448)

* agent: add config var to deploy mce

* Managed cluster image set template

* Add doc for the AGENT_DEPLOY_MCE config var

* Fixed conflicts and getReleaseImage

agent/04_agent_configure.sh

@@ -63,7 +63,6 @@ function get_static_ips_and_macs() {
 function generate_cluster_manifests() {
 
   MANIFESTS_PATH="${OCP_DIR}/cluster-manifests"
-  EXTRA_MANIFESTS_PATH="${OCP_DIR}/openshift"
   MIRROR_PATH="${OCP_DIR}/mirror"
 
   # Fetch current OpenShift version from the release payload
@@ -73,7 +72,6 @@ function generate_cluster_manifests() {
   if [ ! -z "${MIRROR_IMAGES}" ]; then
     mkdir -p ${MIRROR_PATH}
   fi
-  mkdir -p ${EXTRA_MANIFESTS_PATH}
 
   if [[ "$IP_STACK" = "v4" ]]; then
     CLUSTER_NETWORK=${CLUSTER_SUBNET_V4}
@@ -135,23 +133,13 @@ spec:
     name: pull-secret
 EOF
 
-    local releaseImage=${OPENSHIFT_RELEASE_IMAGE}
-    if [ ! -z "${MIRROR_IMAGES}" ]; then
-        releaseImage="${OPENSHIFT_INSTALL_RELEASE_IMAGE_OVERRIDE}"
-    # If not installing from src, let's use the current version from the binary
-    elif [ -z "$KNI_INSTALL_FROM_GIT" ]; then
-      local openshift_install="$(realpath "${OCP_DIR}/openshift-install")"
-      releaseImage=$("${openshift_install}" --dir="${OCP_DIR}" version | grep "release image" | cut -d " " -f 3)
-      echo "Setting release image to ${releaseImage}"
-    fi
-
     cat > "${MANIFESTS_PATH}/cluster-image-set.yaml" << EOF
 apiVersion: hive.openshift.io/v1
 kind: ClusterImageSet
 metadata:
   name: openshift-${VERSION}
 spec:
-  releaseImage: $releaseImage
+  releaseImage: $(getReleaseImage)
 EOF
 
 if [ ! -z "${MIRROR_IMAGES}" ]; then
@@ -260,8 +248,17 @@ spec:
       macAddress: ${AGENT_NODES_MACS[i]}
 ---
 EOF
+    done
+
+    set -x
+}
 
- cat > "${EXTRA_MANIFESTS_PATH}/agent-test.yaml" << EOF
+function generate_extra_cluster_manifests() {
+
+  EXTRA_MANIFESTS_PATH="${OCP_DIR}/openshift"
+  mkdir -p ${EXTRA_MANIFESTS_PATH}
+
+cat > "${EXTRA_MANIFESTS_PATH}/agent-test.yaml" << EOF
 apiVersion: v1
 kind: ConfigMap
 metadata:
@@ -271,9 +268,9 @@ data:
   value: agent-test
 EOF
 
-    done
-
-    set -x
+  if [ ! -z "${AGENT_DEPLOY_MCE}" ]; then
+    cp ${SCRIPTDIR}/agent/assets/mce/agent_mce_0_*.yaml ${EXTRA_MANIFESTS_PATH}
+  fi
 }
 
 write_pull_secret
@@ -298,3 +295,4 @@ if [[ "${NUM_MASTERS}" > "1" ]]; then
 fi
 
 generate_cluster_manifests
+generate_extra_cluster_manifests

agent/05_agent_create_cluster.sh

@@ -9,6 +9,7 @@ source $SCRIPTDIR/common.sh
 source $SCRIPTDIR/network.sh
 source $SCRIPTDIR/utils.sh
 source $SCRIPTDIR/validation.sh
+source $SCRIPTDIR/ocp_install_env.sh
 source $SCRIPTDIR/agent/common.sh
 
 early_deploy_validation
@@ -96,6 +97,51 @@ function wait_for_cluster_ready() {
   echo "Cluster is ready!"
 }
 
+function mce_prepare_postinstallation_manifests() {
+  local mceManifests=$1
+
+  # Copy all the manifests required after the installation completed
+  cp ${SCRIPTDIR}/agent/assets/mce/agent_mce_1*.yaml ${mceManifests}
+
+  # Render the cluster image set template
+  local clusterImageSetTemplate=${mceManifests}/agent_mce_1_04_clusterimageset.yaml
+  local version="$(openshift_version ${OCP_DIR})"
+  local releaseImage=$(getReleaseImage)
+
+  sed -i "s/<version>/${version}/g" ${clusterImageSetTemplate}
+  sed -i "s/<releaseImage>/${releaseImage//\//\\/}/g" ${clusterImageSetTemplate}
+}
+
+function mce_apply_postinstallation_manifests() {
+  local mceManifests=$1
+
+  wait_for_crd "localvolumes.local.storage.openshift.io"
+  apply_manifest "$mceManifests/agent_mce_1_01_localvolumes.yaml"
+  oc wait localvolume -n openshift-local-storage assisted-service --for condition=Available --timeout 10m || exit 1
+
+  wait_for_crd "multiclusterengines.multicluster.openshift.io"
+  apply_manifest "$mceManifests/agent_mce_1_02_mce.yaml"
+
+  wait_for_crd "agentserviceconfigs.agent-install.openshift.io"
+  apply_manifest "$mceManifests/agent_mce_1_03_agentserviceconfig.yaml"
+
+  wait_for_crd "clusterimagesets.hive.openshift.io"
+  apply_manifest "$mceManifests/agent_mce_1_04_clusterimageset.yaml"
+
+  apply_manifest "$mceManifests/agent_mce_1_05_autoimport.yaml"
+  oc wait -n multicluster-engine managedclusters local-cluster --for condition=ManagedClusterJoined=True --timeout 10m || exit 1
+
+  echo "MCE deployment completed"
+}
+
+function mce_complete_deployment() {
+  local mceManifests="${OCP_DIR}/mce"
+  mkdir -p ${mceManifests}
+
+  mce_prepare_postinstallation_manifests ${mceManifests}
+  mce_apply_postinstallation_manifests ${mceManifests}
+}
+
 create_image
 
 attach_agent_iso master $NUM_MASTERS
@@ -109,11 +155,15 @@ if [ ! -z "${AGENT_ENABLE_GUI:-}" ]; then
   enable_assisted_service_ui
 fi
 
-
 wait_for_cluster_ready
 
 # Temporary fix for the CI. To be removed once we'll 
 # be able to generate the cluster credentials
 if [ ! -f "${OCP_DIR}/auth/kubeadmin-password" ]; then
     oc patch --kubeconfig="${OCP_DIR}/auth/kubeconfig" secret -n kube-system kubeadmin --type json -p '[{"op": "replace", "path": "/data/kubeadmin", "value": "'"$(openssl rand -base64 18 | tr -d '\n' | tee "${OCP_DIR}/auth/kubeadmin-password" | htpasswd -nBi -C 10 "" | tr -d ':\n' | sed -e 's/\$2y\$/$2a$/' | base64 -w 0 -)"'"}]'
 fi
+
+if [ ! -z "${AGENT_DEPLOY_MCE}" ]; then
+  mce_complete_deployment
+fi
+

agent/assets/mce/agent_mce_0_01_lso_namespace.yaml

@@ -0,0 +1,6 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  annotations:
+    openshift.io/cluster-monitoring: "true"
+  name: openshift-local-storage

agent/assets/mce/agent_mce_0_02_lso_operatorgroup.yaml

@@ -0,0 +1,9 @@
+
+apiVersion: operators.coreos.com/v1alpha2
+kind: OperatorGroup
+metadata:
+  name: local-operator-group
+  namespace: openshift-local-storage
+spec:
+  targetNamespaces:
+    - openshift-local-storage

agent/assets/mce/agent_mce_0_03_lso_subscription.yaml

@@ -0,0 +1,11 @@
+
+apiVersion: operators.coreos.com/v1alpha1
+kind: Subscription
+metadata:
+  name: local-storage-operator
+  namespace: openshift-local-storage
+spec:
+  installPlanApproval: Automatic
+  name: local-storage-operator
+  source: redhat-operators
+  sourceNamespace: openshift-marketplace

agent/assets/mce/agent_mce_0_03_mce_namespace.yaml

@@ -0,0 +1,6 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  labels:
+    openshift.io/cluster-monitoring: "true"
+  name: multicluster-engine

agent/assets/mce/agent_mce_0_04_mce_operatorgroup.yaml

@@ -0,0 +1,8 @@
+apiVersion: operators.coreos.com/v1
+kind: OperatorGroup
+metadata:
+  name: multicluster-engine-operatorgroup
+  namespace: multicluster-engine
+spec:
+  targetNamespaces:
+  - multicluster-engine

agent/assets/mce/agent_mce_0_05_mce_subscription.yaml

@@ -0,0 +1,10 @@
+apiVersion: operators.coreos.com/v1alpha1
+kind: Subscription
+metadata:
+  name: multicluster-engine
+  namespace: multicluster-engine
+spec:
+  channel: "stable-2.1"
+  name: multicluster-engine
+  source: redhat-operators
+  sourceNamespace: openshift-marketplace

agent/assets/mce/agent_mce_1_01_localvolumes.yaml

@@ -0,0 +1,14 @@
+apiVersion: local.storage.openshift.io/v1
+kind: LocalVolume
+metadata:
+  name: assisted-service
+  namespace: openshift-local-storage
+spec:
+  logLevel: Normal
+  managementState: Managed
+  storageClassDevices:
+    - devicePaths:
+        - /dev/vda
+        - /dev/vdb
+      storageClassName: assisted-service
+      volumeMode: Filesystem

agent/assets/mce/agent_mce_1_02_mce.yaml

@@ -0,0 +1,5 @@
+apiVersion: multicluster.openshift.io/v1
+kind: MultiClusterEngine
+metadata:
+  name: multiclusterengine
+spec: {}

agent/assets/mce/agent_mce_1_03_agentserviceconfig.yaml

@@ -0,0 +1,20 @@
+apiVersion: agent-install.openshift.io/v1beta1
+kind: AgentServiceConfig
+metadata:
+  name: agent
+  namespace: assisted-installer
+spec:
+ databaseStorage:
+  storageClassName: assisted-service
+  accessModes:
+  - ReadWriteOnce
+  resources:
+   requests:
+    storage: 8Gi
+ filesystemStorage:
+  storageClassName: assisted-service
+  accessModes:
+  - ReadWriteOnce
+  resources:
+   requests:
+    storage: 8Gi

agent/assets/mce/agent_mce_1_04_clusterimageset.yaml

@@ -0,0 +1,6 @@
+apiVersion: hive.openshift.io/v1
+kind: ClusterImageSet
+metadata:
+  name: "<version>"
+spec:
+  releaseImage: <releaseImage>

agent/assets/mce/agent_mce_1_05_autoimport.yaml

@@ -0,0 +1,10 @@
+apiVersion: cluster.open-cluster-management.io/v1
+kind: ManagedCluster
+metadata:
+ labels:
+   local-cluster: "true"
+   cloud: auto-detect
+   vendor: auto-detect
+ name: local-cluster
+spec:
+ hubAcceptsClient: true

agent/common.sh

@@ -10,3 +10,15 @@ if [ -n "$MIRROR_IMAGES" ]; then
     # We're going to be using a locally modified release image
     export OPENSHIFT_INSTALL_RELEASE_IMAGE_OVERRIDE="${LOCAL_REGISTRY_DNS_NAME}:${LOCAL_REGISTRY_PORT}/localimages/local-release-image:${OPENSHIFT_RELEASE_TAG}"
 fi
+
+function getReleaseImage() {
+    local releaseImage=${OPENSHIFT_RELEASE_IMAGE}
+    if [ ! -z "${MIRROR_IMAGES}" ]; then
+        releaseImage="${OPENSHIFT_INSTALL_RELEASE_IMAGE_OVERRIDE}"
+    # If not installing from src, let's use the current version from the binary
+    elif [ -z "$KNI_INSTALL_FROM_GIT" ]; then
+      local openshift_install="$(realpath "${OCP_DIR}/openshift-install")"
+      releaseImage=$("${openshift_install}" --dir="${OCP_DIR}" version | grep "release image" | cut -d " " -f 3)      
+    fi
+    echo ${releaseImage}
+}