create-instance.yml

name: Create Instance

on:
  issue_comment:
    types: [created]

jobs:
  create:
    runs-on: self-hosted
    if:
      ${{ !github.event.issue.pull_request && (
      contains(github.event.comment.body, '/create') ) }}
    steps:
      - name: create command
        id: create_command
        uses: github/command@v1.2.0
        with:
          command: "/create"
          reaction: "rocket"
          allowed_contexts: "issue"
          permissions: "write,maintain,admin"
          allowlist: "${{ vars.MORPHOCLOUD_GITHUB_ADMINS }}"

      - uses: actions/checkout@v4

      - name: Add "request:approved" label
        uses: actions/github-script@v7
        with:
          script: |
            github.rest.issues.addLabels({
              issue_number: context.payload.issue.number,
              owner: context.repo.owner,
              repo: context.repo.repo,
              labels: ["request:approved"]
            })

      - name: comment (progress)
        id: couc_progress
        uses: ./.github/actions/comment-progress
        with:
          create_ip: ⏳
          create_volume: ""
          create_instance: ""
          associate_ip: ""
          setup_instance: ""
          attach_volume: ""
          send_email: ""

      - name: Extract labels
        id: labels
        run: |
          # If there are multiple "flavor:" labels, select the first one
          instance_flavor=$(
            gh issue view $ISSUE_NUMBER \
              --json labels \
              --jq '[.labels[].name | select(. | startswith("flavor:")) | split(":")[1]][0]'
          )
          echo "instance_flavor=$instance_flavor" >> $GITHUB_OUTPUT
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          GH_REPO: ${{ github.repository }}
          ISSUE_NUMBER: ${{ github.event.issue.number }}

      - name: Define instance name
        id: define
        uses: ./.github/actions/define-instance-name
        with:
          prefix: ${{ vars.INSTANCE_NAME_PREFIX }}
          issue_number: ${{ github.event.issue.number }}

      - name: Check instance exists
        id: check_instance
        uses: ./.github/actions/check-instance-exists
        with:
          os_cloud: ${{ vars.MORPHOCLOUD_OS_CLOUD }}
          instance_name: ${{ steps.define.outputs.instance_name }}

      - name: comment (Instance already created)
        if: ${{ fromJSON(steps.check_instance.outputs.exists) }}
        uses: peter-evans/create-or-update-comment@v4.0.0
        with:
          issue-number: ${{ github.event.issue.number }}
          body: |
            ### Instance Creation Results ❌

            Instance **${{ steps.define.outputs.instance_name }}** already created.

            See details at https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}

      - name: instance already exists (failure)
        if: ${{ fromJSON(steps.check_instance.outputs.exists) }}
        run: |
          echo "::error ::Instance $INSTANCE_NAME already created"
          exit 1
        env:
          INSTANCE_NAME: ${{ steps.define.outputs.instance_name }}

      - name: Create floating IP
        id: ip_create
        run: |
          source ~/venv/bin/activate

          json_output=$(openstack floating ip create public -f json)
          echo $json_output

          floating_ip_uuid=$(
            echo $json_output |
            jq -r ".id"
          )
          echo "floating_ip_uuid [$floating_ip_uuid]"
          echo "floating_ip_uuid=$floating_ip_uuid" >> $GITHUB_OUTPUT

          floating_ip_address=$(
            echo $json_output |
            jq -r ".floating_ip_address"
          )
          echo "floating_ip_address [$floating_ip_address]"
          echo "floating_ip_address=$floating_ip_address" >> $GITHUB_OUTPUT
        env:
          OS_CLOUD: ${{ vars.MORPHOCLOUD_OS_CLOUD }}

      - name: comment (floating IP creation failed)
        if: ${{ steps.ip_create.outcome == 'failure' && failure() }}
        uses: peter-evans/create-or-update-comment@v4.0.0
        with:
          issue-number: ${{ github.event.issue.number }}
          body: |
            ### Instance Creation Results ❌

            Failed to create floating IP for **${{ steps.define.outputs.instance_name }}** instance.

            See details at https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}

      - name: comment (progress)
        uses: ./.github/actions/comment-progress
        with:
          comment-id: ${{ steps.couc_progress.outputs.comment-id }}
          create_ip: ✅
          create_volume: ⏳
          create_instance: ""
          associate_ip: ""
          setup_instance: ""
          attach_volume: ""
          send_email: ""

      - name: Define volume name
        id: define_volume_name
        uses: ./.github/actions/define-volume-name
        with:
          issue_number: ${{ github.event.issue.number }}
          suffix: ${{ vars.VOLUME_NAME_SUFFIX }}

      - name: Check volume exists
        id: check_volume
        uses: ./.github/actions/check-volume-exists
        with:
          os_cloud: ${{ vars.MORPHOCLOUD_OS_CLOUD }}
          volume_name: ${{ steps.define_volume_name.outputs.volume_name }}

      - name: comment (volume already exists)
        if: ${{ fromJSON(steps.check_volume.outputs.exists) }}
        uses: peter-evans/create-or-update-comment@v4.0.0
        with:
          issue-number: ${{ github.event.issue.number }}
          body: |
            ### Instance Creation Results ⚠️

            Volume **${{ steps.define_volume_name.outputs.volume_name }}** already exists.

      - name: Create Volume
        if: ${{ ! fromJSON(steps.check_volume.outputs.exists) }}
        id: create_volume
        run: |
          source ~/venv/bin/activate

          openstack volume create --size 100 "${VOLUME_NAME}"
        env:
          OS_CLOUD: ${{ vars.MORPHOCLOUD_OS_CLOUD }}
          VOLUME_NAME: ${{ steps.define_volume_name.outputs.volume_name }}

      - name: comment (failed to create volume)
        if: ${{ steps.create_volume.outcome == 'failure' && failure() }}
        uses: peter-evans/create-or-update-comment@v4.0.0
        with:
          issue-number: ${{ github.event.issue.number }}
          body: |
            ### Instance Creation Results ❌

            Failed to create volume.

            See details at https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}

      - name: comment (progress)
        uses: ./.github/actions/comment-progress
        with:
          comment-id: ${{ steps.couc_progress.outputs.comment-id }}
          create_ip: ✅
          create_volume: ✅
          create_instance: ⏳
          associate_ip: ""
          setup_instance: ""
          attach_volume: ""
          send_email: ""

      - name: Create instance
        id: create_instance
        run: |
          source ~/venv/bin/activate

          echo Creating instance "$INSTANCE_NAME"

          # The "Exosphere client UUID" is unique to each authenticated connection and is
          # stored in the browser's local storage. It can be inspected at:
          #   https://jetstream2.exosphere.app/exosphere/helpabout
          # The associated code is located in the Exosphere project at:
          #   src/LocalStorage/LocalStorage.elm
          #
          # The MorphCloud workflow uses a unique UUID generated with:
          #   python3 -c "import uuid; print(uuid.uuid4())"
          exoClientUuid=b7b75870-ecb5-4aca-ae73-49474a7f531e

          # See "currentExoServerVersion" in exosphere/src/Types/Server.elm
          exoServerVersion=5

          openstack server create "$INSTANCE_NAME" \
            --nic net-id="auto_allocated_network" \
            --security-group "exosphere" \
            --flavor $INSTANCE_FLAVOR \
            --image "Featured-Ubuntu22" \
            --key-name "jcfr" \
            --property "exoGuac={\"v\":1,\"ssh\":true,\"vnc\":true}" \
            --property "exoClientUuid=$exoClientUuid" \
            --property "exoServerVersion=$exoServerVersion" \
            --property "exoCreatorUsername=jfillionrobin@access-ci.org" \
            --property "exoFloatingIpOption=useFloatingIp" \
            --property "exoFloatingIpReuseOption=$FLOATING_IP_UUID" \
            --property "exoSetup={\"status\":\"waiting\",\"epoch\":null}" \
            --user-data ./cloud-config \
            --wait \
            --column created \
            --column flavor \
            --column image \
            --column name \
            --column status

        env:
          OS_CLOUD: ${{ vars.MORPHOCLOUD_OS_CLOUD }}
          INSTANCE_NAME: ${{ steps.define.outputs.instance_name }}
          INSTANCE_FLAVOR: ${{ steps.labels.outputs.instance_flavor }}
          FLOATING_IP_UUID: ${{ steps.ip_create.outputs.floating_ip_uuid }}

      - name: comment (instance creation failed)
        if: ${{ steps.create_instance.outcome == 'failure' && failure() }}
        uses: peter-evans/create-or-update-comment@v4.0.0
        with:
          issue-number: ${{ github.event.issue.number }}
          body: |
            ### Instance Creation Results ❌

            Failed to create instance **${{ steps.define.outputs.instance_name }}**.

            See details at https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}

      - name: comment (progress)
        uses: ./.github/actions/comment-progress
        with:
          comment-id: ${{ steps.couc_progress.outputs.comment-id }}
          create_ip: ✅
          create_volume: ✅
          create_instance: ✅
          associate_ip: ⏳
          setup_instance: ""
          attach_volume: ""
          send_email: ""

      - name: Associate floating IP with created instance
        id: associated_ip
        run: |
          source ~/venv/bin/activate

          has_ip=$(
            openstack server show $INSTANCE_NAME -c addresses -f json | \
            jq -r '.addresses.auto_allocated_network[1] != null'
          )
          echo "has_ip [$has_ip]"

          if [[ $has_ip != "true" ]]; then
            openstack server add floating ip "$INSTANCE_NAME" $FLOATING_IP_ADDRESS
          fi
        env:
          OS_CLOUD: ${{ vars.MORPHOCLOUD_OS_CLOUD }}
          FLOATING_IP_ADDRESS:
            ${{ steps.ip_create.outputs.floating_ip_address }}
          INSTANCE_NAME: ${{ steps.define.outputs.instance_name }}

      - name: comment (failed to associate IP with instance)
        if: ${{ steps.associated_ip.outcome == 'failure' && failure() }}
        uses: peter-evans/create-or-update-comment@v4.0.0
        with:
          issue-number: ${{ github.event.issue.number }}
          body: |
            ### Instance Creation Results ❌

            Failed to associate IP with instance **${{ steps.define.outputs.instance_name }}**.

            See details at https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}

      - name: comment (progress)
        uses: ./.github/actions/comment-progress
        with:
          comment-id: ${{ steps.couc_progress.outputs.comment-id }}
          create_ip: ✅
          create_volume: ✅
          create_instance: ✅
          associate_ip: ✅
          setup_instance: ⏳
          attach_volume: ""
          send_email: ""

      - name: Poll instance setup status
        id: instance_poll
        run: |
          source ~/venv/bin/activate

          echo Polling "$INSTANCE_NAME" setup status

          max_wait_time=1200  # Maximum wait time in seconds (1200s -> 20mins)
          wait_interval=5    # Interval between status checks in seconds
          total_wait_time=0

          while [ $total_wait_time -lt $max_wait_time ]; do

              status=$(openstack console log show $INSTANCE_NAME | \
                grep "^\{\"status\":\"" | \
                tail -1 | \
                jq -r '.status // "pending"')

              echo -n "setup status [$status]. "

              if [[ "$status" == "complete" ]]; then
                  echo "Exiting loop."
                  break
              else
                  echo "Waiting for completion..."
                  sleep $wait_interval
                  total_wait_time=$((total_wait_time + wait_interval))
              fi
          done

          if [ $total_wait_time -ge $max_wait_time ]; then
              echo "::error ::Maximum wait time ($max_wait_time seconds) exceeded."
              exit 1
          fi

          # Explicitly set status to "complete"
          exoSetup='{"status":"complete", "epoch": '$(date '+%s')'000}'

          openstack server set \
              --property "exoSetup=$exoSetup" \
            $INSTANCE_NAME

          echo "status=$status" >> $GITHUB_OUTPUT
        env:
          OS_CLOUD: ${{ vars.MORPHOCLOUD_OS_CLOUD }}
          INSTANCE_NAME: ${{ steps.define.outputs.instance_name }}

      - name: comment (maximum wait time exceeded)
        if: ${{ steps.instance_poll.outcome == 'failure' && failure() }}
        uses: peter-evans/create-or-update-comment@v4.0.0
        with:
          issue-number: ${{ github.event.issue.number }}
          body: |
            ### Instance Creation Results ❌

            Maximum wait time for instance **${{ steps.define.outputs.instance_name }}** setup to complete exceeded.

            See details at https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}

      - name: comment (progress)
        uses: ./.github/actions/comment-progress
        with:
          comment-id: ${{ steps.couc_progress.outputs.comment-id }}
          create_ip: ✅
          create_volume: ✅
          create_instance: ✅
          associate_ip: ✅
          setup_instance: ✅
          attach_volume: ⏳
          send_email: ""

      - name: Rename MyData directory to MyData-tmp
        id: rename_mydata
        run: |
          ssh \
            -o StrictHostKeyChecking=no \
            -o UserKnownHostsFile=/dev/null \
            -o LogLevel=ERROR \
            exouser@$INSTANCE_IP \
            'mv /media/volume/MyData /media/volume/MyData-tmp'
        env:
          INSTANCE_IP: ${{ steps.ip_create.outputs.floating_ip_address }}

      - name: comment (failed to rename MyData)
        if: ${{ steps.rename_mydata.outcome == 'failure' && failure() }}
        uses: peter-evans/create-or-update-comment@v4.0.0
        with:
          issue-number: ${{ github.event.issue.number }}
          body: |
            ### Instance Creation Results ❌

            Failed to rename `/media/volume/MyData` to `/media/volume/MyData-tmp` for instance **${{ steps.define.outputs.instance_name }}**.

            See details at https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}

      - name: Attach Volume
        id: attach_volume
        run: |
          source ~/venv/bin/activate

          instance_id=$(openstack server list -f json | \
            jq \
            --arg instance_name "$INSTANCE_NAME" \
            -c '.[] | select(.Name == $instance_name)' | \
            jq -r '.ID' | tail -1)
          echo "instance_id [$instance_id]"

          volume_id=$(openstack volume list -f json | \
            jq \
            --arg volume_name "$VOLUME_NAME" \
            -c '.[] | select(.Name == $volume_name)' | \
            jq -r '.ID' | tail -1)
          echo "volume_id [$volume_id]"

          openstack server set \
            --property "exoVolumes::$volume_id={\"name\":\"MyData\"}" \
            $instance_id

          openstack server add volume \
            $instance_id \
            $volume_id
        env:
          OS_CLOUD: ${{ vars.MORPHOCLOUD_OS_CLOUD }}
          INSTANCE_NAME: ${{ steps.define.outputs.instance_name }}
          VOLUME_NAME: ${{ steps.define_volume_name.outputs.volume_name }}

      - name: comment (failed to attach volume)
        if: ${{ steps.attach_volume.outcome == 'failure' && failure() }}
        uses: peter-evans/create-or-update-comment@v4.0.0
        with:
          issue-number: ${{ github.event.issue.number }}
          body: |
            ### Instance Creation Results ❌

            Failed to attach volume **${{ steps.define_volume_name.outputs.volume_name }}** to instance **${{ steps.define.outputs.instance_name }}**.

            See details at https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}

      - name: Copy installed files into attached volume
        id: copy_installed_files
        run: |
          function check_mountpoint_ready {
            ssh \
              -o StrictHostKeyChecking=no \
              -o UserKnownHostsFile=/dev/null \
              -o LogLevel=ERROR \
              exouser@$INSTANCE_IP \
              'mountpoint -q /media/volume/MyData'
          }
          set +e
          max_attempts=5
          for attempt in $(seq 1 $max_attempts); do
            echo "Checking if mount point is ready ($attempt/$max_attempts)"
            check_mountpoint_ready
            if check_mountpoint_ready; then
              echo "Mount point /media/volume/MyData is ready."
              break
            else
              echo "Mount point not ready. Retrying in 5 seconds..."
              sleep 5
            fi
          done
          if ! check_mountpoint_ready; then
            echo "::error ::Mount point /media/volume/MyData not ready after $((max_attempts * 5)) seconds."
            exit 1
          fi
          set -e
          ssh \
            -o StrictHostKeyChecking=no \
            -o UserKnownHostsFile=/dev/null \
            -o LogLevel=ERROR \
            exouser@$INSTANCE_IP \
            '[ ! -d /media/volume/MyData/Slicer ] && mv /media/volume/MyData-tmp/Slicer /media/volume/MyData/Slicer && rmdir /media/volume/MyData-tmp || rm -rf /media/volume/MyData-tmp'
        env:
          INSTANCE_IP: ${{ steps.ip_create.outputs.floating_ip_address }}

      - name: comment (failed to copy installed files from MyData-tmp to MyData)
        if: ${{ steps.copy_installed_files.outcome == 'failure' && failure() }}
        uses: peter-evans/create-or-update-comment@v4.0.0
        with:
          issue-number: ${{ github.event.issue.number }}
          body: |
            ### Instance Creation Results ❌

            Failed to copy installed files from `/media/volume/MyData-tmp` to `/media/volume/MyData` for instance **${{ steps.define.outputs.instance_name }}**.

            See details at https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}

      - name: Create Renviron file
        id: create_renviron_file
        run: |
          ssh \
            -o StrictHostKeyChecking=no \
            -o UserKnownHostsFile=/dev/null \
            -o LogLevel=ERROR \
            exouser@$INSTANCE_IP \
            'echo -e "R_LIBS_SITE=/media/share/MorphoCloudCephShare/R/x86_64-pc-linux-gnu-library/4.4/\nR_LIBS_USER=/media/volume/MyData/R/x86_64-pc-linux-gnu-library/4.4/" > /home/exouser/.Renviron && mkdir -p /media/volume/MyData/R/x86_64-pc-linux-gnu-library/4.4/'
        env:
          INSTANCE_IP: ${{ steps.ip_create.outputs.floating_ip_address }}

      - name: comment (failed to create /home/exouser/.Renviron)
        if: ${{ steps.create_renviron_file.outcome == 'failure' && failure() }}
        uses: peter-evans/create-or-update-comment@v4.0.0
        with:
          issue-number: ${{ github.event.issue.number }}
          body: |
            ### Instance Creation Results ❌

            Failed to create `/home/exouser/.Renviron` file for instance **${{ steps.define.outputs.instance_name }}**.

            See details at https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}

      - name: Reboot instance
        id: reboot_instance
        run: |
          set +e

          ssh \
            -o StrictHostKeyChecking=no \
            -o UserKnownHostsFile=/dev/null \
            -o LogLevel=ERROR \
            exouser@$INSTANCE_IP \
            'sudo shutdown -r now'

          sleep 10
          function check_instance_ready {
            ssh \
              -o StrictHostKeyChecking=no \
              -o UserKnownHostsFile=/dev/null \
              -o LogLevel=ERROR \
              exouser@$INSTANCE_IP \
              'true'
          }

          max_attempts=3
          instance_ready=false
          for attempt in $(seq 1 $max_attempts); do
            echo "Checking if instance is ready ($attempt/$max_attempts)"
            if check_instance_ready; then
              instance_ready=true
              echo "Instance '$INSTANCE_NAME' is ready."
              break
            else
              echo "Instance '$INSTANCE_NAME' is not ready. Retrying in 5 seconds..."
              sleep 10
            fi
          done
          if ! $instance_ready; then
            echo "::error ::Instance '$INSTANCE_NAME' is not ready after $max_attempts attempts to connect."
            exit 1
          fi
          set -e
        env:
          INSTANCE_IP: ${{ steps.ip_create.outputs.floating_ip_address }}
          INSTANCE_NAME: ${{ steps.define.outputs.instance_name }}

      - name: comment (failed to reboot instance)
        if: ${{ steps.reboot_instance.outcome == 'failure' && failure() }}
        uses: peter-evans/create-or-update-comment@v4.0.0
        with:
          issue-number: ${{ github.event.issue.number }}
          body: |
            ### Instance Creation Results ❌

            Failed to reboot instance **${{ steps.define.outputs.instance_name }}**.

            See details at https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}

      - name: Set instance properties
        id: set_instance_properties
        run: |
          source ~/venv/bin/activate

          # Explicitly set status to complete
          exoSetup='{"status":"complete", "epoch": '$(date '+%s')'000}'

          openstack server set \
              --property "exoSetup=$exoSetup" \
            $INSTANCE_NAME
        env:
          OS_CLOUD: ${{ vars.MORPHOCLOUD_OS_CLOUD }}
          INSTANCE_NAME: ${{ steps.define.outputs.instance_name }}

      - name: comment (failed to set instance properties)
        if:
          ${{ steps.set_instance_properties.outcome == 'failure' && failure() }}
        uses: peter-evans/create-or-update-comment@v4.0.0
        with:
          issue-number: ${{ github.event.issue.number }}
          body: |
            ### Instance Creation Results ❌

            Failed to set instance **${{ steps.define.outputs.instance_name }}** properties.

            See details at https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}

      - name: Update Request Status Label
        id: update-request-status-label
        uses: ./.github/actions/update-request-status-label
        with:
          os_cloud: ${{ vars.MORPHOCLOUD_OS_CLOUD }}
          token: ${{ secrets.GITHUB_TOKEN }}
          instance_name: ${{ steps.define.outputs.instance_name }}
          issue_number: ${{ github.event.issue.number }}

      - name: comment (failed to update request status label)
        if:
          ${{ steps.update-request-status-label.outcome == 'failure' &&
          failure() }}
        uses: peter-evans/create-or-update-comment@v4.0.0
        with:
          issue-number: ${{ github.event.issue.number }}
          body: |
            ### Instance Creation Results ❌

            Failed to update request status label for instance **${{ steps.define.outputs.instance_name }}**.

            See details at https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}

      - name: comment (progress)
        uses: ./.github/actions/comment-progress
        with:
          comment-id: ${{ steps.couc_progress.outputs.comment-id }}
          create_ip: ✅
          create_volume: ✅
          create_instance: ✅
          associate_ip: ✅
          setup_instance: ✅
          attach_volume: ✅
          send_email: ⏳

      - name: Send mail
        id: send_email
        if: ${{ steps.instance_poll.outputs.status == 'complete' }}
        uses: ./.github/actions/send-email
        with:
          os_cloud: ${{ vars.MORPHOCLOUD_OS_CLOUD }}
          token: ${{ secrets.GITHUB_TOKEN }}
          string_encryption_key: ${{ secrets.STRING_ENCRYPTION_KEY }}
          mail_server_username: ${{secrets.MAIL_USERNAME}}
          mail_server_password: ${{secrets.MAIL_PASSWORD}}
          instance_name: ${{ steps.define.outputs.instance_name }}
          instance_issue_number: ${{ github.event.issue.number }}

      - name: comment (failed to send email)
        if: ${{ steps.send_email.outcome == 'failure' && failure() }}
        uses: peter-evans/create-or-update-comment@v4.0.0
        with:
          issue-number: ${{ github.event.issue.number }}
          body: |
            ### Instance Creation Results ❌

            Failed to send connection email for instance **${{ steps.define.outputs.instance_name }}**. ${{ steps.send_email.outputs.error_message }}

            See details at https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}

      - name: comment (progress)
        uses: ./.github/actions/comment-progress
        with:
          comment-id: ${{ steps.couc_progress.outputs.comment-id }}
          create_ip: ✅
          create_volume: ✅
          create_instance: ✅
          associate_ip: ✅
          setup_instance: ✅
          attach_volume: ✅
          send_email: ✅