From 3ab8af965723b5cfc381bfac9a0baf70060d8006 Mon Sep 17 00:00:00 2001 From: Jean-Christophe Fillion-Robin Date: Sun, 15 Dec 2024 18:06:53 -0500 Subject: [PATCH] fix: Update to MorphoCloud/MorphoCloudWorkflow@1a7ea11 List of MorphoCloudWorkflow changes: ``` $ git shortlog 2dbdd12..1a7ea11 --no-merges Jean-Christophe Fillion-Robin (2): fix(create-instance): Reboot to ensure correct torch version is installed test ``` See https://github.com/MorphoCloud/MorphoCloudWorkflow/compare/2dbdd12...1a7ea11 --- .github/workflows/create-instance.yml | 370 ++------------------------ 1 file changed, 28 insertions(+), 342 deletions(-) diff --git a/.github/workflows/create-instance.yml b/.github/workflows/create-instance.yml index ce531511..0af02e5a 100644 --- a/.github/workflows/create-instance.yml +++ b/.github/workflows/create-instance.yml @@ -68,69 +68,6 @@ jobs: prefix: ${{ vars.INSTANCE_NAME_PREFIX }} issue_number: ${{ github.event.issue.number }} - - name: Check instance exists - id: check_instance - uses: ./.github/actions/check-instance-exists - with: - os_cloud: ${{ vars.MORPHOCLOUD_OS_CLOUD }} - instance_name: ${{ steps.define.outputs.instance_name }} - - - name: comment (Instance already created) - if: ${{ fromJSON(steps.check_instance.outputs.exists) }} - uses: peter-evans/create-or-update-comment@v4.0.0 - with: - issue-number: ${{ github.event.issue.number }} - body: | - ### Instance Creation Results ❌ - - Instance **${{ steps.define.outputs.instance_name }}** already created. - - See details at https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} - - - name: instance already exists (failure) - if: ${{ fromJSON(steps.check_instance.outputs.exists) }} - run: | - echo "::error ::Instance $INSTANCE_NAME already created" - exit 1 - env: - INSTANCE_NAME: ${{ steps.define.outputs.instance_name }} - - - name: Create floating IP - id: ip_create - run: | - source ~/venv/bin/activate - - json_output=$(openstack floating ip create public -f json) - echo $json_output - - floating_ip_uuid=$( - echo $json_output | - jq -r ".id" - ) - echo "floating_ip_uuid [$floating_ip_uuid]" - echo "floating_ip_uuid=$floating_ip_uuid" >> $GITHUB_OUTPUT - - floating_ip_address=$( - echo $json_output | - jq -r ".floating_ip_address" - ) - echo "floating_ip_address [$floating_ip_address]" - echo "floating_ip_address=$floating_ip_address" >> $GITHUB_OUTPUT - env: - OS_CLOUD: ${{ vars.MORPHOCLOUD_OS_CLOUD }} - - - name: comment (floating IP creation failed) - if: ${{ steps.ip_create.outcome == 'failure' && failure() }} - uses: peter-evans/create-or-update-comment@v4.0.0 - with: - issue-number: ${{ github.event.issue.number }} - body: | - ### Instance Creation Results ❌ - - Failed to create floating IP for **${{ steps.define.outputs.instance_name }}** instance. - - See details at https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} - - name: comment (progress) uses: ./.github/actions/comment-progress with: @@ -202,316 +139,65 @@ jobs: attach_volume: "" send_email: "" - - name: Create instance - id: create_instance - run: | - source ~/venv/bin/activate - - echo Creating instance "$INSTANCE_NAME" - - # See https://jetstream2.exosphere.app/exosphere/helpabout - exoClientUuid=67296a2e-069b-49ca-9ca4-5dd296869ada - - # See "currentExoServerVersion" in exosphere/src/Types/Server.elm - exoServerVersion=5 - - openstack server create "$INSTANCE_NAME" \ - --nic net-id="auto_allocated_network" \ - --security-group "exosphere" \ - --flavor $INSTANCE_FLAVOR \ - --image "Featured-Ubuntu22" \ - --key-name "jcfr" \ - --property "exoGuac={\"v\":1,\"ssh\":true,\"vnc\":true}" \ - --property "exoClientUuid=$exoClientUuid" \ - --property "exoServerVersion=$exoServerVersion" \ - --property "exoCreatorUsername=jfillionrobin@access-ci.org" \ - --property "exoFloatingIpOption=useFloatingIp" \ - --property "exoFloatingIpReuseOption=$FLOATING_IP_UUID" \ - --property "exoSetup={\"status\":\"waiting\",\"epoch\":null}" \ - --user-data ./cloud-config \ - --wait \ - --column created \ - --column flavor \ - --column image \ - --column name \ - --column status - - env: - OS_CLOUD: ${{ vars.MORPHOCLOUD_OS_CLOUD }} - INSTANCE_NAME: ${{ steps.define.outputs.instance_name }} - INSTANCE_FLAVOR: ${{ steps.labels.outputs.instance_flavor }} - FLOATING_IP_UUID: ${{ steps.ip_create.outputs.floating_ip_uuid }} - - - name: comment (instance creation failed) - if: ${{ steps.create_instance.outcome == 'failure' && failure() }} - uses: peter-evans/create-or-update-comment@v4.0.0 - with: - issue-number: ${{ github.event.issue.number }} - body: | - ### Instance Creation Results ❌ - - Failed to create instance **${{ steps.define.outputs.instance_name }}**. - - See details at https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} - - - name: comment (progress) - uses: ./.github/actions/comment-progress - with: - comment-id: ${{ steps.couc_progress.outputs.comment-id }} - create_ip: ✅ - create_volume: ✅ - create_instance: ✅ - associate_ip: ⏳ - setup_instance: "" - attach_volume: "" - send_email: "" - - - name: Associate floating IP with created instance - id: associated_ip - run: | - source ~/venv/bin/activate - - has_ip=$( - openstack server show $INSTANCE_NAME -c addresses -f json | \ - jq -r '.addresses.auto_allocated_network[1] != null' - ) - echo "has_ip [$has_ip]" - - if [[ $has_ip != "true" ]]; then - openstack server add floating ip "$INSTANCE_NAME" $FLOATING_IP_ADDRESS - fi - env: - OS_CLOUD: ${{ vars.MORPHOCLOUD_OS_CLOUD }} - FLOATING_IP_ADDRESS: - ${{ steps.ip_create.outputs.floating_ip_address }} - INSTANCE_NAME: ${{ steps.define.outputs.instance_name }} - - - name: comment (failed to associate IP with instance) - if: ${{ steps.associated_ip.outcome == 'failure' && failure() }} - uses: peter-evans/create-or-update-comment@v4.0.0 - with: - issue-number: ${{ github.event.issue.number }} - body: | - ### Instance Creation Results ❌ - - Failed to associate IP with instance **${{ steps.define.outputs.instance_name }}**. - - See details at https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} - - - name: comment (progress) - uses: ./.github/actions/comment-progress - with: - comment-id: ${{ steps.couc_progress.outputs.comment-id }} - create_ip: ✅ - create_volume: ✅ - create_instance: ✅ - associate_ip: ✅ - setup_instance: ⏳ - attach_volume: "" - send_email: "" - - - name: Poll instance setup status - id: instance_poll - run: | - source ~/venv/bin/activate - - echo Polling "$INSTANCE_NAME" setup status - - max_wait_time=1200 # Maximum wait time in seconds (1200s -> 20mins) - wait_interval=5 # Interval between status checks in seconds - total_wait_time=0 - - while [ $total_wait_time -lt $max_wait_time ]; do - - status=$(openstack console log show $INSTANCE_NAME | \ - grep "^\{\"status\":\"" | \ - tail -1 | \ - jq -r '.status // "pending"') - - echo -n "setup status [$status]. " - - if [[ "$status" == "complete" ]]; then - echo "Exiting loop." - break - else - echo "Waiting for completion..." - sleep $wait_interval - total_wait_time=$((total_wait_time + wait_interval)) - fi - done - - if [ $total_wait_time -ge $max_wait_time ]; then - echo "::error ::Maximum wait time ($max_wait_time seconds) exceeded." - exit 1 - fi - - echo "status=$status" >> $GITHUB_OUTPUT - env: - OS_CLOUD: ${{ vars.MORPHOCLOUD_OS_CLOUD }} - INSTANCE_NAME: ${{ steps.define.outputs.instance_name }} - - - name: comment (maximum wait time exceeded) - if: ${{ steps.instance_poll.outcome == 'failure' && failure() }} - uses: peter-evans/create-or-update-comment@v4.0.0 - with: - issue-number: ${{ github.event.issue.number }} - body: | - ### Instance Creation Results ❌ - - Maximum wait time for instance **${{ steps.define.outputs.instance_name }}** setup to complete exceeded. - - See details at https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} + - name: Retrieve metadata + id: instance_metadata + uses: ./.github/actions/retrieve-metadata + with: + os_cloud: ${{ vars.MORPHOCLOUD_OS_CLOUD }} + instance_name: ${{ steps.define.outputs.instance_name }} - - name: comment (progress) - uses: ./.github/actions/comment-progress - with: - comment-id: ${{ steps.couc_progress.outputs.comment-id }} - create_ip: ✅ - create_volume: ✅ - create_instance: ✅ - associate_ip: ✅ - setup_instance: ✅ - attach_volume: ⏳ - send_email: "" - - - name: Rename MyData directory to MyData-tmp - id: rename_mydata + - name: Reboot instance + id: reboot_instance run: | ssh \ -o StrictHostKeyChecking=no \ -o UserKnownHostsFile=/dev/null \ -o LogLevel=ERROR \ exouser@$INSTANCE_IP \ - 'mv /media/volume/MyData /media/volume/MyData-tmp' - env: - INSTANCE_IP: ${{ steps.ip_create.outputs.floating_ip_address }} - - - name: comment (failed to rename MyData) - if: ${{ steps.rename_mydata.outcome == 'failure' && failure() }} - uses: peter-evans/create-or-update-comment@v4.0.0 - with: - issue-number: ${{ github.event.issue.number }} - body: | - ### Instance Creation Results ❌ - - Failed to rename `/media/volume/MyData` to `/media/volume/MyData-tmp` for instance **${{ steps.define.outputs.instance_name }}**. - - See details at https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} - - - name: Attach Volume - id: attach_volume - run: | - source ~/venv/bin/activate - - instance_id=$(openstack server list -f json | \ - jq \ - --arg instance_name "$INSTANCE_NAME" \ - -c '.[] | select(.Name == $instance_name)' | \ - jq -r '.ID' | tail -1) - echo "instance_id [$instance_id]" - - volume_id=$(openstack volume list -f json | \ - jq \ - --arg volume_name "$VOLUME_NAME" \ - -c '.[] | select(.Name == $volume_name)' | \ - jq -r '.ID' | tail -1) - echo "volume_id [$volume_id]" - - openstack server set \ - --property "exoVolumes::$volume_id={\"name\":\"MyData\"}" \ - $instance_id - - openstack server add volume \ - $instance_id \ - $volume_id - env: - OS_CLOUD: ${{ vars.MORPHOCLOUD_OS_CLOUD }} - INSTANCE_NAME: ${{ steps.define.outputs.instance_name }} - VOLUME_NAME: ${{ steps.define_volume_name.outputs.volume_name }} - - - name: comment (failed to attach volume) - if: ${{ steps.attach_volume.outcome == 'failure' && failure() }} - uses: peter-evans/create-or-update-comment@v4.0.0 - with: - issue-number: ${{ github.event.issue.number }} - body: | - ### Instance Creation Results ❌ - - Failed to attach volume **${{ steps.define_volume_name.outputs.volume_name }}** to instance **${{ steps.define.outputs.instance_name }}**. - - See details at https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} - - - name: Copy installed files into attached volume - id: copy_installed_files - run: | - function check_mountpoint_ready { + 'sudo shutdown -r now' + sleep 5 + function check_instance_ready { ssh \ -o StrictHostKeyChecking=no \ -o UserKnownHostsFile=/dev/null \ -o LogLevel=ERROR \ exouser@$INSTANCE_IP \ - 'mountpoint -q /media/volume/MyData' + 'true' } set +e - max_attempts=5 + max_attempts=3 + instance_ready=false for attempt in $(seq 1 $max_attempts); do - echo "Checking if mount point is ready ($attempt/$max_attempts)" - check_mountpoint_ready - if check_mountpoint_ready; then - echo "Mount point /media/volume/MyData is ready." + echo "Checking if instance is ready ($attempt/$max_attempts)" + if check_instance_ready; then + instance_ready=true + echo "Instance '$INSTANCE_NAME' is ready." break else - echo "Mount point not ready. Retrying in 5 seconds..." + echo "Instance '$INSTANCE_NAME' is not ready. Retrying in 5 seconds..." sleep 5 fi done - if ! check_mountpoint_ready; then - echo "::error ::Mount point /media/volume/MyData not ready after $((max_attempts * 5)) seconds." + if ! $instance_ready; then + echo "::error ::Instance '$INSTANCE_NAME' is not ready after $max_attempts attempts to connect." exit 1 fi set -e - ssh \ - -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null \ - -o LogLevel=ERROR \ - exouser@$INSTANCE_IP \ - '[ ! -d /media/volume/MyData/Slicer ] && mv /media/volume/MyData-tmp/Slicer /media/volume/MyData/Slicer && rmdir /media/volume/MyData-tmp || rm -rf /media/volume/MyData-tmp' - env: - INSTANCE_IP: ${{ steps.ip_create.outputs.floating_ip_address }} - - - name: comment (failed to copy installed files from MyData-tmp to MyData) - if: ${{ steps.copy_installed_files.outcome == 'failure' && failure() }} - uses: peter-evans/create-or-update-comment@v4.0.0 - with: - issue-number: ${{ github.event.issue.number }} - body: | - ### Instance Creation Results ❌ - - Failed to copy installed files from `/media/volume/MyData-tmp` to `/media/volume/MyData` for instance **${{ steps.define.outputs.instance_name }}**. - - See details at https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} - - - name: Create Renviron file - id: create_renviron_file - run: | - ssh \ - -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null \ - -o LogLevel=ERROR \ - exouser@$INSTANCE_IP \ - 'echo -e "R_LIBS_SITE=/media/share/MorphoCloudCephShare/R/x86_64-pc-linux-gnu-library/4.4/\nR_LIBS_USER=/media/volume/MyData/R/x86_64-pc-linux-gnu-library/4.4/" > /home/exouser/.Renviron && mkdir -p /media/volume/MyData/R/x86_64-pc-linux-gnu-library/4.4/' env: - INSTANCE_IP: ${{ steps.ip_create.outputs.floating_ip_address }} + INSTANCE_IP: ${{ steps.instance_metadata.outputs.instance_ip }} + INSTANCE_NAME: ${{ steps.define.outputs.instance_name }} - - name: comment (failed to create /home/exouser/.Renviron) - if: ${{ steps.create_renviron_file.outcome == 'failure' && failure() }} + - name: comment (failed to install Slicer extension dependencies) + if: + ${{ steps.slicer_install_extension_dependencies.outcome == 'failure' + && failure() }} uses: peter-evans/create-or-update-comment@v4.0.0 with: issue-number: ${{ github.event.issue.number }} body: | ### Instance Creation Results ❌ - Failed to create `/home/exouser/.Renviron` file for instance **${{ steps.define.outputs.instance_name }}**. + Failed to install Slicer extension dependencies on instance **${{ steps.define.outputs.instance_name }}**. See details at https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}