From ed0bc8f1efef85df3f47e43c43ef6cea447cda2c Mon Sep 17 00:00:00 2001
From: Jean-Christophe Fillion-Robin <jchris.fillionr@kitware.com>
Date: Thu, 22 Aug 2024 19:24:26 -0400
Subject: [PATCH] feat: Add "Automatic Instance Shelving" workflow

---
 .../workflows/automatic-instance-shelving.yml | 67 +++++++++++++++++++
 1 file changed, 67 insertions(+)
 create mode 100644 .github/workflows/automatic-instance-shelving.yml

diff --git a/.github/workflows/automatic-instance-shelving.yml b/.github/workflows/automatic-instance-shelving.yml
new file mode 100644
index 00000000..ca8a1cd7
--- /dev/null
+++ b/.github/workflows/automatic-instance-shelving.yml
@@ -0,0 +1,67 @@
+name: Automatic Instance Shelving
+on:
+  schedule:
+    # Run every 5 mins
+    - cron: "* * * * *"
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  auto-shelve:
+    runs-on: self-hosted
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Auto Shelve
+        run: |
+          export OS_CLOUD=BIO180006_IU # Select openstack auth settings defined in ".config/openstack/clouds.yaml"
+
+          source ~/venv/bin/activate
+
+          instance_prefix=${PREFIX:+${PREFIX}_}
+          instance_basename="${instance_prefix}instance"
+
+          openstack server list --name "^${instance_basename}-\d+" -f json | \
+          jq -r '.[] | [.Name, .Status, ."OS-EXT-STS:task_state"] | @tsv' | \
+            while IFS=$'\t' read -r instance_name status task_state; do
+              echo "instance_name [$instance_name] status [$status] task_state [$task_state]"
+              if [[ "$status" != "ACTIVE" ]]; then
+                # Skip because instance is not active
+                continue
+              fi
+              if [[ "$task_state" != "" ]]; then
+                # Skip because instance status is being updated
+                continue
+              fi
+              # Get instance IP
+              instance_ip=$(
+                openstack server show $instance_name -c addresses -f json | \
+                jq -r '.addresses.auto_allocated_network[1]'
+              )
+              echo "instance_ip [$instance_ip]"
+              if [[ "$instance_ip" == "null" ]]; then
+                echo "::warning ::Failed to retrieve $instance_name IP"
+                continue
+              fi
+              # Retrieve uptime
+              uptime_seconds=$(ssh \
+                -o StrictHostKeyChecking=no \
+                -o UserKnownHostsFile=/dev/null \
+                -o LogLevel=ERROR \
+                exouser@$instance_ip \
+                  'cat /proc/uptime | awk  "{print \$1}"')
+              uptime_hours=$(echo "scale=2; $uptime_seconds / 3600" | bc)
+              # Check uptime
+              if $(python3 -c "valid=($uptime_hours > 3.5 and $uptime_hours <= 4.0); EXIT_SUCCESS=0; EXIT_FAILURE=1; exit(EXIT_SUCCESS if valid else EXIT_FAILURE)"); then
+                action="notify"
+              elif $(python3 -c "valid=($uptime_hours > 4.0); EXIT_SUCCESS=0; EXIT_FAILURE=1; exit(EXIT_SUCCESS if valid else EXIT_FAILURE)"); then
+                action="shelve"
+              else
+                action=""
+              fi
+              echo "instance_name [$instance_name] -> action[$action]"
+            done
+        env:
+          PREFIX: ${{ vars.INSTANCE_NAME_PREFIX }}