collect-instance-uptime.yml

name: Collect Instance Uptime
on:
  schedule:
    # Run every 5 mins
    - cron: "* * * * *"
  workflow_dispatch:

concurrency:
  group: ${{ github.workflow }}-${{ vars.MORPHOCLOUD_OS_CLOUD }}
  cancel-in-progress: true

jobs:
  collect-instance-uptime:
    runs-on: self-hosted
    steps:
      - uses: actions/checkout@v4

      - name: Define metadata
        id: define-metadata
        run: |
          echo "branch=instance-uptime-data" >> $GITHUB_OUTPUT
          echo "directory=data" >> $GITHUB_OUTPUT

      - name: Create & push branch
        run: |
          exist_in_remote=$(git ls-remote --heads origin $BRANCH)
          if [[ -z ${exist_in_remote} ]]; then
            echo "Creating $BRANCH branch"
            mkdir $DIRECTORY
            cp -r .git $DIRECTORY/.git
            cd $DIRECTORY
            user_name='github-actions[bot]'
            git config user.name  "$user_name" && echo "user.name:  ${user_name}"
            user_email='41898282+github-actions[bot]@users.noreply.github.com'
            git config user.email "$user_email" && echo "user.email: ${user_email}"
            git checkout --orphan $BRANCH
            git reset --hard
            git commit --allow-empty -m "Initializing $BRANCH branch"
            git push origin $BRANCH
          fi
        env:
          BRANCH: ${{ steps.define-metadata.outputs.branch }}
          DIRECTORY: ${{ steps.define-metadata.outputs.directory }}

      - uses: actions/checkout@v4
        with:
          ref: instance-uptime-data
          path: ${{ steps.define-metadata.outputs.directory }}

      - name: Collect Instance Uptime
        id: collect-instance-uptime
        run: |
          source ~/venv/bin/activate

          instance_prefix=${PREFIX:+${PREFIX}_}
          instance_basename="${instance_prefix}instance"

          # Define the JSON file paths for storing/updating uptimes
          JSON_FILE=$JSON_DIR/uptime.json
          JSON_FILE_TMP=$(mktemp $RUNNER_TEMP/uptime.XXXXXX.json)

          # Initialize the JSON file if it doesn't exist, with a basic structure
          if [[ ! -f $JSON_FILE ]]; then
            jq --null-input \
              --arg allocation_id "$OS_CLOUD" \
              '{"allocations": [{"id": $allocation_id, "instances": []}]}' > $JSON_FILE
          fi

          # List all instances matching the naming pattern and process each one
          openstack server list --name "^${instance_basename}-\d+" -f json | \
          jq -r '.[] | [.Name, .Status, ."OS-EXT-STS:task_state"] | @tsv' | \
            while IFS=$'\t' read -r instance_name status task_state; do
              echo "instance_name [$instance_name] status [$status] task_state [$task_state]"

              # Skip the instance if it is not active
              if [[ "$status" != "ACTIVE" ]]; then
                continue
              fi

              if [[ "$task_state" != "" ]]; then
                continue
              fi

              # Extract issue number
              issue_number=${instance_name##*-}
              echo "issue_number [$issue_number]"

              # Retrieve the IP address of the instance
              instance_ip=$(
                openstack server show $instance_name -c addresses -f json | \
                jq -r '.addresses.auto_allocated_network[1]'
              )
              echo "instance_ip [$instance_ip]"

              # Skip the instance if the IP address could not be retrieved
              if [[ "$instance_ip" == "null" ]]; then
                echo "::warning ::Failed to retrieve $instance_name IP"
                continue
              fi

              # Notes on SSH usage:
              # * Redirecting SSH standard input to /dev/null ('< /dev/null') is required to work around
              #   an issue where SSH breaks out of the while loop in Bash.
              #   Reference: https://stackoverflow.com/questions/9393038/ssh-breaks-out-of-while-loop-in-bash

              # Retrieve the instance uptime using SSH
              uptime=$(ssh \
                -o StrictHostKeyChecking=no \
                -o UserKnownHostsFile=/dev/null \
                -o LogLevel=ERROR \
                exouser@$instance_ip \
                  'cat /proc/uptime | awk  "{print \$1}"' < /dev/null)

              if [[ $? -ne 0 ]]; then
                echo "::warning ::Failed to retrieve uptime for $instance_name using IP $instance_ip"
                continue
              fi

              # Retrieve the startup time of the instance in yyyy-mm-dd HH:MM:SS format
              startup_time=$(ssh \
                -o StrictHostKeyChecking=no \
                -o UserKnownHostsFile=/dev/null \
                -o LogLevel=ERROR \
                exouser@$instance_ip \
                  'uptime -s' < /dev/null)
              if [[ $? -ne 0 ]]; then
                echo "::warning ::Failed to retrieve startup time for $instance_name using IP $instance_ip"
                continue
              fi

              # Check if the instance is already recorded in the JSON file
              has_instance=$(cat $JSON_FILE | jq \
                --arg allocation_id "$OS_CLOUD" \
                --arg instance_name "$instance_name" \
                --arg startup_time "$startup_time" \
                'any(.allocations[] | select(.id == $allocation_id) | .instances[]; .name == $instance_name)')

              # If the instance is not found, add it to the JSON file
              if [[ "$has_instance" == "false" ]]; then
                cat $JSON_FILE | jq \
                  --arg allocation_id "$OS_CLOUD" \
                  --arg instance_name "$instance_name" \
                  '(.allocations[] | select(.id == $allocation_id) | .instances)
                  += [{"name": $instance_name, "uptimes": {}}]' > $JSON_FILE_TMP \
                  && mv $JSON_FILE_TMP $JSON_FILE
              fi

              # Check if there is an uptime entry for the startup time
              has_startup_time=$(cat $JSON_FILE | jq \
                --arg allocation_id "$OS_CLOUD" \
                --arg instance_name "$instance_name" \
                --arg startup_time "$startup_time" \
                '(.allocations[] | select(.id == $allocation_id) | .instances[] | select(.name == $instance_name) | .uptimes | has($startup_time))')

              if [[ "$has_startup_time" == "false" ]]; then
                # If the startup time is not recorded, add the uptime entry
                cat $JSON_FILE | jq \
                  --arg allocation_id "$OS_CLOUD" \
                  --arg instance_name "$instance_name" \
                  --arg startup_time "$startup_time" \
                  --arg uptime "$uptime" \
                  '(.allocations[] | select(.id == $allocation_id) | .instances[] | select(.name == $instance_name) | .uptimes)
                  += {($startup_time): $uptime|tonumber}' > $JSON_FILE_TMP \
                  && mv $JSON_FILE_TMP $JSON_FILE
              else
                # If the startup time is already recorded, update the uptime entry
                cat $JSON_FILE | jq \
                  --arg allocation_id "$OS_CLOUD" \
                  --arg instance_name "$instance_name" \
                  --arg startup_time "$startup_time" \
                  --arg uptime "$uptime" \
                  '(.allocations[] | select(.id == $allocation_id) | .instances[] | select(.name == $instance_name) | .uptimes[$startup_time])
                  = ($uptime|tonumber)' > $JSON_FILE_TMP \
                  && mv $JSON_FILE_TMP $JSON_FILE
              fi
            done

            # Display the final JSON content for verification
            echo "--------"
            cat $JSON_FILE
            echo "--------"
        env:
          OS_CLOUD: ${{ vars.MORPHOCLOUD_OS_CLOUD }}
          PREFIX: ${{ vars.INSTANCE_NAME_PREFIX }}
          JSON_DIR: ${{ steps.define-metadata.outputs.directory }}

      - name: Publish
        uses: s0/git-publish-subdir-action@61eb9e6420447e7cbf010f7cce37103665c46bfa # develop
        env:
          REPO: self
          BRANCH: ${{ steps.define-metadata.outputs.branch }}
          FOLDER: ${{ steps.define-metadata.outputs.directory }}
          SQUASH_HISTORY: true
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}