Add CI/CD workflows for terraform automation

.github/workflows/devnet_main_deploy.yml

@@ -0,0 +1,29 @@
+name: Main CD for Devnet Deployment
+
+on:
+  workflow_dispatch:
+  pull_request:
+    branches:
+      - main
+  push:
+    branches:
+      - "!main"
+    paths:
+      - "aws/devnet/**"
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: false
+
+jobs:
+  deploy:
+    uses: ./.github/workflows/terraform_template_deploy.yml
+    with:
+      project: aws
+      resource: devnet
+      tf_workspace_name: devnet-aws
+      tf_version: 1.5.7
+      tf_organization: subspace
+    secrets:
+      TRANSCRYPT: ${{ secrets.TRANSCRYPT }}
+      TF_API_TOKEN: ${{ secrets.TF_API_TOKEN }}

.github/workflows/ephemeral_devnet_aws_deploy.yml

@@ -0,0 +1,49 @@
+name: Main CD for Ephememeral Devnet Deployment
+
+on:
+  workflow_dispatch:
+    inputs:
+      branch:
+        required: true
+        type: string
+      genesis_hash:
+        required: true
+        type: string
+      run_apply:
+        description: "The code needs to be deployed or not"
+        type: string
+        default: "no"
+      run_destroy:
+        description: "The resources need to be destroyed or not"
+        type: string
+        default: "no"
+  pull_request:
+    branches:
+      - main
+  push:
+    branches:
+      - "!main"
+    paths:
+      - "testing-framework/ec2/network/**"
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: false
+
+jobs:
+  deploy:
+    uses: ./.github/workflows/terraform_template_ephemeral_deploy.yml
+    with:
+      project: testing-framework
+      branch: ${{ github.event.inputs.branch }}
+      genesis_hash: ${{ github.event.inputs.genesis_hash }}
+      instance: ec2
+      resource: network
+      tf_workspace_name: ephemeral-devnet
+      tf_version: 1.5.7
+      tf_organization: subspace
+      run_apply: ${{ github.event.inputs.run_apply }}
+      run_destroy: ${{ github.event.inputs.run_destroy }}
+    secrets:
+      TRANSCRYPT: ${{ secrets.TRANSCRYPT }}
+      TF_API_TOKEN: ${{ secrets.TF_API_TOKEN }}

.github/workflows/ephemeral_devnet_hetzner_deploy.yml

@@ -0,0 +1,49 @@
+name: Main CD for Ephememeral Devnet Deployment
+
+on:
+  workflow_dispatch:
+    inputs:
+      branch:
+        required: true
+        type: string
+      genesis_hash:
+        required: true
+        type: string
+      run_apply:
+        description: "The code needs to be deployed or not"
+        type: string
+        default: "no"
+      run_destroy:
+        description: "The resources need to be destroyed or not"
+        type: string
+        default: "no"
+  pull_request:
+    branches:
+      - main
+  push:
+    branches:
+      - "!main"
+    paths:
+      - "testing-framework/hetzner/network/**"
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: false
+
+jobs:
+  deploy:
+    uses: ./.github/workflows/terraform_template_ephemeral_deploy.yml
+    with:
+      project: testing-framework
+      branch: ${{ github.event.inputs.branch }}
+      genesis_hash: ${{ github.event.inputs.genesis_hash }}
+      instance: hetzner
+      resource: network
+      tf_workspace_name: ephemeral-devnet-hetzner
+      tf_version: 1.5.7
+      tf_organization: subspace
+      run_apply: ${{ github.event.inputs.run_apply }}
+      run_destroy: ${{ github.event.inputs.run_destroy }}
+    secrets:
+      TRANSCRYPT: ${{ secrets.TRANSCRYPT }}
+      TF_API_TOKEN: ${{ secrets.TF_API_TOKEN }}

.github/workflows/gemini_3g_main_deploy.yml

@@ -0,0 +1,29 @@
+name: Main CD for Gemini Deployment
+
+on:
+  workflow_dispatch:
+  pull_request:
+    branches:
+      - main
+  push:
+    branches:
+      - "!main"
+    paths:
+      - "aws/gemini-3g/**"
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: false
+
+jobs:
+  deploy:
+    uses: ./.github/workflows/terraform_template_deploy.yml
+    with:
+      project: aws
+      resource: gemini-3g
+      tf_workspace_name: gemini-3g
+      tf_version: 1.5.7
+      tf_organization: subspace
+    secrets:
+      TRANSCRYPT: ${{ secrets.TRANSCRYPT }}
+      TF_API_TOKEN: ${{ secrets.TF_API_TOKEN }}

.github/workflows/terraform_gh_runner.yml

@@ -1,16 +1,18 @@
-name: Terraform Workflow
+name: Terraform GH Runner Deployment
 
 on:
-  push:
+  workflow_dispatch:
+  pull_request:
     branches:
       - main
     paths:
-      - './github-runners/terraform/base/**'
-  workflow_dispatch:
+      - "github-runners/terraform/dedicated/**"
 
 jobs:
   terraform_gh_runner:
     runs-on: ubuntu-latest
+    env:
+      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
     steps:
       - name: Checkout repository
@@ -19,40 +21,42 @@ jobs:
       - name: Set up Terraform
         uses: hashicorp/setup-terraform@v1
         with:
-          terraform_version: "1.4.2"
-          cli_config_credentials_token: ${{ secrets.TF_CLOUD_TOKEN }}
+          terraform_version: "1.5.7"
+          cli_config_credentials_token: ${{ secrets.TF_API_TOKEN }}
 
-      - name: Install dependencies
+      - name: Decrypt the secrets
         run: |
-          # Install any dependencies required by your Terraform code
+          bash scripts/transcrypt -c aes-256-cbc -p ${{ secrets.TRANSCRYPT }} -y
 
       - name: Run Bash Script
+        id: generate_runner_token
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
-            # GitHub repository and access token
-            repo="subspace/infra"
-            token=${{ secrets.PAT_TOKEN }}
-
-            # API endpoint
-            url="https://api.github.com/repos/$repo/actions/runners/registration-token"
+          # GitHub repository and access token for github api calls, since GITHUB_TOKEN can't
+          # be used for this purpose.
+          repo="subspace/infra"
+          token=${{ secrets.PAT_TOKEN }}
 
-            # Send POST request to get the registration token
-            response=$(curl -X POST -H "Authorization: token $token" -s "$url")
+          # API endpoint
+          url="https://api.github.com/repos/$repo/actions/runners/registration-token"
 
-            # Extract the token value from the response
-            runner_token=$(echo "$response" | jq -r '.token')
+          # Send POST request to get the registration token
+          response=$(curl -X POST -H "Authorization: token $token" -s "$url")
 
-            # Export the token as an environment variable
-            echo "export RUNNER_TOKEN=$runner_token" >> $GITHUB_ENV
+          # Extract the token value from the response
+          runner_token=$(echo "$response" | jq -r '.token')
 
-            # Set the runner token as an environment variable
-            export RUNNER_TOKEN="$runner_token"
+          # Mask the runner token in the logs
+          echo "::add-mask::$runner_token"
 
-            # Store the token as a secret in GitHub Actions
-            gh secret set RUNNER_TOKEN -r "$repo" -b "$runner_token"
+          # Pass the runner token in GitHub Actions for use
+          # in subsequent steps with terraform for runner registration
+          echo "::set-output name=runner_token::$runner_token"
 
       - name: Run Terraform
-        working-directory: ./github-runners/terraform/base
+        working-directory: ./github-runners/terraform/dedicated
         run: |
-          terraform init-backend-config="organization=${{ secrets.ORGANIZATION_NAME }}" -backend-config="workspaces=${{ secrets.WORKSPACE_NAME }}"
-          terraform plan -var-file=${{ secrets.VAR_FILE }}
-          terraform apply -auto-approve -var "gh_token=${{ secrets.RUNNER_TOKEN }}"
+          terraform init
+          terraform plan -var-file=terraform.tfvars
+          terraform apply -auto-approve -var "gh_token=${{ steps.generate_runner_token.outputs.runner_token }}" -var-file=terraform.tfvars

.github/workflows/terraform_template_deploy.yml

@@ -0,0 +1,91 @@
+name: Template Deploy
+
+on:
+  workflow_call:
+    inputs:
+      project:
+        required: true
+        type: string
+      resource:
+        required: true
+        type: string
+      tf_workspace_name:
+        description: "Name of the workspace in terraform cloud"
+        required: false
+        type: string
+      tf_version:
+        description: "Version of the terraform"
+        required: true
+        type: string
+      tf_organization:
+        description: "Name of the TF organization"
+        default: "subspace"
+        type: string
+        required: true
+      run_apply:
+        description: "The code needs to be deployed or not"
+        type: string
+        default: "no"
+      run_destroy:
+        description: "The resources need to be destroyed or not"
+        type: string
+        default: "no"
+    secrets:
+      TRANSCRYPT:
+        required: true
+      TF_API_TOKEN:
+        required: false
+env:
+  TF_CLOUD_ORGANIZATION: "${{ inputs.tf_organization }}"
+  TF_API_TOKEN: "${{ secrets.TF_API_TOKEN }}"
+  TF_VERSION: "${{ inputs.tf_version }}"
+
+jobs:
+  template-deploy:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout the repo
+        uses: actions/checkout@v3
+
+      - name: Decrypt the secrets
+        run: |
+          bash scripts/transcrypt -c aes-256-cbc -p ${{ secrets.TRANSCRYPT }} -y
+
+      - uses: hashicorp/setup-terraform@v2
+        with:
+          terraform_version: ${{ env.TF_VERSION }}
+          terraform_wrapper: false
+          cli_config_credentials_token: ${{ secrets.TF_API_TOKEN }}
+
+      - name: Terraform fmt
+        working-directory: ${{ inputs.project }}/${{ inputs.resource }}
+        run: terraform fmt -check
+        continue-on-error: true
+
+      - name: Terraform Init for ${{ inputs.project }}/${{ inputs.resource }}
+        working-directory: ${{ inputs.project }}/${{ inputs.resource }}
+        run: |
+          terraform init
+
+      - name: Terraform Validate
+        working-directory: ${{ inputs.project }}/${{ inputs.resource }}
+        run: terraform validate
+
+      - name: Terraform Plan for ${{ inputs.project }}/${{ inputs.resource }}
+        if: ${{ (inputs.run_destroy == 'no') }}
+        working-directory: ${{ inputs.project }}/${{ inputs.resource }}
+        run: |
+          terraform plan -var-file=terraform.tfvars
+
+      - name: Terraform Apply for ${{ inputs.project }}/${{ inputs.resource }}
+        if: ${{ (inputs.run_apply == 'yes') && (inputs.run_destroy == 'no') }}
+        working-directory: ${{ inputs.project }}/${{ inputs.resource }}
+        run: |
+          terraform apply -auto-approve -var-file=terraform.tfvars
+
+      - name: Terraform Destroy for ${{ inputs.project }}/${{ inputs.resource }}
+        if: ${{ (inputs.run_destroy == 'yes') }}
+        working-directory: ${{ inputs.project }}/${{ inputs.resource }}
+        run: |
+          terraform plan -destroy -var-file=terraform.tfvars
+          terraform destroy -auto-approve -var-file=terraform.tfvars