worker.yml

name: Worker CI/CD

on:
  push:
    branches: [ main ]
    paths:
      - 'apps/worker/**'
      - 'apps/backend/**'
      - 'sdk/**'
      - '.github/workflows/worker.yml'
  pull_request:
    branches: [ main ]
    paths:
      - 'apps/worker/**'
      - 'apps/backend/**'
      - 'sdk/**'
      - '.github/workflows/worker.yml'
  workflow_dispatch:
    inputs:
      environment:
        description: 'Environment to deploy to'
        required: true
        default: 'dev'
        type: choice
        options:
          - dev
          - stg
          - prd
      deploy_only:
        description: 'Skip build and only deploy latest image'
        required: false
        default: false
        type: boolean
      reason:
        description: 'Reason for manual deployment'
        required: false
        type: string
        default: 'Manual deployment'

env:
  SA_KEY_PATH: 'gcp-sa-key.json'

jobs:
  build:
    if: (github.ref == 'refs/heads/main' && github.event_name == 'push') || (github.event_name == 'workflow_dispatch' && github.event.inputs.deploy_only != 'true')
    runs-on: ubuntu-latest
    permissions:
      contents: read
      id-token: write
    environment: ${{ github.event.inputs.environment || 'dev' }}
    
    env:
      ENVIRONMENT: ${{ github.event.inputs.environment || 'dev' }}
      SERVICE: worker

    outputs:
      image_name: ${{ steps.set_env.outputs.image_name }}
      service_name: ${{ steps.set_env.outputs.service_name }}
      environment: ${{ env.ENVIRONMENT }}

    steps:
      - uses: actions/checkout@v4

      - name: Create and validate service account key file
        id: sa_key
        run: |
          # Write the key to a file, ensuring it's properly quoted
          echo '${{ secrets.GCP_SA_KEY }}' > ${{ env.SA_KEY_PATH }}
          
          # Validate that the file contains valid JSON
          if ! jq . ${{ env.SA_KEY_PATH }} > /dev/null 2>&1; then
            echo "❌ Error: Service account key is not valid JSON"
            echo "First few characters:"
            head -c 20 ${{ env.SA_KEY_PATH }}
            exit 1
          fi
          
          echo "✅ Service account key validated as proper JSON"
          echo "GOOGLE_APPLICATION_CREDENTIALS=$(realpath ${{ env.SA_KEY_PATH }})" >> $GITHUB_ENV

      - name: Setup Google Cloud SDK
        uses: google-github-actions/setup-gcloud@v2

      - name: Authenticate to Google Cloud
        uses: google-github-actions/auth@v2
        with:
          credentials_json: '${{ secrets.GCP_SA_KEY }}'

      - name: Set environment variables
        id: set_env
        run: |
          echo "PROJECT_ID=${{ secrets.PROJECT_ID }}" >> $GITHUB_ENV
          echo "REGION=${{ secrets.REGION }}" >> $GITHUB_ENV
          if [ "${{ env.ENVIRONMENT }}" = "prd" ]; then
            echo "SERVICE_NAME=rhesis-worker" >> $GITHUB_ENV
            echo "IMAGE_NAME=gcr.io/${{ secrets.PROJECT_ID }}/rhesis-worker" >> $GITHUB_ENV
            echo "service_name=rhesis-worker" >> $GITHUB_OUTPUT
            echo "image_name=gcr.io/${{ secrets.PROJECT_ID }}/rhesis-worker" >> $GITHUB_OUTPUT
          else
            echo "SERVICE_NAME=rhesis-worker-${{ env.ENVIRONMENT }}" >> $GITHUB_ENV
            echo "IMAGE_NAME=gcr.io/${{ secrets.PROJECT_ID }}/rhesis-worker-${{ env.ENVIRONMENT }}" >> $GITHUB_ENV
            echo "service_name=rhesis-worker-${{ env.ENVIRONMENT }}" >> $GITHUB_OUTPUT
            echo "image_name=gcr.io/${{ secrets.PROJECT_ID }}/rhesis-worker-${{ env.ENVIRONMENT }}" >> $GITHUB_OUTPUT
          fi
          echo "CLOUDSQL_INSTANCE=${{ secrets.CLOUDSQL_INSTANCE }}" >> $GITHUB_ENV

      - name: Configure Docker for GCR
        run: |
          gcloud auth configure-docker

      - name: Build Container
        run: |
          docker build -t ${{ env.IMAGE_NAME }}:latest -f apps/worker/Dockerfile .

      - name: Push Container
        run: |
          docker push ${{ env.IMAGE_NAME }}:latest

  deploy:
    needs: build
    if: (github.ref == 'refs/heads/main' && github.event_name == 'push') || github.event_name == 'workflow_dispatch'
    runs-on: ubuntu-latest
    permissions:
      contents: read
      id-token: write
    environment: ${{ needs.build.outputs.environment || github.event.inputs.environment }}
    
    env:
      ENVIRONMENT: ${{ needs.build.outputs.environment || github.event.inputs.environment }}
      
    steps:
      - uses: actions/checkout@v4

      - name: Create and validate service account key file
        id: sa_key
        run: |
          # Write the key to a file, ensuring it's properly quoted
          echo '${{ secrets.GCP_SA_KEY }}' > ${{ env.SA_KEY_PATH }}
          
          # Validate that the file contains valid JSON
          if ! jq . ${{ env.SA_KEY_PATH }} > /dev/null 2>&1; then
            echo "❌ Error: Service account key is not valid JSON"
            echo "First few characters:"
            head -c 20 ${{ env.SA_KEY_PATH }}
            exit 1
          fi
          
          echo "✅ Service account key validated as proper JSON"
          echo "GOOGLE_APPLICATION_CREDENTIALS=$(realpath ${{ env.SA_KEY_PATH }})" >> $GITHUB_ENV

      - name: Setup Google Cloud SDK
        uses: google-github-actions/setup-gcloud@v2

      - name: Authenticate to Google Cloud
        uses: google-github-actions/auth@v2
        with:
          credentials_json: '${{ secrets.GCP_SA_KEY }}'

      - name: Install gke-gcloud-auth-plugin
        run: |
          gcloud components install gke-gcloud-auth-plugin

      - name: Set environment variables
        run: |
          echo "PROJECT_ID=${{ secrets.PROJECT_ID }}" >> $GITHUB_ENV
          echo "REGION=${{ secrets.REGION }}" >> $GITHUB_ENV
          if [ "${{ env.ENVIRONMENT }}" = "prd" ]; then
            echo "CLUSTER_NAME=rhesis-worker" >> $GITHUB_ENV
            echo "NAMESPACE=rhesis-worker" >> $GITHUB_ENV
            echo "IMAGE_NAME=gcr.io/${{ secrets.PROJECT_ID }}/rhesis-worker" >> $GITHUB_ENV
          else
            echo "CLUSTER_NAME=rhesis-worker-${{ env.ENVIRONMENT }}" >> $GITHUB_ENV
            echo "NAMESPACE=rhesis-worker-${{ env.ENVIRONMENT }}" >> $GITHUB_ENV
            echo "IMAGE_NAME=gcr.io/${{ secrets.PROJECT_ID }}/rhesis-worker-${{ env.ENVIRONMENT }}" >> $GITHUB_ENV
          fi

      - name: Create or get GKE Autopilot cluster
        run: |
          # Check if cluster exists
          if ! gcloud container clusters describe ${{ env.CLUSTER_NAME }} --region=${{ env.REGION }} --project=${{ env.PROJECT_ID }} >/dev/null 2>&1; then
            echo "Creating GKE Autopilot cluster..."
            gcloud container clusters create-auto ${{ env.CLUSTER_NAME }} \
              --region=${{ env.REGION }} \
              --project=${{ env.PROJECT_ID }} \
              --network=default \
              --subnetwork=default \
              --release-channel=regular \
              --enable-ip-alias \
              --workload-pool=${{ env.PROJECT_ID }}.svc.id.goog
          else
            echo "Cluster ${{ env.CLUSTER_NAME }} already exists"
          fi

      - name: Get GKE credentials
        run: |
          gcloud container clusters get-credentials ${{ env.CLUSTER_NAME }} \
            --region=${{ env.REGION }} \
            --project=${{ env.PROJECT_ID }}

      - name: Setup Workload Identity
        run: |
          # Create Kubernetes service account binding with Google service account
          gcloud iam service-accounts add-iam-policy-binding \
            --role roles/iam.workloadIdentityUser \
            --member "serviceAccount:${{ env.PROJECT_ID }}.svc.id.goog[${{ env.NAMESPACE }}/rhesis-worker-sa]" \
            ${{ secrets.GCP_SERVICE_ACCOUNT_EMAIL }}

      - name: Create namespace
        run: |
          kubectl create namespace ${{ env.NAMESPACE }} --dry-run=client -o yaml | kubectl apply -f -

      - name: Create or update secrets
        run: |
          kubectl create secret generic rhesis-worker-secrets \
            --namespace=${{ env.NAMESPACE }} \
            --from-literal=BROKER_URL="${{ secrets.BROKER_URL }}" \
            --from-literal=CELERY_RESULT_BACKEND="${{ secrets.CELERY_RESULT_BACKEND }}" \
            --from-literal=SQLALCHEMY_DATABASE_URL="${{ secrets.SQLALCHEMY_DATABASE_URL }}" \
            --from-literal=SQLALCHEMY_DB_MODE="${{ secrets.SQLALCHEMY_DB_MODE }}" \
            --from-literal=SQLALCHEMY_DB_DRIVER="${{ secrets.SQLALCHEMY_DB_DRIVER }}" \
            --from-literal=SQLALCHEMY_DB_USER="${{ secrets.SQLALCHEMY_DB_USER }}" \
            --from-literal=SQLALCHEMY_DB_PASS="${{ secrets.SQLALCHEMY_DB_PASS }}" \
            --from-literal=SQLALCHEMY_DB_HOST="${{ secrets.SQLALCHEMY_DB_HOST || '127.0.0.1' }}" \
            --from-literal=SQLALCHEMY_DB_NAME="${{ secrets.SQLALCHEMY_DB_NAME }}" \
            --from-literal=LOG_LEVEL="${{ secrets.LOG_LEVEL }}" \
            --from-literal=CELERY_WORKER_LOGLEVEL="${{ secrets.LOG_LEVEL || 'INFO' }}" \
            --from-literal=CELERY_WORKER_CONCURRENCY="${{ secrets.CELERY_WORKER_CONCURRENCY || '8' }}" \
            --from-literal=GEMINI_API_KEY="${{ secrets.GEMINI_API_KEY }}" \
            --from-literal=GOOGLE_API_KEY="${{ secrets.GOOGLE_API_KEY }}" \
            --from-literal=GEMINI_MODEL_NAME="${{ secrets.GEMINI_MODEL_NAME }}" \
            --from-literal=AZURE_OPENAI_ENDPOINT="${{ secrets.AZURE_OPENAI_ENDPOINT }}" \
            --from-literal=AZURE_OPENAI_API_KEY="${{ secrets.AZURE_OPENAI_API_KEY }}" \
            --from-literal=AZURE_OPENAI_DEPLOYMENT_NAME="${{ secrets.AZURE_OPENAI_DEPLOYMENT_NAME }}" \
            --from-literal=AZURE_OPENAI_API_VERSION="${{ secrets.AZURE_OPENAI_API_VERSION }}" \
            --from-literal=SMTP_HOST="${{ secrets.SMTP_HOST }}" \
            --from-literal=SMTP_PORT="${{ secrets.SMTP_PORT }}" \
            --from-literal=SMTP_USER="${{ secrets.SMTP_USER }}" \
            --from-literal=SMTP_PASSWORD="${{ secrets.SMTP_PASSWORD }}" \
            --from-literal=FROM_EMAIL="${{ secrets.FROM_EMAIL }}" \
            --from-literal=BACKEND_ENV="${{ secrets.BACKEND_ENV }}" \
            --from-literal=WORKER_ENV="${{ secrets.WORKER_ENV }}" \
            --from-literal=FRONTEND_URL="${{ secrets.FRONTEND_URL }}" \
            --dry-run=client -o yaml | kubectl apply -f -

      - name: Deploy to GKE
        run: |
          echo "Deploying image: ${{ env.IMAGE_NAME }}:latest"
          echo "Cloud SQL instance: ${{ env.PROJECT_ID }}:${{ env.REGION }}:${{ secrets.CLOUDSQL_INSTANCE }}"
          
          # Update kustomization with correct image and environment
          cd apps/worker/k8s
          
          # Update deployment with actual values
          sed -i "s|gcr.io/PROJECT_ID/rhesis-worker|${{ env.IMAGE_NAME }}|g" deployment.yaml
          sed -i "s|PROJECT_ID:REGION:CLOUDSQL_INSTANCE|${{ env.PROJECT_ID }}:${{ env.REGION }}:${{ secrets.CLOUDSQL_INSTANCE }}|g" deployment.yaml
          sed -i "s|GCP_SERVICE_ACCOUNT_EMAIL|${{ secrets.GCP_SERVICE_ACCOUNT_EMAIL }}|g" serviceaccount.yaml
          
          # Always force pod restart by updating timestamp annotation
          sed -i "s|deployment-timestamp: \"placeholder\"|deployment-timestamp: \"$(date +%s)\"|g" deployment.yaml
          
          # Update kustomization for environment
          cat > kustomization.yaml << EOF
          apiVersion: kustomize.config.k8s.io/v1beta1
          kind: Kustomization
          
          resources:
          - deployment.yaml
          - service.yaml
          - serviceaccount.yaml
          - networkpolicy.yaml
          
          labels:
          - includeSelectors: true
            pairs:
              environment: ${{ env.ENVIRONMENT }}
          
          namespace: ${{ env.NAMESPACE }}
          
          images:
          - name: ${{ env.IMAGE_NAME }}
            newTag: latest
          EOF
          
          # Apply manifests
          kubectl apply -k . --namespace=${{ env.NAMESPACE }}
          
          # Wait for deployment to be ready
          echo "Waiting for deployment to be ready..."
          if ! kubectl rollout status deployment/rhesis-worker --namespace=${{ env.NAMESPACE }} --timeout=300s; then
            echo "❌ Deployment failed to become ready. Debugging information:"
            echo ""
            echo "=== Pod Status ==="
            kubectl get pods --namespace=${{ env.NAMESPACE }} -o wide
            echo ""
            echo "=== Pod Events ==="
            kubectl get events --namespace=${{ env.NAMESPACE }} --sort-by='.lastTimestamp' | tail -20
            echo ""
            echo "=== Pod Logs (if any pods exist) ==="
            for pod in $(kubectl get pods --namespace=${{ env.NAMESPACE }} -o jsonpath='{.items[*].metadata.name}'); do
              echo "--- Logs for $pod ---"
              kubectl logs $pod --namespace=${{ env.NAMESPACE }} --tail=50 || echo "No logs available"
              echo ""
            done
            exit 1
          fi
          
          echo "✅ Deployment completed successfully"