try chunked query/csv #177
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Apply Terraform for Warehouse and DAG changes | |
| on: | |
| push: | |
| branches: | |
| - 'main' | |
| - 'staging/**' | |
| paths: | |
| - 'airflow/**' | |
| - 'warehouse/**' | |
| jobs: | |
| airflow-changes: | |
| name: Detect Airflow Terraform changes | |
| runs-on: ubuntu-latest | |
| outputs: | |
| staging: ${{ steps.staging-changes.outputs.all_changed_files }} | |
| production: ${{ steps.production-changes.outputs.all_changed_files }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Staging Changes | |
| uses: tj-actions/changed-files@v46 | |
| id: staging-changes | |
| with: | |
| matrix: true | |
| dir_names: true | |
| files: 'iac/cal-itp-data-infra-staging/airflow/**/*.tf' | |
| - name: Production Changes | |
| uses: tj-actions/changed-files@v46 | |
| id: production-changes | |
| with: | |
| matrix: true | |
| dir_names: true | |
| files: 'iac/cal-itp-data-infra/airflow/**/*.tf' | |
| composer-changes: | |
| name: Detect Composer Terraform changes | |
| runs-on: ubuntu-latest | |
| outputs: | |
| staging: ${{ steps.staging-changes.outputs.all_changed_files }} | |
| production: ${{ steps.production-changes.outputs.all_changed_files }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Staging Changes | |
| uses: tj-actions/changed-files@v46 | |
| id: staging-changes | |
| with: | |
| matrix: true | |
| dir_names: true | |
| files: 'iac/cal-itp-data-infra-staging/composer/**/*.tf' | |
| - name: Production Changes | |
| uses: tj-actions/changed-files@v46 | |
| id: production-changes | |
| with: | |
| matrix: true | |
| dir_names: true | |
| files: 'iac/cal-itp-data-infra/composer/**/*.tf' | |
| airflow-staging: | |
| name: Staging Airflow | |
| runs-on: ubuntu-latest | |
| needs: [airflow-changes] | |
| if: ${{ needs.airflow-changes.outputs.staging == '[]' || needs.airflow-changes.outputs.staging == '' }} | |
| permissions: | |
| contents: read | |
| id-token: write | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - uses: google-github-actions/auth@v2 | |
| with: | |
| create_credentials_file: 'true' | |
| project_id: cal-itp-data-infra-staging | |
| workload_identity_provider: 'projects/473674835135/locations/global/workloadIdentityPools/github-actions/providers/data-infra' | |
| service_account: 'github-actions-terraform@cal-itp-data-infra-staging.iam.gserviceaccount.com' | |
| - name: Terraform Apply | |
| uses: dflook/terraform-apply@v1 | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| with: | |
| path: iac/cal-itp-data-infra-staging/airflow/us | |
| auto_approve: ${{ contains(github.ref, 'refs/heads/staging') }} | |
| composer-staging: | |
| name: Staging Composer | |
| runs-on: ubuntu-latest | |
| needs: [composer-changes] | |
| if: ${{ needs.composer-changes.outputs.staging == '[]' || needs.composer-changes.outputs.staging == '' }} | |
| permissions: | |
| contents: read | |
| id-token: write | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - uses: google-github-actions/auth@v2 | |
| with: | |
| create_credentials_file: 'true' | |
| project_id: cal-itp-data-infra-staging | |
| workload_identity_provider: 'projects/473674835135/locations/global/workloadIdentityPools/github-actions/providers/data-infra' | |
| service_account: 'github-actions-terraform@cal-itp-data-infra-staging.iam.gserviceaccount.com' | |
| - name: Terraform Apply | |
| uses: dflook/terraform-apply@v1 | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| with: | |
| path: iac/cal-itp-data-infra-staging/composer/us | |
| auto_approve: ${{ contains(github.ref, 'refs/heads/staging') }} | |
| airflow-production: | |
| name: Production Airflow | |
| runs-on: ubuntu-latest | |
| needs: [airflow-changes] | |
| if: ${{ github.ref == 'refs/heads/main' && (needs.airflow-changes.outputs.production == '[]' || needs.airflow-changes.outputs.production == '') }} | |
| permissions: | |
| contents: read | |
| id-token: write | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - uses: google-github-actions/auth@v2 | |
| with: | |
| create_credentials_file: 'true' | |
| project_id: cal-itp-data-infra | |
| workload_identity_provider: 'projects/1005246706141/locations/global/workloadIdentityPools/github-actions/providers/data-infra' | |
| service_account: 'github-actions-terraform@cal-itp-data-infra.iam.gserviceaccount.com' | |
| - name: Terraform Apply | |
| uses: dflook/terraform-apply@v1 | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| with: | |
| path: iac/cal-itp-data-infra/airflow/us | |
| composer-production: | |
| name: Production Composer | |
| runs-on: ubuntu-latest | |
| needs: [composer-changes] | |
| if: ${{ github.ref == 'refs/heads/main' && (needs.composer-changes.outputs.production == '[]' || needs.composer-changes.outputs.production == '') }} | |
| permissions: | |
| contents: read | |
| id-token: write | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - uses: google-github-actions/auth@v2 | |
| with: | |
| create_credentials_file: 'true' | |
| project_id: cal-itp-data-infra | |
| workload_identity_provider: 'projects/1005246706141/locations/global/workloadIdentityPools/github-actions/providers/data-infra' | |
| service_account: 'github-actions-terraform@cal-itp-data-infra.iam.gserviceaccount.com' | |
| - name: Terraform Apply | |
| uses: dflook/terraform-apply@v1 | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| with: | |
| path: iac/cal-itp-data-infra/composer/us |