Merge pull request #11 from kingsdigitallab/BoglarkaAFarkas-patch-4 #21
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI - ETL and Frontend | |
| on: | |
| push: | |
| branches: | |
| - develop | |
| - main | |
| paths: | |
| - ".github/workflows/**" | |
| - "data/0_raw/**" | |
| - "etl/**" | |
| - "frontend/**" | |
| workflow_dispatch: | |
| permissions: | |
| contents: write | |
| jobs: | |
| changes: | |
| name: Detect changed paths | |
| runs-on: ubuntu-22.04 | |
| outputs: | |
| workflows_changed: ${{ steps.filter.outputs.workflows }} | |
| raw_changed: ${{ steps.filter.outputs.raw }} | |
| etl_changed: ${{ steps.filter.outputs.etl }} | |
| frontend_changed: ${{ steps.filter.outputs.frontend }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| submodules: true | |
| - name: Paths filter | |
| id: filter | |
| uses: dorny/paths-filter@v3 | |
| with: | |
| base: main | |
| filters: | | |
| workflows: | |
| - '.github/workflows/**' | |
| raw: | |
| - 'data/0_raw/**' | |
| etl: | |
| - 'etl/**' | |
| frontend: | |
| - 'frontend/**' | |
| validate-raw: | |
| name: Validate raw data | |
| needs: changes | |
| if: | | |
| needs.changes.outputs.workflows_changed == 'true' || | |
| needs.changes.outputs.raw_changed == 'true' || | |
| needs.changes.outputs.etl_changed == 'true' || | |
| github.event_name == 'workflow_dispatch' | |
| runs-on: ubuntu-22.04 | |
| defaults: | |
| run: | |
| working-directory: etl | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| submodules: true | |
| - name: Set up uv (Python) | |
| uses: astral-sh/setup-uv@v7 | |
| with: | |
| python-version: "3.11" | |
| - name: Validate CSV files | |
| env: | |
| PYTHONUNBUFFERED: "1" | |
| run: | | |
| uv run scripts/validate_raw_data.py | |
| etl: | |
| name: Run ETL | |
| needs: [changes, validate-raw] | |
| if: | | |
| needs.changes.outputs.workflows_changed == 'true' || | |
| needs.changes.outputs.raw_changed == 'true' || | |
| needs.changes.outputs.etl_changed == 'true' || | |
| github.event_name == 'workflow_dispatch' | |
| runs-on: ubuntu-22.04 | |
| defaults: | |
| run: | |
| working-directory: etl | |
| outputs: | |
| data_changed: ${{ steps.auto_commit.outputs.changes_detected }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| submodules: true | |
| - name: Set up uv (Python) | |
| uses: astral-sh/setup-uv@v7 | |
| with: | |
| python-version: "3.11" | |
| - name: Run ETL | |
| env: | |
| PYTHONUNBUFFERED: "1" | |
| run: | | |
| uv run main.py | |
| - name: Auto-commit processed data | |
| id: auto_commit | |
| uses: stefanzweifel/git-auto-commit-action@v5 | |
| with: | |
| commit_message: "chore(etl): update processed data [skip ci]" | |
| file_pattern: | | |
| data/1_interim/** | |
| data/2_final/** | |
| validate-final: | |
| name: Validate final data | |
| needs: [changes, etl] | |
| if: | | |
| needs.changes.outputs.workflows_changed == 'true' || | |
| needs.changes.outputs.raw_changed == 'true' || | |
| needs.changes.outputs.etl_changed == 'true' || | |
| github.event_name == 'workflow_dispatch' | |
| runs-on: ubuntu-22.04 | |
| defaults: | |
| run: | |
| working-directory: etl | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| submodules: true | |
| - name: Set up uv (Python) | |
| uses: astral-sh/setup-uv@v7 | |
| with: | |
| python-version: "3.11" | |
| - name: Validate JSON files | |
| env: | |
| PYTHONUNBUFFERED: "1" | |
| run: | | |
| uv run scripts/validate_final_data.py | |
| frontend: | |
| name: Build and deploy frontend | |
| needs: [changes, etl, validate-final] | |
| if: | | |
| needs.changes.outputs.workflows_changed == 'true' || | |
| needs.etl.outputs.data_changed == 'true' || | |
| needs.changes.outputs.frontend_changed == 'true' || | |
| github.event_name == 'workflow_dispatch' | |
| runs-on: ubuntu-22.04 | |
| defaults: | |
| run: | |
| working-directory: frontend | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| submodules: true | |
| - name: Set up bun | |
| uses: oven-sh/setup-bun@v2 | |
| - name: Install dependencies | |
| run: | | |
| bun install | |
| - name: Build | |
| env: | |
| GITHUB_ACTIONS_BUILD: true | |
| run: | | |
| bun run build | |
| - name: Deploy to GitHub pages | |
| uses: peaceiris/actions-gh-pages@v3 | |
| with: | |
| publish_dir: ./frontend/build | |
| github_token: ${{ secrets.GITHUB_TOKEN }} |