Merge pull request #265 from allenai/gabi/wandb_bugfix #1401
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build, Test, and Publish Docker Image | |
| on: | |
| push: | |
| branches: | |
| - master | |
| pull_request: | |
| branches: | |
| - master | |
| workflow_dispatch: | |
| env: | |
| REGISTRY: ghcr.io | |
| IMAGE_NAME: ${{ github.repository }} | |
| SERVICE_NAME: "rslearn_projects" | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| build: | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| packages: write | |
| outputs: | |
| ghcr_docker_image: ${{ steps.image-names.outputs.ghcr_image_name }} | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Log in to the Container registry | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: ${{ env.REGISTRY }} | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Extract metadata (tags, labels) for Docker | |
| id: meta | |
| uses: docker/metadata-action@v5 | |
| with: | |
| images: | | |
| ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} | |
| tags: | | |
| type=sha,format=long | |
| type=sha,format=short | |
| type=raw,value=latest,enable={{is_default_branch}} | |
| - name: Cleanup disk space | |
| run: | | |
| sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true | |
| sudo docker image prune --all --force >/dev/null 2>&1 || true | |
| sudo rm -rf /usr/share/dotnet | |
| sudo rm -rf /opt/ghc | |
| sudo rm -rf /usr/local/share/boost | |
| # Same thing for olmoearth_run repository. | |
| - name: Start ssh-agent and add olmoearth_run deploy key | |
| uses: webfactory/[email protected] | |
| with: | |
| ssh-private-key: ${{ secrets.DEPLOY_KEY_FOR_OLMOEARTH_RUN_CLONE }} | |
| - name: Clone olmoearth_run and update requirements-extra.txt. | |
| run: | | |
| git clone [email protected]:allenai/olmoearth_run.git docker_build/olmoearth_run | |
| echo "olmoearth_run @ /opt/rslearn_projects/docker_build/olmoearth_run/" >> requirements-extra.txt | |
| - name: Build and push Docker image | |
| id: build-push | |
| uses: docker/build-push-action@v6 | |
| with: | |
| context: . | |
| push: true | |
| tags: ${{ steps.meta.outputs.tags }} | |
| labels: ${{ steps.meta.outputs.labels }} | |
| build-args: | | |
| GIT_USERNAME=${{ secrets.GIT_USERNAME }} | |
| GIT_TOKEN=${{ secrets.GIT_TOKEN }} | |
| - name: Store Image Names | |
| # We need the docker image name downstream in test & deploy. This saves the full docker image names to outputs | |
| id: image-names | |
| run: |- | |
| GHCR_IMAGE="${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}@${{ steps.build-push.outputs.digest }}" | |
| GHCR_IMAGE=`echo ${GHCR_IMAGE} | tr '[:upper:]' '[:lower:]'` # docker requires that all image names be lowercase | |
| echo "ghcr.io Docker image name is ${GHCR_IMAGE}" | |
| echo "ghcr_image_name=\"${GHCR_IMAGE}\"" >> $GITHUB_OUTPUT | |
| test: | |
| needs: build | |
| runs-on: ubuntu-latest-m | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Log in to the Container registry | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: ${{ env.REGISTRY }} | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Cleanup disk space | |
| run: | | |
| sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true | |
| sudo docker image prune --all --force >/dev/null 2>&1 || true | |
| sudo rm -rf /usr/share/dotnet | |
| sudo rm -rf /opt/ghc | |
| sudo rm -rf /usr/local/share/boost | |
| - name: Pull the latest image | |
| run: | | |
| docker pull ${{ needs.build.outputs.ghcr_docker_image }} | |
| - name: Authenticate into gcp | |
| uses: "google-github-actions/auth@v2" | |
| with: | |
| credentials_json: ${{ secrets.GOOGLE_CREDENTIALS }} | |
| - name: Run unit tests | |
| run: | | |
| echo "Executing unit tests" | |
| docker run --rm \ | |
| -e CI="true" \ | |
| ${{ needs.build.outputs.ghcr_docker_image }} \ | |
| pytest tests/unit -vv | |
| - name: Run integration tests | |
| run: | | |
| echo "Executing integration tests" | |
| docker run --rm \ | |
| -e CI="true" \ | |
| -e AWS_ACCESS_KEY_ID="${AWS_ACCESS_KEY_ID}" \ | |
| -e AWS_SECRET_ACCESS_KEY="${AWS_SECRET_ACCESS_KEY}" \ | |
| -e BEAKER_TOKEN="${BEAKER_TOKEN}" \ | |
| -v ${{env.GOOGLE_GHA_CREDS_PATH}}:/tmp/gcp-credentials.json:ro \ | |
| -e GOOGLE_APPLICATION_CREDENTIALS=/tmp/gcp-credentials.json \ | |
| -e RSLP_BUCKET=rslearn-eai \ | |
| -e RSLP_PREFIX=gs://rslearn-eai \ | |
| -e BKT_PROJECT_ID=earthsystem-dev-c3po \ | |
| -e BKT_BUCKET_NAME=rslp-tests \ | |
| -e BKT_BIGTABLE_PROJECT_ID=earthsystem-dev-c3po \ | |
| -e BKT_BIGTABLE_INSTANCE_ID=rslp-bigtable-test-instance \ | |
| -e TEST_PUBSUB_PROJECT=earthsystem-dev-c3po \ | |
| -e TEST_PUBSUB_TOPIC=rslearn_projects_test_topic \ | |
| -e TEST_PUBSUB_SUBSCRIPTION=rslearn_projects_test_subscription \ | |
| -e TEST_QUEUE_NAME=favyen/rslearn-projects-test-queue \ | |
| ${{ needs.build.outputs.ghcr_docker_image }} \ | |
| pytest tests/integration -vv --forked | |
| env: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| BEAKER_TOKEN: ${{ secrets.BEAKER_TOKEN_2 }} | |
| - name: Clean up | |
| if: always() | |
| run: | | |
| docker image prune -f | |
| # test-slow: | |
| # runs-on: GPU-Enabled-Runner | |
| # steps: | |
| # - name: Checkout repository | |
| # uses: actions/checkout@v4 | |
| # - name: Log in to the Container registry | |
| # uses: docker/login-action@v3 | |
| # with: | |
| # registry: ${{ env.REGISTRY }} | |
| # username: ${{ github.actor }} | |
| # password: ${{ secrets.GITHUB_TOKEN }} | |
| # - name: Log in to the Container registry | |
| # uses: docker/login-action@v3 | |
| # with: | |
| # registry: ${{ env.REGISTRY }} | |
| # username: ${{ github.actor }} | |
| # password: ${{ secrets.GITHUB_TOKEN }} | |
| # - name: Cleanup disk space | |
| # run: | | |
| # sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true | |
| # sudo docker image prune --all --force >/dev/null 2>&1 || true | |
| # sudo rm -rf /usr/share/dotnet | |
| # sudo rm -rf /opt/ghc | |
| # sudo rm -rf /usr/local/share/boost | |
| # - name: Build docker images | |
| # run: | | |
| # COMPOSE_DOCKER_CLI_BUILD=1 DOCKER_BUILDKIT=1 docker compose -f docker-compose.yaml build | |
| # - name: Authenticate into gcp | |
| # uses: "google-github-actions/auth@v2" | |
| # with: | |
| # credentials_json: ${{ secrets.GOOGLE_CREDENTIALS }} | |
| # - name: Run tests with Docker Compose | |
| # run: | | |
| # docker run \ | |
| # --gpus all \ | |
| # --shm-size=15g \ | |
| # -e AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }} \ | |
| # -e AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }} \ | |
| # -v ${{env.GOOGLE_GHA_CREDS_PATH}}:/tmp/gcp-credentials.json:ro \ | |
| # -e GOOGLE_APPLICATION_CREDENTIALS=/tmp/gcp-credentials.json \ | |
| # -e RSLP_BUCKET=rslearn-eai \ | |
| # -e RSLP_PREFIX=gs://rslearn-eai \ | |
| # rslearn_projects-test pytest tests/integration_slow/ -vv | |
| # - name: Clean up | |
| # if: always() | |
| # run: | | |
| # docker compose -f docker-compose.yaml down |