migrated to public git repo

.github/push_docker.py

@@ -0,0 +1,66 @@
+#!/usr/bin/env python
+
+import subprocess
+import sys
+import re
+import os
+
+DOCKER_REPO = sys.argv[1]
+IMAGE_TAG = sys.argv[2]
+BRANCH_NAME = sys.argv[3]
+
+quarterly_release_branch_match = re.match(
+    "(dmc|external|peddep)-(\\d\\d.\\d)", BRANCH_NAME
+)
+test_deploy_branch_match = re.match("test-(.*)", BRANCH_NAME)
+dest_tags = set()
+
+# figure out what the tag on the docker image should be. If nothing than don't bother pushing this build
+if (
+    BRANCH_NAME == "master"
+    or BRANCH_NAME == "internal"
+    or BRANCH_NAME == "qa"
+    or test_deploy_branch_match is not None
+    or quarterly_release_branch_match is not None
+):
+    dest_tags.add(BRANCH_NAME)
+
+if quarterly_release_branch_match is not None:
+    env_name = quarterly_release_branch_match.group(1)
+    # find the last branch with this prefix, because and only tag builds for the "latest" one
+    # that is, if we are if the repo contains dmc-20q4 and dmc-21q1, only push an image when we're
+    # building dmc-21q1, not dmc-20q4
+    all_branch_names = (
+        subprocess.check_output(
+            "git for-each-ref --format '%(refname:lstrip=3)'", shell=True
+        )
+        .decode("utf8")
+        .split("\n")
+    )
+    matching_branch_names = sorted(
+        [x for x in all_branch_names if x.startswith(env_name + "-")]
+    )
+    last_release_branch = matching_branch_names[-1]
+
+    if BRANCH_NAME == last_release_branch:
+        dest_tags.add(env_name)
+
+
+def run_each(text):
+    statements = [x.strip() for x in text.split("\n") if x.strip() != ""]
+    for statement in statements:
+        print(f"Executing: {statement}")
+        subprocess.check_call(statement, shell=True)
+
+
+if "master" in dest_tags:
+    # use latest tag for master branch builds as well because that's what people usually grab
+    dest_tags.add("latest")
+
+for dest_tag in dest_tags:
+    run_each(
+        f"""
+        docker tag {IMAGE_TAG} {DOCKER_REPO}:{dest_tag}
+        docker push {DOCKER_REPO}:{dest_tag}
+        """
+    )

.github/workflows/build_app.yml

@@ -0,0 +1,114 @@
+name: Build portal-backend
+
+on:
+  pull_request:
+  workflow_dispatch:
+  push:
+    branches:
+      - "master"
+      - "internal"
+      - "qa"
+      - "peddep*"
+      - "dmc*"
+      - "external*"
+      - "test-*"
+
+env:
+  DOCKER_REPO: us.gcr.io/broad-achilles/depmap
+  DOCKER_TAG: ga-build-${{ github.run_number }}
+
+jobs:
+  run-frontend-tests:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Setup Node 19.4.0
+        uses: actions/setup-node@v3
+        with:
+          node-version: 19.4.0
+      - name: Install yarn dependencies
+        run: "cd frontend && NODE_OPTIONS='--openssl-legacy-provider' CI=true yarn install"
+      - name: Build webpack
+        run: "cd frontend && NODE_OPTIONS='--openssl-legacy-provider --max_old_space_size=4096' yarn build:portal"
+      - name: Run tests
+        run: yarn --cwd frontend test
+
+  build-docker:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Get yarn cache directory path
+        id: yarn-cache-dir-path
+        run: echo "dir=$(yarn --cwd frontend cache dir)" >> $GITHUB_OUTPUT
+
+      - uses: actions/cache@v3
+        id: yarn-cache # use this to check for `cache-hit` (`steps.yarn-cache.outputs.cache-hit != 'true'`)
+        with:
+          path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
+          key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-yarn-
+      - name: Update build id
+        run: |
+          echo "SHA: ${{ github.sha }} Build: ${{ github.run_number	}}" > portal-backend/depmap/templates/build_info.html
+          echo "SHA=\"${{ github.sha }}\"" > portal-backend/depmap/settings/build.py
+          echo "BUILD=\"${{ github.run_number	}}\"" >> portal-backend/depmap/settings/build.py
+      - name: Login to GCR
+        uses: docker/login-action@v2
+        with:
+          registry: us.gcr.io
+          username: _json_key
+          password: ${{ secrets.DEPMAP_DEPLOY_SVC_ACCT }}
+      - name: Build Docker image
+        run: cd portal-backend && bash build-docker-image.sh ${{ env.DOCKER_REPO }}:${{ env.DOCKER_TAG }}
+      - name: Run pyright-ratchet
+        run: docker run -e HOME=/install/pyright ${{ env.DOCKER_REPO }}:${{ env.DOCKER_TAG }} pyright-ratchet
+      - name: Run import linter
+        run: docker run ${{ env.DOCKER_REPO }}:${{ env.DOCKER_TAG }} lint-imports
+      - name: Push Docker image
+        run: docker push ${{ env.DOCKER_REPO }}:${{ env.DOCKER_TAG }}
+
+  run-pytest:
+    runs-on: ubuntu-latest
+    needs: build-docker
+    strategy:
+      matrix:
+        pytest-group: [1, 2, 3, 4, 5]
+    steps:
+      - uses: actions/checkout@v3
+      - name: Login to GCR
+        uses: docker/login-action@v2
+        with:
+          registry: us.gcr.io
+          username: _json_key
+          password: ${{ secrets.DEPMAP_DEPLOY_SVC_ACCT }}
+      - name: Pull Docker image
+        run: docker pull ${{ env.DOCKER_REPO }}:${{ env.DOCKER_TAG }}
+      - name: Get pytest durations
+        uses: actions/cache@v3
+        with:
+          path: .test_durations
+          key: ${{ runner.os }}-pytest-durations
+      - name: Run pytest
+        run: cd portal-backend && docker run -v $PWD/../.test_durations:/install/.test_durations -e PYTHONWARNINGS="ignore::DeprecationWarning" ${{ env.DOCKER_REPO }}:${{ env.DOCKER_TAG }} pytest --splits 5 --group ${{ matrix.pytest-group }}
+
+  push-docker-tags:
+    runs-on: ubuntu-latest
+    needs:
+      - run-pytest
+      - run-frontend-tests
+    steps:
+      - uses: actions/checkout@v3
+      - name: Login to GCR
+        uses: docker/login-action@v2
+        with:
+          registry: us.gcr.io
+          username: _json_key
+          password: ${{ secrets.DEPMAP_DEPLOY_SVC_ACCT }}
+      - name: Pull Docker image
+        run: docker pull ${{ env.DOCKER_REPO }}:${{ env.DOCKER_TAG }}
+      - name: Push Docker tags
+        run: |
+          chmod +x ./.github/push_docker.py
+          # GITHUB_REF##*/ is github actions magic which expands to the branch name
+          ./.github/push_docker.py ${{env.DOCKER_REPO}} ${{ env.DOCKER_REPO }}:${{ env.DOCKER_TAG }} ${GITHUB_REF##*/}