From 642ea11a6a571bcf7628647622cbd55b9e6dca8a Mon Sep 17 00:00:00 2001 From: Gustavo Valverde Date: Fri, 28 Jun 2024 10:30:17 -0400 Subject: [PATCH 01/33] add: uptime-kuma and workflow files --- .github/dependabot.yml | 22 ++++ .github/workflows/cd-deploy-to-dev.yml | 52 ++++++++ .github/workflows/cd-deploy-to-prod.yml | 46 +++++++ .github/workflows/cd-deploy-to-test.yml | 54 +++++++++ .github/workflows/chore-clean-dev.yml | 30 +++++ .github/workflows/ci-lint-codebase.patch.yml | 18 +++ .github/workflows/ci-lint-codebase.yml | 57 +++++++++ .github/workflows/sub-build-docker-image.yml | 117 ++++++++++++++++++ .github/workflows/sub-cloudrun-deploy.yml | 119 +++++++++++++++++++ .github/workflows/sub-unit-tests.yml | 52 ++++++++ Dockerfile | 45 +++++++ etc/litestream.yml | 4 + scripts/run.sh | 13 ++ 13 files changed, 629 insertions(+) create mode 100644 .github/dependabot.yml create mode 100644 .github/workflows/cd-deploy-to-dev.yml create mode 100644 .github/workflows/cd-deploy-to-prod.yml create mode 100644 .github/workflows/cd-deploy-to-test.yml create mode 100644 .github/workflows/chore-clean-dev.yml create mode 100644 .github/workflows/ci-lint-codebase.patch.yml create mode 100644 .github/workflows/ci-lint-codebase.yml create mode 100644 .github/workflows/sub-build-docker-image.yml create mode 100644 .github/workflows/sub-cloudrun-deploy.yml create mode 100644 .github/workflows/sub-unit-tests.yml create mode 100644 Dockerfile create mode 100644 etc/litestream.yml create mode 100755 scripts/run.sh diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..3356b82 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,22 @@ +version: 2 +updates: + - package-ecosystem: npm + directory: / + schedule: + interval: monthly + commit-message: + prefix: "bump(deps): " + + - package-ecosystem: docker + directory: / + schedule: + interval: monthly + commit-message: + prefix: "bump(docker) " + + - package-ecosystem: github-actions + directory: /.github/workflows + schedule: + interval: monthly + commit-message: + prefix: "bump(actions) " diff --git a/.github/workflows/cd-deploy-to-dev.yml b/.github/workflows/cd-deploy-to-dev.yml new file mode 100644 index 0000000..c6c3226 --- /dev/null +++ b/.github/workflows/cd-deploy-to-dev.yml @@ -0,0 +1,52 @@ +name: Deploy to dev + +on: + pull_request: + types: [opened, synchronize, reopened, labeled] + paths: + - '**.sh*' + - '**.ts*' + - Dockerfile + - package.json + - pnpm-lock.yaml + - .github/workflows/cd-deploy-to-dev.yml + - .github/workflows/sub-cloudrun-deploy.yml + +concurrency: + # Ensures that only one workflow task will run at a time. Previous builds, if + # already in process, will get cancelled. Only the latest commit will be allowed + # to run, cancelling any workflows in between + group: ${{ github.workflow }}-${{ github.job }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + test: + uses: ./.github/workflows/sub-unit-tests.yml + with: + node_env: development + + build: + uses: ./.github/workflows/sub-build-docker-image.yml + with: + environment: dev + dockerfile_path: ./docker/Dockerfile + dockerfile_target: runner + app_name: ${{ vars.APP_NAME }} + registry: ${{ vars.GAR_BASE }} + secrets: inherit + + deploy: + needs: [build] + uses: ./.github/workflows/sub-cloudrun-deploy.yml + with: + environment: dev + project_id: ${{ vars.GCP_PROJECT }} + region: ${{ vars.GCP_REGION }} + app_name: ${{ vars.APP_NAME }} + registry: ${{ vars.GAR_BASE }} + image_digest: ${{ needs.build.outputs.image_digest }} + min_instances: '0' + max_instances: '30' + cpu: '1' + memory: 1Gi + secrets: inherit diff --git a/.github/workflows/cd-deploy-to-prod.yml b/.github/workflows/cd-deploy-to-prod.yml new file mode 100644 index 0000000..76aa3c3 --- /dev/null +++ b/.github/workflows/cd-deploy-to-prod.yml @@ -0,0 +1,46 @@ +name: Deploy to prod + +on: + release: + types: + - published + +concurrency: + # Ensures that only one workflow task will run at a time. Previous builds, if + # already in process, will get cancelled. Only the latest commit will be allowed + # to run, cancelling any workflows in between + group: ${{ github.workflow }}-${{ github.job }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + test: + uses: ./.github/workflows/sub-unit-tests.yml + with: + node_env: production + + build: + # needs: [test] + uses: ./.github/workflows/sub-build-docker-image.yml + with: + environment: prod + dockerfile_path: ./docker/Dockerfile + dockerfile_target: runner + app_name: ${{ vars.APP_NAME }} + registry: ${{ vars.GAR_BASE }} + secrets: inherit + + deploy: + needs: [build] + uses: ./.github/workflows/sub-cloudrun-deploy.yml + with: + environment: prod + project_id: ${{ vars.GCP_PROJECT }} + region: ${{ vars.GCP_REGION }} + app_name: ${{ vars.APP_NAME }} + registry: ${{ vars.GAR_BASE }} + image_digest: ${{ needs.build.outputs.image_digest }} + min_instances: '1' + max_instances: '100' + cpu: '1' + memory: 1Gi + secrets: inherit diff --git a/.github/workflows/cd-deploy-to-test.yml b/.github/workflows/cd-deploy-to-test.yml new file mode 100644 index 0000000..01fd82a --- /dev/null +++ b/.github/workflows/cd-deploy-to-test.yml @@ -0,0 +1,54 @@ +name: Deploy to test + +on: + push: + branches: + - main + paths: + - '**.sh*' + - '**.ts*' + - Dockerfile + - package.json + - pnpm-lock.yaml + - .github/workflows/cd-deploy-to-test.yml + - .github/workflows/sub-cloudrun-deploy.yml + +concurrency: + # Ensures that only one workflow task will run at a time. Previous builds, if + # already in process, will get cancelled. Only the latest commit will be allowed + # to run, cancelling any workflows in between + group: ${{ github.workflow }}-${{ github.job }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + test: + uses: ./.github/workflows/sub-unit-tests.yml + with: + node_env: production + + build: + needs: [test] + uses: ./.github/workflows/sub-build-docker-image.yml + with: + environment: test + dockerfile_path: ./docker/Dockerfile + dockerfile_target: runner + app_name: ${{ vars.APP_NAME }} + registry: ${{ vars.GAR_BASE }} + secrets: inherit + + deploy: + needs: [build] + uses: ./.github/workflows/sub-cloudrun-deploy.yml + with: + environment: test + project_id: ${{ vars.GCP_PROJECT }} + region: ${{ vars.GCP_REGION }} + app_name: ${{ vars.APP_NAME }} + registry: ${{ vars.GAR_BASE }} + image_digest: ${{ needs.build.outputs.image_digest }} + min_instances: '0' + max_instances: '30' + cpu: '1' + memory: 1Gi + secrets: inherit diff --git a/.github/workflows/chore-clean-dev.yml b/.github/workflows/chore-clean-dev.yml new file mode 100644 index 0000000..d476d14 --- /dev/null +++ b/.github/workflows/chore-clean-dev.yml @@ -0,0 +1,30 @@ +name: Clean dev instances + +on: + delete: + pull_request: + branches: + - main + types: + - closed + +jobs: + delete: + runs-on: ubuntu-latest + steps: + - name: Inject slug/short variables + uses: rlespinasse/github-slug-action@v4.4.1 + + - name: Authenticate to Google Cloud + id: auth + uses: google-github-actions/auth@v2.1.2 + with: + workload_identity_provider: '${{ vars.GCP_WIF }}' + service_account: '${{ vars.GCP_DEPLOYMENTS_SA }}' + + - name: Set up Cloud SDK + uses: google-github-actions/setup-gcloud@v1.1.1 + + - name: Removing CR service + run: | + gcloud run services delete ${{ vars.APP_NAME }}-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }} --region=${{ vars.GOOGLE_CLOUD_REGION }} --quiet diff --git a/.github/workflows/ci-lint-codebase.patch.yml b/.github/workflows/ci-lint-codebase.patch.yml new file mode 100644 index 0000000..8f61ff1 --- /dev/null +++ b/.github/workflows/ci-lint-codebase.patch.yml @@ -0,0 +1,18 @@ +name: Lint Code Base + +on: + pull_request: + branches: [main] + paths-ignore: + - '**.sh*' + - '**.ts*' + - Dockerfile + - package.json + - pnpm-lock.yaml + - .github/workflows/ci-lint-codebase.yml + +jobs: + linter: + runs-on: ubuntu-latest + steps: + - run: echo "Job not required" diff --git a/.github/workflows/ci-lint-codebase.yml b/.github/workflows/ci-lint-codebase.yml new file mode 100644 index 0000000..dbcfdc3 --- /dev/null +++ b/.github/workflows/ci-lint-codebase.yml @@ -0,0 +1,57 @@ +name: Lint Code Base + +on: + pull_request: + branches: [main] + paths: + - '**.sh*' + - '**.ts*' + - Dockerfile + - package.json + - pnpm-lock.yaml + - .github/workflows/ci-lint-codebase.yml + + push: + branches: [main] + paths: + - '**.sh*' + - '**.ts*' + - Dockerfile + - package.json + - pnpm-lock.yaml + - .github/workflows/ci-lint-codebase.yml + +concurrency: + # Ensures that only one workflow task will run at a time. Previous builds, if + # already in process, will get cancelled. Only the latest commit will be allowed + # to run, cancelling any workflows in between + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + linter: + runs-on: ubuntu-latest + steps: + - name: Checkout Code Repository + uses: actions/checkout@v4.1.1 + with: + # Full git history is needed to get a proper + # list of changed files within `super-linter` + fetch-depth: 0 + + - name: Lint Code Base + uses: super-linter/super-linter/slim@v5.2.1 + env: + LOG_LEVEL: ERROR + VALIDATE_ALL_CODEBASE: false + VALIDATE_SHELL_SHFMT: false + VALIDATE_JSCPD: false + VALIDATE_CSS: false + VALIDATE_EDITORCONFIG: false + VALIDATE_MARKDOWN: false + VALIDATE_DOCKERFILE_HADOLINT: false + LINTER_RULES_PATH: / + JAVASCRIPT_DEFAULT_STYLE: prettier + TYPESCRIPT_DEFAULT_STYLE: prettier + DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/sub-build-docker-image.yml b/.github/workflows/sub-build-docker-image.yml new file mode 100644 index 0000000..6b4bacb --- /dev/null +++ b/.github/workflows/sub-build-docker-image.yml @@ -0,0 +1,117 @@ +name: Build docker image + +on: + workflow_call: + inputs: + app_name: + required: true + type: string + dockerfile_path: + required: true + type: string + dockerfile_target: + required: true + type: string + registry: + required: true + type: string + environment: + required: true + type: string + outputs: + image_digest: + description: The image digest to be used on a caller workflow + value: ${{ jobs.build.outputs.image_digest }} + +jobs: + build: + name: Build images + timeout-minutes: 15 + runs-on: ubuntu-latest + outputs: + image_digest: ${{ steps.docker_build.outputs.digest }} + permissions: + contents: 'read' + id-token: 'write' + steps: + - uses: actions/checkout@v4.1.1 + with: + persist-credentials: false + + - name: Inject slug/short variables + uses: rlespinasse/github-slug-action@v4.4.1 + with: + short-length: 7 + + # Automatic tag management and OCI Image Format Specification for labels + - name: Docker meta + id: meta + uses: docker/metadata-action@v5.5.0 + with: + # list of Docker images to use as base name for tags + images: | + ${{ inputs.registry }}/${{ inputs.app_name }} + # generate Docker tags based on the following events/attributes + tags: | + type=schedule + # semver and ref,tag automatically add a "latest" tag, but only on stable releases + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + type=semver,pattern={{major}} + type=ref,event=tag + type=ref,event=branch + type=ref,event=pr + type=sha + # edge is the latest commit on the default branch. + type=edge,enable={{is_default_branch}} + + # Setup Docker Buildx to allow use of docker cache layers from GH + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v3.0.0 + + - name: Authenticate to Google Cloud + id: auth + uses: google-github-actions/auth@v2.1.2 + with: + workload_identity_provider: '${{ vars.GCP_WIF }}' + service_account: '${{ vars.GCP_ARTIFACTS_SA }}' + token_format: 'access_token' + # Some builds might take over an hour, and Google's default lifetime duration for + # an access token is 1 hour (3600s). We increase this to 3 hours (10800s) + # as some builds take over an hour. + access_token_lifetime: 10800s + + - name: Login to Google Artifact Registry + uses: docker/login-action@v3.0.0 + with: + registry: us-docker.pkg.dev + username: oauth2accesstoken + password: ${{ steps.auth.outputs.access_token }} + + # Build and push image to Google Artifact Registry, and possibly DockerHub + - name: Build & push + id: docker_build + uses: docker/build-push-action@v5.1.0 + with: + target: ${{ inputs.dockerfile_target }} + context: . + file: ${{ inputs.dockerfile_path }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + push: true + # To improve build speeds, for each branch we push an additional image to the registry, + # to be used as the caching layer, using the `max` caching mode. + # + # We use multiple cache sources to confirm a cache hit, starting from a per-branch cache, + # and if there's no hit, then continue with the `main` branch. When changes are added to a PR, + # they are usually smaller than the diff between the PR and `main` branch. So this provides the + # best performance. + # + # The caches are tried in top-down order, the first available cache is used: + # https://github.com/moby/moby/pull/26839#issuecomment-277383550 + cache-from: | + type=registry,ref=${{ inputs.registry }}/${{ inputs.app_name }}:${{ env.GITHUB_REF_SLUG_URL }}-cache + type=registry,ref=${{ inputs.registry }}/${{ inputs.app_name }}:${{ github.event.repository.default_branch }}-cache + cache-to: | + type=registry,ref=${{ inputs.registry }}/${{ inputs.app_name }}:${{ env.GITHUB_REF_SLUG_URL }}-cache,mode=min diff --git a/.github/workflows/sub-cloudrun-deploy.yml b/.github/workflows/sub-cloudrun-deploy.yml new file mode 100644 index 0000000..69b2e10 --- /dev/null +++ b/.github/workflows/sub-cloudrun-deploy.yml @@ -0,0 +1,119 @@ +name: Deploy to Cloud Run + +on: + workflow_call: + inputs: + app_name: + required: true + type: string + registry: + required: true + type: string + image_digest: + required: true + type: string + description: 'The image digest to deploy' + project_id: + required: true + type: string + description: 'The project to deploy to' + region: + required: true + type: string + description: 'The region to deploy to' + environment: + required: false + type: string + description: 'The environment to deploy to' + min_instances: + required: false + type: string + description: 'The minimum number of instances to deploy' + max_instances: + required: false + type: string + description: 'The maximum number of instances to deploy' + cpu: + required: false + type: string + description: 'The number of CPUs to use for the service' + memory: + required: false + type: string + description: 'The amount of memory to use for the service' + +jobs: + versioning: + runs-on: ubuntu-latest + outputs: + version: ${{ steps.set.outputs.version }} + steps: + - name: Getting API Version + id: get + uses: actions/github-script@v7 + if: ${{ github.event_name == 'release' }} + with: + result-encoding: string + script: | + return context.payload.release.tag_name.substring(0,2) + - name: Setting API Version + id: set + run: echo "version=${{ steps.get.outputs.result }}" >> "$GITHUB_OUTPUT" + + deploy: + name: Deploy to Cloud Run + needs: [versioning] + timeout-minutes: 10 + runs-on: ubuntu-latest + environment: + name: ${{ inputs.environment }} + url: ${{ steps.deploy.outputs.url }} + permissions: + contents: 'read' + id-token: 'write' + steps: + - name: Inject slug/short variables + uses: rlespinasse/github-slug-action@v4.4.1 + + - name: Authenticate to Google Cloud + id: auth + uses: google-github-actions/auth@v2.1.2 + with: + workload_identity_provider: '${{ vars.GCP_WIF }}' + service_account: '${{ vars.GCP_DEPLOYMENTS_SA }}' + + - name: Set up Cloud SDK + uses: google-github-actions/setup-gcloud@v2.1.0 + + - name: Deploy to cloud run + id: deploy + uses: google-github-actions/deploy-cloudrun@v2.2.0 + with: + service: ${{ inputs.app_name }}-${{ needs.versioning.outputs.version || env.GITHUB_HEAD_REF_SLUG || inputs.environment }} + image: ${{ inputs.registry }}/${{ inputs.app_name }}@${{ inputs.image_digest }} + region: ${{ inputs.region }} + gcloud_component: alpha + # env_vars: | + # secrets: | + flags: | + --min-instances=${{ inputs.min_instances }} + --max-instances=${{ inputs.max_instances }} + --cpu=${{ inputs.cpu }} + --memory=${{ inputs.memory }} + --network=projects/zfnd-dev-net-spoke-0/global/networks/dev-spoke-0 + --subnet=projects/zfnd-dev-net-spoke-0/regions/us-east1/subnetworks/dev-default-ue1 + + - name: Allow unauthenticated calls to the service + run: | + gcloud run services add-iam-policy-binding ${{ inputs.app_name }}-${{ needs.versioning.outputs.version || env.GITHUB_HEAD_REF_SLUG || inputs.environment }} \ + --region=${{ inputs.region }} --member=allUsers --role=roles/run.invoker --quiet + + - name: Test service with cURL + run: curl "${{ steps.deploy.outputs.url }}" + + - name: Add reaction + if: ${{ github.event_name == 'issue_comment' }} + uses: peter-evans/create-or-update-comment@v4.0.0 + with: + comment-id: ${{ github.event.comment.id }} + reactions: rocket diff --git a/.github/workflows/sub-unit-tests.yml b/.github/workflows/sub-unit-tests.yml new file mode 100644 index 0000000..b11992b --- /dev/null +++ b/.github/workflows/sub-unit-tests.yml @@ -0,0 +1,52 @@ +name: Unit Tests + +on: + workflow_call: + inputs: + node_env: + required: true + type: string + +concurrency: + # Ensures that only one workflow task will run at a time. Previous builds, if + # already in process, will get cancelled. Only the latest commit will be allowed + # to run, cancelling any workflows in between + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +env: + NODE_ENV: ${{ inputs.node_env }} + CI: true + +jobs: + test: + name: Test with Node.js ${{ matrix.node }} + timeout-minutes: 10 + runs-on: ubuntu-latest + + strategy: + fail-fast: false + matrix: + node: [lts/*, latest] + + steps: + - name: Checkout Code Repository + uses: actions/checkout@v4.1.1 + + - uses: pnpm/action-setup@v2.4.0 + + - name: Use Node.js ${{ matrix.node }} + uses: actions/setup-node@v4.0.1 + with: + node-version: ${{ matrix.node }} + cache: pnpm + + # - name: Audit for vulnerabilities + # run: npx audit-ci@^6 --config ./audit-ci.jsonc + + - name: Install and Build + run: | + pnpm install --frozen-lockfile + pnpm build + env: + NODE_OPTIONS: --max_old_space_size=4096 diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..a10b4b7 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,45 @@ +# syntax=docker/dockerfile:1 +# ===================== Create base stage ===================== +ARG NODE_VERSION=lts +ARG UPTIME_KUMA_VERSION=1.23.13 +ARG LITESTREAM_VERSION=0.3.13 +ARG WORK_DIR=/app +FROM louislam/uptime-kuma:${UPTIME_KUMA_VERSION} AS base + +ARG PORT=3001 +ARG WORK_DIR + +ENV WORK_DIR=${WORK_DIR} + +WORKDIR ${WORK_DIR} + +# ==== App specific variables + +ENV UPTIME_KUMA_IS_CONTAINER=1 +ENV UPTIME_KUMA_VERSION=${UPTIME_KUMA_VERSION} +ENV LITESTREAM_VERSION=${LITESTREAM_VERSION} + +# ===================== App Runner Stage ===================== +FROM base AS runner + +RUN apt-get update --no-install-recommends \ + && apt-get install -y ca-certificates iputils-ping wget \ + && rm -rf /var/lib/apt/lists/* + +# Copy all necessary files +COPY --from=litestream/litestream:${LITESTREAM_VERSION} /usr/local/bin/litestream /usr/local/bin/litestream + +USER node + +EXPOSE ${PORT} + +ENV PORT ${PORT} +ENV HOSTNAME 0.0.0.0 + +HEALTHCHECK --interval=60s --timeout=30s --start-period=180s --retries=5 CMD extra/healthcheck + +# Copy Litestream configuration file & startup script. +COPY etc/litestream.yml /etc/litestream.yml +COPY scripts/run.sh /scripts/run.sh + +CMD [ "/scripts/run.sh" ] diff --git a/etc/litestream.yml b/etc/litestream.yml new file mode 100644 index 0000000..742d56a --- /dev/null +++ b/etc/litestream.yml @@ -0,0 +1,4 @@ +dbs: + - path: /data/db + replicas: + - url: ${REPLICA_URL} diff --git a/scripts/run.sh b/scripts/run.sh new file mode 100755 index 0000000..2a8e9be --- /dev/null +++ b/scripts/run.sh @@ -0,0 +1,13 @@ +#!/bin/bash +set -e + +# Restore the database if it does not already exist. +if [ -f /data/db ]; then + echo "Database already exists, skipping restore" +else + echo "No database found, restoring from replica if exists" + litestream restore -v -if-replica-exists -o /data/db "${REPLICA_URL}" +fi + +# Run litestream with your app as the subprocess. +exec litestream replicate -exec "node server/server.js" From 814ee1f5c7ce88b5800284f49edcd33fcd959143 Mon Sep 17 00:00:00 2001 From: Gustavo Valverde Date: Fri, 28 Jun 2024 10:35:24 -0400 Subject: [PATCH 02/33] fix(deploy): remove extra files --- .github/workflows/cd-deploy-to-dev.yml | 5 --- .github/workflows/cd-deploy-to-prod.yml | 5 --- .github/workflows/cd-deploy-to-test.yml | 6 --- .github/workflows/sub-unit-tests.yml | 52 ------------------------- .gitignore | 2 + 5 files changed, 2 insertions(+), 68 deletions(-) delete mode 100644 .github/workflows/sub-unit-tests.yml diff --git a/.github/workflows/cd-deploy-to-dev.yml b/.github/workflows/cd-deploy-to-dev.yml index c6c3226..8bec0f8 100644 --- a/.github/workflows/cd-deploy-to-dev.yml +++ b/.github/workflows/cd-deploy-to-dev.yml @@ -20,11 +20,6 @@ concurrency: cancel-in-progress: true jobs: - test: - uses: ./.github/workflows/sub-unit-tests.yml - with: - node_env: development - build: uses: ./.github/workflows/sub-build-docker-image.yml with: diff --git a/.github/workflows/cd-deploy-to-prod.yml b/.github/workflows/cd-deploy-to-prod.yml index 76aa3c3..8dc787e 100644 --- a/.github/workflows/cd-deploy-to-prod.yml +++ b/.github/workflows/cd-deploy-to-prod.yml @@ -13,11 +13,6 @@ concurrency: cancel-in-progress: true jobs: - test: - uses: ./.github/workflows/sub-unit-tests.yml - with: - node_env: production - build: # needs: [test] uses: ./.github/workflows/sub-build-docker-image.yml diff --git a/.github/workflows/cd-deploy-to-test.yml b/.github/workflows/cd-deploy-to-test.yml index 01fd82a..9a09fe2 100644 --- a/.github/workflows/cd-deploy-to-test.yml +++ b/.github/workflows/cd-deploy-to-test.yml @@ -21,13 +21,7 @@ concurrency: cancel-in-progress: true jobs: - test: - uses: ./.github/workflows/sub-unit-tests.yml - with: - node_env: production - build: - needs: [test] uses: ./.github/workflows/sub-build-docker-image.yml with: environment: test diff --git a/.github/workflows/sub-unit-tests.yml b/.github/workflows/sub-unit-tests.yml deleted file mode 100644 index b11992b..0000000 --- a/.github/workflows/sub-unit-tests.yml +++ /dev/null @@ -1,52 +0,0 @@ -name: Unit Tests - -on: - workflow_call: - inputs: - node_env: - required: true - type: string - -concurrency: - # Ensures that only one workflow task will run at a time. Previous builds, if - # already in process, will get cancelled. Only the latest commit will be allowed - # to run, cancelling any workflows in between - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} - cancel-in-progress: true - -env: - NODE_ENV: ${{ inputs.node_env }} - CI: true - -jobs: - test: - name: Test with Node.js ${{ matrix.node }} - timeout-minutes: 10 - runs-on: ubuntu-latest - - strategy: - fail-fast: false - matrix: - node: [lts/*, latest] - - steps: - - name: Checkout Code Repository - uses: actions/checkout@v4.1.1 - - - uses: pnpm/action-setup@v2.4.0 - - - name: Use Node.js ${{ matrix.node }} - uses: actions/setup-node@v4.0.1 - with: - node-version: ${{ matrix.node }} - cache: pnpm - - # - name: Audit for vulnerabilities - # run: npx audit-ci@^6 --config ./audit-ci.jsonc - - - name: Install and Build - run: | - pnpm install --frozen-lockfile - pnpm build - env: - NODE_OPTIONS: --max_old_space_size=4096 diff --git a/.gitignore b/.gitignore index aecbf5b..52e02a0 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ +.trunk/ + # Created by https://www.toptal.com/developers/gitignore/api/windows,linux,macos,visualstudiocode,jetbrains,node,nextjs,vercel,amplify # Edit at https://www.toptal.com/developers/gitignore?templates=windows,linux,macos,visualstudiocode,jetbrains,node,nextjs,vercel,amplify From 25d494e685b4ee56be0910e2b2358ce2b65e6d78 Mon Sep 17 00:00:00 2001 From: Gustavo Valverde Date: Fri, 28 Jun 2024 10:46:42 -0400 Subject: [PATCH 03/33] fix(deployment): move `Dockerfile` to right location --- Dockerfile => docker/Dockerfile | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename Dockerfile => docker/Dockerfile (100%) diff --git a/Dockerfile b/docker/Dockerfile similarity index 100% rename from Dockerfile rename to docker/Dockerfile From 8a90602d691a7be2a4edfbd79b6400ab0c36256d Mon Sep 17 00:00:00 2001 From: Gustavo Valverde Date: Fri, 28 Jun 2024 11:22:06 -0400 Subject: [PATCH 04/33] fix(build): use correct parameters --- docker/Dockerfile | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index a10b4b7..2647582 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -2,7 +2,6 @@ # ===================== Create base stage ===================== ARG NODE_VERSION=lts ARG UPTIME_KUMA_VERSION=1.23.13 -ARG LITESTREAM_VERSION=0.3.13 ARG WORK_DIR=/app FROM louislam/uptime-kuma:${UPTIME_KUMA_VERSION} AS base @@ -16,8 +15,6 @@ WORKDIR ${WORK_DIR} # ==== App specific variables ENV UPTIME_KUMA_IS_CONTAINER=1 -ENV UPTIME_KUMA_VERSION=${UPTIME_KUMA_VERSION} -ENV LITESTREAM_VERSION=${LITESTREAM_VERSION} # ===================== App Runner Stage ===================== FROM base AS runner @@ -27,14 +24,13 @@ RUN apt-get update --no-install-recommends \ && rm -rf /var/lib/apt/lists/* # Copy all necessary files -COPY --from=litestream/litestream:${LITESTREAM_VERSION} /usr/local/bin/litestream /usr/local/bin/litestream +COPY --from=litestream/litestream:0.3.13 /usr/local/bin/litestream /usr/local/bin/litestream USER node EXPOSE ${PORT} -ENV PORT ${PORT} -ENV HOSTNAME 0.0.0.0 +ENV PORT=${PORT} HEALTHCHECK --interval=60s --timeout=30s --start-period=180s --retries=5 CMD extra/healthcheck From 81e89fcd59318dec49ea5650cc9ab7dacc8398be Mon Sep 17 00:00:00 2001 From: Gustavo Valverde Date: Fri, 28 Jun 2024 12:30:16 -0400 Subject: [PATCH 05/33] fix(deploy): use correct execution values --- .github/workflows/sub-cloudrun-deploy.yml | 3 ++- docker/Dockerfile | 2 -- scripts/run.sh | 2 +- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/sub-cloudrun-deploy.yml b/.github/workflows/sub-cloudrun-deploy.yml index 69b2e10..05be339 100644 --- a/.github/workflows/sub-cloudrun-deploy.yml +++ b/.github/workflows/sub-cloudrun-deploy.yml @@ -93,7 +93,8 @@ jobs: image: ${{ inputs.registry }}/${{ inputs.app_name }}@${{ inputs.image_digest }} region: ${{ inputs.region }} gcloud_component: alpha - # env_vars: | + env_vars: | + REPLICA_URL=${{ vars.REPLICA_URL }} # secrets: | flags: | --min-instances=${{ inputs.min_instances }} diff --git a/docker/Dockerfile b/docker/Dockerfile index 2647582..ea0f97d 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -26,8 +26,6 @@ RUN apt-get update --no-install-recommends \ # Copy all necessary files COPY --from=litestream/litestream:0.3.13 /usr/local/bin/litestream /usr/local/bin/litestream -USER node - EXPOSE ${PORT} ENV PORT=${PORT} diff --git a/scripts/run.sh b/scripts/run.sh index 2a8e9be..986f0a0 100755 --- a/scripts/run.sh +++ b/scripts/run.sh @@ -6,7 +6,7 @@ if [ -f /data/db ]; then echo "Database already exists, skipping restore" else echo "No database found, restoring from replica if exists" - litestream restore -v -if-replica-exists -o /data/db "${REPLICA_URL}" + litestream restore -if-replica-exists -o /data/db "${REPLICA_URL}" fi # Run litestream with your app as the subprocess. From ddb83fe69e997ab11719f910b96dd4ec9cb3d250 Mon Sep 17 00:00:00 2001 From: Gustavo Valverde Date: Fri, 28 Jun 2024 13:25:24 -0400 Subject: [PATCH 06/33] fix(deploy): do not use a litestream config --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index ea0f97d..fa3827c 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -33,7 +33,7 @@ ENV PORT=${PORT} HEALTHCHECK --interval=60s --timeout=30s --start-period=180s --retries=5 CMD extra/healthcheck # Copy Litestream configuration file & startup script. -COPY etc/litestream.yml /etc/litestream.yml +# COPY etc/litestream.yml /etc/litestream.yml COPY scripts/run.sh /scripts/run.sh CMD [ "/scripts/run.sh" ] From bb0666c26d872c396ef8efbad46846b83be45c33 Mon Sep 17 00:00:00 2001 From: Gustavo Valverde Date: Fri, 28 Jun 2024 14:20:35 -0400 Subject: [PATCH 07/33] fix(deploy): allow deploying to cloud run without mounting volumes --- docker/Dockerfile | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index fa3827c..76e4242 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -26,7 +26,10 @@ RUN apt-get update --no-install-recommends \ # Copy all necessary files COPY --from=litestream/litestream:0.3.13 /usr/local/bin/litestream /usr/local/bin/litestream -EXPOSE ${PORT} +# Create data directory (although this will likely be mounted too) as some services won't mount it. +RUN mkdir -p /data + + EXPOSE ${PORT} ENV PORT=${PORT} From e46d6daa88cf9af5aae7e4cc241fc7640d6d6dad Mon Sep 17 00:00:00 2001 From: Gustavo Valverde Date: Fri, 28 Jun 2024 14:26:38 -0400 Subject: [PATCH 08/33] fix(pipeline): deploy with correct triggers --- .github/workflows/cd-deploy-to-test.yml | 8 +++----- .github/workflows/ci-lint-codebase.patch.yml | 8 +++----- .github/workflows/ci-lint-codebase.yml | 8 +++----- 3 files changed, 9 insertions(+), 15 deletions(-) diff --git a/.github/workflows/cd-deploy-to-test.yml b/.github/workflows/cd-deploy-to-test.yml index 9a09fe2..3a793c0 100644 --- a/.github/workflows/cd-deploy-to-test.yml +++ b/.github/workflows/cd-deploy-to-test.yml @@ -5,11 +5,9 @@ on: branches: - main paths: - - '**.sh*' - - '**.ts*' - - Dockerfile - - package.json - - pnpm-lock.yaml + - '**/Dockerfile' + - 'scripts/**' + - 'etc/litestream.yml' - .github/workflows/cd-deploy-to-test.yml - .github/workflows/sub-cloudrun-deploy.yml diff --git a/.github/workflows/ci-lint-codebase.patch.yml b/.github/workflows/ci-lint-codebase.patch.yml index 8f61ff1..8a6115f 100644 --- a/.github/workflows/ci-lint-codebase.patch.yml +++ b/.github/workflows/ci-lint-codebase.patch.yml @@ -4,11 +4,9 @@ on: pull_request: branches: [main] paths-ignore: - - '**.sh*' - - '**.ts*' - - Dockerfile - - package.json - - pnpm-lock.yaml + - '**/Dockerfile' + - 'scripts/**' + - 'etc/litestream.yml' - .github/workflows/ci-lint-codebase.yml jobs: diff --git a/.github/workflows/ci-lint-codebase.yml b/.github/workflows/ci-lint-codebase.yml index dbcfdc3..0e057d0 100644 --- a/.github/workflows/ci-lint-codebase.yml +++ b/.github/workflows/ci-lint-codebase.yml @@ -4,11 +4,9 @@ on: pull_request: branches: [main] paths: - - '**.sh*' - - '**.ts*' - - Dockerfile - - package.json - - pnpm-lock.yaml + - '**/Dockerfile' + - 'scripts/**' + - 'etc/litestream.yml' - .github/workflows/ci-lint-codebase.yml push: From 3d1ff6f2ed9ee68342c41acfe260e1f6cf4cd115 Mon Sep 17 00:00:00 2001 From: Gustavo Valverde Date: Fri, 28 Jun 2024 14:30:07 -0400 Subject: [PATCH 09/33] fix(pipeline): deploy with correct triggers --- .github/workflows/cd-deploy-to-dev.yml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/.github/workflows/cd-deploy-to-dev.yml b/.github/workflows/cd-deploy-to-dev.yml index 8bec0f8..8446e15 100644 --- a/.github/workflows/cd-deploy-to-dev.yml +++ b/.github/workflows/cd-deploy-to-dev.yml @@ -4,11 +4,9 @@ on: pull_request: types: [opened, synchronize, reopened, labeled] paths: - - '**.sh*' - - '**.ts*' - - Dockerfile - - package.json - - pnpm-lock.yaml + - '**/Dockerfile' + - 'scripts/**' + - 'etc/litestream.yml' - .github/workflows/cd-deploy-to-dev.yml - .github/workflows/sub-cloudrun-deploy.yml From 74957ac8690b07a399ea10d17c1d9e43855b350f Mon Sep 17 00:00:00 2001 From: Gustavo Valverde Date: Fri, 28 Jun 2024 14:38:35 -0400 Subject: [PATCH 10/33] fix(build): `/etc/litestream.yml` is required --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 76e4242..ea9c19a 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -36,7 +36,7 @@ ENV PORT=${PORT} HEALTHCHECK --interval=60s --timeout=30s --start-period=180s --retries=5 CMD extra/healthcheck # Copy Litestream configuration file & startup script. -# COPY etc/litestream.yml /etc/litestream.yml +COPY etc/litestream.yml /etc/litestream.yml COPY scripts/run.sh /scripts/run.sh CMD [ "/scripts/run.sh" ] From 8f92818d58cfc90c4f158bf1b8fc706d549aacbe Mon Sep 17 00:00:00 2001 From: Gustavo Valverde Date: Fri, 28 Jun 2024 17:25:42 -0400 Subject: [PATCH 11/33] fix(deploy): fix db state and bucket sync --- .github/workflows/cd-deploy-to-prod.yml | 2 +- .github/workflows/sub-cloudrun-deploy.yml | 6 +++++- docker/Dockerfile | 15 +++++++++------ etc/litestream.yml | 2 +- scripts/run.sh | 4 ++-- 5 files changed, 18 insertions(+), 11 deletions(-) diff --git a/.github/workflows/cd-deploy-to-prod.yml b/.github/workflows/cd-deploy-to-prod.yml index 8dc787e..ab6c97f 100644 --- a/.github/workflows/cd-deploy-to-prod.yml +++ b/.github/workflows/cd-deploy-to-prod.yml @@ -35,7 +35,7 @@ jobs: registry: ${{ vars.GAR_BASE }} image_digest: ${{ needs.build.outputs.image_digest }} min_instances: '1' - max_instances: '100' + max_instances: '10' cpu: '1' memory: 1Gi secrets: inherit diff --git a/.github/workflows/sub-cloudrun-deploy.yml b/.github/workflows/sub-cloudrun-deploy.yml index 05be339..aebddc5 100644 --- a/.github/workflows/sub-cloudrun-deploy.yml +++ b/.github/workflows/sub-cloudrun-deploy.yml @@ -87,7 +87,7 @@ jobs: - name: Deploy to cloud run id: deploy - uses: google-github-actions/deploy-cloudrun@v2.2.0 + uses: google-github-actions/deploy-cloudrun@v2.6.0 with: service: ${{ inputs.app_name }}-${{ needs.versioning.outputs.version || env.GITHUB_HEAD_REF_SLUG || inputs.environment }} image: ${{ inputs.registry }}/${{ inputs.app_name }}@${{ inputs.image_digest }} @@ -95,12 +95,16 @@ jobs: gcloud_component: alpha env_vars: | REPLICA_URL=${{ vars.REPLICA_URL }} + env_vars_update_strategy: overwrite # secrets: | flags: | --min-instances=${{ inputs.min_instances }} --max-instances=${{ inputs.max_instances }} --cpu=${{ inputs.cpu }} --memory=${{ inputs.memory }} + --service-account=${{ vars.GCP_BUCKET_SA }} + --add-volume=name=files,type=in-memory + --add-volume-mount=volume=files,mount-path=/data --network=projects/zfnd-dev-net-spoke-0/global/networks/dev-spoke-0 --subnet=projects/zfnd-dev-net-spoke-0/regions/us-east1/subnetworks/dev-default-ue1 diff --git a/docker/Dockerfile b/docker/Dockerfile index ea9c19a..e33d33c 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -2,19 +2,22 @@ # ===================== Create base stage ===================== ARG NODE_VERSION=lts ARG UPTIME_KUMA_VERSION=1.23.13 -ARG WORK_DIR=/app +ARG APP_HOME=/app FROM louislam/uptime-kuma:${UPTIME_KUMA_VERSION} AS base ARG PORT=3001 -ARG WORK_DIR +ARG APP_HOME -ENV WORK_DIR=${WORK_DIR} +ENV APP_HOME=${APP_HOME} -WORKDIR ${WORK_DIR} +WORKDIR ${APP_HOME} # ==== App specific variables ENV UPTIME_KUMA_IS_CONTAINER=1 +ARG DATA_DIR=/data/ +ENV DATA_DIR=${DATA_DIR} +ENV DB_PATH=${DATA_DIR}kuma.db # ===================== App Runner Stage ===================== FROM base AS runner @@ -27,9 +30,9 @@ RUN apt-get update --no-install-recommends \ COPY --from=litestream/litestream:0.3.13 /usr/local/bin/litestream /usr/local/bin/litestream # Create data directory (although this will likely be mounted too) as some services won't mount it. -RUN mkdir -p /data +RUN mkdir -p ${DATA_DIR} - EXPOSE ${PORT} +EXPOSE ${PORT} ENV PORT=${PORT} diff --git a/etc/litestream.yml b/etc/litestream.yml index 742d56a..5716932 100644 --- a/etc/litestream.yml +++ b/etc/litestream.yml @@ -1,4 +1,4 @@ dbs: - - path: /data/db + - path: ${DB_PATH} replicas: - url: ${REPLICA_URL} diff --git a/scripts/run.sh b/scripts/run.sh index 986f0a0..61476b1 100755 --- a/scripts/run.sh +++ b/scripts/run.sh @@ -2,11 +2,11 @@ set -e # Restore the database if it does not already exist. -if [ -f /data/db ]; then +if [ -f "${DB_PATH}" ]; then echo "Database already exists, skipping restore" else echo "No database found, restoring from replica if exists" - litestream restore -if-replica-exists -o /data/db "${REPLICA_URL}" + litestream restore -if-replica-exists -o "${DB_PATH}" "${REPLICA_URL}" fi # Run litestream with your app as the subprocess. From c433d58e60415ff7c0b07442eafabc01b2798ad5 Mon Sep 17 00:00:00 2001 From: Gustavo Valverde Date: Wed, 10 Jul 2024 19:05:23 +0100 Subject: [PATCH 12/33] imp: use latest uptime kuma for better DB management --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index e33d33c..8420b52 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,7 +1,7 @@ # syntax=docker/dockerfile:1 # ===================== Create base stage ===================== ARG NODE_VERSION=lts -ARG UPTIME_KUMA_VERSION=1.23.13 +ARG UPTIME_KUMA_VERSION=nightly2 ARG APP_HOME=/app FROM louislam/uptime-kuma:${UPTIME_KUMA_VERSION} AS base From 1c13af44c4a239ee52010e94dcf71c2d9948340e Mon Sep 17 00:00:00 2001 From: Gustavo Valverde Date: Thu, 11 Jul 2024 10:34:04 +0100 Subject: [PATCH 13/33] fix(build): do not change inherited service folders/services --- docker/Dockerfile | 6 +----- scripts/run.sh | 2 +- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 8420b52..1f53203 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -15,17 +15,13 @@ WORKDIR ${APP_HOME} # ==== App specific variables ENV UPTIME_KUMA_IS_CONTAINER=1 -ARG DATA_DIR=/data/ +ARG DATA_DIR=./data/ ENV DATA_DIR=${DATA_DIR} ENV DB_PATH=${DATA_DIR}kuma.db # ===================== App Runner Stage ===================== FROM base AS runner -RUN apt-get update --no-install-recommends \ - && apt-get install -y ca-certificates iputils-ping wget \ - && rm -rf /var/lib/apt/lists/* - # Copy all necessary files COPY --from=litestream/litestream:0.3.13 /usr/local/bin/litestream /usr/local/bin/litestream diff --git a/scripts/run.sh b/scripts/run.sh index 61476b1..6460f72 100755 --- a/scripts/run.sh +++ b/scripts/run.sh @@ -2,7 +2,7 @@ set -e # Restore the database if it does not already exist. -if [ -f "${DB_PATH}" ]; then +if [[ -f "${DB_PATH}" ]]; then echo "Database already exists, skipping restore" else echo "No database found, restoring from replica if exists" From ccf4bbb81f2fe606ab7599ec4ce72f7cd8b5403f Mon Sep 17 00:00:00 2001 From: Gustavo Valverde Date: Thu, 11 Jul 2024 12:34:09 +0100 Subject: [PATCH 14/33] ref(deploy): allow an external `mariadb` database --- .github/workflows/sub-cloudrun-deploy.yml | 28 ++++++++++++++--------- docker/Dockerfile | 4 +++- scripts/run.sh | 10 +++++--- 3 files changed, 27 insertions(+), 15 deletions(-) diff --git a/.github/workflows/sub-cloudrun-deploy.yml b/.github/workflows/sub-cloudrun-deploy.yml index aebddc5..d102c62 100644 --- a/.github/workflows/sub-cloudrun-deploy.yml +++ b/.github/workflows/sub-cloudrun-deploy.yml @@ -12,35 +12,35 @@ on: image_digest: required: true type: string - description: 'The image digest to deploy' + description: The image digest to deploy project_id: required: true type: string - description: 'The project to deploy to' + description: The project to deploy to region: required: true type: string - description: 'The region to deploy to' + description: The region to deploy to environment: required: false type: string - description: 'The environment to deploy to' + description: The environment to deploy to min_instances: required: false type: string - description: 'The minimum number of instances to deploy' + description: The minimum number of instances to deploy max_instances: required: false type: string - description: 'The maximum number of instances to deploy' + description: The maximum number of instances to deploy cpu: required: false type: string - description: 'The number of CPUs to use for the service' + description: The number of CPUs to use for the service memory: required: false type: string - description: 'The amount of memory to use for the service' + description: The amount of memory to use for the service jobs: versioning: @@ -69,8 +69,8 @@ jobs: name: ${{ inputs.environment }} url: ${{ steps.deploy.outputs.url }} permissions: - contents: 'read' - id-token: 'write' + contents: read + id-token: write steps: - name: Inject slug/short variables uses: rlespinasse/github-slug-action@v4.4.1 @@ -95,6 +95,12 @@ jobs: gcloud_component: alpha env_vars: | REPLICA_URL=${{ vars.REPLICA_URL }} + UPTIME_KUMA_DB_TYPE=${{ vars.UPTIME_KUMA_DB_TYPE }} + UPTIME_KUMA_DB_HOSTNAME=${{ vars.UPTIME_KUMA_DB_HOSTNAME }} + UPTIME_KUMA_DB_PORT=${{ vars.UPTIME_KUMA_DB_PORT }} + UPTIME_KUMA_DB_NAME=${{ vars.UPTIME_KUMA_DB_NAME }} + UPTIME_KUMA_DB_USERNAME=${{ vars.UPTIME_KUMA_DB_USERNAME }} + UPTIME_KUMA_DB_PASSWORD=${{ secrets.UPTIME_KUMA_DB_PASSWORD }} env_vars_update_strategy: overwrite # secrets: | flags: | @@ -104,7 +110,7 @@ jobs: --memory=${{ inputs.memory }} --service-account=${{ vars.GCP_BUCKET_SA }} --add-volume=name=files,type=in-memory - --add-volume-mount=volume=files,mount-path=/data + --add-volume-mount=volume=files,mount-path=/app/data --network=projects/zfnd-dev-net-spoke-0/global/networks/dev-spoke-0 --subnet=projects/zfnd-dev-net-spoke-0/regions/us-east1/subnetworks/dev-default-ue1 diff --git a/docker/Dockerfile b/docker/Dockerfile index 1f53203..758a6df 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -26,7 +26,7 @@ FROM base AS runner COPY --from=litestream/litestream:0.3.13 /usr/local/bin/litestream /usr/local/bin/litestream # Create data directory (although this will likely be mounted too) as some services won't mount it. -RUN mkdir -p ${DATA_DIR} +RUN mkdir -p "${DATA_DIR}" EXPOSE ${PORT} @@ -38,4 +38,6 @@ HEALTHCHECK --interval=60s --timeout=30s --start-period=180s --retries=5 CMD ext COPY etc/litestream.yml /etc/litestream.yml COPY scripts/run.sh /scripts/run.sh +USER node + CMD [ "/scripts/run.sh" ] diff --git a/scripts/run.sh b/scripts/run.sh index 6460f72..547e9d7 100755 --- a/scripts/run.sh +++ b/scripts/run.sh @@ -2,12 +2,16 @@ set -e # Restore the database if it does not already exist. -if [[ -f "${DB_PATH}" ]]; then +if [[ -f "${DB_PATH}" || "${UPTIME_KUMA_DB_TYPE}" != 'mariadb' ]]; then echo "Database already exists, skipping restore" else echo "No database found, restoring from replica if exists" litestream restore -if-replica-exists -o "${DB_PATH}" "${REPLICA_URL}" fi -# Run litestream with your app as the subprocess. -exec litestream replicate -exec "node server/server.js" +if [[ "${UPTIME_KUMA_DB_TYPE}" != 'mariadb' ]]; then + exec node server/server.js +else + # Run litestream with your app as the subprocess. + exec litestream replicate -exec "node server/server.js" +fi From 20a9f8354522a669908f2b3d56aab13e51697133 Mon Sep 17 00:00:00 2001 From: Gustavo Valverde Date: Thu, 11 Jul 2024 14:51:02 +0100 Subject: [PATCH 15/33] fix(db): allow bigger dns results --- docker/Dockerfile | 1 + scripts/db/2024-07-11-0000-dns-results.js | 11 +++++++++++ 2 files changed, 12 insertions(+) create mode 100644 scripts/db/2024-07-11-0000-dns-results.js diff --git a/docker/Dockerfile b/docker/Dockerfile index 758a6df..762f3ea 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -37,6 +37,7 @@ HEALTHCHECK --interval=60s --timeout=30s --start-period=180s --retries=5 CMD ext # Copy Litestream configuration file & startup script. COPY etc/litestream.yml /etc/litestream.yml COPY scripts/run.sh /scripts/run.sh +COPY scripts/db/2024-07-11-0000-dns-results.js ./db/2024-07-11-0000-dns-results.js USER node diff --git a/scripts/db/2024-07-11-0000-dns-results.js b/scripts/db/2024-07-11-0000-dns-results.js new file mode 100644 index 0000000..fa3fbf7 --- /dev/null +++ b/scripts/db/2024-07-11-0000-dns-results.js @@ -0,0 +1,11 @@ +exports.up = function(knex) { + return knex.schema.table('monitor', function(table) { + table.string('dns_last_result', 2000).alter(); + }); + }; + + exports.down = function(knex) { + return knex.schema.table('monitor', function(table) { + table.string('dns_last_result', 255).alter(); + }); + }; From 4569b34245d9da24ab707ccc5106be06b1e141b7 Mon Sep 17 00:00:00 2001 From: Gustavo Valverde Date: Thu, 11 Jul 2024 14:51:13 +0100 Subject: [PATCH 16/33] fix(startup): use correct logic --- scripts/run.sh | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/scripts/run.sh b/scripts/run.sh index 547e9d7..c36aa88 100755 --- a/scripts/run.sh +++ b/scripts/run.sh @@ -1,17 +1,16 @@ #!/bin/bash set -e -# Restore the database if it does not already exist. -if [[ -f "${DB_PATH}" || "${UPTIME_KUMA_DB_TYPE}" != 'mariadb' ]]; then - echo "Database already exists, skipping restore" -else - echo "No database found, restoring from replica if exists" - litestream restore -if-replica-exists -o "${DB_PATH}" "${REPLICA_URL}" -fi - if [[ "${UPTIME_KUMA_DB_TYPE}" != 'mariadb' ]]; then exec node server/server.js else + # Restore the database if it does not already exist. + if [[ -f "${DB_PATH}" ]]; then + echo "Database already exists, skipping restore" + else + echo "No database found, restoring from replica if exists" + litestream restore -if-replica-exists -o "${DB_PATH}" "${REPLICA_URL}" + fi # Run litestream with your app as the subprocess. exec litestream replicate -exec "node server/server.js" fi From 6afbc5e83ecd63f5face72b43563a9d711f6b14b Mon Sep 17 00:00:00 2001 From: Gustavo Valverde Date: Thu, 11 Jul 2024 16:38:57 +0100 Subject: [PATCH 17/33] chore: linting --- scripts/db/2024-07-11-0000-dns-results.js | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/scripts/db/2024-07-11-0000-dns-results.js b/scripts/db/2024-07-11-0000-dns-results.js index fa3fbf7..d07a1e8 100644 --- a/scripts/db/2024-07-11-0000-dns-results.js +++ b/scripts/db/2024-07-11-0000-dns-results.js @@ -1,11 +1,11 @@ -exports.up = function(knex) { - return knex.schema.table('monitor', function(table) { - table.string('dns_last_result', 2000).alter(); - }); - }; +exports.up = function (knex) { + return knex.schema.table('monitor', function (table) { + table.string('dns_last_result', 2000).alter(); + }); +}; - exports.down = function(knex) { - return knex.schema.table('monitor', function(table) { - table.string('dns_last_result', 255).alter(); - }); - }; +exports.down = function (knex) { + return knex.schema.table('monitor', function (table) { + table.string('dns_last_result', 255).alter(); + }); +}; From 83d8756b242d89082b19c953dade881251fa0de1 Mon Sep 17 00:00:00 2001 From: Gustavo Valverde Date: Thu, 11 Jul 2024 16:45:37 +0100 Subject: [PATCH 18/33] fix(startup): wrong condition --- scripts/run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/run.sh b/scripts/run.sh index c36aa88..f4c05cf 100755 --- a/scripts/run.sh +++ b/scripts/run.sh @@ -1,7 +1,7 @@ #!/bin/bash set -e -if [[ "${UPTIME_KUMA_DB_TYPE}" != 'mariadb' ]]; then +if [[ "${UPTIME_KUMA_DB_TYPE}" == 'mariadb' ]]; then exec node server/server.js else # Restore the database if it does not already exist. From d36f65449b63d8471af8d819b76c67c1da0a00fd Mon Sep 17 00:00:00 2001 From: Gustavo Valverde Date: Thu, 11 Jul 2024 17:13:25 +0100 Subject: [PATCH 19/33] fix(db): patch `knex_init_db.js` file --- docker/Dockerfile | 1 + scripts/db/knex_init_db.js | 654 +++++++++++++++++++++++++++++++++++++ 2 files changed, 655 insertions(+) create mode 100644 scripts/db/knex_init_db.js diff --git a/docker/Dockerfile b/docker/Dockerfile index 762f3ea..74c2f1e 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -38,6 +38,7 @@ HEALTHCHECK --interval=60s --timeout=30s --start-period=180s --retries=5 CMD ext COPY etc/litestream.yml /etc/litestream.yml COPY scripts/run.sh /scripts/run.sh COPY scripts/db/2024-07-11-0000-dns-results.js ./db/2024-07-11-0000-dns-results.js +COPY scripts/db/knex_init_db.js ./db/knex_init_db.js USER node diff --git a/scripts/db/knex_init_db.js b/scripts/db/knex_init_db.js new file mode 100644 index 0000000..d1b0d24 --- /dev/null +++ b/scripts/db/knex_init_db.js @@ -0,0 +1,654 @@ +const { R } = require('redbean-node'); +const { log } = require('../src/util'); + +/** + * ⚠️⚠️⚠️⚠️⚠️⚠️ DO NOT ADD ANYTHING HERE! + * IF YOU NEED TO ADD FIELDS, ADD IT TO ./db/knex_migrations + * See ./db/knex_migrations/README.md for more information + * @returns {Promise} + */ +async function createTables() { + log.info('mariadb', 'Creating basic tables for MariaDB'); + const knex = R.knex; + + // TODO: Should check later if it is really the final patch sql file. + + // docker_host + await knex.schema.createTable('docker_host', (table) => { + table.increments('id'); + table.integer('user_id').unsigned().notNullable(); + table.string('docker_daemon', 255); + table.string('docker_type', 255); + table.string('name', 255); + }); + + // group + await knex.schema.createTable('group', (table) => { + table.increments('id'); + table.string('name', 255).notNullable(); + table.datetime('created_date').notNullable().defaultTo(knex.fn.now()); + table.boolean('public').notNullable().defaultTo(false); + table.boolean('active').notNullable().defaultTo(true); + table.integer('weight').notNullable().defaultTo(1000); + table.integer('status_page_id').unsigned(); + }); + + // proxy + await knex.schema.createTable('proxy', (table) => { + table.increments('id'); + table.integer('user_id').unsigned().notNullable(); + table.string('protocol', 10).notNullable(); + table.string('host', 255).notNullable(); + table.smallint('port').notNullable(); // TODO: Maybe a issue with MariaDB, need migration to int + table.boolean('auth').notNullable(); + table.string('username', 255).nullable(); + table.string('password', 255).nullable(); + table.boolean('active').notNullable().defaultTo(true); + table.boolean('default').notNullable().defaultTo(false); + table.datetime('created_date').notNullable().defaultTo(knex.fn.now()); + + table.index('user_id', 'proxy_user_id'); + }); + + // user + await knex.schema.createTable('user', (table) => { + table.increments('id'); + table + .string('username', 255) + .notNullable() + .unique() + .collate('utf8_general_ci'); + table.string('password', 255); + table.boolean('active').notNullable().defaultTo(true); + table.string('timezone', 150); + table.string('twofa_secret', 64); + table.boolean('twofa_status').notNullable().defaultTo(false); + table.string('twofa_last_token', 6); + }); + + // monitor + await knex.schema.createTable('monitor', (table) => { + table.increments('id'); + table.string('name', 150); + table.boolean('active').notNullable().defaultTo(true); + table + .integer('user_id') + .unsigned() + .references('id') + .inTable('user') + .onDelete('SET NULL') + .onUpdate('CASCADE'); + table.integer('interval').notNullable().defaultTo(20); + table.text('url'); + table.string('type', 20); + table.integer('weight').defaultTo(2000); + table.string('hostname', 255); + table.integer('port'); + table.datetime('created_date').notNullable().defaultTo(knex.fn.now()); + table.string('keyword', 255); + table.integer('maxretries').notNullable().defaultTo(0); + table.boolean('ignore_tls').notNullable().defaultTo(false); + table.boolean('upside_down').notNullable().defaultTo(false); + table.integer('maxredirects').notNullable().defaultTo(10); + table + .text('accepted_statuscodes_json') + .notNullable() + .defaultTo('["200-299"]'); + table.string('dns_resolve_type', 5); + table.string('dns_resolve_server', 255); + table.string('dns_last_result', 2000); + table.integer('retry_interval').notNullable().defaultTo(0); + table.string('push_token', 20).defaultTo(null); + table.text('method').notNullable().defaultTo('GET'); + table.text('body').defaultTo(null); + table.text('headers').defaultTo(null); + table.text('basic_auth_user').defaultTo(null); + table.text('basic_auth_pass').defaultTo(null); + table + .integer('docker_host') + .unsigned() + .references('id') + .inTable('docker_host'); + table.string('docker_container', 255); + table.integer('proxy_id').unsigned().references('id').inTable('proxy'); + table.boolean('expiry_notification').defaultTo(true); + table.text('mqtt_topic'); + table.string('mqtt_success_message', 255); + table.string('mqtt_username', 255); + table.string('mqtt_password', 255); + table.string('database_connection_string', 2000); + table.text('database_query'); + table.string('auth_method', 250); + table.text('auth_domain'); + table.text('auth_workstation'); + table.string('grpc_url', 255).defaultTo(null); + table.text('grpc_protobuf').defaultTo(null); + table.text('grpc_body').defaultTo(null); + table.text('grpc_metadata').defaultTo(null); + table.text('grpc_method').defaultTo(null); + table.text('grpc_service_name').defaultTo(null); + table.boolean('grpc_enable_tls').notNullable().defaultTo(false); + table.string('radius_username', 255); + table.string('radius_password', 255); + table.string('radius_calling_station_id', 50); + table.string('radius_called_station_id', 50); + table.string('radius_secret', 255); + table.integer('resend_interval').notNullable().defaultTo(0); + table.integer('packet_size').notNullable().defaultTo(56); + table.string('game', 255); + }); + + // heartbeat + await knex.schema.createTable('heartbeat', (table) => { + table.increments('id'); + table.boolean('important').notNullable().defaultTo(false); + table + .integer('monitor_id') + .unsigned() + .notNullable() + .references('id') + .inTable('monitor') + .onDelete('CASCADE') + .onUpdate('CASCADE'); + table.smallint('status').notNullable(); + + table.text('msg'); + table.datetime('time').notNullable(); + table.integer('ping'); + table.integer('duration').notNullable().defaultTo(0); + table.integer('down_count').notNullable().defaultTo(0); + + table.index('important'); + table.index(['monitor_id', 'time'], 'monitor_time_index'); + table.index('monitor_id'); + table.index( + ['monitor_id', 'important', 'time'], + 'monitor_important_time_index' + ); + }); + + // incident + await knex.schema.createTable('incident', (table) => { + table.increments('id'); + table.string('title', 255).notNullable(); + table.text('content', 255).notNullable(); + table.string('style', 30).notNullable().defaultTo('warning'); + table.datetime('created_date').notNullable().defaultTo(knex.fn.now()); + table.datetime('last_updated_date'); + table.boolean('pin').notNullable().defaultTo(true); + table.boolean('active').notNullable().defaultTo(true); + table.integer('status_page_id').unsigned(); + }); + + // maintenance + await knex.schema.createTable('maintenance', (table) => { + table.increments('id'); + table.string('title', 150).notNullable(); + table.text('description').notNullable(); + table + .integer('user_id') + .unsigned() + .references('id') + .inTable('user') + .onDelete('SET NULL') + .onUpdate('CASCADE'); + table.boolean('active').notNullable().defaultTo(true); + table.string('strategy', 50).notNullable().defaultTo('single'); + table.datetime('start_date'); + table.datetime('end_date'); + table.time('start_time'); + table.time('end_time'); + table.string('weekdays', 250).defaultTo('[]'); + table.text('days_of_month').defaultTo('[]'); + table.integer('interval_day'); + + table.index('active'); + table.index(['strategy', 'active'], 'manual_active'); + table.index('user_id', 'maintenance_user_id'); + }); + + // status_page + await knex.schema.createTable('status_page', (table) => { + table.increments('id'); + table.string('slug', 255).notNullable().unique().collate('utf8_general_ci'); + table.string('title', 255).notNullable(); + table.text('description'); + table.string('icon', 255).notNullable(); + table.string('theme', 30).notNullable(); + table.boolean('published').notNullable().defaultTo(true); + table.boolean('search_engine_index').notNullable().defaultTo(true); + table.boolean('show_tags').notNullable().defaultTo(false); + table.string('password'); + table.datetime('created_date').notNullable().defaultTo(knex.fn.now()); + table.datetime('modified_date').notNullable().defaultTo(knex.fn.now()); + table.text('footer_text'); + table.text('custom_css'); + table.boolean('show_powered_by').notNullable().defaultTo(true); + table.string('google_analytics_tag_id'); + }); + + // maintenance_status_page + await knex.schema.createTable('maintenance_status_page', (table) => { + table.increments('id'); + + table + .integer('status_page_id') + .unsigned() + .notNullable() + .references('id') + .inTable('status_page') + .onDelete('CASCADE') + .onUpdate('CASCADE'); + + table + .integer('maintenance_id') + .unsigned() + .notNullable() + .references('id') + .inTable('maintenance') + .onDelete('CASCADE') + .onUpdate('CASCADE'); + }); + + // maintenance_timeslot + await knex.schema.createTable('maintenance_timeslot', (table) => { + table.increments('id'); + table + .integer('maintenance_id') + .unsigned() + .notNullable() + .references('id') + .inTable('maintenance') + .onDelete('CASCADE') + .onUpdate('CASCADE'); + table.datetime('start_date').notNullable(); + table.datetime('end_date'); + table.boolean('generated_next').defaultTo(false); + + table.index('maintenance_id'); + table.index( + ['maintenance_id', 'start_date', 'end_date'], + 'active_timeslot_index' + ); + table.index('generated_next', 'generated_next_index'); + }); + + // monitor_group + await knex.schema.createTable('monitor_group', (table) => { + table.increments('id'); + table + .integer('monitor_id') + .unsigned() + .notNullable() + .references('id') + .inTable('monitor') + .onDelete('CASCADE') + .onUpdate('CASCADE'); + table + .integer('group_id') + .unsigned() + .notNullable() + .references('id') + .inTable('group') + .onDelete('CASCADE') + .onUpdate('CASCADE'); + table.integer('weight').notNullable().defaultTo(1000); + table.boolean('send_url').notNullable().defaultTo(false); + + table.index(['monitor_id', 'group_id'], 'fk'); + }); + // monitor_maintenance + await knex.schema.createTable('monitor_maintenance', (table) => { + table.increments('id'); + table + .integer('monitor_id') + .unsigned() + .notNullable() + .references('id') + .inTable('monitor') + .onDelete('CASCADE') + .onUpdate('CASCADE'); + table + .integer('maintenance_id') + .unsigned() + .notNullable() + .references('id') + .inTable('maintenance') + .onDelete('CASCADE') + .onUpdate('CASCADE'); + + table.index('maintenance_id', 'maintenance_id_index2'); + table.index('monitor_id', 'monitor_id_index'); + }); + + // notification + await knex.schema.createTable('notification', (table) => { + table.increments('id'); + table.string('name', 255); + table.boolean('active').notNullable().defaultTo(true); + table.integer('user_id').unsigned(); + table.boolean('is_default').notNullable().defaultTo(false); + table.text('config', 'longtext'); + }); + + // monitor_notification + await knex.schema.createTable('monitor_notification', (table) => { + table.increments('id').unsigned(); // TODO: no auto increment???? + table + .integer('monitor_id') + .unsigned() + .notNullable() + .references('id') + .inTable('monitor') + .onDelete('CASCADE') + .onUpdate('CASCADE'); + table + .integer('notification_id') + .unsigned() + .notNullable() + .references('id') + .inTable('notification') + .onDelete('CASCADE') + .onUpdate('CASCADE'); + + table.index( + ['monitor_id', 'notification_id'], + 'monitor_notification_index' + ); + }); + + // tag + await knex.schema.createTable('tag', (table) => { + table.increments('id'); + table.string('name', 255).notNullable(); + table.string('color', 255).notNullable(); + table.datetime('created_date').notNullable().defaultTo(knex.fn.now()); + }); + + // monitor_tag + await knex.schema.createTable('monitor_tag', (table) => { + table.increments('id'); + table + .integer('monitor_id') + .unsigned() + .notNullable() + .references('id') + .inTable('monitor') + .onDelete('CASCADE') + .onUpdate('CASCADE'); + table + .integer('tag_id') + .unsigned() + .notNullable() + .references('id') + .inTable('tag') + .onDelete('CASCADE') + .onUpdate('CASCADE'); + table.text('value'); + }); + + // monitor_tls_info + await knex.schema.createTable('monitor_tls_info', (table) => { + table.increments('id'); + table + .integer('monitor_id') + .unsigned() + .notNullable() + .references('id') + .inTable('monitor') + .onDelete('CASCADE') + .onUpdate('CASCADE'); + table.text('info_json'); + }); + + // notification_sent_history + await knex.schema.createTable('notification_sent_history', (table) => { + table.increments('id'); + table.string('type', 50).notNullable(); + table.integer('monitor_id').unsigned().notNullable(); + table.integer('days').notNullable(); + table.unique(['type', 'monitor_id', 'days']); + table.index(['type', 'monitor_id', 'days'], 'good_index'); + }); + + // setting + await knex.schema.createTable('setting', (table) => { + table.increments('id'); + table.string('key', 200).notNullable().unique().collate('utf8_general_ci'); + table.text('value'); + table.string('type', 20); + }); + + // status_page_cname + await knex.schema.createTable('status_page_cname', (table) => { + table.increments('id'); + table + .integer('status_page_id') + .unsigned() + .references('id') + .inTable('status_page') + .onDelete('CASCADE') + .onUpdate('CASCADE'); + table.string('domain').notNullable().unique().collate('utf8_general_ci'); + }); + + /********************* + * Converted Patch here + *********************/ + + // 2023-06-30-1348-http-body-encoding.js + // ALTER TABLE monitor ADD http_body_encoding VARCHAR(25); + // UPDATE monitor SET http_body_encoding = 'json' WHERE (type = 'http' or type = 'keyword') AND http_body_encoding IS NULL; + await knex.schema.table('monitor', function (table) { + table.string('http_body_encoding', 25); + }); + + await knex('monitor') + .where(function () { + this.where('type', 'http').orWhere('type', 'keyword'); + }) + .whereNull('http_body_encoding') + .update({ + http_body_encoding: 'json', + }); + + // 2023-06-30-1354-add-description-monitor.js + // ALTER TABLE monitor ADD description TEXT default null; + await knex.schema.table('monitor', function (table) { + table.text('description').defaultTo(null); + }); + + // 2023-06-30-1357-api-key-table.js + /* + CREATE TABLE [api_key] ( + [id] INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, + [key] VARCHAR(255) NOT NULL, + [name] VARCHAR(255) NOT NULL, + [user_id] INTEGER NOT NULL, + [created_date] DATETIME DEFAULT (DATETIME('now')) NOT NULL, + [active] BOOLEAN DEFAULT 1 NOT NULL, + [expires] DATETIME DEFAULT NULL, + CONSTRAINT FK_user FOREIGN KEY ([user_id]) REFERENCES [user]([id]) ON DELETE CASCADE ON UPDATE CASCADE + ); + */ + await knex.schema.createTable('api_key', function (table) { + table.increments('id').primary(); + table.string('key', 255).notNullable(); + table.string('name', 255).notNullable(); + table + .integer('user_id') + .unsigned() + .notNullable() + .references('id') + .inTable('user') + .onDelete('CASCADE') + .onUpdate('CASCADE'); + table.dateTime('created_date').defaultTo(knex.fn.now()).notNullable(); + table.boolean('active').defaultTo(1).notNullable(); + table.dateTime('expires').defaultTo(null); + }); + + // 2023-06-30-1400-monitor-tls.js + /* + ALTER TABLE monitor + ADD tls_ca TEXT default null; + + ALTER TABLE monitor + ADD tls_cert TEXT default null; + + ALTER TABLE monitor + ADD tls_key TEXT default null; + */ + await knex.schema.table('monitor', function (table) { + table.text('tls_ca').defaultTo(null); + table.text('tls_cert').defaultTo(null); + table.text('tls_key').defaultTo(null); + }); + + // 2023-06-30-1401-maintenance-cron.js + /* + -- 999 characters. https://stackoverflow.com/questions/46134830/maximum-length-for-cron-job + DROP TABLE maintenance_timeslot; + ALTER TABLE maintenance ADD cron TEXT; + ALTER TABLE maintenance ADD timezone VARCHAR(255); + ALTER TABLE maintenance ADD duration INTEGER; + */ + await knex.schema + .dropTableIfExists('maintenance_timeslot') + .table('maintenance', function (table) { + table.text('cron'); + table.string('timezone', 255); + table.integer('duration'); + }); + + // 2023-06-30-1413-add-parent-monitor.js. + /* + ALTER TABLE monitor + ADD parent INTEGER REFERENCES [monitor] ([id]) ON DELETE SET NULL ON UPDATE CASCADE; + */ + await knex.schema.table('monitor', function (table) { + table + .integer('parent') + .unsigned() + .references('id') + .inTable('monitor') + .onDelete('SET NULL') + .onUpdate('CASCADE'); + }); + + /* + patch-add-invert-keyword.sql + ALTER TABLE monitor + ADD invert_keyword BOOLEAN default 0 not null; + */ + await knex.schema.table('monitor', function (table) { + table.boolean('invert_keyword').defaultTo(0).notNullable(); + }); + + /* + patch-added-json-query.sql + ALTER TABLE monitor + ADD json_path TEXT; + + ALTER TABLE monitor + ADD expected_value VARCHAR(255); + */ + await knex.schema.table('monitor', function (table) { + table.text('json_path'); + table.string('expected_value', 255); + }); + + /* + patch-added-kafka-producer.sql + + ALTER TABLE monitor + ADD kafka_producer_topic VARCHAR(255); + +ALTER TABLE monitor + ADD kafka_producer_brokers TEXT; + +ALTER TABLE monitor + ADD kafka_producer_ssl INTEGER; + +ALTER TABLE monitor + ADD kafka_producer_allow_auto_topic_creation VARCHAR(255); + +ALTER TABLE monitor + ADD kafka_producer_sasl_options TEXT; + +ALTER TABLE monitor + ADD kafka_producer_message TEXT; + */ + await knex.schema.table('monitor', function (table) { + table.string('kafka_producer_topic', 255); + table.text('kafka_producer_brokers'); + + // patch-fix-kafka-producer-booleans.sql + table.boolean('kafka_producer_ssl').defaultTo(0).notNullable(); + table + .boolean('kafka_producer_allow_auto_topic_creation') + .defaultTo(0) + .notNullable(); + + table.text('kafka_producer_sasl_options'); + table.text('kafka_producer_message'); + }); + + /* + patch-add-certificate-expiry-status-page.sql + ALTER TABLE status_page + ADD show_certificate_expiry BOOLEAN default 0 NOT NULL; + */ + await knex.schema.table('status_page', function (table) { + table.boolean('show_certificate_expiry').defaultTo(0).notNullable(); + }); + + /* + patch-monitor-oauth-cc.sql + ALTER TABLE monitor + ADD oauth_client_id TEXT default null; + +ALTER TABLE monitor + ADD oauth_client_secret TEXT default null; + +ALTER TABLE monitor + ADD oauth_token_url TEXT default null; + +ALTER TABLE monitor + ADD oauth_scopes TEXT default null; + +ALTER TABLE monitor + ADD oauth_auth_method TEXT default null; + */ + await knex.schema.table('monitor', function (table) { + table.text('oauth_client_id').defaultTo(null); + table.text('oauth_client_secret').defaultTo(null); + table.text('oauth_token_url').defaultTo(null); + table.text('oauth_scopes').defaultTo(null); + table.text('oauth_auth_method').defaultTo(null); + }); + + /* + patch-add-timeout-monitor.sql + ALTER TABLE monitor + ADD timeout DOUBLE default 0 not null; + */ + await knex.schema.table('monitor', function (table) { + table.double('timeout').defaultTo(0).notNullable(); + }); + + /* + patch-add-gamedig-given-port.sql + ALTER TABLE monitor + ADD gamedig_given_port_only BOOLEAN default 1 not null; + */ + await knex.schema.table('monitor', function (table) { + table.boolean('gamedig_given_port_only').defaultTo(1).notNullable(); + }); + + log.info('mariadb', 'Created basic tables for MariaDB'); +} + +module.exports = { + createTables, +}; From 15e949710750b33e0bbd3013f1ffe2196be865ee Mon Sep 17 00:00:00 2001 From: Gustavo Valverde Date: Sun, 14 Jul 2024 19:53:12 +0100 Subject: [PATCH 20/33] fix(runtime): avoid spawning zombie processes --- docker/Dockerfile | 1 + scripts/run.sh | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 74c2f1e..009cdef 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -42,4 +42,5 @@ COPY scripts/db/knex_init_db.js ./db/knex_init_db.js USER node +ENTRYPOINT ["/usr/bin/dumb-init", "--"] CMD [ "/scripts/run.sh" ] diff --git a/scripts/run.sh b/scripts/run.sh index f4c05cf..5569d3b 100755 --- a/scripts/run.sh +++ b/scripts/run.sh @@ -2,7 +2,7 @@ set -e if [[ "${UPTIME_KUMA_DB_TYPE}" == 'mariadb' ]]; then - exec node server/server.js + node server/server.js else # Restore the database if it does not already exist. if [[ -f "${DB_PATH}" ]]; then From 700fbc1a0a6f4c3c8e70819b30128e65dd069a3f Mon Sep 17 00:00:00 2001 From: Gustavo Valverde Date: Sun, 14 Jul 2024 19:53:29 +0100 Subject: [PATCH 21/33] chore: do not commit `trunk` linting confs --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 52e02a0..dc58286 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,4 @@ -.trunk/ +.trunk # Created by https://www.toptal.com/developers/gitignore/api/windows,linux,macos,visualstudiocode,jetbrains,node,nextjs,vercel,amplify # Edit at https://www.toptal.com/developers/gitignore?templates=windows,linux,macos,visualstudiocode,jetbrains,node,nextjs,vercel,amplify From c8c890835273e4bdc24e91ebad6a6a7360a3bbad Mon Sep 17 00:00:00 2001 From: Gustavo Valverde Date: Sun, 14 Jul 2024 19:58:59 +0100 Subject: [PATCH 22/33] fix(ci): allow instance deletion on merge --- .github/workflows/chore-clean-dev.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/chore-clean-dev.yml b/.github/workflows/chore-clean-dev.yml index d476d14..edad9eb 100644 --- a/.github/workflows/chore-clean-dev.yml +++ b/.github/workflows/chore-clean-dev.yml @@ -11,6 +11,9 @@ on: jobs: delete: runs-on: ubuntu-latest + permissions: + contents: 'read' + id-token: 'write' steps: - name: Inject slug/short variables uses: rlespinasse/github-slug-action@v4.4.1 From 1f2a4089422b5a26b09f7d69034110dd4115bed5 Mon Sep 17 00:00:00 2001 From: Gustavo Valverde Date: Sun, 14 Jul 2024 20:15:27 +0100 Subject: [PATCH 23/33] chore: update all github actions --- .github/workflows/chore-clean-dev.yml | 6 +++--- .github/workflows/ci-lint-codebase.yml | 7 +++---- .github/workflows/sub-build-docker-image.yml | 14 +++++++------- .github/workflows/sub-cloudrun-deploy.yml | 11 ++--------- 4 files changed, 15 insertions(+), 23 deletions(-) diff --git a/.github/workflows/chore-clean-dev.yml b/.github/workflows/chore-clean-dev.yml index edad9eb..4f812c9 100644 --- a/.github/workflows/chore-clean-dev.yml +++ b/.github/workflows/chore-clean-dev.yml @@ -16,17 +16,17 @@ jobs: id-token: 'write' steps: - name: Inject slug/short variables - uses: rlespinasse/github-slug-action@v4.4.1 + uses: rlespinasse/github-slug-action@v4.5.0 - name: Authenticate to Google Cloud id: auth - uses: google-github-actions/auth@v2.1.2 + uses: google-github-actions/auth@v2.1.3 with: workload_identity_provider: '${{ vars.GCP_WIF }}' service_account: '${{ vars.GCP_DEPLOYMENTS_SA }}' - name: Set up Cloud SDK - uses: google-github-actions/setup-gcloud@v1.1.1 + uses: google-github-actions/setup-gcloud@v2.1.0 - name: Removing CR service run: | diff --git a/.github/workflows/ci-lint-codebase.yml b/.github/workflows/ci-lint-codebase.yml index 0e057d0..08b66f9 100644 --- a/.github/workflows/ci-lint-codebase.yml +++ b/.github/workflows/ci-lint-codebase.yml @@ -31,14 +31,14 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout Code Repository - uses: actions/checkout@v4.1.1 + uses: actions/checkout@v4.1.7 with: # Full git history is needed to get a proper # list of changed files within `super-linter` fetch-depth: 0 - name: Lint Code Base - uses: super-linter/super-linter/slim@v5.2.1 + uses: super-linter/super-linter/slim@v6.7.0 env: LOG_LEVEL: ERROR VALIDATE_ALL_CODEBASE: false @@ -47,9 +47,8 @@ jobs: VALIDATE_CSS: false VALIDATE_EDITORCONFIG: false VALIDATE_MARKDOWN: false + VALIDATE_JAVASCRIPT_ES: false VALIDATE_DOCKERFILE_HADOLINT: false LINTER_RULES_PATH: / - JAVASCRIPT_DEFAULT_STYLE: prettier - TYPESCRIPT_DEFAULT_STYLE: prettier DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/sub-build-docker-image.yml b/.github/workflows/sub-build-docker-image.yml index 6b4bacb..fea09c3 100644 --- a/.github/workflows/sub-build-docker-image.yml +++ b/.github/workflows/sub-build-docker-image.yml @@ -34,19 +34,19 @@ jobs: contents: 'read' id-token: 'write' steps: - - uses: actions/checkout@v4.1.1 + - uses: actions/checkout@v4.1.7 with: persist-credentials: false - name: Inject slug/short variables - uses: rlespinasse/github-slug-action@v4.4.1 + uses: rlespinasse/github-slug-action@v4.5.0 with: short-length: 7 # Automatic tag management and OCI Image Format Specification for labels - name: Docker meta id: meta - uses: docker/metadata-action@v5.5.0 + uses: docker/metadata-action@v5.5.1 with: # list of Docker images to use as base name for tags images: | @@ -68,11 +68,11 @@ jobs: # Setup Docker Buildx to allow use of docker cache layers from GH - name: Set up Docker Buildx id: buildx - uses: docker/setup-buildx-action@v3.0.0 + uses: docker/setup-buildx-action@v3.4.0 - name: Authenticate to Google Cloud id: auth - uses: google-github-actions/auth@v2.1.2 + uses: google-github-actions/auth@v2.1.3 with: workload_identity_provider: '${{ vars.GCP_WIF }}' service_account: '${{ vars.GCP_ARTIFACTS_SA }}' @@ -83,7 +83,7 @@ jobs: access_token_lifetime: 10800s - name: Login to Google Artifact Registry - uses: docker/login-action@v3.0.0 + uses: docker/login-action@v3.2.0 with: registry: us-docker.pkg.dev username: oauth2accesstoken @@ -92,7 +92,7 @@ jobs: # Build and push image to Google Artifact Registry, and possibly DockerHub - name: Build & push id: docker_build - uses: docker/build-push-action@v5.1.0 + uses: docker/build-push-action@v6.3.0 with: target: ${{ inputs.dockerfile_target }} context: . diff --git a/.github/workflows/sub-cloudrun-deploy.yml b/.github/workflows/sub-cloudrun-deploy.yml index d102c62..4627886 100644 --- a/.github/workflows/sub-cloudrun-deploy.yml +++ b/.github/workflows/sub-cloudrun-deploy.yml @@ -73,11 +73,11 @@ jobs: id-token: write steps: - name: Inject slug/short variables - uses: rlespinasse/github-slug-action@v4.4.1 + uses: rlespinasse/github-slug-action@v4.5.0 - name: Authenticate to Google Cloud id: auth - uses: google-github-actions/auth@v2.1.2 + uses: google-github-actions/auth@v2.1.3 with: workload_identity_provider: '${{ vars.GCP_WIF }}' service_account: '${{ vars.GCP_DEPLOYMENTS_SA }}' @@ -121,10 +121,3 @@ jobs: - name: Test service with cURL run: curl "${{ steps.deploy.outputs.url }}" - - - name: Add reaction - if: ${{ github.event_name == 'issue_comment' }} - uses: peter-evans/create-or-update-comment@v4.0.0 - with: - comment-id: ${{ github.event.comment.id }} - reactions: rocket From 944d9a023ff9f44ba4aad1e9ad00464ddd34f7fb Mon Sep 17 00:00:00 2001 From: Gustavo Valverde Date: Sun, 14 Jul 2024 20:19:39 +0100 Subject: [PATCH 24/33] imp(dependabot): group actions update --- .github/dependabot.yml | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 3356b82..e5e151a 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -1,22 +1,19 @@ version: 2 updates: - - package-ecosystem: npm - directory: / - schedule: - interval: monthly - commit-message: - prefix: "bump(deps): " - - package-ecosystem: docker directory: / schedule: interval: monthly commit-message: - prefix: "bump(docker) " + prefix: "deps(docker) " - package-ecosystem: github-actions - directory: /.github/workflows + directory: / schedule: interval: monthly commit-message: - prefix: "bump(actions) " + prefix: "deps(actions) " + groups: + devops: + patterns: + - "*" From bc29d0d248c6cc70d56afa0f5da7be436370b0ff Mon Sep 17 00:00:00 2001 From: Gustavo Valverde Date: Sun, 14 Jul 2024 20:28:26 +0100 Subject: [PATCH 25/33] fix(actions): permissions --- .github/workflows/cd-deploy-to-dev.yml | 4 ++++ .github/workflows/cd-deploy-to-prod.yml | 2 ++ .github/workflows/cd-deploy-to-test.yml | 2 ++ .github/workflows/chore-clean-dev.yml | 2 ++ .github/workflows/ci-lint-codebase.patch.yml | 2 ++ .github/workflows/ci-lint-codebase.yml | 3 +++ .github/workflows/sub-build-docker-image.yml | 6 ++++-- .github/workflows/sub-cloudrun-deploy.yml | 2 ++ 8 files changed, 21 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cd-deploy-to-dev.yml b/.github/workflows/cd-deploy-to-dev.yml index 8446e15..a61814f 100644 --- a/.github/workflows/cd-deploy-to-dev.yml +++ b/.github/workflows/cd-deploy-to-dev.yml @@ -1,5 +1,7 @@ name: Deploy to dev +permissions: read-all + on: pull_request: types: [opened, synchronize, reopened, labeled] @@ -17,6 +19,8 @@ concurrency: group: ${{ github.workflow }}-${{ github.job }}-${{ github.head_ref || github.run_id }} cancel-in-progress: true +permissions: read-all + jobs: build: uses: ./.github/workflows/sub-build-docker-image.yml diff --git a/.github/workflows/cd-deploy-to-prod.yml b/.github/workflows/cd-deploy-to-prod.yml index ab6c97f..4f550f0 100644 --- a/.github/workflows/cd-deploy-to-prod.yml +++ b/.github/workflows/cd-deploy-to-prod.yml @@ -1,5 +1,7 @@ name: Deploy to prod +permissions: read-all + on: release: types: diff --git a/.github/workflows/cd-deploy-to-test.yml b/.github/workflows/cd-deploy-to-test.yml index 3a793c0..89894b3 100644 --- a/.github/workflows/cd-deploy-to-test.yml +++ b/.github/workflows/cd-deploy-to-test.yml @@ -1,5 +1,7 @@ name: Deploy to test +permissions: read-all + on: push: branches: diff --git a/.github/workflows/chore-clean-dev.yml b/.github/workflows/chore-clean-dev.yml index 4f812c9..bbacb73 100644 --- a/.github/workflows/chore-clean-dev.yml +++ b/.github/workflows/chore-clean-dev.yml @@ -8,6 +8,8 @@ on: types: - closed +permissions: read-all + jobs: delete: runs-on: ubuntu-latest diff --git a/.github/workflows/ci-lint-codebase.patch.yml b/.github/workflows/ci-lint-codebase.patch.yml index 8a6115f..e50c1a1 100644 --- a/.github/workflows/ci-lint-codebase.patch.yml +++ b/.github/workflows/ci-lint-codebase.patch.yml @@ -1,5 +1,7 @@ name: Lint Code Base +permissions: read-all + on: pull_request: branches: [main] diff --git a/.github/workflows/ci-lint-codebase.yml b/.github/workflows/ci-lint-codebase.yml index 08b66f9..b750e21 100644 --- a/.github/workflows/ci-lint-codebase.yml +++ b/.github/workflows/ci-lint-codebase.yml @@ -26,6 +26,8 @@ concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} cancel-in-progress: true +permissions: read-all + jobs: linter: runs-on: ubuntu-latest @@ -48,6 +50,7 @@ jobs: VALIDATE_EDITORCONFIG: false VALIDATE_MARKDOWN: false VALIDATE_JAVASCRIPT_ES: false + VALIDATE_JAVASCRIPT_STANDARD: false VALIDATE_DOCKERFILE_HADOLINT: false LINTER_RULES_PATH: / DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} diff --git a/.github/workflows/sub-build-docker-image.yml b/.github/workflows/sub-build-docker-image.yml index fea09c3..21b6987 100644 --- a/.github/workflows/sub-build-docker-image.yml +++ b/.github/workflows/sub-build-docker-image.yml @@ -23,6 +23,8 @@ on: description: The image digest to be used on a caller workflow value: ${{ jobs.build.outputs.image_digest }} +permissions: read-all + jobs: build: name: Build images @@ -31,8 +33,8 @@ jobs: outputs: image_digest: ${{ steps.docker_build.outputs.digest }} permissions: - contents: 'read' - id-token: 'write' + contents: read + id-token: write steps: - uses: actions/checkout@v4.1.7 with: diff --git a/.github/workflows/sub-cloudrun-deploy.yml b/.github/workflows/sub-cloudrun-deploy.yml index 4627886..996abb5 100644 --- a/.github/workflows/sub-cloudrun-deploy.yml +++ b/.github/workflows/sub-cloudrun-deploy.yml @@ -42,6 +42,8 @@ on: type: string description: The amount of memory to use for the service +permissions: read-all + jobs: versioning: runs-on: ubuntu-latest From c74a9b0d0cc4530884a295dbfcb3bec67891b422 Mon Sep 17 00:00:00 2001 From: Gustavo Valverde Date: Sun, 14 Jul 2024 20:32:04 +0100 Subject: [PATCH 26/33] chore: put permissions above jobs --- .github/workflows/cd-deploy-to-dev.yml | 2 -- .github/workflows/cd-deploy-to-prod.yml | 4 ++-- .github/workflows/cd-deploy-to-test.yml | 4 ++-- .github/workflows/ci-lint-codebase.patch.yml | 4 ++-- 4 files changed, 6 insertions(+), 8 deletions(-) diff --git a/.github/workflows/cd-deploy-to-dev.yml b/.github/workflows/cd-deploy-to-dev.yml index a61814f..3509162 100644 --- a/.github/workflows/cd-deploy-to-dev.yml +++ b/.github/workflows/cd-deploy-to-dev.yml @@ -1,7 +1,5 @@ name: Deploy to dev -permissions: read-all - on: pull_request: types: [opened, synchronize, reopened, labeled] diff --git a/.github/workflows/cd-deploy-to-prod.yml b/.github/workflows/cd-deploy-to-prod.yml index 4f550f0..2b08b95 100644 --- a/.github/workflows/cd-deploy-to-prod.yml +++ b/.github/workflows/cd-deploy-to-prod.yml @@ -1,7 +1,5 @@ name: Deploy to prod -permissions: read-all - on: release: types: @@ -14,6 +12,8 @@ concurrency: group: ${{ github.workflow }}-${{ github.job }}-${{ github.head_ref || github.run_id }} cancel-in-progress: true +permissions: read-all + jobs: build: # needs: [test] diff --git a/.github/workflows/cd-deploy-to-test.yml b/.github/workflows/cd-deploy-to-test.yml index 89894b3..5952d6b 100644 --- a/.github/workflows/cd-deploy-to-test.yml +++ b/.github/workflows/cd-deploy-to-test.yml @@ -1,7 +1,5 @@ name: Deploy to test -permissions: read-all - on: push: branches: @@ -20,6 +18,8 @@ concurrency: group: ${{ github.workflow }}-${{ github.job }}-${{ github.head_ref || github.run_id }} cancel-in-progress: true +permissions: read-all + jobs: build: uses: ./.github/workflows/sub-build-docker-image.yml diff --git a/.github/workflows/ci-lint-codebase.patch.yml b/.github/workflows/ci-lint-codebase.patch.yml index e50c1a1..5aaf35a 100644 --- a/.github/workflows/ci-lint-codebase.patch.yml +++ b/.github/workflows/ci-lint-codebase.patch.yml @@ -1,7 +1,5 @@ name: Lint Code Base -permissions: read-all - on: pull_request: branches: [main] @@ -11,6 +9,8 @@ on: - 'etc/litestream.yml' - .github/workflows/ci-lint-codebase.yml +permissions: read-all + jobs: linter: runs-on: ubuntu-latest From ad23cf85a24ef80ee123fad79d06a77f2bc42483 Mon Sep 17 00:00:00 2001 From: Gustavo Valverde Date: Sun, 14 Jul 2024 21:03:09 +0100 Subject: [PATCH 27/33] fix(actions): permissions --- .github/workflows/cd-deploy-to-dev.yml | 4 +++- .github/workflows/cd-deploy-to-prod.yml | 4 +++- .github/workflows/cd-deploy-to-test.yml | 4 +++- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/.github/workflows/cd-deploy-to-dev.yml b/.github/workflows/cd-deploy-to-dev.yml index 3509162..c458c80 100644 --- a/.github/workflows/cd-deploy-to-dev.yml +++ b/.github/workflows/cd-deploy-to-dev.yml @@ -17,7 +17,9 @@ concurrency: group: ${{ github.workflow }}-${{ github.job }}-${{ github.head_ref || github.run_id }} cancel-in-progress: true -permissions: read-all +permissions: + contents: read + id-token: write jobs: build: diff --git a/.github/workflows/cd-deploy-to-prod.yml b/.github/workflows/cd-deploy-to-prod.yml index 2b08b95..c366714 100644 --- a/.github/workflows/cd-deploy-to-prod.yml +++ b/.github/workflows/cd-deploy-to-prod.yml @@ -12,7 +12,9 @@ concurrency: group: ${{ github.workflow }}-${{ github.job }}-${{ github.head_ref || github.run_id }} cancel-in-progress: true -permissions: read-all +permissions: + contents: read + id-token: write jobs: build: diff --git a/.github/workflows/cd-deploy-to-test.yml b/.github/workflows/cd-deploy-to-test.yml index 5952d6b..bd1f7b2 100644 --- a/.github/workflows/cd-deploy-to-test.yml +++ b/.github/workflows/cd-deploy-to-test.yml @@ -18,7 +18,9 @@ concurrency: group: ${{ github.workflow }}-${{ github.job }}-${{ github.head_ref || github.run_id }} cancel-in-progress: true -permissions: read-all +permissions: + contents: read + id-token: write jobs: build: From c1d21c91873f2e7aa63bc73084f17addf5df8078 Mon Sep 17 00:00:00 2001 From: Gustavo Valverde Date: Sun, 14 Jul 2024 21:12:42 +0100 Subject: [PATCH 28/33] fix(actions): permissions --- .github/workflows/cd-deploy-to-dev.yml | 12 ++++++++++++ .github/workflows/cd-deploy-to-prod.yml | 12 ++++++++++++ .github/workflows/cd-deploy-to-test.yml | 12 ++++++++++++ 3 files changed, 36 insertions(+) diff --git a/.github/workflows/cd-deploy-to-dev.yml b/.github/workflows/cd-deploy-to-dev.yml index c458c80..ae6abdd 100644 --- a/.github/workflows/cd-deploy-to-dev.yml +++ b/.github/workflows/cd-deploy-to-dev.yml @@ -18,8 +18,20 @@ concurrency: cancel-in-progress: true permissions: + actions: read + attestations: read + checks: read contents: read + deployments: read id-token: write + issues: read + discussions: read + packages: read + pages: read + pull-requests: read + repository-projects: read + security-events: read + statuses: read jobs: build: diff --git a/.github/workflows/cd-deploy-to-prod.yml b/.github/workflows/cd-deploy-to-prod.yml index c366714..00b9f7e 100644 --- a/.github/workflows/cd-deploy-to-prod.yml +++ b/.github/workflows/cd-deploy-to-prod.yml @@ -13,8 +13,20 @@ concurrency: cancel-in-progress: true permissions: + actions: read + attestations: read + checks: read contents: read + deployments: read id-token: write + issues: read + discussions: read + packages: read + pages: read + pull-requests: read + repository-projects: read + security-events: read + statuses: read jobs: build: diff --git a/.github/workflows/cd-deploy-to-test.yml b/.github/workflows/cd-deploy-to-test.yml index bd1f7b2..803556a 100644 --- a/.github/workflows/cd-deploy-to-test.yml +++ b/.github/workflows/cd-deploy-to-test.yml @@ -19,8 +19,20 @@ concurrency: cancel-in-progress: true permissions: + actions: read + attestations: read + checks: read contents: read + deployments: read id-token: write + issues: read + discussions: read + packages: read + pages: read + pull-requests: read + repository-projects: read + security-events: read + statuses: read jobs: build: From a0877f670332b58d130061b5dddbdf46d525df83 Mon Sep 17 00:00:00 2001 From: Gustavo Valverde Date: Mon, 15 Jul 2024 07:10:28 +0100 Subject: [PATCH 29/33] fix(deploy): use correct principalSet instead of SA --- .github/workflows/chore-clean-dev.yml | 2 +- .github/workflows/sub-build-docker-image.yml | 2 +- .github/workflows/sub-cloudrun-deploy.yml | 8 ++++++-- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/.github/workflows/chore-clean-dev.yml b/.github/workflows/chore-clean-dev.yml index bbacb73..75594c0 100644 --- a/.github/workflows/chore-clean-dev.yml +++ b/.github/workflows/chore-clean-dev.yml @@ -25,7 +25,7 @@ jobs: uses: google-github-actions/auth@v2.1.3 with: workload_identity_provider: '${{ vars.GCP_WIF }}' - service_account: '${{ vars.GCP_DEPLOYMENTS_SA }}' + project_id: '${{ vars.GCP_PROJECT }}' - name: Set up Cloud SDK uses: google-github-actions/setup-gcloud@v2.1.0 diff --git a/.github/workflows/sub-build-docker-image.yml b/.github/workflows/sub-build-docker-image.yml index 21b6987..fd99468 100644 --- a/.github/workflows/sub-build-docker-image.yml +++ b/.github/workflows/sub-build-docker-image.yml @@ -77,7 +77,7 @@ jobs: uses: google-github-actions/auth@v2.1.3 with: workload_identity_provider: '${{ vars.GCP_WIF }}' - service_account: '${{ vars.GCP_ARTIFACTS_SA }}' + project_id: '${{ vars.GCP_PROJECT }}' token_format: 'access_token' # Some builds might take over an hour, and Google's default lifetime duration for # an access token is 1 hour (3600s). We increase this to 3 hours (10800s) diff --git a/.github/workflows/sub-cloudrun-deploy.yml b/.github/workflows/sub-cloudrun-deploy.yml index 996abb5..5aa2a60 100644 --- a/.github/workflows/sub-cloudrun-deploy.yml +++ b/.github/workflows/sub-cloudrun-deploy.yml @@ -14,7 +14,7 @@ on: type: string description: The image digest to deploy project_id: - required: true + required: false type: string description: The project to deploy to region: @@ -77,12 +77,16 @@ jobs: - name: Inject slug/short variables uses: rlespinasse/github-slug-action@v4.5.0 + - uses: actions/checkout@v4.1.7 + with: + persist-credentials: false + - name: Authenticate to Google Cloud id: auth uses: google-github-actions/auth@v2.1.3 with: workload_identity_provider: '${{ vars.GCP_WIF }}' - service_account: '${{ vars.GCP_DEPLOYMENTS_SA }}' + project_id: '${{ vars.GCP_PROJECT }}' - name: Set up Cloud SDK uses: google-github-actions/setup-gcloud@v2.1.0 From 2d7b8b53dd52bf9923706c8115f6dcc216c6fafa Mon Sep 17 00:00:00 2001 From: Gustavo Valverde Date: Mon, 15 Jul 2024 07:34:35 +0100 Subject: [PATCH 30/33] fix(build): docker login requires a SA --- .github/workflows/sub-build-docker-image.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/sub-build-docker-image.yml b/.github/workflows/sub-build-docker-image.yml index fd99468..21b6987 100644 --- a/.github/workflows/sub-build-docker-image.yml +++ b/.github/workflows/sub-build-docker-image.yml @@ -77,7 +77,7 @@ jobs: uses: google-github-actions/auth@v2.1.3 with: workload_identity_provider: '${{ vars.GCP_WIF }}' - project_id: '${{ vars.GCP_PROJECT }}' + service_account: '${{ vars.GCP_ARTIFACTS_SA }}' token_format: 'access_token' # Some builds might take over an hour, and Google's default lifetime duration for # an access token is 1 hour (3600s). We increase this to 3 hours (10800s) From 4263b4233a46cb131c97b24a215a768b670cee84 Mon Sep 17 00:00:00 2001 From: Gustavo Valverde Date: Mon, 15 Jul 2024 08:21:55 +0100 Subject: [PATCH 31/33] fix(deploy): attach the CloudSQL instance --- .github/workflows/sub-cloudrun-deploy.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/sub-cloudrun-deploy.yml b/.github/workflows/sub-cloudrun-deploy.yml index 5aa2a60..84b5893 100644 --- a/.github/workflows/sub-cloudrun-deploy.yml +++ b/.github/workflows/sub-cloudrun-deploy.yml @@ -119,6 +119,7 @@ jobs: --add-volume-mount=volume=files,mount-path=/app/data --network=projects/zfnd-dev-net-spoke-0/global/networks/dev-spoke-0 --subnet=projects/zfnd-dev-net-spoke-0/regions/us-east1/subnetworks/dev-default-ue1 + --set-cloudsql-instances=${{ vars.CLOUDSQL_INSTANCE }} - name: Allow unauthenticated calls to the service run: | From 2f0db5da0d794801fdc3b55169b35e672344bddf Mon Sep 17 00:00:00 2001 From: Gustavo Valverde Date: Mon, 15 Jul 2024 08:29:04 +0100 Subject: [PATCH 32/33] chore: organize flags --- .github/workflows/sub-cloudrun-deploy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/sub-cloudrun-deploy.yml b/.github/workflows/sub-cloudrun-deploy.yml index 84b5893..f7e4ab5 100644 --- a/.github/workflows/sub-cloudrun-deploy.yml +++ b/.github/workflows/sub-cloudrun-deploy.yml @@ -115,11 +115,11 @@ jobs: --cpu=${{ inputs.cpu }} --memory=${{ inputs.memory }} --service-account=${{ vars.GCP_BUCKET_SA }} + --set-cloudsql-instances=${{ vars.CLOUDSQL_INSTANCE }} --add-volume=name=files,type=in-memory --add-volume-mount=volume=files,mount-path=/app/data --network=projects/zfnd-dev-net-spoke-0/global/networks/dev-spoke-0 --subnet=projects/zfnd-dev-net-spoke-0/regions/us-east1/subnetworks/dev-default-ue1 - --set-cloudsql-instances=${{ vars.CLOUDSQL_INSTANCE }} - name: Allow unauthenticated calls to the service run: | From 69c0fa421e6701160db02da9c0bf2acbeff1eab8 Mon Sep 17 00:00:00 2001 From: Gustavo Valverde Date: Mon, 15 Jul 2024 10:00:28 +0100 Subject: [PATCH 33/33] imp(deploy): use secrets from GCP secret manager --- .github/workflows/sub-cloudrun-deploy.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/sub-cloudrun-deploy.yml b/.github/workflows/sub-cloudrun-deploy.yml index f7e4ab5..b1b1c66 100644 --- a/.github/workflows/sub-cloudrun-deploy.yml +++ b/.github/workflows/sub-cloudrun-deploy.yml @@ -106,9 +106,9 @@ jobs: UPTIME_KUMA_DB_PORT=${{ vars.UPTIME_KUMA_DB_PORT }} UPTIME_KUMA_DB_NAME=${{ vars.UPTIME_KUMA_DB_NAME }} UPTIME_KUMA_DB_USERNAME=${{ vars.UPTIME_KUMA_DB_USERNAME }} - UPTIME_KUMA_DB_PASSWORD=${{ secrets.UPTIME_KUMA_DB_PASSWORD }} env_vars_update_strategy: overwrite - # secrets: | + secrets: | + UPTIME_KUMA_DB_PASSWORD=UPTIME_KUMA_DB_PASSWORD:latest flags: | --min-instances=${{ inputs.min_instances }} --max-instances=${{ inputs.max_instances }}