Skip to content

Commit 578bdbf

Browse files
jgadlingjgadling
committed
Merge remote-tracking branch 'origin/main' into daniel-ji/fetch-enqueue-logs
2 parents 3f1566c + 8231b99 commit 578bdbf

File tree

595 files changed

+106769
-35604
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

595 files changed

+106769
-35604
lines changed

‎.gitattributes

+2
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
**/poetry.lock linguist-generated
2+
**/codegen/* linguist-generated

‎.github/workflows/apiv2-test.yml

+23
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
name: APIv2 tests
2+
3+
on:
4+
push:
5+
branches:
6+
- main
7+
paths:
8+
- "apiv2/**"
9+
pull_request:
10+
paths:
11+
- "apiv2/**"
12+
13+
jobs:
14+
py-test:
15+
runs-on: [ARM64, self-hosted, Linux]
16+
name: "Run tests"
17+
steps:
18+
- uses: actions/checkout@v3
19+
# Set up a local test env and run tests
20+
- name: Run tests
21+
run: |
22+
make apiv2-init
23+
make apiv2-test

‎.github/workflows/argus-stack-prod-upsert.yaml

+24
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
name: Update prod
2+
3+
on:
4+
release:
5+
types: [published]
6+
7+
concurrency:
8+
group: ${{ github.workflow }}-${{ github.ref }}
9+
cancel-in-progress: true
10+
11+
jobs:
12+
prod_stack_upsert:
13+
runs-on: [ARM64, self-hosted, Linux]
14+
permissions:
15+
id-token: write
16+
issues: write
17+
pull-requests: write
18+
19+
steps:
20+
- name: Update Prod Stack
21+
uses: chanzuckerberg/argus-artifacts/ci/packages/[email protected]
22+
with:
23+
appName: cryoet-apiv2
24+
envName: prod

‎.github/workflows/argus-stack-rdev-create.yaml

+27
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
name: Create an RDev Stack
2+
3+
on:
4+
pull_request:
5+
types: [ labeled, synchronize ]
6+
7+
concurrency:
8+
group: ${{ github.workflow }}-${{ github.ref }}
9+
cancel-in-progress: true
10+
11+
jobs:
12+
create-stack:
13+
runs-on: [ARM64, self-hosted, Linux]
14+
if: contains(github.event.pull_request.labels.*.name, 'stack')
15+
permissions:
16+
id-token: write
17+
issues: write
18+
pull-requests: write
19+
20+
steps:
21+
- name: Create Stack
22+
id: upsert
23+
uses: chanzuckerberg/argus-artifacts/ci/packages/[email protected]
24+
with:
25+
appName: cryoet-apiv2
26+
envName: rdev
27+
waitForDeploymentSeconds: 300

‎.github/workflows/argus-stack-rdev-delete.yaml

+21
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
name: Delete an RDev Stack
2+
3+
on:
4+
pull_request:
5+
types: [ unlabeled, closed ]
6+
7+
jobs:
8+
delete-stack:
9+
runs-on: [ARM64, self-hosted, Linux]
10+
if: ${{ !contains(github.event.pull_request.labels.*.name, 'stack') || github.event.pull_request.state == 'closed' }}
11+
permissions:
12+
id-token: write
13+
issues: write
14+
pull-requests: write
15+
16+
steps:
17+
- name: Delete Stack
18+
uses: chanzuckerberg/argus-artifacts/ci/packages/[email protected]
19+
with:
20+
appName: cryoet-apiv2
21+
envName: rdev

‎.github/workflows/argus-stack-staging-upsert.yaml

+35
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
name: Update staging
2+
3+
on:
4+
pull_request:
5+
types: [ opened, synchronize ]
6+
paths:
7+
- '.infra/common.yaml'
8+
- '.infra/staging/values.yaml'
9+
10+
concurrency:
11+
group: ${{ github.workflow }}-${{ github.ref }}
12+
cancel-in-progress: true
13+
14+
jobs:
15+
run_if:
16+
if: github.head_ref == 'release-please--branches--main--components--apiv2'
17+
runs-on: [ARM64, self-hosted, Linux]
18+
steps:
19+
- run: echo "The head branch of this PR is the release please branch"
20+
21+
staging_stack_upsert:
22+
needs: run_if
23+
runs-on: [ARM64, self-hosted, Linux]
24+
permissions:
25+
id-token: write
26+
issues: write
27+
pull-requests: write
28+
29+
steps:
30+
- name: Update Staging Stack
31+
uses: chanzuckerberg/argus-artifacts/ci/packages/[email protected]
32+
with:
33+
appName: cryoet-apiv2
34+
envName: staging
35+
postStackDetails: false

‎.github/workflows/conventional-commits.yml

+19
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
# Validates PR title follows conventional commits
2+
name: conventional-commits
3+
on:
4+
pull_request:
5+
branches: [main]
6+
types:
7+
- edited
8+
- opened
9+
- synchronize
10+
- reopened
11+
12+
concurrency:
13+
group: ${{ github.workflow }}-${{ github.ref }}
14+
cancel-in-progress: true
15+
jobs:
16+
conventional_commit_title:
17+
runs-on: [ARM64, self-hosted, Linux]
18+
steps:
19+
- uses: chanzuckerberg/github-actions/.github/actions/[email protected]

‎.github/workflows/docker-build-rdev.yaml

+20
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
name: Docker Build Rdev
2+
3+
on:
4+
pull_request:
5+
types: [ labeled, synchronize ]
6+
paths:
7+
- 'apiv2/**'
8+
9+
concurrency:
10+
group: ${{ github.workflow }}-${{ github.ref }}
11+
cancel-in-progress: true
12+
13+
jobs:
14+
rdev_argus_builder:
15+
uses: ./.github/workflows/workflow-argus-docker-build.yaml
16+
secrets: inherit
17+
if: contains(github.event.pull_request.labels.*.name, 'stack') && github.event.sender.type != 'Bot'
18+
with:
19+
envs: rdev
20+
path_filters: 'apiv2/**'

‎.github/workflows/docker-build-staging-prod.yaml

+17
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
name: Docker Build Staging/Prod
2+
3+
on:
4+
push:
5+
paths:
6+
- 'apiv2/**'
7+
branches:
8+
- release-please--branches--main--components--apiv2
9+
10+
jobs:
11+
argus_builder:
12+
uses: ./.github/workflows/workflow-argus-docker-build.yaml
13+
secrets: inherit
14+
with:
15+
envs: staging,prod
16+
path_filters: '!.infra/**,apiv2/**'
17+
branches_include: release-please--branches--main--components--apiv2

‎.github/workflows/dataset-config-validate.yaml ‎.github/workflows/ingestion-config-validate.yaml

+13-11
Original file line numberDiff line numberDiff line change
@@ -6,17 +6,19 @@ on:
66
- main
77
paths:
88
- ingestion_tools/dataset_configs/**
9-
- schema/v1.1.0/Makefile
10-
- schema/v1.1.0/*.yaml
11-
- schema/v1.1.0/*.py
12-
- schema/v1.1.0/*.json
9+
- schema/**/*.py
10+
- schema/**/*.yaml
11+
- schema/**/Makefile
12+
- schema/**/*.json # ingestion_config_validation_exclusions.json
13+
- schema/.version
1314
pull_request:
1415
paths:
1516
- ingestion_tools/dataset_configs/**
16-
- schema/v1.1.0/Makefile
17-
- schema/v1.1.0/*.yaml
18-
- schema/v1.1.0/*.py
19-
- schema/v1.1.0/*.json
17+
- schema/**/*.py
18+
- schema/**/*.yaml
19+
- schema/**/Makefile
20+
- schema/**/*.json # ingestion_config_validation_exclusions.json
21+
- schema/.version
2022
workflow_dispatch:
2123

2224
jobs:
@@ -38,14 +40,14 @@ jobs:
3840
working-directory: schema/
3941
- name: Build Pydantic models
4042
run: make build
41-
working-directory: schema/v1.1.0
43+
working-directory: schema/
4244
- name: Validate dataset configs
4345
run: |
4446
make validate-configs-with-network
45-
working-directory: schema/v1.1.0
47+
working-directory: schema/
4648
- name: Archive validation results
4749
if: failure()
4850
uses: actions/upload-artifact@v4
4951
with:
5052
name: dataset-configs-validation-logs
51-
path: schema/v1.1.0/dataset_config_validate_errors/
53+
path: schema/ingestion_config/latest/ingestion_config_validate_errors/

‎.github/workflows/py-formatting.yml

+7-2
Original file line numberDiff line numberDiff line change
@@ -13,8 +13,13 @@ jobs:
1313
- uses: actions/checkout@v3
1414
- uses: actions/setup-python@v4
1515
with:
16-
python-version: "3.11"
16+
python-version: "3.12.5"
17+
- name: Install action-validator with asdf
18+
uses: asdf-vm/actions/install@v3
19+
with:
20+
tool_versions: |
21+
action-validator 0.6.0
1722
- name: disable pre-commit validation
1823
run: echo "SKIP=dataset-config-validation" >> $GITHUB_ENV
1924
- name: check backend
20-
uses: pre-commit/[email protected].0
25+
uses: pre-commit/[email protected].1

‎.github/workflows/release-please.yaml

+26
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
on:
2+
push:
3+
branches:
4+
- main
5+
concurrency:
6+
group: ${{ github.workflow }}-${{ github.ref }}
7+
cancel-in-progress: true
8+
name: release-please
9+
jobs:
10+
release-please:
11+
runs-on: [ARM64, self-hosted, Linux]
12+
steps:
13+
# See https://docs.github.com/en/actions/security-guides/automatic-token-authentication#using-the-github_token-in-a-workflow
14+
# For why we need to generate a token and not use the default
15+
- name: Generate token
16+
id: generate_token
17+
uses: tibdex/github-app-token@v2
18+
with:
19+
app_id: ${{ secrets.CZI_RELEASE_PLEASE_APP_ID }}
20+
private_key: ${{ secrets.CZI_RELEASE_PLEASE_PK }}
21+
22+
- name: release please
23+
uses: googleapis/release-please-action@v4
24+
id: release
25+
with:
26+
token: ${{ steps.generate_token.outputs.token }}

‎.github/workflows/workflow-argus-docker-build.yaml

+41
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
name: Docker Image Build
2+
3+
on:
4+
workflow_call:
5+
inputs:
6+
envs:
7+
description: 'Env names, comma delimited'
8+
required: true
9+
type: string
10+
path_filters:
11+
description: 'Glob patterns to match against changed files in the repository, comma delimited'
12+
required: true
13+
type: string
14+
branches_include:
15+
description: 'Branches to build on, comma delimited'
16+
required: false
17+
type: string
18+
default: '*'
19+
branches_ignore:
20+
description: 'Branches to skip build on, comma delimited'
21+
required: false
22+
type: string
23+
default: ''
24+
25+
jobs:
26+
argus_builder:
27+
uses: chanzuckerberg/github-actions/.github/workflows/[email protected]
28+
secrets: inherit
29+
with:
30+
branches_include: ${{ inputs.branches_include }}
31+
branches_ignore: ${{ inputs.branches_ignore }}
32+
path_filters: ${{ inputs.path_filters }}
33+
envs: ${{ inputs.envs }}
34+
images: |
35+
{ "apiv2": {
36+
"name": "apiv2",
37+
"context": "apiv2",
38+
"dockerfile": "apiv2/Dockerfile",
39+
"platform": "linux/arm64"
40+
}
41+
}

‎.gitignore

+3
Original file line numberDiff line numberDiff line change
@@ -160,3 +160,6 @@ cython_debug/
160160
.idea/
161161

162162
**/local_data
163+
**.DS_Store
164+
165+
.moto_recording

0 commit comments

Comments
 (0)