Skip to content

Commit 3e6f4a4

Browse files
filsfils
authored
Merge pull request #28
Many changes
2 parents a963137 + d04c017 commit 3e6f4a4

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

47 files changed

+12857
-4601
lines changed

‎.github/workflows/contanerize.yaml

Lines changed: 21 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -4,8 +4,13 @@ name: Dockerize Dagster
44
# overwrites any files created.
55
on:
66
push:
7+
# be wary of using ** for the branch...
8+
# it will fail, and only pull the main branch... which means old code, usually
79
branches:
8-
- "**"
10+
- main
11+
- dev
12+
- dev_eco
13+
- dev_dv
914
tags:
1015
- "v*.*.*"
1116

@@ -14,15 +19,23 @@ on:
1419
defaults:
1520
run:
1621
working-directory: dagster/implnets
22+
1723
jobs:
18-
deploy:
24+
build:
1925
name: Dockerize Scheduler for Project
2026
runs-on: ubuntu-latest
2127
strategy:
2228
matrix:
2329
# project: [ "eco" ]
2430
project: [ "eco", "iow", "oih" ]
31+
#platform: ["linux/amd64","linux/arm64"]
32+
#platform: ["linux/amd64"] #linux/arm64 issues with building
2533
steps:
34+
- name: Set variables
35+
run: |
36+
REGISTRY_IMAGE=nsfearthcube/dagster-${{ matrix.project }}
37+
echo "REGISTRY_IMAGE=$REGISTRY_IMAGE" >> $GITHUB_ENV
38+
working-directory: /
2639
- name: Checkout Repo
2740
uses: actions/checkout@v3
2841
- name: Set up QEMU
@@ -38,7 +51,7 @@ jobs:
3851
id: meta
3952
uses: docker/metadata-action@v4
4053
with:
41-
images: nsfearthcube/dagster-${{ matrix.project }}
54+
images: ${{ env.REGISTRY_IMAGE }}
4255
flavor: |
4356
latest=true
4457
tags: |
@@ -83,7 +96,7 @@ jobs:
8396
# cat ./generatedCode/implnet-${{ matrix.project }}/output/ops/implnet_ops_amgeo.py
8497

8598
- name: Build and push
86-
id: docker_build
99+
id: build
87100
uses: docker/build-push-action@v4
88101
with:
89102
#context: ./dagster/implnets
@@ -98,5 +111,8 @@ jobs:
98111
tags: ${{ steps.meta.outputs.tags }}
99112
# tags: nsfearthcube/ec_facets_client:latest
100113
labels: ${{ steps.meta.outputs.labels }}
114+
#platforms: ${{ matrix.platform }}
115+
#outputs: type=image,name=${{ env.REGISTRY_IMAGE }},push-by-digest=true,name-canonical=true,push=true
116+
101117
- name: Image digest
102-
run: echo ${{ steps.docker_build.outputs.digest }}
118+
run: echo ${{ steps.build.outputs.digest }}

‎.github/workflows/contanerize_multi.yaml_

Lines changed: 176 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,176 @@
1+
name: Dockerize Dagster
2+
# this will build based on what is committed to the branch
3+
# dockerbuildandpush pulls the repo.
4+
# overwrites any files created.
5+
on:
6+
push:
7+
# be wary of using ** for the branch...
8+
# it will fail, and only pull the main branch... which means old code, usually
9+
branches:
10+
- main
11+
- dev
12+
- dev_eco
13+
tags:
14+
- "v*.*.*"
15+
16+
# https://github.com/marketplace/actions/publish-docker
17+
# https://github.com/docker/build-push-action
18+
defaults:
19+
run:
20+
working-directory: dagster/implnets
21+
22+
jobs:
23+
build:
24+
name: Dockerize Scheduler for Project
25+
runs-on: ubuntu-latest
26+
strategy:
27+
matrix:
28+
# project: [ "eco" ]
29+
project: [ "eco", "iow", "oih" ]
30+
#platform: ["linux/amd64","linux/arm64"]
31+
platform: ["linux/amd64"] #linux/arm64 issues with building
32+
steps:
33+
- name: Set variables
34+
run: |
35+
REGISTRY_IMAGE=nsfearthcube/dagster-${{ matrix.project }}
36+
echo "REGISTRY_IMAGE=$REGISTRY_IMAGE" >> $GITHUB_ENV
37+
working-directory: /
38+
- name: Checkout Repo
39+
uses: actions/checkout@v3
40+
- name: Set up QEMU
41+
uses: docker/setup-qemu-action@v2
42+
- name: Set up Docker Buildx
43+
uses: docker/setup-buildx-action@v2
44+
- name: Login to DockerHub
45+
uses: docker/login-action@v2
46+
with:
47+
username: ${{ secrets.DOCKERHUB_USERNAME }}
48+
password: ${{ secrets.DOCKERHUB_TOKEN }}
49+
- name: Extract metadata (tags, labels) for Docker
50+
id: meta
51+
uses: docker/metadata-action@v4
52+
with:
53+
images: ${{ env.REGISTRY_IMAGE }}
54+
flavor: |
55+
latest=true
56+
tags: |
57+
type=ref,event=tag
58+
type=ref,event=branch
59+
type=semver,pattern={{version}}
60+
type=sha
61+
62+
# - name: Set up Python 3.10
63+
# uses: actions/setup-python@v4
64+
# with:
65+
# python-version: '3.10'
66+
# - name: Install dependencies
67+
# run: |
68+
# python -m pip install --upgrade pip
69+
# if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
70+
# long version with lint,
71+
# run: |
72+
# python -m pip install --upgrade pip
73+
# pip install flake8 pytest
74+
# if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
75+
# pip install build
76+
# - name: Lint with flake8
77+
# run: |
78+
# # stop the build if there are Python syntax errors or undefined names
79+
# flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
80+
# # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
81+
# flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
82+
# - name: Build package
83+
# run: python -m build
84+
# - name: Build tooling
85+
# run: |
86+
# cd ./tooling/cfgBuilder/${{ matrix.project }}
87+
# python cfgBuilder.py -s https://foo.us/sitemap.xml
88+
# - name: Generate
89+
# working-directory: dagster/implnets
90+
# run: |
91+
# python pygen.py -cf ./configs/${{ matrix.project }}/gleanerconfig.yaml -od /temp/generatedDocker/implnet-${{ matrix.project }}/output -td ./templates/v1 -d 7
92+
# - name: check
93+
# working-directory: dagster/implnets
94+
# run: |
95+
# cat ./generatedCode/implnet-${{ matrix.project }}/output/ops/implnet_ops_amgeo.py
96+
97+
- name: Build and push
98+
id: build
99+
uses: docker/build-push-action@v4
100+
with:
101+
#context: ./dagster/implnets
102+
# grr https://github.com/docker/build-push-action#git-context
103+
#context: "{{defaultContext}}"
104+
# push: true
105+
build-args:
106+
implnet=${{ matrix.project }}
107+
#file: ./dagster/implnets/build/Dockerfile
108+
file: ./build/Dockerfile_ci
109+
context: "{{defaultContext}}:dagster/implnets"
110+
#tags: ${{ steps.meta.outputs.tags }}
111+
# tags: nsfearthcube/ec_facets_client:latest
112+
labels: ${{ steps.meta.outputs.labels }}
113+
platforms: ${{ matrix.platform }}
114+
outputs: type=image,name=${{ env.REGISTRY_IMAGE }},push-by-digest=true,name-canonical=true,push=true
115+
116+
- name: Image digest
117+
run: echo ${{ steps.build.outputs.digest }}
118+
- name: Export digest
119+
run: |
120+
mkdir -p /tmp/digests
121+
digest="${{ steps.build.outputs.digest }}"
122+
touch "/tmp/digests/${digest#sha256:}"
123+
- name: Upload digest
124+
uses: actions/upload-artifact@v3
125+
with:
126+
name: digests
127+
path: /tmp/digests/*
128+
if-no-files-found: error
129+
retention-days: 1
130+
merge:
131+
runs-on: ubuntu-latest
132+
needs:
133+
- build
134+
steps:
135+
- name: Set variables
136+
run: |
137+
REGISTRY_IMAGE=nsfearthcube/dagster-${{ matrix.project }}
138+
echo "REGISTRY_IMAGE=$REGISTRY_IMAGE" >> $GITHUB_ENV
139+
working-directory: /
140+
- name: Download digests
141+
uses: actions/download-artifact@v3
142+
with:
143+
name: digests
144+
path: /tmp/digests
145+
-
146+
name: Set up Docker Buildx
147+
uses: docker/setup-buildx-action@v2
148+
-
149+
name: Docker meta
150+
id: meta
151+
uses: docker/metadata-action@v4
152+
with:
153+
images: ${{ env.REGISTRY_IMAGE }}
154+
flavor: |
155+
latest=true
156+
tags: |
157+
type=ref,event=tag
158+
type=ref,event=branch
159+
type=semver,pattern={{version}}
160+
161+
-
162+
name: Login to Docker Hub
163+
uses: docker/login-action@v2
164+
with:
165+
username: ${{ secrets.DOCKERHUB_USERNAME }}
166+
password: ${{ secrets.DOCKERHUB_TOKEN }}
167+
-
168+
name: Create manifest list and push
169+
working-directory: /tmp/digests
170+
run: |
171+
docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \
172+
$(printf '${{ env.REGISTRY_IMAGE }}@sha256:%s ' *)
173+
-
174+
name: Inspect image
175+
run: |
176+
docker buildx imagetools inspect ${{ env.REGISTRY_IMAGE }}:${{ steps.meta.outputs.version }}

‎.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,3 +7,4 @@ venv/**
77
/dagster/.logs_queue/
88
/dagster/.telemetry/
99
/dagster/.telemetry/
10+
.env

‎.idea/misc.xml

Lines changed: 2 additions & 2 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎.idea/scheduler.iml

Lines changed: 2 additions & 2 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎NOTES.md

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
# Notes
2+
3+
4+
5+
## Some articles to review
6+
7+
[Medium on Dagster with configurable API and asset examples](https://medium.com/@alexandreguitton_12701/notes-1-2-dagster-data-orchestrator-hands-on-2af6772b13d9)

‎dagster/implnets/build/Dockerfile_local

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
FROM python:3.10-slim
2+
# context is fron implnets
3+
# for local, code is compiled. to generatedCode/implnet-PROJECT
4+
5+
ARG implnet=eco
6+
RUN mkdir -p /usr/src/app/output
7+
8+
COPY ./requirements.txt .
9+
10+
RUN pip install -r requirements.txt
11+
12+
# these are the configs.
13+
# might mount locally in docker_compose
14+
COPY . scheduler
15+
COPY ./configs/${implnet}/gleanerconfig.yaml scheduler/gleanerconfig.yaml
16+
#COPY ./dagster.yaml /usr/src/app/output/dagster.yaml
17+
COPY ./dagster.yaml /usr/src/app/dagster.yaml
18+
# Change working directory
19+
20+
21+
WORKDIR /usr/src/app
22+
ENV DAGSTER_HOME=/usr/src/app
23+
24+
25+
CMD ["dagit", "-w", "./output/workspace.yaml", "-h", "0.0.0.0", "-p", "3000"]

‎dagster/implnets/configs/eco/gleanerconfig.yaml

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3,11 +3,9 @@ context:
33
strict: true
44
contextmaps:
55
- prefix: "https://schema.org/"
6-
# file: "./assets/schemaorg-current-https.jsonld" # wget http://schema.org/docs/jsonldcontext.jsonld
7-
file: "./jsonldcontext.json" # wget http://schema.org/docs/jsonldcontext.jsonld
6+
file: "/assets/schemaorg-current-https.jsonld" # wget http://schema.org/docs/jsonldcontext.jsonld
87
- prefix: "http://schema.org/"
9-
# file: "./assets/schemaorg-current-https.jsonld" # wget http://schema.org/docs/jsonldcontext.jsonld
10-
file: "./jsonldcontext.json" # wget http://schema.org/docs/jsonldcontext.jsonld
8+
file: "/assets/schemaorg-current-http.jsonld" # wget http://schema.org/docs/jsonldcontext.jsonld
119
gleaner:
1210
mill: true
1311
runid: runX

‎dagster/implnets/configs/eco/nabuconfig.yaml

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
minio:
2+
bucket:
3+
address:
4+
port:
5+
accesskey:
6+
secretkey:
7+
ssl: true
8+
context:
9+
cache: true
10+
strict: true
11+
contextmaps:
12+
- prefix: "https://schema.org/"
13+
file: "/assets/schemaorg-current-https.jsonld" # wget http://schema.org/docs/jsonldcontext.jsonld
14+
- prefix: "http://schema.org/"
15+
file: "/assets/schemaorg-current-http.jsonld" # wget http://schema.org/docs/jsonldcontext.jsonld
16+
sparql:
17+
endpoint:
18+
objects:
19+
domain: us-east-1
20+
prefix:
21+
- summoned/aquadocs
22+
- summoned/bcodmo
23+
- summoned/cchdo
24+
- summoned/earthchem
25+
- summoned/edi
26+
- summoned/hydroshare
27+
- summoned/linkedearth
28+
- summoned/magic
29+
- summoned/opentopography
30+
- summoned/r2r
31+
- summoned/ssdbiodp
32+
- summoned/unavco
33+
- prov/aquadocs
34+
- prov/bcodmo
35+
- prov/cchdo
36+
- prov/earthchem
37+
- prov/edi
38+
- prov/hydroshare
39+
- prov/linkedearth
40+
- prov/magic
41+
- prov/opentopography
42+
- prov/r2r
43+
- prov/ssdbiodp
44+
- prov/unavco
45+
46+

0 commit comments

Comments
 (0)