Merge pull request #28

Many changes
gleanerio · Jul 24, 2023 · 3e6f4a4 · 3e6f4a4
2 parents a963137 + d04c017
commit 3e6f4a4
Show file tree

Hide file tree

Showing 47 changed files with 12,857 additions and 4,601 deletions.
diff --git a/.github/workflows/contanerize.yaml b/.github/workflows/contanerize.yaml
@@ -4,8 +4,13 @@ name: Dockerize  Dagster
 # overwrites any files created.
 on:
   push:
+    # be wary of using **  for the branch...
+    # it will fail, and only pull the main branch... which means old code, usually
     branches:
-      - "**"
+      - main
+      - dev
+      - dev_eco
+      -  dev_dv
     tags:
       - "v*.*.*"
 
@@ -14,15 +19,23 @@ on:
 defaults:
   run:
     working-directory:  dagster/implnets
+
 jobs:
-  deploy:
+  build:
     name: Dockerize  Scheduler for Project
     runs-on: ubuntu-latest
     strategy:
       matrix:
        # project: [ "eco" ]
         project: [ "eco", "iow", "oih" ]
+        #platform: ["linux/amd64","linux/arm64"]
+        #platform: ["linux/amd64"] #linux/arm64 issues with building
     steps:
+      - name: Set variables
+        run: |
+            REGISTRY_IMAGE=nsfearthcube/dagster-${{ matrix.project }}
+            echo "REGISTRY_IMAGE=$REGISTRY_IMAGE" >> $GITHUB_ENV
+        working-directory:  /
       - name: Checkout Repo
         uses: actions/checkout@v3
       - name: Set up QEMU
@@ -38,7 +51,7 @@ jobs:
         id: meta
         uses: docker/metadata-action@v4
         with:
-          images: nsfearthcube/dagster-${{ matrix.project }}
+          images:  ${{ env.REGISTRY_IMAGE }}
           flavor: |
             latest=true
           tags: |
@@ -83,7 +96,7 @@ jobs:
 #           cat ./generatedCode/implnet-${{ matrix.project }}/output/ops/implnet_ops_amgeo.py
 
       - name: Build and push
-        id: docker_build
+        id: build
         uses: docker/build-push-action@v4
         with:
           #context:  ./dagster/implnets
@@ -98,5 +111,8 @@ jobs:
           tags: ${{ steps.meta.outputs.tags }}
 #          tags: nsfearthcube/ec_facets_client:latest
           labels: ${{ steps.meta.outputs.labels }}
+          #platforms: ${{ matrix.platform }}
+          #outputs: type=image,name=${{ env.REGISTRY_IMAGE }},push-by-digest=true,name-canonical=true,push=true
+
       - name: Image digest
-        run: echo ${{ steps.docker_build.outputs.digest }}
+        run: echo ${{ steps.build.outputs.digest }}
diff --git a/.github/workflows/contanerize_multi.yaml_ b/.github/workflows/contanerize_multi.yaml_
@@ -0,0 +1,176 @@
+name: Dockerize  Dagster
+# this will build based on what is committed to the branch
+# dockerbuildandpush pulls the repo.
+# overwrites any files created.
+on:
+  push:
+    # be wary of using **  for the branch...
+    # it will fail, and only pull the main branch... which means old code, usually
+    branches:
+      - main
+      - dev
+      - dev_eco
+    tags:
+      - "v*.*.*"
+
+# https://github.com/marketplace/actions/publish-docker
+# https://github.com/docker/build-push-action
+defaults:
+  run:
+    working-directory:  dagster/implnets
+
+jobs:
+  build:
+    name: Dockerize  Scheduler for Project
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+       # project: [ "eco" ]
+        project: [ "eco", "iow", "oih" ]
+        #platform: ["linux/amd64","linux/arm64"]
+        platform: ["linux/amd64"] #linux/arm64 issues with building
+    steps:
+      - name: Set variables
+        run: |
+            REGISTRY_IMAGE=nsfearthcube/dagster-${{ matrix.project }}
+            echo "REGISTRY_IMAGE=$REGISTRY_IMAGE" >> $GITHUB_ENV
+        working-directory:  /
+      - name: Checkout Repo
+        uses: actions/checkout@v3
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+      - name: Login to DockerHub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@v4
+        with:
+          images:  ${{ env.REGISTRY_IMAGE }}
+          flavor: |
+            latest=true
+          tags: |
+            type=ref,event=tag
+            type=ref,event=branch
+            type=semver,pattern={{version}}
+            type=sha
+
+#      - name: Set up Python 3.10
+#        uses: actions/setup-python@v4
+#        with:
+#          python-version:  '3.10'
+#      - name: Install dependencies
+#        run: |
+#          python -m pip install --upgrade pip
+#          if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+# long version with lint,
+#        run: |
+#          python -m pip install --upgrade pip
+#          pip install flake8 pytest
+#          if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+#          pip install build
+#      - name: Lint with flake8
+#        run: |
+#          # stop the build if there are Python syntax errors or undefined names
+#          flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
+#          # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
+#          flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
+#      - name: Build package
+#        run: python -m build
+#      - name: Build tooling
+#        run: |
+#          cd ./tooling/cfgBuilder/${{ matrix.project }}
+#          python cfgBuilder.py -s https://foo.us/sitemap.xml
+#      - name: Generate
+#        working-directory:  dagster/implnets
+#        run: |
+#          python pygen.py -cf ./configs/${{ matrix.project }}/gleanerconfig.yaml -od /temp/generatedDocker/implnet-${{ matrix.project }}/output  -td ./templates/v1   -d 7
+#      - name: check
+#        working-directory:  dagster/implnets
+#        run: |
+#           cat ./generatedCode/implnet-${{ matrix.project }}/output/ops/implnet_ops_amgeo.py
+
+      - name: Build and push
+        id: build
+        uses: docker/build-push-action@v4
+        with:
+          #context:  ./dagster/implnets
+          # grr https://github.com/docker/build-push-action#git-context
+          #context:  "{{defaultContext}}"
+         # push: true
+          build-args:
+            implnet=${{ matrix.project }}
+          #file: ./dagster/implnets/build/Dockerfile
+          file: ./build/Dockerfile_ci
+          context:  "{{defaultContext}}:dagster/implnets"
+          #tags: ${{ steps.meta.outputs.tags }}
+#          tags: nsfearthcube/ec_facets_client:latest
+          labels: ${{ steps.meta.outputs.labels }}
+          platforms: ${{ matrix.platform }}
+          outputs: type=image,name=${{ env.REGISTRY_IMAGE }},push-by-digest=true,name-canonical=true,push=true
+
+      - name: Image digest
+        run: echo ${{ steps.build.outputs.digest }}
+      - name: Export digest
+        run: |
+          mkdir -p /tmp/digests
+          digest="${{ steps.build.outputs.digest }}"
+          touch "/tmp/digests/${digest#sha256:}"
+      - name: Upload digest
+        uses: actions/upload-artifact@v3
+        with:
+          name: digests
+          path: /tmp/digests/*
+          if-no-files-found: error
+          retention-days: 1
+  merge:
+    runs-on: ubuntu-latest
+    needs:
+      - build
+    steps:
+      - name: Set variables
+        run: |
+            REGISTRY_IMAGE=nsfearthcube/dagster-${{ matrix.project }}
+            echo "REGISTRY_IMAGE=$REGISTRY_IMAGE" >> $GITHUB_ENV
+        working-directory:  /
+      - name: Download digests
+        uses: actions/download-artifact@v3
+        with:
+          name: digests
+          path: /tmp/digests
+      -
+        name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+      -
+        name: Docker meta
+        id: meta
+        uses: docker/metadata-action@v4
+        with:
+          images: ${{ env.REGISTRY_IMAGE }}
+          flavor: |
+            latest=true
+          tags: |
+            type=ref,event=tag
+            type=ref,event=branch
+            type=semver,pattern={{version}}
+
+      -
+        name: Login to Docker Hub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      -
+        name: Create manifest list and push
+        working-directory: /tmp/digests
+        run: |
+          docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \
+            $(printf '${{ env.REGISTRY_IMAGE }}@sha256:%s ' *)
+      -
+        name: Inspect image
+        run: |
+          docker buildx imagetools inspect ${{ env.REGISTRY_IMAGE }}:${{ steps.meta.outputs.version }}
diff --git a/.gitignore b/.gitignore
@@ -7,3 +7,4 @@ venv/**
 /dagster/.logs_queue/
 /dagster/.telemetry/
 /dagster/.telemetry/
+.env
diff --git a/.idea/misc.xml b/.idea/misc.xml
diff --git a/.idea/scheduler.iml b/.idea/scheduler.iml
diff --git a/NOTES.md b/NOTES.md
@@ -0,0 +1,7 @@
+# Notes
+
+
+
+## Some articles to review
+
+[Medium on Dagster with configurable API and asset examples](https://medium.com/@alexandreguitton_12701/notes-1-2-dagster-data-orchestrator-hands-on-2af6772b13d9)
diff --git a/dagster/implnets/build/Dockerfile_local b/dagster/implnets/build/Dockerfile_local
@@ -0,0 +1,25 @@
+FROM python:3.10-slim
+# context is fron implnets
+# for local, code is compiled. to generatedCode/implnet-PROJECT
+
+ARG implnet=eco
+RUN mkdir -p /usr/src/app/output
+
+COPY ./requirements.txt .
+
+RUN pip install -r requirements.txt
+
+# these are the configs.
+# might mount locally in docker_compose
+COPY . scheduler
+COPY ./configs/${implnet}/gleanerconfig.yaml scheduler/gleanerconfig.yaml
+#COPY ./dagster.yaml /usr/src/app/output/dagster.yaml
+COPY ./dagster.yaml /usr/src/app/dagster.yaml
+# Change working directory
+
+
+WORKDIR /usr/src/app
+ENV DAGSTER_HOME=/usr/src/app
+
+
+CMD ["dagit", "-w", "./output/workspace.yaml", "-h", "0.0.0.0", "-p", "3000"]
diff --git a/dagster/implnets/configs/eco/gleanerconfig.yaml b/dagster/implnets/configs/eco/gleanerconfig.yaml
@@ -3,11 +3,9 @@ context:
   strict: true
 contextmaps:
   - prefix: "https://schema.org/"
-#    file: "./assets/schemaorg-current-https.jsonld"  # wget http://schema.org/docs/jsonldcontext.jsonld
-    file: "./jsonldcontext.json"  # wget http://schema.org/docs/jsonldcontext.jsonld
+    file: "/assets/schemaorg-current-https.jsonld"  # wget http://schema.org/docs/jsonldcontext.jsonld
   - prefix: "http://schema.org/"
-#    file: "./assets/schemaorg-current-https.jsonld"  # wget http://schema.org/docs/jsonldcontext.jsonld
-    file: "./jsonldcontext.json"  # wget http://schema.org/docs/jsonldcontext.jsonld
+    file: "/assets/schemaorg-current-http.jsonld"  # wget http://schema.org/docs/jsonldcontext.jsonld
 gleaner:
   mill: true
   runid: runX

diff --git a/dagster/implnets/configs/eco/nabuconfig.yaml b/dagster/implnets/configs/eco/nabuconfig.yaml
@@ -0,0 +1,46 @@
+minio:
+  bucket:
+  address:
+  port:
+  accesskey:
+  secretkey:
+  ssl: true
+context:
+    cache: true
+    strict: true
+contextmaps:
+  - prefix: "https://schema.org/"
+    file: "/assets/schemaorg-current-https.jsonld"  # wget http://schema.org/docs/jsonldcontext.jsonld
+  - prefix: "http://schema.org/"
+    file: "/assets/schemaorg-current-http.jsonld"  # wget http://schema.org/docs/jsonldcontext.jsonld
+sparql:
+  endpoint:
+objects:
+  domain: us-east-1
+  prefix:
+    - summoned/aquadocs
+    - summoned/bcodmo
+    - summoned/cchdo
+    - summoned/earthchem
+    - summoned/edi
+    - summoned/hydroshare
+    - summoned/linkedearth
+    - summoned/magic
+    - summoned/opentopography
+    - summoned/r2r
+    - summoned/ssdbiodp
+    - summoned/unavco
+    - prov/aquadocs
+    - prov/bcodmo
+    - prov/cchdo
+    - prov/earthchem
+    - prov/edi
+    - prov/hydroshare
+    - prov/linkedearth
+    - prov/magic
+    - prov/opentopography
+    - prov/r2r
+    - prov/ssdbiodp
+    - prov/unavco
+
+