consolidate GX docs to a single docsite

Sage-Bionetworks · Oct 23, 2024 · 694eb2a · 694eb2a
1 parent b6bd430
commit 694eb2a
Show file tree

Hide file tree

Showing 12 changed files with 277 additions and 529 deletions.
diff --git a/.github/workflows/upload-and-deploy-to-prod-main.yaml b/.github/workflows/upload-and-deploy-to-prod-main.yaml
@@ -41,7 +41,12 @@ jobs:
           python_version: ${{ env.PYTHON_VERSION }}
 
       - name: Copy files to templates bucket
-        run: python src/scripts/manage_artifacts/artifacts.py --upload --namespace $NAMESPACE --cfn_bucket ${{ vars.CFN_BUCKET }}
+        run: >
+          python src/scripts/manage_artifacts/artifacts.py
+          --upload
+          --namespace $NAMESPACE
+          --cfn_bucket ${{ vars.CFN_BUCKET }}
+          --shareable-artifacts-bucket ${{ vars.SHAREABLE_ARTIFACTS_BUCKET }}
 
 
   sceptre-deploy-main:

diff --git a/.github/workflows/upload-and-deploy.yaml b/.github/workflows/upload-and-deploy.yaml
@@ -115,6 +115,7 @@ jobs:
           --upload
           --namespace $NAMESPACE
           --cfn_bucket ${{ vars.CFN_BUCKET }}
+          --shareable-artifacts-bucket ${{ vars.SHAREABLE_ARTIFACTS_BUCKET }}
 
   nonglue-unit-tests:
     name: Runs unit tests that are not dependent on aws-glue package resources
@@ -437,6 +438,7 @@ jobs:
           --upload
           --namespace staging
           --cfn_bucket ${{ vars.CFN_BUCKET }}
+          --shareable-artifacts-bucket ${{ vars.SHAREABLE_ARTIFACTS_BUCKET }}
 
       - name: Create directory for remote sceptre templates
         run: mkdir -p templates/remote/

diff --git a/config/develop/namespaced/glue-job-run-great-expectations-on-parquet.yaml b/config/develop/namespaced/glue-job-run-great-expectations-on-parquet.yaml
@@ -7,12 +7,15 @@ parameters:
   Namespace: {{ stack_group_config.namespace }}
   JobDescription: Runs great expectations on a set of data
   JobRole: !stack_output_external glue-job-role::RoleArn
-  TempS3Bucket: {{ stack_group_config.processed_data_bucket_name }}
+  ParquetBucket: {{ stack_group_config.processed_data_bucket_name }}
+  ShareableArtifactsBucket: {{ stack_group_config.shareable_artifacts_vpn_bucket_name }}
   S3ScriptBucket: {{ stack_group_config.template_bucket_name }}
   S3ScriptKey: '{{ stack_group_config.namespace }}/src/glue/jobs/run_great_expectations_on_parquet.py'
+  ExpectationSuiteKey: "{{ stack_group_config.namespace }}/src/glue/resources/data_values_expectations.json"
   GlueVersion: "{{ stack_group_config.great_expectations_job_glue_version }}"
   AdditionalPythonModules: "great_expectations~=0.18,urllib3<2"
 stack_tags:
   {{ stack_group_config.default_stack_tags }}
 sceptre_user_data:
   dataset_schemas: !file src/glue/resources/table_columns.yaml
+  data_values_expectations: !file src/glue/resources/data_values_expectations.json
diff --git a/config/develop/namespaced/glue-workflow.yaml b/config/develop/namespaced/glue-workflow.yaml
@@ -20,8 +20,6 @@ parameters:
   CompareParquetMainNamespace: "main"
   S3SourceBucketName: {{ stack_group_config.input_bucket_name }}
   CloudformationBucketName: {{ stack_group_config.template_bucket_name }}
-  ShareableArtifactsBucketName: {{ stack_group_config.shareable_artifacts_vpn_bucket_name }}
-  ExpectationSuiteKey: "{{ stack_group_config.namespace }}/src/glue/resources/data_values_expectations.json"
 stack_tags:
   {{ stack_group_config.default_stack_tags }}
 sceptre_user_data:

diff --git a/config/prod/namespaced/glue-job-run-great-expectations-on-parquet.yaml b/config/prod/namespaced/glue-job-run-great-expectations-on-parquet.yaml
@@ -7,12 +7,15 @@ parameters:
   Namespace: {{ stack_group_config.namespace }}
   JobDescription: Runs great expectations on a set of data
   JobRole: !stack_output_external glue-job-role::RoleArn
-  TempS3Bucket: {{ stack_group_config.processed_data_bucket_name }}
+  ParquetBucket: {{ stack_group_config.processed_data_bucket_name }}
+  ShareableArtifactsBucket: {{ stack_group_config.shareable_artifacts_vpn_bucket_name }}
   S3ScriptBucket: {{ stack_group_config.template_bucket_name }}
   S3ScriptKey: '{{ stack_group_config.namespace }}/src/glue/jobs/run_great_expectations_on_parquet.py'
+  ExpectationSuiteKey: "{{ stack_group_config.namespace }}/src/glue/resources/data_values_expectations.json"
   GlueVersion: "{{ stack_group_config.great_expectations_job_glue_version }}"
   AdditionalPythonModules: "great_expectations~=0.18,urllib3<2"
 stack_tags:
   {{ stack_group_config.default_stack_tags }}
 sceptre_user_data:
   dataset_schemas: !file src/glue/resources/table_columns.yaml
+  data_values_expectations: !file src/glue/resources/data_values_expectations.json
diff --git a/config/prod/namespaced/glue-workflow.yaml b/config/prod/namespaced/glue-workflow.yaml
@@ -20,8 +20,6 @@ parameters:
   CompareParquetMainNamespace: "main"
   S3SourceBucketName: {{ stack_group_config.input_bucket_name }}
   CloudformationBucketName: {{ stack_group_config.template_bucket_name }}
-  ShareableArtifactsBucketName: {{ stack_group_config.shareable_artifacts_vpn_bucket_name }}
-  ExpectationSuiteKey: "{{ stack_group_config.namespace }}/src/glue/resources/data_values_expectations.json"
 stack_tags:
   {{ stack_group_config.default_stack_tags }}
 sceptre_user_data: