Gg foo

.dockstore.yml

@@ -101,7 +101,7 @@ workflows:
              - /.*/
    - name: GvsBenchmarkExtractTask
      subclass: WDL
-     primaryDescriptorPath: /scripts/variantstore/wdl/GvsBenchmarkExtractTask.wdl
+     primaryDescriptorPath: /scripts/variantstore/wdl/test/GvsBenchmarkExtractTask.wdl
      filters:
          branches:
              - master
@@ -151,6 +151,35 @@ workflows:
    - name: GvsCreateVDS
      subclass: WDL
      primaryDescriptorPath: /scripts/variantstore/wdl/GvsCreateVDS.wdl
+     filters:
+       branches:
+         - master
+         - ah_var_store
+         - vs_1730_clusters_leaking
+       tags:
+         - /.*/
+   - name: GvsValidateVDS
+     subclass: WDL
+     primaryDescriptorPath: /scripts/variantstore/wdl/GvsValidateVDS.wdl
+     filters:
+       branches:
+         - master
+         - ah_var_store
+       tags:
+         - /.*/
+   - name: GvsMergeAndRescoreVDSes
+     subclass: WDL
+     primaryDescriptorPath: /scripts/variantstore/wdl/GvsMergeAndRescoreVDSes.wdl
+     filters:
+       branches:
+         - master
+         - ah_var_store
+         - vs_1730_clusters_leaking
+       tags:
+         - /.*/
+   - name: GvsTieOutVDS
+     subclass: WDL
+     primaryDescriptorPath: /scripts/variantstore/wdl/test/GvsTieOutVDS.wdl
      filters:
        branches:
          - master
@@ -166,6 +195,15 @@ workflows:
              - ah_var_store
          tags:
              - /.*/
+   - name: GvsExtractCallsetShard30603
+     subclass: WDL
+     primaryDescriptorPath: /scripts/variantstore/wdl/GvsExtractCallset.wdl
+     filters:
+       branches:
+         - master
+         - ah_var_store
+       tags:
+         - /.*/
    - name: GvsImportGenomes
      subclass: WDL
      primaryDescriptorPath: /scripts/variantstore/wdl/GvsImportGenomes.wdl
@@ -182,7 +220,6 @@ workflows:
          branches:
              - master
              - ah_var_store
-             - vs_1187_headers_only_ingest
          tags:
              - /.*/
    - name: GvsPrepareRangesCallset
@@ -198,17 +235,16 @@ workflows:
              - /.*/
    - name: GvsCreateVATfromVDS
      subclass: WDL
-     primaryDescriptorPath: /scripts/variantstore/wdl/GvsCreateVATfromVDS.wdl
+     primaryDescriptorPath: /scripts/variantstore/variant-annotations-table/GvsCreateVATfromVDS.wdl
      filters:
          branches:
              - master
              - ah_var_store
-             - gg_VS-1229_DeDupTheVatUponCreation
          tags:
              - /.*/
    - name: GvsCreateVATFilesFromBigQuery
      subclass: WDL
-     primaryDescriptorPath: /scripts/variantstore/variant_annotations_table/GvsCreateVATFilesFromBigQuery.wdl
+     primaryDescriptorPath: /scripts/variantstore/variant-annotations-table/GvsCreateVATFilesFromBigQuery.wdl
      filters:
          branches:
              - master
@@ -217,9 +253,9 @@ workflows:
              - /.*/
    - name: GvsValidateVat
      subclass: WDL
-     primaryDescriptorPath: /scripts/variantstore/variant_annotations_table/GvsValidateVAT.wdl
+     primaryDescriptorPath: /scripts/variantstore/variant-annotations-table/GvsValidateVAT.wdl
      testParameterFiles:
-         - /scripts/variantstore/variant_annotations_table/GvsValidateVat.example.inputs.json
+         - /scripts/variantstore/variant-annotations-table/GvsValidateVat.example.inputs.json
      filters:
          branches:
              - master
@@ -251,6 +287,7 @@ workflows:
          branches:
              - master
              - ah_var_store
+             - VS-1737
          tags:
              - /.*/
    - name: GvsBeta
@@ -261,7 +298,6 @@ workflows:
              # `master` is here so there will be *something* under the branches filter, but GVS WDLs are not on `master`
              # so this shouldn't show up in the list of available versions in Dockstore or Terra.
              - master
-             - vs_1099_composite_object_inputs
          tags:
              - /.*/
    - name: GvsJointVariantCallsetCost
@@ -284,7 +320,7 @@ workflows:
              - /.*/
    - name: GvsQuickstartVcfIntegration
      subclass: WDL
-     primaryDescriptorPath: /scripts/variantstore/wdl/GvsQuickstartVcfIntegration.wdl
+     primaryDescriptorPath: /scripts/variantstore/wdl/test/GvsQuickstartVcfIntegration.wdl
      filters:
          branches:
              - master
@@ -293,7 +329,7 @@ workflows:
              - /.*/
    - name: GvsQuickstartHailIntegration
      subclass: WDL
-     primaryDescriptorPath: /scripts/variantstore/wdl/GvsQuickstartHailIntegration.wdl
+     primaryDescriptorPath: /scripts/variantstore/wdl/test/GvsQuickstartHailIntegration.wdl
      filters:
          branches:
              - master
@@ -302,17 +338,17 @@ workflows:
              - /.*/
    - name: GvsQuickstartIntegration
      subclass: WDL
-     primaryDescriptorPath: /scripts/variantstore/wdl/GvsQuickstartIntegration.wdl
+     primaryDescriptorPath: /scripts/variantstore/wdl/test/GvsQuickstartIntegration.wdl
      filters:
          branches:
              - master
              - ah_var_store
-             - vs_1187_headers_only_ingest
+             - vs_1634_pgen_exome
          tags:
              - /.*/
    - name: GvsIngestTieout
      subclass: WDL
-     primaryDescriptorPath: /scripts/variantstore/wdl/GvsIngestTieout.wdl
+     primaryDescriptorPath: /scripts/variantstore/wdl/test/GvsIngestTieout.wdl
      filters:
          branches:
              - master
@@ -355,14 +391,15 @@ workflows:
              - ah_var_store
          tags:
              - /.*/
-   - name: HailFromWdl
+   - name: GvsTieoutVcfMaxAltAlleles
      subclass: WDL
-     primaryDescriptorPath: /scripts/variantstore/wdl/HailFromWdl.wdl
+     primaryDescriptorPath: /scripts/variantstore/wdl/test/GvsTieoutVcfMaxAltAlleles.wdl
      filters:
-         branches:
-             - master
-         tags:
-             - /.*/
+       branches:
+         - ah_var_store
+         - master
+       tags:
+         - /.*/
    - name: MitochondriaPipeline
      subclass: WDL
      primaryDescriptorPath: /scripts/mitochondria_m2_wdl/MitochondriaPipeline.wdl
@@ -411,3 +448,36 @@ workflows:
              - master
          tags:
              - /.*/
+   - name: GvsExtractCallsetPgen
+     subclass: WDL
+     primaryDescriptorPath: /scripts/variantstore/wdl/GvsExtractCallsetPgen.wdl
+     filters:
+       branches:
+         - master
+         - ah_var_store
+   - name: GvsExtractCallsetPgenMerged
+     subclass: WDL
+     primaryDescriptorPath: /scripts/variantstore/wdl/GvsExtractCallsetPgenMerged.wdl
+     filters:
+       branches:
+         - master
+         - ah_var_store
+   - name: MergePgenHierarchicalWdl
+     subclass: WDL
+     primaryDescriptorPath: /scripts/variantstore/wdl/MergePgenHierarchical.wdl
+     filters:
+       branches:
+         - master
+         - ah_var_store
+   - name: GvsTieoutPgenToVcf
+     subclass: WDL
+     primaryDescriptorPath: /scripts/variantstore/wdl/test/GvsTieoutPgenToVcf.wdl
+     filters:
+       branches:
+         - ah_var_store
+   - name: SearchGVCFsForUnmappedVIDs
+     subclass: WDL
+     primaryDescriptorPath: /scripts/variantstore/variant-annotations-table/pseudo_vids_only_in_vat/SearchGVCFsForUnmappedVIDs.wdl
+     filters:
+       branches:
+         - ah_var_store

.github/actions/upload-gatk-test-results/action.yml

@@ -34,22 +34,23 @@ runs:
   steps:
     - name: Upload test results
       if: always()
-      uses: actions/upload-artifact@v3
+      uses: actions/upload-artifact@v4
       with:
         # ternary expression syntax is workaround found here https://github.com/actions/runner/issues/409
         name: test-results-${{ inputs.is-docker == 'true' && 'docker-' ||  '' }}${{ matrix.Java }}-${{ matrix.testType }}
         path: build/reports/tests
 
-    - name: Upload to codecov
-      run: bash <(curl -s https://raw.githubusercontent.com/broadinstitute/codecov-bash-uploader/main/codecov-verified.bash)
-      shell: bash
+          # Disabling codecov because it is timing out and failing builds that otherwise succeed.
+          ##    - name: Upload to codecov
+          ##      run: bash <(curl -s https://raw.githubusercontent.com/broadinstitute/codecov-bash-uploader/main/codecov-verified.bash)
+          ##      shell: bash
 
     - name: Upload Reports
       if: ${{ inputs.only-artifact != 'true' }}
       id: uploadreports
       run: |
         gsutil -m cp -z html -z js -z xml -z css -r build/reports/tests gs:/${{ env.HELLBENDER_TEST_LOGS }}${{ inputs.repo-path }}/;
-        VIEW_URL=https://storage.googleapis.com${{ env.HELLBENDER_TEST_LOGS }}${{ inputs.repo-path }}/tests/testOnPackagedReleaseJar/index.html
+        VIEW_URL=https://storage.googleapis.com${{ env.HELLBENDER_TEST_LOGS }}${{ inputs.repo-path }}/tests/test/index.html
         echo "See the test report at ${VIEW_URL}";
         echo view_url="${VIEW_URL}" >> $GITHUB_OUTPUT
 
@@ -91,4 +92,4 @@ runs:
       run: |
         pip install --user PyGithub;
         python scripts/github_actions/Reporter.py ${{ steps.uploadreports.outputs.view_url }};
-      shell: bash
+      shell: bash

.github/workflows/dependency_submission.yml

@@ -0,0 +1,22 @@
+name: Dependency Submission
+
+on:
+  push:
+    branches: [ 'master' ]
+
+permissions:
+  contents: write
+
+jobs:
+  dependency-submission:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout sources
+      uses: actions/checkout@v4
+    - name: Setup Java
+      uses: actions/setup-java@v4
+      with:
+        distribution: 'temurin'
+        java-version: 17
+    - name: Generate and submit dependency graph
+      uses: gradle/actions/dependency-submission@v4

.github/workflows/gatk-nightly-docker.yml

@@ -10,7 +10,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: checkout gatk
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0