GEOMESA-3540 Spark - Move runtime JARs to classifier (#3451)

* Split CI jobs into separate files
* Reduce unnecessary dash checks in CI

Author: elahrvivaz

.github/workflows/dash-scheduled.yml renamed to .github/workflows/assembly.yml

@@ -1,43 +1,47 @@
-name: weekly-dash-check
+name: assembly
 
 on:
-  workflow_dispatch:
-  schedule:
-    - cron: '20 21 * * 0'
+  push:
+    branches:
+      - main
+      - '[0-9]+.[0-9]+.x'
+    tags:
+      - '**'
+  pull_request:
 
 permissions:  # added using https://github.com/step-security/secure-repo
   contents: read
 
 env:
   JAVA_VERSION: 17
-  MAVEN_CLI_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false --batch-mode
+  MAVEN_CLI_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false --batch-mode -Dlicense.skip=true
+  MAVEN_COMPILE_NO_OP_ARGS: clean install -Dmaven.test.skip -Dmaven.assembly.skip=true -Dmaven.source.skip -Pskip-spark-runtimes -T4 -Dmaven.main.skip
+  MAVEN_ASSEMBLY_ARGS: assembly:single@make-assembly -Dassembly.ignoreMissingDescriptor=true -T2
 
 jobs:
-  dash:
+  assembly:
+    name: build-assembly / ${{ matrix.scala-version }}
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:
-        version: [ "2.12", "2.13" ]
+        scala-version: [ "2.12", "2.13" ]
     steps:
       - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
       - uses: actions/setup-java@dded0888837ed1f317902acf8a20df0ad188d165 # v5.0.0
         with:
           distribution: temurin
           java-version: "${{ env.JAVA_VERSION }}"
-      - uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
+      - uses: actions/cache@0400d5f644dc74513175e3cd8d07132dd4860809 # v4.2.4
         with:
-          key: ${{ hashFiles('**/pom.xml') }}-dash-scheduled-${{ matrix.version }}
+          key: ${{ hashFiles('**/pom.xml') }}-assembly-${{ matrix.scala-version }}
           path: ~/.m2/repository/
       - name: Set Scala version
-        run: ./build/scripts/change-scala-version.sh ${{ matrix.version }}
+        run: ./build/scripts/change-scala-version.sh ${{ matrix.scala-version }}
       - name: Build with Maven
-        run: mvn clean install -Dmaven.assembly.skip=true -Dmaven.source.skip -T4 $MAVEN_CLI_OPTS
-      - name: License check
-        run: mvn org.eclipse.dash:license-tool-plugin:license-check -Ddash.fail=true -Ddash.projectId=locationtech.geomesa -DexcludeGroupIds=org.locationtech.geomesa $MAVEN_CLI_OPTS
-      - name: Show license
-        if: success() || failure()
-        run: cat target/dash/summary
+        run: mvn $MAVEN_COMPILE_NO_OP_ARGS $MAVEN_CLI_OPTS
+      - name: Build assemblies
+        run: mvn $MAVEN_ASSEMBLY_ARGS $MAVEN_CLI_OPTS
       - name: Remove geomesa artifacts
         if: success() || failure()
         run: rm -rf ~/.m2/repository/org/locationtech/geomesa

.github/workflows/build-and-test.yml

@@ -0,0 +1,81 @@
+name: dash
+
+on:
+  pull_request:
+  workflow_dispatch:
+  schedule:
+    - cron: '20 21 * * 0'
+
+permissions:  # added using https://github.com/step-security/secure-repo
+  contents: read
+
+env:
+  JAVA_VERSION: 17
+  MAVEN_CLI_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false --batch-mode -Dlicense.skip=true
+  MAVEN_COMPILE_NO_OP_ARGS: clean install -Dmaven.test.skip -Dmaven.assembly.skip=true -Dmaven.source.skip -Pskip-spark-runtimes -T4 -Dmaven.main.skip
+  MAVEN_DASH_ARGS: org.eclipse.dash:license-tool-plugin:license-check -Ddash.fail=true -Ddash.projectId=locationtech.geomesa -DexcludeGroupIds=org.locationtech.geomesa
+
+jobs:
+  license-check:
+    name: license-check
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
+      - uses: actions/setup-java@dded0888837ed1f317902acf8a20df0ad188d165 # v5.0.0
+        with:
+          distribution: temurin
+          java-version: "${{ env.JAVA_VERSION }}"
+      - uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
+        with:
+          key: ${{ hashFiles('**/pom.xml') }}-dash
+          path: ~/.m2/repository/
+      - name: Check for dependency changes
+        id: dependency_changes
+        run: |
+          changed="false"
+          if [[ "${{ github.event_name }}" == 'pull_request' ]]; then
+            # get the changed files
+            git fetch --depth=1 origin ${{ github.base_ref }}
+            IFS=$'\n' read -r -d '' -a changedFiles < <( git diff --name-only origin/${{ github.base_ref }} && printf '\0' )
+            echo "Changed files:"
+            for file in "${changedFiles[@]}"; do
+              echo "  $file"
+              if [[ -z "$changed" ]] && [[ $file =~ .*pom.xml ]]; then
+                changed="pom"
+              elif [[ "$file" == "build/dependencies.txt" ]]; then
+                changed="true"
+              fi
+            done
+            if [[ "$changed" == "pom" ]]; then
+              mvn $MAVEN_COMPILE_NO_OP_ARGS $MAVEN_CLI_OPTS
+              ./build/scripts/calculate-cqs.sh
+              git diff --exit-code --quiet || {
+                echo 'Detected dependency changes - please run `./build/scripts/calculate-cqs.sh` and commit the results'
+                exit 1
+              }
+              changed="false"
+            fi
+          else
+            # weekly or dispatch requests
+            changed="true"
+          fi
+          echo "changed=$changed" >> "$GITHUB_OUTPUT"
+      - name: License check
+        if: steps.dependency_changes.outputs.changed == 'true'
+        run: |
+          mvn $MAVEN_COMPILE_NO_OP_ARGS $MAVEN_CLI_OPTS
+          mvn $MAVEN_DASH_ARGS $MAVEN_CLI_OPTS
+      - name: Show license
+        if: success() || failure()
+        run: |
+          if [[ -f target/dash/summary ]]; then
+            cat target/dash/summary
+          else
+            echo "No dependency changes detected - skipping license check"
+          fi
+      - name: Remove geomesa artifacts
+        if: success() || failure()
+        run: |
+          if [[ -d ~/.m2/repository/org/locationtech/geomesa ]]; then
+            rm -rf ~/.m2/repository/org/locationtech/geomesa
+          fi

.github/workflows/dash.yml

@@ -0,0 +1,103 @@
+name: javadocs
+
+on:
+  push:
+    branches:
+      - main
+      - '[0-9]+.[0-9]+.x'
+    tags:
+      - '**'
+  pull_request:
+
+permissions:  # added using https://github.com/step-security/secure-repo
+  contents: read
+
+env:
+  JAVA_VERSION: 17
+  MAVEN_CLI_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false --batch-mode -Dlicense.skip=true
+  MAVEN_COMPILE_ARGS: clean install -Dmaven.test.skip -Dmaven.assembly.skip=true -Dmaven.source.skip -Pskip-spark-runtimes -T4
+  MAVEN_SCALADOC_ARGS: generate-sources scala:doc-jar
+  MAVEN_JAVADOC_ARGS: generate-sources javadoc:jar -Psite
+
+jobs:
+  javadocs:
+    name: build-javadocs / ${{ matrix.projects.name }} / ${{ matrix.scala-version }}
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        scala-version: [ "2.12", "2.13" ]
+        projects:
+          - name: accumulo
+            list: geomesa-accumulo/geomesa-accumulo-datastore,geomesa-accumulo/geomesa-accumulo-dist,geomesa-accumulo/geomesa-accumulo-distributed-runtime,geomesa-accumulo/geomesa-accumulo-gs-plugin,geomesa-accumulo/geomesa-accumulo-indices,geomesa-accumulo/geomesa-accumulo-iterators,geomesa-accumulo/geomesa-accumulo-jobs,geomesa-accumulo/geomesa-accumulo-spark,geomesa-accumulo/geomesa-accumulo-tools
+          - name: arrow
+            list: geomesa-arrow/geomesa-arrow-datastore,geomesa-arrow/geomesa-arrow-dist,geomesa-arrow/geomesa-arrow-gt,geomesa-arrow/geomesa-arrow-jts,geomesa-arrow/geomesa-arrow-tools
+          - name: cassandra
+            list: geomesa-cassandra/geomesa-cassandra-datastore,geomesa-cassandra/geomesa-cassandra-dist,geomesa-cassandra/geomesa-cassandra-gs-plugin,geomesa-cassandra/geomesa-cassandra-tools
+          - name: convert
+            list: geomesa-convert/geomesa-convert-all,geomesa-convert/geomesa-convert-avro,geomesa-convert/geomesa-convert-avro-schema-registry,geomesa-convert/geomesa-convert-common,geomesa-convert/geomesa-convert-fixedwidth,geomesa-convert/geomesa-convert-jdbc,geomesa-convert/geomesa-convert-json,geomesa-convert/geomesa-convert-parquet,geomesa-convert/geomesa-convert-redis-cache,geomesa-convert/geomesa-convert-shp,geomesa-convert/geomesa-convert-simplefeature,geomesa-convert/geomesa-convert-text,geomesa-convert/geomesa-convert-xml
+          - name: features
+            list: geomesa-features/geomesa-feature-all,geomesa-features/geomesa-feature-avro,geomesa-features/geomesa-feature-common,geomesa-features/geomesa-feature-exporters,geomesa-features/geomesa-feature-kryo
+          - name: fs
+            list: geomesa-fs/geomesa-fs-datastore,geomesa-fs/geomesa-fs-dist,geomesa-fs/geomesa-fs-gs-plugin,geomesa-fs/geomesa-fs-spark,geomesa-fs/geomesa-fs-tools
+          - name: fs-storage
+            list: geomesa-fs/geomesa-fs-storage/geomesa-fs-storage-api,geomesa-fs/geomesa-fs-storage/geomesa-fs-storage-common,geomesa-fs/geomesa-fs-storage/geomesa-fs-storage-convert,geomesa-fs/geomesa-fs-storage/geomesa-fs-storage-orc,geomesa-fs/geomesa-fs-storage/geomesa-fs-storage-orc-io,geomesa-fs/geomesa-fs-storage/geomesa-fs-storage-parquet,geomesa-fs/geomesa-fs-storage/geomesa-fs-storage-parquet-io
+          - name: gt
+            list: geomesa-gt/geomesa-gt-dist,geomesa-gt/geomesa-gt-gs-plugin,geomesa-gt/geomesa-gt-partitioning,geomesa-gt/geomesa-gt-spark,geomesa-gt/geomesa-gt-tools
+          - name: hbase
+            list: geomesa-hbase/geomesa-hbase-datastore,geomesa-hbase/geomesa-hbase-dist,geomesa-hbase/geomesa-hbase-distributed-runtime-hbase2,geomesa-hbase/geomesa-hbase-gs-plugin,geomesa-hbase/geomesa-hbase-jobs,geomesa-hbase/geomesa-hbase-rpc,geomesa-hbase/geomesa-hbase-server,geomesa-hbase/geomesa-hbase-server-hbase2,geomesa-hbase/geomesa-hbase-spark,geomesa-hbase/geomesa-hbase-tools
+          - name: kafka
+            list: geomesa-kafka/geomesa-kafka-confluent,geomesa-kafka/geomesa-kafka-datastore,geomesa-kafka/geomesa-kafka-dist,geomesa-kafka/geomesa-kafka-gs-plugin,geomesa-kafka/geomesa-kafka-tools,geomesa-kafka/geomesa-kafka-utils
+          - name: lambda
+            list: geomesa-lambda/geomesa-lambda-datastore,geomesa-lambda/geomesa-lambda-dist,geomesa-lambda/geomesa-lambda-gs-plugin,geomesa-lambda/geomesa-lambda-tools
+          - name: misc-1
+            list: geomesa-memory/geomesa-cqengine,geomesa-memory/geomesa-cqengine-datastore,geomesa-metrics/geomesa-metrics-micrometer,geomesa-process/geomesa-process-vector
+          - name: misc-2
+            list: geomesa-filter,geomesa-index-api,geomesa-jobs,geomesa-security,geomesa-tools,geomesa-z3
+          - name: redis
+            list: geomesa-redis/geomesa-redis-datastore,geomesa-redis/geomesa-redis-dist,geomesa-redis/geomesa-redis-gs-plugin,geomesa-redis/geomesa-redis-tools
+          - name: spark
+            list: geomesa-spark/geomesa_pyspark,geomesa-spark/geomesa-spark-converter,geomesa-spark/geomesa-spark-core,geomesa-spark/geomesa-spark-jts,geomesa-spark/geomesa-spark-jupyter-leaflet,geomesa-spark/geomesa-spark-sql,geomesa-spark/geomesa-spark-test,geomesa-spark/geomesa-spark-test-with-sedona
+          - name: utils
+            list: geomesa-utils-parent/geomesa-bom,geomesa-utils-parent/geomesa-hadoop-utils,geomesa-utils-parent/geomesa-utils
+    steps:
+      - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
+      - uses: actions/setup-java@dded0888837ed1f317902acf8a20df0ad188d165 # v5.0.0
+        with:
+          distribution: temurin
+          java-version: "${{ env.JAVA_VERSION }}"
+      - name: Generate cache key
+        run: echo "MOD_SELECTOR=$(echo '${{ matrix.projects.list }}' | sed 's/[^a-zA-Z0-9_.-]/-/g' | head -c 256)" >> "$GITHUB_ENV"
+      - uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
+        with:
+          key: ${{ hashFiles('**/pom.xml') }}-javadocs-${{ matrix.scala-version }}-${{ env.MOD_SELECTOR }}
+          path: ~/.m2/repository/
+      - name: Set Scala version
+        run: ./build/scripts/change-scala-version.sh ${{ matrix.scala-version }}
+      - name: Build with Maven
+        run: mvn $MAVEN_COMPILE_ARGS $MAVEN_CLI_OPTS -am -pl ${{ matrix.projects.list }},geomesa-arrow/geomesa-arrow-jts
+      - name: Build scaladocs
+        run: mvn $MAVEN_SCALADOC_ARGS $MAVEN_CLI_OPTS -pl ${{ matrix.projects.list }}
+      - name: Build javadocs
+        run: mvn $MAVEN_JAVADOC_ARGS $MAVEN_CLI_OPTS -pl geomesa-arrow/geomesa-arrow-jts
+      - name: Remove geomesa artifacts
+        if: success() || failure()
+        run: rm -rf ~/.m2/repository/org/locationtech/geomesa
+  validate-ci:
+    # validates that we haven't missed any projects in our build matrix
+    name: validate-ci
+    runs-on: ubuntu-latest
+    if: github.event_name == 'pull_request'
+    steps:
+      - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
+      - name: Validate project list
+        run: |
+          while IFS= read -r -d '' pom; do
+            grep -q 'packaging>pom' "$pom" || {
+              module="${pom%/pom.xml}"
+              module="${module#./}"
+              echo "Checking $module"
+              yq -e '.jobs.javadocs.strategy.matrix.projects[].list | select(test("(^|[, ])'"${module}"'($|[, ])"))' .github/workflows/javadocs.yml >/dev/null 2>&1 || exit 1
+            }
+          done < <(find . -name pom.xml -print0)
+          echo "done"

.github/workflows/javadocs.yml

@@ -49,25 +49,19 @@ geospatial analytics.
 
 ### Verifying Downloads
 
-Downloads hosted on GitHub include SHA-256 hashes and gpg signatures (.asc files). To verify a download using gpg,
-import the appropriate key:
+Downloads hosted on GitHub include SHA-256 hashes and `gpg` signatures (`.asc` files). To verify a download using `gpg`,
+import the appropriate key then verify:
 
 ```bash
-gpg2 --keyserver hkp://pool.sks-keyservers.net --recv-keys CD24F317
+gpg --keyserver hkp://pool.sks-keyservers.net --recv-keys CD24F317
+gpg --verify geomesa-accumulo_2.12-5.4.0-bin.tar.gz.asc geomesa-accumulo_2.12-5.4.0-bin.tar.gz
 ```
 
-Then verify the file:
-
-```bash
-gpg2 --verify geomesa-accumulo_2.12-5.4.0-bin.tar.gz.asc geomesa-accumulo_2.12-5.4.0-bin.tar.gz
+The keys used for signing are:
+```
+CD24F317 Emilio Lahr-Vivaz <elahrvivaz(-at-)ccri.com>
+1E679A56 James Hughes <jnh5y(-at-)ccri.com>
 ```
-
-The keys currently used for signing are:
-
-| Key ID     | Name                                               |
-|------------|----------------------------------------------------|
-| `CD24F317` | Emilio Lahr-Vivaz &lt;elahrvivaz(-at-)ccri.com&gt; |
-| `1E679A56` | James Hughes &lt;jnh5y(-at-)ccri.com&gt;           |
 
 ## Maven Integration
 
@@ -144,11 +138,11 @@ DataStore implementations:
 ```xml
 <dependency>
   <groupId>org.locationtech.geomesa</groupId>
-  <artifactId>geomesa-gt-spark-runtime_2.12</artifactId>
+  <artifactId>geomesa-gt-spark_2.12</artifactId>
+  <classifier>runtime</classifier>
   <version>5.4.0</version>
   <exclusions>
     <exclusion>
-      <!-- if groupId wildcards are not supported, the two main ones are jline:* and org.geotools:* -->
       <groupId>*</groupId>
       <artifactId>*</artifactId>
     </exclusion>