Merge branch 'task/replace-nonpublic-interruptableiterator' of github…

….com:NationalSecurityAgency/datawave into task/replace-nonpublic-interruptableiterator

.github/workflows/build-accumulo.yml

@@ -0,0 +1,99 @@
+name: Build Accumulo snapshot and update DataWave to use
+
+on:
+  workflow_dispatch:
+    inputs:
+      accumuloBranch:
+        required: true
+        default: "2.1"
+        description: "Branch name to build. Will be used as image tag."
+      accumuloRepo:
+        required: true
+        default: "apache/accumulo"
+        description: "Accumulo Repo to use. Expected to be at Github. Example: apache/accumulo"
+      deployAccumulo:
+        required: true
+        default: "false"
+        description: "Set to false if this accumulo version has already been pushed to Github Packages"
+
+# Defines two custom environment variables for the workflow. These are used for the Container registry domain, and a name for the Docker image that this workflow builds.
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository.lowercase }}
+  ACCUMULO_JAVA_VERSION: '17'
+  DATAWAVE_JAVA_VERSION: '11'
+  JAVA_DISTRIBUTION: 'zulu' #This is the default on v1 of the action for 1.8
+  USER_NAME: ${{ secrets.GHCR_WRITE_USER_NAME }}
+  ACCESS_TOKEN: ${{ secrets.GHCR_WRITE_ACCESS_TOKEN }}
+
+jobs:
+  build-and-deploy-accumulo:
+    runs-on: ubuntu-latest
+    # Sets the permissions granted to the `GITHUB_TOKEN` for the actions in this job.
+    permissions:
+      contents: read
+      packages: write
+      #
+    steps:
+    - name: Checkout DataWave
+      uses: actions/checkout@v4
+      with:
+        path: datawave
+
+    - name: Checkout Accumulo
+      uses: actions/checkout@v4
+      id: accumuloCheckout
+      with:
+        repository: ${{ github.event.inputs.accumuloRepo }}
+        path: accumulo
+        ref: ${{ github.event.inputs.accumuloBranch }}
+
+    - name: Set up JDK ${{env.ACCUMULO_JAVA_VERSION}}
+      uses: actions/setup-java@v4
+      with:
+        distribution: ${{env.JAVA_DISTRIBUTION}}
+        java-version: ${{env.ACCUMULO_JAVA_VERSION}}
+        cache: 'maven'
+    - run: echo "ACCUMULO_JAVA=$JAVA_HOME" >> $GITHUB_ENV
+    - name: Set up JDK ${{env.DATAWAVE_JAVA_VERSION}}
+      uses: actions/setup-java@v4
+      with:
+        distribution: ${{env.JAVA_DISTRIBUTION}}
+        java-version: ${{env.DATAWAVE_JAVA_VERSION}}
+        cache: 'maven'
+    - run: echo "DATAWAVE_JAVA=$JAVA_HOME" >> $GITHUB_ENV
+
+    - name: Get Accumulo Version
+      id: get-accumulo-version
+      run: |
+        export JAVA_HOME="$ACCUMULO_JAVA"
+        cd "$GITHUB_WORKSPACE/accumulo"
+        mvn build-helper:parse-version versions:set -DgenerateBackupPoms=false -DnewVersion=\${parsedVersion.majorVersion}.\${parsedVersion.minorVersion}.\${parsedVersion.incrementalVersion}-dwv-$(git rev-parse --short HEAD)
+        export newVersion=$(mvn -q help:evaluate -DforceStdout -Dexpression=project.version)
+        echo accumuloVersion=$newVersion >> $GITHUB_OUTPUT
+    - name: Deploy Accumulo
+      if: ${{ github.event.inputs.deployAccumulo == 'true'}}
+      run: | 
+        export JAVA_HOME="$ACCUMULO_JAVA"
+        cd "$GITHUB_WORKSPACE/accumulo"
+        mvn -DaltDeploymentRepository=github-datawave::https://maven.pkg.github.com/NationalSecurityAgency/datawave -V -B -e -ntp "-Dstyle.color=always" -DskipTests -T1C clean source:jar deploy -s "$GITHUB_WORKSPACE/datawave/.github/workflows/settings.xml"
+    - name: Log in to the Container registry
+      uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1
+      with:
+        registry: ${{ env.REGISTRY }}
+        username: ${{ env.USER_NAME }}
+        password: ${{ env.ACCESS_TOKEN }}
+
+    - name: Update DataWave Dependency Version
+      run: |
+
+        export JAVA_HOME="$DATAWAVE_JAVA"
+        cd "$GITHUB_WORKSPACE/datawave"
+        mvn -s "$GITHUB_WORKSPACE/datawave/.github/workflows/settings.xml" versions:set-property -Dproperty=version.accumulo -DnewVersion=${{ steps.get-accumulo-version.outputs.accumuloVersion }} -DgenerateBackupPoms=false
+    - name: Build Web and Ingest Docker Images (Maven)
+      run: |
+        export JAVA_HOME="$DATAWAVE_JAVA"
+        cd "$GITHUB_WORKSPACE/datawave"
+        mvn -s "$GITHUB_WORKSPACE/datawave/.github/workflows/settings.xml" clean install -Prpm,kubernetes,assemble,deploy-ws -Ddist -Pdocker -DpushImage -Ddocker-release -DskipTests -Ddocker.image.accumulo.tag=${{ steps.get-accumulo-version.outputs.accumuloVersion }}
+
+ 

.github/workflows/microservice-build-image.yaml

@@ -0,0 +1,41 @@
+#
+name: Create and publish a Docker image
+
+on:
+  workflow_call:
+    secrets:
+      USER_NAME:
+        description: "User Name for maven pulls"
+        required: true
+      ACCESS_TOKEN:
+        description: "Access token for maven pulls"
+        required: true
+
+
+jobs:
+  build-and-push-datawave-images:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Log in to the Container registry
+      uses: docker/login-action@v3
+      with:
+        registry: ghcr.io
+        username: ${{ secrets.USER_NAME }}
+        password: ${{ secrets.ACCESS_TOKEN }}
+    - name: Checkout Code
+      uses: actions/checkout@v4
+    - name: Set up JDK 11
+      uses: actions/setup-java@v4
+      with:
+        distribution: "zulu"
+        java-version: 11
+        cache: 'maven'
+    - name: Build And Push Docker Image (Maven)
+      env:
+        MAVEN_OPTS: "-Dhttps.protocols=TLSv1.2 -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=WARN -Djava.awt.headless=true"
+        USER_NAME: ${{ secrets.USER_NAME }}
+        ACCESS_TOKEN: ${{ secrets.ACCESS_TOKEN }}
+      run: |
+        mvn -s $GITHUB_WORKSPACE/.github/workflows/settings.xml -V -B -e clean install -Pdocker,exec -Ddocker.image.prefix=ghcr.io/nationalsecurityagency/ -DpushImage
+
+ 

.github/workflows/microservice-maven-tests.yaml

@@ -0,0 +1,72 @@
+name: Tests
+
+on:
+  workflow_call:
+    secrets:
+      USER_NAME:
+        description: "User Name for maven pulls"
+        required: true
+      ACCESS_TOKEN:
+        description: "Access token for maven pulls"
+        required: true
+
+env:
+  MAVEN_OPTS: "-Djansi.force=true -Dhttps.protocols=TLSv1.2 -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=WARN -Djava.awt.headless=true -XX:ThreadStackSize=1m"
+
+jobs:
+  # Runs the pom sorter and code formatter to ensure that the code
+  # is formatted and poms are sorted according to project rules. This
+  # will fail if the formatter makes any changes.
+  check-code-formatting:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout Code
+      uses: actions/checkout@v4
+    - name: Set up JDK 11
+      uses: actions/setup-java@v4
+      with:
+        java-version: 11
+        distribution: 'zulu'
+    - uses: actions/cache@v4
+      with:
+        path: ~/.m2/repository
+        key: ${{ runner.os }}-maven-format-${{ hashFiles('**/pom.xml') }}
+        restore-keys: |
+          ${{ runner.os }}-maven-format-
+          ${{ runner.os }}-maven-
+    - name: Format code
+      run: |
+        mvn -s $GITHUB_WORKSPACE/.github/workflows/settings.xml -V -B -e clean formatter:format sortpom:sort -Pautoformat
+        git status
+        git diff-index --quiet HEAD || (echo "Error! There are modified files after formatting." && false)
+      env:
+        MAVEN_OPTS: "-Dhttps.protocols=TLSv1.2 -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=WARN -Djava.awt.headless=true"
+        USER_NAME: ${{ secrets.USER_NAME }}
+        ACCESS_TOKEN: ${{ secrets.ACCESS_TOKEN }}
+
+  # Build the code and run the unit/integration tests.
+  build-and-test:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout Code
+      uses: actions/checkout@v4
+    - name: Set up JDK 11
+      uses: actions/setup-java@v4
+      with:
+        java-version: 11
+        distribution: 'zulu'
+    - uses: actions/cache@v4
+      with:
+        path: ~/.m2/repository
+        key: ${{ runner.os }}-maven-build-${{ hashFiles('**/pom.xml') }}
+        restore-keys: |
+          ${{ runner.os }}-maven-build-
+          ${{ runner.os }}-maven-format-
+          ${{ runner.os }}-maven-
+    - name: Build and Run Unit Tests
+      run: mvn -s $GITHUB_WORKSPACE/.github/workflows/settings.xml -V -B -e -Ddist clean verify
+      env:
+        MAVEN_OPTS: "-Dhttps.protocols=TLSv1.2 -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=WARN -Djava.awt.headless=true"
+        USER_NAME: ${{ secrets.USER_NAME }}
+        ACCESS_TOKEN: ${{ secrets.ACCESS_TOKEN }}
+

.github/workflows/tests.yml

@@ -7,7 +7,7 @@ on:
     - 'integration'
     - 'release/version*'
   pull_request:
-    paths-ignore: ['*.md', 'CODEOWNERS', 'LICENSE']
+    paths-ignore: ['*.md', 'CODEOWNERS', 'LICENSE', '.github/workflows/microservice*.yaml']
   workflow_dispatch:
 
 env:

.gitmodules

@@ -85,3 +85,6 @@
 [submodule "microservices/services/map"]
 	path = microservices/services/map
 	url = [email protected]:NationalSecurityAgency/datawave-map-service.git
+[submodule "microservices/services/file-provider"]
+	path = microservices/services/file-provider
+	url = [email protected]:NationalSecurityAgency/datawave-file-provider-service.git

common-test/pom.xml

@@ -4,7 +4,7 @@
     <parent>
         <groupId>gov.nsa.datawave</groupId>
         <artifactId>datawave-parent</artifactId>
-        <version>7.8.0-SNAPSHOT</version>
+        <version>7.10.0-SNAPSHOT</version>
     </parent>
     <artifactId>datawave-common-test</artifactId>
     <name>${project.artifactId}</name>

contrib/datawave-quickstart/bin/query.sh

@@ -131,21 +131,8 @@ function setQueryIdFromResponse() {
 }
 
 function prettyPrintJson() {
-    local PY=$( which python )
-    if [ -n "${PY}" ] ; then
-        echo "${1}" | ${PY} -c 'from __future__ import print_function;import sys,json;data=json.loads(sys.stdin.read()); print(json.dumps(data, indent=2, sort_keys=True))'
-        local exitStatus=$?
-        echo
-        if [ "${exitStatus}" != "0" ] ; then
-           printRawResponse "${1}"
-           warn "Python encountered error. Printed response without formatting"
-           echo
-        fi
-    else
-        printRawResponse "${1}"
-        warn "Couldn't find python in your environment. Json response was printed without formatting"
-        echo
-    fi
+    PY_CMD='from __future__ import print_function; import sys,json; data=json.loads(sys.stdin.read()); print(json.dumps(data, indent=2, sort_keys=True))'
+    echo "${1}" | ( python3 -c "${PY_CMD}" 2>/dev/null || python2 -c "${PY_CMD}" 2>/dev/null ) || ( warn "Python encountered error. Printed response without formatting" && printRawResponse "${1}" )
 }
 
 function printRawResponse() {

contrib/datawave-quickstart/bin/services/datawave/ingest-examples/tvmaze-api-query.sh

@@ -38,10 +38,11 @@ TVMAZE_RESPONSE_STATUS=$( echo ${CURL_RESPONSE} | tr -d '\n' | sed -e 's/.*HTTP_
 [ "${TVMAZE_RESPONSE_STATUS}" != "200" ] && error "api.tvmaze.com returned invalid response status: ${TVMAZE_RESPONSE_STATUS}" && exit 1
 [ -z "${TVMAZE_RESPONSE_BODY}" ] && error "Response body is empty!" && exit 1
 
+PY_CMD='from __future__ import print_function; import sys,json; data=json.loads(sys.stdin.read()); print(json.dumps(data, indent=2, sort_keys=True))'
 if [ "${PRETTY}" == true ] ; then
-    echo "${TVMAZE_RESPONSE_BODY}" | python -c 'from __future__ import print_function;import sys,json;data=json.loads(sys.stdin.read()); print(json.dumps(data, indent=2, sort_keys=True))'
+  echo "${TVMAZE_RESPONSE_BODY}" | ( python3 -c "${PY_CMD}" 2>/dev/null || python2 -c "${PY_CMD}" 2>/dev/null ) || ( warn "Unable to pretty print, Python not detected" && echo "${TVMAZE_RESPONSE_BODY}" )
 else
-    echo "${TVMAZE_RESPONSE_BODY}"
+  echo "${TVMAZE_RESPONSE_BODY}"
 fi
 
 exit 0

contrib/datawave-quickstart/docker/Dockerfile

@@ -35,7 +35,7 @@ COPY . /opt/datawave
 
 # Install dependencies, configure password-less/zero-prompt SSH...
 
-RUN dnf -y install gcc-c++ openssl openssh openssh-server openssh-clients openssl-libs which bc wget git java-11-openjdk-devel iproute  && \
+RUN dnf -y install gcc-c++ openssl python3 openssh openssh-server openssh-clients openssl-libs which bc wget git java-11-openjdk-devel iproute  && \
     dnf clean all && \
     ssh-keygen -q -N "" -t rsa -f ~/.ssh/id_rsa && \
     cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys && \

contrib/datawave-quickstart/docker/pom.xml

@@ -4,7 +4,7 @@
     <parent>
         <groupId>gov.nsa.datawave</groupId>
         <artifactId>datawave-parent</artifactId>
-        <version>7.8.0-SNAPSHOT</version>
+        <version>7.10.0-SNAPSHOT</version>
         <relativePath>../../../pom.xml</relativePath>
     </parent>
     <artifactId>quickstart</artifactId>

core/cached-results/pom.xml

@@ -4,7 +4,7 @@
     <parent>
         <groupId>gov.nsa.datawave.core</groupId>
         <artifactId>datawave-core-parent</artifactId>
-        <version>7.8.0-SNAPSHOT</version>
+        <version>7.10.0-SNAPSHOT</version>
     </parent>
     <artifactId>datawave-core-cached-results</artifactId>
     <name>${project.artifactId}</name>

core/common-util/pom.xml

@@ -4,7 +4,7 @@
     <parent>
         <groupId>gov.nsa.datawave.core</groupId>
         <artifactId>datawave-core-parent</artifactId>
-        <version>7.8.0-SNAPSHOT</version>
+        <version>7.10.0-SNAPSHOT</version>
     </parent>
     <artifactId>datawave-core-common-util</artifactId>
     <name>${project.artifactId}</name>

core/common/pom.xml

@@ -4,7 +4,7 @@
     <parent>
         <groupId>gov.nsa.datawave.core</groupId>
         <artifactId>datawave-core-parent</artifactId>
-        <version>7.8.0-SNAPSHOT</version>
+        <version>7.10.0-SNAPSHOT</version>
     </parent>
     <artifactId>datawave-core-common</artifactId>
     <name>${project.artifactId}</name>

core/connection-pool/pom.xml

@@ -4,7 +4,7 @@
     <parent>
         <groupId>gov.nsa.datawave.core</groupId>
         <artifactId>datawave-core-parent</artifactId>
-        <version>7.8.0-SNAPSHOT</version>
+        <version>7.10.0-SNAPSHOT</version>
     </parent>
     <artifactId>datawave-core-connection-pool</artifactId>
     <name>${project.artifactId}</name>

core/map-reduce/pom.xml

@@ -4,7 +4,7 @@
     <parent>
         <groupId>gov.nsa.datawave.core</groupId>
         <artifactId>datawave-core-parent</artifactId>
-        <version>7.8.0-SNAPSHOT</version>
+        <version>7.10.0-SNAPSHOT</version>
     </parent>
     <artifactId>datawave-core-map-reduce</artifactId>
     <name>${project.artifactId}</name>

core/modification/pom.xml

@@ -4,7 +4,7 @@
     <parent>
         <groupId>gov.nsa.datawave.core</groupId>
         <artifactId>datawave-core-parent</artifactId>
-        <version>7.8.0-SNAPSHOT</version>
+        <version>7.10.0-SNAPSHOT</version>
     </parent>
     <artifactId>datawave-core-modification</artifactId>
     <name>${project.artifactId}</name>

core/pom.xml

@@ -4,7 +4,7 @@
     <parent>
         <groupId>gov.nsa.datawave</groupId>
         <artifactId>datawave-parent</artifactId>
-        <version>7.8.0-SNAPSHOT</version>
+        <version>7.10.0-SNAPSHOT</version>
     </parent>
     <groupId>gov.nsa.datawave.core</groupId>
     <artifactId>datawave-core-parent</artifactId>

core/query/pom.xml

@@ -4,7 +4,7 @@
     <parent>
         <groupId>gov.nsa.datawave.core</groupId>
         <artifactId>datawave-core-parent</artifactId>
-        <version>7.8.0-SNAPSHOT</version>
+        <version>7.10.0-SNAPSHOT</version>
     </parent>
     <artifactId>datawave-core-query</artifactId>
     <name>${project.artifactId}</name>

core/query/src/main/java/datawave/core/query/configuration/GenericQueryConfiguration.java

@@ -77,6 +77,9 @@ public class GenericQueryConfiguration implements Serializable {
 
     // either IMMEDIATE or EVENTUAL
     private Map<String,ScannerBase.ConsistencyLevel> tableConsistencyLevels = new HashMap<>();
+    // provides default scan hints
+    // NOTE: accumulo reserves the execution hint name 'meta'
+    // NOTE: datawave reserves the execution hint name 'expansion' for index expansion
     private Map<String,Map<String,String>> tableHints = new HashMap<>();
 
     /**