Adds a new shard table balancer #14645
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Tests | |
| on: | |
| push: | |
| paths-ignore: ['*.md', 'CODEOWNERS', 'LICENSE'] | |
| branches: | |
| - 'integration' | |
| - 'release/version*' | |
| pull_request: | |
| paths-ignore: ['*.md', 'CODEOWNERS', 'LICENSE'] | |
| merge_group: | |
| workflow_dispatch: | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} | |
| cancel-in-progress: true | |
| env: | |
| JAVA_VERSION: '11' | |
| JAVA_DISTRIBUTION: 'zulu' #This is the default on v1 of the action for 1.8 | |
| MAVEN_OPTS: "-Djansi.force=true -Dhttps.protocols=TLSv1.2 -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=WARN -Djava.awt.headless=true -XX:ThreadStackSize=1m" | |
| REGISTRY: ghcr.io | |
| USER_NAME: ${{ secrets.GHCR_WRITE_USER_NAME }} | |
| ACCESS_TOKEN: ${{ secrets.GHCR_WRITE_ACCESS_TOKEN }} | |
| jobs: | |
| # Runs the pom sorter and code formatter to ensure that the code | |
| # is formatted and poms are sorted according to project rules. If changes are found, | |
| # they are committed back to the branch | |
| check-code-formatting: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout Code | |
| uses: actions/checkout@v4 | |
| - name: Set up JDK ${{env.JAVA_VERSION}} | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: ${{env.JAVA_DISTRIBUTION}} | |
| java-version: ${{env.JAVA_VERSION}} | |
| cache: 'maven' | |
| - name: Extract branch name | |
| shell: bash | |
| run: | | |
| if [[ "${{ github.event_name }}" != "merge_group" ]]; then | |
| echo "branch=${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}" >> $GITHUB_OUTPUT | |
| else | |
| echo "Nothing to do" | |
| fi | |
| id: extract_branch | |
| - name: Format code | |
| env: | |
| USER_NAME: ${{ secrets.USER_NAME }} | |
| ACCESS_TOKEN: ${{ secrets.ACCESS_TOKEN }} | |
| run: | | |
| if [[ "${{ github.event_name }}" != "merge_group" ]]; then | |
| mvn -s $GITHUB_WORKSPACE/.github/workflows/settings.xml --show-version --batch-mode --errors --no-transfer-progress "-Dstyle.color=always" clean formatter:format sortpom:sort impsort:sort -Dmaven.build.cache.enabled=false -Pautoformat -Dutils -Dservices -Dstarters | |
| if ! git diff-index --quiet HEAD; then | |
| echo "Code formatting issues detected. Please format locally."; | |
| exit 1; | |
| fi | |
| else | |
| echo "Nothing to do" | |
| fi | |
| # Build the code and run the unit/integration tests. | |
| build-and-test: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout Code | |
| uses: actions/checkout@v4 | |
| - name: Set up JDK ${{env.JAVA_VERSION}} | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: ${{env.JAVA_DISTRIBUTION}} | |
| java-version: ${{env.JAVA_VERSION}} | |
| cache: 'maven' | |
| # Clean up free space | |
| - uses: ./.github/actions/free-space | |
| with: | |
| tool-cache: false | |
| - name: Build and Run Unit Tests | |
| env: | |
| USER_NAME: ${{ secrets.USER_NAME }} | |
| ACCESS_TOKEN: ${{ secrets.ACCESS_TOKEN }} | |
| run: | | |
| mvn -s $GITHUB_WORKSPACE/.github/workflows/settings.xml --show-version --batch-mode --errors --no-transfer-progress "-Dstyle.color=always" -Pexamples,assemble,spotbugs -Dmaven.build.cache.enabled=false -Ddeploy -Ddist -T1C clean install -Dsurefire.rerunFailingTestsCount=3 -Dutils -Dservices -Dstarters -DskipITs -DskipFormat | |
| - name: Run Integration Tests | |
| env: | |
| USER_NAME: ${{ secrets.USER_NAME }} | |
| ACCESS_TOKEN: ${{ secrets.ACCESS_TOKEN }} | |
| run: | | |
| mvn -s $GITHUB_WORKSPACE/.github/workflows/settings.xml --show-version --batch-mode --errors --no-transfer-progress "-Dstyle.color=always" -Pexamples -Dmaven.build.cache.enabled=false -Ddeploy -T1C verify -Dfailsafe.rerunFailingTestsCount=3 -Dutils -Dservices -Dstarters -DskipUTs -DskipFormat -DskipSpotbugs -Denforcer.skip=true -Dcheckstyle.skip=true | |
| quickstart-build-and-test: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout Code | |
| uses: actions/checkout@v4 | |
| - name: Set up JDK ${{env.JAVA_VERSION}} | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: ${{env.JAVA_DISTRIBUTION}} | |
| java-version: ${{env.JAVA_VERSION}} | |
| cache: 'maven' | |
| # Allow us to use the "--squash" option below | |
| - name: Turn on Docker experimental features and move Docker data root | |
| run: | | |
| if [[ -f /etc/docker/daemon.json ]]; then | |
| sudo sed -ri 's|\s*}\s*$|, "experimental": true, "data-root": "/mnt/docker" }|' /etc/docker/daemon.json | |
| else | |
| echo $'{\n "experimental": true, "data-root": "/mnt/docker"\n}' | sudo tee /etc/docker/daemon.json | |
| fi | |
| sudo systemctl restart docker | |
| echo Docker Experimental Features: $(docker version -f '{{.Server.Experimental}}') | |
| # Clean up free space | |
| - uses: ./.github/actions/free-space | |
| with: | |
| tool-cache: false | |
| # Builds the quickstart docker image and run the query tests | |
| - name: Quickstart Query Tests | |
| env: | |
| DW_DATAWAVE_BUILD_COMMAND: "mvn -s $GITHUB_WORKSPACE/.github/workflows/settings.xml --show-version --batch-mode --errors --no-transfer-progress -Dstyle.color=always -Dmaven.build.cache.enabled=false -Ddeploy -Dtar -DskipTests clean package" | |
| DOCKER_BUILD_OPTS: "--squash --force-rm" | |
| USER_NAME: ${{ secrets.USER_NAME }} | |
| ACCESS_TOKEN: ${{ secrets.ACCESS_TOKEN }} | |
| run: | | |
| TAG=$(mvn -s $GITHUB_WORKSPACE/.github/workflows/settings.xml -q -N -Dmaven.build.cache.enabled=false -Dexec.executable='echo' -Dexec.args='${project.version}' exec:exec) | |
| contrib/datawave-quickstart/docker/docker-build.sh ${TAG} --docker-opts "${DOCKER_BUILD_OPTS}" | |
| compose-build-and-test-latest-snapshots: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Extract branch name | |
| shell: bash | |
| run: | | |
| echo "branch=${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}" >> $GITHUB_OUTPUT | |
| id: extract_branch | |
| - name: Checkout Code | |
| uses: actions/checkout@v4 | |
| # Clean up free space | |
| - uses: ./.github/actions/free-space | |
| with: | |
| tool-cache: false | |
| - name: Set up JDK ${{env.JAVA_VERSION}} | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: ${{env.JAVA_DISTRIBUTION}} | |
| java-version: ${{env.JAVA_VERSION}} | |
| cache: 'maven' | |
| # Builds the quickstart and microservice docker images and runs a query test | |
| - name: Docker Compose Query Tests | |
| env: | |
| USER_NAME: ${{ secrets.USER_NAME }} | |
| ACCESS_TOKEN: ${{ secrets.ACCESS_TOKEN }} | |
| run: | | |
| if [[ ${{ steps.extract_branch.outputs.branch }} =~ ^release/version* ]] then | |
| echo "Nothing to do since this is a release branch." | |
| else | |
| # update datawave dependencies to use the latest snapshots | |
| mvn -s $GITHUB_WORKSPACE/.github/workflows/settings.xml --show-version --batch-mode --errors -Dutils -Dservices -Dstarters versions:update-properties versions:update-parent -DallowSnapshots=true -Dincludes=gov.nsa.* | |
| mvn -s $GITHUB_WORKSPACE/.github/workflows/settings.xml --show-version --batch-mode --errors -Dutils -Dservices -Dstarters -Pcompose -Dmicroservice-docker -Dquickstart-docker -Ddeploy -Dtar -DskipTests clean install | |
| # free up some space so that we don't run out | |
| docker system prune -f | |
| mvn -s $GITHUB_WORKSPACE/.github/workflows/settings.xml --show-version --batch-mode --errors -Dutils -Dservices -Dstarters -Pcompose -Dmicroservice-docker -Dquickstart-docker -Ddeploy -Dtar -DskipTests clean | |
| cd docker | |
| ./bootstrap.sh | |
| attempt=0 | |
| max_attempts=20 | |
| while [ $attempt -lt $max_attempts ]; do | |
| attempt=$((attempt+1)) | |
| echo "Starting docker compose (Attempt ${attempt}/${max_attempts})" | |
| nohup docker compose up -d --no-recreate < /dev/null > compose.out 2>&1 & | |
| sleep 60s | |
| cat compose.out | |
| # check to see if the query service is running | |
| QUERY="$(docker compose ps --status running --services | grep query || true)" | |
| if [ "$QUERY" == "query" ] ; then | |
| echo "Docker compose started successfully" | |
| break | |
| elif [ $attempt -eq $max_attempts ] ; then | |
| echo "Failed to start docker compose" | |
| exit 1 | |
| fi | |
| done | |
| cd scripts | |
| ./testAll.sh | |
| fi | |
| - name: Dump Logs | |
| if: failure() | |
| run: | | |
| cd docker | |
| docker compose logs | |
| compose-build-and-test: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout Code | |
| uses: actions/checkout@v4 | |
| # Clean up free space | |
| - uses: ./.github/actions/free-space | |
| with: | |
| tool-cache: false | |
| - name: Set up JDK ${{env.JAVA_VERSION}} | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: ${{env.JAVA_DISTRIBUTION}} | |
| java-version: ${{env.JAVA_VERSION}} | |
| - uses: actions/cache@v4 | |
| with: | |
| path: ~/.m2/repository | |
| key: ${{ runner.os }}-maven-build-${{ hashFiles('**/pom.xml') }} | |
| restore-keys: | | |
| ${{ runner.os }}-maven-build- | |
| ${{ runner.os }}-maven-format- | |
| ${{ runner.os }}-maven- | |
| # Builds the quickstart and microservice docker images and runs a query test | |
| - name: Docker Compose Query Tests | |
| env: | |
| USER_NAME: ${{ secrets.USER_NAME }} | |
| ACCESS_TOKEN: ${{ secrets.ACCESS_TOKEN }} | |
| run: | | |
| # set some bogus URLs to trigger dependency download via maven | |
| DIST_URLS="-Durl.zookeeper=https://bogus.apache.org/zookeeper/zookeeper-3.7.2/apache-zookeeper-3.7.2-bin.tar.gz.tar.gz \ | |
| -Durl.accumulo=https://bogus.apache.org/accumulo/2.1.3/accumulo-2.1.3-bin.tar.gz \ | |
| -Durl.wildfly=https://bogus.jboss.org/wildfly/17.0.1.Final/wildfly-17.0.1.Final.tar.gz \ | |
| -Durl.hadoop=https://bogus.apache.org/hadoop/common/hadoop-3.3.6/hadoop-3.3.6.tar.gz \ | |
| -Durl.maven=https://bogus.apache.org/maven/maven-3/3.9.10/binaries/apache-maven-3.9.10-bin.tar.gz" | |
| mvn -s $GITHUB_WORKSPACE/.github/workflows/settings.xml --show-version --batch-mode --errors -Dutils -Dservices -Dstarters -Pcompose -Dmicroservice-docker -Dquickstart-docker -Dquickstart-maven ${DIST_URLS} -Ddeploy -Dtar -DskipTests -Dmaven.build.cache.enabled=false clean install | |
| # free up some space so that we don't run out | |
| docker system prune -f | |
| mvn -s $GITHUB_WORKSPACE/.github/workflows/settings.xml --show-version --batch-mode --errors -Dutils -Dservices -Dstarters -Pcompose -Dmicroservice-docker -Dquickstart-docker -Dquickstart-maven ${DIST_URLS} -Ddeploy -Dtar -DskipTests -Dmaven.build.cache.enabled=false clean | |
| cd docker | |
| ./bootstrap.sh | |
| attempt=0 | |
| max_attempts=20 | |
| while [ $attempt -lt $max_attempts ]; do | |
| attempt=$((attempt+1)) | |
| echo "Starting docker compose (Attempt ${attempt}/${max_attempts})" | |
| nohup docker compose up -d --no-recreate < /dev/null > compose.out 2>&1 & | |
| sleep 60s | |
| cat compose.out | |
| # check to see if the query service is running | |
| QUERY="$(docker compose ps --status running --services | grep query || true)" | |
| if [ "$QUERY" == "query" ] ; then | |
| echo "Docker compose started successfully" | |
| break | |
| elif [ $attempt -eq $max_attempts ] ; then | |
| echo "Failed to start docker compose" | |
| exit 1 | |
| fi | |
| done | |
| cd scripts | |
| ./testAll.sh | |
| - name: Dump Logs | |
| if: failure() | |
| run: | | |
| cd docker | |
| docker compose logs |