Merge branch 'master' of github.com:apache/druid into add_simulation_tests

Author: kfaraz

.github/scripts/create-jacoco-coverage-report.sh

@@ -20,11 +20,6 @@ set -x
 
 echo "GITHUB_BASE_REF: ${GITHUB_BASE_REF}"
 
-if [ "$GITHUB_BASE_REF" == "" ] ;then
-  echo "GITHUB_BASE_REF is not set; skipping this check!"
-  exit 0
-fi
-
 echo "Setting up git remote"
 git remote set-branches --add origin ${GITHUB_BASE_REF}
 git fetch

.github/scripts/run-unit-tests.sh renamed to .github/scripts/run_unit-tests

@@ -24,10 +24,10 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        pattern: [ "A*,G*,R*", "B*,O*,S*,X*,Y*,Z*", "C*,E*", "D*,J*,K*", "F*,H*,U*", "I*,N*,T*", "L*,Q*,W*", "M*,P*,V*,_*,$*"]
+        pattern: [ "C*", "H*,U*,V*", "N*,Q*,S*", "B*,O*,R*", "G*,J*,K*", "F*,L*,M*", "A*,D*,I*,X*,Y*,Z*", "E*,P*,T*,W*"]
     uses: ./.github/workflows/worker.yml
     with:
-      script: .github/scripts/run-unit-tests.sh -Dtest='${{ matrix.pattern }}' -Dmaven.test.failure.ignore=true
+      script: .github/scripts/run_unit-tests -Dtest=!QTest,'${{ matrix.pattern }}' -Dmaven.test.failure.ignore=true
       jdk: 17
       artifact_prefix: "unit-test-reports"
       key: "test-jdk17-[${{ matrix.pattern }}]"
@@ -73,9 +73,17 @@ jobs:
       fail-fast: false
       matrix:
         jdk: [ "11", "21.0.4" ]
-        pattern: [ "A*,G*,R*", "B*,O*,S*,X*,Y*,Z*", "C*,E*", "D*,J*,K*", "F*,H*,U*", "I*,N*,T*", "L*,Q*,W*", "M*,P*,V*"]
+        pattern: [ "C*", "H*,U*,V*", "N*,Q*,S*", "B*,O*,R*", "G*,J*,K*", "F*,L*,M*", "A*,D*,I*,X*,Y*,Z*", "E*,P*,T*,W*"]
     uses: ./.github/workflows/worker.yml
     with:
-      script: .github/scripts/run-unit-tests.sh -Dtest='${{ matrix.pattern }}' -fae
+      script: .github/scripts/run_unit-tests -Dtest='${{ matrix.pattern }}' -fae
       jdk: ${{ matrix.jdk }}
       key: "test-jdk${{ matrix.jdk }}-[${{ matrix.pattern }}]"
+
+  # this will be running in parallel with the ITs later; but until that migration happens - run them in parallel with normal tests
+  run-qtest:
+    uses: ./.github/workflows/worker.yml
+    with:
+      script: .github/scripts/run_unit-tests -Dtest=QTest -fae
+      key: quidem-QTest
+      jdk: 17

.github/workflows/ci.yml

@@ -17,6 +17,10 @@ on:
   schedule:
     - cron: '18 15 * * 4'
 
+concurrency:
+  group: '${{ github.workflow }} @ ${{ github.event.pull_request.head.label || github.head_ref || github.run_id }}' # group workflows only on pull_requests and not on branch commits
+  cancel-in-progress: true
+
 jobs:
   analyze:
     name: Analyze

.github/workflows/codeql.yml

@@ -66,10 +66,8 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        # jdk: [11, 17]
         jdk: [17]
         it: [HighAvailability, MultiStageQuery, Catalog, BatchIndex, MultiStageQueryWithMM, InputSource, InputFormat, Security, Query]
-        #indexer: [indexer, middleManager]
         indexer: [middleManager]
     uses: ./.github/workflows/reusable-revised-its.yml
     if: ${{ needs.changes.outputs.core == 'true' || needs.changes.outputs.common-extensions == 'true' }}

.github/workflows/revised-its.yml

@@ -171,12 +171,11 @@ jobs:
       - name: Run IT
         id: test
         run: |
-          echo "Running mvn -B -ff install -pl '!web-console' -Pdist,bundle-contrib-exts -Pskip-static-checks,skip-tests -Dmaven.javadoc.skip=true -T1C"
+          set -x
           mvn -B -ff install -pl '!web-console' -Pdist,bundle-contrib-exts -Pskip-static-checks,skip-tests -Dmaven.javadoc.skip=true -T1C
           # Note: The above command relies on the correct version of the JARs being installed in the local m2 repository.
           # For any changes, please rebuild it using the command from the previous step (./it.sh ci).
 
-          echo "MAVEN_OPTS='-Xmx2048m' ${MVN} verify -pl integration-tests -P int-tests-config-file ${IT_TEST} ${MAVEN_SKIP} -Dpod.name=${POD_NAME} -Dpod.namespace=${POD_NAMESPACE} -Dbuild.druid.cluster=${BUILD_DRUID_CLUSTER}"
           MAVEN_OPTS='-Xmx2048m' ${MVN} verify -pl integration-tests -P int-tests-config-file ${IT_TEST} ${MAVEN_SKIP} -Dpod.name=${POD_NAME} -Dpod.namespace=${POD_NAMESPACE} -Dbuild.druid.cluster=${BUILD_DRUID_CLUSTER}
 
       - name: Debug on failure

.github/workflows/standard-its.yml

@@ -157,60 +157,16 @@ jobs:
           echo ${{ needs.set-env-var.outputs.DRUID_PREVIOUS_IT_IMAGE_NAME }}
           docker save "${{ needs.set-env-var.outputs.DRUID_PREVIOUS_IT_IMAGE_NAME }}" | gzip > druid-container-jdk${{ matrix.jdk }}-version${{ needs.set-env-var.outputs.DRUID_PREVIOUS_VERSION }}.tar.gz    
 
-  # check if it is a PR and if it is approved. For approved PRs, we do not want to run tests sequentially
-  check-approval:
-    runs-on: ubuntu-latest
-    outputs:
-      approved: ${{ steps.check.outputs.isApproved }}
-    steps:
-      - name: Check if Triggered by a PR
-        id: determine-trigger
-        run: |
-          if [ "${{ github.event_name }}" == "pull_request" ]; then
-            echo "isPR=true" >> $GITHUB_ENV
-          else
-            echo "isPR=false" >> $GITHUB_ENV
-          fi
-
-      - name: Check PR Approval (if applicable)
-        id: check
-        if: ${{ env.isPR == 'true' }}
-        uses: actions/github-script@v7
-        with:
-          script: |
-            const reviews = await github.rest.pulls.listReviews({
-              owner: context.repo.owner,
-              repo: context.repo.repo,
-              pull_number: context.payload.pull_request.number
-            });
-            const approved = reviews.data.some(review => review.state === 'APPROVED');
-            core.setOutput("isApproved", approved);
-
-      - name: Default to Approved for Branch
-        id: default-check
-        if: ${{ env.isPR == 'false' }}
-        run: echo "::set-output name=isApproved::true"
-
-  unit-tests-unapproved:
-    name: "unit tests - PR unapproved"
-    uses: ./.github/workflows/ci.yml
-    needs: [check-approval]
-    if: ${{ needs.check-approval.outputs.approved != 'true' }}
-
-  unit-tests-approved:
-    name: "unit tests - PR approved"
+  unit-tests:
+    name: "unit tests"
     uses: ./.github/workflows/ci.yml
-    needs: [check-approval]
-    if: ${{ needs.check-approval.outputs.approved == 'true' }}
 
   standard-its:
-    needs: [build, unit-tests-unapproved, check-approval]
-    if: ${{ !cancelled() && (needs.check-approval.outputs.approved == 'true' || needs.unit-tests-unapproved.result == 'success') }}
+    needs: [build, unit-tests]
     uses: ./.github/workflows/standard-its.yml
 
   revised-its:
-    needs: [build, unit-tests-unapproved, check-approval]
-    if: ${{ !cancelled() && (needs.check-approval.outputs.approved == 'true' || needs.unit-tests-unapproved.result == 'success') }}
+    needs: [build, unit-tests]
     uses: ./.github/workflows/revised-its.yml
     with:
       BACKWARD_COMPATIBILITY_IT_ENABLED: ${{ needs.set-env-var.outputs.BACKWARD_COMPATIBILITY_IT_ENABLED }}
@@ -219,7 +175,7 @@ jobs:
       DRUID_PREVIOUS_IT_IMAGE_NAME: ${{ needs.set-env-var.outputs.DRUID_PREVIOUS_IT_IMAGE_NAME }}
 
   actions-timeline:
-    needs: [build, unit-tests-approved, unit-tests-unapproved, revised-its, standard-its]
+    needs: [build, unit-tests, revised-its, standard-its]
     runs-on: ubuntu-latest
     if: ${{ !cancelled() }}
     steps: