feat: replace legacy replays with PRAGUE replays (#2461)

* refactor legacy replay tests

This refactors the existing legacy replay tests, such they are: (1) all
disabled; (2) live in their own package to avoid clutter; (3) all the
actual replays have been moved into their own folder as well.

* add initial set of prague replays

The goal here is simply to figure out what the batch size is for the
various testing regimes (fast replay, nightly, weekly).

* updating gradle tests

This simplifies the way the existing gradle test tasks are written, and
also adds a notion of "weekly replay tests".

Author: DavePearce

.github/workflows/gradle-nightly-replay-tests.yml

@@ -18,3 +18,41 @@ jobs:
     with:
       zkevm_fork: PRAGUE
       tests-with-ssh: ${{ inputs.tests-with-ssh || false }}
+
+  # ==================================================================
+  # Replay Tests
+  # ==================================================================
+  nightly-replay-tests:
+    runs-on: gha-runner-scale-set-ubuntu-22.04-amd64-xxl
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
+        with:
+          submodules: recursive
+
+      - name: Setup Test Environment
+        uses: ./.github/actions/setup-environment
+
+      - name: Run Nightly Replay tests
+        run: GOMAXPROCS=10 GOMEMLIMIT=20GiB ./gradlew nightlyReplayTests
+        env:
+          JAVA_OPTS: -Dorg.gradle.daemon=false
+          GOCORSET_FLAGS: -v --ansi-escapes=false --report --mir
+          JUNIT_TESTS_PARALLELISM: 2
+
+      - name: Upload test report
+        if: ${{ always() }}
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        with:
+          name: nightly-replay-tests-report
+          path: arithmetization/build/reports/tests/**/*
+
+      - name: Failure Notification
+        if: github.ref == 'refs/heads/arith-dev' && (failure() || cancelled())
+        uses: slackapi/slack-github-action@91efab103c0de0a537f72a35f6b8cda0ee76bf0a # v2.1.1
+        with:
+          webhook: ${{ secrets.SLACK_WEBHOOK_URL }}
+          webhook-type: webhook-trigger
+          payload: |
+            name: "NightlyReplay"
+            status: "${{ job.status }}"

.github/workflows/gradle-nightly-tests.yml

@@ -87,9 +87,8 @@ jobs:
         run: GOMEMLIMIT=26GiB ./gradlew :arithmetization:fastReplayTests
         env:
           JAVA_OPTS: -Dorg.gradle.daemon=false
-          REPLAY_TESTS_PARALLELISM: 4
+          JUNIT_TESTS_PARALLELISM: 4
           GOCORSET_FLAGS: -b1024 -v --ansi-escapes=false --report --air
-          ZKEVM_FORK: LONDON
 
       - name: Upload test report
         if: ${{ always() }}

.github/workflows/gradle-tests.yml

@@ -23,3 +23,39 @@ jobs:
     secrets: inherit
     with:
       zkevm_fork: PRAGUE
+
+  weekly-replay-tests:
+    runs-on: gha-runner-scale-set-ubuntu-22.04-amd64-xxl
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
+        with:
+          submodules: recursive
+
+      - name: Setup Test Environment
+        uses: ./.github/actions/setup-environment
+
+      - name: Run Weekly Replay tests
+        run: GOMAXPROCS=10 GOMEMLIMIT=20GiB ./gradlew weeklyReplayTests
+        env:
+          JAVA_OPTS: -Dorg.gradle.daemon=false
+          GOCORSET_FLAGS: -v --ansi-escapes=false --report --air
+          JUNIT_TESTS_PARALLELISM: 2
+
+      - name: Upload test report
+        if: ${{ always() }}
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        with:
+          name: weekly-replay-tests-report
+          path: arithmetization/build/reports/tests/**/*
+
+      - name: Failure Notification
+        if: github.ref == 'refs/heads/arith-dev' && (failure() || cancelled())
+        uses: slackapi/slack-github-action@91efab103c0de0a537f72a35f6b8cda0ee76bf0a # v2.1.1
+        with:
+          webhook: ${{ secrets.SLACK_WEBHOOK_URL }}
+          webhook-type: webhook-trigger
+          payload: |
+            name: "WeeklyReplay"
+            status: "${{ job.status }}"
+      

.github/workflows/gradle-weekly-tests.yml

@@ -128,7 +128,7 @@ jobs:
         run: GOMEMLIMIT=26GiB ./gradlew :arithmetization:fastReplayTests
         env:
           JAVA_OPTS: -Dorg.gradle.daemon=false
-          REPLAY_TESTS_PARALLELISM: 4
+          JUNIT_TESTS_PARALLELISM: 4
           GOCORSET_FLAGS: -b1024 -v --ansi-escapes=false --report --air
           ZKEVM_FORK: LONDON
 

.github/workflows/manual-release.yml

@@ -67,7 +67,7 @@ jobs:
         run: GOMAXPROCS=10 GOMEMLIMIT=20GiB ./gradlew referenceExecutionSpecBlockchainTests -x spotlessCheck --tests "${{ inputs.test_filter || 'BlockchainReferenceTest_*' }}"
         timeout-minutes: 360
         env:
-          REFERENCE_TESTS_PARALLELISM: 3
+          JUNIT_TESTS_PARALLELISM: 3
           JAVA_OPTS: -Dorg.gradle.daemon=false
           GOCORSET_FLAGS: -b1024 -v --ansi-escapes=false --report --air
           FAILED_TEST_JSON_DIRECTORY: ${{ github.workspace }}/tmp/${{ steps.extract_branch.outputs.branch }}/

.github/workflows/reusable-blockchain-tests.yml

@@ -30,7 +30,7 @@ jobs:
         env:
           JAVA_OPTS: -Dorg.gradle.daemon=false
           GOCORSET_FLAGS: -v --ansi-escapes=false --report --mir
-          NIGHTLY_TESTS_PARALLELISM: 2
+          JUNIT_TESTS_PARALLELISM: 2
           ZKEVM_FORK: ${{ inputs.zkevm_fork }}
 
       - name: Upload test report

.github/workflows/reusable-nightly-tests.yml

@@ -32,7 +32,7 @@ jobs:
         env:
           JAVA_OPTS: -Dorg.gradle.daemon=false
           GOCORSET_FLAGS: -b1024 -v --ansi-escapes=false --report --air
-          UNIT_TESTS_PARALLELISM: 4
+          JUNIT_TESTS_PARALLELISM: 4
           ZKEVM_FORK: ${{ inputs.zkevm_fork }}
 
       - name: Upload test report

.github/workflows/reusable-unit-tests.yml

@@ -26,7 +26,7 @@ jobs:
         run: GOMAXPROCS=1 GOMEMLIMIT=10GiB ./gradlew prcCallTests
         env:
           JAVA_OPTS: -Dorg.gradle.daemon=false
-          WEEKLY_TESTS_PARALLELISM: 24
+          JUNIT_TESTS_PARALLELISM: 24
           GOCORSET_FLAGS: -v -b1024 --ansi-escapes=false --report --mir
           ZKEVM_FORK: ${{ inputs.zkevm_fork }}
 

.github/workflows/reusable-weekly-prc-tests.yml

@@ -25,7 +25,7 @@ jobs:
       - name: Run Weekly tests
         run: GOMAXPROCS=10 GOMEMLIMIT=20GiB ./gradlew weeklyTests
         env:
-          WEEKLY_TESTS_PARALLELISM: 2
+          JUNIT_TESTS_PARALLELISM: 2
           GOCORSET_FLAGS: -v -b1024 --ansi-escapes=false --report --mir
           ZKEVM_FORK: ${{ inputs.zkevm_fork }}