Initial reorg e2e tests and verifier adjustment

.github/workflows/test-finality.yaml

@@ -0,0 +1,170 @@
+name: End-to-end CCV Finality Test
+
+on:
+  pull_request:
+
+defaults:
+  run:
+    working-directory: build/devenv
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}-${{ github.sha }}
+  cancel-in-progress: true
+
+jobs:
+  e2e-finality:
+    permissions:
+      id-token: write
+      contents: read
+      pull-requests: write
+    runs-on: runs-on=${{ github.run_id }}/family=c6i/cpu=32+48/ram=64+96/spot=false/image=ubuntu24-full-x64/extras=s3-cache+tmpfs
+    steps:
+      - name: Enable S3 Cache for Self-Hosted Runners
+        # these env vars are set (and exposed) when it is a self-hosted runner with extras=s3-cache
+        if: ${{ env.RUNS_ON_INSTANCE_ID != '' && env.ACTIONS_CACHE_URL != '' }}
+        uses: runs-on/action@cd2b598b0515d39d78c38a02d529db87d2196d1e # v2.0.3
+
+      - name: Checkout code
+        uses: actions/checkout@v5
+        with:
+          fetch-depth: 0
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1
+
+      - name: Install Just
+        uses: extractions/setup-just@e33e0265a09d6d736e2ee1e0eb685ef1de4669ff # v3
+        with:
+          just-version: '1.40.0'
+
+      - name: Authenticate to AWS ECR
+        uses: ./.github/actions/aws-ecr-auth
+        with:
+          role-to-assume: ${{ secrets.CCV_IAM_ROLE }}
+          aws-region: us-east-1
+          registry-type: public
+
+      - name: Authenticate to AWS ECR (JD)
+        uses: ./.github/actions/aws-ecr-auth
+        with:
+          role-to-assume: ${{ secrets.CCV_IAM_ROLE }}
+          aws-region: us-west-2
+          registry-type: private
+          registries: ${{ secrets.JD_REGISTRY }}
+
+      - name: Set up Go
+        uses: actions/setup-go@v6 # v6
+        with:
+          cache: true
+          go-version-file: build/devenv/go.mod
+          cache-dependency-path: build/devenv/go.sum
+
+      - name: Download Go dependencies
+        run: |
+          go mod download
+
+      - name: Build Docker Images
+        run: just build-docker-dev-ci
+        env:
+          DOCKER_BUILDKIT: 1
+
+      - name: Run CCV environment with finality configuration
+        env:
+          JD_IMAGE: ${{ secrets.JD_IMAGE }}
+        run: |
+          cd cmd/ccv && go install . && cd -
+          ccv u env.toml,env-finality-test.toml && ccv obs up -m loki
+
+      - name: Run finality tests
+        id: finality_test
+        working-directory: build/devenv/tests/e2e
+        run: |
+          set -o pipefail
+          go test -v -timeout 15m -count=1 -run 'TestSimpleReorgWithMessageOrdering' -json \
+            | tee test-results.json \
+            | jq -r '
+              select(.Action != null) |
+              if .Action=="run" and .Test then
+                "=== RUN   \(.Test)"
+              elif .Action=="pause" and .Test then
+                "=== PAUSE \(.Test)"
+              elif .Action=="cont" and .Test then
+                "=== CONT  \(.Test)"
+              elif .Action=="output" and .Test then
+                (.Output | rtrimstr("\n") | "    " + .)
+              elif (.Action=="pass" or .Action=="fail" or .Action=="skip") and .Test then
+                ("--- \(.Action | ascii_upcase): \(.Test)" + (if .Elapsed then " (\(.Elapsed)s)" else "" end))
+              elif .Action=="output" and (.Test|not) then
+                (.Output | rtrimstr("\n"))
+              elif (.Action=="pass" or .Action=="fail") and (.Test|not) then
+                if .Action=="pass" then
+                  "PASS\nok  \(.Package) \(.Elapsed)s"
+                else
+                  "FAIL\nFAIL\t\(.Package) \(.Elapsed)s"
+                end
+              else empty end
+            '
+        continue-on-error: true
+
+      - name: Create test summary
+        if: always()
+        working-directory: build/devenv/tests/e2e
+        run: |
+          echo "### E2E Finality Test Results" > test-summary.md
+          echo "| Test Case | Status | Duration |" >> test-summary.md
+          echo "|-----------|--------|----------|" >> test-summary.md
+          cat test-results.json | jq -r '. | select(.Test != null and (.Action=="pass" or .Action=="fail")) | "| `\(.Test)` | `\(.Action)` | `\(.Elapsed)s` |"' >> test-summary.md
+          echo "" >> test-summary.md
+          echo "Full logs are available in the workflow artifacts." >> test-summary.md
+
+      - name: Remove previous finality test comments
+        uses: actions/github-script@v6
+        if: always() && github.event_name == 'pull_request'
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          script: |
+            const { owner, repo, number: issue_number } = context.issue;
+            const comments = await github.rest.issues.listComments({
+              owner,
+              repo,
+              issue_number
+            });
+            const commentPrefix = "### E2E Finality Test Results";
+            for (const comment of comments.data) {
+              if (comment.body.startsWith(commentPrefix)) {
+                await github.rest.issues.deleteComment({
+                  owner,
+                  repo,
+                  comment_id: comment.id
+                });
+              }
+            }
+
+      - name: Post test summary comment
+        uses: actions/github-script@v6
+        if: always() && github.event_name == 'pull_request'
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          script: |
+            const fs = require('fs');
+            const body = fs.readFileSync('build/devenv/tests/e2e/test-summary.md', 'utf8');
+            github.rest.issues.createComment({
+              issue_number: context.issue.number,
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              body: body
+            });
+
+      - name: Upload Logs
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: container-logs-finality
+          path: build/devenv/tests/e2e/logs
+          retention-days: 1
+
+      - name: Check test results
+        if: always() && steps.finality_test.outcome == 'failure'
+        run: |
+          echo "Finality tests failed."
+          exit 1

build/devenv/env-finality-test.toml

@@ -0,0 +1,23 @@
+[[blockchains]]
+  container_name = "blockchain-src"
+  chain_id = "1337"
+  docker_cmd_params = ["--no-mining", "--slots-in-an-epoch", "1"]
+  image = "ghcr.io/foundry-rs/foundry:latest"
+  port = "8545"
+  type = "anvil"
+
+[[blockchains]]
+  container_name = "blockchain-dst"
+  chain_id = "2337"
+  docker_cmd_params = ["--no-mining", "--slots-in-an-epoch", "1"]
+  image = "ghcr.io/foundry-rs/foundry:latest"
+  port = "8555"
+  type = "anvil"
+
+[[blockchains]]
+  container_name = "blockchain-3rd"
+  chain_id = "3337"
+  docker_cmd_params = ["--no-mining", "--slots-in-an-epoch", "1"]
+  image = "ghcr.io/foundry-rs/foundry:latest"
+  port = "8565"
+  type = "anvil"

build/devenv/environment.go

@@ -10,6 +10,7 @@ import (
 	"strings"
 
 	"github.com/ethereum/go-ethereum/common/hexutil"
+	"github.com/ethereum/go-ethereum/ethclient"
 
 	"github.com/smartcontractkit/chainlink-ccv/devenv/cciptestinterfaces"
 	"github.com/smartcontractkit/chainlink-ccv/devenv/internal/util"
@@ -71,10 +72,10 @@ const (
 
 type Cfg struct {
 	Mode               services.Mode               `toml:"mode"`
-	CLDF               CLDF                        `toml:"cldf"                  validate:"required"`
-	JD                 *jd.Input                   `toml:"jd"                    validate:"required"`
-	Fake               *services.FakeInput         `toml:"fake"                  validate:"required"`
-	Verifier           []*services.VerifierInput   `toml:"verifier"              validate:"required"`
+	CLDF               CLDF                        `toml:"cldf"     validate:"required"`
+	JD                 *jd.Input                   `toml:"jd"       validate:"required"`
+	Fake               *services.FakeInput         `toml:"fake"     validate:"required"`
+	Verifier           []*services.VerifierInput   `toml:"verifier" validate:"required"`
 	Executor           *services.ExecutorInput     `toml:"executor"              validate:"required"`
 	Indexer            *services.IndexerInput      `toml:"indexer"               validate:"required"`
 	Aggregator         []*services.AggregatorInput `toml:"aggregator"            validate:"required"`
@@ -176,6 +177,11 @@ func NewEnvironment() (in *Cfg, err error) {
 		}
 	}
 
+	// Enable automine for Anvil chains during setup to ensure transactions can be processed
+	if err = enableAutomineForAnvilChains(in.Blockchains); err != nil {
+		return nil, fmt.Errorf("failed to enable automine: %w", err)
+	}
+
 	////////////////////////////
 	// Start: Launch CL Nodes //
 	// We launch the CL nodes first because they don't require any configuration from
@@ -545,3 +551,54 @@ func prefixWith0xIfNeeded(s string) string {
 	}
 	return "0x" + s
 }
+
+// enableAutomineForAnvilChains enables automine for all Anvil chains during setup.
+// This is necessary when Anvil is started with the --no-mining flag,
+// as transactions won't be mined automatically.
+func enableAutomineForAnvilChains(blockchains []*blockchain.Input) error {
+	for _, bc := range blockchains {
+		if bc.Type != "anvil" {
+			continue
+		}
+
+		// Check if --no-mining flag is present in docker_cmd_params
+		hasNoMining := false
+		for _, param := range bc.DockerCmdParamsOverrides {
+			if param == "--no-mining" {
+				hasNoMining = true
+				break
+			}
+		}
+
+		// If --no-mining is set, we need to enable automine
+		if hasNoMining {
+			Plog.Info().
+				Str("chain", bc.ChainID).
+				Msg("Enabling automine for Anvil chain (--no-mining flag detected)")
+
+			if bc.Out == nil || len(bc.Out.Nodes) == 0 {
+				return fmt.Errorf("blockchain output not initialized for chain %s", bc.ChainID)
+			}
+
+			// Use the HTTP URL to send RPC call
+			httpURL := bc.Out.Nodes[0].ExternalHTTPUrl
+			client, err := ethclient.Dial(httpURL)
+			if err != nil {
+				return fmt.Errorf("failed to connect to chain %s: %w", bc.ChainID, err)
+			}
+			defer client.Close()
+
+			// Enable automine using anvil_setAutomine RPC
+			var result any
+			err = client.Client().CallContext(context.Background(), &result, "anvil_setAutomine", true)
+			if err != nil {
+				return fmt.Errorf("failed to enable automine for chain %s: %w", bc.ChainID, err)
+			}
+
+			Plog.Info().
+				Str("chain", bc.ChainID).
+				Msg("Successfully enabled automine for Anvil chain")
+		}
+	}
+	return nil
+}