build/devenv: support multiple executors in deployment

.github/workflows/test-smoke.yaml

@@ -78,81 +78,9 @@ jobs:
         working-directory: build/devenv/tests/e2e
         run: |
           set -o pipefail
-          go test -v -timeout 15m -count=1 -run 'TestE2ESmoke' -json \
-            | tee test-results.json \
-            | jq -r '
-              select(.Action != null) |
-              if .Action=="run" and .Test then
-                "=== RUN   \(.Test)"
-              elif .Action=="pause" and .Test then
-                "=== PAUSE \(.Test)"
-              elif .Action=="cont" and .Test then
-                "=== CONT  \(.Test)"
-              elif .Action=="output" and .Test then
-                (.Output | rtrimstr("\n") | "    " + .)
-              elif (.Action=="pass" or .Action=="fail" or .Action=="skip") and .Test then
-                ("--- \(.Action | ascii_upcase): \(.Test)" + (if .Elapsed then " (\(.Elapsed)s)" else "" end))
-              elif .Action=="output" and (.Test|not) then
-                (.Output | rtrimstr("\n"))
-              elif (.Action=="pass" or .Action=="fail") and (.Test|not) then
-                if .Action=="pass" then
-                  "PASS\nok  \(.Package) \(.Elapsed)s"
-                else
-                  "FAIL\nFAIL\t\(.Package) \(.Elapsed)s"
-                end
-              else empty end
-            '
+          go test -v -timeout 15m -count=1 -run 'TestE2ESmoke'
         continue-on-error: true
 
-      - name: Create test summary
-        if: always()
-        working-directory: build/devenv/tests/e2e
-        run: |
-          echo "### E2E Smoke Test Results" > test-summary.md
-          echo "| Test Case | Status | Duration |" >> test-summary.md
-          echo "|-----------|--------|----------|" >> test-summary.md
-          cat test-results.json | jq -r '. | select(.Test != null and (.Action=="pass" or .Action=="fail")) | "| `\(.Test)` | `\(.Action)` | `\(.Elapsed)s` |"' >> test-summary.md
-          echo "" >> test-summary.md
-          echo "Full logs are available in the workflow artifacts." >> test-summary.md
-
-      - name: Remove previous smoke test comments
-        uses: actions/github-script@v6
-        if: always() && github.event_name == 'pull_request'
-        with:
-          github-token: ${{ secrets.GITHUB_TOKEN }}
-          script: |
-            const { owner, repo, number: issue_number } = context.issue;
-            const comments = await github.rest.issues.listComments({
-              owner,
-              repo,
-              issue_number
-            });
-            const commentPrefix = "### E2E Smoke Test Results";
-            for (const comment of comments.data) {
-              if (comment.body.startsWith(commentPrefix)) {
-                await github.rest.issues.deleteComment({
-                  owner,
-                  repo,
-                  comment_id: comment.id
-                });
-              }
-            }
-
-      - name: Post test summary comment
-        uses: actions/github-script@v6
-        if: always() && github.event_name == 'pull_request'
-        with:
-          github-token: ${{ secrets.GITHUB_TOKEN }}
-          script: |
-            const fs = require('fs');
-            const body = fs.readFileSync('build/devenv/tests/e2e/test-summary.md', 'utf8');
-            github.rest.issues.createComment({
-              issue_number: context.issue.number,
-              owner: context.repo.owner,
-              repo: context.repo.repo,
-              body: body
-            });
-
       - name: Upload Logs
         if: always()
         uses: actions/upload-artifact@v4

build/devenv/cciptestinterfaces/interface.go

@@ -157,7 +157,10 @@ type OffChainConfigurable interface {
 	// ConfigureNodes configure CL nodes from blockchain data
 	// returns a piece of TOML config as a string that the framework inject into final configuration
 	ConfigureNodes(ctx context.Context, blockchain *blockchain.Input) (string, error)
-	// FundNodes Fund Chainlink nodes for some amount of native/LINK currency
+	// FundNodes funds Chainlink nodes for some amount of native/LINK currency
 	// using chain-specific clients or CLDF
 	FundNodes(ctx context.Context, cls []*nodeset.Input, bc *blockchain.Input, linkAmount, nativeAmount *big.Int) error
+	// FundAddresses funds addresses for some amount of native currency
+	// using chain-specific clients or CLDF
+	FundAddresses(ctx context.Context, bc *blockchain.Input, addresses []protocol.UnknownAddress, nativeAmount *big.Int) error
 }

build/devenv/env-cl.toml

@@ -144,9 +144,18 @@
     aggregator_api_key = "dev-api-key-tertiary-verifier-2"
     aggregator_secret_key = "dev-secret-tertiary-verifier-2"
 
-[executor]
+[[executor]]
   mode = "cl"
   image = "executor:dev"
   port = 8101
   source_code_path = "../executor"
   root_path = "../../"
+  container_name = "default-executor-1"
+
+[[executor]]
+  mode = "cl"
+  image = "executor:dev"
+  port = 8102
+  source_code_path = "../executor"
+  root_path = "../../"
+  container_name = "default-executor-2"

build/devenv/env.toml

@@ -149,15 +149,23 @@ cl_nodes_funding_link = 50
     aggregator_api_key = "dev-api-key-tertiary-verifier-2"
     aggregator_secret_key = "dev-secret-tertiary-verifier-2"
 
-[executor]
+[[executor]]
   image = "executor:dev"
   port = 8101
   source_code_path = "../executor"
   root_path = "../../"
+  container_name = "default-executor-1"
+
+[[executor]]
+  image = "executor:dev"
+  port = 8102
+  source_code_path = "../executor"
+  root_path = "../../"
+  container_name = "default-executor-2"
 
 [indexer]
   image = "indexer:dev"
-  port = 8102
+  port = 8103
   source_code_path = "../indexer"
   root_path = "../../"
   [indexer.db]

build/devenv/environment.go

@@ -18,6 +18,7 @@ import (
 	"github.com/smartcontractkit/chainlink-ccv/devenv/cciptestinterfaces"
 	"github.com/smartcontractkit/chainlink-ccv/devenv/internal/util"
 	"github.com/smartcontractkit/chainlink-ccv/devenv/services"
+	"github.com/smartcontractkit/chainlink-ccv/protocol"
 	"github.com/smartcontractkit/chainlink-ccv/verifier/commit"
 	"github.com/smartcontractkit/chainlink-deployments-framework/datastore"
 	"github.com/smartcontractkit/chainlink-deployments-framework/deployment"
@@ -79,7 +80,7 @@ type Cfg struct {
 	JD                 *jd.Input                   `toml:"jd"                    validate:"required"`
 	Fake               *services.FakeInput         `toml:"fake"                  validate:"required"`
 	Verifier           []*services.VerifierInput   `toml:"verifier"              validate:"required"`
-	Executor           *services.ExecutorInput     `toml:"executor"              validate:"required"`
+	Executor           []*services.ExecutorInput   `toml:"executor"              validate:"required"`
 	Indexer            *services.IndexerInput      `toml:"indexer"               validate:"required"`
 	Aggregator         []*services.AggregatorInput `toml:"aggregator"            validate:"required"`
 	Blockchains        []*blockchain.Input         `toml:"blockchains"           validate:"required"`
@@ -101,7 +102,7 @@ func checkKeys(in *Cfg) error {
 func NewProductConfigurationFromNetwork(typ string) (cciptestinterfaces.CCIP17ProductConfiguration, error) {
 	switch typ {
 	case "anvil":
-		return &evm.CCIP17EVM{}, nil
+		return evm.NewEmptyCCIP17EVM(), nil
 	case "canton":
 		// see devenv-evm implementation and add Canton
 		return nil, nil
@@ -146,7 +147,9 @@ func NewEnvironment() (in *Cfg, err error) {
 
 	// Executor config...
 	if in.Executor != nil {
-		services.ApplyExecutorDefaults(in.Executor)
+		for _, exec := range in.Executor {
+			services.ApplyExecutorDefaults(exec)
+		}
 	}
 
 	/////////////////////////////
@@ -429,14 +432,38 @@ func NewEnvironment() (in *Cfg, err error) {
 		return nil, fmt.Errorf("failed to create indexer service: %w", err)
 	}
 
-	if in.Executor != nil {
-		exec, err := services.ResolveContractsForExecutor(e.DataStore, in.Blockchains, in.Executor)
+	if len(in.Executor) > 0 {
+		execs, err := services.ResolveContractsForExecutor(e.DataStore, in.Blockchains, in.Executor)
 		if err != nil {
 			return nil, fmt.Errorf("failed to lookup contracts for executor: %w", err)
 		}
-		in.Executor = exec
+		execs, err = services.SetExecutorPoolAndID(execs)
+		if err != nil {
+			return nil, fmt.Errorf("failed to set executor pool and ID: %w", err)
+		}
+		execs, err = services.SetTransmitterPrivateKey(execs)
+		if err != nil {
+			return nil, fmt.Errorf("failed to set transmitter private key: %w", err)
+		}
+
+		// fund the keys used by the executors to send transactions in standalone mode.
+		addresses := make([]protocol.UnknownAddress, 0, len(execs))
+		for _, exec := range execs {
+			addresses = append(addresses, exec.GetTransmitterAddress())
+		}
+		Plog.Info().Any("Addresses", addresses).Int("ImplsLen", len(impls)).Msg("Funding executors")
+		for i, impl := range impls {
+			Plog.Info().Int("ImplIndex", i).Msg("Funding executor")
+			err = impl.FundAddresses(ctx, in.Blockchains[i], addresses, big.NewInt(5))
+			if err != nil {
+				return nil, fmt.Errorf("failed to fund addresses for executors: %w", err)
+			}
+			Plog.Info().Int("ImplIndex", i).Msg("Funded executors")
+		}
+
+		in.Executor = execs
 	}
-	_, err = launchStandaloneExecutor(in.Executor)
+	_, err = launchStandaloneExecutors(in.Executor)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create standalone executor: %w", err)
 	}
@@ -477,7 +504,7 @@ func NewEnvironment() (in *Cfg, err error) {
 }
 
 // createJobs creates the jobs for the verifiers and executors on the CL nodes if they're in CL mode.
-func createJobs(in *Cfg, vIn []*services.VerifierInput, executorIn *services.ExecutorInput) error {
+func createJobs(in *Cfg, vIn []*services.VerifierInput, executorIn []*services.ExecutorInput) error {
 	// Exit early, there are no nodes configured.
 	if len(in.NodeSets) == 0 {
 		return nil
@@ -514,25 +541,30 @@ func createJobs(in *Cfg, vIn []*services.VerifierInput, executorIn *services.Exe
 		}
 	}
 
-	if executorIn != nil && executorIn.Mode == services.CL {
-		index, clClient := roundRobin.GetNext()
+	for _, exec := range executorIn {
+		switch exec.Mode {
+		case services.CL:
+			index, clClient := roundRobin.GetNext()
 
-		tomlConfig, err := executorIn.GenerateConfig()
-		if err != nil {
-			return fmt.Errorf("failed to generate executor config: %w", err)
-		}
+			tomlConfig, err := exec.GenerateConfig()
+			if err != nil {
+				return fmt.Errorf("failed to generate executor config: %w", err)
+			}
 
-		jb, resp, err := clClient.CreateJobRaw(executorSpec(string(tomlConfig)))
-		if err != nil {
-			return fmt.Errorf("failed to create executor job: %w", err)
-		}
-		if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusCreated {
-			return fmt.Errorf("failed to create executor job: %s", resp.Status)
+			jb, resp, err := clClient.CreateJobRaw(executorSpec(string(tomlConfig)))
+			if err != nil {
+				return fmt.Errorf("failed to create executor job: %w", err)
+			}
+			if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusCreated {
+				return fmt.Errorf("failed to create executor job: %s", resp.Status)
+			}
+			Plog.Info().
+				Int("CurrentIndex", index).
+				Any("ExecutorJob", jb).
+				Msg("Created executor job on node")
+		case services.Standalone:
+			continue
 		}
-		Plog.Info().
-			Int("CurrentIndex", index).
-			Any("ExecutorJob", jb).
-			Msg("Created executor job on node")
 	}
 
 	return nil
@@ -586,8 +618,8 @@ func launchCLNodes(
 		hasAService = hasAService || (ver.Mode == services.CL)
 	}
 
-	if in.Executor != nil {
-		hasAService = hasAService || (in.Executor.Mode == services.CL)
+	for _, exec := range in.Executor {
+		hasAService = hasAService || (exec.Mode == services.CL)
 	}
 
 	// Exit early, there are no services configured to deploy on a CL node.
@@ -710,15 +742,17 @@ func launchCLNodes(
 	return onchainPublicKeys, nil
 }
 
-func launchStandaloneExecutor(in *services.ExecutorInput) ([]*services.ExecutorOutput, error) {
+func launchStandaloneExecutors(in []*services.ExecutorInput) ([]*services.ExecutorOutput, error) {
 	var outs []*services.ExecutorOutput
-	// Start standalone executor if in standalone mode.
-	if in != nil && in.Mode == services.Standalone {
-		out, err := services.NewExecutor(in)
-		if err != nil {
-			return nil, fmt.Errorf("failed to create executor service: %w", err)
+	// Start standalone executors if they are in standalone mode.
+	for _, exec := range in {
+		if exec != nil && exec.Mode == services.Standalone {
+			out, err := services.NewExecutor(exec)
+			if err != nil {
+				return nil, fmt.Errorf("failed to create executor service: %w", err)
+			}
+			outs = append(outs, out)
 		}
-		outs = append(outs, out)
 	}
 	return outs, nil
 }

build/devenv/evm/impl.go

@@ -6,6 +6,7 @@ import (
 	"errors"
 	"fmt"
 	"math/big"
+	"os"
 	"time"
 
 	"github.com/Masterminds/semver/v3"
@@ -18,6 +19,7 @@ import (
 	"github.com/google/uuid"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/rs/zerolog"
+	"github.com/rs/zerolog/log"
 
 	"github.com/smartcontractkit/chainlink-ccip/ccv/chains/evm/deployment/v1_7_0/operations/burn_mint_token_pool"
 	"github.com/smartcontractkit/chainlink-ccip/ccv/chains/evm/deployment/v1_7_0/operations/committee_verifier"
@@ -250,6 +252,18 @@ type CCIP17EVM struct {
 	offRampBySelector      map[uint64]*offramp.OffRamp
 }
 
+// NewEmptyCCIP17EVM creates a new CCIP17EVM with a logger that logs to the console.
+func NewEmptyCCIP17EVM() *CCIP17EVM {
+	return &CCIP17EVM{
+		logger: log.
+			Output(zerolog.ConsoleWriter{Out: os.Stderr}).
+			Level(zerolog.DebugLevel).
+			With().
+			Fields(map[string]any{"component": "CCIP17EVM"}).
+			Logger(),
+	}
+}
+
 // NewCCIP17EVM creates new smart-contracts wrappers with utility functions for CCIP17EVM implementation.
 func NewCCIP17EVM(ctx context.Context, logger zerolog.Logger, e *deployment.Environment, chainIDs, wsURLs []string) (*CCIP17EVM, error) {
 	if len(chainIDs) != len(wsURLs) {
@@ -1530,6 +1544,34 @@ func (m *CCIP17EVM) ConnectContractsWithSelectors(ctx context.Context, e *deploy
 	return nil
 }
 
+func (m *CCIP17EVM) FundAddresses(ctx context.Context, bc *blockchain.Input, addresses []protocol.UnknownAddress, nativeAmount *big.Int) error {
+	client, _, _, err := ETHClient(ctx, bc.Out.Nodes[0].ExternalWSUrl, &GasSettings{
+		FeeCapMultiplier: 2,
+		TipCapMultiplier: 2,
+	})
+	if err != nil {
+		return fmt.Errorf("could not create basic eth client: %w", err)
+	}
+	chainInfo, err := chainsel.GetChainDetailsByChainIDAndFamily(bc.ChainID, chainsel.FamilyEVM)
+	if err != nil {
+		return fmt.Errorf("could not get chain details: %w", err)
+	}
+	for _, addr := range addresses {
+		a, _ := nativeAmount.Float64()
+		addrStr := common.BytesToAddress(addr).Hex()
+		m.logger.Info().Uint64("ChainSelector", chainInfo.ChainSelector).Str("Address", addrStr).Msg("Funding address")
+		if err := FundNodeEIP1559(ctx, client, getNetworkPrivateKey(), addrStr, a); err != nil {
+			return fmt.Errorf("failed to fund address %s: %w", addrStr, err)
+		}
+		bal, err := client.BalanceAt(ctx, common.HexToAddress(addrStr), nil)
+		if err != nil {
+			return fmt.Errorf("failed to get balance: %w", err)
+		}
+		m.logger.Info().Uint64("ChainSelector", chainInfo.ChainSelector).Str("Address", addrStr).Int64("Balance", bal.Int64()).Msg("Address balance")
+	}
+	return nil
+}
+
 func (m *CCIP17EVM) FundNodes(ctx context.Context, ns []*simple_node_set.Input, bc *blockchain.Input, linkAmount, nativeAmount *big.Int) error {
 	l := m.logger
 	l.Info().Msg("Funding CL nodes with ETH and LINK")