feat: ✨ Add docker support (#456)

* update pkgs

* update pkgs

* chore: 📦 Update packages

* removed anys

* chore

* feat: ✨ added docker option and launch feature

* fix

* fix: 🐛 added port mapping feature to docker environments

* add pull logic

* feat: ✨ Add check for existing containers

Author: timbrinded

.changeset/chatty-stingrays-prove.md

@@ -0,0 +1,8 @@
+---
+"@moonwall/types": minor
+"@moonwall/cli": minor
+"@moonwall/docs": minor
+"@moonwall/tests": minor
+---
+
+Add Docker Support

.github/workflows/main.yml

@@ -114,7 +114,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        suite: ["dev_test", "dev_multi", "dev_seq", "dev_smoke", "papi_dev", "fork_test"]
+        suite: ["dev_test", "dev_multi", "dev_seq", "dev_smoke", "papi_dev", "fork_test", "dev_docker"]
         shard: [1, 2, 3, 4]
     steps:
       - uses: actions/checkout@v4

package.json

@@ -65,7 +65,7 @@
   },
   "pnpm": {
     "overrides": {
-       "@moonbeam-network/api-augment": "0.3401.2",
+      "@moonbeam-network/api-augment": "0.3401.2",
       "@polkadot/api": "15.0.1",
       "@polkadot/api-base": "15.0.1",
       "@polkadot/api-derive": "15.0.1",

packages/cli/package.json

@@ -83,6 +83,7 @@
     "cli-progress": "3.12.0",
     "colors": "1.4.0",
     "debug": "4.4.0",
+    "dockerode": "4.0.4",
     "dotenv": "16.4.7",
     "ethers": "*",
     "get-port": "^7.1.0",
@@ -104,6 +105,7 @@
   },
   "devDependencies": {
     "@biomejs/biome": "*",
+    "@types/dockerode": "3.3.34",
     "@types/clear": "^0.1.4",
     "@types/cli-progress": "3.11.6",
     "@types/debug": "*",

packages/cli/src/cmds/runTests.ts

@@ -143,7 +143,9 @@ export async function executeTests(env: Environment, testRunArgs?: testRunArgs &
         ...vitestOptions,
       } satisfies UserConfig;
 
-      console.log(`Options to use: ${JSON.stringify(optionsToUse, null, 2)}`);
+      if (env.printVitestOptions) {
+        console.log(`Options to use: ${JSON.stringify(optionsToUse, null, 2)}`);
+      }
       resolve((await startVitest("test", folders, optionsToUse)) as Vitest);
     } catch (e) {
       console.error(e);

packages/cli/src/internal/cmdFunctions/downloader.ts

@@ -57,7 +57,7 @@ export async function downloader(url: string, outputPath: string): Promise<void>
   fs.rmSync(tempPath);
 }
 
-function initializeProgressBar(): SingleBar {
+export function initializeProgressBar(): SingleBar {
   const options: ProgressBarOptions = {
     etaAsynchronousUpdate: true,
     etaBuffer: 40,

packages/cli/src/internal/commandParsers.ts

@@ -62,7 +62,9 @@ export class LaunchCommandParser {
     this.cmd = launchSpec.binPath;
     this.args = launchSpec.options
       ? [...launchSpec.options]
-      : fetchDefaultArgs(path.basename(launchSpec.binPath), additionalRepos);
+      : launchSpec.useDocker
+        ? []
+        : fetchDefaultArgs(path.basename(launchSpec.binPath), additionalRepos);
   }
 
   private overrideArg(newArg: string): void {

packages/cli/src/internal/launcherCommon.ts

@@ -5,6 +5,8 @@ import fs from "node:fs";
 import path from "node:path";
 import { importAsyncConfig, parseZombieConfigForBins } from "../lib/configReader";
 import { checkAlreadyRunning, downloadBinsIfMissing, promptAlreadyRunning } from "./fileCheckers";
+import Docker from "dockerode";
+import { select } from "@inquirer/prompts";
 
 export async function commonChecks(env: Environment) {
   const globalConfig = await importAsyncConfig();
@@ -45,12 +47,85 @@ async function devBinCheck(env: Environment) {
     throw new Error("This function is only for dev environments");
   }
 
+  if (!env.foundation.launchSpec || !env.foundation.launchSpec[0]) {
+    throw new Error("Dev environment requires a launchSpec configuration");
+  }
+
+  if (env.foundation.launchSpec[0].useDocker) {
+    const docker = new Docker();
+    const imageName = env.foundation.launchSpec[0].binPath;
+
+    console.log(`Checking if ${imageName} is running...`);
+    const matchingContainers = (
+      await docker.listContainers({
+        filters: { ancestor: [imageName] },
+      })
+    ).flat();
+
+    if (matchingContainers.length === 0) {
+      return;
+    }
+
+    if (!process.env.CI) {
+      await promptKillContainers(matchingContainers);
+      return;
+    }
+
+    const runningContainers = matchingContainers.map(({ Id, Ports }) => ({
+      Id: Id.slice(0, 12),
+      Ports: Ports.map(({ PublicPort, PrivatePort }) =>
+        PublicPort ? `${PublicPort} -> ${PrivatePort}` : `${PrivatePort}`
+      ).join(", "),
+    }));
+
+    console.table(runningContainers);
+
+    throw new Error(`${imageName} is already running, aborting`);
+  }
+
   const binName = path.basename(env.foundation.launchSpec[0].binPath);
   const pids = checkAlreadyRunning(binName);
   pids.length === 0 || process.env.CI || (await promptAlreadyRunning(pids));
   await downloadBinsIfMissing(env.foundation.launchSpec[0].binPath);
 }
 
+async function promptKillContainers(matchingContainers: Docker.ContainerInfo[]) {
+  const answer = await select({
+    message: `The following containers are already running image ${matchingContainers[0].Image}: ${matchingContainers.map(({ Id }) => Id).join(", ")}\n Would you like to kill them?`,
+    choices: [
+      { name: "🪓  Kill containers", value: "kill" },
+      { name: "👋   Quit", value: "goodbye" },
+    ],
+  });
+
+  if (answer === "goodbye") {
+    console.log("Goodbye!");
+    process.exit(0);
+  }
+
+  if (answer === "kill") {
+    const docker = new Docker();
+    for (const { Id } of matchingContainers) {
+      const container = docker.getContainer(Id);
+      await container.stop();
+      await container.remove();
+    }
+
+    const containers = await docker.listContainers({
+      filters: { ancestor: matchingContainers.map(({ Image }) => Image) },
+    });
+
+    if (containers.length > 0) {
+      console.error(
+        `The following containers are still running: ${containers.map(({ Id }) => Id).join(", ")}`
+      );
+      process.exit(1);
+    }
+
+    return;
+  }
+}
+
 export async function executeScript(scriptCommand: string, args?: string) {
   const scriptsDir = (await importAsyncConfig()).scriptsDir;
 

packages/cli/src/internal/localNode.ts

@@ -6,11 +6,103 @@ import { checkAccess, checkExists } from "./fileCheckers";
 import Debug from "debug";
 import { setTimeout as timer } from "node:timers/promises";
 import util from "node:util";
+import type { DevLaunchSpec } from "@moonwall/types";
+import Docker from "dockerode";
+import invariant from "tiny-invariant";
 
 const execAsync = util.promisify(exec);
 const debug = Debug("global:localNode");
 
-export async function launchNode(cmd: string, args: string[], name: string) {
+// TODO: Add multi-threading support
+async function launchDockerContainer(
+  imageName: string,
+  args: string[],
+  name: string,
+  dockerConfig?: DevLaunchSpec["dockerConfig"]
+) {
+  const docker = new Docker();
+  const port = args.find((a) => a.includes("port"))?.split("=")[1];
+  debug(`\x1b[36mStarting Docker container ${imageName} on port ${port}...\x1b[0m`);
+
+  const dirPath = path.join(process.cwd(), "tmp", "node_logs");
+  const logLocation = path.join(dirPath, `${name}_docker_${Date.now()}.log`);
+  const fsStream = fs.createWriteStream(logLocation);
+  process.env.MOON_LOG_LOCATION = logLocation;
+
+  const portBindings = dockerConfig?.exposePorts?.reduce<Record<string, { HostPort: string }[]>>(
+    (acc, { hostPort, internalPort }) => {
+      acc[`${internalPort}/tcp`] = [{ HostPort: hostPort.toString() }];
+      return acc;
+    },
+    {}
+  );
+
+  const rpcPort = args.find((a) => a.includes("rpc-port"))?.split("=")[1];
+  invariant(rpcPort, "RPC port not found, this is a bug");
+
+  const containerOptions = {
+    Image: imageName,
+    platform: "linux/amd64",
+    Cmd: args,
+    name: dockerConfig?.containerName || `moonwall_${name}_${Date.now()}`,
+    ExposedPorts: {
+      ...Object.fromEntries(
+        dockerConfig?.exposePorts?.map(({ internalPort }) => [`${internalPort}/tcp`, {}]) || []
+      ),
+      [`${rpcPort}/tcp`]: {},
+    },
+    HostConfig: {
+      PortBindings: {
+        ...portBindings,
+        [`${rpcPort}/tcp`]: [{ HostPort: rpcPort }],
+      },
+    },
+    Env: dockerConfig?.runArgs?.filter((arg) => arg.startsWith("env:")).map((arg) => arg.slice(4)),
+  } satisfies Docker.ContainerCreateOptions;
+
+  try {
+    await pullImage(imageName, docker);
+
+    const container = await docker.createContainer(containerOptions);
+    await container.start();
+
+    const containerInfo = await container.inspect();
+    if (!containerInfo.State.Running) {
+      const errorMessage = `Container failed to start: ${containerInfo.State.Error}`;
+      console.error(errorMessage);
+      fs.appendFileSync(logLocation, `${errorMessage}\n`);
+      throw new Error(errorMessage);
+    }
+
+    for (let i = 0; i < 300; i++) {
+      if (await checkWebSocketJSONRPC(Number.parseInt(rpcPort))) {
+        break;
+      }
+      await timer(100);
+    }
+
+    return { runningNode: container, fsStream };
+  } catch (error: unknown) {
+    if (error instanceof Error) {
+      console.error(`Docker container launch failed: ${error.message}`);
+      fs.appendFileSync(logLocation, `Docker launch error: ${error.message}\n`);
+    }
+    throw error;
+  }
+}
+
+export async function launchNode(options: {
+  command: string;
+  args: string[];
+  name: string;
+  launchSpec?: DevLaunchSpec;
+}) {
+  const { command: cmd, args, name, launchSpec: config } = options;
+
+  if (config?.useDocker) {
+    return launchDockerContainer(cmd, args, name, config.dockerConfig);
+  }
+
   if (cmd.includes("moonbeam")) {
     await checkExists(cmd);
     checkAccess(cmd);
@@ -193,3 +285,19 @@ async function findPortsByPid(pid: number, retryCount = 600, retryDelay = 100):
 
   return [];
 }
+
+async function pullImage(imageName: string, docker: Docker) {
+  console.log(`Pulling Docker image: ${imageName}`);
+
+  const pullStream = await docker.pull(imageName);
+  // Dockerode pull doesn't wait for completion by default 🫠
+  await new Promise((resolve, reject) => {
+    docker.modem.followProgress(pullStream, (err: Error | null, output: any[]) => {
+      if (err) {
+        reject(err);
+      } else {
+        resolve(output);
+      }
+    });
+  });
+}

packages/cli/src/internal/providerFactories.ts

@@ -95,7 +95,9 @@ export class ProviderFactory {
       name: this.providerConfig.name,
       type: this.providerConfig.type,
       connect: () => {
-        const provider = new ethers.WebSocketProvider(this.url);
+        const provider = this.url.startsWith("ws")
+          ? new ethers.WebSocketProvider(this.url)
+          : new ethers.JsonRpcProvider(this.url);
         return new Wallet(this.privateKey, provider);
       },
     };