dxvm: experimental support for Devbox VMs

This is a prototype for using the macOS Virtualization Framework to
launch Linux VMs on macOS. See the README.md for details.

Author: gcurtis

devbox.json

@@ -3,6 +3,14 @@
     "go@latest",
     "golangci-lint@latest"
   ],
+  "env": {
+	  "AR": "/usr/bin/ar",
+	  "AS": "/usr/bin/as",
+	  "CC": "/usr/bin/cc",
+	  "CXX": "/usr/bin/cxx",
+	  "LD": "/usr/bin/ld",
+	  "NM": "/usr/bin/nm"
+  },
   "shell": {
     "init_hook": [
       "echo 'Welcome to devbox!' > /dev/null"

devbox.lock

@@ -2,16 +2,44 @@
   "lockfile_version": "1",
   "packages": {
     "go@latest": {
-      "last_modified": "2023-07-30T12:29:02Z",
-      "resolved": "github:NixOS/nixpkgs/3acb5c4264c490e7714d503c7166a3fde0c51324#go",
+      "last_modified": "2023-09-15T06:49:28Z",
+      "resolved": "github:NixOS/nixpkgs/46688f8eb5cd6f1298d873d4d2b9cf245e09e88e#go_1_21",
       "source": "devbox-search",
-      "version": "1.20.6"
+      "version": "1.21.1",
+      "systems": {
+        "aarch64-darwin": {
+          "store_path": "/nix/store/jkhg33806wygpwpix47d2h5scfgn01i8-go-1.21.1"
+        },
+        "aarch64-linux": {
+          "store_path": "/nix/store/sgkkfw6saficch0mviqyqyw6nj64kzf9-go-1.21.1"
+        },
+        "x86_64-darwin": {
+          "store_path": "/nix/store/w67nj5iqgnz0msi8i12kyh9nhsp2ci9n-go-1.21.1"
+        },
+        "x86_64-linux": {
+          "store_path": "/nix/store/jk0bqfsjijia52vks1wxqnn4s6dxaiqp-go-1.21.1"
+        }
+      }
     },
     "golangci-lint@latest": {
-      "last_modified": "2023-06-30T04:44:22Z",
-      "resolved": "github:NixOS/nixpkgs/3c614fbc76fc152f3e1bc4b2263da6d90adf80fb#golangci-lint",
+      "last_modified": "2023-09-15T06:49:28Z",
+      "resolved": "github:NixOS/nixpkgs/46688f8eb5cd6f1298d873d4d2b9cf245e09e88e#golangci-lint",
       "source": "devbox-search",
-      "version": "1.53.3"
+      "version": "1.54.2",
+      "systems": {
+        "aarch64-darwin": {
+          "store_path": "/nix/store/xvmmv6mzzpx1krr05zmpkzpc1q6dnlcn-golangci-lint-1.54.2"
+        },
+        "aarch64-linux": {
+          "store_path": "/nix/store/x581cpf86b0jdpmdn8shx8vm2rw4c0qb-golangci-lint-1.54.2"
+        },
+        "x86_64-darwin": {
+          "store_path": "/nix/store/wdrdk7xv3fmblhg1grfw5c665mgnsz9z-golangci-lint-1.54.2"
+        },
+        "x86_64-linux": {
+          "store_path": "/nix/store/9nd1nv54dsmkjy0lad38xrilzh71lvjc-golangci-lint-1.54.2"
+        }
+      }
     }
   }
 }

go.work

@@ -1,4 +1,4 @@
-go 1.20
+go 1.21.1
 
 use (
 	./envsec

pkg/go.mod

@@ -1,8 +1,9 @@
 module go.jetpack.io/pkg
 
-go 1.20
+go 1.21
 
 require (
+	github.com/Code-Hex/vz/v3 v3.0.6
 	github.com/cavaliergopher/grab/v3 v3.0.1
 	github.com/codeclysm/extract/v3 v3.1.1
 	github.com/coreos/go-oidc/v3 v3.6.0
@@ -20,9 +21,11 @@ require (
 	github.com/spf13/cobra v1.7.0
 	github.com/stretchr/testify v1.8.4
 	golang.org/x/oauth2 v0.12.0
+	golang.org/x/sys v0.12.0
 )
 
 require (
+	github.com/Code-Hex/go-infinity-channel v1.0.0 // indirect
 	github.com/ProtonMail/go-crypto v0.0.0-20230217124315-7d5c6f04bbb8 // indirect
 	github.com/cloudflare/circl v1.3.3 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
@@ -43,7 +46,7 @@ require (
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/ulikunitz/xz v0.5.11 // indirect
 	golang.org/x/crypto v0.13.0 // indirect
-	golang.org/x/sys v0.12.0 // indirect
+	golang.org/x/mod v0.13.0 // indirect
 	google.golang.org/appengine v1.6.8 // indirect
 	google.golang.org/protobuf v1.31.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect

pkg/go.sum

@@ -1,6 +1,11 @@
+github.com/Code-Hex/go-infinity-channel v1.0.0 h1:M8BWlfDOxq9or9yvF9+YkceoTkDI1pFAqvnP87Zh0Nw=
+github.com/Code-Hex/go-infinity-channel v1.0.0/go.mod h1:5yUVg/Fqao9dAjcpzoQ33WwfdMWmISOrQloDRn3bsvY=
+github.com/Code-Hex/vz/v3 v3.0.6 h1:YoW0ZHbdb9G1lYDw9h/QrbBC5lAI1k9LAZMmTGR/Rpw=
+github.com/Code-Hex/vz/v3 v3.0.6/go.mod h1:xUfvg1VJ5A6ZQNuzQERwXJ7l2ZdTnY6eCy9CIS6/DYQ=
 github.com/ProtonMail/go-crypto v0.0.0-20230217124315-7d5c6f04bbb8 h1:wPbRQzjjwFc0ih8puEVAOFGELsn1zoIIYdxvML7mDxA=
 github.com/ProtonMail/go-crypto v0.0.0-20230217124315-7d5c6f04bbb8/go.mod h1:I0gYDMZ6Z5GRU7l58bNFSkPTFN6Yl12dsUlAZ8xy98g=
 github.com/arduino/go-paths-helper v1.2.0 h1:qDW93PR5IZUN/jzO4rCtexiwF8P4OIcOmcSgAYLZfY4=
+github.com/arduino/go-paths-helper v1.2.0/go.mod h1:HpxtKph+g238EJHq4geEPv9p+gl3v5YYu35Yb+w31Ck=
 github.com/bwesterb/go-ristretto v1.2.0/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0=
 github.com/cavaliergopher/grab/v3 v3.0.1 h1:4z7TkBfmPjmLAAmkkAZNX/6QJ1nNFdv3SdIHXju0Fr4=
 github.com/cavaliergopher/grab/v3 v3.0.1/go.mod h1:1U/KNnD+Ft6JJiYoYBAimKH2XrYptb8Kl3DFGmsjpq4=
@@ -30,6 +35,7 @@ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
 github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
+github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/go-github/v53 v53.2.0 h1:wvz3FyF53v4BK+AsnvCmeNhf8AkTaeh2SoYu/XUvTtI=
 github.com/google/go-github/v53 v53.2.0/go.mod h1:XhFRObz+m/l+UCm9b7KSIC3lT3NWSXGt7mOsAWEloao=
 github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8=
@@ -97,6 +103,8 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y
 golang.org/x/crypto v0.13.0 h1:mvySKfSWJ+UKUii46M40LOvyWfN0s2U+46/jDd0e6Ck=
 golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
+golang.org/x/mod v0.13.0 h1:I/DsJXRlw/8l/0c24sM9yb0T4z9liZTduXvdAWYiysY=
+golang.org/x/mod v0.13.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
@@ -137,6 +145,7 @@ google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqw
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
 gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=

pkg/sandbox/vm/README.md

@@ -0,0 +1,97 @@
+# Experimental Devbox VM
+
+Experimental support for Devbox virtual machines on macOS.
+
+## Usage
+
+The `dxvm` command acts like `devbox shell` except that it launches the Devbox
+environment in a VM.
+
+To create a new VM, run the following:
+
+	cd ~/my/project
+	dxvm -install
+	# Wait for the prompt. This might appear to hang the first time it's run
+	# while downloading the NixOS installer.
+
+	mkdir bootstrap
+	sudo mount -t virtiofs bootstrap bootstrap
+	sudo bootstrap/install.sh
+	sudo shutdown now
+	# ^C to exit
+
+Now that the VM is bootstrapped, you can launch it any time with:
+
+	cd ~/my/project
+	dxvm
+
+The NixOS installer files are cached in `~/.local/state/devbox/vm`. You can
+monitor the ISO in this directory to estimate how far along the download is. The
+final size should be around 800 MiB.
+
+The first time `dxvm` is run in a Devbox project, it creates a `.devbox/vm`
+directory that contains the VM's state and configuration files:
+
+- `log` - error and debug log messages
+- `disk.img` - main disk image, typically mounted as root
+- `id` - an opaque Virtualization Framework machine ID
+
+The following files can be edited (for example, `echo 4 > cpu`) to adjust the
+VM's resources:
+
+- `mem` - the amount of memory (in bytes) to allocate to the VM
+- `cpu` - the number of CPUs to allocate to the VM
+
+## Building
+
+This package uses the macOS Virtualization Framework, and therefore needs CGO.
+Devbox and Nix are unable to download the macOS SDK directly, so some extra
+setup is required:
+
+- macOS Ventura (13) or later
+- XCode command line tools (open Xcode at least once to accept the EULA)
+
+To compile and sign `dxvm` run:
+
+	devbox run build
+
+The `devbox run build` script uses `./cmd/dxvmsign` to sign the Go binary, which
+allows it to use the Virtualization Framework. It's a small wrapper around
+Apple's `codesign` utility.
+
+## Limitations
+
+- Mounting the Devbox project directory was temporarily removed while cleaning
+things up. Needs to be brought back.
+- Only aarch64-linux is implemented right now. Other systems have been tested,
+but they aren't an option in the dxvm command.
+- Using ctrl-c to exit has the unfortunate side-effect of making it impossible
+to interrupt a program in the VM.
+- The host terminal has no way of telling the guest when it has resized (usually
+this is done with SIGWINCH). Running less/vim/etc. in the VM might look messed
+up. Run `stty cols X rows Y` in the VM to manually set the size of your terminal
+window.
+
+# Todo/Ideas
+
+- Support macOS and x86_64-linux.
+- macOS Sonoma added support for VM suspend/resume. This would probably make VM
+start times a lot faster (maybe instant?).
+- Clipboard sharing.
+- Expose sockets for services.
+- Mount /nix/store as an overlay to share packages between VMs.
+- Communicate directly with the host Nix daemon?
+- Disk resizing.
+- Memory balloon (adjust VM memory at runtime).
+- Multiple consoles.
+
+## Docs
+
+Some useful links for learning more about the Virtualization Framework:
+
+- `vz` - Go bindings for Apple's Virtualization Framework
+	- <https://github.com/Code-Hex/vz>
+	- <https://github.com/Code-Hex/vz/wiki>
+	- <https://pkg.go.dev/github.com/Code-Hex/vz/v3>
+- Virtualization Framework
+	- <https://developer.apple.com/documentation/virtualization>

pkg/sandbox/vm/bin/dxvm

@@ -0,0 +1,74 @@
+{ config, pkgs, lib, ... }:
+
+{
+  boot = {
+    consoleLogLevel = 0;
+    kernelParams = [ "quiet" "udev.log_level=3" ];
+
+    initrd = {
+      enable = true;
+      verbose = false;
+    };
+
+    loader = {
+      grub.enable = false;
+      generationsDir = {
+        enable = true;
+        copyKernels = true;
+      };
+    };
+  };
+
+  environment = {
+    # defaultPackages = [ ];
+    systemPackages = with pkgs; [ curl vim ];
+  };
+
+  fileSystems = {
+    "/" = {
+      device = "/dev/vda";
+      fsType = "ext4";
+    };
+  };
+
+  nix = {
+    settings = {
+      auto-optimise-store = true;
+      experimental-features = [ "ca-derivations" "flakes" "nix-command" "repl-flake" ];
+    };
+  };
+
+  nixpkgs = {
+    config = {
+      allowInsecure = true;
+      allowUnfree = true;
+    };
+    hostPlatform = lib.mkDefault "{{.System}}";
+  };
+
+  security.sudo = {
+    extraConfig = "Defaults lecture = never";
+    wheelNeedsPassword = false;
+  };
+
+  services.getty = {
+    autologinUser = "{{.User.Username}}";
+    greetingLine = "";
+  };
+
+  system.stateVersion = "23.05";
+
+  time.timeZone = "{{.TimeZone}}";
+
+  users.users = {
+    root = {
+      hashedPassword = "";
+    };
+    "{{.User.Username}}" = {
+      isNormalUser = true;
+      description = "{{.User.Name}}";
+      hashedPassword = ""; # passwordless login
+      extraGroups = [ "wheel" ];
+    };
+  };
+}

pkg/sandbox/vm/bootstrap/configuration.nix

@@ -0,0 +1,10 @@
+#!/bin/sh
+
+mkfs.ext4 /dev/vda
+mount -t ext4 /dev/vda /mnt
+mkdir -p /mnt/nix /mnt/boot /mnt/etc/nixos
+mount -t virtiofs boot /mnt/boot
+cat << 'EOF' > /mnt/etc/nixos/configuration.nix
+{{ template "configuration.nix" . -}}
+EOF
+nixos-install --no-root-password --show-trace --root /mnt

pkg/sandbox/vm/bootstrap/install.sh

@@ -0,0 +1,39 @@
+package main
+
+import (
+	"context"
+	"flag"
+	"log/slog"
+	"os"
+	"os/signal"
+	"time"
+
+	"go.jetpack.io/pkg/sandbox/vm"
+)
+
+func main() {
+	ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt)
+	defer cancel()
+
+	vm := vm.VM{}
+	flag.StringVar(&vm.HostDataDir, "datadir", ".devbox/vm", "`path` to the directory for saving VM state")
+	flag.BoolVar(&vm.Install, "install", false, "mount NixOS install image")
+	flag.Parse()
+
+	if vm.Install {
+		slog.Debug("downloading the NixOS installer, this make take a few minutes")
+	}
+
+	err := vm.Start(ctx)
+	if err != nil {
+		slog.Error("start virtual machine", "err", err)
+		os.Exit(1)
+	}
+
+	<-ctx.Done()
+	ctx, cancel = context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+	if err := vm.Stop(ctx); err != nil {
+		slog.Error("stop virtual machine", "err", err)
+	}
+}

pkg/sandbox/vm/cmd/dxvm/main.go

@@ -0,0 +1,12 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+	<dict>
+		<key>com.apple.security.network.server</key>
+		<true/>
+		<key>com.apple.security.network.client</key>
+		<true/>
+		<key>com.apple.security.virtualization</key>
+		<true/>
+	</dict>
+</plist>