kola/qemuexec: add --netboot option

For local testing, it's useful to have an easy way to netboot a system.
Enable this by exposing QEMU's builtin support for TFTP serving and
BOOTP option.

For iPXE, one can just directly pass the iPXE script. For PXELINUX/GRUB,
you'll likely want to prepare a netboot directory with your artifacts.

Probably this should be streamlined more in the future, and also deduped
more with the related bits in `metal.go`. But anyway, for now this is
immediately useful in helping to test root on iSCSI locally (via iPXE's
`sanboot` option).

Author: jlebon

docs/cosa/run.md

@@ -192,3 +192,91 @@ TARGET   SOURCE              FSTYPE OPTIONS
 installed system automatically just as the live environment itself was set up.)
 
 This is equivalent to our `kola testiso` multipath tests.
+
+## Netbooting
+
+You can use the `--netboot` option to boot via BOOTP (e.g. iPXE, PXELINUX, GRUB).
+
+### iPXE
+
+This is the simplest since it's the default firmware and doesn't require
+chaining. You can just point to the iPXE script, e.g.:
+
+```
+$ cat tmp/ipxe/boot.ipxe
+#!ipxe
+kernel /<relative path to kernel> initrd=main coreos.live.rootfs_url=<URL> ignition.firstboot ignition.platform.id=metal console=ttyS0 ignition.config.url=<URL>
+initrd --name main /<relative path to initrd>
+boot
+$ cosa run -c --netboot tmp/ipxe/boot.ipxe
+```
+
+(That example requires hosting the rootfs separately, but you can also combine with the initrd.)
+
+Or doing an iSCSI boot:
+
+```
+#!ipxe
+sanboot iscsi:192.168.10.1::::iqn.2023-10.coreos.target.vm:coreos
+```
+
+See [this section](https://docs.fedoraproject.org/en-US/fedora-coreos/live-booting/#_booting_via_ipxe) of the official docs for more info.
+
+### PXELINUX
+
+Point to the `pxelinux.0` binary, likely symlinked, e.g.:
+
+```
+$ tree tmp/pxelinux/
+tmp/pxelinux/
+├── fedora-coreos-38.20231010.dev.0-live-initramfs.x86_64.img -> ../../builds/latest/x86_64/fedora-coreos-38.20231010.dev.0-live-initramfs.x86_64.img
+├── fedora-coreos-38.20231010.dev.0-live-kernel-x86_64 -> ../../builds/latest/x86_64/fedora-coreos-38.20231010.dev.0-live-kernel-x86_64
+├── fedora-coreos-38.20231010.dev.0-live-rootfs.x86_64.img -> ../../builds/latest/x86_64/fedora-coreos-38.20231010.dev.0-live-rootfs.x86_64.img
+├── ldlinux.c32 -> /usr/share/syslinux/ldlinux.c32
+├── pxelinux.0 -> /usr/share/syslinux/pxelinux.0
+└── pxelinux.cfg
+    └── default
+
+2 directories, 6 files
+$ cat tmp/pxelinux/pxelinux.cfg/default
+DEFAULT pxeboot
+TIMEOUT 20
+PROMPT 0
+LABEL pxeboot
+    KERNEL fedora-coreos-38.20231010.dev.0-live-kernel-x86_64
+    APPEND initrd=fedora-coreos-38.20231010.dev.0-live-initramfs.x86_64.img,fedora-coreos-38.20231010.dev.0-live-rootfs.x86_64.img ignition.firstboot ignition.platform.id=metal ignition.config.url=<URL> console=ttyS0
+IPAPPEND 2
+
+$ cosa run -c --netboot tmp/pxelinux/pxelinux.0 -m 4096
+```
+
+See [this section](https://docs.fedoraproject.org/en-US/fedora-coreos/live-booting/#_booting_via_pxe) of the official docs for more info.
+
+### GRUB
+
+Create the netboot dir if not already created:
+
+```
+$ mkdir tmp/grub-netboot
+$ grub2-mknetdir --net-directory tmp/grub-netboot
+```
+
+Create your GRUB config, e.g.:
+
+```
+$ cat tmp/grub-netboot/boot/grub2/grub.cfg
+default=0
+timeout=1
+menuentry "CoreOS (BIOS/UEFI)" {
+        echo "Loading kernel"
+        linux /fedora-coreos-38.20231010.dev.0-live-kernel-x86_64 coreos.live.rootfs_url=<URL> ignition.firstboot ignition.platform.id=metal console=ttyS0 ignition.config.url=<URL>
+        echo "Loading initrd"
+        initrd fedora-coreos-38.20231010.dev.0-live-initramfs.x86_64.img
+}
+```
+
+And point to it and the `core.0` binary:
+
+```
+$ cosa run -c --netboot-dir tmp/grub-netboot --netboot boot/grub2/i386-pc/core.0 -m 4096
+```

mantle/cmd/kola/qemuexec.go

@@ -70,6 +70,9 @@ var (
 	sshCommand string
 
 	additionalNics int
+
+	netboot    string
+	netbootDir string
 )
 
 const maxAdditionalNics = 16
@@ -97,7 +100,8 @@ func init() {
 	cmdQemuExec.Flags().StringVarP(&consoleFile, "console-to-file", "", "", "Filepath in which to save serial console logs")
 	cmdQemuExec.Flags().IntVarP(&additionalNics, "additional-nics", "", 0, "Number of additional NICs to add")
 	cmdQemuExec.Flags().StringVarP(&sshCommand, "ssh-command", "x", "", "Command to execute instead of spawning a shell")
-
+	cmdQemuExec.Flags().StringVarP(&netboot, "netboot", "", "", "Filepath to BOOTP program (e.g. PXELINUX/GRUB binary or iPXE script")
+	cmdQemuExec.Flags().StringVarP(&netbootDir, "netboot-dir", "", "", "Directory to serve over TFTP (default: BOOTP parent dir). If specified, --netboot is relative to this dir.")
 }
 
 func renderFragments(fragments []string, c *conf.Conf) error {
@@ -315,15 +319,15 @@ func runQemuExec(cmd *cobra.Command, args []string) error {
 	if kola.QEMUOptions.Firmware != "" {
 		builder.Firmware = kola.QEMUOptions.Firmware
 	}
-	if kola.QEMUOptions.DiskImage != "" {
+	if kola.QEMUOptions.DiskImage != "" && netboot == "" {
 		if err := builder.AddBootDisk(buildDiskFromOptions()); err != nil {
 			return err
 		}
 		if err != nil {
 			return err
 		}
 	}
-	if kola.QEMUIsoOptions.IsoPath != "" {
+	if kola.QEMUIsoOptions.IsoPath != "" && netboot == "" {
 		err := builder.AddIso(kola.QEMUIsoOptions.IsoPath, "bootindex=3", kola.QEMUIsoOptions.AsDisk)
 		if err != nil {
 			return err
@@ -358,6 +362,9 @@ func runQemuExec(cmd *cobra.Command, args []string) error {
 		}
 		builder.EnableUsermodeNetworking(h)
 	}
+	if netboot != "" {
+		builder.SetNetbootP(netboot, netbootDir)
+	}
 	if additionalNics != 0 {
 		if additionalNics < 0 || additionalNics > maxAdditionalNics {
 			return errors.Wrapf(nil, "additional-nics value cannot be negative or greater than %d", maxAdditionalNics)

mantle/platform/qemu.go

@@ -481,6 +481,8 @@ type QemuBuilder struct {
 	RestrictNetworking        bool
 	requestedHostForwardPorts []HostForwardPort
 	additionalNics            int
+	netbootP                  string
+	netbootDir                string
 
 	finalized bool
 	diskID    uint
@@ -602,6 +604,12 @@ func (builder *QemuBuilder) EnableUsermodeNetworking(h []HostForwardPort) {
 	builder.requestedHostForwardPorts = h
 }
 
+func (builder *QemuBuilder) SetNetbootP(filename, dir string) {
+	builder.UsermodeNetworking = true
+	builder.netbootP = filename
+	builder.netbootDir = dir
+}
+
 func (builder *QemuBuilder) AddAdditionalNics(additionalNics int) {
 	builder.additionalNics = additionalNics
 }
@@ -629,6 +637,31 @@ func (builder *QemuBuilder) setupNetworking() error {
 	if builder.RestrictNetworking {
 		netdev += ",restrict=on"
 	}
+	if builder.netbootP != "" {
+		// do an early stat so we fail with a nicer error now instead of in the VM
+		if _, err := os.Stat(filepath.Join(builder.netbootDir, builder.netbootP)); err != nil {
+			return err
+		}
+		tftpDir := ""
+		relpath := ""
+		if builder.netbootDir == "" {
+			absPath, err := filepath.Abs(builder.netbootP)
+			if err != nil {
+				return err
+			}
+			tftpDir = filepath.Dir(absPath)
+			relpath = filepath.Base(absPath)
+		} else {
+			absPath, err := filepath.Abs(builder.netbootDir)
+			if err != nil {
+				return err
+			}
+			tftpDir = absPath
+			relpath = builder.netbootP
+		}
+		netdev += fmt.Sprintf(",tftp=%s,bootfile=/%s", tftpDir, relpath)
+		builder.Append("-boot", "order=n")
+	}
 
 	builder.Append("-netdev", netdev, "-device", virtio(builder.architecture, "net", "netdev=eth0"))
 	return nil