diff --git a/.gitignore b/.gitignore index 47eca825..b1d85784 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ # ignore files by Jetbrains IDEs *.idea +bin/ diff --git a/devbox.json b/devbox.json index e81be812..001cf65f 100644 --- a/devbox.json +++ b/devbox.json @@ -3,6 +3,14 @@ "go@latest", "golangci-lint@latest" ], + "env": { + "AR": "/usr/bin/ar", + "AS": "/usr/bin/as", + "CC": "/usr/bin/cc", + "CXX": "/usr/bin/cxx", + "LD": "/usr/bin/ld", + "NM": "/usr/bin/nm" + }, "shell": { "init_hook": [ "echo 'Welcome to devbox!' > /dev/null" diff --git a/devbox.lock b/devbox.lock index 7e44c983..1809a786 100644 --- a/devbox.lock +++ b/devbox.lock @@ -2,16 +2,44 @@ "lockfile_version": "1", "packages": { "go@latest": { - "last_modified": "2023-07-30T12:29:02Z", - "resolved": "github:NixOS/nixpkgs/3acb5c4264c490e7714d503c7166a3fde0c51324#go", + "last_modified": "2023-09-15T06:49:28Z", + "resolved": "github:NixOS/nixpkgs/46688f8eb5cd6f1298d873d4d2b9cf245e09e88e#go_1_21", "source": "devbox-search", - "version": "1.20.6" + "version": "1.21.1", + "systems": { + "aarch64-darwin": { + "store_path": "/nix/store/jkhg33806wygpwpix47d2h5scfgn01i8-go-1.21.1" + }, + "aarch64-linux": { + "store_path": "/nix/store/sgkkfw6saficch0mviqyqyw6nj64kzf9-go-1.21.1" + }, + "x86_64-darwin": { + "store_path": "/nix/store/w67nj5iqgnz0msi8i12kyh9nhsp2ci9n-go-1.21.1" + }, + "x86_64-linux": { + "store_path": "/nix/store/jk0bqfsjijia52vks1wxqnn4s6dxaiqp-go-1.21.1" + } + } }, "golangci-lint@latest": { - "last_modified": "2023-06-30T04:44:22Z", - "resolved": "github:NixOS/nixpkgs/3c614fbc76fc152f3e1bc4b2263da6d90adf80fb#golangci-lint", + "last_modified": "2023-09-15T06:49:28Z", + "resolved": "github:NixOS/nixpkgs/46688f8eb5cd6f1298d873d4d2b9cf245e09e88e#golangci-lint", "source": "devbox-search", - "version": "1.53.3" + "version": "1.54.2", + "systems": { + "aarch64-darwin": { + "store_path": "/nix/store/xvmmv6mzzpx1krr05zmpkzpc1q6dnlcn-golangci-lint-1.54.2" + }, + "aarch64-linux": { + "store_path": "/nix/store/x581cpf86b0jdpmdn8shx8vm2rw4c0qb-golangci-lint-1.54.2" + }, + "x86_64-darwin": { + "store_path": "/nix/store/wdrdk7xv3fmblhg1grfw5c665mgnsz9z-golangci-lint-1.54.2" + }, + "x86_64-linux": { + "store_path": "/nix/store/9nd1nv54dsmkjy0lad38xrilzh71lvjc-golangci-lint-1.54.2" + } + } } } } diff --git a/go.work b/go.work index 48115733..ede4b92b 100644 --- a/go.work +++ b/go.work @@ -1,4 +1,4 @@ -go 1.20 +go 1.21.1 use ( ./envsec diff --git a/pkg/go.mod b/pkg/go.mod index 5f39c52c..046ca598 100644 --- a/pkg/go.mod +++ b/pkg/go.mod @@ -1,8 +1,9 @@ module go.jetpack.io/pkg -go 1.20 +go 1.21 require ( + github.com/Code-Hex/vz/v3 v3.0.6 github.com/cavaliergopher/grab/v3 v3.0.1 github.com/codeclysm/extract/v3 v3.1.1 github.com/coreos/go-oidc/v3 v3.6.0 @@ -20,9 +21,11 @@ require ( github.com/spf13/cobra v1.7.0 github.com/stretchr/testify v1.8.4 golang.org/x/oauth2 v0.12.0 + golang.org/x/sys v0.12.0 ) require ( + github.com/Code-Hex/go-infinity-channel v1.0.0 // indirect github.com/ProtonMail/go-crypto v0.0.0-20230217124315-7d5c6f04bbb8 // indirect github.com/cloudflare/circl v1.3.3 // indirect github.com/davecgh/go-spew v1.1.1 // indirect @@ -43,7 +46,7 @@ require ( github.com/spf13/pflag v1.0.5 // indirect github.com/ulikunitz/xz v0.5.11 // indirect golang.org/x/crypto v0.13.0 // indirect - golang.org/x/sys v0.12.0 // indirect + golang.org/x/mod v0.13.0 // indirect google.golang.org/appengine v1.6.8 // indirect google.golang.org/protobuf v1.31.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/pkg/go.sum b/pkg/go.sum index d4411039..847665cc 100644 --- a/pkg/go.sum +++ b/pkg/go.sum @@ -1,6 +1,11 @@ +github.com/Code-Hex/go-infinity-channel v1.0.0 h1:M8BWlfDOxq9or9yvF9+YkceoTkDI1pFAqvnP87Zh0Nw= +github.com/Code-Hex/go-infinity-channel v1.0.0/go.mod h1:5yUVg/Fqao9dAjcpzoQ33WwfdMWmISOrQloDRn3bsvY= +github.com/Code-Hex/vz/v3 v3.0.6 h1:YoW0ZHbdb9G1lYDw9h/QrbBC5lAI1k9LAZMmTGR/Rpw= +github.com/Code-Hex/vz/v3 v3.0.6/go.mod h1:xUfvg1VJ5A6ZQNuzQERwXJ7l2ZdTnY6eCy9CIS6/DYQ= github.com/ProtonMail/go-crypto v0.0.0-20230217124315-7d5c6f04bbb8 h1:wPbRQzjjwFc0ih8puEVAOFGELsn1zoIIYdxvML7mDxA= github.com/ProtonMail/go-crypto v0.0.0-20230217124315-7d5c6f04bbb8/go.mod h1:I0gYDMZ6Z5GRU7l58bNFSkPTFN6Yl12dsUlAZ8xy98g= github.com/arduino/go-paths-helper v1.2.0 h1:qDW93PR5IZUN/jzO4rCtexiwF8P4OIcOmcSgAYLZfY4= +github.com/arduino/go-paths-helper v1.2.0/go.mod h1:HpxtKph+g238EJHq4geEPv9p+gl3v5YYu35Yb+w31Ck= github.com/bwesterb/go-ristretto v1.2.0/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= github.com/cavaliergopher/grab/v3 v3.0.1 h1:4z7TkBfmPjmLAAmkkAZNX/6QJ1nNFdv3SdIHXju0Fr4= github.com/cavaliergopher/grab/v3 v3.0.1/go.mod h1:1U/KNnD+Ft6JJiYoYBAimKH2XrYptb8Kl3DFGmsjpq4= @@ -30,6 +35,7 @@ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-github/v53 v53.2.0 h1:wvz3FyF53v4BK+AsnvCmeNhf8AkTaeh2SoYu/XUvTtI= github.com/google/go-github/v53 v53.2.0/go.mod h1:XhFRObz+m/l+UCm9b7KSIC3lT3NWSXGt7mOsAWEloao= github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= @@ -97,6 +103,8 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.13.0 h1:mvySKfSWJ+UKUii46M40LOvyWfN0s2U+46/jDd0e6Ck= golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.13.0 h1:I/DsJXRlw/8l/0c24sM9yb0T4z9liZTduXvdAWYiysY= +golang.org/x/mod v0.13.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= @@ -137,6 +145,7 @@ google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqw gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/pkg/sandbox/vm/README.md b/pkg/sandbox/vm/README.md new file mode 100644 index 00000000..2599e4d7 --- /dev/null +++ b/pkg/sandbox/vm/README.md @@ -0,0 +1,101 @@ +# Experimental Devbox VM + +Experimental support for Devbox virtual machines on macOS. + +## Usage + +The `dxvm` command acts like `devbox shell` except that it launches the Devbox +environment in a VM. + +To create a new VM, run with the `-install` flag inside a Devbox project +directory: + + cd ~/my/project + dxvm -install + +The VM will start, install NixOS, and then reboot into a shell. This step might +appear to hang at times while it downloads NixOS. + +After the VM is created, you no longer need the `-install` flag: + + cd ~/my/project + dxvm + +The NixOS installer files are cached in `~/.local/state/devbox/vm`. You can +monitor the ISO in this directory to estimate how far along the download is. The +final size should be around 800 MiB. + +The first time `dxvm` is run in a Devbox project, it creates a `.devbox/vm` +directory that contains the VM's state and configuration files: + +- `log` - error and debug log messages +- `console` - the Linux kernel console output +- `disk.img` - main disk image, typically mounted as root +- `id` - an opaque Virtualization Framework machine ID + +The following files can be edited (for example, `echo 4 > cpu`) to adjust the +VM's resources: + +- `mem` - the amount of memory (in bytes) to allocate to the VM +- `cpu` - the number of CPUs to allocate to the VM + +There are two directories shared between the host and guest machines: + +- `boot -> /boot` - gives the host access to the NixOS kernel and initrd so it + can create a bootloader +- `bootstrap -> ~/bootstrap` - contains a script for bootstrapping a new VM from + a vanilla NixOS installer ISO (only mounted with `-install`) + +## Building + +This package uses the macOS Virtualization Framework, and therefore needs CGO. +Devbox and Nix are unable to download the macOS SDK directly, so some extra +setup is required: + +- macOS Ventura (13) or later +- XCode command line tools (open Xcode at least once to accept the EULA) + +To compile and sign `dxvm` run: + + devbox run build + +It's okay if it prints a couple of warnings about duplicate libraries and +replacing the code-signing signature. + +The `devbox run build` script uses `./cmd/dxvmsign` to sign the Go binary, which +allows it to use the Virtualization Framework. It's a small wrapper around +Apple's `codesign` utility. + +## Limitations + +- Intel macOS hasn't been tested yet. +- Using ctrl-c to exit has the unfortunate side-effect of making it impossible +to interrupt a program in the VM. +- The host terminal has no way of telling the guest when it has resized (usually +this is done with SIGWINCH). Running less/vim/etc. in the VM might look messed +up. Run `stty cols X rows Y` in the VM to manually set the size of your terminal +window. + +# Todo/Ideas + +- Support macOS and x86_64-linux. +- macOS Sonoma added support for VM suspend/resume. This would probably make VM +start times a lot faster (maybe instant?). +- Clipboard sharing. +- Expose sockets for services. +- Mount /nix/store as an overlay to share packages between VMs. +- Communicate directly with the host Nix daemon? +- Disk resizing. +- Memory balloon (adjust VM memory at runtime). +- Multiple consoles. + +## Docs + +Some useful links for learning more about the Virtualization Framework: + +- `vz` - Go bindings for Apple's Virtualization Framework + - + - + - +- Virtualization Framework + - diff --git a/pkg/sandbox/vm/bootstrap/configuration.nix b/pkg/sandbox/vm/bootstrap/configuration.nix new file mode 100644 index 00000000..55e9606b --- /dev/null +++ b/pkg/sandbox/vm/bootstrap/configuration.nix @@ -0,0 +1,94 @@ +{ config, pkgs, lib, ... }: + +{ + boot = { + consoleLogLevel = 0; + kernelParams = [ "quiet" "udev.log_level=3" ]; + + initrd = { + enable = true; + verbose = false; + }; + + loader = { + grub.enable = false; + generationsDir = { + enable = true; + copyKernels = true; + }; + }; + }; + + environment = { + defaultPackages = [ ]; + systemPackages = with pkgs; [ + curl + git + vim + ((builtins.getFlake "github:jetpack-io/devbox?ref=gcurtis/flake").packages."{{.System}}".default) + ]; + }; + + fileSystems = { + "/" = { + device = "/dev/vda"; + fsType = "ext4"; + }; + "/boot" = { + device = "boot"; + fsType = "virtiofs"; + }; + "/home/{{.User.Username}}/devbox" = { + device = "home"; + fsType = "virtiofs"; + options = [ "nofail" ]; + }; + }; + + nix = { + settings = { + auto-optimise-store = true; + experimental-features = [ "ca-derivations" "flakes" "nix-command" "repl-flake" ]; + }; + }; + + nixpkgs = { + config = { + allowInsecure = true; + allowUnfree = true; + }; + hostPlatform = lib.mkDefault "{{.System}}"; + }; + + programs.bash.promptInit = "PS1='dxvm\$ '"; + + security.sudo = { + extraConfig = "Defaults lecture = never"; + wheelNeedsPassword = false; + }; + + services.getty = { + autologinUser = "{{.User.Username}}"; + greetingLine = lib.mkForce ""; + helpLine = lib.mkForce ""; + extraArgs = [ "--skip-login" "--nohostname" "--noissue" "--noclear" "--nonewline" "--8bits" ]; + }; + + system.stateVersion = "23.05"; + + time.timeZone = "{{.TimeZone}}"; + + users.users = { + root = { + hashedPassword = ""; + }; + "{{.User.Username}}" = { + isNormalUser = true; + description = "{{.User.Name}}"; + hashedPassword = ""; # passwordless login + extraGroups = [ "wheel" ]; + }; + }; + + virtualisation.rosetta.enable = {{.Rosetta}}; +} diff --git a/pkg/sandbox/vm/bootstrap/install.sh b/pkg/sandbox/vm/bootstrap/install.sh new file mode 100644 index 00000000..9bdca663 --- /dev/null +++ b/pkg/sandbox/vm/bootstrap/install.sh @@ -0,0 +1,10 @@ +#!/bin/sh + +mkfs.ext4 /dev/vda +mount -t ext4 /dev/vda /mnt +mkdir -p /mnt/nix /mnt/boot /mnt/etc/nixos +mount -t virtiofs boot /mnt/boot +cat << 'EOF' > /mnt/etc/nixos/configuration.nix +{{ template "configuration.nix" . -}} +EOF +NIX_CONFIG="experimental-features = nix-command flakes" nixos-install --no-root-password --show-trace --root /mnt diff --git a/pkg/sandbox/vm/cmd/dxvm/main.go b/pkg/sandbox/vm/cmd/dxvm/main.go new file mode 100644 index 00000000..eaa83995 --- /dev/null +++ b/pkg/sandbox/vm/cmd/dxvm/main.go @@ -0,0 +1,99 @@ +package main + +import ( + "cmp" + "context" + "flag" + "fmt" + "io/fs" + "log/slog" + "os" + "os/signal" + "path/filepath" + "slices" + "time" + + "go.jetpack.io/pkg/sandbox/vm" +) + +func main() { + ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt) + defer cancel() + + dataDir := "./dxvm" + devboxDir, devboxDirFound, err := findDevboxDir() + if err != nil { + fmt.Fprintf(os.Stderr, "no devbox.json found, using %s for state: %v\n", dataDir, err) + } else if !devboxDirFound { + fmt.Fprintf(os.Stderr, "no devbox.json found, using %s for state: searched up to %s\n", dataDir, devboxDir) + } else { + dataDir = filepath.Join(devboxDir, ".devbox", "vm") + } + + dxvm := vm.VM{} + flag.StringVar(&dxvm.HostDataDir, "datadir", dataDir, "`path` to the directory for saving VM state") + flag.BoolVar(&dxvm.Install, "install", false, "mount NixOS install image") + flag.Parse() + + if dxvm.Install { + slog.Debug("downloading the NixOS installer, this make take a few minutes") + } else if devboxDirFound { + dxvm.SharedDirectories = append(dxvm.SharedDirectories, vm.SharedDirectory{ + Path: devboxDir, + HomeDir: true, + ReadOnly: false, + }) + fmt.Fprintln(os.Stderr, "booting virtual machine") + } + + go func() { + <-ctx.Done() + ctx, cancel = context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + if err := dxvm.Stop(ctx); err != nil { + slog.Error("stop virtual machine", "err", err) + } + }() + if err := dxvm.Run(ctx); err != nil { + slog.Error("run virtual machine install", "err", err) + os.Exit(1) + } + + // Restart if we just finished bootrapping a new VM. + if dxvm.Install { + fmt.Fprintln(os.Stderr, "virtual machine created successfully") + dxvm.Install = false + if err := dxvm.Run(ctx); err != nil { + slog.Error("run virtual machine", "err", err) + os.Exit(1) + } + } +} + +func findDevboxDir() (dir string, found bool, err error) { + dir = "." + if wd, err := os.Getwd(); err == nil { + dir = wd + } + + home, _ := os.UserHomeDir() + vol := filepath.VolumeName(dir) + for { + // Refuse to go past the user's home directory or search root. + if dir == "" || dir == "/" || dir == home || dir == vol { + return dir, false, nil + } + entries, err := os.ReadDir(dir) + if err != nil { + return "", false, nil + } + _, found := slices.BinarySearchFunc(entries, "devbox.json", func(e fs.DirEntry, t string) int { + return cmp.Compare(e.Name(), t) + }) + if found { + return dir, true, nil + } + dir = filepath.Dir(dir) + } +} diff --git a/pkg/sandbox/vm/cmd/dxvmsign/dxvm.entitlements b/pkg/sandbox/vm/cmd/dxvmsign/dxvm.entitlements new file mode 100644 index 00000000..6336579b --- /dev/null +++ b/pkg/sandbox/vm/cmd/dxvmsign/dxvm.entitlements @@ -0,0 +1,12 @@ + + + + + com.apple.security.network.server + + com.apple.security.network.client + + com.apple.security.virtualization + + + diff --git a/pkg/sandbox/vm/cmd/dxvmsign/main.go b/pkg/sandbox/vm/cmd/dxvmsign/main.go new file mode 100644 index 00000000..0afc67dc --- /dev/null +++ b/pkg/sandbox/vm/cmd/dxvmsign/main.go @@ -0,0 +1,68 @@ +// codesign signs a dxvm binary using the macOS codesign utility. +package main + +import ( + "context" + _ "embed" + "errors" + "fmt" + "os" + "os/exec" + "os/signal" +) + +//go:embed dxvm.entitlements +var entitlements []byte + +func main() { + ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt) + defer cancel() + + if len(os.Args) < 2 { + fmt.Fprintf(os.Stderr, "usage: %s path\n\n%[1]s signs a dxvm binary using the macOS codesign utility.\n", os.Args[0]) + os.Exit(2) + } + + exe, err := exec.LookPath("/usr/bin/codesign") + if err != nil { + fmt.Fprintln(os.Stderr, "error: /usr/bin/codesign not found (did you run xcode-select --install)") + os.Exit(1) + } + + entitlements, err := entitlementsPath() + if err != nil { + fmt.Fprintf(os.Stderr, "error: create entitlements file: %v\n", err) + os.Exit(1) + } + defer os.Remove(entitlements) + + const adhocIdentity = "-" + cmd := exec.CommandContext(ctx, exe, "--force", "--entitlements", entitlements, "--sign", adhocIdentity, "bin/dxvm") + cmd.Stderr = os.Stderr + err = cmd.Run() + + var exitErr *exec.ExitError + if errors.As(err, &exitErr) { + os.Exit(exitErr.ExitCode()) + } + if err != nil { + fmt.Fprintf(os.Stderr, "error: run %s: %v\n", exe, err) + os.Exit(1) + } +} + +func entitlementsPath() (string, error) { + f, err := os.CreateTemp("", "dxvm-codesign-") + if err != nil { + fmt.Fprintln(os.Stderr, "error: /usr/bin/codesign not found (did you run xcode-select --install)") + os.Exit(1) + } + if _, err := f.Write(entitlements); err != nil { + f.Close() + return "", err + } + if err := f.Close(); err != nil { + return "", err + } + return f.Name(), nil +} diff --git a/pkg/sandbox/vm/console.go b/pkg/sandbox/vm/console.go new file mode 100644 index 00000000..ae67c704 --- /dev/null +++ b/pkg/sandbox/vm/console.go @@ -0,0 +1,70 @@ +package vm + +import ( + "bufio" + "context" + "fmt" + "io" + "log/slog" + "os" + "strings" + "sync" + "time" + + "github.com/Code-Hex/vz/v3" +) + +func scriptedConsole(ctx context.Context, logger *slog.Logger, prompt string, script []string) (*vz.VirtioConsoleDeviceSerialPortConfiguration, error) { + stdinr, stdinw, err := os.Pipe() + if err != nil { + return nil, fmt.Errorf("create stdin pipe: %v", err) + } + stdoutr, stdoutw, err := os.Pipe() + if err != nil { + return nil, fmt.Errorf("create stdout pipe: %v", err) + } + + go func() { + var idle *time.Timer + idleDur := time.Second + sawPrompt := false + doneWriting := false + scanner := bufio.NewScanner(io.TeeReader(stdoutr, os.Stdout)) + for scanner.Scan() && ctx.Err() == nil { + logger.Debug("install console", "stdout", scanner.Text()) + + if doneWriting { + continue + } + if idle != nil { + doneWriting = !idle.Reset(idleDur) + continue + } + + sawPrompt = sawPrompt || strings.Contains(scanner.Text(), prompt) + if !sawPrompt { + continue + } + idle = time.AfterFunc(idleDur, sync.OnceFunc(func() { + _, err := stdinw.WriteString(strings.Join(script, " && ") + "\n") + if err != nil { + logger.Error("error writing to VM standard input", "err", err) + } + stdinw.Close() + })) + } + if err := scanner.Err(); err != nil { + logger.Error("error reading install console stdout", "err", err) + } + }() + + attach, err := vz.NewFileHandleSerialPortAttachment(stdinr, stdoutw) + if err != nil { + return nil, fmt.Errorf("create serial port attachment: %v", err) + } + config, err := vz.NewVirtioConsoleDeviceSerialPortConfiguration(attach) + if err != nil { + return nil, fmt.Errorf("create serial port configuration: %v", err) + } + return config, nil +} diff --git a/pkg/sandbox/vm/devbox.json b/pkg/sandbox/vm/devbox.json new file mode 100644 index 00000000..0ae97928 --- /dev/null +++ b/pkg/sandbox/vm/devbox.json @@ -0,0 +1,24 @@ +{ + "packages": [ + "go@latest", + "golangci-lint@latest" + ], + "env": { + "AR": "/usr/bin/ar", + "AS": "/usr/bin/as", + "CC": "/usr/bin/cc", + "CXX": "/usr/bin/c++", + "LD": "/usr/bin/ld", + "NM": "/usr/bin/nm", + "PATH": "$PWD/bin:$PATH" + }, + "shell": { + "init_hook": "unset LIBRARY_PATH LD_LIBRARY_PATH", + "scripts": { + "build": [ + "go build -o bin/dxvm ./cmd/dxvm", + "go run ./cmd/dxvmsign bin/dxvm" + ] + } + } +} diff --git a/pkg/sandbox/vm/devbox.lock b/pkg/sandbox/vm/devbox.lock new file mode 100644 index 00000000..1809a786 --- /dev/null +++ b/pkg/sandbox/vm/devbox.lock @@ -0,0 +1,45 @@ +{ + "lockfile_version": "1", + "packages": { + "go@latest": { + "last_modified": "2023-09-15T06:49:28Z", + "resolved": "github:NixOS/nixpkgs/46688f8eb5cd6f1298d873d4d2b9cf245e09e88e#go_1_21", + "source": "devbox-search", + "version": "1.21.1", + "systems": { + "aarch64-darwin": { + "store_path": "/nix/store/jkhg33806wygpwpix47d2h5scfgn01i8-go-1.21.1" + }, + "aarch64-linux": { + "store_path": "/nix/store/sgkkfw6saficch0mviqyqyw6nj64kzf9-go-1.21.1" + }, + "x86_64-darwin": { + "store_path": "/nix/store/w67nj5iqgnz0msi8i12kyh9nhsp2ci9n-go-1.21.1" + }, + "x86_64-linux": { + "store_path": "/nix/store/jk0bqfsjijia52vks1wxqnn4s6dxaiqp-go-1.21.1" + } + } + }, + "golangci-lint@latest": { + "last_modified": "2023-09-15T06:49:28Z", + "resolved": "github:NixOS/nixpkgs/46688f8eb5cd6f1298d873d4d2b9cf245e09e88e#golangci-lint", + "source": "devbox-search", + "version": "1.54.2", + "systems": { + "aarch64-darwin": { + "store_path": "/nix/store/xvmmv6mzzpx1krr05zmpkzpc1q6dnlcn-golangci-lint-1.54.2" + }, + "aarch64-linux": { + "store_path": "/nix/store/x581cpf86b0jdpmdn8shx8vm2rw4c0qb-golangci-lint-1.54.2" + }, + "x86_64-darwin": { + "store_path": "/nix/store/wdrdk7xv3fmblhg1grfw5c665mgnsz9z-golangci-lint-1.54.2" + }, + "x86_64-linux": { + "store_path": "/nix/store/9nd1nv54dsmkjy0lad38xrilzh71lvjc-golangci-lint-1.54.2" + } + } + } + } +} diff --git a/pkg/sandbox/vm/install_darwin.go b/pkg/sandbox/vm/install_darwin.go new file mode 100644 index 00000000..e74f1aff --- /dev/null +++ b/pkg/sandbox/vm/install_darwin.go @@ -0,0 +1,215 @@ +package vm + +import ( + "context" + "embed" + "errors" + "fmt" + "io" + "io/fs" + "log/slog" + "net/http" + "os" + "os/exec" + "os/user" + "path/filepath" + "runtime" + "strings" + "text/template" + + "github.com/Code-Hex/vz/v3" +) + +var installerDir = os.Expand("$XDG_STATE_HOME/devbox/vm", xdgStateHome) + +//go:embed bootstrap +var bootstrapFiles embed.FS + +var nixosConfigTmpl = template.Must(template.ParseFS(bootstrapFiles, "bootstrap/install.sh", "bootstrap/configuration.nix")) + +func (vm *VM) generateBootstrapScript() (dir string, err error) { + currentUser, err := user.Current() + if err != nil { + return "", fmt.Errorf("get current user: %v", err) + } + + localtime, err := os.Readlink("/etc/localtime") + if err != nil { + return "", fmt.Errorf("get current time zone: %v", err) + } + tz, ok := strings.CutPrefix(localtime, "/var/db/timezone/zoneinfo/") + if !ok { + return "", fmt.Errorf("/etc/localtime symlink missing /var/db/timezone/zoneinfo/ prefix") + } + + f, err := os.Create(vm.filePaths.bootstrap) + if errors.Is(err, os.ErrNotExist) { + if err := os.MkdirAll(filepath.Dir(vm.filePaths.bootstrap), 0o700); err != nil { + return "", fmt.Errorf("create directory for install script: %v", err) + } + f, err = os.Create(vm.filePaths.bootstrap) + } + if err != nil { + return "", err + } + defer f.Close() + + system := "aarch64-linux" + if runtime.GOARCH == "amd64" { + system = "x86_64-linux" + } + data := struct { + Rosetta bool + System string + TimeZone string + User *user.User + }{ + Rosetta: runtime.GOOS == "darwin" && runtime.GOARCH == "arm64", + System: system, + TimeZone: tz, + User: currentUser, + } + err = nixosConfigTmpl.Execute(f, data) + if err != nil { + return "", fmt.Errorf("execute template: %v", err) + } + if err := f.Chmod(0o755); err != nil { + return "", fmt.Errorf("make install script executable: %v", err) + } + return filepath.Dir(vm.filePaths.bootstrap), nil +} + +func (vm *VM) attachInstallConsole(ctx context.Context) error { + script := []string{ + "mkdir bootstrap", + "sudo mount -t virtiofs bootstrap bootstrap", + "sudo bootstrap/install.sh", + "sudo shutdown now", + } + config, err := scriptedConsole(ctx, vm.Logger, "nixos login: nixos (automatic login)", script) + if err != nil { + return err + } + vm.config.SetSerialPortsVirtualMachineConfiguration([]*vz.VirtioConsoleDeviceSerialPortConfiguration{config}) + vm.Logger.Debug("attached install console device") + return nil +} + +func (vm *VM) installerBootLoader(ctx context.Context) (*vz.LinuxBootLoader, error) { + vm.Logger.Debug("downloading linux kernel") + kernel, init, initrd, err := downloadInstallerKernel(ctx) + if err != nil { + return nil, fmt.Errorf("download installer kernel: %v", err) + } + vm.Logger.Debug("linux kernel downloaded") + + params := fmt.Sprintf("console=hvc0 root=/dev/sda init=%s boot.shell_on_fail", init) + vm.Logger.Debug("created installer boot loader", "params", params, "installer", vm.Install) + return vz.NewLinuxBootLoader(kernel, + vz.WithInitrd(initrd), + vz.WithCommandLine(params), + ) +} + +func downloadInstallerKernel(ctx context.Context) (kernel, init, initrd string, err error) { + switch runtime.GOARCH { + case "arm64": + kernel = "/nix/store/kp0454y12fhlivdnv6vpbc0drdijmh32-nixos-system-nixos-23.05.3701.e9b4b56e5a20/kernel" + init = "/nix/store/kp0454y12fhlivdnv6vpbc0drdijmh32-nixos-system-nixos-23.05.3701.e9b4b56e5a20/init" + initrd = "/nix/store/kp0454y12fhlivdnv6vpbc0drdijmh32-nixos-system-nixos-23.05.3701.e9b4b56e5a20/initrd" + default: + return "", "", "", fmt.Errorf("unsupported system %s", runtime.GOARCH) + } + + cmd := exec.CommandContext(ctx, "nix-store", "--realise", kernel, init, initrd) + if err := cmd.Run(); err != nil { + return "", "", "", fmt.Errorf("command nix-store --realise: %v", err) + } + return kernel, init, initrd, nil +} + +func (vm *VM) installerDisk(ctx context.Context) (vz.StorageDeviceConfiguration, error) { + iso, err := downloadInstallerISO(ctx, vm.Logger) + if err != nil { + return nil, fmt.Errorf("download installer iso: %v", err) + } + attach, err := vz.NewDiskImageStorageDeviceAttachment(iso, true) + if err != nil { + return nil, fmt.Errorf("create disk image storage device: %v", err) + } + config, err := vz.NewUSBMassStorageDeviceConfiguration(attach) + if err != nil { + return nil, fmt.Errorf("configure disk image as USB mass storage device: %v", err) + } + return config, nil +} + +func downloadInstallerISO(ctx context.Context, logger *slog.Logger) (string, error) { + system := "" + switch runtime.GOARCH { + case "amd64": + system = "x86_64-linux" + case "arm64": + system = "aarch64-linux" + } + url := fmt.Sprintf("https://releases.nixos.org/nixos/23.05/nixos-23.05.3701.e9b4b56e5a20/nixos-minimal-23.05.3701.e9b4b56e5a20-%s.iso", system) + logger.Debug("downloading installer iso", "url", url) + + path := filepath.Join(installerDir, filepath.Base(url)) + flag := os.O_WRONLY | os.O_CREATE | os.O_EXCL + perm := fs.FileMode(0o644) + f, err := os.OpenFile(path, flag, perm) + if errors.Is(err, os.ErrExist) { + return path, nil + } + if errors.Is(err, os.ErrNotExist) { + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + return "", fmt.Errorf("create directory for ISO: %v", err) + } + f, err = os.OpenFile(path, flag, perm) + } + if err != nil { + return "", err + } + defer f.Close() + + req, err := http.NewRequestWithContext(ctx, "GET", url, nil) + if err != nil { + return "", fmt.Errorf("new request: %v", err) + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + return "", fmt.Errorf("do request: %v", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("response status %s", resp.Status) + } + if _, err := io.Copy(f, resp.Body); err != nil { + return "", err + } + if err := f.Close(); err != nil { + return "", fmt.Errorf("close ISO file: %v", err) + } + + logger.Debug("installer iso downloaded") + return path, nil +} + +// xdgStateHome returns the path for XDG_STATE_HOME or ~/.local/state if it +// isn't set. It can be used with os.Expand. +func xdgStateHome(s string) string { + switch s { + case "XDG_STATE_HOME": + if xdg := os.Getenv("XDG_STATE_HOME"); xdg != "" { + return xdg + } + home, err := os.UserHomeDir() + if err != nil { + return os.TempDir() + } + return filepath.Join(home, ".local", "state") + } + return "" +} diff --git a/pkg/sandbox/vm/vm_darwin.go b/pkg/sandbox/vm/vm_darwin.go new file mode 100644 index 00000000..3d5a3b40 --- /dev/null +++ b/pkg/sandbox/vm/vm_darwin.go @@ -0,0 +1,712 @@ +// Package vm implements experimental support for Devbox virtual machines on +// macOS. +package vm + +import ( + "cmp" + "context" + "errors" + "fmt" + "io" + "io/fs" + "log/slog" + "os" + "path/filepath" + "runtime" + "strings" + + "github.com/Code-Hex/vz/v3" + "golang.org/x/sys/unix" +) + +var boot fs.FS + +// Default host resources to allocate to new VMs. +var ( + DefaultCPUs int = 1 + DefaultMemory int = 1 << 31 // 2 GiB + DefaultDisk int64 = 1 << 34 // 16 GiB +) + +// readyMarker is a message that signals to the host that the VM has reached a +// shell prompt. +const readyMarker = "devbox is ready" + +// VM is a Devbox virtual machine. The zero value is a temporary VM that is +// deleted after it stops. +type VM struct { + // ID is a Virtualization Framework machine ID. + ID []byte + + // CPUs is the number of CPU cores to allocate to the VM. For a new VM, + // it defaults to the system-allowed minimum or DefaultCPUs, whichever + // is larger. For existing VMs, it defaults to the value from the + // previous run. + CPUs int + + // Memory is the amount of memory in bytes to allocate to the VM. For a + // new VM, it defaults to the system-allowed minimum or DefaultMemory, + // whichever is larger. For existing VMs, it defaults to the value from + // the previous run. + Memory int + + // DiskSize is the size in bytes of the root disk. It's not possible to + // change the size of an image after it's created. Setting DiskSize has + // no effect on existing VMs. + DiskSize int64 + + // OS is the guest operating system. It must be either "darwin" or + // "linux". Setting OS has no effect on existing VMs. + OS string + + // Arch is the guest machine's architecture. It must be either "aarch64" + // or "x86_64". Setting Arch has no effect on existing VMs. + Arch string + + // Install boots from the NixOS installer ISO instead of the main image. + Install bool + + // SharedDirectories is a list of host directories to share with the + // guest operating system. + SharedDirectories []SharedDirectory + + // HostDataDir is a directory containing the VM's state and + // configuration. If HostDataDir is empty, it is set to a temporary + // directory that is created the first time the VM starts and deleted + // after the VM stops. + HostDataDir string + + // Logger is where the host machine writes logs. It defaults to writing + // them to a file named "log" inside HostDataDir. The logger's handler + // should avoid writing to standard out or standard error so as to not + // interfere with the VM's console output. Set it to a logger with any + // level above slog.LevelError to disable logging. + Logger *slog.Logger + + vzvm *vz.VirtualMachine + config *vz.VirtualMachineConfiguration + filePaths dataDirectory +} + +func (vm *VM) Run(ctx context.Context) error { + var err error + vm.filePaths, err = dataDir(vm.HostDataDir) + if err != nil { + return fmt.Errorf("create directory for virtual machine data: %v", err) + } + if vm.OS == "" { + vm.OS = "linux" + } + if vm.Arch == "" { + vm.Arch = "aarch64" + } + + vm.initLogger() + vm.configureCPUs() + vm.configureMemory() + + loader, err := vm.linuxBootLoader(ctx) + if err != nil { + return fmt.Errorf("create boot loader: %v", err) + } + + vm.Logger.Debug("creating virtual machine", "cpus", vm.CPUs, "memory", vm.Memory) + vm.config, err = vz.NewVirtualMachineConfiguration(loader, uint(vm.CPUs), uint64(vm.Memory)) + if err != nil { + return fmt.Errorf("create virtual machine configuration: %v", err) + } + if vm.Install { + if err := vm.attachInstallConsole(ctx); err != nil { + return fmt.Errorf("attach install console: %v", err) + } + } else { + if err := vm.attachConsole(); err != nil { + return fmt.Errorf("attach console: %v", err) + } + } + if err := vm.attachDisks(ctx); err != nil { + return fmt.Errorf("attach disks: %v", err) + } + if err := vm.attachNetwork(); err != nil { + return fmt.Errorf("attach network: %v", err) + } + if err := vm.attachEntropy(); err != nil { + return fmt.Errorf("attach entropy: %v", err) + } + if err := vm.attachSharedDirs(); err != nil { + return fmt.Errorf("attach shared directories: %v", err) + } + if err := vm.configureLinuxPlatform(); err != nil { + return fmt.Errorf("configure linux platform: %v", err) + } + + valid, err := vm.config.Validate() + if err != nil { + return fmt.Errorf("invalid configuration: %v", err) + } + if !valid { + return fmt.Errorf("invalid configuration") + } + vm.vzvm, err = vz.NewVirtualMachine(vm.config) + if err != nil { + return fmt.Errorf("create virtual machine: %v", err) + } + if err := vm.vzvm.Start(); err != nil { + return err + } + + for state := range vm.vzvm.StateChangedNotify() { + switch state { + case vz.VirtualMachineStateStarting: + vm.Logger.Debug("virtual machine state changed", "state", "starting") + case vz.VirtualMachineStateRunning: + vm.Logger.Debug("virtual machine state changed", "state", "running") + case vz.VirtualMachineStateError: + vm.Logger.Debug("virtual machine state changed", "state", "error") + return nil + case vz.VirtualMachineStateStopping: + vm.Logger.Debug("virtual machine state changed", "state", "stopping") + case vz.VirtualMachineStateStopped: + vm.Logger.Debug("virtual machine state changed", "state", "stopped") + return nil + } + } + return nil +} + +func (vm *VM) Stop(ctx context.Context) error { + if vm == nil || vm.vzvm == nil || vm.vzvm.State() == vz.VirtualMachineStateStopped { + return nil + } + + ch := make(chan error) + go func() { + ok, err := vm.vzvm.RequestStop() + if !ok || err != nil { + vm.vzvm.Stop() + if err != nil { + ch <- fmt.Errorf("request stop: %v", err) + return + } + ch <- fmt.Errorf("invalid machine state for stopping") + return + } + close(ch) + }() + select { + case <-ctx.Done(): + return ctx.Err() + case err := <-ch: + return err + } +} + +func (vm *VM) configureCPUs() { + minCPU := int(vz.VirtualMachineConfigurationMinimumAllowedCPUCount()) + maxCPU := int(vz.VirtualMachineConfigurationMaximumAllowedCPUCount()) + if vm.CPUs == 0 { + vm.loadStateData(vm.filePaths.cpu, &vm.CPUs) + if vm.CPUs == 0 { + vm.CPUs = clamp(DefaultCPUs, minCPU, maxCPU) + vm.saveStateData(vm.filePaths.cpu, vm.CPUs) + return + } + } + clamped := clamp(vm.CPUs, minCPU, maxCPU) + if vm.CPUs != clamped { + vm.CPUs = clamped + vm.saveStateData(vm.filePaths.cpu, vm.CPUs) + } +} + +func (vm *VM) configureMemory() { + minMemory := int(vz.VirtualMachineConfigurationMinimumAllowedMemorySize()) + maxMemory := int(vz.VirtualMachineConfigurationMaximumAllowedMemorySize()) + if vm.Memory == 0 { + vm.loadStateData(vm.filePaths.memory, &vm.Memory) + if vm.Memory == 0 { + vm.Memory = clamp(DefaultMemory, minMemory, maxMemory) + vm.saveStateData(vm.filePaths.memory, vm.Memory) + return + } + } + clamped := clamp(vm.Memory, minMemory, maxMemory) + if vm.Memory != clamped { + vm.Memory = clamped + vm.saveStateData(vm.filePaths.memory, vm.Memory) + } +} + +func clamp[T cmp.Ordered](value, min, max T) T { + if value < min { + return min + } + if value > max { + return max + } + return value +} + +func (vm *VM) linuxBootLoader(ctx context.Context) (*vz.LinuxBootLoader, error) { + if vm.Install { + return vm.installerBootLoader(ctx) + } + + guestInitPath, err := os.Readlink(vm.filePaths.init) + if err != nil { + return nil, fmt.Errorf("determine path to kernel init file inside vm: %v", err) + } + params := fmt.Sprintf("console=hvc1 console=hvc0 root=/dev/vda init=%s quiet boot.shell_on_fail rd.systemd.show_status=false rd.udev.log_level=3 rd.udev.log_priority=3", guestInitPath) + vm.Logger.Debug("created boot loader", "params", params, "installer", vm.Install) + return vz.NewLinuxBootLoader(vm.filePaths.kernel, + vz.WithInitrd(vm.filePaths.initrd), + vz.WithCommandLine(params), + ) +} + +func (vm *VM) efiBootLoader() (*vz.EFIBootLoader, error) { + nvram, err := vm.nvram() + if err != nil { + return nil, err + } + return vz.NewEFIBootLoader(vz.WithEFIVariableStore(nvram)) +} + +func (vm *VM) nvram() (*vz.EFIVariableStore, error) { + f, err := os.OpenFile(vm.filePaths.nvram, os.O_RDWR|os.O_CREATE|os.O_EXCL, os.FileMode(0o600)) + if err != nil && !errors.Is(err, os.ErrExist) { + return nil, fmt.Errorf("open nvram file: %v", err) + } + + created := err == nil + if created { + f.Close() + nvram, err := vz.NewEFIVariableStore(vm.filePaths.nvram, vz.WithCreatingEFIVariableStore()) + if err != nil { + return nil, fmt.Errorf("create nvram variable store: %v", err) + } + return nvram, nil + } + nvram, err := vz.NewEFIVariableStore(vm.filePaths.nvram) + if err != nil { + return nil, fmt.Errorf("load nvram variable store: %v", err) + } + return nvram, nil +} + +func (vm *VM) attachConsole() error { + // Create two consoles: one for the kernel to log to and one for the user. + consoleAttach, err := vz.NewFileSerialPortAttachment(vm.filePaths.console, false) + if err != nil { + return fmt.Errorf("create console serial port attachment: %v", err) + } + consoleConfig, err := vz.NewVirtioConsoleDeviceSerialPortConfiguration(consoleAttach) + if err != nil { + return fmt.Errorf("create console serial port configuration: %v", err) + } + + fd := int(os.Stdin.Fd()) + term, err := unix.IoctlGetTermios(int(fd), unix.TIOCGETA) + if err != nil { + return fmt.Errorf("put stdin in raw mode: get terminal attributes: %v", err) + } + + // See `man termios` for reference. + term.Iflag &^= unix.ICRNL // disable CR-NL mapping + term.Lflag &^= unix.ICANON | unix.ECHO // disable input canoncialization and echo + + // VMIN and VTIME control when a system read() call returns. VMIN is the minimum + // number of characters to wait for and VTIME is the time to wait for them (in + // tenths of a second). Here we're saying to only return after at least 1 byte is + // available and to ignore time entirely. These settings are usually the default, + // but explicitly set them anyway to be safe. + term.Cc[unix.VMIN] = 1 + term.Cc[unix.VTIME] = 0 + + if err := unix.IoctlSetTermios(fd, unix.TIOCSETA, term); err != nil { + return fmt.Errorf("put stdin in raw mode: set terminal attributes: %v", err) + } + attach, err := vz.NewFileHandleSerialPortAttachment(os.Stdin, os.Stdout) + if err != nil { + return fmt.Errorf("create serial port attachment: %v", err) + } + config, err := vz.NewVirtioConsoleDeviceSerialPortConfiguration(attach) + if err != nil { + return fmt.Errorf("create serial port configuration: %v", err) + } + vm.config.SetSerialPortsVirtualMachineConfiguration([]*vz.VirtioConsoleDeviceSerialPortConfiguration{config, consoleConfig}) + vm.Logger.Debug("attached console device") + return nil +} + +func (vm *VM) attachKeyboard() error { + config, err := vz.NewUSBKeyboardConfiguration() + if err != nil { + return fmt.Errorf("create keyboard configuration: %w", err) + } + vm.config.SetKeyboardsVirtualMachineConfiguration([]vz.KeyboardConfiguration{config}) + vm.Logger.Debug("attached usb keyboard device") + return nil +} + +func (vm *VM) attachNetwork() error { + attach, err := vz.NewNATNetworkDeviceAttachment() + if err != nil { + return fmt.Errorf("create network attachment: %v", err) + } + config, err := vz.NewVirtioNetworkDeviceConfiguration(attach) + if err != nil { + return fmt.Errorf("create network configuration: %v", err) + } + mac, err := vz.NewRandomLocallyAdministeredMACAddress() + if err != nil { + return fmt.Errorf("create random MAC address: %v", err) + } + config.SetMACAddress(mac) + vm.config.SetNetworkDevicesVirtualMachineConfiguration([]*vz.VirtioNetworkDeviceConfiguration{config}) + vm.Logger.Debug("attached network device") + return nil +} + +func (vm *VM) attachEntropy() error { + config, err := vz.NewVirtioEntropyDeviceConfiguration() + if err != nil { + return fmt.Errorf("create entropy configuration: %v", err) + } + vm.config.SetEntropyDevicesVirtualMachineConfiguration([]*vz.VirtioEntropyDeviceConfiguration{config}) + vm.Logger.Debug("attached entropy device") + return nil +} + +func (vm *VM) configureLinuxPlatform() error { + err := vm.loadStateData(vm.filePaths.id, &vm.ID) + if err != nil { + return fmt.Errorf("load machine identifier: %v", err) + } + + var id *vz.GenericMachineIdentifier + if len(vm.ID) == 0 { + id, err = vz.NewGenericMachineIdentifier() + if err != nil { + return fmt.Errorf("create machine identifier: %v", err) + } + vm.ID = id.DataRepresentation() + if err := vm.saveStateData(vm.filePaths.id, vm.ID); err != nil { + return fmt.Errorf("save machine identifier: %v", err) + } + vm.Logger.Debug("created new machine identifier") + } else { + id, err = vz.NewGenericMachineIdentifierWithData(vm.ID) + if err != nil { + return fmt.Errorf("load machine identifier: %v", err) + } + vm.Logger.Debug("loaded machine identifier") + } + + platform, err := vz.NewGenericPlatformConfiguration(vz.WithGenericMachineIdentifier(id)) + if err != nil { + return fmt.Errorf("create platform configuration: %v", err) + } + vm.config.SetPlatformVirtualMachineConfiguration(platform) + return nil +} + +func (vm *VM) configureRosetta() (*vz.VirtioFileSystemDeviceConfiguration, error) { + switch vz.LinuxRosettaDirectoryShareAvailability() { + case vz.LinuxRosettaAvailabilityNotSupported: + return nil, fmt.Errorf("this version of macOS doesn't support rosetta") + case vz.LinuxRosettaAvailabilityNotInstalled: + vm.Logger.Debug("starting rosetta install") + if err := vz.LinuxRosettaDirectoryShareInstallRosetta(); err != nil { + return nil, fmt.Errorf("install rosetta: %v", err) + } + vm.Logger.Debug("rosetta installed") + } + + share, err := vz.NewLinuxRosettaDirectoryShare() + if err != nil { + return nil, fmt.Errorf("create rosetta directory share: %v", err) + } + tag := "rosetta" + config, err := vz.NewVirtioFileSystemDeviceConfiguration(tag) + if err != nil { + return nil, fmt.Errorf("create virtiofs configuration %s: %v", tag, err) + } + config.SetDirectoryShare(share) + return config, nil +} + +func (vm *VM) attachSharedDirs() error { + var configs []vz.DirectorySharingDeviceConfiguration + if vm.Install { + bootstrapDir, err := vm.generateBootstrapScript() + if err != nil { + return fmt.Errorf("generate bootstrap files: %v", err) + } + config, err := vm.configureSingleDirShare(SharedDirectory{ + Path: bootstrapDir, + ReadOnly: true, + }) + if err != nil { + return err + } + configs = append(configs, config) + } + + bootDir := filepath.Dir(vm.filePaths.kernel) + if err := os.MkdirAll(bootDir, 0o700); err != nil { + return err + } + config, err := vm.configureSingleDirShare(SharedDirectory{ + Path: bootDir, + ReadOnly: false, + }) + if err != nil { + return err + } + configs = append(configs, config) + + // Track home directories separately so we can share them via a multiple + // directory share that gets mounted in /home/{{.User.Username}}. + userDirs := make([]SharedDirectory, 0, len(vm.SharedDirectories)) + for _, dir := range vm.SharedDirectories { + if dir.HomeDir { + userDirs = append(userDirs, dir) + continue + } + config, err := vm.configureSingleDirShare(dir) + if err != nil { + return err + } + configs = append(configs, config) + } + config, err = vm.configureMultipleDirShare("home", userDirs...) + if err != nil { + return err + } + configs = append(configs, config) + + if runtime.GOARCH == "arm64" { + rosetta, err := vm.configureRosetta() + if err != nil { + return fmt.Errorf("configure rosetta: %v", err) + } + configs = append(configs, rosetta) + } + + vm.config.SetDirectorySharingDevicesVirtualMachineConfiguration(configs) + vm.Logger.Debug("attached shared directories", "count", len(configs)) + return nil +} + +func (vm *VM) configureSingleDirShare(sd SharedDirectory) (*vz.VirtioFileSystemDeviceConfiguration, error) { + dir, err := vz.NewSharedDirectory(sd.Path, sd.ReadOnly) + if err != nil { + return nil, fmt.Errorf("create shared directory %s: %v", sd.Path, err) + } + share, err := vz.NewSingleDirectoryShare(dir) + if err != nil { + return nil, fmt.Errorf("create directory share %s: %v", sd.Path, err) + } + tag := filepath.Base(sd.Path) + config, err := vz.NewVirtioFileSystemDeviceConfiguration(tag) + if err != nil { + return nil, fmt.Errorf("create virtiofs configuration %s -> %s: %v", sd.Path, tag, err) + } + config.SetDirectoryShare(share) + vm.Logger.Debug("configured shared directory", "dir", sd.Path, "readonly", sd.ReadOnly) + return config, nil +} + +func (vm *VM) configureMultipleDirShare(tag string, sd ...SharedDirectory) (*vz.VirtioFileSystemDeviceConfiguration, error) { + dirNames := make([]string, len(sd)) + dirs := make(map[string]*vz.SharedDirectory, len(sd)) + for i, dir := range sd { + vzdir, err := vz.NewSharedDirectory(dir.Path, dir.ReadOnly) + if err != nil { + return nil, fmt.Errorf("create shared directory %s: %v", dir.Path, err) + } + dirs[filepath.Base(dir.Path)] = vzdir + dirNames[i] = dir.String() + } + share, err := vz.NewMultipleDirectoryShare(dirs) + if err != nil { + return nil, fmt.Errorf("create multiple directory share for %s: %v", strings.Join(dirNames, ", "), err) + } + config, err := vz.NewVirtioFileSystemDeviceConfiguration(tag) + if err != nil { + return nil, fmt.Errorf("create virtiofs configuration (%s) -> %s: %v", strings.Join(dirNames, ", "), tag, err) + } + config.SetDirectoryShare(share) + vm.Logger.Debug("configured multiple shared directories", "dirs", dirNames) + return config, nil +} + +func (vm *VM) attachDisks(ctx context.Context) error { + root, err := vm.rootDisk() + if err != nil { + return fmt.Errorf("create root disk: %v", err) + } + + disks := []vz.StorageDeviceConfiguration{root} + if vm.Install { + iso, err := vm.installerDisk(ctx) + if err != nil { + return fmt.Errorf("create installer disk: %v", err) + } + disks = append(disks, iso) + } + vm.config.SetStorageDevicesVirtualMachineConfiguration(disks) + vm.Logger.Debug("attached disks", "count", len(disks)) + return nil +} + +func (vm *VM) rootDisk() (vz.StorageDeviceConfiguration, error) { + f, err := os.OpenFile(vm.filePaths.image, os.O_RDWR|os.O_CREATE|os.O_EXCL, os.FileMode(0o600)) + if err != nil && !errors.Is(err, os.ErrExist) { + return nil, err + } + + created := err == nil + if created { + if vm.DiskSize == 0 { + vm.DiskSize = DefaultDisk + } + err := f.Truncate(vm.DiskSize) + f.Close() + if err != nil { + return nil, fmt.Errorf("truncate new root disk image: %v", err) + } + } + + attach, err := vz.NewDiskImageStorageDeviceAttachment(vm.filePaths.image, false) + if err != nil { + return nil, fmt.Errorf("create root disk image storage device: %v", err) + } + config, err := vz.NewVirtioBlockDeviceConfiguration(attach) + if err != nil { + return nil, fmt.Errorf("configure root disk image as block device: %v", err) + } + return config, nil +} + +func (vm *VM) loadStateData(path string, value any) error { + f, err := os.OpenFile(path, os.O_RDONLY, 0) + if errors.Is(err, os.ErrNotExist) { + return nil + } + if err != nil { + return err + } + defer f.Close() + + switch value.(type) { + case []byte: + _, err = fmt.Fscanf(f, "%x", value) + default: + _, err = fmt.Fscanf(f, "%v", value) + } + if errors.Is(err, io.ErrUnexpectedEOF) { + return nil + } + return err +} + +func (vm *VM) saveStateData(path string, value any) error { + f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, os.FileMode(0o644)) + if err != nil { + return err + } + defer f.Close() + + switch value.(type) { + case []byte: + _, err = fmt.Fprintf(f, "%x\n", value) + default: + _, err = fmt.Fprintf(f, "%v\n", value) + } + return err +} + +func (vm *VM) initLogger() { + f, err := os.OpenFile(vm.filePaths.log, os.O_WRONLY|os.O_CREATE|os.O_APPEND, os.FileMode(0o644)) + if err != nil { + vm.Logger = slog.Default() + vm.Logger.Error("could not create log file, using slog.Default()", "err", err) + return + } + vm.Logger = slog.New(slog.NewTextHandler(f, &slog.HandlerOptions{ + AddSource: true, + Level: slog.LevelDebug, + })) +} + +type SharedDirectory struct { + Path string + HomeDir bool + ReadOnly bool +} + +func (sd SharedDirectory) String() string { + if sd.ReadOnly { + return sd.Path + ":ro" + } + return sd.Path + ":rw" +} + +type dataDirectory struct { + path string + isTemp bool + + init string + initrd string + kernel string + nvram string + bootstrap string + image string + cpu string + memory string + id string + log string + console string +} + +func dataDir(dir string) (dataDirectory, error) { + isTemp := false + if dir == "" { + var err error + dir, err = os.MkdirTemp("", "devboxvm-") + if err != nil { + return dataDirectory{}, fmt.Errorf("create temporary directory for virtual machine data: %v", err) + } + isTemp = true + } else { + if err := os.MkdirAll(dir, 0o700); err != nil { + return dataDirectory{}, fmt.Errorf("create directory for virtual machine data: %v", err) + } + } + return dataDirectory{ + path: dir, + isTemp: isTemp, + init: filepath.Join(dir, "boot", "default", "init"), + initrd: filepath.Join(dir, "boot", "nixos-initrd"), + kernel: filepath.Join(dir, "boot", "nixos-kernel"), + nvram: filepath.Join(dir, "nvram"), + bootstrap: filepath.Join(dir, "bootstrap", "install.sh"), + cpu: filepath.Join(dir, "cpu"), + memory: filepath.Join(dir, "mem"), + image: filepath.Join(dir, "disk.img"), + id: filepath.Join(dir, "id"), + log: filepath.Join(dir, "log"), + console: filepath.Join(dir, "console"), + }, nil +} + +func (d dataDirectory) cleanup() error { + if !d.isTemp { + return nil + } + slog.Debug("clean up temporary data directory", "dir", d.path) + // TODO(gcurtis): actually do the os.RemoveAll after this is tested. + return nil +}