diff --git a/ci/README.md b/ci/README.md new file mode 100644 index 00000000000000..fbd89a34dd78fc --- /dev/null +++ b/ci/README.md @@ -0,0 +1,84 @@ +# CI + +This directory contains scripts for building CI images for Bun. + +## Building + +### `macOS` + +On macOS, images are built using [`tart`](https://tart.run/), a tool that abstracts over the [`Virtualization.Framework`](https://developer.apple.com/documentation/virtualization) APIs, to run macOS VMs. + +To install the dependencies required, run: + +```sh +$ cd ci +$ bun run bootstrap +``` + +To build a vanilla macOS VM, run: + +```sh +$ bun run build:darwin-aarch64-vanilla +``` + +This builds a vanilla macOS VM with the current macOS release on your machine. It runs scripts to disable things like spotlight and siri, but it does not install any software. + +> Note: The image size is 50GB, so make sure you have enough disk space. + +If you want to build a specific macOS release, you can run: + +```sh +$ bun run build:darwin-aarch64-vanilla-15 +``` + +> Note: You cannot build a newer release of macOS on an older macOS machine. + +To build a macOS VM with software installed to build and test Bun, run: + +```sh +$ bun run build:darwin-aarch64 +``` + +## Running + +### `macOS` + +## How To + +### Support a new macOS release + +1. Visit [`ipsw.me`](https://ipsw.me/VirtualMac2,1) and find the IPSW of the macOS release you want to build. + +2. Add an entry to [`ci/darwin/variables.pkr.hcl`](/ci/darwin/variables.pkr.hcl) with the following format: + +```hcl +sonoma = { + distro = "sonoma" + release = "15" + ipsw = "https://updates.cdn-apple.com/..." +} +``` + +3. Add matching scripts to [`ci/package.json`](/ci/package.json) to build the image, then test it: + +```sh +$ bun run build:darwin-aarch64-vanilla-15 +``` + +> Note: If you need to troubleshoot the build, you can remove the `headless = true` property from [`ci/darwin/image-vanilla.pkr.hcl`](/ci/darwin/image-vanilla.pkr.hcl) and the VM's screen will be displayed. + +4. Test and build the non-vanilla image: + +```sh +$ bun run build:darwin-aarch64-15 +``` + +This will use the vanilla image and run the [`scripts/bootstrap.sh`](/scripts/bootstrap.sh) script to install the required software to build and test Bun. + +5. Publish the images: + +```sh +$ bun run login +$ bun run publish:darwin-aarch64-vanilla-15 +$ bun run publish:darwin-aarch64-15 +``` diff --git a/ci/darwin/image-vanilla.pkr.hcl b/ci/darwin/image-vanilla.pkr.hcl new file mode 100644 index 00000000000000..40455713b4a9b6 --- /dev/null +++ b/ci/darwin/image-vanilla.pkr.hcl @@ -0,0 +1,46 @@ +# Generates a vanilla macOS VM with optimized settings for virtualized environments. +# See login.sh and optimize.sh for details. + +data "external-raw" "boot-script" { + program = ["sh", "-c", templatefile("scripts/boot-image.sh", var)] +} + +source "tart-cli" "bun-darwin-aarch64-vanilla" { + vm_name = "bun-darwin-aarch64-vanilla-${local.release.distro}-${local.release.release}" + from_ipsw = local.release.ipsw + cpu_count = local.cpu_count + memory_gb = local.memory_gb + disk_size_gb = local.disk_size_gb + ssh_username = local.username + ssh_password = local.password + ssh_timeout = "120s" + create_grace_time = "30s" + boot_command = split("\n", data.external-raw.boot-script.result) + headless = true # Disable if you need to debug why the boot_command is not working +} + +build { + sources = ["source.tart-cli.bun-darwin-aarch64-vanilla"] + + provisioner "file" { + content = file("scripts/setup-login.sh") + destination = "/tmp/setup-login.sh" + } + + provisioner "shell" { + inline = ["echo \"${local.password}\" | sudo -S sh -c 'sh /tmp/setup-login.sh \"${local.username}\" \"${local.password}\"'"] + } + + provisioner "file" { + content = file("scripts/optimize-machine.sh") + destination = "/tmp/optimize-machine.sh" + } + + provisioner "shell" { + inline = ["sudo sh /tmp/optimize-machine.sh"] + } + + provisioner "shell" { + inline = ["sudo rm -rf /tmp/*"] + } +} diff --git a/ci/darwin/image.pkr.hcl b/ci/darwin/image.pkr.hcl new file mode 100644 index 00000000000000..b536efbecb36e2 --- /dev/null +++ b/ci/darwin/image.pkr.hcl @@ -0,0 +1,44 @@ +# Generates a macOS VM with software installed to build and test Bun. + +source "tart-cli" "bun-darwin-aarch64" { + vm_name = "bun-darwin-aarch64-${local.release.distro}-${local.release.release}" + vm_base_name = "bun-darwin-aarch64-vanilla-${local.release.distro}-${local.release.release}" + cpu_count = local.cpu_count + memory_gb = local.memory_gb + disk_size_gb = local.disk_size_gb + ssh_username = local.username + ssh_password = local.password + ssh_timeout = "120s" + headless = true +} + +build { + sources = ["source.tart-cli.bun-darwin-aarch64"] + + provisioner "file" { + content = file("../../scripts/bootstrap.sh") + destination = "/tmp/bootstrap.sh" + } + + provisioner "shell" { + inline = ["CI=true sh /tmp/bootstrap.sh"] + } + + provisioner "file" { + source = "darwin/plists/" + destination = "/tmp/" + } + + provisioner "shell" { + inline = [ + "sudo ls /tmp/", + "sudo mv /tmp/*.plist /Library/LaunchDaemons/", + "sudo chown root:wheel /Library/LaunchDaemons/*.plist", + "sudo chmod 644 /Library/LaunchDaemons/*.plist", + ] + } + + provisioner "shell" { + inline = ["sudo rm -rf /tmp/*"] + } +} diff --git a/ci/darwin/plists/buildkite-agent.plist b/ci/darwin/plists/buildkite-agent.plist new file mode 100644 index 00000000000000..23c058913f7e3c --- /dev/null +++ b/ci/darwin/plists/buildkite-agent.plist @@ -0,0 +1,44 @@ + + + + + Label + com.buildkite.buildkite-agent + + ProgramArguments + + /usr/local/bin/buildkite-agent + start + + + KeepAlive + + SuccessfulExit + + + + RunAtLoad + + + StandardOutPath + /var/buildkite-agent/logs/buildkite-agent.log + + StandardErrorPath + /var/buildkite-agent/logs/buildkite-agent.log + + EnvironmentVariables + + BUILDKITE_AGENT_CONFIG + /etc/buildkite-agent/buildkite-agent.cfg + + + LimitLoadToSessionType + + Aqua + LoginWindow + Background + StandardIO + System + + + \ No newline at end of file diff --git a/ci/darwin/plists/tailscale.plist b/ci/darwin/plists/tailscale.plist new file mode 100644 index 00000000000000..cbe3f001b0c4ae --- /dev/null +++ b/ci/darwin/plists/tailscale.plist @@ -0,0 +1,20 @@ + + + + + Label + com.tailscale.tailscaled + + ProgramArguments + + /usr/local/bin/tailscale + up + --ssh + --authkey + ${TAILSCALE_AUTHKEY} + + + RunAtLoad + + + \ No newline at end of file diff --git a/ci/darwin/plists/tailscaled.plist b/ci/darwin/plists/tailscaled.plist new file mode 100644 index 00000000000000..12d316f1abaad1 --- /dev/null +++ b/ci/darwin/plists/tailscaled.plist @@ -0,0 +1,16 @@ + + + + + Label + com.tailscale.tailscaled + + ProgramArguments + + /usr/local/bin/tailscaled + + + RunAtLoad + + + \ No newline at end of file diff --git a/ci/darwin/scripts/boot-image.sh b/ci/darwin/scripts/boot-image.sh new file mode 100755 index 00000000000000..02ae01db0345a3 --- /dev/null +++ b/ci/darwin/scripts/boot-image.sh @@ -0,0 +1,124 @@ +#!/bin/sh + +# This script generates the boot commands for the macOS installer GUI. +# It is run on your local machine, not inside the VM. + +# Sources: +# - https://github.com/cirruslabs/macos-image-templates/blob/master/templates/vanilla-sequoia.pkr.hcl + +if ! [ "${release}" ] || ! [ "${username}" ] || ! [ "${password}" ]; then + echo "Script must be run with variables: release, username, and password" >&2 + exit 1 +fi + +# Hello, hola, bonjour, etc. +echo "" + +# Select Your Country and Region +echo "italianoenglish" +echo "united states" + +# Written and Spoken Languages +echo "" + +# Accessibility +echo "" + +# Data & Privacy +echo "" + +# Migration Assistant +echo "" + +# Sign In with Your Apple ID +echo "" + +# Are you sure you want to skip signing in with an Apple ID? +echo "" + +# Terms and Conditions +echo "" + +# I have read and agree to the macOS Software License Agreement +echo "" + +# Create a Computer Account +echo "${username}${password}${password}" + +# Enable Location Services +echo "" + +# Are you sure you don't want to use Location Services? +echo "" + +# Select Your Time Zone +echo "UTC" + +# Analytics +echo "" + +# Screen Time +echo "" + +# Siri +echo "" + +# Choose Your Look +echo "" + +if [ "${release}" = "13" ] || [ "${release}" = "14" ]; then + # Enable Voice Over + echo "v" +else + # Welcome to Mac + echo "" + + # Enable Keyboard navigation + echo "Terminal" + echo "defaults write NSGlobalDomain AppleKeyboardUIMode -int 3" + echo "q" +fi + +# Now that the installation is done, open "System Settings" +echo "System Settings" + +# Navigate to "Sharing" +echo "fsharing" + +if [ "${release}" = "13" ]; then + # Navigate to "Screen Sharing" and enable it + echo "" + + # Navigate to "Remote Login" and enable it + echo "" + + # Open "Remote Login" details + echo "" + + # Enable "Full Disk Access" + echo "" + + # Click "Done" + echo "" + + # Disable Voice Over + echo "" +elif [ "${release}" = "14" ]; then + # Navigate to "Screen Sharing" and enable it + echo "" + + # Navigate to "Remote Login" and enable it + echo "" + + # Disable Voice Over + echo "" +elif [ "${release}" = "15" ]; then + # Navigate to "Screen Sharing" and enable it + echo "" + + # Navigate to "Remote Login" and enable it + echo "" +fi + +# Quit System Settings +echo "q" diff --git a/ci/darwin/scripts/optimize-machine.sh b/ci/darwin/scripts/optimize-machine.sh new file mode 100644 index 00000000000000..1d58ff4bb349c0 --- /dev/null +++ b/ci/darwin/scripts/optimize-machine.sh @@ -0,0 +1,122 @@ +#!/bin/sh + +# This script optimizes macOS for virtualized environments. +# It disables things like spotlight, screen saver, and sleep. + +# Sources: +# - https://github.com/sickcodes/osx-optimizer +# - https://github.com/koding88/MacBook-Optimization-Script +# - https://www.macstadium.com/blog/simple-optimizations-for-macos-and-ios-build-agents + +if [ "$(id -u)" != "0" ]; then + echo "This script must be run using sudo." >&2 + exit 1 +fi + +execute() { + echo "$ $@" >&2 + if ! "$@"; then + echo "Command failed: $@" >&2 + exit 1 + fi +} + +disable_software_update() { + execute softwareupdate --schedule off + execute defaults write com.apple.SoftwareUpdate AutomaticDownload -bool false + execute defaults write com.apple.SoftwareUpdate AutomaticCheckEnabled -bool false + execute defaults write com.apple.SoftwareUpdate ConfigDataInstall -int 0 + execute defaults write com.apple.SoftwareUpdate CriticalUpdateInstall -int 0 + execute defaults write com.apple.SoftwareUpdate ScheduleFrequency -int 0 + execute defaults write com.apple.SoftwareUpdate AutomaticDownload -int 0 + execute defaults write com.apple.commerce AutoUpdate -bool false + execute defaults write com.apple.commerce AutoUpdateRestartRequired -bool false +} + +disable_spotlight() { + execute mdutil -i off -a + execute mdutil -E / +} + +disable_siri() { + execute launchctl unload -w /System/Library/LaunchAgents/com.apple.Siri.agent.plist + execute defaults write com.apple.Siri StatusMenuVisible -bool false + execute defaults write com.apple.Siri UserHasDeclinedEnable -bool true + execute defaults write com.apple.assistant.support "Assistant Enabled" 0 +} + +disable_sleep() { + execute systemsetup -setsleep Never + execute systemsetup -setcomputersleep Never + execute systemsetup -setdisplaysleep Never + execute systemsetup -setharddisksleep Never +} + +disable_screen_saver() { + execute defaults write com.apple.screensaver loginWindowIdleTime 0 + execute defaults write com.apple.screensaver idleTime 0 +} + +disable_screen_lock() { + execute defaults write com.apple.loginwindow DisableScreenLock -bool true +} + +disable_wallpaper() { + execute defaults write com.apple.loginwindow DesktopPicture "" +} + +disable_application_state() { + execute defaults write com.apple.loginwindow TALLogoutSavesState -bool false +} + +disable_accessibility() { + execute defaults write com.apple.Accessibility DifferentiateWithoutColor -int 1 + execute defaults write com.apple.Accessibility ReduceMotionEnabled -int 1 + execute defaults write com.apple.universalaccess reduceMotion -int 1 + execute defaults write com.apple.universalaccess reduceTransparency -int 1 +} + +disable_dashboard() { + execute defaults write com.apple.dashboard mcx-disabled -boolean YES + execute killall Dock +} + +disable_animations() { + execute defaults write NSGlobalDomain NSAutomaticWindowAnimationsEnabled -bool false + execute defaults write -g QLPanelAnimationDuration -float 0 + execute defaults write com.apple.finder DisableAllAnimations -bool true +} + +disable_time_machine() { + execute tmutil disable +} + +enable_performance_mode() { + # https://support.apple.com/en-us/101992 + if ! [ $(nvram boot-args 2>/dev/null | grep -q serverperfmode) ]; then + execute nvram boot-args="serverperfmode=1 $(nvram boot-args 2>/dev/null | cut -f 2-)" + fi +} + +add_terminal_to_desktop() { + execute ln -sf /System/Applications/Utilities/Terminal.app ~/Desktop/Terminal +} + +main() { + disable_software_update + disable_spotlight + disable_siri + disable_sleep + disable_screen_saver + disable_screen_lock + disable_wallpaper + disable_application_state + disable_accessibility + disable_dashboard + disable_animations + disable_time_machine + enable_performance_mode + add_terminal_to_desktop +} + +main diff --git a/ci/darwin/scripts/setup-login.sh b/ci/darwin/scripts/setup-login.sh new file mode 100755 index 00000000000000..f68beb26f2f2d8 --- /dev/null +++ b/ci/darwin/scripts/setup-login.sh @@ -0,0 +1,78 @@ +#!/bin/sh + +# This script generates a /etc/kcpassword file to enable auto-login on macOS. +# Yes, this stores your password in plain text. Do NOT do this on your local machine. + +# Sources: +# - https://github.com/xfreebird/kcpassword/blob/master/kcpassword + +if [ "$(id -u)" != "0" ]; then + echo "This script must be run using sudo." >&2 + exit 1 +fi + +execute() { + echo "$ $@" >&2 + if ! "$@"; then + echo "Command failed: $@" >&2 + exit 1 + fi +} + +kcpassword() { + passwd="$1" + key="7d 89 52 23 d2 bc dd ea a3 b9 1f" + passwd_hex=$(printf "%s" "$passwd" | xxd -p | tr -d '\n') + + key_len=33 + passwd_len=${#passwd_hex} + remainder=$((passwd_len % key_len)) + if [ $remainder -ne 0 ]; then + padding=$((key_len - remainder)) + passwd_hex="${passwd_hex}$(printf '%0*x' $((padding / 2)) 0)" + fi + + result="" + i=0 + while [ $i -lt ${#passwd_hex} ]; do + for byte in $key; do + [ $i -ge ${#passwd_hex} ] && break + p="${passwd_hex:$i:2}" + r=$(printf '%02x' $((0x$p ^ 0x$byte))) + result="${result}${r}" + i=$((i + 2)) + done + done + + echo "$result" +} + +login() { + username="$1" + password="$2" + + enable_passwordless_sudo() { + execute mkdir -p /etc/sudoers.d/ + echo "${username} ALL=(ALL) NOPASSWD: ALL" | EDITOR=tee execute visudo "/etc/sudoers.d/${username}-nopasswd" + } + + enable_auto_login() { + echo "00000000: 1ced 3f4a bcbc ba2c caca 4e82" | execute xxd -r - /etc/kcpassword + execute defaults write /Library/Preferences/com.apple.loginwindow autoLoginUser "${username}" + } + + disable_screen_lock() { + execute sysadminctl -screenLock off -password "${password}" + } + + enable_passwordless_sudo + enable_auto_login + disable_screen_lock +} + +if [ $# -ne 2 ]; then + echo "Usage: $0 " >&2 + exit 1 +fi + +login "$@" diff --git a/ci/darwin/variables.pkr.hcl b/ci/darwin/variables.pkr.hcl new file mode 100644 index 00000000000000..d1133eb04a5f21 --- /dev/null +++ b/ci/darwin/variables.pkr.hcl @@ -0,0 +1,78 @@ +packer { + required_plugins { + tart = { + version = ">= 1.12.0" + source = "github.com/cirruslabs/tart" + } + external = { + version = ">= 0.0.2" + source = "github.com/joomcode/external" + } + } +} + +variable "release" { + type = number + default = 13 +} + +variable "username" { + type = string + default = "admin" +} + +variable "password" { + type = string + default = "admin" +} + +variable "cpu_count" { + type = number + default = 2 +} + +variable "memory_gb" { + type = number + default = 4 +} + +variable "disk_size_gb" { + type = number + default = 50 +} + +locals { + sequoia = { + tier = 1 + distro = "sequoia" + release = "15" + ipsw = "https://updates.cdn-apple.com/2024FallFCS/fullrestores/062-78489/BDA44327-C79E-4608-A7E0-455A7E91911F/UniversalMac_15.0_24A335_Restore.ipsw" + } + + sonoma = { + tier = 2 + distro = "sonoma" + release = "14" + ipsw = "https://updates.cdn-apple.com/2023FallFCS/fullrestores/042-54934/0E101AD6-3117-4B63-9BF1-143B6DB9270A/UniversalMac_14.0_23A344_Restore.ipsw" + } + + ventura = { + tier = 2 + distro = "ventura" + release = "13" + ipsw = "https://updates.cdn-apple.com/2022FallFCS/fullrestores/012-92188/2C38BCD1-2BFF-4A10-B358-94E8E28BE805/UniversalMac_13.0_22A380_Restore.ipsw" + } + + releases = { + 15 = local.sequoia + 14 = local.sonoma + 13 = local.ventura + } + + release = local.releases[var.release] + username = var.username + password = var.password + cpu_count = var.cpu_count + memory_gb = var.memory_gb + disk_size_gb = var.disk_size_gb +} diff --git a/ci/package.json b/ci/package.json new file mode 100644 index 00000000000000..ffb1297dcdd3a9 --- /dev/null +++ b/ci/package.json @@ -0,0 +1,27 @@ +{ + "private": true, + "scripts": { + "bootstrap": "brew install gh jq cirruslabs/cli/tart cirruslabs/cli/sshpass hashicorp/tap/packer && packer init darwin", + "login": "gh auth token | tart login ghcr.io --username $(gh api user --jq .login) --password-stdin", + "fetch:image-name": "echo ghcr.io/oven-sh/bun-vm", + "fetch:darwin-version": "echo 1", + "fetch:macos-version": "sw_vers -productVersion | cut -d. -f1", + "fetch:script-version": "cat ../scripts/bootstrap.sh | grep 'v=' | sed 's/v=\"//;s/\"//' | head -n 1", + "build:darwin-aarch64-vanilla": "packer build '-only=*.bun-darwin-aarch64-vanilla' -var release=$(bun fetch:macos-version) darwin/", + "build:darwin-aarch64-vanilla-15": "packer build '-only=*.bun-darwin-aarch64-vanilla' -var release=15 darwin/", + "build:darwin-aarch64-vanilla-14": "packer build '-only=*.bun-darwin-aarch64-vanilla' -var release=14 darwin/", + "build:darwin-aarch64-vanilla-13": "packer build '-only=*.bun-darwin-aarch64-vanilla' -var release=13 darwin/", + "build:darwin-aarch64": "packer build '-only=*.bun-darwin-aarch64' -var release=$(bun fetch:macos-version) darwin/", + "build:darwin-aarch64-15": "packer build '-only=*.bun-darwin-aarch64' -var release=15 darwin/", + "build:darwin-aarch64-14": "packer build '-only=*.bun-darwin-aarch64' -var release=14 darwin/", + "build:darwin-aarch64-13": "packer build '-only=*.bun-darwin-aarch64' -var release=13 darwin/", + "publish:darwin-aarch64-vanilla": "image=$(tart list --format json | jq -r \".[] | select(.Name | test(\\\"^bun-darwin-aarch64-vanilla-.*-$(bun fetch:macos-version)$\\\")) | .Name\" | head -n 1 | sed 's/bun-//'); tart push \"bun-$image\" \"ghcr.io/oven-sh/bun-vm:$image-v$(bun fetch:darwin-version)\"", + "publish:darwin-aarch64-vanilla-15": "tart push bun-darwin-aarch64-vanilla-sequoia-15 \"$(bun fetch:image-name):darwin-aarch64-vanilla-sequoia-15-v$(bun fetch:darwin-version)\"", + "publish:darwin-aarch64-vanilla-14": "tart push bun-darwin-aarch64-vanilla-sonoma-14 \"$(bun fetch:image-name):darwin-aarch64-vanilla-sonoma-14-v$(bun fetch:darwin-version)\"", + "publish:darwin-aarch64-vanilla-13": "tart push bun-darwin-aarch64-vanilla-ventura-13 \"$(bun fetch:image-name):darwin-aarch64-vanilla-ventura-13-v$(bun fetch:darwin-version)\"", + "publish:darwin-aarch64": "image=$(tart list --format json | jq -r \".[] | select(.Name | test(\\\"^bun-darwin-aarch64-.*-$(bun fetch:macos-version)$\\\")) | .Name\" | head -n 1 | sed 's/bun-//'); tart push \"bun-$image\" \"ghcr.io/oven-sh/bun-vm:$image-v$(bun fetch:script-version)\"", + "publish:darwin-aarch64-15": "tart push bun-darwin-aarch64-sequoia-15 \"$(bun fetch:image-name):darwin-aarch64-sequoia-15-v$(bun fetch:script-version)\"", + "publish:darwin-aarch64-14": "tart push bun-darwin-aarch64-sonoma-14 \"$(bun fetch:image-name):darwin-aarch64-sonoma-14-v$(bun fetch:script-version)\"", + "publish:darwin-aarch64-13": "tart push bun-darwin-aarch64-ventura-13 \"$(bun fetch:image-name):darwin-aarch64-ventura-13-v$(bun fetch:script-version)\"" + } +} diff --git a/scripts/bootstrap.sh b/scripts/bootstrap.sh new file mode 100755 index 00000000000000..f809e4d734befc --- /dev/null +++ b/scripts/bootstrap.sh @@ -0,0 +1,714 @@ +#!/bin/sh + +# A script that installs the dependencies needed to build and test Bun. +# This should work on macOS and Linux with a POSIX shell. + +# If this script does not work on your machine, please open an issue: +# https://github.com/oven-sh/bun/issues + +# If you need to make a change to this script, such as upgrading a dependency, +# increment the version number to indicate that a new image should be built. +# Otherwise, the existing image will be retroactively updated. +v="3" +pid=$$ +script="$(realpath "$0")" + +print() { + echo "$@" +} + +error() { + echo "error: $@" >&2 + kill -s TERM "$pid" + exit 1 +} + +execute() { + print "$ $@" >&2 + if ! "$@"; then + error "Command failed: $@" + fi +} + +execute_sudo() { + if [ "$sudo" = "1" ]; then + execute "$@" + else + execute sudo "$@" + fi +} + +execute_non_root() { + if [ "$sudo" = "1" ]; then + execute sudo -u "$user" "$@" + else + execute "$@" + fi +} + +which() { + command -v "$1" +} + +require() { + path="$(which "$1")" + if ! [ -f "$path" ]; then + error "Command \"$1\" is required, but is not installed." + fi + echo "$path" +} + +fetch() { + curl=$(which curl) + if [ -f "$curl" ]; then + execute "$curl" -fsSL "$1" + else + wget=$(which wget) + if [ -f "$wget" ]; then + execute "$wget" -qO- "$1" + else + error "Command \"curl\" or \"wget\" is required, but is not installed." + fi + fi +} + +download_file() { + url="$1" + filename="${2:-$(basename "$url")}" + path="$(mktemp -d)/$filename" + + fetch "$url" > "$path" + print "$path" +} + +compare_version() { + if [ "$1" = "$2" ]; then + echo "0" + elif [ "$1" = "$(echo -e "$1\n$2" | sort -V | head -n1)" ]; then + echo "-1" + else + echo "1" + fi +} + +append_to_file() { + file="$1" + content="$2" + + if ! [ -f "$file" ]; then + execute mkdir -p "$(dirname "$file")" + execute touch "$file" + fi + + echo "$content" | while read -r line; do + if ! grep -q "$line" "$file"; then + echo "$line" >> "$file" + fi + done +} + +append_to_profile() { + content="$1" + profiles=".profile .zprofile .bash_profile .bashrc .zshrc" + for profile in $profiles; do + file="$HOME/$profile" + if [ "$ci" = "1" ] || [ -f "$file" ]; then + append_to_file "$file" "$content" + fi + done +} + +append_to_path() { + path="$1" + if ! [ -d "$path" ]; then + error "Could not find directory: \"$path\"" + fi + + append_to_profile "export PATH=\"$path:\$PATH\"" + export PATH="$path:$PATH" +} + +check_system() { + uname="$(require uname)" + + os="$($uname -s)" + case "$os" in + Linux*) os="linux" ;; + Darwin*) os="darwin" ;; + *) error "Unsupported operating system: $os" ;; + esac + + arch="$($uname -m)" + case "$arch" in + x86_64 | x64 | amd64) arch="x64" ;; + aarch64 | arm64) arch="aarch64" ;; + *) error "Unsupported architecture: $arch" ;; + esac + + kernel="$(uname -r)" + + if [ "$os" = "darwin" ]; then + sw_vers="$(which sw_vers)" + if [ -f "$sw_vers" ]; then + distro="$($sw_vers -productName)" + release="$($sw_vers -productVersion)" + fi + + if [ "$arch" = "x64" ]; then + sysctl="$(which sysctl)" + if [ -f "$sysctl" ] && [ "$($sysctl -n sysctl.proc_translated 2>/dev/null)" = "1" ]; then + arch="aarch64" + rosetta="1" + fi + fi + fi + + if [ "$os" = "linux" ] && [ -f /etc/os-release ]; then + . /etc/os-release + if [ -n "$ID" ]; then + distro="$ID" + fi + if [ -n "$VERSION_ID" ]; then + release="$VERSION_ID" + fi + fi + + if [ "$os" = "linux" ]; then + rpm="$(which rpm)" + if [ -f "$rpm" ]; then + glibc="$($rpm -q glibc --queryformat '%{VERSION}\n')" + else + ldd="$(which ldd)" + awk="$(which awk)" + if [ -f "$ldd" ] && [ -f "$awk" ]; then + glibc="$($ldd --version | $awk 'NR==1{print $NF}')" + fi + fi + fi + + if [ "$os" = "darwin" ]; then + brew="$(which brew)" + pm="brew" + fi + + if [ "$os" = "linux" ]; then + apt="$(which apt-get)" + if [ -f "$apt" ]; then + pm="apt" + else + dnf="$(which dnf)" + if [ -f "$dnf" ]; then + pm="dnf" + else + yum="$(which yum)" + if [ -f "$yum" ]; then + pm="yum" + fi + fi + fi + + if [ -z "$pm" ]; then + error "No package manager found. (apt, dnf, yum)" + fi + fi + + if [ -n "$SUDO_USER" ]; then + user="$SUDO_USER" + else + whoami="$(which whoami)" + if [ -f "$whoami" ]; then + user="$($whoami)" + else + error "Could not determine the current user, set \$USER." + fi + fi + + id="$(which id)" + if [ -f "$id" ] && [ "$($id -u)" = "0" ]; then + sudo=1 + fi + + if [ "$CI" = "true" ]; then + ci=1 + fi + + print "System information:" + if [ -n "$distro" ]; then + print "| Distro: $distro $release" + fi + print "| Operating system: $os" + print "| Architecture: $arch" + if [ -n "$rosetta" ]; then + print "| Rosetta: true" + fi + if [ -n "$glibc" ]; then + print "| Glibc: $glibc" + fi + print "| Package manager: $pm" + print "| User: $user" + if [ -n "$sudo" ]; then + print "| Sudo: true" + fi + if [ -n "$ci" ]; then + print "| CI: true" + fi +} + +package_manager() { + case "$pm" in + apt) DEBIAN_FRONTEND=noninteractive \ + execute "$apt" "$@" ;; + dnf) execute dnf "$@" ;; + yum) execute "$yum" "$@" ;; + brew) + if ! [ -f "$(which brew)" ]; then + install_brew + fi + execute_non_root brew "$@" + ;; + *) error "Unsupported package manager: $pm" ;; + esac +} + +update_packages() { + case "$pm" in + apt) + package_manager update + ;; + esac +} + +check_package() { + case "$pm" in + apt) + apt-cache policy "$1" + ;; + dnf | yum | brew) + package_manager info "$1" + ;; + *) + error "Unsupported package manager: $pm" + ;; + esac +} + +install_packages() { + case "$pm" in + apt) + package_manager install --yes --no-install-recommends "$@" + ;; + dnf) + package_manager install --assumeyes --nodocs --noautoremove --allowerasing "$@" + ;; + yum) + package_manager install -y "$@" + ;; + brew) + package_manager install --force --formula "$@" + package_manager link --force --overwrite "$@" + ;; + *) + error "Unsupported package manager: $pm" + ;; + esac +} + +get_version() { + command="$1" + path="$(which "$command")" + + if [ -f "$path" ]; then + case "$command" in + go | zig) "$path" version ;; + *) "$path" --version ;; + esac + else + print "not found" + fi +} + +install_brew() { + bash="$(require bash)" + script=$(download_file "https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh") + NONINTERACTIVE=1 execute_non_root "$bash" "$script" + + case "$arch" in + x64) + append_to_path "/usr/local/bin" + ;; + aarch64) + append_to_path "/opt/homebrew/bin" + ;; + esac + + case "$ci" in + 1) + append_to_profile "export HOMEBREW_NO_INSTALL_CLEANUP=1" + append_to_profile "export HOMEBREW_NO_AUTO_UPDATE=1" + append_to_profile "export HOMEBREW_NO_ANALYTICS=1" + ;; + esac +} + +install_common_software() { + case "$pm" in + apt) install_packages \ + apt-transport-https \ + software-properties-common + ;; + dnf) install_packages \ + dnf-plugins-core \ + tar + ;; + esac + + install_packages \ + bash \ + ca-certificates \ + curl \ + jq \ + htop \ + gnupg \ + git \ + unzip \ + wget \ + zip + + install_rosetta + install_nodejs + install_bun +} + +install_nodejs() { + version="${1:-"22"}" + + if ! [ "$(compare_version "$glibc" "2.27")" = "1" ]; then + version="16" + fi + + case "$pm" in + dnf | yum) + bash="$(require bash)" + script=$(download_file "https://rpm.nodesource.com/setup_$version.x") + execute "$bash" "$script" + ;; + apt) + bash="$(require bash)" + script=$(download_file "https://deb.nodesource.com/setup_$version.x") + execute "$bash" "$script" + ;; + esac + + install_packages nodejs +} + +install_bun() { + bash="$(require bash)" + script=$(download_file "https://bun.sh/install") + + version="${1:-"latest"}" + case "$version" in + latest) + execute "$bash" "$script" + ;; + *) + execute "$bash" "$script" -s "$version" + ;; + esac + + append_to_path "$HOME/.bun/bin" +} + +install_rosetta() { + case "$os" in + darwin) + if ! [ "$(which arch)" ]; then + execute softwareupdate \ + --install-rosetta \ + --agree-to-license + fi + ;; + esac +} + +install_build_essentials() { + case "$pm" in + apt) install_packages \ + build-essential \ + ninja-build \ + xz-utils + ;; + dnf | yum) install_packages \ + ninja-build \ + gcc-c++ \ + xz + ;; + brew) install_packages \ + ninja + ;; + esac + + install_packages \ + make \ + cmake \ + pkg-config \ + python3 \ + libtool \ + ruby \ + perl \ + golang + + install_llvm + install_ccache + install_rust + install_docker +} + +llvm_version_exact() { + case "$os" in + linux) + print "16.0.6" + ;; + darwin | windows) + print "18.1.8" + ;; + esac +} + +llvm_version() { + echo "$(llvm_version_exact)" | cut -d. -f1 +} + +install_llvm() { + case "$pm" in + apt) + bash="$(require bash)" + script=$(download_file "https://apt.llvm.org/llvm.sh") + execute "$bash" "$script" "$(llvm_version)" all + ;; + brew) + install_packages "llvm@$(llvm_version)" + ;; + esac +} + +install_ccache() { + case "$pm" in + apt | brew) + install_packages ccache + ;; + esac +} + +install_rust() { + sh="$(require sh)" + script=$(download_file "https://sh.rustup.rs") + execute "$sh" "$script" -y + append_to_path "$HOME/.cargo/bin" +} + +install_docker() { + case "$pm" in + brew) + if ! [ -d "/Applications/Docker.app" ]; then + package_manager install docker --cask + fi + ;; + *) + case "$distro-$release" in + amzn-2 | amzn-1) + execute amazon-linux-extras install docker + ;; + amzn-*) + install_packages docker + ;; + *) + sh="$(require sh)" + script=$(download_file "https://get.docker.com") + execute "$sh" "$script" + ;; + esac + ;; + esac + + systemctl="$(which systemctl)" + if [ -f "$systemctl" ]; then + execute "$systemctl" enable docker + fi +} + +install_ci_dependencies() { + if ! [ "$ci" = "1" ]; then + return + fi + + install_tailscale + install_buildkite +} + +install_tailscale() { + case "$os" in + linux) + sh="$(require sh)" + script=$(download_file "https://tailscale.com/install.sh") + execute "$sh" "$script" + ;; + darwin) + install_packages go + execute_non_root go install tailscale.com/cmd/tailscale{,d}@latest + append_to_path "$HOME/go/bin" + ;; + esac +} + +install_buildkite() { + home_dir="/var/lib/buildkite-agent" + config_dir="/etc/buildkite-agent" + config_file="$config_dir/buildkite-agent.cfg" + + if ! [ -d "$home_dir" ]; then + execute_sudo mkdir -p "$home_dir" + fi + + if ! [ -d "$config_dir" ]; then + execute_sudo mkdir -p "$config_dir" + fi + + case "$os" in + linux) + getent="$(require getent)" + if [ -z "$("$getent" passwd buildkite-agent)" ]; then + useradd="$(require useradd)" + execute "$useradd" buildkite-agent \ + --system \ + --no-create-home \ + --home-dir "$home_dir" + fi + + if [ -n "$("$getent" group docker)" ]; then + usermod="$(require usermod)" + execute "$usermod" -aG docker buildkite-agent + fi + + execute chown -R buildkite-agent:buildkite-agent "$home_dir" + execute chown -R buildkite-agent:buildkite-agent "$config_dir" + ;; + darwin) + execute_sudo chown -R "$user:admin" "$home_dir" + execute_sudo chown -R "$user:admin" "$config_dir" + ;; + esac + + if ! [ -f "$config_file" ]; then + cat <"$config_file" +# This is generated by scripts/bootstrap.sh +# https://buildkite.com/docs/agent/v3/configuration + +name="%hostname-%random" +tags="v=$v,os=$os,arch=$arch,distro=$distro,release=$release,kernel=$kernel,glibc=$glibc" + +build-path="$home_dir/builds" +git-mirrors-path="$home_dir/git" +job-log-path="$home_dir/logs" +plugins-path="$config_dir/plugins" +hooks-path="$config_dir/hooks" + +no-ssh-keyscan=true +cancel-grace-period=3600000 # 1 hour +enable-job-log-tmpfile=true +experiment="normalised-upload-paths,resolve-commit-after-checkout,agent-api" +EOF + fi + + bash="$(require bash)" + script=$(download_file "https://raw.githubusercontent.com/buildkite/agent/main/install.sh") + execute "$bash" "$script" + + out_dir="$HOME/.buildkite-agent" + execute_sudo mv -f "$out_dir/bin/buildkite-agent" "/usr/local/bin/buildkite-agent" + execute rm -rf "$out_dir" +} + +install_chrome_dependencies() { + # https://github.com/puppeteer/puppeteer/blob/main/docs/troubleshooting.md#chrome-doesnt-launch-on-linux + # https://github.com/puppeteer/puppeteer/blob/main/docs/troubleshooting.md#running-puppeteer-in-the-cloud + case "$pm" in + apt) + install_packages \ + fonts-liberation \ + libatk-bridge2.0-0 \ + libatk1.0-0 \ + libc6 \ + libcairo2 \ + libcups2 \ + libdbus-1-3 \ + libexpat1 \ + libfontconfig1 \ + libgbm1 \ + libgcc1 \ + libglib2.0-0 \ + libgtk-3-0 \ + libnspr4 \ + libnss3 \ + libpango-1.0-0 \ + libpangocairo-1.0-0 \ + libstdc++6 \ + libx11-6 \ + libx11-xcb1 \ + libxcb1 \ + libxcomposite1 \ + libxcursor1 \ + libxdamage1 \ + libxext6 \ + libxfixes3 \ + libxi6 \ + libxrandr2 \ + libxrender1 \ + libxss1 \ + libxtst6 \ + xdg-utils + + # Fixes issue in newer version of Ubuntu: + # Package 'libasound2' has no installation candidate + if [ "$(check_package "libasound2t64")" ]; then + install_packages libasound2t64 + else + install_packages libasound2 + fi + ;; + dnf | yum) + install_packages \ + alsa-lib \ + atk \ + cups-libs \ + gtk3 \ + ipa-gothic-fonts \ + libXcomposite \ + libXcursor \ + libXdamage \ + libXext \ + libXi \ + libXrandr \ + libXScrnSaver \ + libXtst \ + pango \ + xorg-x11-fonts-100dpi \ + xorg-x11-fonts-75dpi \ + xorg-x11-fonts-cyrillic \ + xorg-x11-fonts-misc \ + xorg-x11-fonts-Type1 \ + xorg-x11-utils + ;; + esac +} + +main() { + check_system + update_packages + install_common_software + install_build_essentials + install_chrome_dependencies + install_ci_dependencies +} + +main