From dc542adb938333c3d4ef5227ebcd1d367c0d90ae Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Thu, 31 Jan 2019 21:49:10 +0000 Subject: [PATCH 1/4] Add metal-bios target Split out the "net.ifnames" stuff into a separate kickstart, and generate a raw disk image `metal-bios` if specified. Note to really be useful this requires Ignition to load `/boot/config.ign` or so. --- src/cmd-build | 41 +++++++++++++++++++++++++++++++---------- src/image-base.ks | 22 ---------------------- src/image-cloud.ks | 20 ++++++++++++++++++++ src/image-metal.ks | 4 ++++ src/virt-install | 12 +++++++++++- 5 files changed, 66 insertions(+), 33 deletions(-) create mode 100644 src/image-cloud.ks create mode 100644 src/image-metal.ks diff --git a/src/cmd-build b/src/cmd-build index 7294725c78..89c0655878 100755 --- a/src/cmd-build +++ b/src/cmd-build @@ -17,6 +17,7 @@ Usage: coreos-assembler build --help Valid image types: - qemu + - metal-bios EOF } @@ -227,23 +228,38 @@ else extraargs="${extraargs} --kickstart ${image_input}" fi -# This generates the "base image"; not specific to a platform. -# We want extraargs var to be split on words -# shellcheck disable=SC2086 -/usr/lib/coreos-assembler/virt-install --dest="${PWD}"/"${img_base}" \ - --create-disk --kickstart-out "${PWD}"/tmp/flattened.ks \ - --ostree-remote="${name}" --ostree-stateroot="${name}" \ - --ostree-ref="${ref:-${commit}}" \ - --location "${workdir}"/installer/*.iso \ - --ostree-repo="${workdir}"/repo ${extraargs-} -/usr/lib/coreos-assembler/gf-anaconda-cleanup "$(pwd)"/"${img_base}" set +x +# This generates the "base image"; not specific to a platform. +run_virtinstall() { + dest=$1 + shift + tmpdest="${dest}.tmp" + # We want extraargs var to be split on words + # shellcheck disable=SC2086 + /usr/lib/coreos-assembler/virt-install --create-disk --dest=${tmpdest} \ + --kickstart-out "${PWD}"/tmp/flattened.ks \ + --ostree-remote="${name}" --ostree-stateroot="${name}" \ + --ostree-ref="${ref:-${commit}}" \ + --location "${workdir}"/installer/*.iso \ + --ostree-repo="${workdir}"/repo ${extraargs-} "$@" + # And strip out all of the Anaconda stuff in /var + /usr/lib/coreos-assembler/gf-anaconda-cleanup "${dest}" + mv "${tmpdest}" "${dest}" +} + +build_cloud_base() { + if [ -f "${PWD}/${img_base}" ]; then + return + fi + run_virtinstall "${PWD}"/"${img_base}" --variant=cloud +} declare -A images for itype in "${IMAGE_TYPES[@]}"; do case $itype in qemu) img_qemu=${imageprefix}-qemu.qcow2 images[$itype]="${img_qemu}" + build_cloud_base /usr/lib/coreos-assembler/gf-oemid "$(pwd)"/"${img_base}" "$(pwd)"/"${img_qemu}" qemu # Clear the MCS SELinux labels # See https://github.com/coreos/coreos-assembler/issues/292 @@ -252,6 +268,11 @@ for itype in "${IMAGE_TYPES[@]}"; do # TODO: Remove this, things should be parsing the metadata ln -s "${img_qemu}" "${name}"-qemu.qcow2 ;; + metal-bios) + img_metalbios=${imageprefix}-metal-bios.raw + images[$itype]="${img_metalbios}" + run_virtinstall "$(pwd)"/"${img_metalbios}" --variant=metal + ;; *) fatal "Unrecognized image type: $itype" ;; esac diff --git a/src/image-base.ks b/src/image-base.ks index 465b6812ad..d2d2f1cf55 100644 --- a/src/image-base.ks +++ b/src/image-base.ks @@ -19,13 +19,6 @@ network --bootproto=dhcp --onboot=on zerombr clearpart --initlabel --all -# Add the following to kernel boot args: -# - ip=dhcp # how to get network -# - rd.neednet=1 # tell dracut we need network -# - rootflags=defaults,prjquota # enable prjquota for quota enablement for containers -# # https://bugzilla.redhat.com/show_bug.cgi?id=1658386 -# - $ignition_firstboot # This is actually a GRUB variable -bootloader --timeout=1 --append="no_timer_check console=ttyS0,115200n8 console=tty0 net.ifnames=0 biosdevname=0 ip=dhcp rd.neednet=1 rw rootflags=defaults,prjquota $ignition_firstboot" # https://github.com/coreos/fedora-coreos-tracker/issues/18 # See also coreos-growpart.service defined in fedora-coreos-base.yaml @@ -44,19 +37,4 @@ reboot # Remove any persistent NIC rules generated by udev rm -vf /etc/udev/rules.d/*persistent-net*.rules - -# By default, we do DHCP. Also, due to the above disabling -# of biosdevname/net.ifnames, this uses eth0. -cat < /etc/sysconfig/network-scripts/ifcfg-eth0 -DEVICE="eth0" -BOOTPROTO="dhcp" -ONBOOT="yes" -TYPE="Ethernet" -PERSISTENT_DHCLIENT="yes" -NM_CONTROLLED="yes" -EOF - -# Anaconda is writing a /etc/resolv.conf from the generating environment. -# The system should start out with an empty file. -truncate -s 0 /etc/resolv.conf %end diff --git a/src/image-cloud.ks b/src/image-cloud.ks new file mode 100644 index 0000000000..ec1bb35b9d --- /dev/null +++ b/src/image-cloud.ks @@ -0,0 +1,20 @@ +# ip=dhcp and rd.neednet=1 enable networking in the initramfs +# We use net.ifnames in cloud environments +# no_timer_check is something we're cargo culting around +# console= args are also for clouds +# The other ones are for Ignition and are also in image-metal.ks; +# change them there first. +bootloader --timeout=1 --append="no_timer_check console=ttyS0,115200n8 console=tty0 net.ifnames=0 biosdevname=0 ip=dhcp rd.neednet=1 rootflags=defaults,prjquota rw $ignition_firstboot" + +%post --erroronfail +# By default, we do DHCP. Also, due to the above disabling +# of biosdevname/net.ifnames, this uses eth0. +cat < /etc/sysconfig/network-scripts/ifcfg-eth0 +DEVICE="eth0" +BOOTPROTO="dhcp" +ONBOOT="yes" +TYPE="Ethernet" +PERSISTENT_DHCLIENT="yes" +NM_CONTROLLED="yes" +EOF +%end diff --git a/src/image-metal.ks b/src/image-metal.ks new file mode 100644 index 0000000000..09523180aa --- /dev/null +++ b/src/image-metal.ks @@ -0,0 +1,4 @@ +# If you add something here, also update image-cloud.ks. +# prjquota is for quota enablement for containers: https://bugzilla.redhat.com/show_bug.cgi?id=1658386 +# rw and $ignition_firstboot are used by https://github.com/coreos/ignition-dracut/ +bootloader --append="rootflags=defaults,prjquota rw $ignition_firstboot" diff --git a/src/virt-install b/src/virt-install index 4a91938222..3fd52cda2d 100755 --- a/src/virt-install +++ b/src/virt-install @@ -22,6 +22,8 @@ parser.add_argument("--create-disk", help="Automatically create disk as qcow2, p action='store_true') parser.add_argument("--image-config", help="coreos-assembler image.yaml", action='store') +parser.add_argument("--variant", help="Use an internal (kickstart) config", + choices=('metal', 'cloud'), default=None) parser.add_argument("--kickstart", help="Kickstart path", action='store') parser.add_argument("--kickstart-out", help="Save flattened kickstart", @@ -86,6 +88,10 @@ else: disk_size = None +if args.variant is not None: + with open(f'/usr/lib/coreos-assembler/image-{args.variant}.ks') as f: + shutil.copyfileobj(f, ks_tmp) + if args.image_config: with open('/usr/lib/coreos-assembler/image-base.ks') as basef: shutil.copyfileobj(basef, ks_tmp) @@ -155,7 +161,11 @@ if args.create_disk and args.kickstart: fatal("--create-disk specified, but failed to find '{}' in kickstart".format(magic_virt_install_size_str)) if args.create_disk: - run_sync_verbose(['qemu-img', 'create', '-f', 'qcow2', args.dest, '{}G'.format(disk_size)]) + if args.variant == 'metal': + fmt = 'raw' + else: + fmt = 'qcow2' + run_sync_verbose(['qemu-img', 'create', '-f', fmt, args.dest, '{}G'.format(disk_size)]) print("Created initial disk: {} with size {}G".format(args.dest, disk_size)) # Now a hack to avoid libvirt race conditions; it's From 13a3e18a81ca2cac92b4287274768162e3cb1274 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Wed, 6 Feb 2019 15:12:45 +0000 Subject: [PATCH 2/4] Move the "gf-anaconda-cleanup" invocation into virt-install Let's be clear these two things are tightly coupled. Also virt-install is already in Python and it makes sense to do more that way than via bash. --- src/cmd-build | 2 -- src/gf-anaconda-cleanup | 1 + src/virt-install | 3 +++ 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/cmd-build b/src/cmd-build index 89c0655878..b7fb14475e 100755 --- a/src/cmd-build +++ b/src/cmd-build @@ -242,8 +242,6 @@ run_virtinstall() { --ostree-ref="${ref:-${commit}}" \ --location "${workdir}"/installer/*.iso \ --ostree-repo="${workdir}"/repo ${extraargs-} "$@" - # And strip out all of the Anaconda stuff in /var - /usr/lib/coreos-assembler/gf-anaconda-cleanup "${dest}" mv "${tmpdest}" "${dest}" } diff --git a/src/gf-anaconda-cleanup b/src/gf-anaconda-cleanup index b12291f959..8a2c93f459 100755 --- a/src/gf-anaconda-cleanup +++ b/src/gf-anaconda-cleanup @@ -1,4 +1,5 @@ #!/usr/bin/env bash +# This runs after virt-install to undo things leftover from Anaconda. set -euo pipefail dn=$(dirname "$0") diff --git a/src/virt-install b/src/virt-install index 3fd52cda2d..748c2e8018 100755 --- a/src/virt-install +++ b/src/virt-install @@ -226,6 +226,9 @@ try: "--initrd-inject={}".format(ks_tmp.name), "--extra-args", "ks=file://{} console=tty0 console=ttyS0,115200n8 inst.cmdline inst.notmux".format(os.path.basename(ks_tmp.name))]) run_sync_verbose(vinstall_args) + # And strip out all of the Anaconda stuff in /var; this uses libguestfs + # to ensure we fully control the code. + run_sync_verbose(['/usr/lib/coreos-assembler/gf-anaconda-cleanup', args.dest]) finally: subprocess.call(['virsh', '--connect=qemu:///session', 'undefine', domain]) if tail_proc is not None: From 5ea948095ae32d52abbbce7efd5b69ec96c3830e Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Wed, 6 Feb 2019 15:04:04 +0000 Subject: [PATCH 3/4] Switch to GPT partitioning With a `/boot` partition. But today Anaconda doesn't expose an API to explicitly label it `/boot` (or even change the filesystem type). We'll live with that for now. Prep for UEFI. --- src/gf-anaconda-cleanup | 12 ++++++++++++ src/image-base.ks | 4 ++-- src/libguestfish.sh | 16 ++++++++++++++-- src/virt-install | 5 ++++- 4 files changed, 32 insertions(+), 5 deletions(-) diff --git a/src/gf-anaconda-cleanup b/src/gf-anaconda-cleanup index 8a2c93f459..bbb9622379 100755 --- a/src/gf-anaconda-cleanup +++ b/src/gf-anaconda-cleanup @@ -9,8 +9,20 @@ dn=$(dirname "$0") . "${dn}"/libguestfish.sh src="$1" +shift +skip_partition_fixups= +if [ "${1:-}" = "--skip-partition-fixups" ]; then + skip_partition_fixups=1 + shift +fi set -x +if [ "${skip_partition_fixups}" != 1 ]; then + coreos_gf_run "${src}" + # We don't have a way to do this with Anaconda/kickstart right now. + # This bootstraps us to be able to do all of the mounts. + coreos_gf set-label /dev/sda2 boot +fi coreos_gf_run_mount "${src}" # Both of these are written by Anaconda diff --git a/src/image-base.ks b/src/image-base.ks index d2d2f1cf55..4f98391bf5 100644 --- a/src/image-base.ks +++ b/src/image-base.ks @@ -18,13 +18,13 @@ firewall --disabled network --bootproto=dhcp --onboot=on zerombr -clearpart --initlabel --all +clearpart --initlabel --all --disklabel=gpt # https://github.com/coreos/fedora-coreos-tracker/issues/18 # See also coreos-growpart.service defined in fedora-coreos-base.yaml # You can change this partition layout, but note that the `boot` and `root` # filesystem labels are currently mandatory (they're interpreted by coreos-assembler). -part /boot --size=300 --fstype="xfs" --label=boot +reqpart --add-boot # Note no reflinks for /boot since the bootloader may not understand them part / --size=3000 --fstype="xfs" --label=root --grow --mkfsoptions="-m reflink=1" diff --git a/src/libguestfish.sh b/src/libguestfish.sh index 7041c3ae7d..18cb041b05 100755 --- a/src/libguestfish.sh +++ b/src/libguestfish.sh @@ -9,6 +9,9 @@ export LIBGUESTFS_BACKEND=direct # http://libguestfs.org/guestfish.1.html#using-remote-control-robustly-from-shell-scripts GUESTFISH_PID= coreos_gf_launch() { + if [ -n "$GUESTFISH_PID" ]; then + return + fi local src=$1 shift local guestfish @@ -32,11 +35,20 @@ coreos_gf() { guestfish --remote -- "$@" } +GUESTFISH_RUNNING= +coreos_gf_run() { + if [ -n "$GUESTFISH_RUNNING" ]; then + return + fi + coreos_gf_launch "$@" + coreos_gf run + GUESTFISH_RUNNING=1 +} + # Run libguestfs and mount the root and boot partitions. # Export `stateroot` and `deploydir` variables. coreos_gf_run_mount() { - coreos_gf_launch "$@" - coreos_gf run + coreos_gf_run "$@" local root root=$(coreos_gf findfs-label root) coreos_gf mount "${root}" / diff --git a/src/virt-install b/src/virt-install index 748c2e8018..26cb1b4a4f 100755 --- a/src/virt-install +++ b/src/virt-install @@ -228,7 +228,10 @@ try: run_sync_verbose(vinstall_args) # And strip out all of the Anaconda stuff in /var; this uses libguestfs # to ensure we fully control the code. - run_sync_verbose(['/usr/lib/coreos-assembler/gf-anaconda-cleanup', args.dest]) + cleanup_argv = ['/usr/lib/coreos-assembler/gf-anaconda-cleanup', args.dest] + if args.image_config is None: + cleanup_argv.append('--skip-partition-fixups') + run_sync_verbose(cleanup_argv) finally: subprocess.call(['virsh', '--connect=qemu:///session', 'undefine', domain]) if tail_proc is not None: From 9b38d6a8d52dd23e5a0973b2c7f25e54d163ec83 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Wed, 6 Feb 2019 16:32:47 +0000 Subject: [PATCH 4/4] Add metal-uefi target This builds but I haven't tested running it yet. --- src/cmd-build | 6 ++++++ src/virt-install | 12 +++++++++--- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/src/cmd-build b/src/cmd-build index b7fb14475e..1dc65ff3c0 100755 --- a/src/cmd-build +++ b/src/cmd-build @@ -18,6 +18,7 @@ Usage: coreos-assembler build --help - qemu - metal-bios + - metal-uefi EOF } @@ -271,6 +272,11 @@ for itype in "${IMAGE_TYPES[@]}"; do images[$itype]="${img_metalbios}" run_virtinstall "$(pwd)"/"${img_metalbios}" --variant=metal ;; + metal-uefi) + img_metaluefi=${imageprefix}-metal-uefi.raw + images[$itype]="${img_metaluefi}" + run_virtinstall "$(pwd)"/"${img_metaluefi}" --variant=metal-uefi + ;; *) fatal "Unrecognized image type: $itype" ;; esac diff --git a/src/virt-install b/src/virt-install index 26cb1b4a4f..e212b1388d 100755 --- a/src/virt-install +++ b/src/virt-install @@ -23,7 +23,7 @@ parser.add_argument("--create-disk", help="Automatically create disk as qcow2, p parser.add_argument("--image-config", help="coreos-assembler image.yaml", action='store') parser.add_argument("--variant", help="Use an internal (kickstart) config", - choices=('metal', 'cloud'), default=None) + choices=('metal', 'metal-uefi', 'cloud'), default=None) parser.add_argument("--kickstart", help="Kickstart path", action='store') parser.add_argument("--kickstart-out", help="Save flattened kickstart", @@ -89,7 +89,11 @@ else: disk_size = None if args.variant is not None: - with open(f'/usr/lib/coreos-assembler/image-{args.variant}.ks') as f: + if args.variant == 'metal-uefi': + variant_ks = 'metal' + else: + variant_ks = args.variant + with open(f'/usr/lib/coreos-assembler/image-{variant_ks}.ks') as f: shutil.copyfileobj(f, ks_tmp) if args.image_config: @@ -161,7 +165,7 @@ if args.create_disk and args.kickstart: fatal("--create-disk specified, but failed to find '{}' in kickstart".format(magic_virt_install_size_str)) if args.create_disk: - if args.variant == 'metal': + if args.variant in ('metal', 'metal-uefi'): fmt = 'raw' else: fmt = 'qcow2' @@ -225,6 +229,8 @@ try: "--disk=path={},cache=unsafe".format(args.dest), "--initrd-inject={}".format(ks_tmp.name), "--extra-args", "ks=file://{} console=tty0 console=ttyS0,115200n8 inst.cmdline inst.notmux".format(os.path.basename(ks_tmp.name))]) + if args.variant == 'metal-uefi': + vinstall_args.append('--boot=uefi') run_sync_verbose(vinstall_args) # And strip out all of the Anaconda stuff in /var; this uses libguestfs # to ensure we fully control the code.