-
Notifications
You must be signed in to change notification settings - Fork 305
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
lib/sysroot-deploy: Add experimental support for automatic early prune
During the early design of FCOS and RHCOS, we chose a value of 384M for the boot partition. This turned out to be too small: some arches other than x86_64 have larger initrds, kernel binaries, or additional artifacts (like device tree blobs). We'll likely bump the boot partition size in the future, but we don't want to abandon all the nodes deployed with the current size.[[1]] Because stale entries in `/boot` are cleaned up after new entries are written, there is a window in the update process during which the bootfs temporarily must host all the `(kernel, initrd)` pairs for the union of current and new deployments. This patch determines if the bootfs is capable of holding all the pairs. If it can't but it could hold all the pairs from just the new deployments, the outgoing deployments (e.g. rollbacks) are deleted *before* new deployments are written. This is done by updating the bootloader in two steps to maintain atomicity. Since this is a lot of new logic in an important section of the code, this feature is gated for now behind an environment variable (`OSTREE_EXP_AUTO_EARLY_PRUNE`). Once we gain more experience with it, we can consider turning it on by default. This strategy increases the fallibility of the update system since one would no longer be able to rollback to the previous deployment if a bug is present in the bootloader update logic after auto-pruning. This is however mitigated by the fact that the heuristic is opportunistic: the rollback is pruned *only if* it's the only way for the system to update. [1]: coreos/fedora-coreos-tracker#1247 Closes: #2670
- Loading branch information
Showing
3 changed files
with
332 additions
and
3 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,94 @@ | ||
#!/bin/bash | ||
set -xeuo pipefail | ||
|
||
# https://github.com/ostreedev/ostree/issues/2670 | ||
|
||
. ${KOLA_EXT_DATA}/libinsttest.sh | ||
|
||
# make two fake ostree commits with modified kernels of about the same size | ||
cd /root | ||
mkdir -p rootfs/usr/lib/modules/`uname -r` | ||
cp /usr/lib/modules/`uname -r`/vmlinuz rootfs/usr/lib/modules/`uname -r` | ||
echo 1 >> rootfs/usr/lib/modules/`uname -r`/vmlinuz | ||
ostree commit --base "${host_refspec}" -P --tree=dir=rootfs -b modkernel1 | ||
echo 1 >> rootfs/usr/lib/modules/`uname -r`/vmlinuz | ||
ostree commit --base "${host_refspec}" -P --tree=dir=rootfs -b modkernel2 | ||
|
||
assert_bootfs_has_n_bootcsum_dirs() { | ||
local expected=$1; shift | ||
local actual | ||
actual=$(ls -d /boot/ostree/${host_osname}-* | wc -l) | ||
if [ "$expected" != "$actual" ]; then | ||
ls -l /boot/ostree | ||
assert_not_reached "expected $expected bootcsum dirs, found $actual" | ||
fi | ||
} | ||
|
||
consume_bootfs_space() { | ||
local free_blocks=$(stat --file-system /boot -c '%a') | ||
local block_size=$(stat --file-system /boot -c '%s') | ||
# leave 1 block free | ||
unshare -m bash -c \ | ||
"mount -o rw,remount /boot && \ | ||
dd if=/dev/zero of=/boot/bigfile count=$((free_blocks-1)) bs=${block_size}" | ||
} | ||
|
||
unconsume_bootfs_space() { | ||
unshare -m bash -c "mount -o rw,remount /boot && rm /boot/bigfile" | ||
} | ||
|
||
assert_bootfs_has_n_bootcsum_dirs 1 | ||
|
||
# first, deploy our second deployment on a filled up bootfs | ||
# the booted deployment is never pruned, so this is a hopeless case and auto-pruning can't save us | ||
consume_bootfs_space | ||
rpm-ostree rebase :modkernel1 | ||
if OSTREE_EXP_AUTO_EARLY_PRUNE=1 ostree admin finalize-staged |& tee out.txt; then | ||
assert_not_reached "successfully wrote to filled up bootfs" | ||
fi | ||
assert_file_has_content out.txt "No space left on device" | ||
rm out.txt | ||
unconsume_bootfs_space | ||
rpm-ostree cleanup -bpr | ||
|
||
# OK, now deploy our second deployment for realsies on a bootfs with ample space | ||
# and sanity-check that auto-pruning doesn't kick in | ||
assert_bootfs_has_n_bootcsum_dirs 1 | ||
|
||
rpm-ostree rebase :modkernel1 | ||
OSTREE_EXP_AUTO_EARLY_PRUNE=1 ostree admin finalize-staged |& tee out.txt | ||
assert_not_file_has_content out.txt "updating bootloader in two steps" | ||
rm out.txt | ||
|
||
# and put it in rollback position; this is the deployment that'll get auto-pruned | ||
rpm-ostree rollback | ||
|
||
assert_bootfs_has_n_bootcsum_dirs 2 | ||
bootloader_orig=$(sha256sum /boot/loader/entries/*) | ||
|
||
# now try to deploy a third deployment without early pruning; we should hit ENOSPC | ||
consume_bootfs_space | ||
rpm-ostree rebase :modkernel2 | ||
if ostree admin finalize-staged |& tee out.txt; then | ||
assert_not_reached "successfully wrote kernel without auto-pruning" | ||
fi | ||
assert_file_has_content out.txt "No space left on device" | ||
rm out.txt | ||
|
||
# there's 3 bootcsums now because it'll also have the partially written | ||
# bootcsum dir we were creating when we hit ENOSPC; this verifies that all the | ||
# deployments have different bootcsums | ||
assert_bootfs_has_n_bootcsum_dirs 3 | ||
# but the bootloader wasn't updated | ||
assert_streq "$bootloader_orig" "$(sha256sum /boot/loader/entries/*)" | ||
|
||
# now, try again but with auto-pruning enabled | ||
rpm-ostree rebase :modkernel2 | ||
OSTREE_EXP_AUTO_EARLY_PRUNE=1 ostree admin finalize-staged |& tee out.txt | ||
assert_file_has_content out.txt "updating bootloader in two steps" | ||
rm out.txt | ||
|
||
assert_bootfs_has_n_bootcsum_dirs 2 | ||
assert_not_streq "$bootloader_orig" "$(sha256sum /boot/loader/entries/*)" | ||
|
||
echo "ok bootfs auto-prune" |