core: Add knob to make it easier to test out state overlay model

I want people to be able to test out the state overlay approach without
having to switch over the base compose in e.g. FCOS to use it.

Hackily add an env var that will forcibly fix the symlinks in the OSTree
commit when doing a local assembly. This is enough to allow properly
checking out RPMs that install into `/opt` and `/usr/local`. Have it also
enable the systemd services for the overlays. (In `/usr`, so that if the
knob is removed, the services are no longer enabled.)

src/libpriv/rpmostree-core.cxx

@@ -4062,6 +4062,55 @@ rpmostree_context_assemble (RpmOstreeContext *self, GCancellable *cancellable, G
   if (!build_rootfs_usrlinks (self, error))
     return FALSE;
 
+  /* This is purely for making it easier for people to test out the
+   * state-overlay stuff until it's stabilized and part of base composes. */
+  if (g_getenv ("RPMOSTREE_EXPERIMENTAL_FORCE_OPT_USRLOCAL_OVERLAY"))
+    {
+      rpmostree_output_message (
+          "Enabling experimental state overlay support for /opt and /usr/local");
+
+      struct stat stbuf;
+
+      if (!glnx_fstatat_allow_noent (tmprootfs_dfd, "opt", &stbuf, AT_SYMLINK_NOFOLLOW, error))
+        return FALSE;
+      if (errno == ENOENT || (errno == 0 && S_ISLNK (stbuf.st_mode)))
+        {
+          if (errno == 0 && !glnx_unlinkat (tmprootfs_dfd, "opt", 0, error))
+            return FALSE;
+          if (symlinkat ("usr/lib/opt", tmprootfs_dfd, "opt") < 0)
+            return glnx_throw_errno_prefix (error, "symlinkat(/opt)");
+        }
+
+      if (!glnx_fstatat_allow_noent (tmprootfs_dfd, "usr/local", &stbuf, AT_SYMLINK_NOFOLLOW,
+                                     error))
+        return FALSE;
+      if (errno == ENOENT || (errno == 0 && S_ISLNK (stbuf.st_mode)))
+        {
+          if (errno == 0 && !glnx_unlinkat (tmprootfs_dfd, "usr/local", 0, error))
+            return FALSE;
+          if (mkdirat (tmprootfs_dfd, "usr/local", 0755) < 0)
+            return glnx_throw_errno_prefix (error, "mkdirat(/usr/local)");
+        }
+
+      if (!glnx_shutil_mkdir_p_at (tmprootfs_dfd, "usr/lib/systemd/system/local-fs.target.wants",
+                                   0755, cancellable, error))
+        return FALSE;
+
+      if (symlinkat ("[email protected]", tmprootfs_dfd,
+                     "usr/lib/systemd/system/local-fs.target.wants/"
+                     "[email protected]")
+              < 0
+          && errno != EEXIST)
+        return glnx_throw_errno_prefix (error, "enabling ostree-state-overlay for /usr/lib/opt");
+
+      if (symlinkat (
+              "[email protected]", tmprootfs_dfd,
+              "usr/lib/systemd/system/local-fs.target.wants/[email protected]")
+              < 0
+          && errno != EEXIST)
+        return glnx_throw_errno_prefix (error, "enabling ostree-state-overlay for /usr/local");
+    }
+
   /* We need up to date labels; the set of things needing relabeling
    * will have been calculated in sort_packages()
    */

tests/kolainst/destructive/state-overlays

@@ -0,0 +1,104 @@
+#!/bin/bash
+## kola:
+##   tags: "needs-internet"
+
+set -euo pipefail
+
+. ${KOLA_EXT_DATA}/libtest.sh
+
+rm -rf /etc/yum.repos.d/*
+cat > /etc/yum.repos.d/vmcheck.repo << EOF
+[test-repo]
+name=test-repo
+baseurl=file:///${KOLA_EXT_DATA}/rpm-repos/0
+gpgcheck=0
+enabled=1
+EOF
+
+
+case "${AUTOPKGTEST_REBOOT_MARK:-}" in
+  "")
+    # switch over to local ref so upgrades are purely about package changes
+    booted_commit=$(rpm-ostree status --json | jq -r '.deployments[0].checksum')
+    ostree refs --create kolatest "${booted_commit}"
+    systemctl stop rpm-ostreed
+    unshare -m /bin/bash -c 'mount -o rw,remount /sysroot && sed -i -e "s/refspec=.*/refspec=kolatest/" /ostree/deploy/*/deploy/*.origin'
+
+    # XXX: until ostree v2024.1 hits FCOS
+    ostree_ver=$(rpm -q ostree  --qf '%{version}')
+    if [ "${ostree_ver}" != "2024.1" ] && \
+       [ "$(echo -e "${ostree_ver}\n2024.1" | sort -V | tail -n1)" = "2024.1" ]; then
+      rpm-ostree override replace https://bodhi.fedoraproject.org/updates/FEDORA-2024-6c7480dd2f
+    fi
+
+    # FCOS doesn't enable opt-usrlocal-overlays so use the hack instead
+    mkdir -p /etc/systemd/system/rpm-ostreed.service.d/
+    cat > /etc/systemd/system/rpm-ostreed.service.d/state-overlay.conf <<EOF
+[Service]
+Environment=RPMOSTREE_EXPERIMENTAL_FORCE_OPT_USRLOCAL_OVERLAY=1
+EOF
+
+    # This script itself is in /usr/local, so we need to move it back on top
+    # of the overlay. This simultaneously demos one way upgrading nodes could
+    # retain content if we turn on opt-usrlocal-overlays in FCOS.
+    cat > /etc/systemd/system/move-usr-local.service <<EOF
+[Unit]
+Description=Move Previous /usr/local content back into /usr/local
+After=local-fs.target
+After=systemd-tmpfiles-setup.service
+Before=kola-runext.service
+
+[Service]
+Type=oneshot
+# I previously used rsync here to sync all of /var/usrlocal, but hit SELinux
+# issues; it seems like it runs as rsync_t which can't do things on top of
+# overlay? To investigate...
+ExecStart=cp /var/usrlocal/bin/kola-runext-state-overlays /usr/local/bin/
+RemainAfterExit=yes
+
+[Install]
+WantedBy=multi-user.target
+EOF
+    systemctl daemon-reload
+    systemctl restart rpm-ostreed
+    systemctl enable move-usr-local.service
+
+    rpm-ostree install test-opt
+
+    /tmp/autopkgtest-reboot 1
+    ;;
+  1)
+    test -f /opt/megacorp/bin/test-opt
+    test -f /opt/megacorp/lib/mylib
+    test -d /opt/megacorp/state
+
+    /opt/megacorp/bin/test-opt > /tmp/out.txt
+    assert_file_has_content /tmp/out.txt 'test-opt'
+    assert_file_has_content /opt/megacorp/lib/mylib 'lib1'
+
+    # add some state files
+    echo 'foobar' > /opt/megacorp/state/mystate
+
+    # change some base files
+    echo 'badlib' > /opt/megacorp/lib/mylib
+    /tmp/autopkgtest-reboot 2
+    ;;
+  2)
+    # check our state is still there
+    assert_file_has_content /opt/megacorp/state/mystate 'foobar'
+    assert_file_has_content /opt/megacorp/lib/mylib 'badlib'
+
+    # upgrade to -2
+    sed -i -e 's,rpm-repos/0,rpm-repos/1,' /etc/yum.repos.d/vmcheck.repo
+    rpm-ostree upgrade
+    /tmp/autopkgtest-reboot 3
+    ;;
+  3)
+    # check our state is still there
+    assert_file_has_content /opt/megacorp/state/mystate 'foobar'
+
+    # but base content has been reset
+    assert_file_has_content /opt/megacorp/lib/mylib 'lib2'
+    ;;
+  *) echo "unexpected mark: ${AUTOPKGTEST_REBOOT_MARK}"; exit 1;;
+esac

tests/kolainst/kolainst-build.sh

@@ -96,6 +96,13 @@ build_module_defaults foomodular \
 build_rpm zincati version 99.99 release 2
 build_rpm zincati version 99.99 release 3
 
+# An RPM that installs in /opt
+build_rpm test-opt \
+             install "mkdir -p %{buildroot}/opt/megacorp/{bin,lib,state}
+                      install %{name} %{buildroot}/opt/megacorp/bin
+                      echo lib1 > %{buildroot}/opt/megacorp/lib/mylib" \
+             files "/opt/megacorp"
+
 mv ${test_tmpdir}/yumrepo/* ${test_tmpdir}/rpm-repos/${repover}
 
 # To test remote override replace update
@@ -111,6 +118,14 @@ build_rpm pkgsystemd \
   install "mkdir -p %{buildroot}/usr/lib/systemd/system 
            install %{name}.service %{buildroot}/usr/lib/systemd/system" \
   files "/usr/lib/systemd/system/%{name}.service"
+
+# to test updates to RPMs that install in /opt
+build_rpm test-opt release 2 \
+             install "mkdir -p %{buildroot}/opt/megacorp/{bin,lib,state}
+                      install %{name} %{buildroot}/opt/megacorp/bin
+                      echo lib2 > %{buildroot}/opt/megacorp/lib/mylib" \
+             files "/opt/megacorp"
+
 mv ${test_tmpdir}/yumrepo/* ${test_tmpdir}/rpm-repos/${repover}
 
 # Create an empty repo when we want to test inability to find a package