apply: Use staged deployment on booted systems

debian/control

@@ -39,6 +39,7 @@ Multi-arch: no
 Depends:
  gir1.2-glib-2.0,
  mogwai-scheduled,
+ ostree,
  python3-gi,
  systemd (>= 200),
  ${misc:Depends},

debian/eos-updater.install

@@ -2,12 +2,14 @@ lib/systemd/system/eos-autoupdater.service
 lib/systemd/system/eos-autoupdater.timer
 lib/systemd/system/eos-updater-avahi.path
 lib/systemd/system/eos-updater-avahi.service
+lib/systemd/system/eos-updater-autocleanup.service
 lib/systemd/system/eos-updater-flatpak-installer.service
 lib/systemd/system/eos-updater-flatpak-installer-fallback.service
 lib/systemd/system/eos-updater-flatpak-installer-fallback.timer
 lib/systemd/system/eos-updater.service
 lib/systemd/system/eos-update-server.service
 lib/systemd/system/eos-update-server.socket
+lib/systemd/system/ostree-finalize-staged.service.d/autocleanup.conf
 usr/libexec/eos-updater
 usr/libexec/eos-autoupdater
 usr/libexec/eos-updater-avahi

eos-updater/apply.c

@@ -241,6 +241,7 @@ apply_internal (ApplyData     *apply_data,
   g_autoptr(GKeyFile) origin = NULL;
   g_autoptr(OstreeSysroot) sysroot = NULL;
   const gchar *osname = get_test_osname ();
+  gboolean staged_deploy;
   g_autoptr(GError) local_error = NULL;
 
   sysroot = ostree_sysroot_new_default ();
@@ -269,21 +270,62 @@ apply_internal (ApplyData     *apply_data,
 
   origin = ostree_sysroot_origin_new_from_refspec (sysroot, update_refspec);
 
-  if (!ostree_sysroot_deploy_tree (sysroot,
-                                   osname,
-                                   update_id,
-                                   origin,
-                                   booted_deployment,
-                                   NULL,
-                                   &new_deployment,
-                                   cancellable,
-                                   error))
-    return FALSE;
+  /* When booted into an OSTree system, stage the deployment so that the
+   * /etc merge happens during shutdown. Otherwise (primarily the test
+   * suite), deploy the finalized tree immediately.
+   */
+  staged_deploy = ostree_sysroot_is_booted (sysroot);
+  if (staged_deploy)
+    {
+      g_message ("Creating staged deployment for revision %s", update_id);
+      if (!ostree_sysroot_stage_tree (sysroot,
+                                      osname,
+                                      update_id,
+                                      origin,
+                                      booted_deployment,
+                                      NULL,
+                                      &new_deployment,
+                                      cancellable,
+                                      error))
+        return FALSE;
+    }
+  else
+    {
+      g_message ("Creating finalized deployment for revision %s", update_id);
+      if (!ostree_sysroot_deploy_tree (sysroot,
+                                       osname,
+                                       update_id,
+                                       origin,
+                                       booted_deployment,
+                                       NULL,
+                                       &new_deployment,
+                                       cancellable,
+                                       error))
+        return FALSE;
+
+      if (!ostree_sysroot_simple_write_deployment (sysroot,
+                                                   osname,
+                                                   new_deployment,
+                                                   booted_deployment,
+                                                   OSTREE_SYSROOT_SIMPLE_WRITE_DEPLOYMENT_FLAGS_NO_CLEAN,
+                                                   cancellable,
+                                                   error))
+        return FALSE;
+    }
+
+  g_message ("New deployment: index: %d, OS name: %s, deploy serial: %d, "
+             "checksum: %s, boot checksum: %s, boot serial: %d",
+             ostree_deployment_get_index (new_deployment),
+             ostree_deployment_get_osname (new_deployment),
+             ostree_deployment_get_deployserial (new_deployment),
+             ostree_deployment_get_csum (new_deployment),
+             ostree_deployment_get_bootcsum (new_deployment),
+             ostree_deployment_get_bootserial (new_deployment));
 
   /* If the original refspec is not the update refspec, then we may have
    * a ref to a no longer needed tree. Delete that remote ref so the
-   * cleanup done in simple_write_deployment() really removes that tree
-   * if no deployments point to it anymore.
+   * sysroot cleanup below really removes that tree if no deployments
+   * point to it anymore.
    */
   if (g_strcmp0 (update_refspec, orig_refspec) != 0)
     {
@@ -304,28 +346,17 @@ apply_internal (ApplyData     *apply_data,
         }
     }
 
-  if (!ostree_sysroot_simple_write_deployment (sysroot,
-                                               osname,
-                                               new_deployment,
-                                               booted_deployment,
-                                               OSTREE_SYSROOT_SIMPLE_WRITE_DEPLOYMENT_FLAGS_NO_CLEAN,
-                                               cancellable,
-                                               error))
-    return FALSE;
-
-  g_message ("New deployment: index: %d, OS name: %s, deploy serial: %d, "
-             "checksum: %s, boot checksum: %s, boot serial: %d",
-             ostree_deployment_get_index (new_deployment),
-             ostree_deployment_get_osname (new_deployment),
-             ostree_deployment_get_deployserial (new_deployment),
-             ostree_deployment_get_csum (new_deployment),
-             ostree_deployment_get_bootcsum (new_deployment),
-             ostree_deployment_get_bootserial (new_deployment));
-
   /* FIXME: Cleaning up after update should be non-fatal, since we've
    * already successfully deployed the new OS. This clearly is a
    * workaround for a more serious issue, likely related to concurrent
-   * prunes (https://phabricator.endlessm.com/T16736). */
+   * prunes (https://phabricator.endlessm.com/T16736).
+   *
+   * TODO: When using staged deployments, there's likely nothing to
+   * prune since the old rollback deployment isn't removed until the
+   * staged deployment is finalized during system shutdown. Pruning
+   * happens during the subsequent boot, so this cleanup could be
+   * skipped for staged deploys.
+   */
   if (!ostree_sysroot_cleanup (sysroot, cancellable, &local_error))
     g_warning ("Failed to clean up the sysroot after successful deployment: %s",
                local_error->message);

eos-updater/eos-updater-autocleanup.service.in

@@ -0,0 +1,37 @@
+[Unit]
+Description=Automatically cleanup after staged Endless OS Updater deployment
+Documentation=man:ostree-admin-cleanup(1) man:eos-updater(8)
+
+# Run ostree admin cleanup only if /sysroot/.cleanup is present and then
+# delete it when cleanup is successful.
+#
+# FIXME: Drop this when https://github.com/ostreedev/ostree/issues/2510
+# is resolved.
+ConditionPathExists=/sysroot/.cleanup
+
+# We want this to be triggered by multi-user.target but not block it via
+# the default After added to target units since pruning the repo can be
+# slow. See the Default Dependencies sections in systemd.service(5) and
+# systemd.target(5).
+DefaultDependencies=no
+Requires=sysinit.target
+After=sysinit.target basic.target
+Conflicts=shutdown.target
+Before=shutdown.target
+
+[Service]
+Type=oneshot
+RemainAfterExit=yes
+ExecStart=@ostree@ admin cleanup
+ExecStart=/bin/rm -f /sysroot/.cleanup
+
+# Only /sysroot and /boot need to be written to.
+ProtectSystem=strict
+ReadWritePaths=/sysroot /boot
+
+# This will be allowed to run in the background, so try to make it less
+# disruptive while it prunes the repo.
+IOSchedulingClass=idle
+
+[Install]
+WantedBy=multi-user.target

eos-updater/finalize-autocleanup.conf.in

@@ -0,0 +1,9 @@
+# This is a drop-in file for ostree-finalize-staged.service.
+#
+# FIXME: Drop this when https://github.com/ostreedev/ostree/issues/2510
+# is resolved.
+
+[Service]
+# After finalizing the staged deployment, touch the .cleanup file so
+# that the cleanup can be completed on the next boot.
+ExecStop=-/bin/touch /sysroot/.cleanup

eos-updater/meson.build

@@ -105,6 +105,7 @@ install_data(
 # systemd files
 config = configuration_data()
 config.set('libexecdir', join_paths(get_option('prefix'), get_option('libexecdir')))
+config.set('ostree', find_program('ostree').path())
 
 configure_file(
   input: 'eos-updater.service.in',
@@ -113,6 +114,20 @@ configure_file(
   configuration: config,
 )
 
+configure_file(
+  input: 'eos-updater-autocleanup.service.in',
+  output: 'eos-updater-autocleanup.service',
+  install_dir: dependency('systemd').get_pkgconfig_variable('systemdsystemunitdir'),
+  configuration: config,
+)
+
+configure_file(
+  input: 'finalize-autocleanup.conf.in',
+  output: 'autocleanup.conf',
+  install_dir: join_paths(dependency('systemd').get_pkgconfig_variable('systemdsystemunitdir'), 'ostree-finalize-staged.service.d'),
+  configuration: config,
+)
+
 # Example configuration
 install_data(
   files('eos-updater.conf'),

tests/meson.build

@@ -106,7 +106,7 @@ foreach test_name, extra_args : test_programs
     exe,
     env: envs,
     suite: ['eos-updater'] + extra_args.get('suite', []),
-    timeout: extra_args.get('slow', false) ? 360 : 60,
+    timeout: extra_args.get('slow', false) ? 600 : 60,
     is_parallel: extra_args.get('parallel', true),
   )
 endforeach