apply: Use staged deployment on booted systems

When finalizing an OSTree deployment, the current `/etc` is merged with
the new commit's `/usr/etc`. Any changes that happen in the current
`/etc` after the deployment has been finalized will not appear in the
new deployment. Since eos-updater is often run in the background, it's
likely the user will make changes in `/etc` (such as creating a new
user) long before the new deployment is booted into.

To address this issue, OSTree has provided the concept of a staged
deployment since 2018.5. The new deployment is initialized but not
finalized during shutdown via the `ostree-finalize-staged.service`
systemd unit. Since staged deployments only work on OSTree booted
systems that can initiate systemd units, this can't really work in the
current test suite. The old full deployment method is kept for that
case.

Note that staged deployment finalization depends on the
`ostree-finalize-staged.path` systemd unit being activated. Currently,
OSTree does this on demand but in the future it may require the OS to
explicitly activate the unit via a systemd preset or similar mechanism.

https://phabricator.endlessm.com/T5658

eos-updater/apply.c

@@ -241,6 +241,7 @@ apply_internal (ApplyData     *apply_data,
   g_autoptr(GKeyFile) origin = NULL;
   g_autoptr(OstreeSysroot) sysroot = NULL;
   const gchar *osname = get_test_osname ();
+  gboolean staged_deploy;
   g_autoptr(GError) local_error = NULL;
 
   sysroot = ostree_sysroot_new_default ();
@@ -269,21 +270,62 @@ apply_internal (ApplyData     *apply_data,
 
   origin = ostree_sysroot_origin_new_from_refspec (sysroot, update_refspec);
 
-  if (!ostree_sysroot_deploy_tree (sysroot,
-                                   osname,
-                                   update_id,
-                                   origin,
-                                   booted_deployment,
-                                   NULL,
-                                   &new_deployment,
-                                   cancellable,
-                                   error))
-    return FALSE;
+  /* When booted into an OSTree system, stage the deployment so that the
+   * /etc merge happens during shutdown. Otherwise (primarily the test
+   * suite), deploy the finalized tree immediately.
+   */
+  staged_deploy = ostree_sysroot_is_booted (sysroot);
+  if (staged_deploy)
+    {
+      g_message ("Creating staged deployment for revision %s", update_id);
+      if (!ostree_sysroot_stage_tree (sysroot,
+                                      osname,
+                                      update_id,
+                                      origin,
+                                      booted_deployment,
+                                      NULL,
+                                      &new_deployment,
+                                      cancellable,
+                                      error))
+        return FALSE;
+    }
+  else
+    {
+      g_message ("Creating finalized deployment for revision %s", update_id);
+      if (!ostree_sysroot_deploy_tree (sysroot,
+                                       osname,
+                                       update_id,
+                                       origin,
+                                       booted_deployment,
+                                       NULL,
+                                       &new_deployment,
+                                       cancellable,
+                                       error))
+        return FALSE;
+
+      if (!ostree_sysroot_simple_write_deployment (sysroot,
+                                                   osname,
+                                                   new_deployment,
+                                                   booted_deployment,
+                                                   OSTREE_SYSROOT_SIMPLE_WRITE_DEPLOYMENT_FLAGS_NO_CLEAN,
+                                                   cancellable,
+                                                   error))
+        return FALSE;
+    }
+
+  g_message ("New deployment: index: %d, OS name: %s, deploy serial: %d, "
+             "checksum: %s, boot checksum: %s, boot serial: %d",
+             ostree_deployment_get_index (new_deployment),
+             ostree_deployment_get_osname (new_deployment),
+             ostree_deployment_get_deployserial (new_deployment),
+             ostree_deployment_get_csum (new_deployment),
+             ostree_deployment_get_bootcsum (new_deployment),
+             ostree_deployment_get_bootserial (new_deployment));
 
   /* If the original refspec is not the update refspec, then we may have
    * a ref to a no longer needed tree. Delete that remote ref so the
-   * cleanup done in simple_write_deployment() really removes that tree
-   * if no deployments point to it anymore.
+   * sysroot cleanup below really removes that tree if no deployments
+   * point to it anymore.
    */
   if (g_strcmp0 (update_refspec, orig_refspec) != 0)
     {
@@ -304,32 +346,17 @@ apply_internal (ApplyData     *apply_data,
         }
     }
 
-  if (!ostree_sysroot_simple_write_deployment (sysroot,
-                                               osname,
-                                               new_deployment,
-                                               booted_deployment,
-                                               OSTREE_SYSROOT_SIMPLE_WRITE_DEPLOYMENT_FLAGS_NO_CLEAN,
-                                               cancellable,
-                                               error))
-    return FALSE;
-
-  g_message ("New deployment: index: %d, OS name: %s, deploy serial: %d, "
-             "checksum: %s, boot checksum: %s, boot serial: %d",
-             ostree_deployment_get_index (new_deployment),
-             ostree_deployment_get_osname (new_deployment),
-             ostree_deployment_get_deployserial (new_deployment),
-             ostree_deployment_get_csum (new_deployment),
-             ostree_deployment_get_bootcsum (new_deployment),
-             ostree_deployment_get_bootserial (new_deployment));
-
   /* FIXME: Cleaning up after update should be non-fatal, since we've
    * already successfully deployed the new OS. This clearly is a
    * workaround for a more serious issue, likely related to concurrent
    * prunes (https://phabricator.endlessm.com/T16736). */
-  if (!ostree_sysroot_cleanup (sysroot, cancellable, &local_error))
-    g_warning ("Failed to clean up the sysroot after successful deployment: %s",
-               local_error->message);
-  g_clear_error (&local_error);
+  if (!staged_deploy)
+    {
+      if (!ostree_sysroot_cleanup (sysroot, cancellable, &local_error))
+        g_warning ("Failed to clean up the sysroot after finalized deployment: %s",
+                   local_error->message);
+      g_clear_error (&local_error);
+    }
 
   /* Try to update the remote branches option to use the new refspec.
    * This option is almost never used and has no impact on future