diff --git a/.cci.jenkinsfile b/.cci.jenkinsfile
index 82d958b285..d902235a26 100644
--- a/.cci.jenkinsfile
+++ b/.cci.jenkinsfile
@@ -1,39 +1,70 @@
-// Documentation: https://github.com/coreos/coreos-ci/blob/master/README-upstream-ci.md
+// Documentation: https://github.com/coreos/coreos-ci/blob/main/README-upstream-ci.md
 
-cosaPod {
+properties([
+    // abort previous runs when a PR is updated to save resources
+    disableConcurrentBuilds(abortPrevious: true)
+])
+
+// We run `kolaTestIso` which requires at least 8Gi. Add 1Gi for overhead.
+cosaPod(cpus: 4, memory: "9Gi") {
     checkoutToDir(scm, 'config')
 
+    def basearch = shwrapCapture("cosa basearch")
+    def mechanical_streams = ['branched', 'rawhide']
+
     shwrap("cd config && ci/validate")
 
     shwrap("""
-        mkdir -p /srv/fcos && cd /srv/fcos
+        mkdir -p /srv/coreos && cd /srv/coreos
         cosa init ${env.WORKSPACE}/config
-        curl -LO https://raw.githubusercontent.com/coreos/fedora-coreos-releng-automation/master/scripts/download-overrides.py
-        python3 download-overrides.py
+        python3 /usr/lib/coreos-assembler/download-overrides.py
         # prep from the latest builds so that we generate a diff on PRs that add packages
-        cosa buildprep https://builds.coreos.fedoraproject.org/prod/streams/${env.CHANGE_TARGET}/builds
+        cosa buildfetch --stream=${env.CHANGE_TARGET}
     """)
 
-    fcosBuild(skipInit: true, extraFetchArgs: '--with-cosa-overrides')
+    // use a --parent-build arg so we can diff later and it matches prod
+    def parent_arg = ""
+    def parent_commit = ""
+    if (shwrapRc("test -e /srv/coreos/builds/latest/${basearch}/meta.json") == 0) {
+        shwrap("cp /srv/coreos/builds/latest/${basearch}/meta.json .") // readJSON wants it in the WORKSPACE
+        def meta = readJSON file: "meta.json"
+        def version = meta["buildid"]
+        parent_arg = "--parent-build ${version}"
+        parent_commit = meta["ostree-commit"]
+    }
+
+    // do a build. If we are operating on a mechanical stream then we
+    // can pin packages in lockfiles but we don't maintain a full set
+    // so we can't do a strict build.
+    def no_strict_build = false
+    if (env.CHANGE_TARGET in mechanical_streams) {
+        no_strict_build = true
+    }
+    cosaBuild(skipInit: true, noStrict: no_strict_build, extraFetchArgs: '--with-cosa-overrides', extraArgs: parent_arg)
 
     parallel metal: {
-        shwrap("cd /srv/fcos && cosa buildextend-metal")
+        shwrap("cd /srv/coreos && cosa buildextend-metal")
     }, metal4k: {
-        shwrap("cd /srv/fcos && cosa buildextend-metal4k")
+        shwrap("cd /srv/coreos && cosa buildextend-metal4k")
     }
 
     stage("Test ISO") {
-        shwrap("cd /srv/fcos && cosa buildextend-live")
-        try {
-            shwrap("cd /srv/fcos && kola testiso -S --scenarios pxe-install,pxe-offline-install,iso-install,iso-offline-install --output-dir tmp/kola-testiso-metal")
-        } finally {
-            shwrap("cd /srv/fcos && tar -cf - tmp/kola-testiso-metal/ | xz -c9 > ${env.WORKSPACE}/kola-testiso-metal.tar.xz")
-            archiveArtifacts allowEmptyArchive: true, artifacts: 'kola-testiso-metal.tar.xz'
-        }
+        shwrap("cd /srv/coreos && cosa buildextend-live")
+        kolaTestIso()
     }
 
     // also print the pkgdiff as a separate stage to make it more visible
-    stage("RPM Diff") {
-        shwrap("jq .pkgdiff /srv/fcos/builds/latest/x86_64/meta.json")
+    if (parent_arg != "") {
+        stage("RPM Diff") {
+            shwrap("""
+                cd /srv/coreos
+                new_commit=\$(jq -r '.["ostree-commit"]' builds/latest/${basearch}/meta.json)
+                rpm-ostree db diff --repo tmp/repo ${parent_commit} \${new_commit} | tee tmp/diff.txt
+                if grep -q Downgraded tmp/diff.txt; then
+                    echo "Downgrade detected. This is likely unintentional. If not, you may safely ignore this error."
+                    exit 1
+                fi
+            """)
+        }
     }
 }
diff --git a/.github/workflows/add-override.yml b/.github/workflows/add-override.yml
new file mode 100644
index 0000000000..e243037f2f
--- /dev/null
+++ b/.github/workflows/add-override.yml
@@ -0,0 +1,93 @@
+---
+name: Add package override
+on:
+  workflow_dispatch:
+    inputs:
+      target:
+        description: Target branch
+        default: testing-devel
+      what:
+        description: "Bodhi update (fast-track) or SRPM NVR (pin)"
+      pin:
+        description: "Pin (don't remove when stable)"
+        type: boolean
+      reason:
+        description: "Reason URL (optional for routine fast-tracks)"
+
+permissions:
+  # none at all
+  contents: none
+
+# This workflow could almost use the default GITHUB_TOKEN, if we were to
+# push the branch into this repo.  However, GitHub Actions has recursion
+# avoidance that would prevent CI from running on the PR:
+#
+# https://github.com/peter-evans/create-pull-request/blob/28fa4848947e/docs/concepts-guidelines.md#workarounds-to-trigger-further-workflow-runs
+#
+# So we create the PR using a separate Personal Access Token in
+# COREOSBOT_RELENG_TOKEN, belonging to a machine account.  That allows CI to
+# run when the PR is first created.  However, it's also possible to rerun
+# the workflow and have it force-push the branch, reusing the same PR.  In
+# that case the push also cannot come from GITHUB_TOKEN, or CI will not
+# rerun.  Thus we also do the push using COREOSBOT_RELENG_TOKEN.  Since we
+# don't want to give the machine account privileges to this repo, we push
+# to a forked repo owned by the machine account.
+
+jobs:
+  add-override:
+    name: Add package override
+    runs-on: ubuntu-latest
+    container: quay.io/fedora/fedora:latest
+    steps:
+      - name: Install dependencies
+        run: dnf install -y git jq python3-bodhi-client python3-pyyaml
+      - name: Check out repository
+        uses: actions/checkout@v3
+        with:
+          ref: ${{ github.event.inputs.target }}
+          # We need an unbroken commit chain when pushing to the fork.  Don't
+          # make assumptions about which commits are already available there.
+          fetch-depth: 0
+      # https://github.com/actions/checkout/issues/766
+      - name: Mark git checkout as safe
+        run: git config --global --add safe.directory "$GITHUB_WORKSPACE"
+      - name: Update metadata
+        env:
+          TARGET: ${{ github.event.inputs.target }}
+          WHAT: ${{ github.event.inputs.what }}
+          PIN: ${{ github.event.inputs.pin }}
+          REASON: ${{ github.event.inputs.reason }}
+        run: |
+          set -euxo pipefail
+
+          if [ "${PIN}" = true ]; then
+              ci/overrides.py pin "${WHAT}" -r "${REASON}"
+              title="overrides: pin ${WHAT}"
+          else
+              ci/overrides.py fast-track "${WHAT}" ${REASON:+-r "${REASON}"}
+              srpms=$(ci/overrides.py srpms "${WHAT}" | paste -sd,)
+              title="overrides: fast-track ${srpms//,/, }"
+          fi
+
+          if [ "${TARGET}" = testing-devel ]; then
+              pr_title="${title}"
+          else
+              pr_title="[${TARGET}] ${title}"
+          fi
+          branch_name=override-$(echo "${TARGET}:${title}" | sha256sum | cut -c1-8)
+
+          echo "BRANCH_NAME=${branch_name}" >> ${GITHUB_ENV}
+          echo "COMMIT_TITLE=${title}" >> ${GITHUB_ENV}
+          echo "PR_TITLE=${pr_title}" >> ${GITHUB_ENV}
+      - name: Open pull request
+        uses: peter-evans/create-pull-request@v6
+        with:
+          token: ${{ secrets.COREOSBOT_RELENG_TOKEN }}
+          branch: ${{ env.BRANCH_NAME }}
+          commit-message: ${{ env.COMMIT_TITLE }}
+          push-to-fork: coreosbot-releng/fedora-coreos-config
+          title: ${{ env.PR_TITLE }}
+          body: "Requested by @${{ github.actor }} via [GitHub workflow](${{ github.server_url }}/${{ github.repository }}/actions/workflows/add-override.yml) ([source](${{ github.server_url }}/${{ github.repository }}/blob/testing-devel/.github/workflows/add-override.yml))."
+          committer: "CoreOS Bot <coreosbot@fedoraproject.org>"
+          author: "CoreOS Bot <coreosbot@fedoraproject.org>"
+          delete-branch: true
diff --git a/.github/workflows/find-whitespace.yml b/.github/workflows/find-whitespace.yml
new file mode 100644
index 0000000000..a8b5cf84d0
--- /dev/null
+++ b/.github/workflows/find-whitespace.yml
@@ -0,0 +1,22 @@
+name: Find whitespace
+
+on:
+  pull_request:
+    branches: [testing-devel]
+
+permissions:
+  contents: read
+
+jobs:
+  find-whitespace:
+    name: Find whitespace
+    runs-on: ubuntu-latest
+    container: quay.io/coreos-assembler/fcos-buildroot:testing-devel
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v3
+      # https://github.com/actions/checkout/issues/760
+      - name: Mark git checkout as safe
+        run: git config --global --add safe.directory "$GITHUB_WORKSPACE"
+      - name: Look for whitespace at the end of line
+        run: ci/find-whitespace
diff --git a/.github/workflows/next-devel.yml b/.github/workflows/next-devel.yml
new file mode 100644
index 0000000000..1fcc9b7036
--- /dev/null
+++ b/.github/workflows/next-devel.yml
@@ -0,0 +1,34 @@
+---
+name: next-devel
+on:
+  # This is a privileged event!  It runs with a r/w token, even in PRs from
+  # forks.
+  pull_request_target:
+    branches: [next-devel]
+    types: [opened, edited, reopened, ready_for_review]
+
+permissions:
+  pull-requests: write
+
+# Privileged job to comment on next-devel PRs indicating whether next-devel
+# is currently open.  This job must not trust the contents of the PR.
+
+jobs:
+  branch-status:
+    name: "Check branch status"
+    runs-on: ubuntu-latest
+    steps:
+      - name: Post PR comment
+        uses: actions/github-script@v6
+        with:
+          script: |
+            const url = 'https://raw.githubusercontent.com/coreos/fedora-coreos-pipeline/main/next-devel/status.json'
+            const resp = await github.request(url)
+            if (!JSON.parse(resp.data).enabled) {
+              await github.rest.issues.createComment({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                issue_number: context.issue.number,
+                body: ':no_entry: The `next-devel` branch is currently closed.  PRs should target only `testing-devel`.',
+              })
+            }
diff --git a/.github/workflows/openshift-os.yml b/.github/workflows/openshift-os.yml
new file mode 100644
index 0000000000..1e2d5ff744
--- /dev/null
+++ b/.github/workflows/openshift-os.yml
@@ -0,0 +1,129 @@
+name: Sync to openshift/os
+on:
+  # We could do push: branches: [testing-devel] but that would restart
+  # downstream CI a lot
+  schedule:
+    - cron: '0 0 * * *'
+  workflow_dispatch:
+    inputs:
+      branch:
+        # Allow specifying the source branch for backports
+        description: Source branch to use for PR
+        required: true
+        default: testing-devel
+        type: choice
+        options:
+          - testing-devel
+          - rhcos-4.17
+          - rhcos-4.16
+          - rhcos-4.15
+          - rhcos-4.14
+          - rhcos-4.13
+          - rhcos-4.12
+          - rhcos-4.11
+          - rhcos-4.10
+      jira:
+        description: The JIRA reference to put in the PR title. Defaults to "NO-JIRA".
+        required: false
+        default: NO-JIRA
+        type: string
+
+permissions:
+  # none at all
+  contents: none
+
+jobs:
+  update-submodule:
+    name: Update fedora-coreos-config submodule
+    runs-on: ubuntu-latest
+    env:
+      SOURCE_BRANCH: ${{ github.event.inputs.branch }}
+      JIRA: ${{ github.event.inputs.jira }}
+    steps:
+      - name: Set branches values
+        run: |
+          set -euxo pipefail
+
+          case $SOURCE_BRANCH in
+              # in the on.schedule case, the SOURCE_BRANCH is empty
+              testing-devel|"")
+                echo "SOURCE_BRANCH=testing-devel" >> $GITHUB_ENV
+                echo "TARGET_BRANCH=master" >> $GITHUB_ENV
+                echo "BRANCH_NAME=fcc-sync" >> $GITHUB_ENV
+                ;;
+              rhcos-*)
+                # split the string around the -
+                array=(${SOURCE_BRANCH//-/ })
+                OCP_VERSION=${array[1]}
+                echo "TARGET_BRANCH=release-${OCP_VERSION}" >> $GITHUB_ENV
+                echo "BRANCH_NAME=fcc-sync-${SOURCE_BRANCH}" >> $GITHUB_ENV
+                echo "TITLE_PREFIX=[release-${OCP_VERSION}] " >> $GITHUB_ENV
+                ;;
+              *)
+                echo "SOURCE_BRANCH=$SOURCE_BRANCH is invalid" >&2
+                exit 1
+                ;;
+          esac
+
+          echo "JIRA=${JIRA:-NO-JIRA}" >> $GITHUB_ENV
+      - name: Check out repository
+        uses: actions/checkout@v3
+        with:
+          repository: openshift/os
+          # We need an unbroken commit chain when pushing to the fork.  Don't
+          # make assumptions about which commits are already available there.
+          fetch-depth: 0
+          # We need to checkout against the target branch
+          ref: ${{ env.TARGET_BRANCH }}
+
+      - name: Update submodule
+        run: |
+          set -euxo pipefail
+
+          git submodule init
+          git submodule update
+
+          cd fedora-coreos-config
+          # the submodule init only fetch the submodule commit and the default branch `testing-devel`
+          git fetch origin ${SOURCE_BRANCH}
+          # Omit CoreOS Bot commits from the log message, since they generally
+          # only affect FCOS
+          git shortlog "HEAD..FETCH_HEAD" --perl-regexp \
+              --author='^((?!CoreOS Bot <coreosbot@fedoraproject.org>).*)$' \
+              > $RUNNER_TEMP/shortlog
+
+          if [ ! -s $RUNNER_TEMP/shortlog ]; then
+              # Any changes have been made by CoreOS Bot.  Ignore.
+              echo "No non-trivial changes; exiting"
+              exit 0
+          fi
+
+          git checkout $SOURCE_BRANCH
+
+          marker=OPENSHIFT-OS-END-OF-LOG-MARKER-$RANDOM$RANDOM$RANDOM
+          cat >> $GITHUB_ENV <<EOF
+          SHORTLOG<<$marker
+          $(cat $RUNNER_TEMP/shortlog)
+          $marker
+          EOF
+
+      - name: Open pull request
+        uses: peter-evans/create-pull-request@v6
+        with:
+          token: ${{ secrets.COREOSBOT_RELENG_TOKEN }}
+          push-to-fork: coreosbot-releng/os
+          branch: ${{ env.BRANCH_NAME }}
+          base: ${{ env.TARGET_BRANCH }}
+          commit-message: |
+            Bump fedora-coreos-config
+
+            ${{ env.SHORTLOG }}
+          title: "${{ env.TITLE_PREFIX }}${{ env.JIRA }}: Bump fedora-coreos-config"
+          body: |
+            Created by [GitHub workflow](${{ github.server_url }}/${{ github.repository }}/actions/workflows/openshift-os.yml) ([source](${{ github.server_url }}/${{ github.repository }}/blob/testing-devel/.github/workflows/openshift-os.yml)).
+
+            ```
+            ${{ env.SHORTLOG }}
+            ```
+          committer: "CoreOS Bot <coreosbot@fedoraproject.org>"
+          author: "CoreOS Bot <coreosbot@fedoraproject.org>"
diff --git a/.github/workflows/promotion-diff.yml b/.github/workflows/promotion-diff.yml
new file mode 100644
index 0000000000..6be39fddbf
--- /dev/null
+++ b/.github/workflows/promotion-diff.yml
@@ -0,0 +1,50 @@
+---
+name: Check promotion diffs
+on:
+  pull_request:
+    branches: [next, testing, stable]
+permissions:
+  contents: read
+
+jobs:
+  promotion-diff:
+    name: Check promotion diffs
+    runs-on: ubuntu-latest
+    # Only run if this looks like a promotion PR
+    if: "contains(github.event.pull_request.title, 'tree: promote changes from')"
+    steps:
+      - name: Get base commit hash
+        env:
+          COMMIT_TITLE: ${{ github.event.pull_request.title }}
+        run: |
+          echo "ORIGIN_COMMIT=$(echo ${COMMIT_TITLE} | awk '{print $NF}')" >> $GITHUB_ENV
+      - name: Check out origin commit
+        uses: actions/checkout@v3
+        with:
+          path: origin
+          ref: ${{ env.ORIGIN_COMMIT }}
+      - name: Check out base branch
+        uses: actions/checkout@v3
+        with:
+          path: base
+          ref: ${{ github.base_ref }}
+      - name: Check out PR
+        uses: actions/checkout@v3
+        with:
+          path: new
+      - name: Ignore manifest files
+        run: |
+          # manifest.yaml is per-branch, so we care about changes vs. the
+          # one in the base, not the one from the origin
+          cp base/manifest.yaml origin/
+      - name: Normalize kola-denylist.yaml
+        run: |
+          # When we promote to a production branch we strip out the
+          # snooze and warn lines. Let's do the same here so we don't get warnings.
+          # See https://github.com/coreos/fedora-coreos-releng-automation/pull/179
+          sed -E -i 's/^(\s+)((snooze:|warn:)\s+.*)/\1# \2 (disabled on promotion)/' origin/kola-denylist.yaml
+      - name: Compare trees
+        uses: coreos/actions-lib/check-diff@main
+        with:
+          basedir: origin
+          patchdir: new
diff --git a/.github/workflows/remove-graduated-overrides.yml b/.github/workflows/remove-graduated-overrides.yml
new file mode 100644
index 0000000000..aa89e466a6
--- /dev/null
+++ b/.github/workflows/remove-graduated-overrides.yml
@@ -0,0 +1,81 @@
+name: Remove graduated overrides
+
+on:
+  schedule:
+    - cron: '0 */6 * * *'
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  buildmatrix:
+    name: "Build job matrix"
+    runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{steps.build.outputs.matrix}}
+    steps:
+      - name: Build job matrix
+        id: build
+        run: |
+          set -xeuo pipefail
+
+          branches=(testing-devel branched rawhide)
+
+          enabled="$(curl -L https://raw.githubusercontent.com/coreos/fedora-coreos-pipeline/main/next-devel/status.json | jq .enabled)"
+          case "${enabled}" in
+          true) branches+=(next-devel) ;;
+          false) ;;
+          *)
+            echo "Unexpected value: ${enabled}"
+            exit 1
+            ;;
+          esac
+
+          echo "matrix=$(xargs -n 1 echo <<< "${branches[@]}" | jq -cnR '[inputs]')" >> $GITHUB_OUTPUT
+  remove-graduated-overrides:
+    name: Remove graduated overrides
+    needs: buildmatrix
+    runs-on: ubuntu-latest
+    # TODO: use cosa directly here
+    # https://github.com/coreos/coreos-assembler/issues/2223
+    container: quay.io/coreos-assembler/fcos-buildroot:testing-devel
+    strategy:
+      matrix:
+        branch: ${{fromJson(needs.buildmatrix.outputs.matrix)}}
+      fail-fast: false
+    steps:
+      - name: Enable CoreOS continuous repo
+        run: |
+          version_id=$(. /etc/os-release && echo ${VERSION_ID})
+          echo -e "[f${version_id}-coreos-continuous]\nenabled=1\nmetadata_expire=1m\nbaseurl=https://kojipkgs.fedoraproject.org/repos-dist/f${version_id}-coreos-continuous/latest/\$basearch/\ngpgcheck=0\nskip_if_unavailable=False\n" > /etc/yum.repos.d/coreos.repo
+      - name: Install dependencies
+        run: dnf install -y python3-bodhi-client rpm-ostree # see related TODO above
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          ref: ${{ matrix.branch }}
+      # https://github.com/actions/checkout/issues/766
+      - name: Mark git checkout as safe
+        run: git config --global --add safe.directory "$GITHUB_WORKSPACE"
+      - name: Remove graduated overrides
+        run: |
+          git config user.name 'CoreOS Bot'
+          git config user.email coreosbot@fedoraproject.org
+          ci/overrides.py graduate
+      - name: Create commit
+        run: |
+          if ! git diff --quiet --exit-code; then
+              git commit -am "lockfiles: drop graduated overrides 🎓" \
+                -m "Triggered by remove-graduated-overrides GitHub Action."
+          fi
+      - name: Open pull request
+        uses: peter-evans/create-pull-request@v6
+        with:
+          token: ${{ secrets.COREOSBOT_RELENG_TOKEN }}
+          branch: ${{ matrix.branch }}-graduation
+          push-to-fork: coreosbot-releng/fedora-coreos-config
+          title: "[${{ matrix.branch }}] lockfiles: drop graduated overrides 🎓"
+          body: "Created by remove-graduated-overrides [GitHub workflow](${{ github.server_url }}/${{ github.repository }}/actions/workflows/remove-graduated-overrides.yml) ([source](${{ github.server_url }}/${{ github.repository }}/blob/testing-devel/.github/workflows/remove-graduated-overrides.yml))."
+          committer: "CoreOS Bot <coreosbot@fedoraproject.org>"
+          author: "CoreOS Bot <coreosbot@fedoraproject.org>"
diff --git a/.github/workflows/shellcheck.yml b/.github/workflows/shellcheck.yml
new file mode 100644
index 0000000000..4e0497acc7
--- /dev/null
+++ b/.github/workflows/shellcheck.yml
@@ -0,0 +1,24 @@
+# Template generated by https://github.com/coreos/repo-templates; do not edit downstream
+
+name: ShellCheck
+
+on:
+  pull_request:
+    branches: [testing-devel]
+
+permissions:
+  contents: read
+
+jobs:
+  shellcheck:
+    name: Shellcheck
+    runs-on: ubuntu-latest
+    container: quay.io/coreos-assembler/fcos-buildroot:testing-devel
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v3
+      # https://github.com/actions/checkout/issues/760
+      - name: Mark git checkout as safe
+        run: git config --global --add safe.directory "$GITHUB_WORKSPACE"
+      - name: Run ShellCheck
+        run: ci/shellcheck
diff --git a/.gitleaks.toml b/.gitleaks.toml
new file mode 100644
index 0000000000..ca5e283ce8
--- /dev/null
+++ b/.gitleaks.toml
@@ -0,0 +1,5 @@
+[allowlist]
+description = "Exclude test keys from Red Hat secret scanning"
+files = [
+    ".*custom-host-key-permissions.*",
+]
diff --git a/README.md b/README.md
index c93b82a7b2..1809f276c9 100644
--- a/README.md
+++ b/README.md
@@ -1,4 +1,7 @@
 # Fedora CoreOS Config
+
+[![next-devel status](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/coreos/fedora-coreos-pipeline/main/next-devel/badge.json)](https://github.com/coreos/fedora-coreos-pipeline/blob/main/next-devel/README.md)
+
 Base manifest configuration for
 [Fedora CoreOS](https://coreos.fedoraproject.org/).
 
@@ -14,21 +17,24 @@ https://github.com/coreos/fedora-coreos-tracker.
 There is one branch for each stream. The default branch is
 [`testing-devel`](https://github.com/coreos/fedora-coreos-config/commits/testing-devel),
 on which all development happens. See
-[the design](https://github.com/coreos/fedora-coreos-tracker/blob/master/Design.md#release-streams)
-and [tooling](https://github.com/coreos/fedora-coreos-tracker/blob/master/stream-tooling.md)
+[the design](https://github.com/coreos/fedora-coreos-tracker/blob/main/Design.md#release-streams)
+and [tooling](https://github.com/coreos/fedora-coreos-tracker/blob/main/stream-tooling.md)
 docs for more information about streams.
 
 All file changes in `testing-devel` are propagated to other
-branches (to `bodhi-updates` through
-[config-bot](https://github.com/coreos/fedora-coreos-releng-automation/tree/master/config-bot),
-and to `testing` through usual promotion), with the
-following exceptions:
-- `manifest.yaml`: contains the stream "identity", such as
-  the ref, additional commit metadata, and yum input repos.
-- lockfiles (`manifest-lock.*` files): lockfiles are
-  imported from `bodhi-updates` to `testing-devel`.
-  Overrides (`manifest-lock.overrides.*`) are manually
-  curated.
+branches (to `next-devel`, `branched`, and `rawhide` through
+[config-bot](https://github.com/coreos/fedora-coreos-releng-automation/tree/main/config-bot),
+and to `testing` and eventually `stable` through usual
+promotion), with the following exceptions:
+- `manifest.yaml`: contains the stream's name, yum repos
+  used during composes, and the `releasever`.
+- lockfiles (`manifest-lock.*` files): on `testing-devel`
+  and `next-devel`, lockfiles are pushed by
+  [the `bump-lockfile` job](https://github.com/coreos/fedora-coreos-pipeline/blob/main/jobs/bump-lockfile.Jenkinsfile).
+  Production streams receive them as part of usual
+  promotion. Overrides (`manifest-lock.overrides.*`) are
+  managed independently with the help of some GitHub Actions
+  (see sections below).
 
 ## Layout
 
@@ -42,7 +48,7 @@ To derive from this repository, the recommendation is to add it
 as a git submodule.  Then create your own `manifest.yaml` which does
 `include: fedora-coreos-config/ignition-and-ostree.yaml` for example.
 You will also want to create an `overlay.d` and symlink in components
-in this repository's `overlay.d.
+in this repository's `overlay.d`.
 
 ## Overriding packages
 
@@ -50,22 +56,69 @@ By default, all packages for FCOS come from the stable
 Fedora repos. However, it is sometimes necessary to either
 hold back some packages, or pull in fixes ahead of Bodhi. To
 add such overrides, one needs to add the packages to
-`manifest-lock.overrides.$basearch.yaml`. E.g.:
+`manifest-lock.overrides.yaml` (there are also arch-specific
+variants of these files for the rare occasions the override
+should only apply to a specific arch). There is a
+[tool](ci/overrides.py) to help with this, and for simple
+cases, an [automated workflow](https://github.com/coreos/fedora-coreos-config/actions/workflows/add-override.yml)
+that runs the tool and submits a PR.
+
+Note that comments are not preserved in these files. The
+lockfile supports arbitrary keys under the `metadata` key to
+carry information. Some keys are semantically meaningful to
+humans or other tools.
+
+### Fast-tracking
+
+Example:
 
 ```yaml
 packages:
-  # document reason here and link to any Bodhi update
-  foobar:
-    evra: 1.2.3-1.fc31.x86_64
+  selinux-policy:
+    evra: 34.10-1.fc34.noarch
+    metadata:
+      type: fast-track
+      bodhi: https://bodhi.fedoraproject.org/updates/FEDORA-2021-f014ca8326
+      reason: https://github.com/coreos/fedora-coreos-tracker/issues/850
+  selinux-policy-targeted:
+    evra: 34.10-1.fc34.noarch
+    metadata:
+      type: fast-track
+      # you don't have to repeat the other keys for related packages
+```
+
+Whenever possible, it is important that the package be
+submitted as an update to Bodhi so that we don't have to
+carry the override for a long time.
+
+Fast-tracked packages will automatically be removed by the
+`remove-graduated-overrides` GitHub Action in this repo once
+they reach the stable Fedora repos (or newer versions). They
+are detected by the `type: fast-track` key.
+
+### Pinning
+
+Example:
+
+```
+packages:
+  dracut:
+      evr: 053-5.fc34
+      metadata:
+        type: pin
+        reason: https://github.com/coreos/fedora-coreos-tracker/issues/842
+  dracut-network:
+      evr: 053-5.fc34
+      metadata:
+        type: pin
+        reason: https://github.com/coreos/fedora-coreos-tracker/issues/842
 ```
 
-Whenever possible, in the case of pulling in a newer
-package, it is important that the package be submitted as an
-update to Bodhi so that we don't have to carry the override
-forever.
+All pinned packages *must* have a `reason` key containing
+more information about why the pin is necessary.
 
 Once an override PR is merged,
-[`coreos-koji-tagger`](https://github.com/coreos/fedora-coreos-releng-automation/tree/master/coreos-koji-tagger)
+[`coreos-koji-tagger`](https://github.com/coreos/fedora-coreos-releng-automation/tree/main/coreos-koji-tagger)
 will automatically tag overridden packages into the pool.
 
 ## Adding packages to the OS
@@ -85,67 +138,51 @@ the corresponding entries in the lockfiles:
 There will be better tooling to come to enable this, though
 one easy way to do this is for now:
 - add packages to the correct YAML manifest
-- run `cosa fetch --update-lockfile`
-- commit only the new package entries
+- run `cosa fetch --update-lockfile` (this will only update the lockfile for
+  the current architecture, most likely `x86_64`)
+- copy the new lines to the lockfiles for other architectures (i.e. `aarch64`)
+- commit only the new package entries (skip the timestamped changes to avoid
+  merge conflicts with the lockfile updates made by the bot)
 
 ## Moving to a new major version (N) of Fedora
 
-Updating this repo:
-
-1. bump `releasever` in `manifest.yaml`
-2. update the repos in `manifest.yaml` if needed
-3. run `cosa fetch --update-lockfile`
-4. PR the result
-
-Update server changes:
-
-1. Set a new update barrier for N-2 on all streams.
-   In the barrier entry set a link to [the docs](https://docs.fedoraproject.org/en-US/fedora-coreos/update-barrier-signing-keys/).
-   See [discussion](https://github.com/coreos/fedora-coreos-tracker/issues/480#issuecomment-631724629).
-
-CoreOS Installer changes:
-
-1. Update CoreOS Installer to know about the signing key used for the
-   future new major version of Fedora (N+1). Note that the signing
-   keys for N+1 won't get created until releng branches and rawhide
-   becomes N+1.
-
-Release engineering changes:
-
-1. Verify that a few tags have been created. These should have been created
-   by releng scripts on branching: 
-
-- `f${releasever}-coreos-signing-pending`
-- `f${releasever}-coreos-continuous`
-
-2. The tag info for the coreos-pool tag has the new release (N) and
-   next release (N+1) signing keys (just to stay ahead of the curve)
-   and removes the old release (N-2) signing key. The following commands
-   view the current settings and then update the list to 32/33/34 keys.
-   You'll most likely have to get someone from releng to run the second
-   command (`edit-tag`).
-
-- `koji taginfo coreos-pool`
-- `koji edit-tag coreos-pool -x tag2distrepo.keys="12c944d0 9570ff31 45719a39"`
-
-
-3. `koji untag` N-2 packages from the pool (at some point we'll have GC
-   in place to do this for us, but for now we must remember to do this
-   manually or otherwise distRepo will fail once the signed packages are
-   GC'ed). For example the following snippet finds all RPMs signed by the
-   Fedora 31 key and untags them.
-
-```
-f31key=3c3359c4
-key=$f31key
-untaglist=''
-for build in $(koji list-tagged --quiet coreos-pool | cut -f1 -d' '); do
-    if koji buildinfo $build | grep $key 1>/dev/null; then
-        untaglist+="${build} "
-        echo "Adding $build to untag list"
-    fi
-done
-
-# After verifying the list looks good:
-#   - koji untag-build coreos-pool $untaglist
+[Create a rebase checklist](https://github.com/coreos/fedora-coreos-tracker/issues/new?labels=kind/enhancement&template=rebase.md&title=Rebase+onto+Fedora+N) in fedora-coreos-tracker.
+
+## CoreOS CI
+
+Pull requests submitted to this repo are tested by
+[CoreOS CI](https://github.com/coreos/coreos-ci). You can see the pipeline
+executed in `.cci.jenkinsfile`. For more information, including interacting with
+CI, see the [CoreOS CI documentation](https://github.com/coreos/coreos-ci/blob/main/README-upstream-ci.md).
+
+## Tests layout
+Tests should follow the following format:
+
+```bash
+#!/bin/bash
+## kola:
+##   exclusive: false
+##   platforms: aws gcp
+##   # See all options in https://coreos.github.io/coreos-assembler/kola/external-tests/#kolajson
+#
+# Short summary of what the test does, why we need it, etc.
+#
+# Recommended: Link to corresponding issue or PR
+#
+# Explain the reasons behind all the kola options:
+# - distros: fcos
+#   - This test only runs on FCOS due to ...
+# - platforms: qemu
+#   - This test should ...
+# - etc.
+
+set -euxo pipefail
+
+. $KOLA_EXT_DATA/commonlib.sh
+
+foo_bar() <-- Other function definitions
+
+if ...    <-- Actual test code
+          <-- Errors must be raised with `fatal()`
+          <-- Does not need to end with a call to `ok()`
 ```
diff --git a/ci/buildroot/Dockerfile b/ci/buildroot/Dockerfile
new file mode 100644
index 0000000000..aee83d3eec
--- /dev/null
+++ b/ci/buildroot/Dockerfile
@@ -0,0 +1,12 @@
+# This includes the build dependencies for some key packages
+# such as ignition, rpm-ostree, libpod, systemd, and kernel.
+# If you want another package in this list, submit a PR and
+# we can probably add it.
+#
+# This image is used by CoreOS CI to build software like
+# Ignition, rpm-ostree, ostree, coreos-installer, etc...
+FROM quay.io/fedora/fedora:40
+# Work around for https://bugzilla.redhat.com/show_bug.cgi?id=2278652
+ENV container=oci
+COPY . /src
+RUN ./src/install-buildroot.sh && yum clean all && rm /src -rf
diff --git a/ci/buildroot/buildroot-buildreqs.txt b/ci/buildroot/buildroot-buildreqs.txt
new file mode 100644
index 0000000000..ce510912d6
--- /dev/null
+++ b/ci/buildroot/buildroot-buildreqs.txt
@@ -0,0 +1,9 @@
+# This is what the CoreOS developers tend to actively develop/own.
+# If you want to extend this, feel free to file a PR.
+ignition
+ostree
+librepo
+kernel
+systemd
+dracut
+podman
diff --git a/ci/buildroot/buildroot-reqs-aarch64.txt b/ci/buildroot/buildroot-reqs-aarch64.txt
new file mode 100644
index 0000000000..080e3163df
--- /dev/null
+++ b/ci/buildroot/buildroot-reqs-aarch64.txt
@@ -0,0 +1 @@
+lld
diff --git a/ci/buildroot/buildroot-reqs-ppc64le.txt b/ci/buildroot/buildroot-reqs-ppc64le.txt
new file mode 100644
index 0000000000..080e3163df
--- /dev/null
+++ b/ci/buildroot/buildroot-reqs-ppc64le.txt
@@ -0,0 +1 @@
+lld
diff --git a/ci/buildroot/buildroot-reqs-x86_64.txt b/ci/buildroot/buildroot-reqs-x86_64.txt
new file mode 100644
index 0000000000..080e3163df
--- /dev/null
+++ b/ci/buildroot/buildroot-reqs-x86_64.txt
@@ -0,0 +1 @@
+lld
diff --git a/ci/buildroot/buildroot-reqs.txt b/ci/buildroot/buildroot-reqs.txt
new file mode 100644
index 0000000000..afe92d4b73
--- /dev/null
+++ b/ci/buildroot/buildroot-reqs.txt
@@ -0,0 +1,59 @@
+# This is a list of basic buildrequires; it'd be a bit better to
+# yum -y install @buildsys-build but unfortunately that hits a bug:
+# https://fedoraproject.org/wiki/Common_F30_bugs#Conflicts_between_fedora-release_packages_when_installing_package_groups
+# So here we inline it, minus the -release package.
+bash
+bzip2
+coreutils
+cpio
+diffutils
+findutils
+gawk
+glibc-minimal-langpack
+grep
+gzip
+info
+make
+patch
+redhat-rpm-config
+rpm-build
+sed
+shadow-utils
+tar
+unzip
+util-linux
+which
+xz
+
+# For rust projects
+rustfmt
+clippy
+
+# For unit tests at least.
+ostree
+
+# A super common tool
+jq
+
+# For golang projects like mantle and gangplank
+golang
+
+# Used by ostree tests (TODO: add to something like TestBuildRequires in spec files)
+attr
+gjs
+
+# Also, add clang since it's useful at least in CI for C/C++ projects
+clang
+# And the tools such as clang-format, used for style checking
+clang-tools-extra
+# All C/C++ projects should have CI that uses the sanitizers
+libubsan libasan libtsan
+# And all C/C++ projects should use clang-analyzer
+clang-analyzer
+
+# We don't want zombies in our pods
+dumb-init
+
+# Used to check Bash scripts in CI
+ShellCheck
+file
diff --git a/ci/buildroot/buildroot-specs.txt b/ci/buildroot/buildroot-specs.txt
new file mode 100644
index 0000000000..0afa4a9673
--- /dev/null
+++ b/ci/buildroot/buildroot-specs.txt
@@ -0,0 +1,3 @@
+# for projects which have their canonical spec files upstream, use those instead
+# since they're more up to date
+https://raw.githubusercontent.com/coreos/rpm-ostree/main/packaging/rpm-ostree.spec
diff --git a/ci/buildroot/install-buildroot.sh b/ci/buildroot/install-buildroot.sh
new file mode 100755
index 0000000000..2809b8a8f7
--- /dev/null
+++ b/ci/buildroot/install-buildroot.sh
@@ -0,0 +1,47 @@
+#!/bin/bash
+set -euo pipefail
+
+# This is invoked by Dockerfile
+
+dnf -y install dnf-plugins-core
+# We want to avoid a 7 day cycle for e.g. new ostree etc.
+dnf config-manager --set-enabled updates-testing
+
+dn=$(dirname "$0")
+tmpd=$(mktemp -d) && trap 'rm -rf ${tmpd}' EXIT
+
+arch=$(arch)
+
+echo "Installing base build requirements"
+dnf -y install /usr/bin/xargs 'dnf-command(builddep)'
+deps=$(grep -v '^#' "${dn}"/buildroot-reqs.txt)
+if [ -f "${dn}/buildroot-reqs-${arch}.txt" ]; then
+  deps+=" "
+  deps+=$(grep -v '^#' "${dn}/buildroot-reqs-${arch}.txt")
+fi
+echo "${deps}" | xargs dnf -y install
+
+echo "Installing build dependencies of primary packages"
+brs=$(grep -v '^#' "${dn}"/buildroot-buildreqs.txt)
+(cd "${tmpd}" && mkdir rpmbuild
+ echo "${brs}" | xargs dnf download --source
+ # rebuild the SRPM for this arch; see
+ # https://bugzilla.redhat.com/show_bug.cgi?id=1402784#c6
+ find . -name '*.src.rpm' -print0 | xargs -0n 1 rpmbuild -rs --nodeps \
+    -D "%_topdir $PWD/rpmbuild" -D "%_tmppath %{_topdir}/tmp"
+ dnf builddep -y rpmbuild/SRPMS/*.src.rpm)
+rm -rf "${tmpd:?}"/*
+
+echo "Installing build dependencies from canonical spec files"
+specs=$(grep -v '^#' "${dn}"/buildroot-specs.txt)
+(cd "${tmpd}" && echo "${specs}" | xargs curl -L --remote-name-all)
+(cd "${tmpd}" && find . -type f -print0 | xargs -0 dnf -y builddep --spec)
+rm -rf "${tmpd:?}"/*
+
+echo "Installing test dependencies from canonical upstream files"
+testdep_urls=$(grep -v '^#' "${dn}"/testdeps.txt)
+(cd "${tmpd}" && echo "${testdep_urls}" | xargs curl -L --remote-name-all)
+grep -hrv '^#' "${tmpd}" | xargs dnf -y install
+rm -rf "${tmpd:?}"/*
+
+echo 'Done!'
diff --git a/ci/buildroot/testdeps.txt b/ci/buildroot/testdeps.txt
new file mode 100644
index 0000000000..59f7a716b1
--- /dev/null
+++ b/ci/buildroot/testdeps.txt
@@ -0,0 +1,2 @@
+# for projects which have their canonical list of test deps upstream
+https://raw.githubusercontent.com/coreos/rpm-ostree/main/ci/testdeps.txt
diff --git a/ci/continuous/Dockerfile b/ci/continuous/Dockerfile
new file mode 100644
index 0000000000..65a0f293e3
--- /dev/null
+++ b/ci/continuous/Dockerfile
@@ -0,0 +1,4 @@
+FROM quay.io/coreos-assembler/fcos:testing-devel
+ADD fcos-continuous.repo /etc/yum.repos.d
+ADD overrides.yaml /etc/rpm-ostree/origin.d/overrides.yaml
+RUN rpm-ostree ex rebuild
diff --git a/ci/continuous/fcos-continuous.repo b/ci/continuous/fcos-continuous.repo
new file mode 100644
index 0000000000..82a0683540
--- /dev/null
+++ b/ci/continuous/fcos-continuous.repo
@@ -0,0 +1,10 @@
+[copr:copr.fedorainfracloud.org:group_CoreOS:continuous]
+name=Copr repo for continuous owned by @CoreOS
+baseurl=https://download.copr.fedorainfracloud.org/results/@CoreOS/continuous/fedora-$releasever-$basearch/
+type=rpm-md
+skip_if_unavailable=True
+gpgcheck=1
+gpgkey=https://download.copr.fedorainfracloud.org/results/@CoreOS/continuous/pubkey.gpg
+repo_gpgcheck=0
+enabled=1
+enabled_metadata=1
diff --git a/ci/continuous/overrides.yaml b/ci/continuous/overrides.yaml
new file mode 100644
index 0000000000..4e55de029a
--- /dev/null
+++ b/ci/continuous/overrides.yaml
@@ -0,0 +1,11 @@
+ex-override-replace:
+  - from:
+      repo: copr:copr.fedorainfracloud.org:group_CoreOS:continuous
+    packages:
+      - ostree
+      - ostree-libs
+      - rpm-ostree
+      - rpm-ostree-libs
+      - ignition
+      - coreos-installer
+      - coreos-installer-bootinfra
diff --git a/ci/find-whitespace b/ci/find-whitespace
new file mode 100755
index 0000000000..d72b9b7e1c
--- /dev/null
+++ b/ci/find-whitespace
@@ -0,0 +1,65 @@
+#!/bin/bash
+
+set -euo pipefail
+
+main() {
+
+    local files_with_whitespace=""
+    local files_with_missing_empty_line_at_eof=""
+
+    while IFS= read -r -d '' f; do
+        echo "[+] Checking ${f}"
+
+        # Looking for whitespace at end of line
+        if grep -Eq " +$" "${f}"; then
+            # List of files to ignore
+            if \
+                [[ "${f}" == "./live/isolinux/boot.msg" ]] \
+                ; then
+                echo "[+] Checking ${f}: Ignoring whitespace at end of line"
+            else
+                echo "[+] Checking ${f}: Found whitespace at end of line"
+                files_with_whitespace+=" ${f}"
+            fi
+        fi
+
+        # Looking for missing empty line at end of file
+        if [[ -n $(tail -c 1 "${f}") ]]; then
+            # List of files to ignore
+            if \
+                [[ "${f}" == "./tests/kola/ignition/resource/authenticated-gs/data/expected/"* ]] ||\
+                [[ "${f}" == "./tests/kola/ignition/resource/authenticated-s3/data/expected/"* ]] ||\
+                [[ "${f}" == "./tests/kola/ignition/resource/remote/data/expected/"* ]] \
+                ; then
+                echo "[+] Checking ${f}: Ignoring missing empty line at end of file"
+            else
+                echo "[+] Checking ${f}: Missing empty line at end of file"
+                files_with_missing_empty_line_at_eof+=" ${f}"
+            fi
+        fi
+    done< <(find . -path "./.git" -prune -o -type f -print0)
+
+    echo ""
+    if [[ -n "${files_with_whitespace}" ]]; then
+        echo "[+] Found files with whitespace at the end of line"
+        echo "${files_with_whitespace}" | tr ' ' '\n'
+    else
+        echo "[+] No files with whitespace at the end of line"
+    fi
+
+    echo ""
+    if [[ -n "${files_with_missing_empty_line_at_eof}" ]]; then
+        echo "[+] Found files with missing empty line at end of file"
+        echo "${files_with_missing_empty_line_at_eof}" | tr ' ' '\n'
+    else
+        echo "[+] No files with missing empty line at end of file"
+    fi
+
+    if [[ -n "${files_with_whitespace}" ]] || [[ -n "${files_with_missing_empty_line_at_eof}" ]]; then
+        exit 1
+    fi
+
+    exit 0
+}
+
+main "${@}"
diff --git a/ci/overrides.py b/ci/overrides.py
new file mode 100755
index 0000000000..5b8612a093
--- /dev/null
+++ b/ci/overrides.py
@@ -0,0 +1,383 @@
+#!/usr/bin/python3
+
+import argparse
+import functools
+import os
+import sys
+import json
+import requests
+from urllib.parse import urlparse
+import yaml
+import subprocess
+
+import bodhi.client.bindings
+import dnf
+import hawkey
+import koji
+
+KOJI_URL = 'https://koji.fedoraproject.org/kojihub'
+ARCHES = ['s390x', 'x86_64', 'ppc64le', 'aarch64']
+TRIVIAL_FAST_TRACKS = [
+    # Packages that don't need a reason URL when fast-tracking
+    'console-login-helper-messages',
+    'ignition',
+    'ostree',
+    'rpm-ostree',
+    'rust-afterburn',
+    'rust-bootupd',
+    'rust-coreos-installer',
+    'rust-ignition-config',
+    'rust-zincati',
+]
+BUILDS_JSON_URL_TEMPLATE = 'https://builds.coreos.fedoraproject.org/prod/streams/{stream}/builds/builds.json'
+GENERATED_LOCKFILE_URL_TEMPLATE = 'https://builds.coreos.fedoraproject.org/prod/streams/{stream}/builds/{version}/{arch}/manifest-lock.generated.{arch}.json'
+
+OVERRIDES_HEADER = """
+# This lockfile should be used to pin to a package version (`type: pin`) or to
+# fast-track packages ahead of Bodhi (`type: fast-track`). Fast-tracked
+# packages will automatically be removed once they are in the stable repos.
+#
+# IMPORTANT: YAML comments *will not* be preserved. All `pin` overrides *must*
+# include a URL in the `metadata.reason` key. Overrides of type `fast-track`
+# *should* include a Bodhi update URL in the `metadata.bodhi` key and a URL
+# in the `metadata.reason` key, though it's acceptable to omit a `reason`
+# for FCOS-specific packages (ignition, afterburn, etc.).
+"""
+
+basedir = os.path.normpath(os.path.join(os.path.dirname(sys.argv[0]), '..'))
+
+
+def main():
+    parser = argparse.ArgumentParser(description='Manage overrides.')
+    # "dest" to work around https://bugs.python.org/issue29298
+    subcommands = parser.add_subparsers(title='subcommands', required=True,
+            dest='command')
+
+    fast_track = subcommands.add_parser('fast-track',
+            description='Fast-track Bodhi updates.')
+    fast_track.add_argument('update', nargs='+',
+            help='ID or URL of Bodhi update to fast-track')
+    fast_track.add_argument('-r', '--reason',
+            help='URL explaining the reason for the fast-track')
+    fast_track.add_argument('--ignore-dist-mismatch', action='store_true',
+            help='ignore mismatched Fedora major version')
+    fast_track.set_defaults(func=do_fast_track)
+
+    pin = subcommands.add_parser('pin', description='Pin source RPMs.')
+    pin.add_argument('nvr', nargs='+',
+            help='NVR of SRPM to pin')
+    pin.add_argument('-r', '--reason', required=True,
+            help='URL explaining the reason for the pin')
+    pin.add_argument('--ignore-dist-mismatch', action='store_true',
+            help='ignore mismatched Fedora major version')
+    pin.set_defaults(func=do_pin)
+
+    srpms = subcommands.add_parser('srpms',
+            description='Name the relevant source RPMs for a Bodhi update.')
+    srpms.add_argument('update', help='ID or URL of Bodhi update')
+    srpms.set_defaults(func=do_srpms)
+
+    graduate = subcommands.add_parser('graduate',
+            description='Remove graduated overrides.')
+    graduate.set_defaults(func=do_graduate)
+
+    args = parser.parse_args()
+    args.func(args)
+
+
+def do_fast_track(args):
+    overrides = {}
+    dist = get_expected_dist_tag()
+    if args.reason:
+        check_url(args.reason)
+    for update in args.update:
+        update = get_bodhi_update(update)
+        source_nvrs = get_source_nvrs(update)
+        for source_nvr in source_nvrs:
+            source_name = '-'.join(source_nvr.split('-')[:-2])
+            if not args.reason and source_name not in TRIVIAL_FAST_TRACKS:
+                raise Exception(f'No reason URL specified and source package {source_name} not in {TRIVIAL_FAST_TRACKS}')
+        for n, info in get_binary_packages(source_nvrs).items():
+            if not args.ignore_dist_mismatch:
+                check_dist_tag(n, info, dist)
+            info['metadata'] = dict(
+                type='fast-track',
+                bodhi=update['url'],
+            )
+            if args.reason:
+                info['metadata']['reason'] = args.reason
+            overrides[n] = info
+    if not overrides:
+        raise Exception('specified updates contain no binary packages listed in lockfiles')
+    for lockfile_path in get_lockfiles():
+        merge_overrides(lockfile_path, overrides)
+
+
+def do_pin(args):
+    overrides = {}
+    dist = get_expected_dist_tag()
+    check_url(args.reason)
+    for n, info in get_binary_packages(args.nvr).items():
+        if not args.ignore_dist_mismatch:
+            check_dist_tag(n, info, dist)
+        info['metadata'] = dict(
+            type='pin',
+            reason=args.reason,
+        )
+        overrides[n] = info
+    if not overrides:
+        raise Exception('specified source packages produce no binary packages listed in lockfiles')
+    for lockfile_path in get_lockfiles():
+        merge_overrides(lockfile_path, overrides)
+
+
+def do_srpms(args):
+    printed = False
+    for nvr in get_source_nvrs(get_bodhi_update(args.update)):
+        if get_binary_packages([nvr]):
+            print(nvr)
+            printed = True
+    if not printed:
+        raise Exception('specified update contains no binary packages listed in lockfiles')
+
+
+def do_graduate(_args):
+    treefile = get_treefile()
+    base = get_dnf_base(treefile)
+    setup_repos(base, treefile)
+
+    for fn in get_lockfiles():
+        graduate_lockfile(base, fn)
+
+
+def get_treefile():
+    treefile = subprocess.check_output(['rpm-ostree', 'compose', 'tree',
+                                        '--print-only',
+                                        os.path.join(basedir, 'manifest.yaml')])
+    return json.loads(treefile)
+
+
+def get_dnf_base(treefile):
+    base = dnf.Base()
+    base.conf.reposdir = basedir
+    base.conf.releasever = treefile['releasever']
+    base.read_all_repos()
+    return base
+
+
+@functools.cache
+def get_stream():
+    '''Get the current stream name.'''
+    with open(os.path.join(basedir, 'manifest.yaml')) as fh:
+        manifest = yaml.safe_load(fh)
+    return manifest['variables']['stream']
+
+
+@functools.cache
+def get_build_list():
+    '''Return list of official builds fetched from builds.json for the current
+    stream.'''
+    stream_url = BUILDS_JSON_URL_TEMPLATE.format(stream=get_stream())
+    resp = requests.get(stream_url)
+    resp.raise_for_status()
+    return resp.json()['builds']
+
+
+@functools.cache
+def get_manifest_packages(arch):
+    '''Return manifest lock package map for the specified arch.'''
+
+    # If this branch has any lockfiles in it, return the lockfile for the
+    # specified arch, or an empty dict if missing.
+    lockfile_path = lambda arch: os.path.join(basedir, f'manifest-lock.{arch}.json')
+    if any(os.path.exists(lockfile_path(a)) for a in ARCHES):
+        try:
+            with open(lockfile_path(arch)) as f:
+                manifest = json.load(f)
+            return manifest['packages']
+        except FileNotFoundError:
+            return {}
+
+    # Otherwise we're on a mechanical branch.  Pull the generated lockfile
+    # from the most recent successful CI build, or return an empty dict if
+    # we've never built for this arch.  Thus, different arches may return
+    # lockfiles from different builds if a recent build failed on some arches.
+    versions = [b['id'] for b in get_build_list() if arch in b['arches']]
+    if not versions:
+        return {}
+    eprint(f'Reading generated lockfile from build {versions[0]} on {arch}')
+    lockfile_url = GENERATED_LOCKFILE_URL_TEMPLATE.format(stream=get_stream(),
+            version=versions[0], arch=arch)
+    resp = requests.get(lockfile_url)
+    resp.raise_for_status()
+    return resp.json()['packages']
+
+
+def get_bodhi_update(id_or_url):
+    '''Query Bodhi for the specified update ID or URL and return an info
+    dict.'''
+    # discard rest of URL if any
+    id = id_or_url.split('/')[-1]
+    client = bodhi.client.bindings.BodhiClient()
+    result = client.query(updateid=id)
+    if not result.updates:
+        raise Exception(f'Update {id} not found')
+    return result.updates[0]
+
+
+def get_source_nvrs(update):
+    '''Return list of source NVRs from the update info dict.'''
+    return [b['nvr'] for b in update.builds]
+
+
+def get_binary_packages(source_nvrs):
+    '''Return name => info dict for the specified source NVRs.  The info
+    dict contains "evr" for archful packages and "evra" for noarch ones.
+    A binary package is included if it is in the manifest lockfiles.'''
+    binpkgs = {}
+    accepted_in_arch = {}
+    client = koji.ClientSession(KOJI_URL)
+
+    archful = lambda arch: arch != 'noarch'
+
+    def arches_with_package(name, arch):
+        '''For a given package and arch, return the arches that list the
+        package in their lockfiles.  There may be more than one, since we
+        check noarch packages against every candidate architecture.'''
+        candidates = [arch] if archful(arch) else ARCHES
+        return [a for a in candidates if name in get_manifest_packages(a)]
+
+    for source_nvr in source_nvrs:
+        for binpkg in client.listBuildRPMs(source_nvr):
+            name = binpkg['name']
+            evr = f'{binpkg["version"]}-{binpkg["release"]}'
+            if binpkg['epoch'] is not None:
+                evr = f'{binpkg["epoch"]}:{evr}'
+            for arch in arches_with_package(name, binpkg['arch']):
+                if archful(binpkg['arch']):
+                    binpkgs[name] = {'evr': evr}
+                else:
+                    binpkgs[name] = {'evra': evr + '.noarch'}
+                accepted_in_arch.setdefault(arch, set()).add(name)
+
+    # Check that every arch has the same package set
+    if list(accepted_in_arch.values())[:-1] != list(accepted_in_arch.values())[1:]:
+        raise Exception(f'This tool cannot handle arch-specific overrides: {accepted_in_arch}')
+
+    return binpkgs
+
+
+def setup_repos(base, treefile):
+    for repo in base.repos.values():
+        repo.disable()
+
+    eprint("Enabled repos:")
+    for repo in treefile['repos']:
+        base.repos[repo].enable()
+        eprint(f"- {repo}")
+
+
+def get_lockfiles():
+    lockfiles = ['manifest-lock.overrides.yaml']
+    # TODO: for now, we only support the archless variant; supporting
+    # arch-specific lockfiles will require making dnf fetch metadata not just
+    # for the basearch on which we're running
+    # lockfiles += [f'manifest-lock.overrides.{arch}.yaml' for arch in ARCHES]
+    return [os.path.join(basedir, f) for f in lockfiles]
+
+
+def graduate_lockfile(base, fn):
+    if not os.path.exists(fn):
+        return
+
+    with open(fn) as f:
+        lockfile = yaml.safe_load(f)
+    if len(lockfile.get('packages', {})) == 0:
+        return
+
+    if base.sack is None:
+        eprint("Downloading metadata")
+        base.fill_sack(load_system_repo=False)
+
+    new_packages = {}
+    for name, lock in lockfile['packages'].items():
+        if ('metadata' not in lock or
+                lock['metadata'].get('type') != "fast-track"):
+            new_packages[name] = lock
+            continue
+
+        if 'evra' in lock:
+            nevra = f"{name}-{lock['evra']}"
+        else:
+            # it applies to all arches, so we can just check our arch (see
+            # related TODO above)
+            nevra = f"{name}-{lock['evr']}.{base.conf.basearch}"
+        graduated = sack_has_nevra_greater_or_equal(base, nevra)
+        if not graduated:
+            new_packages[name] = lock
+        else:
+            eprint(f"{fn}: {nevra} has graduated")
+
+    if lockfile['packages'] != new_packages:
+        lockfile['packages'] = new_packages
+        write_lockfile(fn, lockfile)
+    else:
+        eprint(f"{fn}: no packages graduated")
+
+
+def sack_has_nevra_greater_or_equal(base, nevra):
+    nevra = hawkey.split_nevra(nevra)
+    pkgs = base.sack.query().filterm(name=nevra.name,
+                                     arch=nevra.arch).latest().run()
+
+    if len(pkgs) == 0:
+        # Odd... the only way I can imagine this happen is if we fast-track a
+        # brand new package from Koji which hasn't hit the updates repo yet.
+        # Corner-case, but let's be nice.
+        eprint(f"couldn't find package {nevra.name}; assuming not graduated")
+        return False
+
+    nevra_latest = hawkey.split_nevra(str(pkgs[0]))
+    return nevra_latest >= nevra
+
+
+def merge_overrides(fn, overrides):
+    '''Modify the file fn by applying the specified package overrides.'''
+    with open(fn) as f:
+        lockfile = yaml.safe_load(f)
+    lockfile.setdefault('packages', {}).update(overrides)
+    write_lockfile(fn, lockfile)
+
+
+def write_lockfile(fn, contents):
+    with open(fn, 'w') as f:
+        f.write(OVERRIDES_HEADER.strip())
+        f.write('\n\n')
+        yaml.dump(contents, f)
+
+
+def check_url(u):
+    p = urlparse(u)
+    if p.scheme not in ('http', 'https'):
+        raise Exception(f'Invalid URL: {u}')
+
+
+def get_expected_dist_tag():
+    with open(os.path.join(basedir, 'manifest.yaml')) as f:
+        releasever = yaml.safe_load(f)['releasever']
+    return f'.fc{releasever}'
+
+
+def check_dist_tag(name, info, dist):
+    if 'evr' in info and not info['evr'].endswith(dist):
+        raise Exception(f"Package {name}-{info['evr']} doesn't match expected dist tag {dist}")
+    if 'evra' in info and not info['evra'].endswith(dist + '.noarch'):
+        raise Exception(f"Package {name}-{info['evra']} doesn't match expected dist tag {dist}")
+
+
+def eprint(*args, **kwargs):
+    print(*args, file=sys.stderr, **kwargs)
+
+
+if __name__ == "__main__":
+    sys.exit(main())
diff --git a/ci/shellcheck b/ci/shellcheck
new file mode 100755
index 0000000000..7543bd3804
--- /dev/null
+++ b/ci/shellcheck
@@ -0,0 +1,35 @@
+#!/bin/bash
+# Template generated by https://github.com/coreos/repo-templates; do not edit downstream
+
+set -euo pipefail
+
+main() {
+    local found_errors="false"
+    # Let's start with error, then we can do warning, info, style
+    local -r severity="error"
+
+    while IFS= read -r -d '' f; do
+        # Skip non-text files that are very unlikely to be shell scripts
+        if [[ "$(file -b --mime-type "${f}" | sed 's|/.*||')" != "text" ]]; then
+            continue
+        fi
+        shebang="$(head -1 "${f}")"
+        if [[ "${f}" == *.sh ]] || \
+            [[ ${shebang} =~ ^#!/.*/bash.* ]] || \
+            [[ ${shebang} =~ ^#!/.*/env\ bash ]]; then
+            echo "[+] Checking ${f}"
+            shellcheck --external-sources --shell bash --severity="${severity}" "${f}" || found_errors="true"
+            bash -n "${f}" || found_errors="true"
+        fi
+    done< <(find . -path "./.git" -prune -o -path "./vendor" -prune -o -type f -print0)
+
+    if [[ "${found_errors}" != "false" ]]; then
+        echo "[+] Found errors with ShellCheck"
+        exit 1
+    fi
+
+    echo "[+] No error found with ShellCheck"
+    exit 0
+}
+
+main "${@}"
diff --git a/ci/validate b/ci/validate
index d4ad1834bd..42f58471c1 100755
--- a/ci/validate
+++ b/ci/validate
@@ -2,17 +2,46 @@
 # Validate basic syntax of shell script and yaml.
 
 import os
+import re
 import stat
 import subprocess
 import yaml
 
+INITRD_SERVICES_WITHOUT_BEFORE = {
+    # Depended on by other services
+    'coreos-livepxe-rootfs.service',
+}
+
 validated=0
 
+def openat(dirfd, name, mode='r'):
+    def opener(path, flags):
+        return os.open(path, flags, dir_fd=dirfd)
+    return open(name, mode, opener=opener)
+
+
+def validate_initrd_service(rootfd, name):
+    with openat(rootfd, name) as fh:
+        if ([l for l in fh.readlines() if l.startswith('Before=')] or
+                name in INITRD_SERVICES_WITHOUT_BEFORE):
+            global validated
+            validated += 1
+        else:
+            raise Exception(
+                f'{name} has no Before= and may race with switch-root'
+            )
+
+
+BASH_UNBRACKETED_IF = re.compile(r'\sif\s+"?\$')
 def validate_shell(rootfd, name):
     subprocess.check_call(['bash', '-n', name], preexec_fn=lambda: os.fchdir(rootfd))
+    with openat(rootfd, name) as fh:
+        if BASH_UNBRACKETED_IF.search(fh.read()):
+            raise Exception(f'Possible unbracketed conditional in {name}')
     global validated
     validated +=1
 
+
 for root, dirs, files, rootfd in os.fwalk('.'):
     # Skip .git
     if '.git' in dirs:
@@ -26,6 +55,8 @@ for root, dirs, files, rootfd in os.fwalk('.'):
         elif name.endswith('.sh'):
             validate_shell(rootfd, name)
             continue
+        elif 'dracut/modules.d' in root and name.endswith('.service'):
+            validate_initrd_service(rootfd, name)
         stbuf = os.lstat(name, dir_fd=rootfd)
         if not stat.S_ISREG(stbuf.st_mode):
             continue
diff --git a/fedora-archive.repo b/fedora-archive.repo
new file mode 100644
index 0000000000..9a80c38321
--- /dev/null
+++ b/fedora-archive.repo
@@ -0,0 +1,39 @@
+# Certain kola tests on older RHCOS branches rely on EOL Fedora containers
+# to set up and run the test environment. The ITUP cluster, being used by
+# the RHCOS pipeline, requires all outbound connections to be explicitly
+# specified in a Firewall Egress file. By using these archive repos on
+# the older RHCOS branches, we can ensure that those kola tests will
+# always download archived content from `https://dl.fedoraproject.org`.
+# We include both EOL and non-EOL repository locations in the baseurl list
+# to simplify maintenance, avoiding the need to remove the fedora.repo file
+# as versions reach EOL.
+
+[fedora-archive]
+name=Fedora $releasever - $basearch
+baseurl=https://dl.fedoraproject.org/pub/fedora/linux/releases/$releasever/Everything/$basearch/os/
+        https://dl.fedoraproject.org/pub/fedora-secondary/releases/$releasever/Everything/$basearch/os/
+        https://dl.fedoraproject.org/pub/archive/fedora/linux/releases/$releasever/Everything/$basearch/os/
+        https://dl.fedoraproject.org/pub/archive/fedora-secondary/releases/$releasever/Everything/$basearch/os/
+#metalink=https://mirrors.fedoraproject.org/metalink?repo=fedora-$releasever&arch=$basearch
+enabled=1
+#metadata_expire=7d
+repo_gpgcheck=0
+type=rpm
+gpgcheck=1
+gpgkey=file:///usr/share/distribution-gpg-keys/fedora/RPM-GPG-KEY-fedora-$releasever-primary
+skip_if_unavailable=False
+
+[fedora-archive-updates]
+name=Fedora $releasever - $basearch
+baseurl=https://dl.fedoraproject.org/pub/fedora/linux/updates/$releasever/Everything/$basearch/
+        https://dl.fedoraproject.org/pub/fedora-secondary/updates/$releasever/Everything/$basearch/
+        https://dl.fedoraproject.org/pub/archive/fedora/linux/updates/$releasever/Everything/$basearch/
+        https://dl.fedoraproject.org/pub/archive/fedora-secondary/updates/$releasever/Everything/$basearch/
+#metalink=https://mirrors.fedoraproject.org/metalink?repo=fedora-$releasever&arch=$basearch
+enabled=1
+#metadata_expire=7d
+repo_gpgcheck=0
+type=rpm
+gpgcheck=1
+gpgkey=file:///usr/share/distribution-gpg-keys/fedora/RPM-GPG-KEY-fedora-$releasever-primary
+skip_if_unavailable=False
diff --git a/fedora-candidate-compose.repo b/fedora-candidate-compose.repo
new file mode 100644
index 0000000000..b408da256f
--- /dev/null
+++ b/fedora-candidate-compose.repo
@@ -0,0 +1,21 @@
+# This yum repo entry represents the latest candidate compose for the given
+# Fedora release. During prep for final release some release blockers and
+# freeze exceptions are actually built in candidate composes first to
+# qualify them before they are promoted to the other repos. In order to pick
+# these packages up ASAP we'll pull from the latest candidate compose
+# as well. Note that if a package doesn't pass testing it will get
+# demoted from a later canddiate compose and never promoted to stable
+# repos. In this case a later bump-lockfile run will revert ot the
+# older NVR package that is currently in the stable repos. This should address:
+# https://github.com/coreos/fedora-coreos-tracker/issues/1602
+
+[fedora-candidate-compose]
+name=Fedora Candidate Compose $releasever - $basearch
+baseurl=https://kojipkgs.fedoraproject.org/compose/$releasever/latest-Fedora-$releasever/compose/Everything/$basearch/os/
+enabled=1
+#metadata_expire=7d
+repo_gpgcheck=0
+type=rpm
+gpgcheck=1
+gpgkey=file:///usr/share/distribution-gpg-keys/fedora/RPM-GPG-KEY-fedora-$releasever-primary
+skip_if_unavailable=False
diff --git a/fedora-next.repo b/fedora-next.repo
index 49f684cbf1..b7cc9bc78d 100644
--- a/fedora-next.repo
+++ b/fedora-next.repo
@@ -3,7 +3,6 @@
 
 [fedora-next]
 name=Fedora $releasever - $basearch
-failovermethod=priority
 baseurl=https://dl.fedoraproject.org/pub/fedora/linux/development/$releasever/Everything/$basearch/os/
         https://dl.fedoraproject.org/pub/fedora-secondary/development/$releasever/Everything/$basearch/os/
 #metalink=https://mirrors.fedoraproject.org/metalink?repo=fedora-$releasever&arch=$basearch
@@ -17,7 +16,6 @@ skip_if_unavailable=False
 
 [fedora-next-updates]
 name=Fedora $releasever - $basearch - Updates
-failovermethod=priority
 baseurl=https://dl.fedoraproject.org/pub/fedora/linux/updates/$releasever/Everything/$basearch/
         https://dl.fedoraproject.org/pub/fedora-secondary/updates/$releasever/Everything/$basearch/
 #metalink=https://mirrors.fedoraproject.org/metalink?repo=updates-released-f$releasever&arch=$basearch
@@ -31,7 +29,6 @@ skip_if_unavailable=False
 
 [fedora-next-updates-testing]
 name=Fedora $releasever - $basearch - Test Updates
-failovermethod=priority
 baseurl=https://dl.fedoraproject.org/pub/fedora/linux/updates/testing/$releasever/Everything/$basearch/
         https://dl.fedoraproject.org/pub/fedora-secondary/updates/testing/$releasever/Everything/$basearch/
 #metalink=https://mirrors.fedoraproject.org/metalink?repo=updates-testing-f$releasever&arch=$basearch
@@ -40,42 +37,3 @@ gpgcheck=1
 metadata_expire=6h
 gpgkey=file:///usr/share/distribution-gpg-keys/fedora/RPM-GPG-KEY-fedora-$releasever-primary
 skip_if_unavailable=False
-
-[fedora-next-modular]
-name=Fedora Modular $releasever - $basearch
-baseurl=https://dl.fedoraproject.org/pub/fedora/linux/development/$releasever/Modular/$basearch/os/
-        https://dl.fedoraproject.org/pub/fedora-secondary/development/$releasever/Modular/$basearch/os/
-#metalink=https://mirrors.fedoraproject.org/metalink?repo=fedora-modular-$releasever&arch=$basearch
-enabled=1
-#metadata_expire=7d
-repo_gpgcheck=0
-type=rpm
-gpgcheck=1
-gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-$releasever-$basearch
-skip_if_unavailable=False
-
-[fedora-next-updates-modular]
-name=Fedora Modular $releasever - $basearch - Updates
-failovermethod=priority
-baseurl=https://dl.fedoraproject.org/pub/fedora/linux/updates/$releasever/Modular/$basearch/
-        https://dl.fedoraproject.org/pub/fedora-secondary/updates/$releasever/Modular/$basearch/
-#metalink=https://mirrors.fedoraproject.org/metalink?repo=updates-released-modular-f$releasever&arch=$basearch
-enabled=1
-repo_gpgcheck=0
-type=rpm
-gpgcheck=1
-metadata_expire=6h
-gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-$releasever-$basearch
-skip_if_unavailable=False
-
-[fedora-next-updates-testing-modular]
-name=Fedora Modular $releasever - $basearch - Test Updates
-failovermethod=priority
-baseurl=https://dl.fedoraproject.org/pub/fedora/linux/updates/testing/$releasever/Modular/$basearch/
-        https://dl.fedoraproject.org/pub/fedora-secondary/updates/testing/$releasever/Modular/$basearch/
-#metalink=https://mirrors.fedoraproject.org/metalink?repo=updates-testing-f$releasever&arch=$basearch
-enabled=1
-gpgcheck=1
-metadata_expire=6h
-gpgkey=file:///usr/share/distribution-gpg-keys/fedora/RPM-GPG-KEY-fedora-$releasever-primary
-skip_if_unavailable=False
diff --git a/fedora.repo b/fedora.repo
index 4a94919a7f..4b1fdd7f99 100644
--- a/fedora.repo
+++ b/fedora.repo
@@ -3,7 +3,6 @@
 
 [fedora]
 name=Fedora $releasever - $basearch
-failovermethod=priority
 baseurl=https://dl.fedoraproject.org/pub/fedora/linux/releases/$releasever/Everything/$basearch/os/
         https://dl.fedoraproject.org/pub/fedora-secondary/releases/$releasever/Everything/$basearch/os/
 #metalink=https://mirrors.fedoraproject.org/metalink?repo=fedora-$releasever&arch=$basearch
@@ -17,7 +16,6 @@ skip_if_unavailable=False
 
 [fedora-updates]
 name=Fedora $releasever - $basearch - Updates
-failovermethod=priority
 baseurl=https://dl.fedoraproject.org/pub/fedora/linux/updates/$releasever/Everything/$basearch/
         https://dl.fedoraproject.org/pub/fedora-secondary/updates/$releasever/Everything/$basearch/
 #metalink=https://mirrors.fedoraproject.org/metalink?repo=updates-released-f$releasever&arch=$basearch
@@ -31,7 +29,6 @@ skip_if_unavailable=False
 
 [fedora-updates-testing]
 name=Fedora $releasever - $basearch - Test Updates
-failovermethod=priority
 baseurl=https://dl.fedoraproject.org/pub/fedora/linux/updates/testing/$releasever/Everything/$basearch/
         https://dl.fedoraproject.org/pub/fedora-secondary/updates/testing/$releasever/Everything/$basearch/
 #metalink=https://mirrors.fedoraproject.org/metalink?repo=updates-testing-f$releasever&arch=$basearch
@@ -40,42 +37,3 @@ gpgcheck=1
 metadata_expire=6h
 gpgkey=file:///usr/share/distribution-gpg-keys/fedora/RPM-GPG-KEY-fedora-$releasever-primary
 skip_if_unavailable=False
-
-[fedora-modular]
-name=Fedora Modular $releasever - $basearch
-baseurl=https://dl.fedoraproject.org/pub/fedora/linux/releases/$releasever/Modular/$basearch/os/
-        https://dl.fedoraproject.org/pub/fedora-secondary/releases/$releasever/Modular/$basearch/os/
-#metalink=https://mirrors.fedoraproject.org/metalink?repo=fedora-modular-$releasever&arch=$basearch
-enabled=1
-#metadata_expire=7d
-repo_gpgcheck=0
-type=rpm
-gpgcheck=1
-gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-$releasever-$basearch
-skip_if_unavailable=False
-
-[fedora-updates-modular]
-name=Fedora Modular $releasever - $basearch - Updates
-failovermethod=priority
-baseurl=https://dl.fedoraproject.org/pub/fedora/linux/updates/$releasever/Modular/$basearch/
-        https://dl.fedoraproject.org/pub/fedora-secondary/updates/$releasever/Modular/$basearch/
-#metalink=https://mirrors.fedoraproject.org/metalink?repo=updates-released-modular-f$releasever&arch=$basearch
-enabled=1
-repo_gpgcheck=0
-type=rpm
-gpgcheck=1
-metadata_expire=6h
-gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-$releasever-$basearch
-skip_if_unavailable=False
-
-[fedora-updates-testing-modular]
-name=Fedora Modular $releasever - $basearch - Test Updates
-failovermethod=priority
-baseurl=https://dl.fedoraproject.org/pub/fedora/linux/updates/testing/$releasever/Modular/$basearch/
-        https://dl.fedoraproject.org/pub/fedora-secondary/updates/testing/$releasever/Modular/$basearch/
-#metalink=https://mirrors.fedoraproject.org/metalink?repo=updates-testing-f$releasever&arch=$basearch
-enabled=1
-gpgcheck=1
-metadata_expire=6h
-gpgkey=file:///usr/share/distribution-gpg-keys/fedora/RPM-GPG-KEY-fedora-$releasever-primary
-skip_if_unavailable=False
diff --git a/image-base.yaml b/image-base.yaml
new file mode 100644
index 0000000000..cc038b04b8
--- /dev/null
+++ b/image-base.yaml
@@ -0,0 +1,26 @@
+# This file is shared by all streams. For a stream-specific change, use
+# image.yaml instead.
+
+# Target disk size in GB.
+# Make it at least 10G because we want the rootfs to be at least 8G:
+# https://github.com/coreos/fedora-coreos-tracker/issues/586
+size: 10
+
+extra-kargs:
+    # Disable SMT on systems vulnerable to MDS or any similar future issue.
+    - mitigations=auto,nosmt
+
+# Disable networking by default on firstboot. We can drop this once cosa stops
+# defaulting to `ip=auto rd.neednet=1` when it doesn't see this key.
+ignition-network-kcmdline: []
+
+# Optional remote by which to prefix the deployed OSTree ref
+ostree-remote: fedora
+
+vmware-os-type: fedora64Guest
+# VMware hardware versions: https://kb.vmware.com/s/article/1003746
+# We use the newest version allowed by the oldest non-EOL VMware
+# Workstation/Player/Fusion/ESXi release: https://lifecycle.vmware.com/
+vmware-hw-version: 17
+
+compressor: xz
diff --git a/image.yaml b/image.yaml
index a91ab32025..863d4e1207 100644
--- a/image.yaml
+++ b/image.yaml
@@ -1,27 +1,11 @@
-# This replaces image.ks
-# size is the target disk size in GB.
-size: 8
-
-extra-kargs:
-    # Disable SMT on systems vulnerable to MDS or any similar future issue.
-    - mitigations=auto,nosmt
-    # https://github.com/coreos/fedora-coreos-tracker/issues/292
-    # https://fedoraproject.org/wiki/Changes/CGroupsV2
-    - systemd.unified_cgroup_hierarchy=0
-
-# Disable networking by default on firstboot. We can drop this once cosa stops
-# defaulting to `ip=dhcp,dhcp6 rd.neednet=1` when it doesn't see this key.
-ignition-network-kcmdline: []
-
-# Optional remote by which to prefix the deployed OSTree ref
-ostree-remote: fedora
-
-# We want read-only /sysroot to protect from unintentional damage.
-# https://github.com/ostreedev/ostree/issues/1265
-sysroot-readonly: true
-
-# After this, we plan to add support for the Ignition
-# storage/filesystems sections.  (Although one can do
-# that on boot as well)
+# This file can optionally contain configuration specific to the stream,
+# similarly to manifest.yaml. Unlike image-base.yaml, which is shared by all
+# streams.
+include: image-base.yaml
 
 
+# We don't have zincati for rawhide, so we can bypass the issues in
+# https://github.com/coreos/fedora-coreos-tracker/issues/1263
+# and enable it by default for rawhide!
+deploy-via-container: true
+container-imgref: "ostree-remote-registry:fedora:quay.io/fedora/fedora-coreos:rawhide"
diff --git a/kola-denylist.yaml b/kola-denylist.yaml
index cf3a3d8c88..cce7cb0b9c 100644
--- a/kola-denylist.yaml
+++ b/kola-denylist.yaml
@@ -5,3 +5,35 @@
   tracker: https://github.com/coreos/coreos-assembler/pull/1478
 - pattern: podman.workflow
   tracker: https://github.com/coreos/coreos-assembler/pull/1478
+- pattern: coreos.ignition.ssh.key
+  tracker: https://github.com/coreos/fedora-coreos-tracker/issues/1553
+  snooze: 2024-10-01
+  warn: true
+  platforms:
+    - azure
+- pattern: coreos.boot-mirror*
+  tracker: https://github.com/coreos/fedora-coreos-tracker/issues/1659
+  warn: true
+  arches:
+    - ppc64le
+- pattern: ext.config.kdump.crash
+  tracker: https://github.com/coreos/fedora-coreos-tracker/issues/1791
+  snooze: 2024-09-26
+  warn: true
+  streams:
+    - rawhide
+    - branched
+    - next-devel
+    - next
+- pattern: ext.config.extensions.package
+  tracker: https://github.com/coreos/fedora-coreos-tracker/issues/1795
+  snooze: 2024-09-26
+  streams:
+    - next-devel
+    - next
+- pattern: ext.config.rpm-ostree.kernel-replace
+  tracker: https://github.com/coreos/fedora-coreos-tracker/issues/1795
+  snooze: 2024-09-26
+  streams:
+    - next-devel
+    - next
diff --git a/live/EFI/fedora/grub.cfg b/live/EFI/fedora/grub.cfg
index c833dc2171..6f881fef62 100644
--- a/live/EFI/fedora/grub.cfg
+++ b/live/EFI/fedora/grub.cfg
@@ -4,9 +4,11 @@
 #
 # One diff to note is we use linux and initrd instead of linuxefi and
 # initrdefi. We do this because it works and allows us to use this same
-# file on other architecutres. https://github.com/coreos/fedora-coreos-config/issues/63
+# file on other architectures. https://github.com/coreos/fedora-coreos-config/issues/63
 #
-# This file gets embedded into the efiboot.img on our Fedora CoreOS ISO.
+# This file is loaded directly when booting via El Torito, and indirectly
+# from a stub config in efiboot.img when booting via the hybrid ESP.
+
 set default="1"
 
 function load_video {
diff --git a/live/README-devel.md b/live/README-devel.md
index 6a123986fa..373b17ff19 100644
--- a/live/README-devel.md
+++ b/live/README-devel.md
@@ -1,7 +1,7 @@
 These files will be copied to the target live ISO
 via the CoreOS Assembler buildextend-live call. It
 picks up all files in the coreos/fedora-coreos-config/live/
-directory and copies them to the base of the ISO. 
+directory and copies them to the base of the ISO.
 
 Files currently copied are:
 
@@ -10,4 +10,4 @@ Files currently copied are:
 
 Files that get copied into efiboot.img in the ISO:
 
-- EFI/grub.cfg 
+- EFI/grub.cfg
diff --git a/live/isolinux/isolinux.cfg b/live/isolinux/isolinux.cfg
index bd3d30c345..d4f159fa51 100644
--- a/live/isolinux/isolinux.cfg
+++ b/live/isolinux/isolinux.cfg
@@ -1,4 +1,4 @@
-# Note this file mostly matches the isolinux.cfg file from the Fedora 
+# Note this file mostly matches the isolinux.cfg file from the Fedora
 # Server DVD iso. Diff this file with that file in the future to pick up
 # changes.
 serial 0
diff --git a/manifest-lock.overrides.aarch64.yaml b/manifest-lock.overrides.aarch64.yaml
deleted file mode 100644
index 8e6ad8a364..0000000000
--- a/manifest-lock.overrides.aarch64.yaml
+++ /dev/null
@@ -1,23 +0,0 @@
-packages:
-  # Fast-track systemd update that reverts fallback hostname change
-  # https://github.com/coreos/fedora-coreos-tracker/issues/649#issuecomment-739956843
-  # This is a one-off build, please don't remove it without talking
-  # to dustymabe or jlebon.
-  systemd:
-    evra: 246.7-1.fc33.aarch64
-  systemd-container:
-    evra: 246.7-1.fc33.aarch64
-  systemd-libs:
-    evra: 246.7-1.fc33.aarch64
-  systemd-pam:
-    evra: 246.7-1.fc33.aarch64
-  systemd-rpm-macros:
-    evra: 246.7-1.fc33.noarch
-  systemd-udev:
-    evra: 246.7-1.fc33.aarch64
-  # Fast-track Afterburn release
-  # https://bodhi.fedoraproject.org/updates/FEDORA-2020-94fd991213
-  afterburn:
-    evra: 4.6.0-2.fc33.aarch64
-  afterburn-dracut:
-    evra: 4.6.0-2.fc33.aarch64
diff --git a/manifest-lock.overrides.ppc64le.yaml b/manifest-lock.overrides.ppc64le.yaml
index e9c123f612..f595d1a248 100644
--- a/manifest-lock.overrides.ppc64le.yaml
+++ b/manifest-lock.overrides.ppc64le.yaml
@@ -1,23 +1,16 @@
+# This lockfile should be used to pin to a package version (`type: pin`) or to
+# fast-track packages ahead of Bodhi (`type: fast-track`). Fast-tracked
+# packages will automatically be removed once they are in the stable repos.
+#
+# IMPORTANT: YAML comments *will not* be preserved. All `pin` overrides *must*
+# include a URL in the `metadata.reason` key. Overrides of type `fast-track`
+# *should* include a Bodhi update URL in the `metadata.bodhi` key and a URL
+# in the `metadata.reason` key, though it's acceptable to omit a `reason`
+# for FCOS-specific packages (ignition, afterburn, etc.).
+
 packages:
-  # Fast-track systemd update that reverts fallback hostname change
-  # https://github.com/coreos/fedora-coreos-tracker/issues/649#issuecomment-739956843
-  # This is a one-off build, please don't remove it without talking
-  # to dustymabe or jlebon.
-  systemd:
-    evra: 246.7-1.fc33.ppc64le
-  systemd-container:
-    evra: 246.7-1.fc33.ppc64le
-  systemd-libs:
-    evra: 246.7-1.fc33.ppc64le
-  systemd-pam:
-    evra: 246.7-1.fc33.ppc64le
-  systemd-rpm-macros:
-    evra: 246.7-1.fc33.noarch
-  systemd-udev:
-    evra: 246.7-1.fc33.ppc64le
-  # Fast-track Afterburn release
-  # https://bodhi.fedoraproject.org/updates/FEDORA-2020-94fd991213
-  afterburn:
-    evra: 4.6.0-2.fc33.ppc64le
-  afterburn-dracut:
-    evra: 4.6.0-2.fc33.ppc64le
+  kdump-utils:
+    evr: 1.0.48-1.fc42
+    metadata:
+      reason: https://github.com/coreos/fedora-coreos-tracker/issues/1830
+      type: pin
diff --git a/manifest-lock.overrides.s390x.yaml b/manifest-lock.overrides.s390x.yaml
deleted file mode 100644
index 2ff0adbc3e..0000000000
--- a/manifest-lock.overrides.s390x.yaml
+++ /dev/null
@@ -1,23 +0,0 @@
-packages:
-  # Fast-track systemd update that reverts fallback hostname change
-  # https://github.com/coreos/fedora-coreos-tracker/issues/649#issuecomment-739956843
-  # This is a one-off build, please don't remove it without talking
-  # to dustymabe or jlebon.
-  systemd:
-    evra: 246.7-1.fc33.s390x
-  systemd-container:
-    evra: 246.7-1.fc33.s390x
-  systemd-libs:
-    evra: 246.7-1.fc33.s390x
-  systemd-pam:
-    evra: 246.7-1.fc33.s390x
-  systemd-rpm-macros:
-    evra: 246.7-1.fc33.noarch
-  systemd-udev:
-    evra: 246.7-1.fc33.s390x
-  # Fast-track Afterburn release
-  # https://bodhi.fedoraproject.org/updates/FEDORA-2020-94fd991213
-  afterburn:
-    evra: 4.6.0-2.fc33.s390x
-  afterburn-dracut:
-    evra: 4.6.0-2.fc33.s390x
diff --git a/manifest-lock.overrides.x86_64.yaml b/manifest-lock.overrides.x86_64.yaml
deleted file mode 100644
index 0a82913b1b..0000000000
--- a/manifest-lock.overrides.x86_64.yaml
+++ /dev/null
@@ -1,23 +0,0 @@
-packages:
-  # Fast-track systemd update that reverts fallback hostname change
-  # https://github.com/coreos/fedora-coreos-tracker/issues/649#issuecomment-739956843
-  # This is a one-off build, please don't remove it without talking
-  # to dustymabe or jlebon.
-  systemd:
-    evra: 246.7-1.fc33.x86_64
-  systemd-container:
-    evra: 246.7-1.fc33.x86_64
-  systemd-libs:
-    evra: 246.7-1.fc33.x86_64
-  systemd-pam:
-    evra: 246.7-1.fc33.x86_64
-  systemd-rpm-macros:
-    evra: 246.7-1.fc33.noarch
-  systemd-udev:
-    evra: 246.7-1.fc33.x86_64
-  # Fast-track Afterburn release
-  # https://bodhi.fedoraproject.org/updates/FEDORA-2020-94fd991213
-  afterburn:
-    evra: 4.6.0-2.fc33.x86_64
-  afterburn-dracut:
-    evra: 4.6.0-2.fc33.x86_64
diff --git a/manifest-lock.overrides.yaml b/manifest-lock.overrides.yaml
new file mode 100644
index 0000000000..b7e644cec4
--- /dev/null
+++ b/manifest-lock.overrides.yaml
@@ -0,0 +1,36 @@
+# This lockfile should be used to pin to a package version (`type: pin`) or to
+# fast-track packages ahead of Bodhi (`type: fast-track`). Fast-tracked
+# packages will automatically be removed once they are in the stable repos.
+#
+# IMPORTANT: YAML comments *will not* be preserved. All `pin` overrides *must*
+# include a URL in the `metadata.reason` key. Overrides of type `fast-track`
+# *should* include a Bodhi update URL in the `metadata.bodhi` key and a URL
+# in the `metadata.reason` key, though it's acceptable to omit a `reason`
+# for FCOS-specific packages (ignition, afterburn, etc.).
+
+packages:
+  podman:
+    evr: 5:5.2.2-1.fc42
+    metadata:
+      reason: https://github.com/coreos/fedora-coreos-tracker/issues/1793
+      type: pin
+  shadow-utils:
+    evr: 2:4.15.1-9.fc41
+    metadata:
+      reason: https://github.com/coreos/fedora-coreos-tracker/issues/1793
+      type: pin
+  shadow-utils-subid:
+    evr: 2:4.15.1-9.fc41
+    metadata:
+      reason: https://github.com/coreos/fedora-coreos-tracker/issues/1793
+      type: pin
+  skopeo:
+    evr: 1:1.16.1-1.fc42
+    metadata:
+      reason: https://github.com/coreos/fedora-coreos-tracker/issues/1793
+      type: pin
+  toolbox:
+    evr: 0.0.99.5-17.fc41
+    metadata:
+      reason: https://github.com/coreos/fedora-coreos-tracker/issues/1793
+      type: pin
diff --git a/manifest-lock.x86_64.json b/manifest-lock.x86_64.json
deleted file mode 100644
index b39dd19614..0000000000
--- a/manifest-lock.x86_64.json
+++ /dev/null
@@ -1,1218 +0,0 @@
-{
-  "packages": {
-    "NetworkManager": {
-      "evra": "1:1.26.4-1.fc33.x86_64"
-    },
-    "NetworkManager-libnm": {
-      "evra": "1:1.26.4-1.fc33.x86_64"
-    },
-    "NetworkManager-team": {
-      "evra": "1:1.26.4-1.fc33.x86_64"
-    },
-    "NetworkManager-tui": {
-      "evra": "1:1.26.4-1.fc33.x86_64"
-    },
-    "acl": {
-      "evra": "2.2.53-9.fc33.x86_64"
-    },
-    "adcli": {
-      "evra": "0.9.0-4.fc33.x86_64"
-    },
-    "afterburn": {
-      "evra": "4.6.0-2.fc33.x86_64"
-    },
-    "afterburn-dracut": {
-      "evra": "4.6.0-2.fc33.x86_64"
-    },
-    "alternatives": {
-      "evra": "1.14-3.fc33.x86_64"
-    },
-    "attr": {
-      "evra": "2.4.48-10.fc33.x86_64"
-    },
-    "audit-libs": {
-      "evra": "3.0-0.21.20191104git1c2f876.fc33.x86_64"
-    },
-    "avahi-libs": {
-      "evra": "0.8-7.fc33.x86_64"
-    },
-    "basesystem": {
-      "evra": "11-10.fc33.noarch"
-    },
-    "bash": {
-      "evra": "5.0.17-2.fc33.x86_64"
-    },
-    "bash-completion": {
-      "evra": "1:2.8-9.fc33.noarch"
-    },
-    "bind-libs": {
-      "evra": "32:9.11.25-2.fc33.x86_64"
-    },
-    "bind-libs-lite": {
-      "evra": "32:9.11.25-2.fc33.x86_64"
-    },
-    "bind-license": {
-      "evra": "32:9.11.25-2.fc33.noarch"
-    },
-    "bind-utils": {
-      "evra": "32:9.11.25-2.fc33.x86_64"
-    },
-    "bootupd": {
-      "evra": "0.2.3-2.fc33.x86_64"
-    },
-    "bsdtar": {
-      "evra": "3.5.0-1.fc33.x86_64"
-    },
-    "btrfs-progs": {
-      "evra": "5.9-1.fc33.x86_64"
-    },
-    "bubblewrap": {
-      "evra": "0.4.1-2.fc33.x86_64"
-    },
-    "bzip2": {
-      "evra": "1.0.8-4.fc33.x86_64"
-    },
-    "bzip2-libs": {
-      "evra": "1.0.8-4.fc33.x86_64"
-    },
-    "c-ares": {
-      "evra": "1.17.0-1.fc33.x86_64"
-    },
-    "ca-certificates": {
-      "evra": "2020.2.41-4.fc33.noarch"
-    },
-    "catatonit": {
-      "evra": "0.1.5-3.fc33.x86_64"
-    },
-    "chrony": {
-      "evra": "4.0-1.fc33.x86_64"
-    },
-    "cifs-utils": {
-      "evra": "6.9-4.fc33.x86_64"
-    },
-    "clevis": {
-      "evra": "15-2.fc33.x86_64"
-    },
-    "clevis-dracut": {
-      "evra": "15-2.fc33.x86_64"
-    },
-    "clevis-luks": {
-      "evra": "15-2.fc33.x86_64"
-    },
-    "clevis-systemd": {
-      "evra": "15-2.fc33.x86_64"
-    },
-    "cloud-utils-growpart": {
-      "evra": "0.31-7.fc33.noarch"
-    },
-    "compat-readline5": {
-      "evra": "5.2-37.fc33.x86_64"
-    },
-    "conmon": {
-      "evra": "2:2.0.21-3.fc33.x86_64"
-    },
-    "console-login-helper-messages": {
-      "evra": "0.20.3-1.fc33.noarch"
-    },
-    "console-login-helper-messages-issuegen": {
-      "evra": "0.20.3-1.fc33.noarch"
-    },
-    "console-login-helper-messages-motdgen": {
-      "evra": "0.20.3-1.fc33.noarch"
-    },
-    "console-login-helper-messages-profile": {
-      "evra": "0.20.3-1.fc33.noarch"
-    },
-    "container-selinux": {
-      "evra": "2:2.151.0-1.fc33.noarch"
-    },
-    "containerd": {
-      "evra": "1.4.3-1.fc33.x86_64"
-    },
-    "containernetworking-plugins": {
-      "evra": "0.8.7-1.fc33.x86_64"
-    },
-    "containers-common": {
-      "evra": "1:1.2.0-10.fc33.x86_64"
-    },
-    "coreos-installer": {
-      "evra": "0.7.2-1.fc33.x86_64"
-    },
-    "coreos-installer-bootinfra": {
-      "evra": "0.7.2-1.fc33.x86_64"
-    },
-    "coreutils": {
-      "evra": "8.32-12.fc33.x86_64"
-    },
-    "coreutils-common": {
-      "evra": "8.32-12.fc33.x86_64"
-    },
-    "cpio": {
-      "evra": "2.13-8.fc33.x86_64"
-    },
-    "cracklib": {
-      "evra": "2.9.6-24.fc33.x86_64"
-    },
-    "crun": {
-      "evra": "0.16-1.fc33.x86_64"
-    },
-    "crypto-policies": {
-      "evra": "20200918-1.git85dccc5.fc33.noarch"
-    },
-    "cryptsetup": {
-      "evra": "2.3.4-1.fc33.x86_64"
-    },
-    "cryptsetup-libs": {
-      "evra": "2.3.4-1.fc33.x86_64"
-    },
-    "cups-libs": {
-      "evra": "1:2.3.3op1-1.fc33.x86_64"
-    },
-    "curl": {
-      "evra": "7.71.1-8.fc33.x86_64"
-    },
-    "cyrus-sasl-gssapi": {
-      "evra": "2.1.27-6.fc33.x86_64"
-    },
-    "cyrus-sasl-lib": {
-      "evra": "2.1.27-6.fc33.x86_64"
-    },
-    "dbus": {
-      "evra": "1:1.12.20-2.fc33.x86_64"
-    },
-    "dbus-broker": {
-      "evra": "24-1.fc33.x86_64"
-    },
-    "dbus-common": {
-      "evra": "1:1.12.20-2.fc33.noarch"
-    },
-    "dbus-libs": {
-      "evra": "1:1.12.20-2.fc33.x86_64"
-    },
-    "device-mapper": {
-      "evra": "1.02.173-1.fc33.x86_64"
-    },
-    "device-mapper-event": {
-      "evra": "1.02.173-1.fc33.x86_64"
-    },
-    "device-mapper-event-libs": {
-      "evra": "1.02.173-1.fc33.x86_64"
-    },
-    "device-mapper-libs": {
-      "evra": "1.02.173-1.fc33.x86_64"
-    },
-    "device-mapper-multipath": {
-      "evra": "0.8.4-7.fc33.x86_64"
-    },
-    "device-mapper-multipath-libs": {
-      "evra": "0.8.4-7.fc33.x86_64"
-    },
-    "device-mapper-persistent-data": {
-      "evra": "0.8.5-4.fc33.x86_64"
-    },
-    "diffutils": {
-      "evra": "3.7-7.fc33.x86_64"
-    },
-    "dnsmasq": {
-      "evra": "2.82-3.fc33.x86_64"
-    },
-    "dosfstools": {
-      "evra": "4.1-12.fc33.x86_64"
-    },
-    "dracut": {
-      "evra": "050-64.git20200529.fc33.x86_64"
-    },
-    "dracut-network": {
-      "evra": "050-64.git20200529.fc33.x86_64"
-    },
-    "dracut-squash": {
-      "evra": "050-64.git20200529.fc33.x86_64"
-    },
-    "e2fsprogs": {
-      "evra": "1.45.6-4.fc33.x86_64"
-    },
-    "e2fsprogs-libs": {
-      "evra": "1.45.6-4.fc33.x86_64"
-    },
-    "efi-filesystem": {
-      "evra": "4-5.fc33.noarch"
-    },
-    "efibootmgr": {
-      "evra": "16-9.fc33.x86_64"
-    },
-    "efivar-libs": {
-      "evra": "37-14.fc33.x86_64"
-    },
-    "elfutils-default-yama-scope": {
-      "evra": "0.182-1.fc33.noarch"
-    },
-    "elfutils-libelf": {
-      "evra": "0.182-1.fc33.x86_64"
-    },
-    "elfutils-libs": {
-      "evra": "0.182-1.fc33.x86_64"
-    },
-    "ethtool": {
-      "evra": "2:5.9-1.fc33.x86_64"
-    },
-    "expat": {
-      "evra": "2.2.8-3.fc33.x86_64"
-    },
-    "fedora-coreos-pinger": {
-      "evra": "0.0.4-7.fc33.x86_64"
-    },
-    "fedora-gpg-keys": {
-      "evra": "33-1.noarch"
-    },
-    "fedora-release-common": {
-      "evra": "33-2.noarch"
-    },
-    "fedora-release-coreos": {
-      "evra": "33-2.noarch"
-    },
-    "fedora-release-identity-coreos": {
-      "evra": "33-2.noarch"
-    },
-    "fedora-repos": {
-      "evra": "33-1.noarch"
-    },
-    "fedora-repos-archive": {
-      "evra": "33-1.noarch"
-    },
-    "fedora-repos-modular": {
-      "evra": "33-1.noarch"
-    },
-    "fedora-repos-ostree": {
-      "evra": "33-1.noarch"
-    },
-    "file": {
-      "evra": "5.39-3.fc33.x86_64"
-    },
-    "file-libs": {
-      "evra": "5.39-3.fc33.x86_64"
-    },
-    "filesystem": {
-      "evra": "3.14-3.fc33.x86_64"
-    },
-    "findutils": {
-      "evra": "1:4.7.0-7.fc33.x86_64"
-    },
-    "firewalld-filesystem": {
-      "evra": "0.8.4-1.fc33.noarch"
-    },
-    "flatpak-session-helper": {
-      "evra": "1.8.3-1.fc33.x86_64"
-    },
-    "fstrm": {
-      "evra": "0.6.0-1.fc33.x86_64"
-    },
-    "fuse": {
-      "evra": "2.9.9-10.fc33.x86_64"
-    },
-    "fuse-common": {
-      "evra": "3.9.4-1.fc33.x86_64"
-    },
-    "fuse-libs": {
-      "evra": "2.9.9-10.fc33.x86_64"
-    },
-    "fuse-overlayfs": {
-      "evra": "1.3.0-1.fc33.x86_64"
-    },
-    "fuse-sshfs": {
-      "evra": "3.7.1-1.fc33.x86_64"
-    },
-    "fuse3": {
-      "evra": "3.9.4-1.fc33.x86_64"
-    },
-    "fuse3-libs": {
-      "evra": "3.9.4-1.fc33.x86_64"
-    },
-    "fwupd": {
-      "evra": "1.5.3-1.fc33.x86_64"
-    },
-    "gawk": {
-      "evra": "5.1.0-2.fc33.x86_64"
-    },
-    "gdisk": {
-      "evra": "1.0.5-2.fc33.x86_64"
-    },
-    "gettext": {
-      "evra": "0.21-3.fc33.x86_64"
-    },
-    "gettext-libs": {
-      "evra": "0.21-3.fc33.x86_64"
-    },
-    "git-core": {
-      "evra": "2.29.2-3.fc33.x86_64"
-    },
-    "glib2": {
-      "evra": "2.66.3-1.fc33.x86_64"
-    },
-    "glibc": {
-      "evra": "2.32-2.fc33.x86_64"
-    },
-    "glibc-all-langpacks": {
-      "evra": "2.32-2.fc33.x86_64"
-    },
-    "glibc-common": {
-      "evra": "2.32-2.fc33.x86_64"
-    },
-    "gmp": {
-      "evra": "1:6.2.0-5.fc33.x86_64"
-    },
-    "gnupg2": {
-      "evra": "2.2.25-2.fc33.x86_64"
-    },
-    "gnutls": {
-      "evra": "3.6.15-1.fc33.x86_64"
-    },
-    "gpgme": {
-      "evra": "1.14.0-2.fc33.x86_64"
-    },
-    "grep": {
-      "evra": "3.4-5.fc33.x86_64"
-    },
-    "grub2-common": {
-      "evra": "1:2.04-31.fc33.noarch"
-    },
-    "grub2-efi-x64": {
-      "evra": "1:2.04-31.fc33.x86_64"
-    },
-    "grub2-pc": {
-      "evra": "1:2.04-31.fc33.x86_64"
-    },
-    "grub2-pc-modules": {
-      "evra": "1:2.04-31.fc33.noarch"
-    },
-    "grub2-tools": {
-      "evra": "1:2.04-31.fc33.x86_64"
-    },
-    "grub2-tools-minimal": {
-      "evra": "1:2.04-31.fc33.x86_64"
-    },
-    "gzip": {
-      "evra": "1.10-3.fc33.x86_64"
-    },
-    "hostname": {
-      "evra": "3.23-3.fc33.x86_64"
-    },
-    "hwdata": {
-      "evra": "0.341-1.fc33.noarch"
-    },
-    "ignition": {
-      "evra": "2.8.1-1.gitc733d23.fc33.x86_64"
-    },
-    "iproute": {
-      "evra": "5.8.0-1.fc33.x86_64"
-    },
-    "iproute-tc": {
-      "evra": "5.8.0-1.fc33.x86_64"
-    },
-    "iptables": {
-      "evra": "1.8.5-4.fc33.x86_64"
-    },
-    "iptables-libs": {
-      "evra": "1.8.5-4.fc33.x86_64"
-    },
-    "iptables-nft": {
-      "evra": "1.8.5-4.fc33.x86_64"
-    },
-    "iptables-services": {
-      "evra": "1.8.5-4.fc33.x86_64"
-    },
-    "iputils": {
-      "evra": "20200821-1.fc33.x86_64"
-    },
-    "irqbalance": {
-      "evra": "2:1.7.0-4.fc33.x86_64"
-    },
-    "iscsi-initiator-utils": {
-      "evra": "6.2.1.1-0.gitac87641.fc33.2.x86_64"
-    },
-    "iscsi-initiator-utils-iscsiuio": {
-      "evra": "6.2.1.1-0.gitac87641.fc33.2.x86_64"
-    },
-    "isns-utils-libs": {
-      "evra": "0.97-11.fc33.x86_64"
-    },
-    "jansson": {
-      "evra": "2.13.1-1.fc33.x86_64"
-    },
-    "jose": {
-      "evra": "10-8.fc33.x86_64"
-    },
-    "jq": {
-      "evra": "1.6-5.fc33.x86_64"
-    },
-    "json-c": {
-      "evra": "0.14-7.fc33.x86_64"
-    },
-    "json-glib": {
-      "evra": "1.6.0-1.fc33.x86_64"
-    },
-    "kbd": {
-      "evra": "2.3.0-2.fc33.x86_64"
-    },
-    "kbd-legacy": {
-      "evra": "2.3.0-2.fc33.noarch"
-    },
-    "kbd-misc": {
-      "evra": "2.3.0-2.fc33.noarch"
-    },
-    "kernel": {
-      "evra": "5.9.14-200.fc33.x86_64"
-    },
-    "kernel-core": {
-      "evra": "5.9.14-200.fc33.x86_64"
-    },
-    "kernel-modules": {
-      "evra": "5.9.14-200.fc33.x86_64"
-    },
-    "kexec-tools": {
-      "evra": "2.0.20-17.fc33.x86_64"
-    },
-    "keyutils": {
-      "evra": "1.6-5.fc33.x86_64"
-    },
-    "keyutils-libs": {
-      "evra": "1.6-5.fc33.x86_64"
-    },
-    "kmod": {
-      "evra": "27-3.fc33.x86_64"
-    },
-    "kmod-libs": {
-      "evra": "27-3.fc33.x86_64"
-    },
-    "kpartx": {
-      "evra": "0.8.4-7.fc33.x86_64"
-    },
-    "krb5-libs": {
-      "evra": "1.18.2-29.fc33.x86_64"
-    },
-    "less": {
-      "evra": "551-4.fc33.x86_64"
-    },
-    "libacl": {
-      "evra": "2.2.53-9.fc33.x86_64"
-    },
-    "libaio": {
-      "evra": "0.3.111-10.fc33.x86_64"
-    },
-    "libarchive": {
-      "evra": "3.5.0-1.fc33.x86_64"
-    },
-    "libargon2": {
-      "evra": "20171227-5.fc33.x86_64"
-    },
-    "libassuan": {
-      "evra": "2.5.3-4.fc33.x86_64"
-    },
-    "libattr": {
-      "evra": "2.4.48-10.fc33.x86_64"
-    },
-    "libbasicobjects": {
-      "evra": "0.1.1-46.fc33.x86_64"
-    },
-    "libblkid": {
-      "evra": "2.36-3.fc33.x86_64"
-    },
-    "libbrotli": {
-      "evra": "1.0.9-3.fc33.x86_64"
-    },
-    "libcap": {
-      "evra": "2.26-8.fc33.x86_64"
-    },
-    "libcap-ng": {
-      "evra": "0.8-1.fc33.x86_64"
-    },
-    "libcbor": {
-      "evra": "0.5.0-7.fc33.x86_64"
-    },
-    "libcollection": {
-      "evra": "0.7.0-46.fc33.x86_64"
-    },
-    "libcom_err": {
-      "evra": "1.45.6-4.fc33.x86_64"
-    },
-    "libcurl": {
-      "evra": "7.71.1-8.fc33.x86_64"
-    },
-    "libdaemon": {
-      "evra": "0.14-20.fc33.x86_64"
-    },
-    "libdb": {
-      "evra": "5.3.28-45.fc33.x86_64"
-    },
-    "libdhash": {
-      "evra": "0.5.0-46.fc33.x86_64"
-    },
-    "libeconf": {
-      "evra": "0.3.8-4.fc33.x86_64"
-    },
-    "libedit": {
-      "evra": "3.1-33.20191231cvs.fc33.x86_64"
-    },
-    "libevent": {
-      "evra": "2.1.8-10.fc33.x86_64"
-    },
-    "libfdisk": {
-      "evra": "2.36-3.fc33.x86_64"
-    },
-    "libffi": {
-      "evra": "3.1-26.fc33.x86_64"
-    },
-    "libfido2": {
-      "evra": "1.4.0-3.fc33.x86_64"
-    },
-    "libgcab1": {
-      "evra": "1.4-3.fc33.x86_64"
-    },
-    "libgcc": {
-      "evra": "10.2.1-9.fc33.x86_64"
-    },
-    "libgcrypt": {
-      "evra": "1.8.7-1.fc33.x86_64"
-    },
-    "libgomp": {
-      "evra": "10.2.1-9.fc33.x86_64"
-    },
-    "libgpg-error": {
-      "evra": "1.39-1.fc33.x86_64"
-    },
-    "libgudev": {
-      "evra": "234-1.fc33.x86_64"
-    },
-    "libgusb": {
-      "evra": "0.3.5-1.fc33.x86_64"
-    },
-    "libibverbs": {
-      "evra": "32.0-1.fc33.x86_64"
-    },
-    "libicu": {
-      "evra": "67.1-4.fc33.x86_64"
-    },
-    "libidn2": {
-      "evra": "2.3.0-4.fc33.x86_64"
-    },
-    "libini_config": {
-      "evra": "1.3.1-46.fc33.x86_64"
-    },
-    "libipa_hbac": {
-      "evra": "2.4.0-4.fc33.x86_64"
-    },
-    "libjcat": {
-      "evra": "0.1.4-1.fc33.x86_64"
-    },
-    "libjose": {
-      "evra": "10-8.fc33.x86_64"
-    },
-    "libkcapi": {
-      "evra": "1.2.0-3.fc33.x86_64"
-    },
-    "libkcapi-hmaccalc": {
-      "evra": "1.2.0-3.fc33.x86_64"
-    },
-    "libksba": {
-      "evra": "1.3.5-13.fc33.x86_64"
-    },
-    "libldb": {
-      "evra": "2.2.0-4.fc33.x86_64"
-    },
-    "libluksmeta": {
-      "evra": "9-8.fc33.x86_64"
-    },
-    "libmaxminddb": {
-      "evra": "1.4.2-3.fc33.x86_64"
-    },
-    "libmetalink": {
-      "evra": "0.1.3-13.fc33.x86_64"
-    },
-    "libmnl": {
-      "evra": "1.0.4-12.fc33.x86_64"
-    },
-    "libmodulemd": {
-      "evra": "2.11.0-1.fc33.x86_64"
-    },
-    "libmount": {
-      "evra": "2.36-3.fc33.x86_64"
-    },
-    "libndp": {
-      "evra": "1.7-6.fc33.x86_64"
-    },
-    "libnetfilter_conntrack": {
-      "evra": "1.0.7-5.fc33.x86_64"
-    },
-    "libnfnetlink": {
-      "evra": "1.0.1-18.fc33.x86_64"
-    },
-    "libnfsidmap": {
-      "evra": "1:2.5.2-1.rc1.fc33.x86_64"
-    },
-    "libnftnl": {
-      "evra": "1.1.7-3.fc33.x86_64"
-    },
-    "libnghttp2": {
-      "evra": "1.41.0-3.fc33.x86_64"
-    },
-    "libnl3": {
-      "evra": "3.5.0-5.fc33.x86_64"
-    },
-    "libnl3-cli": {
-      "evra": "3.5.0-5.fc33.x86_64"
-    },
-    "libnsl2": {
-      "evra": "1.2.0-8.20180605git4a062cf.fc33.x86_64"
-    },
-    "libpath_utils": {
-      "evra": "0.2.1-46.fc33.x86_64"
-    },
-    "libpcap": {
-      "evra": "14:1.9.1-6.fc33.x86_64"
-    },
-    "libpkgconf": {
-      "evra": "1.7.3-5.fc33.x86_64"
-    },
-    "libpsl": {
-      "evra": "0.21.1-2.fc33.x86_64"
-    },
-    "libpwquality": {
-      "evra": "1.4.4-1.fc33.x86_64"
-    },
-    "libref_array": {
-      "evra": "0.1.5-46.fc33.x86_64"
-    },
-    "librepo": {
-      "evra": "1.12.1-1.fc33.x86_64"
-    },
-    "libreport-filesystem": {
-      "evra": "2.14.0-15.fc33.noarch"
-    },
-    "libseccomp": {
-      "evra": "2.5.0-3.fc33.x86_64"
-    },
-    "libselinux": {
-      "evra": "3.1-2.fc33.x86_64"
-    },
-    "libselinux-utils": {
-      "evra": "3.1-2.fc33.x86_64"
-    },
-    "libsemanage": {
-      "evra": "3.1-2.fc33.x86_64"
-    },
-    "libsepol": {
-      "evra": "3.1-3.fc33.x86_64"
-    },
-    "libsigsegv": {
-      "evra": "2.11-11.fc33.x86_64"
-    },
-    "libslirp": {
-      "evra": "4.3.1-3.fc33.x86_64"
-    },
-    "libsmartcols": {
-      "evra": "2.36-3.fc33.x86_64"
-    },
-    "libsmbclient": {
-      "evra": "2:4.13.2-2.fc33.x86_64"
-    },
-    "libsmbios": {
-      "evra": "2.4.2-10.fc33.x86_64"
-    },
-    "libsolv": {
-      "evra": "0.7.15-1.fc33.x86_64"
-    },
-    "libss": {
-      "evra": "1.45.6-4.fc33.x86_64"
-    },
-    "libssh": {
-      "evra": "0.9.5-1.fc33.x86_64"
-    },
-    "libssh-config": {
-      "evra": "0.9.5-1.fc33.noarch"
-    },
-    "libsss_certmap": {
-      "evra": "2.4.0-4.fc33.x86_64"
-    },
-    "libsss_idmap": {
-      "evra": "2.4.0-4.fc33.x86_64"
-    },
-    "libsss_nss_idmap": {
-      "evra": "2.4.0-4.fc33.x86_64"
-    },
-    "libsss_sudo": {
-      "evra": "2.4.0-4.fc33.x86_64"
-    },
-    "libstdc++": {
-      "evra": "10.2.1-9.fc33.x86_64"
-    },
-    "libtalloc": {
-      "evra": "2.3.1-5.fc33.x86_64"
-    },
-    "libtasn1": {
-      "evra": "4.16.0-3.fc33.x86_64"
-    },
-    "libtdb": {
-      "evra": "1.4.3-5.fc33.x86_64"
-    },
-    "libteam": {
-      "evra": "1.31-2.fc33.x86_64"
-    },
-    "libtevent": {
-      "evra": "0.10.2-5.fc33.x86_64"
-    },
-    "libtextstyle": {
-      "evra": "0.21-3.fc33.x86_64"
-    },
-    "libtirpc": {
-      "evra": "1.2.6-2.rc4.fc33.x86_64"
-    },
-    "libunistring": {
-      "evra": "0.9.10-9.fc33.x86_64"
-    },
-    "libusbx": {
-      "evra": "1.0.23-2.fc33.x86_64"
-    },
-    "libuser": {
-      "evra": "0.62-26.fc33.x86_64"
-    },
-    "libutempter": {
-      "evra": "1.2.1-2.fc33.x86_64"
-    },
-    "libuuid": {
-      "evra": "2.36-3.fc33.x86_64"
-    },
-    "libvarlink-util": {
-      "evra": "19-3.fc33.x86_64"
-    },
-    "libverto": {
-      "evra": "0.3.0-10.fc33.x86_64"
-    },
-    "libwbclient": {
-      "evra": "2:4.13.2-2.fc33.x86_64"
-    },
-    "libxcrypt": {
-      "evra": "4.4.17-1.fc33.x86_64"
-    },
-    "libxml2": {
-      "evra": "2.9.10-8.fc33.x86_64"
-    },
-    "libxmlb": {
-      "evra": "0.2.1-1.fc33.x86_64"
-    },
-    "libyaml": {
-      "evra": "0.2.5-3.fc33.x86_64"
-    },
-    "libzstd": {
-      "evra": "1.4.5-5.fc33.x86_64"
-    },
-    "linux-atm-libs": {
-      "evra": "2.5.1-27.fc33.x86_64"
-    },
-    "linux-firmware": {
-      "evra": "20201118-115.fc33.noarch"
-    },
-    "linux-firmware-whence": {
-      "evra": "20201118-115.fc33.noarch"
-    },
-    "lmdb-libs": {
-      "evra": "0.9.27-1.fc33.x86_64"
-    },
-    "logrotate": {
-      "evra": "3.17.0-3.fc33.x86_64"
-    },
-    "lsof": {
-      "evra": "4.93.2-4.fc33.x86_64"
-    },
-    "lua-libs": {
-      "evra": "5.4.2-1.fc33.x86_64"
-    },
-    "luksmeta": {
-      "evra": "9-8.fc33.x86_64"
-    },
-    "lvm2": {
-      "evra": "2.03.10-1.fc33.x86_64"
-    },
-    "lvm2-libs": {
-      "evra": "2.03.10-1.fc33.x86_64"
-    },
-    "lz4-libs": {
-      "evra": "1.9.1-3.fc33.x86_64"
-    },
-    "lzo": {
-      "evra": "2.10-3.fc33.x86_64"
-    },
-    "mdadm": {
-      "evra": "4.1-6.fc33.x86_64"
-    },
-    "microcode_ctl": {
-      "evra": "2:2.1-43.fc33.x86_64"
-    },
-    "moby-engine": {
-      "evra": "19.03.13-1.ce.git4484c46.fc33.x86_64"
-    },
-    "mokutil": {
-      "evra": "2:0.4.0-2.fc33.x86_64"
-    },
-    "mozjs78": {
-      "evra": "78.5.0-1.fc33.x86_64"
-    },
-    "mpfr": {
-      "evra": "4.1.0-2.fc33.x86_64"
-    },
-    "ncurses": {
-      "evra": "6.2-3.20200222.fc33.x86_64"
-    },
-    "ncurses-base": {
-      "evra": "6.2-3.20200222.fc33.noarch"
-    },
-    "ncurses-libs": {
-      "evra": "6.2-3.20200222.fc33.x86_64"
-    },
-    "net-tools": {
-      "evra": "2.0-0.58.20160912git.fc33.x86_64"
-    },
-    "nettle": {
-      "evra": "3.6-3.fc33.x86_64"
-    },
-    "newt": {
-      "evra": "0.52.21-8.fc33.x86_64"
-    },
-    "nfs-utils-coreos": {
-      "evra": "1:2.5.2-1.rc1.fc33.x86_64"
-    },
-    "nftables": {
-      "evra": "1:0.9.3-6.fc33.x86_64"
-    },
-    "nmap-ncat": {
-      "evra": "2:7.80-5.fc33.x86_64"
-    },
-    "npth": {
-      "evra": "1.6-5.fc33.x86_64"
-    },
-    "nss-altfiles": {
-      "evra": "2.18.1-17.fc33.x86_64"
-    },
-    "numactl-libs": {
-      "evra": "2.0.14-1.fc33.x86_64"
-    },
-    "nvme-cli": {
-      "evra": "1.11.1-2.fc33.x86_64"
-    },
-    "oniguruma": {
-      "evra": "6.9.6-1.fc33.x86_64"
-    },
-    "openldap": {
-      "evra": "2.4.50-5.fc33.x86_64"
-    },
-    "openssh": {
-      "evra": "8.4p1-4.fc33.x86_64"
-    },
-    "openssh-clients": {
-      "evra": "8.4p1-4.fc33.x86_64"
-    },
-    "openssh-server": {
-      "evra": "8.4p1-4.fc33.x86_64"
-    },
-    "openssl": {
-      "evra": "1:1.1.1i-1.fc33.x86_64"
-    },
-    "openssl-libs": {
-      "evra": "1:1.1.1i-1.fc33.x86_64"
-    },
-    "os-prober": {
-      "evra": "1.77-6.fc33.x86_64"
-    },
-    "ostree": {
-      "evra": "2020.8-1.fc33.x86_64"
-    },
-    "ostree-libs": {
-      "evra": "2020.8-1.fc33.x86_64"
-    },
-    "p11-kit": {
-      "evra": "0.23.22-1.fc33.x86_64"
-    },
-    "p11-kit-trust": {
-      "evra": "0.23.22-1.fc33.x86_64"
-    },
-    "pam": {
-      "evra": "1.4.0-10.fc33.x86_64"
-    },
-    "passwd": {
-      "evra": "0.80-9.fc33.x86_64"
-    },
-    "pciutils": {
-      "evra": "3.6.4-2.fc33.x86_64"
-    },
-    "pciutils-libs": {
-      "evra": "3.6.4-2.fc33.x86_64"
-    },
-    "pcre": {
-      "evra": "8.44-2.fc33.x86_64"
-    },
-    "pcre2": {
-      "evra": "10.35-8.fc33.x86_64"
-    },
-    "pcre2-syntax": {
-      "evra": "10.35-8.fc33.noarch"
-    },
-    "pigz": {
-      "evra": "2.4-7.fc33.x86_64"
-    },
-    "pkgconf": {
-      "evra": "1.7.3-5.fc33.x86_64"
-    },
-    "pkgconf-m4": {
-      "evra": "1.7.3-5.fc33.noarch"
-    },
-    "pkgconf-pkg-config": {
-      "evra": "1.7.3-5.fc33.x86_64"
-    },
-    "podman": {
-      "evra": "2:2.2.1-1.fc33.x86_64"
-    },
-    "podman-plugins": {
-      "evra": "2:2.2.1-1.fc33.x86_64"
-    },
-    "policycoreutils": {
-      "evra": "3.1-4.fc33.x86_64"
-    },
-    "polkit": {
-      "evra": "0.117-2.fc33.x86_64"
-    },
-    "polkit-libs": {
-      "evra": "0.117-2.fc33.x86_64"
-    },
-    "polkit-pkla-compat": {
-      "evra": "0.1-18.fc33.x86_64"
-    },
-    "popt": {
-      "evra": "1.18-2.fc33.x86_64"
-    },
-    "procps-ng": {
-      "evra": "3.3.16-1.fc33.x86_64"
-    },
-    "protobuf-c": {
-      "evra": "1.3.3-3.fc33.x86_64"
-    },
-    "psmisc": {
-      "evra": "23.3-4.fc33.x86_64"
-    },
-    "publicsuffix-list-dafsa": {
-      "evra": "20190417-4.fc33.noarch"
-    },
-    "qrencode-libs": {
-      "evra": "4.0.2-6.fc33.x86_64"
-    },
-    "rdma-core": {
-      "evra": "32.0-1.fc33.x86_64"
-    },
-    "readline": {
-      "evra": "8.0-5.fc33.x86_64"
-    },
-    "rpcbind": {
-      "evra": "1.2.5-5.rc1.fc33.3.x86_64"
-    },
-    "rpm": {
-      "evra": "4.16.0-5.fc33.x86_64"
-    },
-    "rpm-libs": {
-      "evra": "4.16.0-5.fc33.x86_64"
-    },
-    "rpm-ostree": {
-      "evra": "2020.8-1.fc33.x86_64"
-    },
-    "rpm-ostree-libs": {
-      "evra": "2020.8-1.fc33.x86_64"
-    },
-    "rpm-plugin-selinux": {
-      "evra": "4.16.0-5.fc33.x86_64"
-    },
-    "rsync": {
-      "evra": "3.2.3-3.fc33.x86_64"
-    },
-    "runc": {
-      "evra": "2:1.0.0-279.dev.gitdedadbf.fc33.x86_64"
-    },
-    "samba-client-libs": {
-      "evra": "2:4.13.2-2.fc33.x86_64"
-    },
-    "samba-common": {
-      "evra": "2:4.13.2-2.fc33.noarch"
-    },
-    "samba-common-libs": {
-      "evra": "2:4.13.2-2.fc33.x86_64"
-    },
-    "samba-libs": {
-      "evra": "2:4.13.2-2.fc33.x86_64"
-    },
-    "sed": {
-      "evra": "4.8-5.fc33.x86_64"
-    },
-    "selinux-policy": {
-      "evra": "3.14.6-33.fc33.noarch"
-    },
-    "selinux-policy-targeted": {
-      "evra": "3.14.6-33.fc33.noarch"
-    },
-    "setup": {
-      "evra": "2.13.7-2.fc33.noarch"
-    },
-    "sg3_utils": {
-      "evra": "1.45-3.fc33.x86_64"
-    },
-    "sg3_utils-libs": {
-      "evra": "1.45-3.fc33.x86_64"
-    },
-    "shadow-utils": {
-      "evra": "2:4.8.1-5.fc33.x86_64"
-    },
-    "shared-mime-info": {
-      "evra": "2.0-3.fc33.x86_64"
-    },
-    "shim-x64": {
-      "evra": "15-8.x86_64"
-    },
-    "skopeo": {
-      "evra": "1:1.2.0-10.fc33.x86_64"
-    },
-    "slang": {
-      "evra": "2.3.2-8.fc33.x86_64"
-    },
-    "slirp4netns": {
-      "evra": "1.1.8-1.fc33.x86_64"
-    },
-    "snappy": {
-      "evra": "1.1.8-4.fc33.x86_64"
-    },
-    "socat": {
-      "evra": "1.7.3.4-3.fc33.x86_64"
-    },
-    "sqlite-libs": {
-      "evra": "3.34.0-1.fc33.x86_64"
-    },
-    "squashfs-tools": {
-      "evra": "4.4-2.20200513gitc570c61.fc33.x86_64"
-    },
-    "ssh-key-dir": {
-      "evra": "0.1.2-5.fc33.x86_64"
-    },
-    "sssd": {
-      "evra": "2.4.0-4.fc33.x86_64"
-    },
-    "sssd-ad": {
-      "evra": "2.4.0-4.fc33.x86_64"
-    },
-    "sssd-client": {
-      "evra": "2.4.0-4.fc33.x86_64"
-    },
-    "sssd-common": {
-      "evra": "2.4.0-4.fc33.x86_64"
-    },
-    "sssd-common-pac": {
-      "evra": "2.4.0-4.fc33.x86_64"
-    },
-    "sssd-ipa": {
-      "evra": "2.4.0-4.fc33.x86_64"
-    },
-    "sssd-krb5": {
-      "evra": "2.4.0-4.fc33.x86_64"
-    },
-    "sssd-krb5-common": {
-      "evra": "2.4.0-4.fc33.x86_64"
-    },
-    "sssd-ldap": {
-      "evra": "2.4.0-4.fc33.x86_64"
-    },
-    "sudo": {
-      "evra": "1.9.2-1.fc33.x86_64"
-    },
-    "systemd": {
-      "evra": "246.7-1.fc33.x86_64"
-    },
-    "systemd-container": {
-      "evra": "246.7-1.fc33.x86_64"
-    },
-    "systemd-libs": {
-      "evra": "246.7-1.fc33.x86_64"
-    },
-    "systemd-pam": {
-      "evra": "246.7-1.fc33.x86_64"
-    },
-    "systemd-rpm-macros": {
-      "evra": "246.7-1.fc33.noarch"
-    },
-    "systemd-udev": {
-      "evra": "246.7-1.fc33.x86_64"
-    },
-    "tar": {
-      "evra": "2:1.32-6.fc33.x86_64"
-    },
-    "teamd": {
-      "evra": "1.31-2.fc33.x86_64"
-    },
-    "toolbox": {
-      "evra": "0.0.97-1.fc33.x86_64"
-    },
-    "tpm2-tools": {
-      "evra": "4.3.0-1.fc33.x86_64"
-    },
-    "tpm2-tss": {
-      "evra": "3.0.3-1.fc33.x86_64"
-    },
-    "tzdata": {
-      "evra": "2020d-1.fc33.noarch"
-    },
-    "userspace-rcu": {
-      "evra": "0.12.1-2.fc33.x86_64"
-    },
-    "util-linux": {
-      "evra": "2.36-3.fc33.x86_64"
-    },
-    "vim-minimal": {
-      "evra": "2:8.2.2143-1.fc33.x86_64"
-    },
-    "which": {
-      "evra": "2.21-20.fc33.x86_64"
-    },
-    "wireguard-tools": {
-      "evra": "1.0.20200827-2.fc33.x86_64"
-    },
-    "xfsprogs": {
-      "evra": "5.7.0-1.fc33.x86_64"
-    },
-    "xz": {
-      "evra": "5.2.5-3.fc33.x86_64"
-    },
-    "xz-libs": {
-      "evra": "5.2.5-3.fc33.x86_64"
-    },
-    "yajl": {
-      "evra": "2.1.0-15.fc33.x86_64"
-    },
-    "zchunk-libs": {
-      "evra": "1.1.5-3.fc33.x86_64"
-    },
-    "zincati": {
-      "evra": "0.0.14-1.fc33.x86_64"
-    },
-    "zlib": {
-      "evra": "1.2.11-23.fc33.x86_64"
-    },
-    "zram-generator": {
-      "evra": "0.2.0-4.fc33.x86_64"
-    }
-  },
-  "metadata": {
-    "generated": "2020-12-17T21:07:34Z",
-    "rpmmd_repos": {
-      "fedora": {
-        "generated": "2020-10-19T23:27:19Z"
-      },
-      "fedora-coreos-pool": {
-        "generated": "2020-12-16T21:54:56Z"
-      },
-      "fedora-updates": {
-        "generated": "2020-12-17T00:56:01Z"
-      }
-    }
-  }
-}
diff --git a/manifest.yaml b/manifest.yaml
index aaf361dd5b..20e42d6202 100644
--- a/manifest.yaml
+++ b/manifest.yaml
@@ -1,35 +1,15 @@
-ref: fedora/${basearch}/coreos/testing-devel
-include: manifests/fedora-coreos.yaml
-
-releasever: "33"
+variables:
+  stream: rawhide
+  prod: false
 
-rojig:
-  license: MIT
-  name: fedora-coreos
-  summary: Fedora CoreOS testing-devel
+releasever: 42
 
 repos:
-  # these repos are there to make it easier to add new packages to the OS and to
-  # use `cosa fetch --update-lockfile`; but note that all package versions are
-  # still pinned
-  - fedora
-  - fedora-updates
+  - fedora-rawhide
 
-# All Fedora CoreOS streams share the same pool for locked files.
-# This will be in fedora-coreos.yaml in the future so it can be more easily be
-# shared between all the streams
-lockfile-repos:
-  - fedora-coreos-pool
-
-add-commit-metadata:
-  fedora-coreos.stream: testing-devel
+include: manifests/fedora-coreos.yaml
 
-postprocess:
-  # Disable Zincati and fedora-coreos-pinger on non-production streams
-  # https://github.com/coreos/fedora-coreos-tracker/issues/163
-  - |
-    #!/usr/bin/env bash
-    set -xeuo pipefail
-    mkdir -p /etc/fedora-coreos-pinger/config.d /etc/zincati/config.d
-    echo -e '# https://github.com/coreos/fedora-coreos-tracker/issues/163\nreporting.enabled = false' > /etc/fedora-coreos-pinger/config.d/90-disable-on-non-production-stream.toml
-    echo -e '# https://github.com/coreos/fedora-coreos-tracker/issues/163\nupdates.enabled = false' > /etc/zincati/config.d/90-disable-on-non-production-stream.toml
+# Ship this in rawhide because we want to enable it in the future, also
+# to shake out any integration issues.
+packages:
+  - bootc
diff --git a/manifests/bootable-rpm-ostree.yaml b/manifests/bootable-rpm-ostree.yaml
index 809cc77af6..9d5faddbec 100644
--- a/manifests/bootable-rpm-ostree.yaml
+++ b/manifests/bootable-rpm-ostree.yaml
@@ -1,15 +1,19 @@
-# This minimal base starts just from: kernel + systemd + rpm-ostree + bootloader.
+# This minimal base is the userspace: systemd + rpm-ostree + bootloader.
 # The intent of this is to inherit from this if you are doing something highly
 # custom that e.g. might not involve Ignition or podman, but you do want
 # rpm-ostree.
 # We expect most people though using coreos-assembler to inherit from
 # fedora-coreos-base.yaml.
 packages:
- # Kernel + systemd.  Note we explicitly specify kernel-{core,modules}
- # because otherwise depsolving could bring in kernel-debug.
- - kernel kernel-core kernel-modules systemd
+ - systemd
+ # linux-firmware now a recommends so let's explicitly include it
+ # https://gitlab.com/cki-project/kernel-ark/-/commit/32271d0cd9bd52d386eb35497c4876a8f041f70b
+ # https://src.fedoraproject.org/rpms/kernel/c/f55c3e9ed8605ff28cb9a922efbab1055947e213?branch=rawhide
+ - linux-firmware
  # rpm-ostree
  - rpm-ostree nss-altfiles
+ # firmware updates
+ - fwupd
 
 # bootloader
 packages-aarch64:
@@ -20,6 +24,14 @@ packages-s390x:
   # On Fedora, this is provided by s390utils-core. on RHEL, this is for now
   # provided by s390utils-base, but soon will be -core too.
   - /usr/sbin/zipl
+  # for Secure Execution
+  - veritysetup
 packages-x86_64:
   - grub2 grub2-efi-x64 efibootmgr shim
   - microcode_ctl
+
+exclude-packages:
+  # Exclude kernel-debug-core to make sure that it doesn't somehow get
+  # chosen as the package to satisfy the `kernel-core` dependency from
+  # the kernel package.
+  - kernel-debug-core
diff --git a/manifests/bootupd.yaml b/manifests/bootupd.yaml
index ca8c5de64e..fe4aa4bed7 100644
--- a/manifests/bootupd.yaml
+++ b/manifests/bootupd.yaml
@@ -7,7 +7,5 @@ postprocess:
   - |
     #!/bin/bash
     set -xeuo pipefail
-    # Until we have https://github.com/coreos/rpm-ostree/pull/2275
-    mkdir -p /run
     # Transforms /usr/lib/ostree-boot into a bootupd-compatible update payload
-    /usr/bin/bootupctl backend generate-update-metadata /
+    /usr/bin/bootupctl backend generate-update-metadata
diff --git a/manifests/composefs.yaml b/manifests/composefs.yaml
new file mode 100644
index 0000000000..d575c1349b
--- /dev/null
+++ b/manifests/composefs.yaml
@@ -0,0 +1,3 @@
+# Enable composefs by default.
+ostree-layers:
+  - overlay/08composefs
diff --git a/manifests/disable-zincati.yaml b/manifests/disable-zincati.yaml
new file mode 100644
index 0000000000..e7c58fdf2a
--- /dev/null
+++ b/manifests/disable-zincati.yaml
@@ -0,0 +1,2 @@
+ostree-layers:
+  - overlay/16disable-zincati
diff --git a/manifests/exclude-dnf.yaml b/manifests/exclude-dnf.yaml
new file mode 100644
index 0000000000..dc51914b9f
--- /dev/null
+++ b/manifests/exclude-dnf.yaml
@@ -0,0 +1,3 @@
+exclude-packages:
+  - dnf
+  - dnf5
diff --git a/manifests/fedora-coreos-base.yaml b/manifests/fedora-coreos-base.yaml
index 1aa8274099..24fc8017c7 100644
--- a/manifests/fedora-coreos-base.yaml
+++ b/manifests/fedora-coreos-base.yaml
@@ -3,20 +3,23 @@
 # core functionality.
 
 include:
+  - kernel.yaml
+  - system-configuration.yaml
   - ignition-and-ostree.yaml
   - file-transfer.yaml
-
-initramfs-args:
-  - --no-hostonly
-  # We don't support root on NFS, so we don't need it in the initramfs. It also
-  # conflicts with /var mount support in ignition because NFS tries to mount stuff
-  # in /var/ and then ignition can't cleanly unmount it. For example:
-  # https://github.com/dracutdevs/dracut/blob/1856ae95c873a6fe855b3dccd0144f1a96b9e71c/modules.d/95nfs/nfs-start-rpc.sh#L7
-  # See also discussion in https://github.com/coreos/fedora-coreos-config/pull/60
-  - --omit=nfs
-  # Omit these since we don't use them
-  - --omit=lvm
-  - --omit=iscsi
+  - networking-tools.yaml
+  - user-experience.yaml
+  - shared-workarounds.yaml
+  # See https://github.com/coreos/bootupd
+  - bootupd.yaml
+ostree-layers:
+  - overlay/05core
+  - overlay/08nouveau
+  - overlay/09misc
+  - overlay/20platform-chrony
+  - overlay/25azure-udev-rules
+  - overlay/30lvmdevices
+  - overlay/40grub
 
 # Be minimal
 recommends: false
@@ -40,19 +43,12 @@ check-groups:
 
 default-target: multi-user.target
 
+# we can drop this when it's the rpm-ostree default
+rpmdb: sqlite
+
 # ⚠⚠⚠ ONLY TEMPORARY HACKS ALLOWED HERE; ALL ENTRIES NEED TRACKER LINKS ⚠⚠⚠
 # See also the version of this in fedora-coreos.yaml
 postprocess:
-  # This will be dropped once rpm-ostree because module-aware.
-  # https://github.com/projectatomic/rpm-ostree/issues/1542#issuecomment-419684977
-  # https://github.com/projectatomic/rpm-ostree/issues/1435
-  - |
-    #!/usr/bin/env bash
-    set -xeuo pipefail
-    for x in /etc/yum.repos.d/*modular.repo; do
-      sed -i -e 's,enabled=[01],enabled=0,' ${x}
-    done
-
   # Enable SELinux booleans used by OpenShift
   # https://github.com/coreos/fedora-coreos-tracker/issues/284
   - |
@@ -68,134 +64,150 @@ postprocess:
   - |
     #!/usr/bin/env bash
     systemctl mask dnsmasq.service
-  # Mask systemd-repart. Ignition is responsible for partition setup on first
-  # boot and does not use systemd-repart currently. See also
-  # https://github.com/coreos/fedora-coreos-tracker/issues/570
+
+  # Default to iptables-nft. Otherwise, legacy wins. We can drop this once/if we
+  # remove iptables-legacy. This is needed because alternatives don't work
+  # https://github.com/coreos/fedora-coreos-tracker/issues/677
+  # https://github.com/coreos/fedora-coreos-tracker/issues/676
   - |
     #!/usr/bin/env bash
-    systemctl mask systemd-repart.service
+    set -xeuo pipefail
+    ln -sf /usr/sbin/ip6tables-nft         /etc/alternatives/ip6tables
+    ln -sf /usr/sbin/ip6tables-nft-restore /etc/alternatives/ip6tables-restore
+    ln -sf /usr/sbin/ip6tables-nft-save    /etc/alternatives/ip6tables-save
+    ln -sf /usr/sbin/iptables-nft          /etc/alternatives/iptables
+    ln -sf /usr/sbin/iptables-nft-restore  /etc/alternatives/iptables-restore
+    ln -sf /usr/sbin/iptables-nft-save     /etc/alternatives/iptables-save
 
-  # Neuter systemd-resolved for now.
-  # https://github.com/coreos/fedora-coreos-tracker/issues/649#issuecomment-743219353
-  # Note: When removing this, we likely also want to remove
-  # coreos-reset-stub-resolv-selinux-context.{path,service} and their presets.
+  # Force the ssh-host-keys-migration to happen on every boot
+  # to handle cases where someone did a upgrade->rollback->upgrade
+  # See https://github.com/coreos/fedora-coreos-tracker/issues/1473
+  # We should remove this after the next barrier release.
   - |
     #!/usr/bin/env bash
     set -xeuo pipefail
-    # Get us back to Fedora 32's nsswitch.conf settings
-    sed -i 's/^hosts:.*/hosts:      files dns myhostname/' /etc/nsswitch.conf
-    mkdir /usr/lib/systemd/resolved.conf.d/
-    cat > /usr/lib/systemd/resolved.conf.d/fedora-coreos-stub-listener.conf <<'EOF'
-    # Fedora CoreOS is electing to not use systemd-resolved's internal
-    # logic for now because of issues with setting hostnames via reverse DNS.
-    # https://github.com/coreos/fedora-coreos-tracker/issues/649#issuecomment-736104003
-    [Resolve]
-    DNSStubListener=no
+    mkdir -p /usr/lib/systemd/system/ssh-host-keys-migration.service.d
+    cat <<'EOF' > /usr/lib/systemd/system/ssh-host-keys-migration.service.d/coreos-force-migration-on-every-boot.conf
+    # Force the ssh-host-keys-migration to happen on every boot
+    # to handle cases where someone did a upgrade->rollback->upgrade
+    # See https://github.com/coreos/fedora-coreos-tracker/issues/1473
+    [Unit]
+    ConditionPathExists=
     EOF
 
+# Packages listed here should be specific to Fedore CoreOS (as in not yet
+# available in RHCOS or not desired in RHCOS). All other packages should go
+# into one of the sub-manifests listed at the top.
 packages:
   # Security
-  - selinux-policy-targeted
   - polkit
-  # System setup
-  - afterburn
-  - afterburn-dracut
-  - passwd
-  # Dependency of 20live dracut module
-  - bsdtar
-  # SSH
-  - openssh-server openssh-clients
-  - ssh-key-dir
   # Containers
-  - podman skopeo runc systemd-container catatonit
+  - systemd-container catatonit
   - fuse-overlayfs slirp4netns
-  # name resolution for podman containers
+  # Some host applications(i.e. NetworkManager) use dnsmasq
+  # as the binary for some various utility operations.
   # https://github.com/coreos/fedora-coreos-tracker/issues/519
-  - podman-plugins dnsmasq
-  # Remote IPC for podman
-  - libvarlink-util
-  # Networking
+  - dnsmasq
+  # For podman v4 netavark gets pulled in but it only recommends
+  # aardvark-dns (which provides name resolution based on container
+  # names). This functionality was previously provided by dnsname from
+  # podman-plugins in the podman v3 stack.
+  # See https://github.com/containers/netavark/pull/217
+  - aardvark-dns
+  # Since we need `containernetworking-plugins` installed to continue
+  # to support CNI networks we need to also explicitly install
+  # `netavark` so we get both of them installed since both of them
+  # provide `container-network-stack`.
+  # https://github.com/coreos/fedora-coreos-tracker/issues/1128#issuecomment-1071458717
+  - netavark
+  # Minimal NFS client
   - nfs-utils-coreos
-  - NetworkManager hostname
-  - iproute-tc
+  # Active Directory support
   - adcli
-  ## Teaming https://github.com/coreos/fedora-coreos-config/pull/289 and http://bugzilla.redhat.com/1758162
-  - NetworkManager-team teamd
-  # Static firewalling
-  - iptables nftables iptables-nft iptables-services
-  # Interactive Networking configuration during coreos-install
-  - NetworkManager-tui
+  # Additional firewall support; we aren't including these in RHCOS or they
+  # don't exist in RHEL
+  - iptables-nft iptables-services
   # WireGuard https://github.com/coreos/fedora-coreos-tracker/issues/362
   - wireguard-tools
   # Storage
-  - cloud-utils-growpart
-  - lvm2 iscsi-initiator-utils sg3_utils
-  - device-mapper-multipath
-  - xfsprogs e2fsprogs btrfs-progs mdadm
-  - cryptsetup
-  - cifs-utils
-  # Time sync
-  - chrony
+  - btrfs-progs
+  - WALinuxAgent-udev
   # Allow communication between sudo and SSSD
   # for caching sudo rules by SSSD.
   # https://github.com/coreos/fedora-coreos-tracker/issues/445
   - libsss_sudo
-  # Extra runtime
-  - sssd shadow-utils
-  # There are things that write outside of the journal still (such as the classic wtmp, etc.)
-  # (auditd also writes outside the journal but it has its own log rotation.)
-  # Anything package layered will also tend to expect files dropped in
-  # /etc/logrotate.d to work.  Really, this is a legacy thing, but if we don't
-  # have it then people's disks will slowly fill up with logs.
-  - logrotate
+  # SSSD; we only ship a subset of the backends
+  - sssd-client sssd-ad sssd-ipa sssd-krb5 sssd-ldap
   # Used by admins interactively
-  - sudo coreutils attr less tar xz gzip bzip2
-  - socat net-tools bind-utils
-  - bash-completion
+  - attr
   - openssl
-  - vim-minimal
-  - lsof
   # Provides terminal tools like clear, reset, tput, and tset
   - ncurses
   # file-transfer: note fuse-sshfs is not in RHEL
   # so we can't put it in file-transfer.yaml
   - fuse-sshfs
-  # User experience
-  - console-login-helper-messages-issuegen
+  # Improved MOTD experience
   - console-login-helper-messages-motdgen
-  - console-login-helper-messages-profile
-  - toolbox
-  # CoreOS Installer
-  - coreos-installer coreos-installer-bootinfra
   # i18n
   - kbd
-  # Parsing/Interacting with JSON data
-  - jq
-  # nvme-cli for managing nvme disks
-  - nvme-cli
-  # zram-generator (but not zram-generator-defaults) for F33 change
-  # https://github.com/coreos/fedora-coreos-tracker/issues/509
-  - zram-generator
-  # kdump (https://github.com/coreos/fedora-coreos-tracker/issues/622)
-  - kexec-tools
+  # resolved was broken out to its own package in rawhide/f35
+  - systemd-resolved
+  # In F35+ need `iptables-legacy` package
+  # See https://github.com/coreos/fedora-coreos-tracker/issues/676#issuecomment-928028451
+  - iptables-legacy
+  # GPU Firmware files (not broken out into subpackage of linux-firmware in RHEL yet)
+  - amd-gpu-firmware intel-gpu-firmware nvidia-gpu-firmware
+  # NIC firmware we've traditionally shipped but then were split out of linux-firmware in Fedora
+  - qed-firmware # https://github.com/coreos/fedora-coreos-tracker/issues/1746
+  # makedumpfile and kexec were split into subpackages in kexec-tools 0.2.28-7
+  # which is not in RHCOS yet
+  # These should be with kexec-tools in the user-experience manifest, move them
+  # once the split lands in RHCOS.
+  - makedumpfile
+  - kdump-utils
+
 
-# This thing is crying out to be pulled into systemd, but that hasn't happened
-# yet.  Also we may want to add to rpm-ostree something like arch negation;
-# basically right now it doesn't exist on s390x.
-# Anyways, it was requested by the Red Hat perf team for RHCOS, so we have it here.
-# https://serverfault.com/questions/513807/is-there-still-a-use-for-irqbalance-on-modern-hardware
-# https://access.redhat.com/solutions/41535
+# - irqbalance
+#   - This thing is crying out to be pulled into systemd, but that hasn't happened
+#     yet.  Also we may want to add to rpm-ostree something like arch negation;
+#     basically right now it doesn't exist on s390x.
+#     Anyways, it was requested by the Red Hat perf team for RHCOS, so we have it here.
+#     https://serverfault.com/questions/513807/is-there-still-a-use-for-irqbalance-on-modern-hardware
+#     https://access.redhat.com/solutions/41535
+# - qemu-user-static-x86
+#   - Include this on non-x86_64 FCOS images to allow access to the large
+#     inventory of containers only built for x86_64.
+#     https://github.com/coreos/fedora-coreos-tracker/issues/1237
+# - google-compute-engine-guest-configs-udev
+#   - Add this package on x86_64 and aarch64 (the two architectures
+#     GCP supports. https://github.com/coreos/fedora-coreos-tracker/issues/1494
+#     This should be moved to a shared manifest when RHEL has this package.
+# - crun-wasm wasmedge-rt
+#   - Support for wasm runtime: https://github.com/coreos/fedora-coreos-tracker/issues/1375
 packages-x86_64:
   - irqbalance
+  - google-compute-engine-guest-configs-udev
+  - crun-wasm wasmedge-rt
+  # Include AMD microcode updates, see https://github.com/coreos/fedora-coreos-tracker/issues/1618.
+  # This normally should belong in bootable-rpm-ostree.yaml (alongside
+  # `microcode_ctl`), but this change hasn't hit RHCOS yet.
+  - amd-ucode-firmware
 packages-ppc64le:
   - irqbalance
   - librtas
   - powerpc-utils-core
   - ppc64-diag-rtas
+  - qemu-user-static-x86
 packages-aarch64:
   - irqbalance
+  - qemu-user-static-x86
+  - google-compute-engine-guest-configs-udev
+  - crun-wasm wasmedge-rt
+packages-s390x:
+  - qemu-user-static-x86
 
-# See https://github.com/coreos/bootupd
-arch-include:
-  x86_64: bootupd.yaml
-  aarch64: bootupd.yaml
+remove-from-packages:
+  # Hopefully short-term hack -- see https://github.com/coreos/fedora-coreos-config/pull/1206#discussion_r705425869.
+  # This keeps the size down and ensures nothing tries to use it, preventing us
+  # from shedding the dep eventually.
+  - [cracklib-dicts, .*]
diff --git a/manifests/fedora-coreos.yaml b/manifests/fedora-coreos.yaml
index c92effcd71..047f86292a 100644
--- a/manifests/fedora-coreos.yaml
+++ b/manifests/fedora-coreos.yaml
@@ -2,26 +2,66 @@
 # into "official" builds of Fedora CoreOS (such as including `fedora-release-coreos`)
 # or are very "opinionated" like disabling SSH passwords by default.
 
+ref: fedora/${basearch}/coreos/${stream}
+rojig:
+  license: MIT
+  name: fedora-coreos
+  summary: Fedora CoreOS ${stream}
+
+add-commit-metadata:
+  fedora-coreos.stream: ${stream}
+
+
 include: fedora-coreos-base.yaml
+conditional-include:
+  - if: prod == false
+    # long-term, would be good to support specifying a nested TreeComposeConfig
+    include: disable-zincati.yaml
+  - if: basearch != "s390x"
+    # And remove some cruft from grub2
+    include: grub2-removals.yaml
+  # On <41, we want to keep making sure dnf doesn't slip in somehow
+  # On 41+, we do want it
+  # https://github.com/coreos/fedora-coreos-tracker/issues/1687
+  - if: releasever < 41
+    include: exclude-dnf.yaml
+  - if: releasever >= 41
+    include: include-dnf.yaml
+  # Wifi firmwares will be dropped in F41
+  - if: releasever < 41
+    include: wifi-firmwares.yaml
+  - if: releasever >= 41
+    include: composefs.yaml
+  - if: releasever >= 41
+    include: selinux-workaround.yaml
+
+ostree-layers:
+  - overlay/15fcos
 
 automatic-version-prefix: "${releasever}.<date:%Y%m%d>.dev"
 mutate-os-release: "${releasever}"
 
+# All Fedora CoreOS streams share the same pool for locked files.
+lockfile-repos:
+  - fedora-coreos-pool
+
 packages:
   - fedora-release-coreos
   - fedora-repos-ostree
-  # fedora-repos-modular was converted into its own subpackage in f33
-  # Continue to include it in case users want to use it.
-  - fedora-repos-modular
   # the archive repo for more reliable package layering
   # https://github.com/coreos/fedora-coreos-tracker/issues/400
   - fedora-repos-archive
   # CL ships this.
   - moby-engine
-  # User metrics
-  - fedora-coreos-pinger
+  # Already pulled in by moby-engine, but let's be explicit. Typhoon uses it.
+  - containerd
   # Updates
   - zincati
+  # Include and set the default editor
+  - nano nano-default-editor
+  # Introduce a default colored prompt for Fedora's default shell bash.
+  # https://github.com/coreos/fedora-coreos-tracker/issues/1567
+  - bash-color-prompt
 
 etc-group-members:
   # Add the docker group to /etc/group
@@ -30,24 +70,17 @@ etc-group-members:
   # https://github.com/projectatomic/rpm-ostree/issues/49
   - docker
 
-# XXX: this is used by coreos-assembler for artifact naming...
-rojig:
-  license: MIT
-  name: fedora-coreos
-  summary: Fedora CoreOS base image
-
 # ⚠⚠⚠ ONLY TEMPORARY HACKS ALLOWED HERE; ALL ENTRIES NEED TRACKER LINKS ⚠⚠⚠
 # See also the version of this in fedora-coreos-base.yaml
 postprocess:
-  # Disable Zincati and fedora-coreos-pinger on non-release builds
+  # Disable Zincati on non-release builds
   # https://github.com/coreos/fedora-coreos-tracker/issues/212
   - |
     #!/usr/bin/env bash
-    set -xeuo pipefail
+    set -euxo pipefail
     source /etc/os-release
     if [[ $OSTREE_VERSION = *.dev* ]]; then
-      mkdir -p /etc/fedora-coreos-pinger/config.d /etc/zincati/config.d
-      echo -e '# https://github.com/coreos/fedora-coreos-tracker/issues/212\nreporting.enabled = false' > /etc/fedora-coreos-pinger/config.d/95-disable-on-dev.toml
+      mkdir -p /etc/zincati/config.d
       echo -e '# https://github.com/coreos/fedora-coreos-tracker/issues/212\nupdates.enabled = false' > /etc/zincati/config.d/95-disable-on-dev.toml
     fi
   # Users shouldn't be configuring `rpm-ostreed.conf`
@@ -66,11 +99,48 @@ postprocess:
     cat /usr/etc/rpm-ostreed.conf >> /tmp/rpm-ostreed.conf
     cp /tmp/rpm-ostreed.conf /usr/etc/rpm-ostreed.conf
     rm /tmp/rpm-ostreed.conf
+  # Make sure that we do not ship broken symlinks:
+  # https://github.com/coreos/fedora-coreos-config/issues/1782
+  # Remove known broken symlinks that point to non-existing files or directories:
+  # - Remove `.build-id` for binaries that we remove in other parts of the FCOS manifest
+  # - Remove links to man pages that we remove in FCOS
+  # Man pages are removed in FCOS thus the links in alternatives pointing to those are left there broken.
+  # Docs removal comes from manifests/fedora-coreos.yaml
+  # - systemd-firstboot comes from manifests/ignition-and-ostree.yaml
+  # - systemd-gpt-auto-generator comes from ignition-and-ostree.yaml
+  - |
+    #!/usr/bin/env bash
+    set -euo pipefail
+
+    list_broken_symlinks_folders=(
+      '/etc/alternatives/'
+      '/usr/lib/.build-id/'
+    )
+
+    # It is not possible to remove files from usr after first boot so that is
+    # why we are removing them in the postprocess scripts here.
+    # The .build-id links are pointing to binaries that we remove in other parts of the FCOS manifest.
+    list_known_removed_folders=(
+      '/usr/bin/systemd-firstboot'
+      '/usr/lib/systemd/system-generators/systemd-gpt-auto-generator'
+      '/usr/share/doc/'
+      '/usr/share/info/'
+      '/usr/share/man/'
+      )
+    for folder in "${list_broken_symlinks_folders[@]}"; do
+        find "${folder}" -type l | while read -r file_name; do
+            real_path=$(realpath -m "${file_name}");
+            if [[ -e "${real_path}" ]]; then
+              continue
+            fi
+            for element in "${list_known_removed_folders[@]}"; do
+              if [[ "${real_path}" == "${element}"* ]]; then
+                  rm -r "${file_name}"
+              fi
+            done
+        done
+    done
 
-remove-from-packages:
-  # Drop NetworkManager support for ifcfg files, see also corresponding
-  # overlay.d/14NetworkManager-plugins
-  - [NetworkManager, /usr/lib64/NetworkManager/.*/libnm-settings-plugin-ifcfg-rh.so]
 
 remove-files:
   # We don't ship man(1) or info(1)
@@ -85,18 +155,22 @@ remove-files:
 exclude-packages:
   - python
   - python2
+  - python2-libs
   - python3
+  - python3-libs
   - perl
+  - perl-interpreter
   - nodejs
-  - dnf
   - grubby
   - cowsay  # Just in case
   # Let's make sure initscripts doesn't get pulled back in
   # https://github.com/coreos/fedora-coreos-tracker/issues/220#issuecomment-611566254
   - initscripts
-
-# And remove some cruft from grub2
-arch-include:
-  x86_64: grub2-removals.yaml
-  aarch64: grub2-removals.yaml
-  ppc64le: grub2-removals.yaml
+  # nor /usr/sbin/service
+  - initscripts-service
+  # For (datacenter/cloud oriented) servers, we want to see the details by default.
+  # https://lists.fedoraproject.org/archives/list/devel@lists.fedoraproject.org/thread/HSMISZ3ETWQ4ETVLWZQJ55ARZT27AAV3/
+  - plymouth
+  # Do not use legacy ifcfg config format in NetworkManager
+  # See https://github.com/coreos/fedora-coreos-config/pull/1991
+  - NetworkManager-initscripts-ifcfg-rh
diff --git a/manifests/file-transfer.yaml b/manifests/file-transfer.yaml
index 64ae36782c..88e27b90eb 100644
--- a/manifests/file-transfer.yaml
+++ b/manifests/file-transfer.yaml
@@ -3,3 +3,6 @@ packages:
   - git-core
   - gnupg2
   - rsync
+  # Explicit dependency on curl because we use it in coreos-livepxe-rootfs.sh
+  # We need curl and not curl-minimal because we support TFTP.
+  - curl
diff --git a/manifests/group b/manifests/group
index 3ce8c357be..2fd197cb7c 100644
--- a/manifests/group
+++ b/manifests/group
@@ -16,35 +16,29 @@ sudo:x:16:
 dialout:x:18:
 floppy:x:19:
 games:x:20:
-tape:x:30:
+tape:x:33:
 video:x:39:
 ftp:x:50:
 lock:x:54:
 audio:x:63:
 nobody:x:99:
 users:x:100:
-utmp:x:22:
-utempter:x:35:
 ssh_keys:x:999:
 systemd-journal:x:190:
-dbus:x:81:
 polkitd:x:998:
 etcd:x:997:
 dip:x:40:
 cgred:x:996:
-tss:x:59:
 avahi-autoipd:x:170:
-rpc:x:32:
 sssd:x:993:
 dockerroot:x:986:
 rpcuser:x:29:
 nfsnobody:x:65534:
 kube:x:994:
-sshd:x:74:
 chrony:x:992:
 tcpdump:x:72:
 ceph:x:167:
-input:x:995:
+input:x:104:
 systemd-timesync:x:991:
 systemd-network:x:990:
 systemd-resolve:x:989:
diff --git a/manifests/ignition-and-ostree.yaml b/manifests/ignition-and-ostree.yaml
index 3ac0667137..8416a6dbfc 100644
--- a/manifests/ignition-and-ostree.yaml
+++ b/manifests/ignition-and-ostree.yaml
@@ -8,10 +8,6 @@
 # Include rpm-ostree + kernel + bootloader
 include: bootable-rpm-ostree.yaml
 
-initramfs-args:
-  # make it a hard error if Ignition can't be included
-  - --add=ignition
-
 # Modern defaults we want
 boot-location: modules
 tmp-is-dir: true
@@ -27,7 +23,8 @@ packages:
 
 remove-from-packages:
   # We don't want systemd-firstboot.service. It conceptually conflicts with
-  # Ignition.
+  # Ignition.  We also inject runtime bits to disable it in systemd-firstboot.service.d/fcos-disable.conf
+  # to make it easier to use systemd builds from git.
   - [systemd, /usr/bin/systemd-firstboot,
               /usr/lib/systemd/system/systemd-firstboot.service,
               /usr/lib/systemd/system/sysinit.target.wants/systemd-firstboot.service]
diff --git a/manifests/include-dnf.yaml b/manifests/include-dnf.yaml
new file mode 100644
index 0000000000..2c023b112d
--- /dev/null
+++ b/manifests/include-dnf.yaml
@@ -0,0 +1,2 @@
+packages:
+  - dnf5
diff --git a/manifests/kernel.yaml b/manifests/kernel.yaml
new file mode 100644
index 0000000000..1b1f1a4e29
--- /dev/null
+++ b/manifests/kernel.yaml
@@ -0,0 +1,3 @@
+packages:
+  # We use the default kernel package, but note c9s may differ
+  - kernel
diff --git a/manifests/networking-tools.yaml b/manifests/networking-tools.yaml
new file mode 100644
index 0000000000..cbbf77caf8
--- /dev/null
+++ b/manifests/networking-tools.yaml
@@ -0,0 +1,27 @@
+# This defines a set of tools that are useful for configuring, debugging,
+# or manipulating the network of a system.  It is desired to keep this list
+# generic enough to be shared downstream with RHCOS.
+
+packages:
+  # Standard tools for configuring network/hostname
+  - NetworkManager hostname
+  # Interactive Networking configuration during coreos-install
+  - NetworkManager-tui
+  # Teaming https://github.com/coreos/fedora-coreos-config/pull/289
+  # and http://bugzilla.redhat.com/1758162
+  - NetworkManager-team teamd
+  # Support for cloud quirks and dynamic config in real rootfs:
+  # https://github.com/coreos/fedora-coreos-tracker/issues/320
+  - NetworkManager-cloud-setup
+  # Route manipulation and QoS
+  - iproute iproute-tc
+  # Firewall manipulation
+  - iptables nftables
+  # Interactive network tools for admins
+  - socat net-tools bind-utils
+  # Declarative network configuration
+  # https://github.com/coreos/fedora-coreos-tracker/issues/1175
+  - nmstate
+  # Advanced custom networking calculations
+  # https://github.com/coreos/fedora-coreos-tracker/issues/1460
+  - ipcalc
diff --git a/manifests/passwd b/manifests/passwd
index b05ebdbfe8..ea84802caa 100644
--- a/manifests/passwd
+++ b/manifests/passwd
@@ -1,33 +1,32 @@
-root:x:0:0:root:/root:/bin/bash
-bin:x:1:1:bin:/bin:/sbin/nologin
-daemon:x:2:2:daemon:/sbin:/sbin/nologin
-adm:x:3:4:adm:/var/adm:/sbin/nologin
-lp:x:4:7:lp:/var/spool/lpd:/sbin/nologin
-sync:x:5:0:sync:/sbin:/bin/sync
-shutdown:x:6:0:shutdown:/sbin:/sbin/shutdown
+adm:x:3:4:adm:/var/adm:/usr/sbin/nologin
+avahi-autoipd:x:170:170:Avahi IPv4LL Stack:/var/lib/avahi-autoipd:/usr/sbin/nologin
+bin:x:1:1:bin:/bin:/usr/sbin/nologin
+ceph:x:167:167:Ceph daemons:/var/lib/ceph:/usr/sbin/nologin
+chrony:x:994:992::/var/lib/chrony:/usr/sbin/nologin
+cockpit-ws:x:988:987:User for cockpit-ws:/:/usr/sbin/nologin
+daemon:x:2:2:daemon:/sbin:/usr/sbin/nologin
+dbus:x:81:81:System Message Bus:/:/usr/sbin/nologin
+dockerroot:x:997:986:Docker User:/var/lib/docker:/usr/sbin/nologin
+etcd:x:998:997:etcd user:/var/lib/etcd:/usr/sbin/nologin
+ftp:x:14:50:FTP User:/var/ftp:/usr/sbin/nologin
+games:x:12:100:games:/usr/games:/usr/sbin/nologin
 halt:x:7:0:halt:/sbin:/sbin/halt
-mail:x:8:12:mail:/var/spool/mail:/sbin/nologin
-operator:x:11:0:operator:/root:/sbin/nologin
-games:x:12:100:games:/usr/games:/sbin/nologin
-ftp:x:14:50:FTP User:/var/ftp:/sbin/nologin
-nobody:x:99:99:Nobody:/:/sbin/nologin
-dbus:x:81:81:System message bus:/:/sbin/nologin
-polkitd:x:999:998:User for polkitd:/:/sbin/nologin
-etcd:x:998:997:etcd user:/var/lib/etcd:/sbin/nologin
-tss:x:59:59:Account used by the trousers package to sandbox the tcsd daemon:/dev/null:/sbin/nologin
-avahi-autoipd:x:170:170:Avahi IPv4LL Stack:/var/lib/avahi-autoipd:/sbin/nologin
-rpc:x:32:32:Rpcbind Daemon:/var/lib/rpcbind:/sbin/nologin
-sssd:x:995:993:User for sssd:/:/sbin/nologin
-dockerroot:x:997:986:Docker User:/var/lib/docker:/sbin/nologin
-rpcuser:x:29:29:RPC Service User:/var/lib/nfs:/sbin/nologin
-nfsnobody:x:65534:65534:Anonymous NFS User:/var/lib/nfs:/sbin/nologin
-kube:x:996:994:Kubernetes user:/:/sbin/nologin
-sshd:x:74:74:Privilege-separated SSH:/var/empty/sshd:/sbin/nologin
-chrony:x:994:992::/var/lib/chrony:/sbin/nologin
-tcpdump:x:72:72::/:/sbin/nologin
-ceph:x:167:167:Ceph daemons:/var/lib/ceph:/sbin/nologin
-systemd-timesync:x:993:991:systemd Time Synchronization:/:/sbin/nologin
-systemd-network:x:991:990:systemd Network Management:/:/sbin/nologin
-systemd-resolve:x:990:989:systemd Resolver:/:/sbin/nologin
-systemd-bus-proxy:x:989:988:systemd Bus Proxy:/:/sbin/nologin
-cockpit-ws:x:988:987:User for cockpit-ws:/:/sbin/nologin
+kube:x:996:994:Kubernetes user:/:/usr/sbin/nologin
+lp:x:4:7:lp:/var/spool/lpd:/usr/sbin/nologin
+mail:x:8:12:mail:/var/spool/mail:/usr/sbin/nologin
+nfsnobody:x:65534:65534:Anonymous NFS User:/var/lib/nfs:/usr/sbin/nologin
+nobody:x:99:99:Kernel Overflow User:/:/usr/sbin/nologin
+operator:x:11:0:operator:/root:/usr/sbin/nologin
+polkitd:x:999:998:User for polkitd:/:/usr/sbin/nologin
+root:x:0:0:Super User:/root:/bin/bash
+rpc:x:32:32:Rpcbind Daemon:/var/lib/rpcbind:/usr/sbin/nologin
+rpcuser:x:29:29:RPC Service User:/var/lib/nfs:/usr/sbin/nologin
+shutdown:x:6:0:shutdown:/sbin:/sbin/shutdown
+sshd:x:74:74:Privilege-separated SSH:/var/empty/sshd:/usr/sbin/nologin
+sssd:x:995:993:User for sssd:/:/usr/sbin/nologin
+sync:x:5:0:sync:/sbin:/bin/sync
+systemd-bus-proxy:x:989:988:systemd Bus Proxy:/:/usr/sbin/nologin
+systemd-network:x:991:990:systemd Network Management:/:/usr/sbin/nologin
+systemd-resolve:x:990:989:systemd Resolver:/:/usr/sbin/nologin
+systemd-timesync:x:993:991:systemd Time Synchronization:/:/usr/sbin/nologin
+tcpdump:x:72:72::/:/usr/sbin/nologin
diff --git a/manifests/selinux-workaround.yaml b/manifests/selinux-workaround.yaml
new file mode 100644
index 0000000000..391e784e36
--- /dev/null
+++ b/manifests/selinux-workaround.yaml
@@ -0,0 +1,23 @@
+# Recent changes in the SELinux policy have broken a lot of our code.
+# Revert the affected domains back to permissive mode so we can
+# continue to build and test `releasever >= 41` until
+# https://github.com/fedora-selinux/selinux-policy/pull/2257 merges
+# and the domains are reverted upstream or until the issue is resolved
+# altogether
+postprocess:
+  - |
+    #!/usr/bin/env bash
+    set -xeuo pipefail
+    cat > /tmp/fcos-workarounds.cil << EOF
+    ; https://bugzilla.redhat.com/show_bug.cgi?id=2300306
+    (typeattributeset cil_gen_require bootupd_t)
+    (typepermissive bootupd_t)
+    ; https://bugzilla.redhat.com/show_bug.cgi?id=2305385
+    (typeattributeset cil_gen_require coreos_installer_t)
+    (typepermissive coreos_installer_t)
+    ; https://bugzilla.redhat.com/show_bug.cgi?id=2306352
+    (typeattributeset cil_gen_require afterburn_t)
+    (typepermissive afterburn_t)
+    EOF
+    /usr/sbin/semodule -i /tmp/fcos-workarounds.cil
+    rm /tmp/fcos-workarounds.cil
diff --git a/manifests/shared-workarounds.yaml b/manifests/shared-workarounds.yaml
new file mode 100644
index 0000000000..500582942b
--- /dev/null
+++ b/manifests/shared-workarounds.yaml
@@ -0,0 +1,2 @@
+# This manifest is a list of shared workarounds that are needed in both Fedora CoreOS
+# and downstreams (i.e. Red Hat CoreOS).
diff --git a/manifests/system-configuration.yaml b/manifests/system-configuration.yaml
new file mode 100644
index 0000000000..0f982b612d
--- /dev/null
+++ b/manifests/system-configuration.yaml
@@ -0,0 +1,56 @@
+# These are packages that are related to configuring parts of the system.
+# It is intended to be kept generic so that it may be shared downstream with
+# RHCOS.
+
+packages:
+  # Manage and load rules in the audit subsystem in the kernel
+  # https://github.com/coreos/fedora-coreos-tracker/issues/1362
+  - audit
+  # Configuring SSH keys, cloud provider check-in, etc
+  - afterburn afterburn-dracut
+  # NTP support
+  - chrony
+  # Installing CoreOS itself
+  - coreos-installer coreos-installer-bootinfra
+  # Storage configuration/management
+  ## cloud-utils-growpart - For growing root partition
+  - cifs-utils
+  - cloud-utils-growpart
+  - cryptsetup
+  - device-mapper-multipath
+  - e2fsprogs
+  - iscsi-initiator-utils
+  - lvm2
+  - mdadm
+  - sg3_utils
+  - xfsprogs
+  # User configuration
+  - shadow-utils
+  - acl
+  # SELinux policy
+  - selinux-policy-targeted
+  # There are things that write outside of the journal still (such as the
+  # classic wtmp, etc.). auditd also writes outside the journal but it has its
+  # own log rotation.
+  # Anything package layered will also tend to expect files dropped in
+  # /etc/logrotate.d to work. Really, this is a legacy thing, but if we don't
+  # have it then people's disks will slowly fill up with logs.
+  - logrotate
+  # Boost starving threads
+  # https://github.com/coreos/fedora-coreos-tracker/issues/753
+  - stalld
+  # Ignition aware SSH key management
+  - ssh-key-dir
+  # Allow for configuring different timezones
+  - tzdata
+  # zram-generator (but not zram-generator-defaults) for F33 change
+  # https://github.com/coreos/fedora-coreos-tracker/issues/509
+  - zram-generator
+
+postprocess:
+  # Mask systemd-repart. Ignition is responsible for partition setup on first
+  # boot and does not use systemd-repart currently. See also
+  # https://github.com/coreos/fedora-coreos-tracker/issues/570
+  - |
+    #!/usr/bin/env bash
+    systemctl mask systemd-repart.service
diff --git a/manifests/user-experience.yaml b/manifests/user-experience.yaml
new file mode 100644
index 0000000000..863fc1e5b0
--- /dev/null
+++ b/manifests/user-experience.yaml
@@ -0,0 +1,56 @@
+# This file is included in RHEL CoreOS, see
+# https://github.com/openshift/os/blob/71c974b1e456292033e3ef3fe7bcfe17d1855ebc/manifest.yaml#L12
+# Only apply changes here that should apply to both FCOS and RHCOS.
+
+# Default to `bash` in our container, the same as other containers we ship.
+container-cmd:
+  - /usr/bin/bash
+
+# These packages are either widely used utilities/services or
+# are targeted for improving the general CoreOS user experience.
+# It is intended to be kept generic so that it may be shared downstream with
+# RHCOS.
+packages:
+  # Basic user tools
+  ## jq - parsing/interacting with JSON data
+  - bash-completion
+  - coreutils
+  - file
+  - jq
+  - less
+  - sudo
+  - vim-minimal
+  # File compression/decompression
+  ## bsdtar - dependency of 35coreos-live dracut module
+  - bsdtar
+  - bzip2
+  - gzip
+  - tar
+  - xz
+  - zstd
+  # Improved MOTD experience
+  - console-login-helper-messages-issuegen
+  - console-login-helper-messages-profile
+  # kdump support
+  # https://github.com/coreos/fedora-coreos-tracker/issues/622
+  - kexec-tools
+  # Remote Access
+  - openssh-clients openssh-server
+  # Container tooling
+  ## crun recommends but doesn't require criu and criu-libs. We want them for
+  ## checkpoint/restore. https://github.com/coreos/fedora-coreos-tracker/issues/1370
+  - crun criu criu-libs
+  - podman
+  - runc
+  - skopeo
+  - toolbox
+  # passt provides user-mode networking daemons for namespaces
+  - passt
+  # nvme-cli for managing nvme disks
+  - nvme-cli
+  # Used by admins interactively
+  - lsof
+  # Locates executable files' paths in `PATH`
+  - which
+  # provides utilities for inspecting/setting devices connected to the PCI bus
+  - pciutils
diff --git a/manifests/wifi-firmwares.yaml b/manifests/wifi-firmwares.yaml
new file mode 100644
index 0000000000..6875ff7319
--- /dev/null
+++ b/manifests/wifi-firmwares.yaml
@@ -0,0 +1,9 @@
+# Wifi/BT firmware files kept in FCOS until the F41 rebase
+# See: https://github.com/coreos/fedora-coreos-tracker/issues/1575
+packages:
+  - atheros-firmware
+  - brcmfmac-firmware
+  - mt7xxx-firmware
+  - nxpwireless-firmware
+  - realtek-firmware
+  - tiwilink-firmware
diff --git a/overlay.d/05core/statoverride b/overlay.d/05core/statoverride
new file mode 100644
index 0000000000..a726b7ff0a
--- /dev/null
+++ b/overlay.d/05core/statoverride
@@ -0,0 +1,7 @@
+# Config file for overriding permission bits on overlay files/dirs
+# Format: =<file mode in decimal> <absolute path to a file or directory>
+
+# sudo prefers its config files to be mode 440, and some security scanners
+# complain if /etc/sudoers.d files are world-readable.
+# https://bugzilla.redhat.com/show_bug.cgi?id=1981979
+=288 /etc/sudoers.d/coreos-sudo-group
diff --git a/overlay.d/05core/usr/lib/coreos/generator-lib.sh b/overlay.d/05core/usr/lib/coreos/generator-lib.sh
new file mode 100644
index 0000000000..84547ce36b
--- /dev/null
+++ b/overlay.d/05core/usr/lib/coreos/generator-lib.sh
@@ -0,0 +1,30 @@
+# File intended to be sourced by shell script generators shipped with CoreOS systems
+
+# Generators don't have logging right now
+# https://github.com/systemd/systemd/issues/15638
+exec 1>/dev/kmsg; exec 2>&1
+
+UNIT_DIR="${1:-/tmp}"
+
+have_karg() {
+    local arg="$1"
+    IFS=" " read -r -a cmdline <<< "$(</proc/cmdline)"
+    local i
+    for i in "${cmdline[@]}"; do
+        if [[ "$i" =~ "$arg=" ]]; then
+            return 0
+        fi
+    done
+    return 1
+}
+
+karg() {
+    local name="$1" value="${2:-}"
+    IFS=" " read -r -a cmdline <<< "$(</proc/cmdline)"
+    for arg in "${cmdline[@]}"; do
+        if [[ "${arg%%=*}" == "${name}" ]]; then
+            value="${arg#*=}"
+        fi
+    done
+    echo "${value}"
+}
diff --git a/overlay.d/05core/usr/lib/dracut/dracut.conf.d/coreos-nohostonly.conf b/overlay.d/05core/usr/lib/dracut/dracut.conf.d/coreos-nohostonly.conf
new file mode 100644
index 0000000000..cc65c87351
--- /dev/null
+++ b/overlay.d/05core/usr/lib/dracut/dracut.conf.d/coreos-nohostonly.conf
@@ -0,0 +1,2 @@
+# Default rpm-ostree model is server-side generated initramfs
+hostonly=no
diff --git a/overlay.d/05core/usr/lib/dracut/dracut.conf.d/coreos-nostrip.conf b/overlay.d/05core/usr/lib/dracut/dracut.conf.d/coreos-nostrip.conf
new file mode 100644
index 0000000000..cad2269431
--- /dev/null
+++ b/overlay.d/05core/usr/lib/dracut/dracut.conf.d/coreos-nostrip.conf
@@ -0,0 +1,3 @@
+# We don't ship `strip` or `eu-strip` today, and even if we did, it doesn't
+# save much space. So let's disable it to avoid the error-looking message.
+do_strip=no
diff --git a/overlay.d/05core/usr/lib/dracut/dracut.conf.d/coreos-omits.conf b/overlay.d/05core/usr/lib/dracut/dracut.conf.d/coreos-omits.conf
new file mode 100644
index 0000000000..bf07099c46
--- /dev/null
+++ b/overlay.d/05core/usr/lib/dracut/dracut.conf.d/coreos-omits.conf
@@ -0,0 +1,18 @@
+# We don't support root on NFS, so we don't need it in the initramfs. It also
+# conflicts with /var mount support in ignition because NFS tries to mount stuff
+# in /var/ and then ignition can't cleanly unmount it. For example:
+# https://github.com/dracutdevs/dracut/blob/1856ae95c873a6fe855b3dccd0144f1a96b9e71c/modules.d/95nfs/nfs-start-rpc.sh#L7
+# See also discussion in https://github.com/coreos/fedora-coreos-config/pull/60
+# Further, we currently do not use LVM or dmraid
+omit_dracutmodules+=" nfs lvm dmraid "
+# More storage modules we don't use
+omit_dracutmodules+=" fcoe fcoe-uefi nbd "
+# We use NetworkManager
+omit_dracutmodules+=" systemd-networkd network-legacy network-wicked "
+# We only want to propagate NM keyfiles.
+# See also https://github.com/dracutdevs/dracut/pull/2529.
+omit_dracutmodules+=" ifcfg "
+# We use systemd network naming
+omit_dracutmodules+=" biosdevname "
+# Random stuff we don't want
+omit_dracutmodules+=" dbus-daemon memstrack "
diff --git a/overlay.d/05core/usr/lib/dracut/dracut.conf.d/coreos-zstd.conf b/overlay.d/05core/usr/lib/dracut/dracut.conf.d/coreos-zstd.conf
new file mode 100644
index 0000000000..5ff266c9a7
--- /dev/null
+++ b/overlay.d/05core/usr/lib/dracut/dracut.conf.d/coreos-zstd.conf
@@ -0,0 +1,4 @@
+# Compress initrd with zstd.  dracut defaults to -15, but we want the
+# maximum reasonable compression, so override the command line to use
+# dracut's defaults along with -19.
+compress="zstd -19 -q -T0"
diff --git a/overlay.d/05core/usr/lib/dracut/modules.d/01scsi-rules/module-setup.sh b/overlay.d/05core/usr/lib/dracut/modules.d/01scsi-rules/module-setup.sh
new file mode 100644
index 0000000000..d874dc8b27
--- /dev/null
+++ b/overlay.d/05core/usr/lib/dracut/modules.d/01scsi-rules/module-setup.sh
@@ -0,0 +1,13 @@
+# -*- mode: shell-script; indent-tabs-mode: nil; sh-basic-offset: 4; -*-
+# ex: ts=8 sw=4 sts=4 et filetype=sh
+
+# 1) Fix for https://bugzilla.redhat.com/show_bug.cgi?id=1918244
+# On s390x systems with IBM 2810XIV discs multipath couldn't be configured
+# because SCSI_IDENT_* udev properties are not set at boot time
+# 2-) Fix for https://bugzilla.redhat.com/show_bug.cgi?id=1990506
+# Missing symlinks to disk install
+install() {
+    inst_simple sg_inq
+    inst_rules 61-scsi-sg3_id.rules
+    inst_rules 63-scsi-sg3_symlink.rules
+}
diff --git a/overlay.d/05core/usr/lib/dracut/modules.d/15coreos-network/coreos-copy-firstboot-network.service b/overlay.d/05core/usr/lib/dracut/modules.d/15coreos-network/coreos-copy-firstboot-network.service
deleted file mode 100644
index 200032c4c5..0000000000
--- a/overlay.d/05core/usr/lib/dracut/modules.d/15coreos-network/coreos-copy-firstboot-network.service
+++ /dev/null
@@ -1,53 +0,0 @@
-# This unit will run early in boot and detect if the user copied
-# in firstboot networking config files into the installed system
-# (most likely by using `coreos-installer install --copy-network`).
-# Since this unit is modifying network configuration there are some
-# dependencies that we have:
-#
-# - Need to look for networking configuration on the /boot partition
-#     - i.e. after /dev/disk/by-label/boot is available
-#     - and after the ignition-dracut GPT generator (see below)
-# - Need to run before networking is brought up.
-#     - This is done in nm-run.sh [1] that runs as part of dracut-initqueue [2]
-#     - i.e. Before=dracut-initqueue.service
-# - Need to make sure karg networking configuration isn't applied
-#     - There are two ways to do this.
-#         - One is to run *before* the nm-config.sh [3] that runs as part of
-#           dracut-cmdline [4] and `ln -sf /bin/true /usr/libexec/nm-initrd-generator`.
-#             - i.e. Before=dracut-cmdline.service
-#         - Another is to run *after* nm-config.sh [3] in dracut-cmdline [4]
-#           and just delete all the files created by nm-initrd-generator.
-#             - i.e. After=dracut-cmdline.service, but Before=dracut-initqueue.service
-#     - We'll go with the second option here because the need for the /boot
-#       device (mentioned above) means we can't start before dracut-cmdline.service
-#
-# [1] https://github.com/dracutdevs/dracut/blob/master/modules.d/35network-manager/nm-run.sh
-# [2] https://github.com/dracutdevs/dracut/blob/master/modules.d/35network-manager/module-setup.sh#L37
-# [3] https://github.com/dracutdevs/dracut/blob/master/modules.d/35network-manager/nm-config.sh
-# [4] https://github.com/dracutdevs/dracut/blob/master/modules.d/35network-manager/module-setup.sh#L36
-#
-[Unit]
-Description=Copy CoreOS Firstboot Networking Config
-ConditionPathExists=/usr/lib/initrd-release
-DefaultDependencies=false
-Before=ignition-diskful.target
-Before=dracut-initqueue.service
-After=dracut-cmdline.service
-# Any services looking at mounts need to order after this
-# because it causes device re-probing.
-After=coreos-gpt-setup.service
-# Since we are mounting /boot/, require the device first
-Requires=dev-disk-by\x2dlabel-boot.device
-After=dev-disk-by\x2dlabel-boot.device
-# Need to run after coreos-enable-network since it may re-run the NM cmdline
-# hook which will generate NM configs from the network kargs, but we want to
-# have precedence.
-After=coreos-enable-network.service
-
-[Service]
-Type=oneshot
-RemainAfterExit=yes
-# The MountFlags=slave is so the umount of /boot is guaranteed to happen
-# /boot will only be mounted for the lifetime of the unit.
-MountFlags=slave
-ExecStart=/usr/sbin/coreos-copy-firstboot-network
diff --git a/overlay.d/05core/usr/lib/dracut/modules.d/15coreos-network/coreos-copy-firstboot-network.sh b/overlay.d/05core/usr/lib/dracut/modules.d/15coreos-network/coreos-copy-firstboot-network.sh
deleted file mode 100755
index 61c486c89c..0000000000
--- a/overlay.d/05core/usr/lib/dracut/modules.d/15coreos-network/coreos-copy-firstboot-network.sh
+++ /dev/null
@@ -1,31 +0,0 @@
-#!/bin/bash
-set -euo pipefail
-
-# For a description of how this is used see coreos-copy-firstboot-network.service
-
-bootmnt=/mnt/boot_partition
-mkdir -p ${bootmnt}
-bootdev=/dev/disk/by-label/boot
-firstboot_network_dir_basename="coreos-firstboot-network"
-initramfs_firstboot_network_dir="${bootmnt}/${firstboot_network_dir_basename}"
-initramfs_network_dir="/run/NetworkManager/system-connections/"
-realroot_firstboot_network_dir="/boot/${firstboot_network_dir_basename}"
-
-# Mount /boot. Note that we mount /boot but we don't unmount boot because we
-# are run in a systemd unit with MountFlags=slave so it is unmounted for us.
-# Mount as read-only since we don't strictly need write access and we may be
-# running alongside other code that also has it mounted ro
-mount -o ro ${bootdev} ${bootmnt}
-
-if [ -n "$(ls -A ${initramfs_firstboot_network_dir} 2>/dev/null)" ]; then
-    # Clear out any files that may have already been generated from
-    # kargs by nm-initrd-generator
-    rm -f ${initramfs_network_dir}/*
-    # Copy files that were placed into boot (most likely by coreos-installer)
-    # to the appropriate location for NetworkManager to use the configuration.
-    echo "info: copying files from ${initramfs_firstboot_network_dir} to ${initramfs_network_dir}"
-    mkdir -p ${initramfs_network_dir}
-    cp -v ${initramfs_firstboot_network_dir}/* ${initramfs_network_dir}/
-else
-    echo "info: no files to copy from ${initramfs_firstboot_network_dir}. skipping"
-fi
diff --git a/overlay.d/05core/usr/lib/dracut/modules.d/20live/coreos-liveiso-reconfigure-nm-wait-online.service b/overlay.d/05core/usr/lib/dracut/modules.d/20live/coreos-liveiso-reconfigure-nm-wait-online.service
deleted file mode 100644
index 1c910a2eeb..0000000000
--- a/overlay.d/05core/usr/lib/dracut/modules.d/20live/coreos-liveiso-reconfigure-nm-wait-online.service
+++ /dev/null
@@ -1,23 +0,0 @@
-# Configure NetworkManager-wait-online in the real root for the
-# Live ISO to timeout quicker and also not explicitly fail since
-# booting the Live ISO without network is a valid use case.
-#
-# Doing this improves the user experience when booting the
-# Live ISO without network.
-
-[Unit]
-Description=Reconfigure NetworkManager-wait-online service
-DefaultDependencies=no
-# Make sure we are in the initramfs and we are booted to the live ISO
-ConditionPathExists=/usr/lib/initrd-release
-ConditionKernelCommandLine=coreos.liveiso
-ConditionPathExists=/run/ostree-live
-
-[Service]
-Type=oneshot
-RemainAfterExit=yes
-# Note keep this in sync with NetworkManager-wait-online.service
-# Right now we are keeping the same ExecStart but we are making it
-# OK to fail (`-`) and timeout sooner (5 seconds vs 30).
-ExecStartPre=/usr/bin/mkdir -p /run/systemd/system/NetworkManager-wait-online.service.d
-ExecStart=/bin/bash -c 'echo -e "[Service]\nExecStart=\nExecStart=-/usr/bin/nm-online -s -q --timeout=5" > /run/systemd/system/NetworkManager-wait-online.service.d/liveiso.conf'
diff --git a/overlay.d/05core/usr/lib/dracut/modules.d/20live/coreos-livepxe-rootfs.sh b/overlay.d/05core/usr/lib/dracut/modules.d/20live/coreos-livepxe-rootfs.sh
deleted file mode 100755
index 05824f3f24..0000000000
--- a/overlay.d/05core/usr/lib/dracut/modules.d/20live/coreos-livepxe-rootfs.sh
+++ /dev/null
@@ -1,50 +0,0 @@
-#!/bin/bash
-# Ensure that a PXE-booted system has a valid rootfs.
-
-set -euo pipefail
-
-# Get rootfs_url karg
-set +euo pipefail
-. /usr/lib/dracut-lib.sh
-rootfs_url=$(getarg coreos.live.rootfs_url=)
-set -euo pipefail
-
-if [[ -f /etc/coreos-live-rootfs ]]; then
-    # rootfs image was injected via PXE.  Verify that the initramfs and
-    # rootfs versions match.
-    initramfs_ver=$(cat /etc/coreos-live-initramfs)
-    rootfs_ver=$(cat /etc/coreos-live-rootfs)
-    if [[ $initramfs_ver != $rootfs_ver ]]; then
-        echo "Found initramfs version $initramfs_ver but rootfs version $rootfs_ver." >&2
-        echo "Please fix your PXE configuration." >&2
-        exit 1
-    fi
-elif [[ -n "${rootfs_url}" ]]; then
-    # rootfs URL was provided as karg.  Fetch image, check its hash, and
-    # unpack it.
-    echo "Fetching rootfs image from ${rootfs_url}..."
-    if [[ ${rootfs_url} != http:* && ${rootfs_url} != https:* ]]; then
-        # Don't commit to supporting protocols we might not want to expose in
-        # the long term.
-        echo "coreos.live.rootfs_url= supports HTTP and HTTPS only." >&2
-        echo "Please fix your PXE configuration." >&2
-        exit 1
-    fi
-    # We don't need to verify TLS certificates because we're checking the
-    # image hash.
-    # bsdtar can read cpio archives and we already depend on it for
-    # coreos-liveiso-persist-osmet.service, so use it instead of cpio.
-    if ! curl --silent --show-error --insecure --location --retry 5 "${rootfs_url}" | \
-            rdcore stream-hash /etc/coreos-live-want-rootfs | \
-            bsdtar -xf - -C / ; then
-        echo "Couldn't fetch, verify, and unpack image specified by coreos.live.rootfs_url=" >&2
-        echo "Check that the URL is correct and that the rootfs version matches the initramfs." >&2
-        exit 1
-    fi
-else
-    # Nothing.  Fail.
-    echo "No rootfs image found.  Modify your PXE configuration to add the rootfs" >&2
-    echo "image as a second initrd, or use the coreos.live.rootfs_url= kernel parameter" >&2
-    echo "to specify an HTTP or HTTPS URL to the rootfs." >&2
-    exit 1
-fi
diff --git a/overlay.d/05core/usr/lib/dracut/modules.d/30ignition-coreos/coreos-diskful-generator b/overlay.d/05core/usr/lib/dracut/modules.d/30ignition-coreos/coreos-diskful-generator
deleted file mode 100755
index 852cdc34aa..0000000000
--- a/overlay.d/05core/usr/lib/dracut/modules.d/30ignition-coreos/coreos-diskful-generator
+++ /dev/null
@@ -1,57 +0,0 @@
-#!/bin/bash
-# -*- mode: shell-script; indent-tabs-mode: nil; sh-basic-offset: 4; -*-
-# ex: ts=8 sw=4 sts=4 et filetype=sh
-
-# Originally this was known as 'ignition-generator' found in ignition-dracut.
-# With Ignition v 2.5.0, ignition-dracut was merged into Ignition and the CoreOS
-# specific bits were deposited here.
-
-set -e
-
-# Generators don't have logging right now
-# https://github.com/systemd/systemd/issues/15638
-exec 1>/dev/kmsg; exec 2>&1
-
-UNIT_DIR="${1:-/tmp}"
-
-cmdline=( $(</proc/cmdline) )
-cmdline_arg() {
-    local name="$1" value="$2"
-    for arg in "${cmdline[@]}"; do
-        if [[ "${arg%%=*}" == "${name}" ]]; then
-            value="${arg#*=}"
-        fi
-    done
-    echo "${value}"
-}
-
-cmdline_bool() {
-    local value=$(cmdline_arg "$@")
-    case "$value" in
-        ""|0|no|off) return 1;;
-        *) return 0;;
-    esac
-}
-
-add_requires() {
-    local name="$1"; shift
-    local target="$1"; shift
-    local requires_dir="${UNIT_DIR}/${target}.requires"
-    mkdir -p "${requires_dir}"
-    ln -sf "../${name}" "${requires_dir}/${name}"
-}
-
-if ! $(cmdline_bool 'ignition.firstboot' 0); then
-    exit 0
-fi
-
-if ! command -v is-live-image >/dev/null || ! is-live-image; then
-    # ignition-setup-user.service should depend on the boot device node
-    # only on diskful boots
-    mkdir -p "${UNIT_DIR}/ignition-setup-user.service.d"
-    cat > "${UNIT_DIR}/ignition-setup-user.service.d/diskful-gpt.conf" <<EOF
-[Unit]
-Requires=coreos-gpt-setup.service
-After=coreos-gpt-setup.service
-EOF
-fi
diff --git a/overlay.d/05core/usr/lib/dracut/modules.d/30ignition-coreos/coreos-gpt-setup.service b/overlay.d/05core/usr/lib/dracut/modules.d/30ignition-coreos/coreos-gpt-setup.service
deleted file mode 100644
index 6d873665fb..0000000000
--- a/overlay.d/05core/usr/lib/dracut/modules.d/30ignition-coreos/coreos-gpt-setup.service
+++ /dev/null
@@ -1,33 +0,0 @@
-[Unit]
-Description=Generate new UUID for boot disk GPT
-ConditionPathExists=/etc/initrd-release
-DefaultDependencies=no
-Before=local-fs-pre.target systemd-fsck-root.service
-Before=ignition-diskful.target
-Wants=systemd-udevd.service
-After=systemd-udevd.service
-
-# This unit must be the first to run when the disk holding the root partition
-# becomes available. To avoid relying on the name of the root partition which
-# is different between RHCOS LUKS setup and current FCOS setup, we wait for the
-# partition labeled 'boot' to become available. This is reliable as we don't
-# have any plan to support re-provisioning/re-writing the /boot partition,
-#
-# This is the only unit where it is safe to wait only on a specific disk label
-# as this will call udevadm settle after the GPT setup. Units that requires the
-# boot and root partitions to be available should order themselves after this
-# unit.
-Requires=dev-disk-by\x2dlabel-boot.device
-After=dev-disk-by\x2dlabel-boot.device
-
-# Run before services that use device nodes, preventing them from racing
-# with udev activity generated by sgdisk
-Before=ignition-setup-base.service ignition-setup-user.service ignition-disks.service
-
-OnFailure=emergency.target
-OnFailureJobMode=isolate
-
-[Service]
-Type=oneshot
-RemainAfterExit=yes
-ExecStart=/usr/sbin/coreos-gpt-setup /dev/disk/by-label/boot
diff --git a/overlay.d/05core/usr/lib/dracut/modules.d/30ignition-coreos/module-setup.sh b/overlay.d/05core/usr/lib/dracut/modules.d/30ignition-coreos/module-setup.sh
deleted file mode 100755
index 03565e21e6..0000000000
--- a/overlay.d/05core/usr/lib/dracut/modules.d/30ignition-coreos/module-setup.sh
+++ /dev/null
@@ -1,47 +0,0 @@
-#!/bin/bash
-# -*- mode: shell-script; indent-tabs-mode: nil; sh-basic-offset: 4; -*-
-# ex: ts=8 sw=4 sts=4 et filetype=sh
-
-depends() {
-    echo systemd network ignition
-}
-
-install_ignition_unit() {
-    local unit="$1"; shift
-    local target="${1:-ignition-complete.target}"; shift
-    local instantiated="${1:-$unit}"; shift
-    inst_simple "$moddir/$unit" "$systemdsystemunitdir/$unit"
-    systemctl -q --root="$initdir" add-requires "$target" "$instantiated"
-}
-
-install() {
-    inst_multiple \
-        basename \
-        diff \
-        lsblk \
-        sed \
-        sgdisk
-
-    inst_simple "$moddir/coreos-diskful-generator" \
-        "$systemdutildir/system-generators/coreos-diskful-generator"
-
-    inst_script "$moddir/coreos-gpt-setup.sh" \
-        "/usr/sbin/coreos-gpt-setup"
-
-    # For consistency tear down the network and persist multipath between the initramfs and
-    # real root. See https://github.com/coreos/fedora-coreos-tracker/issues/394#issuecomment-599721763
-    inst_script "$moddir/coreos-teardown-initramfs.sh" \
-        "/usr/sbin/coreos-teardown-initramfs"
-    install_ignition_unit coreos-teardown-initramfs.service
-
-    # units only started when we have a boot disk
-    # path generated by systemd-escape --path /dev/disk/by-label/root
-    install_ignition_unit coreos-gpt-setup.service ignition-diskful.target
-
-    inst_script "$moddir/coreos-boot-edit.sh" \
-        "/usr/sbin/coreos-boot-edit"
-    # Only start when the system has disks since we are editing /boot.
-    install_ignition_unit "coreos-boot-edit.service" \
-        "ignition-diskful.target"
-
-}
diff --git a/overlay.d/05core/usr/lib/dracut/modules.d/35coreos-ignition/01-secex.ign b/overlay.d/05core/usr/lib/dracut/modules.d/35coreos-ignition/01-secex.ign
new file mode 100644
index 0000000000..6848038632
--- /dev/null
+++ b/overlay.d/05core/usr/lib/dracut/modules.d/35coreos-ignition/01-secex.ign
@@ -0,0 +1,82 @@
+{
+  "ignition": {
+    "version": "3.2.0"
+  },
+  "storage": {
+    "disks": [
+      {
+        "device": "/dev/disk/by-id/coreos-boot-disk",
+        "partitions": [
+          {
+            "label": "boot",
+            "number": 3
+          },
+          {
+            "label": "root",
+            "number": 4,
+            "resize": true,
+            "sizeMiB": 0
+          },
+          {
+            "number": 5,
+            "shouldExist": false,
+            "wipePartitionEntry": true
+          },
+          {
+            "number": 6,
+            "shouldExist": false,
+            "wipePartitionEntry": true
+          }
+        ]
+      }
+    ],
+    "luks": [
+      {
+        "device": "/dev/disk/by-partlabel/boot",
+        "label": "crypt_bootfs",
+        "name": "boot",
+        "options": [
+          "--integrity",
+          "hmac-sha256"
+        ],
+        "wipeVolume": true
+      },
+      {
+        "device": "/dev/disk/by-partlabel/root",
+        "label": "crypt_rootfs",
+        "name": "root",
+        "options": [
+          "--integrity",
+          "hmac-sha256"
+        ],
+        "wipeVolume": true
+      }
+    ],
+    "filesystems": [
+      {
+        "device": "/dev/mapper/boot",
+        "format": "ext4",
+        "label": "boot",
+        "wipeFilesystem": true
+      },
+      {
+        "device": "/dev/mapper/root",
+        "format": "xfs",
+        "label": "root",
+        "wipeFilesystem": true
+      }
+    ]
+  },
+  "systemd": {
+    "units": [
+      {
+        "mask": true,
+        "name": "serial-getty@.service"
+      },
+      {
+        "mask": true,
+        "name": "autovt@.service"
+      }
+    ]
+  }
+}
diff --git a/overlay.d/05core/usr/lib/dracut/modules.d/35coreos-ignition/80-coreos-boot-disk.rules b/overlay.d/05core/usr/lib/dracut/modules.d/35coreos-ignition/80-coreos-boot-disk.rules
new file mode 100644
index 0000000000..e56ee5b741
--- /dev/null
+++ b/overlay.d/05core/usr/lib/dracut/modules.d/35coreos-ignition/80-coreos-boot-disk.rules
@@ -0,0 +1,10 @@
+# CoreOS-specific symlink for boot disk
+
+ACTION!="add|change", GOTO="stable_boot_end"
+SUBSYSTEM!="block", GOTO="stable_boot_end"
+
+ENV{DEVTYPE}=="disk" \
+  , PROGRAM=="coreos-disk-contains-fs $name boot" \
+  , SYMLINK+="disk/by-id/coreos-boot-disk"
+
+LABEL="stable_boot_end"
diff --git a/overlay.d/05core/usr/lib/dracut/modules.d/30ignition-coreos/coreos-boot-edit.service b/overlay.d/05core/usr/lib/dracut/modules.d/35coreos-ignition/coreos-boot-edit.service
similarity index 69%
rename from overlay.d/05core/usr/lib/dracut/modules.d/30ignition-coreos/coreos-boot-edit.service
rename to overlay.d/05core/usr/lib/dracut/modules.d/35coreos-ignition/coreos-boot-edit.service
index 99a28563c2..bc2d9ec9ea 100644
--- a/overlay.d/05core/usr/lib/dracut/modules.d/30ignition-coreos/coreos-boot-edit.service
+++ b/overlay.d/05core/usr/lib/dracut/modules.d/35coreos-ignition/coreos-boot-edit.service
@@ -14,6 +14,14 @@ Requires=dev-disk-by\x2dlabel-boot.device
 After=dev-disk-by\x2dlabel-boot.device
 # Start after Ignition has finished
 After=ignition-files.service
+# As above, this isn't strictly necessary, but on principle.
+After=coreos-multipath-wait.target
+# Finish before systemd starts tearing down services
+Before=initrd.target
+# initrd-parse-etc.service starts initrd-cleanup.service which will race
+# with us completing before we get nuked.  Need to get to the bottom of it,
+# but for now we need this.
+Before=initrd-parse-etc.service
 
 [Service]
 Type=oneshot
diff --git a/overlay.d/05core/usr/lib/dracut/modules.d/30ignition-coreos/coreos-boot-edit.sh b/overlay.d/05core/usr/lib/dracut/modules.d/35coreos-ignition/coreos-boot-edit.sh
similarity index 55%
rename from overlay.d/05core/usr/lib/dracut/modules.d/30ignition-coreos/coreos-boot-edit.sh
rename to overlay.d/05core/usr/lib/dracut/modules.d/35coreos-ignition/coreos-boot-edit.sh
index 3b16813b46..437a8dee80 100755
--- a/overlay.d/05core/usr/lib/dracut/modules.d/30ignition-coreos/coreos-boot-edit.sh
+++ b/overlay.d/05core/usr/lib/dracut/modules.d/35coreos-ignition/coreos-boot-edit.sh
@@ -3,7 +3,7 @@ set -euo pipefail
 
 # For a description of how this is used, see `coreos-boot-edit.service`.
 
-cmdline=( $(</proc/cmdline) )
+IFS=" " read -r -a cmdline <<< "$(</proc/cmdline)"
 karg() {
     local name="$1" value="${2:-}"
     for arg in "${cmdline[@]}"; do
@@ -16,8 +16,7 @@ karg() {
 
 # Mount /boot. Note that we mount /boot but we don't unmount it because we
 # are run in a systemd unit with MountFlags=slave so it is unmounted for us.
-bootmnt=/mnt/boot_partition
-mkdir -p ${bootmnt}
+bootmnt=/sysroot/boot
 bootdev=/dev/disk/by-label/boot
 mount -o rw ${bootdev} ${bootmnt}
 
@@ -31,5 +30,22 @@ rm -vrf ${initramfs_firstboot_network_dir}
 # append rootmap kargs to the BLS configs.
 root=$(karg root)
 if [ -z "${root}" ]; then
-    /usr/bin/rdcore rootmap /sysroot --boot-mount ${bootmnt}
+    rdcore rootmap /sysroot --boot-mount ${bootmnt}
+    echo "Prepared rootmap"
+fi
+
+# This does a few things:
+# 1. it puts the boot UUID in /run/coreos/bootfs_uuid which is used by the real
+#    root for mounting the bootfs in this boot
+# 2. it adds a boot=UUID= karg which is used by the real root for mounting the
+#    bootfs in subsequent boots
+# 3. it create a .root_uuid stamp file on the bootfs or fails if one exists
+# 4. it adds GRUB bootuuid.cfg dropins so that GRUB selects the boot filesystem
+#    by UUID
+rdcore bind-boot /sysroot ${bootmnt}
+
+# relabel files rdcore created; ideally in the future rdcore does this itself
+coreos-relabel /boot/.root_uuid
+if [ -e /sysroot/boot/grub2/bootuuid.cfg ]; then
+    coreos-relabel /boot/grub2/bootuuid.cfg
 fi
diff --git a/overlay.d/05core/usr/lib/dracut/modules.d/35coreos-ignition/coreos-disk-contains-fs.sh b/overlay.d/05core/usr/lib/dracut/modules.d/35coreos-ignition/coreos-disk-contains-fs.sh
new file mode 100755
index 0000000000..37b5f98bd1
--- /dev/null
+++ b/overlay.d/05core/usr/lib/dracut/modules.d/35coreos-ignition/coreos-disk-contains-fs.sh
@@ -0,0 +1,20 @@
+#!/bin/bash
+# checks whether `disk` contains filesystem labeled `label`
+set -euo pipefail
+
+disk=$1
+label=$2
+
+# during execution of udev rules on disks 'lsblk' returns empty fields
+for pt in /sys/block/$disk/*; do
+    name=$(basename $pt)
+    if [[ "$name" =~ ${disk}p?[[:digit:]] ]] && [[ -e "/sys/block/$disk/$name/start" ]];
+    then
+        eval $(udevadm info --query=property -n /dev/$name | grep -e ID_FS_LABEL -e PARTNAME)
+        if [[ "${ID_FS_LABEL:-}" == "$label" ]] || [[ "${PARTNAME:-}" == "$label" ]]; then
+            exit 0
+        fi
+    fi
+done
+
+exit 1
diff --git a/overlay.d/05core/usr/lib/dracut/modules.d/35coreos-ignition/coreos-diskful-generator b/overlay.d/05core/usr/lib/dracut/modules.d/35coreos-ignition/coreos-diskful-generator
new file mode 100755
index 0000000000..b869866431
--- /dev/null
+++ b/overlay.d/05core/usr/lib/dracut/modules.d/35coreos-ignition/coreos-diskful-generator
@@ -0,0 +1,170 @@
+#!/bin/bash
+# -*- mode: shell-script; indent-tabs-mode: nil; sh-basic-offset: 4; -*-
+# ex: ts=8 sw=4 sts=4 et filetype=sh
+
+# Originally this was known as 'ignition-generator' found in ignition-dracut.
+# With Ignition v 2.5.0, ignition-dracut was merged into Ignition and the CoreOS
+# specific bits were deposited here.
+
+set +euo pipefail
+. /usr/lib/dracut-lib.sh
+set -euo pipefail
+
+# dracut isn't nice with `set -e`
+dracut_func() {
+    set +euo pipefail
+    "$@"; local rc=$?
+    set -euo pipefail
+    return $rc
+}
+
+# Generators don't have logging right now
+# https://github.com/systemd/systemd/issues/15638
+exec 1>/dev/kmsg; exec 2>&1
+
+UNIT_DIR="${1:-/tmp}"
+EARLY_DIR="${2:-/tmp}"
+
+IFS=" " read -r -a cmdline <<< "$(</proc/cmdline)"
+cmdline_arg() {
+    local name="$1" value=""
+    for arg in "${cmdline[@]}"; do
+        if [[ "${arg%%=*}" == "${name}" ]]; then
+            value="${arg#*=}"
+        fi
+    done
+    echo "${value}"
+}
+
+cmdline_bool() {
+    local value
+    value=$(cmdline_arg "$@")
+    case "$value" in
+        ""|0|no|off) return 1;;
+        *) return 0;;
+    esac
+}
+
+add_requires() {
+    local name="$1"; shift
+    local target="$1"; shift
+    local requires_dir="${UNIT_DIR}/${target}.requires"
+    mkdir -p "${requires_dir}"
+    ln -sf "../${name}" "${requires_dir}/${name}"
+}
+
+# If this is a live image then exit early.
+if is-live-image; then
+    exit 0
+fi
+
+# Generate a sysroot.mount unit. We can't do this later by dynamically
+# doing the mount like we used to because systemd 254+ will drop units that
+# require sysroot.mount from the transaction:
+# https://github.com/coreos/fedora-coreos-tracker/issues/1527#issuecomment-1656786452
+#
+# We'll dynamically pick up what mount options to use from the OPTIONS
+# environment variable in /run/coreos-rootflags (generated by
+# coreos-rootflags.service).
+#
+# Note that for new machines this is *only* useful on first boot because
+# coreos-boot-edit.service will add the root= and rootflags= kargs to the
+# kernel command line for subsequent boots. However we haven't migrated
+# all old machines so there are still some that could exist that don't
+# have root= and rootflags= kargs. For that reason we run this bit of
+# code on EVERY diskful boot.
+mkdir -p "${UNIT_DIR}"
+# We want to generate sysroot.mount on ostree systems, but we don't want
+# to run if there's already a root= karg, where the systemd-fstab-generator
+# should win.
+# We also don't want to generate sysroot.mount if we are booting a kdump kernel
+# that aims to upload logs to a remote target, as we may be in a diskless setup
+# https://issues.redhat.com/browse/OCPBUGS-27935
+# FIXME use dracut-lib https://github.com/coreos/fedora-coreos-config/pull/2890#issuecomment-1985953499
+# rather than ConditionPathExists
+# Using dracut-lib seems to interfere with iscsi functionnality but this requires deeper investigation
+# and more tests : https://github.com/coreos/fedora-coreos-tracker/issues/1689
+if test -n "$(cmdline_arg ostree)" && test -z "$(cmdline_arg root)" && ! dracut_func getargbool 0 'kdump_remote_ip'; then
+    cat >"${UNIT_DIR}"/sysroot.mount << 'EOF'
+[Unit]
+Before=initrd-root-fs.target
+After=coreos-rootflags.service
+[Mount]
+EnvironmentFile=/run/coreos-rootflags
+What=/dev/disk/by-label/root
+Where=/sysroot
+Options=${OPTIONS}
+EOF
+    add_requires sysroot.mount initrd-root-fs.target
+    add_requires coreos-rootflags.service initrd-root-fs.target
+fi
+
+if ! cmdline_bool 'ignition.firstboot'; then
+    exit 0
+fi
+
+# coreos-ignition-setup-user.service and `coreos-copy-firstboot-network.service`
+# should depend on the boot device node only on diskful boots
+mkdir -p "${UNIT_DIR}/coreos-ignition-setup-user.service.d"
+mkdir -p "${UNIT_DIR}/coreos-copy-firstboot-network.service.d"
+cat > "${UNIT_DIR}/coreos-ignition-setup-user.service.d/diskful.conf" <<EOF
+[Unit]
+Requires=dev-disk-by\x2dlabel-boot.device
+After=dev-disk-by\x2dlabel-boot.device
+EOF
+cp "${UNIT_DIR}/coreos-ignition-setup-user.service.d/diskful.conf" \
+    "${UNIT_DIR}/coreos-copy-firstboot-network.service.d/diskful.conf"
+
+# If booting from iSCSI, then we need networking first before we
+# can see the bootfs. This has some implications.
+if test -n "$(cmdline_arg rd.iscsi.firmware)" || test -n "$(cmdline_arg netroot)"; then
+    mkdir -p "${EARLY_DIR}/ignition-complete.target.requires"
+
+    # In the injected Ignition config case, `coreos-ignition-setup-user.service`
+    # can't "pass" the config to `ignition-fetch-offline.service`. So we neuter
+    # the latter so that `ignition-fetch.service` (which runs after networking
+    # comes up) can pick it up instead.
+    ln -sf /dev/null "${EARLY_DIR}/ignition-complete.target.requires/ignition-fetch-offline.service"
+    # activate ignition-fetch.service
+    mkdir -p /run/ignition
+    touch /run/ignition/neednet
+
+    # With iSCSI, configuring networking via kargs is a hard requirement
+    # since it can't be picked up from the bootfs (i.e. injected via
+    # `--copy-network`). So neuter `coreos-copy-firstboot-network.service`.
+    ln -sf /dev/null "${EARLY_DIR}/ignition-complete.target.requires/coreos-copy-firstboot-network.service"
+fi
+
+# create symlink for udev rule
+mkdir -p /run/udev/rules.d/
+ln -sf /usr/lib/coreos/80-coreos-boot-disk.rules \
+      /run/udev/rules.d/80-coreos-boot-disk.rules
+
+# IBM Secure Execution case
+# During firstboot we have to reencrypt '/boot' and '/', to do that an Ignition config
+# is injected. 'coreos-boot-disk' is required for this
+secure_execution=0
+if [[ $(uname -m) == s390x ]] && [[ -e /sys/firmware/uv/prot_virt_guest ]]; then
+    secure_execution=$(cat /sys/firmware/uv/prot_virt_guest)
+fi
+if [[ "${secure_execution}" = "1" ]]; then
+    mkdir -p /run/coreos/
+    touch /run/coreos/secure-execution
+    # Add dropins to disable Ignition logging for all stages
+    stages=("fetch-offline" "fetch" "kargs" "disks" "mount" "files")
+    for s in "${stages[@]}"; do
+        dropin="${UNIT_DIR}/ignition-${s}.service.d"
+        mkdir -p "${dropin}"
+        cat > "${dropin}/10-secex.conf" <<EOF
+[Service]
+Environment=IGNITION_ARGS=-log-to-stdout
+StandardOutput=null
+StandardError=null
+EOF
+    done
+
+    # This one is done dynamically because it hard Requires a device to appear
+    # and if it's always part of the transaction, systemd will want the device
+    # to appear regardless of ConditionPathExists.
+    add_requires coreos-secex-ignition-prepare.service ignition-diskful.target
+fi
diff --git a/overlay.d/05core/usr/lib/dracut/modules.d/35coreos-ignition/coreos-gpt-setup.service b/overlay.d/05core/usr/lib/dracut/modules.d/35coreos-ignition/coreos-gpt-setup.service
new file mode 100644
index 0000000000..629bb60f49
--- /dev/null
+++ b/overlay.d/05core/usr/lib/dracut/modules.d/35coreos-ignition/coreos-gpt-setup.service
@@ -0,0 +1,30 @@
+[Unit]
+Description=Generate New UUID For Boot Disk GPT
+ConditionPathExists=/etc/initrd-release
+Before=systemd-fsck-root.service
+Before=ignition-diskful.target
+
+# Have some ordering with this service since it also wants to access the bootfs.
+# We run after it since that service runs early and is skipped if we're on
+# iSCSI, but we still run.
+After=coreos-copy-firstboot-network.service
+
+Requires=dev-disk-by\x2dlabel-boot.device
+After=dev-disk-by\x2dlabel-boot.device
+
+# And since the boot device may be on multipath; optionally wait for it to
+# appear via the dynamic target.
+After=coreos-unique-boot.service
+Before=ignition-ostree-uuid-boot.service
+
+# Run before services that use device nodes, preventing them from racing
+# with udev activity generated by sgdisk
+Before=ignition-kargs.service
+
+OnFailure=emergency.target
+OnFailureJobMode=isolate
+
+[Service]
+Type=oneshot
+RemainAfterExit=yes
+ExecStart=/usr/sbin/coreos-gpt-setup /dev/disk/by-label/boot
diff --git a/overlay.d/05core/usr/lib/dracut/modules.d/30ignition-coreos/coreos-gpt-setup.sh b/overlay.d/05core/usr/lib/dracut/modules.d/35coreos-ignition/coreos-gpt-setup.sh
similarity index 60%
rename from overlay.d/05core/usr/lib/dracut/modules.d/30ignition-coreos/coreos-gpt-setup.sh
rename to overlay.d/05core/usr/lib/dracut/modules.d/35coreos-ignition/coreos-gpt-setup.sh
index 757d8fe96d..11cebd34f3 100755
--- a/overlay.d/05core/usr/lib/dracut/modules.d/30ignition-coreos/coreos-gpt-setup.sh
+++ b/overlay.d/05core/usr/lib/dracut/modules.d/35coreos-ignition/coreos-gpt-setup.sh
@@ -6,14 +6,11 @@ set -euo pipefail
 
 UNINITIALIZED_GUID='00000000-0000-4000-a000-000000000001'
 
-# On RHEL 8 the version of lsblk doesn't have PTUUID. Let's detect
-# if lsblk supports it. In the future we can remove the 'if' and
-# just use the 'else'.
-if ! lsblk --help | grep -q PTUUID; then
-    # Get the PKNAME
-    eval $(lsblk --output PKNAME --pairs --paths --nodeps "$1")
-    # Get the PTUUID
-    eval $(blkid -o export $PKNAME)
+# If it's on multipath, get the parent device from udev properties.
+DM_MPATH=$(eval $(udevadm info --query property --export "$1") && echo "${DM_MPATH:-}")
+if [ -n "${DM_MPATH:-}" ]; then
+    PKNAME=/dev/mapper/$DM_MPATH
+    PTUUID=$(eval $(udevadm info --query property --export "$PKNAME") && echo "${ID_PART_TABLE_UUID:-}")
 else
     # PTUUID is the disk guid, PKNAME is the parent kernel name
     eval $(lsblk --output PTUUID,PKNAME --pairs --paths --nodeps "$1")
@@ -24,7 +21,11 @@ fi
 #  - The PTUUID is empty. This happens on s390x where DASD disks don't
 #    have PTUUID or any of the other traditional partition table
 #    attributes of GPT disks.
-[ "${PTUUID:-}" != "$UNINITIALIZED_GUID" ] && exit 0
+if [ "${PTUUID:-}" != "$UNINITIALIZED_GUID" ]; then
+    echo "Not randomizing disk GUID; found ${PTUUID:-none}"
+    exit 0
+fi
 
+echo "Randomizing disk GUID"
 sgdisk --disk-guid=R --move-second-header "$PKNAME"
-udevadm settle
+udevadm settle || :
diff --git a/overlay.d/05core/usr/lib/dracut/modules.d/35coreos-ignition/coreos-ignition-setup-user.service b/overlay.d/05core/usr/lib/dracut/modules.d/35coreos-ignition/coreos-ignition-setup-user.service
new file mode 100644
index 0000000000..3728641512
--- /dev/null
+++ b/overlay.d/05core/usr/lib/dracut/modules.d/35coreos-ignition/coreos-ignition-setup-user.service
@@ -0,0 +1,25 @@
+[Unit]
+Description=CoreOS Ignition User Config Setup
+Documentation=https://github.com/coreos/ignition
+ConditionPathExists=/etc/initrd-release
+DefaultDependencies=false
+
+# We run before config fetch because we may copy in new/different
+# configs for Ignition to consume. We include both fetch stages since
+# ignition-fetch-offline.service is neutered in a root on iSCSI boot (see
+# `coreos-diskful-generator`).
+Before=ignition-fetch-offline.service ignition-fetch.service
+
+OnFailure=emergency.target
+OnFailureJobMode=isolate
+
+# On diskful boots, coreos-diskful-generator adds Requires/After on
+# dev-disk-by\x2dlabel-boot.device
+
+[Service]
+Type=oneshot
+RemainAfterExit=yes
+# The MountFlags=slave is so the umount of /boot is guaranteed to happen
+# /boot will only be mounted for the lifetime of the unit.
+MountFlags=slave
+ExecStart=/usr/sbin/coreos-ignition-setup-user
diff --git a/overlay.d/05core/usr/lib/dracut/modules.d/35coreos-ignition/coreos-ignition-setup-user.sh b/overlay.d/05core/usr/lib/dracut/modules.d/35coreos-ignition/coreos-ignition-setup-user.sh
new file mode 100755
index 0000000000..af0ce3cda6
--- /dev/null
+++ b/overlay.d/05core/usr/lib/dracut/modules.d/35coreos-ignition/coreos-ignition-setup-user.sh
@@ -0,0 +1,39 @@
+#!/bin/bash
+set -euo pipefail
+
+copy_file_if_exists() {
+    src="${1}"; dst="${2}"
+    if [ -f "${src}" ]; then
+        echo "Copying ${src} to ${dst}"
+        cp "${src}" "${dst}"
+    else
+        echo "File ${src} does not exist.. Skipping copy"
+    fi
+}
+
+# systemd v256 now runs the initrd with ProtectSystem=yes, which makes /usr
+# read-only. Just remount it rw until we have:
+# https://github.com/coreos/ignition/issues/1891
+if [ ! -w /usr ]; then
+    mount -o rw,remount /usr
+fi
+
+destination=/usr/lib/ignition
+mkdir -p $destination
+
+if is-live-image; then
+    # Live image. If the user has supplied a config.ign via an appended
+    # initrd, put it in the right place.
+    copy_file_if_exists "/config.ign" "${destination}/user.ign"
+else
+    # We will support a user embedded config in the boot partition
+    # under $bootmnt/ignition/config.ign. Note that we mount /boot
+    # but we don't unmount boot because we are run in a systemd unit
+    # with MountFlags=slave so it is unmounted for us.
+    bootmnt=/mnt/boot_partition
+    mkdir -p $bootmnt
+    # mount as read-only since we don't strictly need write access and we may be
+    # running alongside other code that also has it mounted ro
+    mount -o ro /dev/disk/by-label/boot $bootmnt
+    copy_file_if_exists "${bootmnt}/ignition/config.ign" "${destination}/user.ign"
+fi
diff --git a/overlay.d/05core/usr/lib/dracut/modules.d/35coreos-ignition/coreos-ignition-unique-boot.service b/overlay.d/05core/usr/lib/dracut/modules.d/35coreos-ignition/coreos-ignition-unique-boot.service
new file mode 100644
index 0000000000..81f099b6fb
--- /dev/null
+++ b/overlay.d/05core/usr/lib/dracut/modules.d/35coreos-ignition/coreos-ignition-unique-boot.service
@@ -0,0 +1,19 @@
+[Unit]
+Description=CoreOS Ignition Ensure Unique Boot Filesystem
+ConditionPathExists=/etc/initrd-release
+OnFailure=emergency.target
+OnFailureJobMode=isolate
+
+# That's a weak dependency, so service won't fail if boot dissaperears
+Wants=dev-disk-by\x2dlabel-boot.device
+After=dev-disk-by\x2dlabel-boot.device
+
+# Start after ignition has finished with disks but before mounting them
+After=ignition-disks.service
+Before=ignition-mount.service
+Before=ignition-ostree-uuid-root.service
+
+[Service]
+Type=oneshot
+RemainAfterExit=yes
+ExecStart=/usr/bin/rdcore verify-unique-fs-label --rereadpt boot
diff --git a/overlay.d/05core/usr/lib/dracut/modules.d/35coreos-ignition/coreos-kargs-reboot.service b/overlay.d/05core/usr/lib/dracut/modules.d/35coreos-ignition/coreos-kargs-reboot.service
new file mode 100644
index 0000000000..23906732bb
--- /dev/null
+++ b/overlay.d/05core/usr/lib/dracut/modules.d/35coreos-ignition/coreos-kargs-reboot.service
@@ -0,0 +1,23 @@
+[Unit]
+Description=CoreOS Kernel Arguments Reboot
+ConditionPathExists=/etc/initrd-release
+ConditionPathExists=/run/coreos-kargs-reboot
+DefaultDependencies=false
+Before=ignition-complete.target
+
+# This runs after ignition-kargs & before ignition-disks so that it can optionally reboot
+# if kargs were modified via Ignition. This is done in a two-stage fashion so that other
+# mechanisms which may want to reboot (e.x. FIPS) can also hook in here and only reboot
+# once from the initrd.
+After=ignition-kargs.service
+Before=ignition-disks.service
+
+OnFailure=emergency.target
+OnFailureJobMode=isolate
+
+[Service]
+Type=oneshot
+RemainAfterExit=yes
+# --force causes a rapid reboot.  Without it, systemd continues running
+# Ignition stages in parallel with shutting down.
+ExecStart=/usr/bin/systemctl reboot --force
diff --git a/overlay.d/05core/usr/lib/dracut/modules.d/35coreos-ignition/coreos-kargs.sh b/overlay.d/05core/usr/lib/dracut/modules.d/35coreos-ignition/coreos-kargs.sh
new file mode 100755
index 0000000000..1dcc3add13
--- /dev/null
+++ b/overlay.d/05core/usr/lib/dracut/modules.d/35coreos-ignition/coreos-kargs.sh
@@ -0,0 +1,12 @@
+#!/bin/bash
+set -euo pipefail
+
+if is-live-image; then
+    /usr/bin/rdcore kargs --current --create-if-changed /run/coreos-kargs-changed "$@"
+    if [ -e /run/coreos-kargs-changed ]; then
+        echo "Need to modify kernel arguments, but cannot affect live system." >&2
+        exit 1
+    fi
+else
+    /usr/bin/rdcore kargs --boot-device /dev/disk/by-label/boot --create-if-changed /run/coreos-kargs-reboot "$@"
+fi
diff --git a/overlay.d/05core/usr/lib/dracut/modules.d/35coreos-ignition/coreos-post-ignition-checks.service b/overlay.d/05core/usr/lib/dracut/modules.d/35coreos-ignition/coreos-post-ignition-checks.service
new file mode 100644
index 0000000000..e37082268d
--- /dev/null
+++ b/overlay.d/05core/usr/lib/dracut/modules.d/35coreos-ignition/coreos-post-ignition-checks.service
@@ -0,0 +1,17 @@
+# This unit will run late in the initrd process after the Ignition files
+# stage has completed successfully so that we may validate ignition changes
+
+[Unit]
+Description=CoreOS Post Ignition Checks
+ConditionPathExists=/usr/lib/initrd-release
+OnFailure=emergency.target
+OnFailureJobMode=isolate
+
+# Start after Ignition has finished creating files and before ignition umount
+After=ignition-files.service
+Before=ignition-complete.target
+
+[Service]
+Type=oneshot
+ExecStart=/usr/sbin/coreos-post-ignition-checks
+RemainAfterExit=yes
diff --git a/overlay.d/05core/usr/lib/dracut/modules.d/35coreos-ignition/coreos-post-ignition-checks.sh b/overlay.d/05core/usr/lib/dracut/modules.d/35coreos-ignition/coreos-post-ignition-checks.sh
new file mode 100755
index 0000000000..12a492618e
--- /dev/null
+++ b/overlay.d/05core/usr/lib/dracut/modules.d/35coreos-ignition/coreos-post-ignition-checks.sh
@@ -0,0 +1,18 @@
+#!/bin/bash
+# See coreos-post-ignition-checks.service for more information about this script
+set -euo pipefail
+
+# Verify that GRUB password directives are only used when GRUB is being used
+arch=$(uname -p)
+# Butane sugar will tell ignition to mount /boot to /sysroot/boot. We can simply check if
+# the file exists to see whether the check needs to be performed.
+# It is possible that the user creates a config, which will mount /boot at a different path
+# but that case is not officially supported.
+if [ -f /sysroot/boot/grub2/user.cfg ]; then
+    # s390x does not use GRUB, ppcle64 uses petitboot with a GRUB config parser which does not support passwords
+    # So in both these cases, GRUB password is not supported
+    if grep -q password_pbkdf2 /sysroot/boot/grub2/user.cfg && [[ "$arch" =~ ^(s390x|ppc64le)$ ]]; then
+        echo "Ignition config provisioned a GRUB password, which is not supported on $arch"
+        exit 1
+    fi
+fi
diff --git a/overlay.d/05core/usr/lib/dracut/modules.d/35coreos-ignition/coreos-rootflags.service b/overlay.d/05core/usr/lib/dracut/modules.d/35coreos-ignition/coreos-rootflags.service
new file mode 100644
index 0000000000..3981f08510
--- /dev/null
+++ b/overlay.d/05core/usr/lib/dracut/modules.d/35coreos-ignition/coreos-rootflags.service
@@ -0,0 +1,25 @@
+[Unit]
+Description=Determine root FS mount option flags
+DefaultDependencies=false
+# Just in case. We shouldn't run twice.
+ConditionPathExists=!/run/coreos-rootflags
+# Since the sysroot.mount depends on the options generated by us
+# we must run before sysroot.mount
+Before=sysroot.mount
+# In case the disk was manipulated by Ignition and we need to pick
+# up mount options from Ignition config.
+After=ignition-disks.service
+# The script checks if the rootfs was reprovisioned so let's wait
+# until after that has completed if it was requested.
+After=ignition-ostree-transposefs-restore.service
+# Needs to run after the root device is available
+Requires=dev-disk-by\x2dlabel-root.device
+After=dev-disk-by\x2dlabel-root.device
+
+[Service]
+Type=oneshot
+StandardError=journal
+StandardOutput=append:/run/coreos-rootflags
+ExecStartPre=echo -n 'OPTIONS='
+ExecStart=/usr/sbin/coreos-rootflags
+RemainAfterExit=yes
diff --git a/overlay.d/05core/usr/lib/dracut/modules.d/35coreos-ignition/coreos-rootflags.sh b/overlay.d/05core/usr/lib/dracut/modules.d/35coreos-ignition/coreos-rootflags.sh
new file mode 100755
index 0000000000..a84c76119d
--- /dev/null
+++ b/overlay.d/05core/usr/lib/dracut/modules.d/35coreos-ignition/coreos-rootflags.sh
@@ -0,0 +1,30 @@
+#!/bin/bash
+set -euo pipefail
+
+# see related comment block in transposefs.sh re. inspecting the config directly
+ignition_cfg=/run/ignition.json
+rootpath=/dev/disk/by-label/root
+
+query_rootfs() {
+    local filter=$1
+    jq -re ".storage?.filesystems? // [] |
+                map(select(.label == \"root\" and .wipeFilesystem == true)) |
+                .[0] | $filter" "${ignition_cfg}"
+}
+
+# If the rootfs was reprovisioned, then the mountOptions from the Ignition
+# config has priority.
+if [ -d /run/ignition-ostree-transposefs/root ]; then
+    if query_rootfs 'has("mountOptions")' >/dev/null; then
+        query_rootfs '.mountOptions | join(",")'
+        exit 0
+    fi
+fi
+
+eval $(blkid -p -o export ${rootpath})
+if [ "${TYPE}" == "xfs" ]; then
+    # We use prjquota on XFS by default to aid multi-tenant Kubernetes (and
+    # other container) clusters.  See
+    # https://github.com/coreos/coreos-assembler/pull/303/commits/6103effbd006bb6109467830d6a3e42dd847668d
+    echo "prjquota"
+fi
diff --git a/overlay.d/05core/usr/lib/dracut/modules.d/35coreos-ignition/coreos-secex-ignition-prepare.service b/overlay.d/05core/usr/lib/dracut/modules.d/35coreos-ignition/coreos-secex-ignition-prepare.service
new file mode 100644
index 0000000000..a9dd23f565
--- /dev/null
+++ b/overlay.d/05core/usr/lib/dracut/modules.d/35coreos-ignition/coreos-secex-ignition-prepare.service
@@ -0,0 +1,20 @@
+# This service is enabled by coreos-diskful-generator
+
+[Unit]
+Description=CoreOS Secex Ignition Config Preparation
+ConditionPathExists=/etc/initrd-release
+ConditionPathExists=/run/coreos/secure-execution
+DefaultDependencies=false
+
+OnFailure=emergency.target
+OnFailureJobMode=isolate
+
+# Run after the crypt device becomes available and before Ignition
+Requires=dev-disk-by\x2did-virtio\x2dignition_crypted.device
+After=dev-disk-by\x2did-virtio\x2dignition_crypted.device
+Before=ignition-fetch-offline.service
+
+[Service]
+Type=oneshot
+RemainAfterExit=yes
+ExecStart=/usr/sbin/coreos-secex-ignition-prepare
diff --git a/overlay.d/05core/usr/lib/dracut/modules.d/35coreos-ignition/coreos-secex-ignition-prepare.sh b/overlay.d/05core/usr/lib/dracut/modules.d/35coreos-ignition/coreos-secex-ignition-prepare.sh
new file mode 100755
index 0000000000..018c640258
--- /dev/null
+++ b/overlay.d/05core/usr/lib/dracut/modules.d/35coreos-ignition/coreos-secex-ignition-prepare.sh
@@ -0,0 +1,30 @@
+#!/bin/bash
+set -euo pipefail
+
+disk=/dev/disk/by-id/virtio-ignition_crypted
+conf=/usr/lib/ignition/user.ign
+pkey=/usr/lib/coreos/ignition.asc
+tmpd=
+
+cleanup() {
+    rm -f "${pkey}"
+    if [[ -n "${tmpd}" ]]; then
+        rm -rf "${tmpd}"
+    fi
+}
+
+trap cleanup EXIT
+
+# copy base Secure Execution config (enables LUKS+dm-verity for boot and root partitions)
+cp /usr/lib/coreos/01-secex.ign /usr/lib/ignition/base.d/01-secex.ign
+
+# decrypt user's config
+tmpd=$(mktemp -d)
+
+if [ ! -e "${disk}" ]; then
+    echo "Ignition config must be encrypted"
+    exit 1
+fi
+
+gpg --homedir "${tmpd}" --import "${pkey}" && rm "${pkey}"
+gpg --homedir "${tmpd}" --skip-verify --output "${conf}" --decrypt "${disk}"
diff --git a/overlay.d/05core/usr/lib/dracut/modules.d/30ignition-coreos/coreos-teardown-initramfs.service b/overlay.d/05core/usr/lib/dracut/modules.d/35coreos-ignition/coreos-teardown-initramfs.service
similarity index 78%
rename from overlay.d/05core/usr/lib/dracut/modules.d/30ignition-coreos/coreos-teardown-initramfs.service
rename to overlay.d/05core/usr/lib/dracut/modules.d/35coreos-ignition/coreos-teardown-initramfs.service
index a2131afcbc..b08c827bc5 100644
--- a/overlay.d/05core/usr/lib/dracut/modules.d/30ignition-coreos/coreos-teardown-initramfs.service
+++ b/overlay.d/05core/usr/lib/dracut/modules.d/35coreos-ignition/coreos-teardown-initramfs.service
@@ -3,7 +3,7 @@
 # https://github.com/coreos/fedora-coreos-tracker/issues/394#issuecomment-599721763
 
 [Unit]
-Description=CoreOS Tear down initramfs
+Description=CoreOS Tear Down Initramfs
 DefaultDependencies=false
 
 # We want to run the teardown after all other Ignition stages
@@ -18,6 +18,14 @@ DefaultDependencies=false
 Before=ignition-mount.service
 Before=ignition-complete.target
 
+# Since we are tearing down networking we need to make sure
+# NetworkManager has been stopped, otherwise it'll be trying
+# to react to our delete/down operations. Since the ordering
+# for ExecStop is the opposite of ExecStart we need to use
+# `Before=nm-initrd.service`.
+# https://issues.redhat.com/browse/OCPBUGS-11052
+Before=nm-initrd.service
+
 # Make sure ExecStop= runs before we switch root
 Conflicts=initrd-switch-root.target umount.target
 Before=initrd-switch-root.target
diff --git a/overlay.d/05core/usr/lib/dracut/modules.d/30ignition-coreos/coreos-teardown-initramfs.sh b/overlay.d/05core/usr/lib/dracut/modules.d/35coreos-ignition/coreos-teardown-initramfs.sh
similarity index 65%
rename from overlay.d/05core/usr/lib/dracut/modules.d/30ignition-coreos/coreos-teardown-initramfs.sh
rename to overlay.d/05core/usr/lib/dracut/modules.d/35coreos-ignition/coreos-teardown-initramfs.sh
index 2c8e5353a5..8312b534a4 100755
--- a/overlay.d/05core/usr/lib/dracut/modules.d/30ignition-coreos/coreos-teardown-initramfs.sh
+++ b/overlay.d/05core/usr/lib/dracut/modules.d/35coreos-ignition/coreos-teardown-initramfs.sh
@@ -22,15 +22,19 @@ dracut_func() {
     return $rc
 }
 
-selinux_relabel() {
-    # If we have access to coreos-relabel then let's use that because
-    # it allows us to set labels on things before switching root
-    # If not, fallback to tmpfiles.
-    if command -v coreos-relabel; then
-        coreos-relabel $1
-    else
-        echo "Z $1 - - -" >> "/run/tmpfiles.d/$(basename $0)-relabel.conf"
+# Get the BOOTIF and rd.bootif kernel arguments from
+# the kernel command line.
+get_bootif_kargs() {
+    bootif_kargs=""
+    bootif_karg=$(dracut_func getarg BOOTIF)
+    if [ ! -z "$bootif_karg" ]; then
+        bootif_kargs+="BOOTIF=${bootif_karg}"
+    fi
+    rdbootif_karg=$(dracut_func getarg rd.bootif)
+    if [ ! -z "$rdbootif_karg" ]; then
+        bootif_kargs+=" rd.bootif=${rdbootif_karg}"
     fi
+    echo $bootif_kargs
 }
 
 # Determine if the generated NM connection profiles match the default
@@ -41,6 +45,12 @@ selinux_relabel() {
 # If it matches then it was the default, if not then the user provided
 # something extra.
 are_default_NM_configs() {
+    # pick up our CoreOS default networking kargs from the afterburn dropin
+    DEFAULT_KARGS_FILE=/usr/lib/systemd/system/afterburn-network-kargs.service.d/50-afterburn-network-kargs-default.conf
+    source <(grep -o 'AFTERBURN_NETWORK_KARGS_DEFAULT=.*' $DEFAULT_KARGS_FILE)
+    # Also pick up BOOTIF/rd.bootif kargs and apply them here.
+    # See https://github.com/coreos/fedora-coreos-tracker/issues/1048
+    BOOTIF_KARGS=$(get_bootif_kargs)
     # Make two dirs for storing files to use in the comparison
     mkdir -p /run/coreos-teardown-initramfs/connections-compare-{1,2}
     # Make another that's just a throwaway for the initrd-data-dir
@@ -48,10 +58,14 @@ are_default_NM_configs() {
     # Copy over the previously generated connection(s) profiles
     cp  /run/NetworkManager/system-connections/* \
         /run/coreos-teardown-initramfs/connections-compare-1/
+    # Delete lo.nmconnection if it was created.
+    # https://github.com/coreos/fedora-coreos-tracker/issues/1385
+    rm -f /run/coreos-teardown-initramfs/connections-compare-1/lo.nmconnection
     # Do a new run with the default input
     /usr/libexec/nm-initrd-generator \
         -c /run/coreos-teardown-initramfs/connections-compare-2 \
-        -i /run/coreos-teardown-initramfs/initrd-data-dir -- ip=dhcp,dhcp6
+        -i /run/coreos-teardown-initramfs/initrd-data-dir \
+        -- $AFTERBURN_NETWORK_KARGS_DEFAULT $BOOTIF_KARGS
     # remove unique identifiers from the files (so our diff can work)
     sed -i '/^uuid=/d' /run/coreos-teardown-initramfs/connections-compare-{1,2}/*
     # currently the output will differ based on whether rd.neednet=1
@@ -81,21 +95,45 @@ are_default_NM_configs() {
 #
 # See https://github.com/coreos/fedora-coreos-tracker/issues/394#issuecomment-599721173
 propagate_initramfs_networking() {
-    # Check the two locations where a user could have provided network configuration
-    # On FCOS we only support keyfiles, but on RHCOS we support keyfiles and ifcfg
+    # Check for any real root config in the two locations where a user could have
+    # provided network configuration. On FCOS we only support keyfiles, but on RHCOS
+    # we support keyfiles and ifcfg. We also need to ignore readme-ifcfg-rh.txt
+    # which is a cosmetic file added in
+    # https://gitlab.freedesktop.org/NetworkManager/NetworkManager/-/commit/96d7362
     if [ -n "$(ls -A /sysroot/etc/NetworkManager/system-connections/)" -o \
-         -n "$(ls -A /sysroot/etc/sysconfig/network-scripts/)" ]; then
+         -n "$(ls -A -I readme-ifcfg-rh.txt /sysroot/etc/sysconfig/network-scripts/)" ]; then
         echo "info: networking config is defined in the real root"
-        echo "info: will not attempt to propagate initramfs networking"
+        realrootconfig=1
     else
         echo "info: no networking config is defined in the real root"
+        realrootconfig=0
+    fi
+
+    # Did the user tell us to force initramfs networking config
+    # propagation even if real root networking config exists?
+    # Hopefully we only need this in rare circumstances.
+    # https://github.com/coreos/fedora-coreos-tracker/issues/853
+    forcepropagate=0
+    if dracut_func getargbool 0 'coreos.force_persist_ip'; then
+        forcepropagate=1
+        echo "info: coreos.force_persist_ip detected: will force network config propagation"
+    fi
+
+    if [ $realrootconfig == 1 -a $forcepropagate == 0 ]; then
+        echo "info: will not attempt to propagate initramfs networking"
+    fi
+
+    if [ $realrootconfig == 0 -o $forcepropagate == 1 ]; then
         if [ -n "$(ls -A /run/NetworkManager/system-connections/)" ]; then
             if are_default_NM_configs; then
                 echo "info: skipping propagation of default networking configs"
             else
                 echo "info: propagating initramfs networking config to the real root"
                 cp -v /run/NetworkManager/system-connections/* /sysroot/etc/NetworkManager/system-connections/
-                selinux_relabel /etc/NetworkManager/system-connections/
+                # Delete lo.nmconnection if it was created.
+                # https://github.com/coreos/fedora-coreos-tracker/issues/1385
+                rm -vf /sysroot/etc/NetworkManager/system-connections/lo.nmconnection
+                coreos-relabel /etc/NetworkManager/system-connections/
             fi
         else
             echo "info: no initramfs networking information to propagate"
@@ -117,43 +155,6 @@ propagate_initramfs_hostname() {
         return 0
     fi
 
-    # COMPAT: keep two code paths, one for older NetworkManager and
-    #         one for newer NetworkManager that supports writing to
-    #         /run/NetworkManager/initrd/hostname. We can delete this
-    #         block once RHCOS and FCOS minimum NM version is >= 1.26.0
-    #         See https://gitlab.freedesktop.org/NetworkManager/NetworkManager/-/commit/ff70adf
-    barrierversion='1.26.0'
-    nmversion=$(/usr/sbin/NetworkManager --version)
-    sorted=$((echo $barrierversion; echo $nmversion) | sort -V | tail -n 1)
-    if [ $sorted == $barrierversion ]; then
-        # The version of NM on the system is older than we need
-        # execute compat code in this block.
-        echo "info: NM version is older than $barrierversion. Executing compat code path."
-
-        # Detect if any hostname was provided via static ip= kargs
-        # run in a subshell so we don't pollute our environment
-        hostnamefile=$(mktemp)
-        (
-            last_nonempty_hostname=''
-            # Inspired from ifup.sh from the 40network dracut module. Note that
-            # $hostname from ip_to_var will only be nonempty for static networking.
-            for iparg in $(dracut_func getargs ip=); do
-                dracut_func ip_to_var $iparg
-                [ -n "${hostname:-}" ] && last_nonempty_hostname="$hostname"
-            done
-            echo -n "$last_nonempty_hostname" > $hostnamefile
-        )
-        hostname=$(<$hostnamefile); rm $hostnamefile
-        if [ -n "$hostname" ]; then
-            echo "info: propagating initramfs hostname (${hostname}) to the real root"
-            echo $hostname > /sysroot/etc/hostname
-            selinux_relabel /etc/hostname
-        else
-            echo "info: no initramfs hostname information to propagate"
-        fi
-        return 0
-    fi
-
     # If any hostname was provided NetworkManager will write it out to
     # /run/NetworkManager/initrd/hostname. See
     # https://gitlab.freedesktop.org/NetworkManager/NetworkManager/-/merge_requests/481
@@ -161,25 +162,23 @@ propagate_initramfs_hostname() {
         hostname=$(</run/NetworkManager/initrd/hostname)
         echo "info: propagating initramfs hostname (${hostname}) to the real root"
         echo $hostname > /sysroot/etc/hostname
-        selinux_relabel /etc/hostname
+        coreos-relabel /etc/hostname
     else
         echo "info: no initramfs hostname information to propagate"
     fi
 }
 
-# Persist automatic multipath configuration, if any.
-# When booting with `rd.multipath=default`, the default multipath
-# configuration is written. We need to ensure that the mutlipath configuration
-# is persisted to the final target.
-propagate_initramfs_multipath() {
-    if [ ! -f /sysroot/etc/multipath.conf ] && [ -f /etc/multipath.conf ]; then
-        echo "info: propagating automatic multipath configuration"
-        cp -v /etc/multipath.conf /sysroot/etc/
-        mkdir -p /sysroot/etc/multipath/multipath.conf.d
-        selinux_relabel /etc/multipath.conf
-        selinux_relabel /etc/multipath/multipath.conf.d
-    else
-        echo "info: no initramfs automatic multipath configuration to propagate"
+# Propagate the ifname= karg udev rules. The policy here is:
+#
+#     - IF ifname karg udev rule file exists
+#     - THEN copy it to the real root
+#
+propagate_ifname_udev_rules() {
+    local udev_file='/etc/udev/rules.d/80-ifname.rules'
+    if [ -e "${udev_file}" ]; then
+        echo "info: propagating ifname karg udev rules to the real root"
+        cp -v "${udev_file}" "/sysroot/${udev_file}"
+        coreos-relabel "${udev_file}"
     fi
 }
 
@@ -198,7 +197,7 @@ down_interface() {
 }
 
 # Iterate through the interfaces in the machine and take them down.
-# Note that in the futre we would like to possibly use `nmcli` networking off`
+# Note that in the future we would like to possibly use `nmcli` networking off`
 # for this. See the following two comments for details:
 # https://github.com/coreos/fedora-coreos-tracker/issues/394#issuecomment-599721763
 # https://github.com/coreos/fedora-coreos-tracker/issues/394#issuecomment-599746049
@@ -230,13 +229,22 @@ main() {
     # Load libraries from dracut
     load_dracut_libs
 
-    # Take down all interfaces set up in the initramfs
-    down_interfaces
+    # If we're using iSCSI, then we can't tear down networking since we'll lose
+    # root. This means in that case that the network config written to the real
+    # root won't be applied "from scratch". But anyway, since networking must
+    # stay on, it's simply not supported to configure the real root in a way
+    # that would require tearing down the connection on the interface involved.
+    if dracut_func getargbool 0 rd.iscsi.firmware || dracut_func getarg netroot; then
+        echo "info: iSCSI in use; not tearing down networking"
+    else
+        # Take down all interfaces set up in the initramfs
+        down_interfaces
 
-    # Clean up all routing
-    echo "info: flushing all routing"
-    ip route flush table main
-    ip route flush cache
+        # Clean up all routing
+        echo "info: flushing all routing"
+        ip route flush table main
+        ip route flush cache
+    fi
 
     # Hopefully our logic is sound enough that this is never needed, but
     # user's can explicitly disable initramfs network/hostname propagation
@@ -247,6 +255,7 @@ main() {
     else
         propagate_initramfs_hostname
         propagate_initramfs_networking
+        propagate_ifname_udev_rules
     fi
 
     # Now that the configuration has been propagated (or not)
@@ -254,9 +263,8 @@ main() {
     # real root is passed on to NetworkManager in the real root
     rm -rf /run/NetworkManager/
 
-    # If automated multipath configuration has been enabled, ensure
-    # that its propagated to the real rootfs.
-    propagate_initramfs_multipath
+    rm -f /run/udev/rules.d/80-coreos-boot-disk.rules
+    rm -f /dev/disk/by-id/coreos-boot-disk
 }
 
 main
diff --git a/overlay.d/05core/usr/lib/dracut/modules.d/35coreos-ignition/coreos-unique-boot.service b/overlay.d/05core/usr/lib/dracut/modules.d/35coreos-ignition/coreos-unique-boot.service
new file mode 100644
index 0000000000..e02d72e588
--- /dev/null
+++ b/overlay.d/05core/usr/lib/dracut/modules.d/35coreos-ignition/coreos-unique-boot.service
@@ -0,0 +1,22 @@
+[Unit]
+Description=CoreOS Ensure Unique Boot Filesystem Label
+ConditionPathExists=/etc/initrd-release
+DefaultDependencies=no
+Before=ignition-diskful.target
+Wants=systemd-udevd.service
+After=systemd-udevd.service
+# And since the boot device may be on multipath; optionally wait for it to
+# appear via the dynamic target.
+After=coreos-multipath-wait.target
+Requires=dev-disk-by\x2dlabel-boot.device
+After=dev-disk-by\x2dlabel-boot.device
+# Run before services that modify/use `boot` partition
+Before=coreos-gpt-setup.service coreos-boot-edit.service
+
+OnFailure=emergency.target
+OnFailureJobMode=isolate
+
+[Service]
+Type=oneshot
+RemainAfterExit=yes
+ExecStart=/usr/bin/rdcore verify-unique-fs-label boot
diff --git a/overlay.d/05core/usr/lib/dracut/modules.d/35coreos-ignition/module-setup.sh b/overlay.d/05core/usr/lib/dracut/modules.d/35coreos-ignition/module-setup.sh
new file mode 100755
index 0000000000..c3fd96bfc5
--- /dev/null
+++ b/overlay.d/05core/usr/lib/dracut/modules.d/35coreos-ignition/module-setup.sh
@@ -0,0 +1,96 @@
+#!/bin/bash
+# -*- mode: shell-script; indent-tabs-mode: nil; sh-basic-offset: 4; -*-
+# ex: ts=8 sw=4 sts=4 et filetype=sh
+
+depends() {
+    echo systemd network ignition coreos-live
+}
+
+install_ignition_unit() {
+    local unit="$1"; shift
+    local target="${1:-ignition-complete.target}"; shift
+    local instantiated="${1:-$unit}"; shift
+    inst_simple "$moddir/$unit" "$systemdsystemunitdir/$unit"
+    # note we `|| exit 1` here so we error out if e.g. the units are missing
+    # see https://github.com/coreos/fedora-coreos-config/issues/799
+    systemctl -q --root="$initdir" add-requires "$target" "$instantiated" || exit 1
+}
+
+install() {
+    inst_multiple \
+        basename \
+        diff \
+        lsblk \
+        sed \
+        grep \
+        sgdisk \
+        uname
+
+    # For IBM SecureExecution
+    if [[ $(uname -m) = s390x ]]; then
+        inst_multiple \
+            gpg \
+            gpg-agent
+    fi
+
+    inst_simple "$moddir/coreos-diskful-generator" \
+        "$systemdutildir/system-generators/coreos-diskful-generator"
+
+    inst_script "$moddir/coreos-gpt-setup.sh" \
+        "/usr/sbin/coreos-gpt-setup"
+
+    # This has to work only on diskful systems during firstboot.
+    # coreos-diskful-generator will create a symlink
+    inst_simple "$moddir/80-coreos-boot-disk.rules" \
+        "/usr/lib/coreos/80-coreos-boot-disk.rules"
+
+    inst_script "$moddir/coreos-disk-contains-fs.sh" \
+        "/usr/lib/udev/coreos-disk-contains-fs"
+
+    inst_script "$moddir/coreos-ignition-setup-user.sh" \
+        "/usr/sbin/coreos-ignition-setup-user"
+
+    inst_script "$moddir/coreos-post-ignition-checks.sh" \
+        "/usr/sbin/coreos-post-ignition-checks"
+
+    install_ignition_unit coreos-post-ignition-checks.service
+
+    # For consistency tear down the network and persist multipath between the initramfs and
+    # real root. See https://github.com/coreos/fedora-coreos-tracker/issues/394#issuecomment-599721763
+    inst_script "$moddir/coreos-teardown-initramfs.sh" \
+        "/usr/sbin/coreos-teardown-initramfs"
+    install_ignition_unit coreos-teardown-initramfs.service
+
+    # units only started when we have a boot disk
+    # path generated by systemd-escape --path /dev/disk/by-label/root
+    install_ignition_unit coreos-gpt-setup.service ignition-diskful.target
+
+    # dracut inst_script doesn't allow overwrites and we are replacing
+    # the default script placed by Ignition
+    binpath="/usr/sbin/ignition-kargs-helper"
+    cp "$moddir/coreos-kargs.sh" "$initdir$binpath"
+    install_ignition_unit coreos-kargs-reboot.service
+
+    inst_script "$moddir/coreos-boot-edit.sh" \
+        "/usr/sbin/coreos-boot-edit"
+    # Only start when the system has disks since we are editing /boot.
+    install_ignition_unit "coreos-boot-edit.service" \
+        "ignition-diskful.target"
+
+    install_ignition_unit coreos-ignition-unique-boot.service ignition-diskful.target
+    install_ignition_unit coreos-unique-boot.service ignition-diskful.target
+    install_ignition_unit coreos-ignition-setup-user.service
+
+    # IBM Secure Execution. Ignition config for reencryption of / and /boot
+    inst_simple "$moddir/01-secex.ign" /usr/lib/coreos/01-secex.ign
+    inst_simple "$moddir/coreos-secex-ignition-prepare.service" \
+        "$systemdsystemunitdir/coreos-secex-ignition-prepare.service"
+    inst_script "$moddir/coreos-secex-ignition-prepare.sh" \
+        "/usr/sbin/coreos-secex-ignition-prepare"
+
+    inst_multiple jq blkid
+    inst_script "$moddir/coreos-rootflags.sh" \
+        "/usr/sbin/coreos-rootflags"
+    # Install unit, but don't enable it. Will be pulled in by diskful generator.
+    inst_simple "$moddir/coreos-rootflags.service" "$systemdsystemunitdir/coreos-rootflags.service"
+}
diff --git a/overlay.d/05core/usr/lib/dracut/modules.d/20live/coreos-live-clear-sssd-cache.service b/overlay.d/05core/usr/lib/dracut/modules.d/35coreos-live/coreos-live-clear-sssd-cache.service
similarity index 90%
rename from overlay.d/05core/usr/lib/dracut/modules.d/20live/coreos-live-clear-sssd-cache.service
rename to overlay.d/05core/usr/lib/dracut/modules.d/35coreos-live/coreos-live-clear-sssd-cache.service
index 758bb617af..e30b62d408 100644
--- a/overlay.d/05core/usr/lib/dracut/modules.d/20live/coreos-live-clear-sssd-cache.service
+++ b/overlay.d/05core/usr/lib/dracut/modules.d/35coreos-live/coreos-live-clear-sssd-cache.service
@@ -4,7 +4,7 @@
 # cache to avoid this.
 
 [Unit]
-Description=Clear SSSD NSS cache in persistent /var
+Description=Clear SSSD NSS Cache in Persistent /var
 DefaultDependencies=false
 ConditionPathExists=/run/ostree-live
 ConditionPathExists=/sysroot/var/lib/sss/mc
diff --git a/overlay.d/05core/usr/lib/dracut/modules.d/20live/coreos-live-unmount-tmpfs-var.service b/overlay.d/05core/usr/lib/dracut/modules.d/35coreos-live/coreos-live-unmount-tmpfs-var.service
similarity index 93%
rename from overlay.d/05core/usr/lib/dracut/modules.d/20live/coreos-live-unmount-tmpfs-var.service
rename to overlay.d/05core/usr/lib/dracut/modules.d/35coreos-live/coreos-live-unmount-tmpfs-var.service
index de5080f611..17ae51faaa 100644
--- a/overlay.d/05core/usr/lib/dracut/modules.d/20live/coreos-live-unmount-tmpfs-var.service
+++ b/overlay.d/05core/usr/lib/dracut/modules.d/35coreos-live/coreos-live-unmount-tmpfs-var.service
@@ -7,7 +7,7 @@
 # before switching roots.
 
 [Unit]
-Description=Unmount live /var if persistent /var is configured
+Description=Unmount Live /var if Persistent /var Is Configured
 DefaultDependencies=false
 ConditionPathExists=/run/ostree-live
 ConditionPathExists=|/sysroot/etc/systemd/system/var.mount
diff --git a/overlay.d/05core/usr/lib/dracut/modules.d/20live/coreos-live-unmount-tmpfs-var.sh b/overlay.d/05core/usr/lib/dracut/modules.d/35coreos-live/coreos-live-unmount-tmpfs-var.sh
similarity index 100%
rename from overlay.d/05core/usr/lib/dracut/modules.d/20live/coreos-live-unmount-tmpfs-var.sh
rename to overlay.d/05core/usr/lib/dracut/modules.d/35coreos-live/coreos-live-unmount-tmpfs-var.sh
diff --git a/overlay.d/05core/usr/lib/dracut/modules.d/20live/coreos-liveiso-persist-osmet.service b/overlay.d/05core/usr/lib/dracut/modules.d/35coreos-live/coreos-liveiso-persist-osmet.service
similarity index 72%
rename from overlay.d/05core/usr/lib/dracut/modules.d/20live/coreos-liveiso-persist-osmet.service
rename to overlay.d/05core/usr/lib/dracut/modules.d/35coreos-live/coreos-liveiso-persist-osmet.service
index 95684102df..e4cc6dba27 100644
--- a/overlay.d/05core/usr/lib/dracut/modules.d/20live/coreos-liveiso-persist-osmet.service
+++ b/overlay.d/05core/usr/lib/dracut/modules.d/35coreos-live/coreos-liveiso-persist-osmet.service
@@ -1,10 +1,12 @@
 [Unit]
-Description=Persist osmet files (ISO)
-DefaultDependencies=false
+Description=Persist Osmet Files (ISO)
 ConditionPathExists=/run/ostree-live
 ConditionKernelCommandLine=coreos.liveiso
 RequiresMountsFor=/run/media/iso
 Before=initrd-switch-root.target
+# DefaultDependencies=true so this unit gets stopped on switchroot to
+# allow for /run/media/iso to get unmounted.
+DefaultDependencies=true
 
 [Service]
 Type=oneshot
diff --git a/overlay.d/05core/usr/lib/dracut/modules.d/20live/coreos-livepxe-persist-osmet.service b/overlay.d/05core/usr/lib/dracut/modules.d/35coreos-live/coreos-livepxe-persist-osmet.service
similarity index 92%
rename from overlay.d/05core/usr/lib/dracut/modules.d/20live/coreos-livepxe-persist-osmet.service
rename to overlay.d/05core/usr/lib/dracut/modules.d/35coreos-live/coreos-livepxe-persist-osmet.service
index 17484e66a1..63baa687df 100644
--- a/overlay.d/05core/usr/lib/dracut/modules.d/20live/coreos-livepxe-persist-osmet.service
+++ b/overlay.d/05core/usr/lib/dracut/modules.d/35coreos-live/coreos-livepxe-persist-osmet.service
@@ -1,5 +1,5 @@
 [Unit]
-Description=Persist osmet files (PXE)
+Description=Persist Osmet Files (PXE)
 DefaultDependencies=false
 ConditionPathExists=/run/ostree-live
 ConditionKernelCommandLine=!coreos.liveiso
diff --git a/overlay.d/05core/usr/lib/dracut/modules.d/20live/coreos-livepxe-rootfs.service b/overlay.d/05core/usr/lib/dracut/modules.d/35coreos-live/coreos-livepxe-rootfs.service
similarity index 79%
rename from overlay.d/05core/usr/lib/dracut/modules.d/20live/coreos-livepxe-rootfs.service
rename to overlay.d/05core/usr/lib/dracut/modules.d/35coreos-live/coreos-livepxe-rootfs.service
index 337b55bcd0..fdb5ad9647 100644
--- a/overlay.d/05core/usr/lib/dracut/modules.d/20live/coreos-livepxe-rootfs.service
+++ b/overlay.d/05core/usr/lib/dracut/modules.d/35coreos-live/coreos-livepxe-rootfs.service
@@ -1,5 +1,5 @@
 [Unit]
-Description=Acquire live PXE rootfs image
+Description=Acquire Live PXE rootfs Image
 DefaultDependencies=false
 ConditionPathExists=/usr/lib/initrd-release
 ConditionPathExists=/run/ostree-live
@@ -7,6 +7,8 @@ ConditionKernelCommandLine=!coreos.liveiso
 
 After=basic.target
 # Network is enabled here
+After=nm-run.service
+# compat: remove when everyone is on dracut 053+
 After=dracut-initqueue.service
 
 # If we fail, the boot will fail.  Be explicit about it.
diff --git a/overlay.d/05core/usr/lib/dracut/modules.d/35coreos-live/coreos-livepxe-rootfs.sh b/overlay.d/05core/usr/lib/dracut/modules.d/35coreos-live/coreos-livepxe-rootfs.sh
new file mode 100755
index 0000000000..d05709e360
--- /dev/null
+++ b/overlay.d/05core/usr/lib/dracut/modules.d/35coreos-live/coreos-livepxe-rootfs.sh
@@ -0,0 +1,88 @@
+#!/bin/bash
+# Ensure that a PXE-booted system has a valid rootfs.
+
+set -euo pipefail
+
+# Get rootfs_url karg
+set +euo pipefail
+. /usr/lib/dracut-lib.sh
+rootfs_url=$(getarg coreos.live.rootfs_url=)
+set -euo pipefail
+
+if [[ -f /etc/coreos-live-rootfs ]]; then
+    # rootfs image was injected via PXE.  Verify that the initramfs and
+    # rootfs versions match.
+    initramfs_ver=$(cat /etc/coreos-live-initramfs)
+    rootfs_ver=$(cat /etc/coreos-live-rootfs)
+    if [[ $initramfs_ver != $rootfs_ver ]]; then
+        echo "Found initramfs version $initramfs_ver but rootfs version $rootfs_ver." >&2
+        echo "Please fix your PXE configuration." >&2
+        exit 1
+    fi
+elif [[ -n "${rootfs_url}" ]]; then
+    # rootfs URL was provided as karg.  Fetch image, check its hash, and
+    # unpack it.
+    echo "Fetching rootfs image from ${rootfs_url}..."
+    if [[ ${rootfs_url} != http:* && ${rootfs_url} != https:* && ${rootfs_url} != tftp:* ]]; then
+        # Don't commit to supporting protocols we might not want to expose in
+        # the long term.
+        echo "Unsupported scheme for image specified by:" >&2
+        echo "coreos.live.rootfs_url=${rootfs_url}" >&2
+        echo "Only HTTP, HTTPS, and TFTP are supported. Please fix your PXE configuration." >&2
+        exit 1
+    fi
+
+    # First, reach out to the server to verify connectivity before
+    # trying to download and pipe content through other programs.
+    # Doing this allows us to retry all errors (including transient
+    # "no route to host" errors during startup). Note we can't use
+    # curl's --retry-all-errors here because it's not in el8's curl yet.
+    # We don't need to verify TLS certificates because we're checking the
+    # image hash. We retry forever, matching Ignition's semantics.
+    curl_common_args="--silent --show-error --insecure --location"
+    while ! curl --head $curl_common_args "${rootfs_url}" >/dev/null; do
+        echo "Couldn't establish connectivity with the server specified by:" >&2
+        echo "coreos.live.rootfs_url=${rootfs_url}" >&2
+        echo "Retrying in 5s..." >&2
+        sleep 5
+    done
+
+    # We shouldn't need a --retry from here on since we've just successfully
+    # HEADed the file, but let's add one just to be safe (e.g. if the
+    # connection just went online and flickers or something).
+    curl_common_args+=" --retry 5"
+
+    # Do a HEAD again but just once and with `--fail` so that if e.g. it's
+    # missing, we get a clearer error than if it were part of the pipeline
+    # below. Otherwise, the `curl` error emitted there would get lost among
+    # all the spurious errors from the other commands in that pipeline and also
+    # wouldn't show up in the journal logs dumped by `emergency-shell.sh` since
+    # it only prints 10 lines.
+    curl_common_args+=" --fail"
+    if ! curl --head $curl_common_args "${rootfs_url}" >/dev/null; then
+        echo "Couldn't query the server for the rootfs specified by:" >&2
+        echo "coreos.live.rootfs_url=${rootfs_url}" >&2
+        exit 1
+    fi
+
+    # bsdtar can read cpio archives and we already depend on it for
+    # coreos-liveiso-persist-osmet.service, so use it instead of cpio.
+    if ! curl $curl_common_args "${rootfs_url}" | \
+            rdcore stream-hash /etc/coreos-live-want-rootfs | \
+            bsdtar -xf - -C / ; then
+        echo "Couldn't fetch, verify, and unpack image specified by:" >&2
+        echo "coreos.live.rootfs_url=${rootfs_url}" >&2
+        echo "Check that the URL is correct and that the rootfs version matches the initramfs." >&2
+        source /etc/os-release
+        if [ -n "${OSTREE_VERSION:-}" ]; then
+            echo "The version of this initramfs is ${OSTREE_VERSION}." >&2
+        fi
+        exit 1
+    fi
+else
+    # Nothing.  Fail.
+    echo "No rootfs image found.  Modify your PXE configuration to add the rootfs" >&2
+    echo "image as a second initrd, or use the coreos.live.rootfs_url kernel parameter" >&2
+    echo "to specify an HTTP or HTTPS URL to the rootfs." >&2
+    exit 1
+fi
diff --git a/overlay.d/05core/usr/lib/dracut/modules.d/20live/is-live-image.sh b/overlay.d/05core/usr/lib/dracut/modules.d/35coreos-live/is-live-image.sh
similarity index 100%
rename from overlay.d/05core/usr/lib/dracut/modules.d/20live/is-live-image.sh
rename to overlay.d/05core/usr/lib/dracut/modules.d/35coreos-live/is-live-image.sh
diff --git a/overlay.d/05core/usr/lib/dracut/modules.d/20live/live-generator b/overlay.d/05core/usr/lib/dracut/modules.d/35coreos-live/live-generator
similarity index 71%
rename from overlay.d/05core/usr/lib/dracut/modules.d/20live/live-generator
rename to overlay.d/05core/usr/lib/dracut/modules.d/35coreos-live/live-generator
index fec5860061..add4e6dcca 100755
--- a/overlay.d/05core/usr/lib/dracut/modules.d/20live/live-generator
+++ b/overlay.d/05core/usr/lib/dracut/modules.d/35coreos-live/live-generator
@@ -30,8 +30,9 @@ fi
 add_requires sysroot.mount     initrd-root-fs.target
 add_requires sysroot-etc.mount initrd-root-fs.target
 add_requires sysroot-var.mount initrd-root-fs.target
-
-add_requires coreos-liveiso-reconfigure-nm-wait-online.service initrd.target
+# make sure we enable network if required for coreos-livepxe-rootfs
+# https://github.com/coreos/fedora-coreos-tracker/issues/1423
+add_requires coreos-enable-network.service initrd-root-fs.target
 
 mkdir -p "${UNIT_DIR}/ostree-prepare-root.service.d"
 cat > "${UNIT_DIR}/ostree-prepare-root.service.d/10-live.conf" <<EOF
@@ -53,6 +54,7 @@ ExecStartPost=/usr/sbin/ostree-cmdline stop
 EOF
 
 isoroot=$(getarg coreos.liveiso= ||:)
+
 if [ -z "${isoroot}" ]; then
     # In this case, the rootfs is already unpacked into the initrd, or we need
     # to retrieve it
@@ -83,6 +85,7 @@ else
 After=${isosrc_escaped}
 Requires=${isosrc_escaped}
 EOF
+
     cat >"${UNIT_DIR}/run-media-iso.mount" <<EOF
 # Automatically generated by live-generator
 
@@ -90,11 +93,14 @@ EOF
 DefaultDependencies=false
 # HACK for https://github.com/coreos/fedora-coreos-config/issues/437
 Wants=systemd-udev-settle.service
-# Note that `man bootup` implies that initrd-root-device is After=basic.target
+# Note that bootup(7) implies that initrd-root-device is After=basic.target
 # but that appears to not be the case.  We explicitly order after sysinit.target
 After=sysinit.target
 After=initrd-root-device.target
 Before=initrd-root-fs.target
+# This will make /run/media/iso get unmounted when no longer needed,
+# which will aid in the coreos.liveiso.fromram case.
+StopWhenUnneeded=true
 
 [Mount]
 What=/${isosrc}
@@ -103,16 +109,44 @@ Options=ro
 Type=iso9660
 EOF
 
+    # Determine what to mount in sysroot.mount based on if we were asked to run
+    # completely from RAM via coreos.liveiso.fromram karg.
+    if getargbool 0 coreos.liveiso.fromram; then
+        sysrootrequiresmountsfor=""
+        sysrootfsimg=/rootfs.img
+        # Add service to copy the rootfs.img from the ISO to memory
+        cat >"${UNIT_DIR}/coreos-liveiso-run-media-iso-cp-rootfsimg.service" <<EOF
+# Automatically generated by live-generator
+
+[Unit]
+RequiresMountsFor=/run/media/iso
+Before=sysroot.mount
+# DefaultDependencies=true so this unit gets stopped on switchroot to
+# allow for /run/media/iso to get unmounted.
+DefaultDependencies=true
+
+[Service]
+Type=oneshot
+ExecStart=cp -v /run/media/iso/images/pxeboot/rootfs.img /rootfs.img
+RemainAfterExit=yes
+EOF
+        add_requires coreos-liveiso-run-media-iso-cp-rootfsimg.service default.target
+    else
+        sysrootrequiresmountsfor="RequiresMountsFor=/run/media/iso"
+        sysrootfsimg=/run/media/iso/images/pxeboot/rootfs.img
+    fi
+
     cat >"${UNIT_DIR}/sysroot.mount" <<EOF
 # Automatically generated by live-generator
 
 [Unit]
 DefaultDependencies=false
 Before=initrd-root-fs.target
-RequiresMountsFor=/run/media/iso
+After=coreos-liveiso-run-media-iso-cp-rootfsimg.service
+${sysrootrequiresmountsfor}
 
 [Mount]
-What=/run/media/iso/images/pxeboot/rootfs.img
+What=${sysrootfsimg}
 Where=/sysroot
 Type=squashfs
 # Offset of the squashfs within the rootfs cpio.  Assumes newc format
@@ -122,9 +156,22 @@ Options=loop,offset=124
 EOF
 fi
 
-# It turns out that `tmpfs` currently munches all SELinux labels
+# The systemd default for /run is 20% of RAM, but we want to use basically all of RAM,
+# so we make a distinct tmpfs that allows more.  Note the file we create there is
+# "thinly provisioned", so we're not actually allocating all of that.
+cat >>"${UNIT_DIR}/run-ephemeral_base.mount" <<EOF
+[Unit]
+DefaultDependencies=false
+[Mount]
+What=tmpfs
+Where=/run/ephemeral_base
+Type=tmpfs
+Options=size=50%%
+EOF
+
+# But it turns out that `tmpfs` currently munches all SELinux labels
 # we set before policy is loaded, so we make an XFS filesystem
-# loopback mounted that's sized the same as /run.
+# loopback mounted on top of the base tmpfs.
 # https://github.com/coreos/fedora-coreos-config/pull/499
 cat >"${UNIT_DIR}/sysroot-xfs-ephemeral-mkfs.service" <<'EOF'
 [Unit]
@@ -133,14 +180,15 @@ DefaultDependencies=false
 # can run really early.
 After=systemd-tmpfiles-setup-dev.service
 ConditionPathExists=/usr/lib/initrd-release
+RequiresMountsFor=/run/ephemeral_base
 # Something seems to be causing us to rerun?
-ConditionPathExists=!/run/ephemeral
+ConditionPathExists=!/run/ephemeral_base/loopfs
 
 [Service]
 Type=oneshot
 RemainAfterExit=yes
-ExecStart=/bin/sh -c 'set -euo pipefail; mem=$$(($$(stat -f -c "%%b * %%s / 1024" /run))) && /bin/truncate -s $${mem}k /run/ephemeral.xfsloop'
-ExecStart=/sbin/mkfs.xfs /run/ephemeral.xfsloop
+ExecStart=/bin/sh -c 'set -euo pipefail; mem=$$(($$(stat -f -c "%%b * %%s / 1024" /run/ephemeral_base))) && /bin/truncate -s $${mem}k /run/ephemeral_base/loopfs'
+ExecStart=/sbin/mkfs.xfs /run/ephemeral_base/loopfs
 ExecStart=/bin/mkdir /run/ephemeral
 EOF
 add_requires sysroot-xfs-ephemeral-mkfs.service initrd-root-fs.target
@@ -151,7 +199,7 @@ DefaultDependencies=false
 Requires=sysroot-xfs-ephemeral-mkfs.service
 After=sysroot-xfs-ephemeral-mkfs.service
 [Mount]
-What=/run/ephemeral.xfsloop
+What=/run/ephemeral_base/loopfs
 Where=/run/ephemeral
 Type=xfs
 Options=loop,discard
@@ -163,11 +211,9 @@ DefaultDependencies=false
 RequiresMountsFor=/run/ephemeral
 ConditionPathExists=/usr/lib/initrd-release
 ConditionPathExists=!/run/ephemeral/var
-# Make sure /sysroot is mounted first, since we're mounting under there
-Requires=sysroot.mount
-After=sysroot.mount
-# And after OSTree has set up the chroot() equivalent
+# We want to run after ostree is set up
 After=ostree-prepare-root.service
+Requires=ostree-prepare-root.service
 
 [Service]
 Type=oneshot
diff --git a/overlay.d/05core/usr/lib/dracut/modules.d/20live/module-setup.sh b/overlay.d/05core/usr/lib/dracut/modules.d/35coreos-live/module-setup.sh
similarity index 85%
rename from overlay.d/05core/usr/lib/dracut/modules.d/20live/module-setup.sh
rename to overlay.d/05core/usr/lib/dracut/modules.d/35coreos-live/module-setup.sh
index 0cf81a2a20..6a91048d7d 100644
--- a/overlay.d/05core/usr/lib/dracut/modules.d/20live/module-setup.sh
+++ b/overlay.d/05core/usr/lib/dracut/modules.d/35coreos-live/module-setup.sh
@@ -7,7 +7,14 @@ install_and_enable_unit() {
     unit="$1"; shift
     target="$1"; shift
     inst_simple "$moddir/$unit" "$systemdsystemunitdir/$unit"
-    systemctl -q --root="$initdir" add-requires "$target" "$unit"
+    # note we `|| exit 1` here so we error out if e.g. the units are missing
+    # see https://github.com/coreos/fedora-coreos-config/issues/799
+    systemctl -q --root="$initdir" add-requires "$target" "$unit" || exit 1
+}
+
+installkernel() {
+    # we do loopmounts
+    instmods -c loop
 }
 
 install() {
@@ -45,7 +52,4 @@ install() {
 
     install_and_enable_unit "coreos-livepxe-persist-osmet.service" \
         "default.target"
-
-    inst_simple "$moddir/coreos-liveiso-reconfigure-nm-wait-online.service" \
-        "$systemdsystemunitdir/coreos-liveiso-reconfigure-nm-wait-online.service"
 }
diff --git a/overlay.d/05core/usr/lib/dracut/modules.d/20live/ostree-cmdline.sh b/overlay.d/05core/usr/lib/dracut/modules.d/35coreos-live/ostree-cmdline.sh
similarity index 62%
rename from overlay.d/05core/usr/lib/dracut/modules.d/20live/ostree-cmdline.sh
rename to overlay.d/05core/usr/lib/dracut/modules.d/35coreos-live/ostree-cmdline.sh
index 444a536e5a..8d4d7914c8 100755
--- a/overlay.d/05core/usr/lib/dracut/modules.d/20live/ostree-cmdline.sh
+++ b/overlay.d/05core/usr/lib/dracut/modules.d/35coreos-live/ostree-cmdline.sh
@@ -11,11 +11,14 @@ set -euo pipefail
 case "${1:-unset}" in
     start)
         treepath="$(echo /sysroot/ostree/boot.1/*/*/0)"
-        echo "$(cat /proc/cmdline) ostree=${treepath#/sysroot}" > /tmp/cmdline
+        # ostree-prepare-root requires /etc and /var to be writeable for composeFS
+        # which cannot happen in the live ISO. Disable composeFS there
+        # https://github.com/coreos/fedora-coreos-config/pull/3009#issuecomment-2235923719
+        echo "$(cat /proc/cmdline) ostree=${treepath#/sysroot} ostree.prepare-root.composefs=0" > /tmp/cmdline
         mount --bind /tmp/cmdline /proc/cmdline
         ;;
     stop)
-        umount /proc/cmdline
+        umount -l /proc/cmdline
         rm /tmp/cmdline
         ;;
     *)
diff --git a/overlay.d/05core/usr/lib/dracut/modules.d/35coreos-multipath/coreos-multipath-generator b/overlay.d/05core/usr/lib/dracut/modules.d/35coreos-multipath/coreos-multipath-generator
new file mode 100755
index 0000000000..f073fee942
--- /dev/null
+++ b/overlay.d/05core/usr/lib/dracut/modules.d/35coreos-multipath/coreos-multipath-generator
@@ -0,0 +1,27 @@
+#!/bin/bash
+
+# Generators don't have logging right now
+# https://github.com/systemd/systemd/issues/15638
+exec 1>/dev/kmsg; exec 2>&1
+
+command -v getargbool >/dev/null || . /usr/lib/dracut-lib.sh
+
+set -e
+
+if is-live-image; then
+    exit 0
+fi
+
+UNIT_DIR="${1:-/tmp}"
+
+add_requires() {
+    local name="$1"; shift
+    local target="$1"; shift
+    local requires_dir="${UNIT_DIR}/${target}.requires"
+    mkdir -p "${requires_dir}"
+    ln -sf "../${name}" "${requires_dir}/${name}"
+}
+
+if getargbool 0 rd.multipath; then
+    add_requires coreos-multipath-wait.target initrd.target
+fi
diff --git a/overlay.d/05core/usr/lib/dracut/modules.d/35coreos-multipath/coreos-multipath-wait.target b/overlay.d/05core/usr/lib/dracut/modules.d/35coreos-multipath/coreos-multipath-wait.target
new file mode 100644
index 0000000000..8cdc3d8ac7
--- /dev/null
+++ b/overlay.d/05core/usr/lib/dracut/modules.d/35coreos-multipath/coreos-multipath-wait.target
@@ -0,0 +1,17 @@
+[Unit]
+Description=CoreOS Wait For Multipathed Boot
+DefaultDependencies=false
+After=dracut-cmdline.service
+Requires=dev-disk-by\x2dlabel-dm\x2dmpath\x2dboot.device
+After=dev-disk-by\x2dlabel-dm\x2dmpath\x2dboot.device
+Requires=multipathd.service
+After=multipathd.service
+
+OnFailure=emergency.target
+OnFailureJobMode=isolate
+
+# This is one of the earliest services that accesses the bootfs so make sure we
+# already have our multipath target.
+Before=coreos-ignition-setup-user.service
+
+Before=cryptsetup-pre.target
diff --git a/overlay.d/05core/usr/lib/dracut/modules.d/35coreos-multipath/coreos-propagate-multipath-conf.service b/overlay.d/05core/usr/lib/dracut/modules.d/35coreos-multipath/coreos-propagate-multipath-conf.service
new file mode 100644
index 0000000000..9eefee8749
--- /dev/null
+++ b/overlay.d/05core/usr/lib/dracut/modules.d/35coreos-multipath/coreos-propagate-multipath-conf.service
@@ -0,0 +1,24 @@
+[Unit]
+Description=CoreOS Propagate Multipath Configuration
+Before=initrd.target
+
+# we write to the rootfs, so run after it's ready
+After=initrd-root-fs.target
+
+# we only propagate if multipath wasn't configured via Ignition
+After=ignition-files.service
+
+# That service starts initrd-cleanup.service which will race with us completing
+# before we get nuked. Need to get to the bottom of it, but for now we need
+# this (XXX: add link to systemd issue here).
+Before=initrd-parse-etc.service
+
+ConditionKernelCommandLine=rd.multipath=default
+
+OnFailure=emergency.target
+OnFailureJobMode=isolate
+
+[Service]
+Type=oneshot
+ExecStart=/usr/sbin/coreos-propagate-multipath-conf
+RemainAfterExit=yes
diff --git a/overlay.d/05core/usr/lib/dracut/modules.d/35coreos-multipath/coreos-propagate-multipath-conf.sh b/overlay.d/05core/usr/lib/dracut/modules.d/35coreos-multipath/coreos-propagate-multipath-conf.sh
new file mode 100755
index 0000000000..eb6d12a1af
--- /dev/null
+++ b/overlay.d/05core/usr/lib/dracut/modules.d/35coreos-multipath/coreos-propagate-multipath-conf.sh
@@ -0,0 +1,23 @@
+#!/bin/bash
+set -euo pipefail
+
+# Persist automatic multipath configuration, if any.
+# When booting with `rd.multipath=default`, the default multipath
+# configuration is written. We need to ensure that the multipath configuration
+# is persisted to the rootfs.
+
+if [ ! -f /etc/multipath.conf ]; then
+    echo "info: initrd file /etc/multipath.conf does not exist"
+    echo "info: no initrd multipath configuration to propagate"
+    exit 0
+fi
+
+if [ -f /sysroot/etc/multipath.conf ]; then
+    echo "info: real root file /etc/multipath.conf exists"
+    echo "info: not propagating initrd multipath configuration"
+    exit 0
+fi
+
+echo "info: propagating initrd multipath configuration"
+cp -v /etc/multipath.conf /sysroot/etc/
+coreos-relabel /etc/multipath.conf
diff --git a/overlay.d/05core/usr/lib/dracut/modules.d/35coreos-multipath/module-setup.sh b/overlay.d/05core/usr/lib/dracut/modules.d/35coreos-multipath/module-setup.sh
new file mode 100755
index 0000000000..b04ba58e96
--- /dev/null
+++ b/overlay.d/05core/usr/lib/dracut/modules.d/35coreos-multipath/module-setup.sh
@@ -0,0 +1,26 @@
+#!/bin/bash
+# -*- mode: shell-script; indent-tabs-mode: nil; sh-basic-offset: 4; -*-
+# ex: ts=8 sw=4 sts=4 et filetype=sh
+
+install_unit() {
+    local unit=$1; shift
+    local target=${1:-initrd}
+    inst_simple "$moddir/$unit" "$systemdsystemunitdir/$unit"
+    # note we `|| exit 1` here so we error out if e.g. the units are missing
+    # see https://github.com/coreos/fedora-coreos-config/issues/799
+    systemctl -q --root="$initdir" add-requires "${target}.target" "$unit" || exit 1
+}
+
+install() {
+    inst_script "$moddir/coreos-propagate-multipath-conf.sh" \
+        "/usr/sbin/coreos-propagate-multipath-conf"
+
+    install_unit coreos-propagate-multipath-conf.service
+
+    inst_simple "$moddir/coreos-multipath-generator" \
+        "$systemdutildir/system-generators/coreos-multipath-generator"
+
+    # we don't enable these; they're enabled dynamically via the generator
+    inst_simple "$moddir/coreos-multipath-wait.target" \
+        "$systemdsystemunitdir/coreos-multipath-wait.target"
+}
diff --git a/overlay.d/05core/usr/lib/dracut/modules.d/15coreos-network/50-afterburn-network-kargs-default.conf b/overlay.d/05core/usr/lib/dracut/modules.d/35coreos-network/50-afterburn-network-kargs-default.conf
similarity index 80%
rename from overlay.d/05core/usr/lib/dracut/modules.d/15coreos-network/50-afterburn-network-kargs-default.conf
rename to overlay.d/05core/usr/lib/dracut/modules.d/35coreos-network/50-afterburn-network-kargs-default.conf
index bad6d14047..8c411e5add 100644
--- a/overlay.d/05core/usr/lib/dracut/modules.d/15coreos-network/50-afterburn-network-kargs-default.conf
+++ b/overlay.d/05core/usr/lib/dracut/modules.d/35coreos-network/50-afterburn-network-kargs-default.conf
@@ -4,4 +4,4 @@
 # https://github.com/coreos/fedora-coreos-tracker/issues/460
 
 [Service]
-Environment=AFTERBURN_NETWORK_KARGS_DEFAULT='ip=dhcp,dhcp6'
+Environment=AFTERBURN_NETWORK_KARGS_DEFAULT='ip=auto'
diff --git a/overlay.d/05core/usr/lib/dracut/modules.d/35coreos-network/coreos-copy-firstboot-network.service b/overlay.d/05core/usr/lib/dracut/modules.d/35coreos-network/coreos-copy-firstboot-network.service
new file mode 100644
index 0000000000..478dd85253
--- /dev/null
+++ b/overlay.d/05core/usr/lib/dracut/modules.d/35coreos-network/coreos-copy-firstboot-network.service
@@ -0,0 +1,62 @@
+# This unit will run early in boot and detect if:
+# - In the diskful case, the user copied in firstboot networking config files
+#   into `/boot` (most likely by using `coreos-installer install
+#   --copy-network`).
+# - In the live case, the user provided firstboot networking config files in
+#   `/etc` (most likely by using `coreos-installer iso network embed`).
+#
+# Since this unit is modifying network configuration there are some
+# dependencies that we have:
+#
+# - In the diskful case, we need to look for networking configuration on the
+#   /boot partition
+#     - i.e. after /dev/disk/by-label/boot is available
+#     - which is injected via coreos-diskful-generator
+# - Need to run before networking is brought up.
+#     - This is done in nm-initrd.service [1]
+#     - i.e. Before=nm-initrd.service
+# - Need to make sure karg networking configuration isn't applied
+#     - There are two ways to do this.
+#         - One is to run *before* the nm-config.sh [2] that runs as part of
+#           dracut-cmdline [3] and `ln -sf /bin/true /usr/libexec/nm-initrd-generator`.
+#             - i.e. Before=dracut-cmdline.service
+#         - Another is to run *after* nm-config.sh [2] in dracut-cmdline [3]
+#           and just delete all the files created by nm-initrd-generator.
+#             - i.e. After=dracut-cmdline.service, but Before=nm-initrd.service
+#     - We'll go with the second option here because the need for the /boot
+#       device (mentioned above) means we can't start before dracut-cmdline.service
+#
+# [1] https://github.com/dracutdevs/dracut/blob/master/modules.d/35network-manager/nm-initrd.service
+# [2] https://github.com/dracutdevs/dracut/blob/master/modules.d/35network-manager/nm-config.sh
+# [3] https://github.com/dracutdevs/dracut/blob/master/modules.d/35network-manager/module-setup.sh#L34
+#
+[Unit]
+Description=Copy CoreOS Firstboot Networking Config
+ConditionPathExists=/usr/lib/initrd-release
+DefaultDependencies=false
+# We're pulled in by ignition-complete.target; as good practice, add a matching
+# Before to be explicit about it gating on this unit passing.
+Before=ignition-complete.target
+Before=nm-initrd.service
+# compat: remove when everyone is on dracut 054+
+Before=dracut-initqueue.service
+After=dracut-cmdline.service
+# And since the boot device may be on multipath; optionally wait for it to
+# appear via the dynamic target.
+After=coreos-multipath-wait.target
+# Need to run after coreos-enable-network since it may re-run the NM cmdline
+# hook which will generate NM configs from the network kargs, but we want to
+# have precedence.
+After=coreos-enable-network.service
+# We've seen races with ignition-kargs.service, which accesses /boot rw.
+# Let's introduce some ordering here. Need to use `Before` because otherwise
+# we get a systemd ordering cycle. https://github.com/coreos/fedora-coreos-tracker/issues/883
+Before=ignition-kargs.service
+
+[Service]
+Type=oneshot
+RemainAfterExit=yes
+# The MountFlags=slave is so the umount of /boot is guaranteed to happen
+# /boot will only be mounted for the lifetime of the unit.
+MountFlags=slave
+ExecStart=/usr/sbin/coreos-copy-firstboot-network
diff --git a/overlay.d/05core/usr/lib/dracut/modules.d/35coreos-network/coreos-copy-firstboot-network.sh b/overlay.d/05core/usr/lib/dracut/modules.d/35coreos-network/coreos-copy-firstboot-network.sh
new file mode 100755
index 0000000000..51ea2830cd
--- /dev/null
+++ b/overlay.d/05core/usr/lib/dracut/modules.d/35coreos-network/coreos-copy-firstboot-network.sh
@@ -0,0 +1,49 @@
+#!/bin/bash
+set -euo pipefail
+
+# For a description of how this is used see coreos-copy-firstboot-network.service
+
+bootmnt=/mnt/boot_partition
+bootdev=/dev/disk/by-label/boot
+firstboot_network_dir_basename="coreos-firstboot-network"
+boot_firstboot_network_dir="${bootmnt}/${firstboot_network_dir_basename}"
+etc_firstboot_network_dir="/etc/${firstboot_network_dir_basename}"
+initramfs_network_dir="/run/NetworkManager/system-connections/"
+
+copy_firstboot_network() {
+    local src=$1; shift
+
+    # Clear out any files that may have already been generated from
+    # kargs by nm-initrd-generator
+    rm -f ${initramfs_network_dir}/*
+    # Copy files that were placed into the source
+    # to the appropriate location for NetworkManager to use the configuration.
+    echo "info: copying files from ${src} to ${initramfs_network_dir}"
+    mkdir -p ${initramfs_network_dir}
+    cp -v ${src}/* ${initramfs_network_dir}/
+}
+
+if ! is-live-image; then
+    # Mount /boot. Note that we mount /boot but we don't unmount boot because we
+    # are run in a systemd unit with MountFlags=slave so it is unmounted for us.
+    # Mount as read-only since we don't strictly need write access and we may be
+    # running alongside other code that also has it mounted ro
+    mkdir -p ${bootmnt}
+    mount -o ro ${bootdev} ${bootmnt}
+
+    if [ -n "$(ls -A ${boot_firstboot_network_dir} 2>/dev/null)" ]; then
+        # Likely placed there by coreos-installer, see:
+        # https://github.com/coreos/coreos-installer/pull/212
+        copy_firstboot_network "${boot_firstboot_network_dir}"
+    else
+        echo "info: no files to copy from ${boot_firstboot_network_dir}; skipping"
+    fi
+else
+    if [ -n "$(ls -A ${etc_firstboot_network_dir} 2>/dev/null)" ]; then
+        # Also placed there by coreos-installer but in a different flow, see:
+        # https://github.com/coreos/coreos-installer/pull/713
+        copy_firstboot_network "${etc_firstboot_network_dir}"
+    else
+        echo "info: no files to copy from ${etc_firstboot_network_dir}; skipping"
+    fi
+fi
diff --git a/overlay.d/05core/usr/lib/dracut/modules.d/15coreos-network/coreos-enable-network.service b/overlay.d/05core/usr/lib/dracut/modules.d/35coreos-network/coreos-enable-network.service
similarity index 76%
rename from overlay.d/05core/usr/lib/dracut/modules.d/15coreos-network/coreos-enable-network.service
rename to overlay.d/05core/usr/lib/dracut/modules.d/35coreos-network/coreos-enable-network.service
index 9f9cc92e4b..baa5d92c77 100644
--- a/overlay.d/05core/usr/lib/dracut/modules.d/15coreos-network/coreos-enable-network.service
+++ b/overlay.d/05core/usr/lib/dracut/modules.d/35coreos-network/coreos-enable-network.service
@@ -2,15 +2,15 @@
 Description=CoreOS Enable Network
 ConditionPathExists=/etc/initrd-release
 DefaultDependencies=false
-After=basic.target
 
 # Triggering conditions for cases where we need network:
 #  * when Ignition signals that it is required for provisioning.
 #  * on live systems fetching the remote rootfs in initramfs.
-#  * on Azure, for hostname fetching (metadata endpoint) and boot check-in (wireserver).
+#  * on Azure and Azure Stack Hub, for hostname fetching (metadata endpoint) and boot check-in (wireserver).
 ConditionPathExists=|/run/ignition/neednet
 ConditionKernelCommandLine=|coreos.live.rootfs_url
 ConditionKernelCommandLine=|ignition.platform.id=azure
+ConditionKernelCommandLine=|ignition.platform.id=azurestack
 
 # Creates /run/ignition/neednet
 After=ignition-fetch-offline.service
@@ -19,6 +19,8 @@ Before=ignition-fetch.service
 
 # See hack in coreos-enable-network, as well as coreos-copy-firstboot-network.service.
 After=dracut-cmdline.service
+Before=nm-initrd.service
+# compat: remove when everyone is on dracut 054+
 Before=dracut-initqueue.service
 
 [Service]
diff --git a/overlay.d/05core/usr/lib/dracut/modules.d/15coreos-network/coreos-enable-network.sh b/overlay.d/05core/usr/lib/dracut/modules.d/35coreos-network/coreos-enable-network.sh
similarity index 100%
rename from overlay.d/05core/usr/lib/dracut/modules.d/15coreos-network/coreos-enable-network.sh
rename to overlay.d/05core/usr/lib/dracut/modules.d/35coreos-network/coreos-enable-network.sh
diff --git a/overlay.d/05core/usr/lib/dracut/modules.d/15coreos-network/module-setup.sh b/overlay.d/05core/usr/lib/dracut/modules.d/35coreos-network/module-setup.sh
similarity index 78%
rename from overlay.d/05core/usr/lib/dracut/modules.d/15coreos-network/module-setup.sh
rename to overlay.d/05core/usr/lib/dracut/modules.d/35coreos-network/module-setup.sh
index 12dec8b1b3..31da969cdd 100644
--- a/overlay.d/05core/usr/lib/dracut/modules.d/15coreos-network/module-setup.sh
+++ b/overlay.d/05core/usr/lib/dracut/modules.d/35coreos-network/module-setup.sh
@@ -2,21 +2,21 @@ install_and_enable_unit() {
     unit="$1"; shift
     target="$1"; shift
     inst_simple "$moddir/$unit" "$systemdsystemunitdir/$unit"
-    systemctl -q --root="$initdir" add-requires "$target" "$unit"
+    # note we `|| exit 1` here so we error out if e.g. the units are missing
+    # see https://github.com/coreos/fedora-coreos-config/issues/799
+    systemctl -q --root="$initdir" add-requires "$target" "$unit" || exit 1
 }
 
 install() {
     inst_simple "$moddir/coreos-enable-network.sh" \
         "/usr/sbin/coreos-enable-network"
     install_and_enable_unit "coreos-enable-network.service" \
-        "initrd.target"
+        "ignition-complete.target"
 
     inst_simple "$moddir/coreos-copy-firstboot-network.sh" \
         "/usr/sbin/coreos-copy-firstboot-network"
-    # Only run this when ignition runs and only when the system
-    # has disks. ignition-diskful.target should suffice.
     install_and_enable_unit "coreos-copy-firstboot-network.service" \
-        "ignition-diskful.target"
+        "ignition-complete.target"
 
     # Dropin with firstboot network configuration kargs, applied via
     # Afterburn.
diff --git a/overlay.d/05core/usr/lib/dracut/modules.d/40ignition-conf/README.md b/overlay.d/05core/usr/lib/dracut/modules.d/40ignition-conf/README.md
index 793e519232..9c4eee089a 100644
--- a/overlay.d/05core/usr/lib/dracut/modules.d/40ignition-conf/README.md
+++ b/overlay.d/05core/usr/lib/dracut/modules.d/40ignition-conf/README.md
@@ -1 +1 @@
-`00-core.ign` is the base config shared between FCOS and RHCOS. The configs specific to FCOS are in [50ignition-conf-fcos](../../../../../../15fcos/usr/lib/dracut/modules.d/50ignition-conf-fcos).
\ No newline at end of file
+`00-core.ign` is the base config shared between FCOS and RHCOS. The configs specific to FCOS are in [50ignition-conf-fcos](../../../../../../15fcos/usr/lib/dracut/modules.d/50ignition-conf-fcos).
diff --git a/overlay.d/05core/usr/lib/dracut/modules.d/40ignition-ostree/coreos-check-rootfs-size b/overlay.d/05core/usr/lib/dracut/modules.d/40ignition-ostree/coreos-check-rootfs-size
new file mode 100755
index 0000000000..d0b38d9bfd
--- /dev/null
+++ b/overlay.d/05core/usr/lib/dracut/modules.d/40ignition-ostree/coreos-check-rootfs-size
@@ -0,0 +1,38 @@
+#!/bin/bash
+set -euo pipefail
+
+# See also ignition-ostree-check-rootfs-size.service
+# https://github.com/coreos/fedora-coreos-tracker/issues/586#issuecomment-777220000
+
+# /sysroot is the mounted deploy root, /sysroot/sysroot is the physical root filesystem
+srcdev=$(findmnt -nvr -o SOURCE /sysroot/sysroot | tail -n1)
+size=$(lsblk --nodeps --noheadings --bytes -o SIZE "${srcdev}")
+
+MINIMUM_GB=8
+MINIMUM_BYTES=$((1024 * 1024 * 1024 * MINIMUM_GB))
+
+MOTD_DROPIN=/etc/motd.d/60-coreos-rootfs-size.motd
+
+YELLOW=$(echo -e '\033[0;33m')
+RESET=$(echo -e '\033[0m')
+
+if [ "${size}" -lt "${MINIMUM_BYTES}" ]; then
+    mkdir -p "/sysroot/$(dirname "${MOTD_DROPIN}")"
+    cat > "/sysroot/${MOTD_DROPIN}" <<EOF
+${YELLOW}
+############################################################################
+WARNING: The root filesystem is too small. It is strongly recommended to
+allocate at least ${MINIMUM_GB} GiB of space to allow for upgrades. From June 2021, this
+condition will trigger a failure in some cases. For more information, see:
+https://docs.fedoraproject.org/en-US/fedora-coreos/storage/
+
+You may delete this warning using:
+sudo rm ${MOTD_DROPIN}
+############################################################################
+${RESET}
+EOF
+
+    # And also write it on stdout for the journal and console
+    cat "/sysroot/${MOTD_DROPIN}"
+    coreos-relabel "${MOTD_DROPIN}"
+fi
diff --git a/overlay.d/05core/usr/lib/dracut/modules.d/40ignition-ostree/coreos-growpart b/overlay.d/05core/usr/lib/dracut/modules.d/40ignition-ostree/coreos-growpart
deleted file mode 100755
index f32fcee065..0000000000
--- a/overlay.d/05core/usr/lib/dracut/modules.d/40ignition-ostree/coreos-growpart
+++ /dev/null
@@ -1,99 +0,0 @@
-#!/bin/bash
-set -euo pipefail
-
-# See also ignition-ostree-growfs.service.
-
-# https://github.com/coreos/fedora-coreos-tracker/issues/18
-# See also image.ks.
-# Growpart /, until we can fix Ignition for separate /var
-# (And eventually we want ignition-disks)
-
-saved_partstate=/run/ignition-ostree-rootfs-partstate.json
-
-path=$1
-shift
-
-# The use of tail is to avoid errors from duplicate mounts;
-# this shouldn't happen for us but we're being conservative.
-src=$(findmnt -nvr -o SOURCE "$path" | tail -n1)
-
-if [ "${path}" == /sysroot ] && [ -f "${saved_partstate}" ]; then
-    # We're still ironing out our rootfs automatic growpart story, see e.g.:
-    # https://github.com/coreos/fedora-coreos-tracker/issues/570
-    # https://github.com/coreos/fedora-coreos-tracker/issues/586
-    #
-    # In the context of rootfs reprovisioning, for now our rule is the
-    # following: if the rootfs partition was moved off of the boot disk or it
-    # was resized, then we don't growpart.
-    #
-    # To detect this, we compare the output of `lsblk --paths -o NAME,SIZE`
-    # before and after `ignition-disks.service`.
-    partstate=$(lsblk "${src}" --nodeps --paths --json -b -o NAME,SIZE | jq -c .)
-    if [ "${partstate}" != "$(cat "${saved_partstate}")" ]; then
-        echo "coreos-growpart: detected rootfs partition changes; not auto-growing"
-        exit 0
-    fi
-fi
-
-# Get the filesystem type before extending the partition.  This matters
-# because the partition, once extended, might include leftover superblocks
-# from the previous contents of the disk (notably ZFS), causing blkid to
-# refuse to return any filesystem type at all.
-eval $(blkid -o export "${src}")
-case "${TYPE:-}" in
-    xfs|ext4|btrfs) ;;
-    *) echo "error: Unsupported filesystem for ${path}: '${TYPE:-}'" 1>&2; exit 1 ;;
-esac
-
-if [[ "${src}" =~ "/dev/mapper" ]]; then
-    eval $(udevadm info --query property --export "${src}")
-    # get the partition, if any, and the name for the device mapper
-    partition="${ID_PART_ENTRY_NUMBER:-}"
-    dm_name="${DM_NAME//$partition/}"
-    # identify the type of device mapper.
-    subsystem=$(dmsetup info ${dm_name} -C -o subsystem --noheadings)
-
-    # for now, we only support multipath devices
-    if [ "${subsystem}" == "mpath" ] && [ -n "${partition}" ]; then
-        # growpart does not understand device mapper, instead of having sfdisk inform the kernel,
-        # use kpartx to inform the kernel and the device mapper linear maps.
-        echo ", +" | sfdisk --no-reread --no-tell-kernel --force -N "${ID_PART_ENTRY_NUMBER}" "/dev/mapper/${dm_name}"
-        kpartx -fu /dev/mapper/${dm_name}
-    else
-        echo "coreos-growpart: unsupported device-mapper target: ${dm_name}"
-        exit 0
-    fi
-else
-    if test "${TYPE:-}" = "btrfs"; then
-        # Theoretically btrfs can have multiple devices, but when
-        # we start we will always have exactly one.
-        devpath=$(btrfs device usage /sysroot | grep /dev | cut -f 1 -d ,)
-        devpath=$(realpath /sys/class/block/${devpath#/dev/})
-    else
-        # Handle traditional disk/partitions
-        majmin=$(findmnt -nvr -o MAJ:MIN "$path" | tail -n1)
-        devpath=$(realpath "/sys/dev/block/$majmin")
-    fi
-    partition="${partition:-$(cat "$devpath/partition")}"
-    parent_path=$(dirname "$devpath")
-    parent_device=/dev/$(basename "${parent_path}")
-
-    # TODO: make this idempotent, and don't error out if
-    # we can't resize.
-    growpart "${parent_device}" "${partition}" || true
-fi
-
-# Wipe any filesystem signatures from the extended partition that don't
-# correspond to the FS type we detected earlier.
-wipefs -af -t "no${TYPE}" "${src}"
-
-# TODO: Add XFS to https://github.com/systemd/systemd/blob/master/src/partition/growfs.c
-# and use it instead.
-case "${TYPE}" in
-    xfs) xfs_growfs "${path}" ;;
-    ext4) resize2fs "${src}" ;;
-    btrfs) btrfs filesystem resize max ${path} ;;
-esac
-
-# this is useful for tests
-touch /run/coreos-growpart.stamp
diff --git a/overlay.d/05core/usr/lib/dracut/modules.d/40ignition-ostree/coreos-rootflags.sh b/overlay.d/05core/usr/lib/dracut/modules.d/40ignition-ostree/coreos-rootflags.sh
deleted file mode 100755
index 332665e4f3..0000000000
--- a/overlay.d/05core/usr/lib/dracut/modules.d/40ignition-ostree/coreos-rootflags.sh
+++ /dev/null
@@ -1,19 +0,0 @@
-#!/bin/bash
-set -euo pipefail
-
-rootpath=/dev/disk/by-label/root
-
-# If the rootfs was reprovisioned, then the user is free to define their own
-# rootflags.
-if [ -d /run/ignition-ostree-transposefs/root ]; then
-    exit 0
-fi
-
-eval $(blkid -o export ${rootpath})
-# this really should always be true, but let's be conservative
-if [ "${TYPE}" == "xfs" ]; then
-    # We use prjquota on XFS by default to aid multi-tenant Kubernetes (and
-    # other container) clusters.  See
-    # https://github.com/coreos/coreos-assembler/pull/303/commits/6103effbd006bb6109467830d6a3e42dd847668d
-    echo "prjquota"
-fi
diff --git a/overlay.d/05core/usr/lib/dracut/modules.d/40ignition-ostree/ignition-ostree-check-rootfs-size.service b/overlay.d/05core/usr/lib/dracut/modules.d/40ignition-ostree/ignition-ostree-check-rootfs-size.service
new file mode 100644
index 0000000000..340b4d4251
--- /dev/null
+++ b/overlay.d/05core/usr/lib/dracut/modules.d/40ignition-ostree/ignition-ostree-check-rootfs-size.service
@@ -0,0 +1,16 @@
+[Unit]
+Description=Ignition OSTree: Check Root Filesystem Size
+Documentation=https://docs.fedoraproject.org/en-US/fedora-coreos/storage/
+DefaultDependencies=false
+ConditionKernelCommandLine=ostree
+ConditionPathExists=!/run/ostree-live
+After=ignition-ostree-growfs.service
+After=ostree-prepare-root.service
+Requires=ostree-prepare-root.service
+# Allow Ignition config to blank out the warning
+Before=ignition-files.service
+
+[Service]
+Type=oneshot
+ExecStart=/usr/libexec/coreos-check-rootfs-size
+RemainAfterExit=yes
diff --git a/overlay.d/05core/usr/lib/dracut/modules.d/40ignition-ostree/ignition-ostree-firstboot-uuid b/overlay.d/05core/usr/lib/dracut/modules.d/40ignition-ostree/ignition-ostree-firstboot-uuid
index 0027bed357..1292b377c1 100755
--- a/overlay.d/05core/usr/lib/dracut/modules.d/40ignition-ostree/ignition-ostree-firstboot-uuid
+++ b/overlay.d/05core/usr/lib/dracut/modules.d/40ignition-ostree/ignition-ostree-firstboot-uuid
@@ -18,7 +18,7 @@ if ! [ -b "${target}" ]; then
   exit 1
 fi
 
-eval $(blkid -o export ${target})
+eval $(blkid -p -o export ${target})
 case "${label}" in
   root) orig_uuid="${rootfs_uuid}"; orig_type=xfs ;;
   boot) orig_uuid="${bootfs_uuid}"; orig_type=ext4 ;;
@@ -27,14 +27,11 @@ esac
 
 if [ "${TYPE}" == "${orig_type}" ] && [ "${UUID}" == "${orig_uuid}" ]; then
   case "${TYPE}" in
-    # For now we need to fsck first, see https://github.com/coreos/coreos-assembler/pull/1452
-    # Basically we're not passing `metadata_csum_seed` as a mkfs.ext4 option
-    # because grub2 barfs on it.
-    ext4) e2fsck -fy "${target}" && tune2fs -U random "${target}" ;;
+    ext4) tune2fs -U random "${target}"    ;;
     xfs) xfs_admin -U generate "${target}" ;;
     *) echo "unexpected filesystem type ${TYPE}" 1>&2; exit 1 ;;
   esac
-  udevadm settle
+  udevadm settle || :
   echo "Regenerated UUID for ${target}"
 else
   echo "No changes required for ${target} TYPE=${TYPE} UUID=${UUID}"
diff --git a/overlay.d/05core/usr/lib/dracut/modules.d/40ignition-ostree/ignition-ostree-growfs.service b/overlay.d/05core/usr/lib/dracut/modules.d/40ignition-ostree/ignition-ostree-growfs.service
index 0dc4cf7950..3674431ff8 100644
--- a/overlay.d/05core/usr/lib/dracut/modules.d/40ignition-ostree/ignition-ostree-growfs.service
+++ b/overlay.d/05core/usr/lib/dracut/modules.d/40ignition-ostree/ignition-ostree-growfs.service
@@ -1,19 +1,15 @@
 [Unit]
-Description=Ignition OSTree: Grow root filesystem
+Description=Ignition OSTree: Grow Root Filesystem
 DefaultDependencies=false
 ConditionKernelCommandLine=ostree
-# Similar to the other mount rules, suppress invocation if we detect
-# we are running from a legacy setup created by Anaconda.
-ConditionKernelCommandLine=!root
 ConditionPathExists=!/run/ostree-live
 Before=initrd-root-fs.target
-After=ignition-ostree-mount-firstboot-sysroot.service
-Requires=ignition-ostree-mount-firstboot-sysroot.service
-# This shouldn't be strictly necessary, but it's cleaner to not have OSTree muck
-# around with moving mounts while we're still resizing the filesystem.
-Before=ostree-prepare-root.service
+Before=sysroot.mount
+After=ignition-ostree-uuid-root.service
 
 [Service]
 Type=oneshot
-ExecStart=/usr/libexec/coreos-growpart /sysroot
+ExecStart=/usr/sbin/ignition-ostree-growfs
 RemainAfterExit=yes
+# So we can transiently mount sysroot
+MountFlags=slave
diff --git a/overlay.d/05core/usr/lib/dracut/modules.d/40ignition-ostree/ignition-ostree-growfs.sh b/overlay.d/05core/usr/lib/dracut/modules.d/40ignition-ostree/ignition-ostree-growfs.sh
new file mode 100755
index 0000000000..5ebb1cfad4
--- /dev/null
+++ b/overlay.d/05core/usr/lib/dracut/modules.d/40ignition-ostree/ignition-ostree-growfs.sh
@@ -0,0 +1,178 @@
+#!/bin/bash
+set -euo pipefail
+
+# This script is run by ignition-ostree-growfs.service. It grows the root
+# partition, unless it determines that either the rootfs was moved or the
+# partition was already resized (e.g. via Ignition).
+
+# In the IBM Secure Execution case we use Ignition to grow and reencrypt rootfs
+# see overlay.d/05core/usr/lib/dracut/modules.d/35coreos-ignition/coreos-diskful-generator
+if [[ -f /run/coreos/secure-execution ]]; then
+    exit 0
+fi
+
+# This is copied from ignition-ostree-transposefs.sh.
+# Sometimes, for some reason the by-label symlinks aren't updated. Detect these
+# cases, and explicitly `udevadm trigger`.
+# See: https://bugzilla.redhat.com/show_bug.cgi?id=1908780
+udev_trigger_on_label_mismatch() {
+    local label=$1; shift
+    local expected_dev=$1; shift
+    local actual_dev
+    expected_dev=$(realpath "${expected_dev}")
+    # We `|| :` here because sometimes /dev/disk/by-label/$label is missing.
+    # We've seen this on Fedora kernels with debug enabled (common in `rawhide`).
+    # See https://github.com/coreos/fedora-coreos-tracker/issues/1092
+    actual_dev=$(realpath "/dev/disk/by-label/$label" || :)
+    if [ "$actual_dev" != "$expected_dev" ]; then
+        echo "Expected /dev/disk/by-label/$label to point to $expected_dev, but points to $actual_dev; triggering udev"
+        udevadm trigger --settle "$expected_dev"
+    fi
+}
+
+# This is also similar to bits from transposefs.sh.
+ignition_cfg=/run/ignition.json
+expected_dev=$(jq -r '.storage?.filesystems? // [] | map(select(.label == "root")) | .[0].device // ""' "${ignition_cfg}")
+if [ -n "${expected_dev}" ]; then
+    udev_trigger_on_label_mismatch root "${expected_dev}"
+fi
+
+# If root reprovisioning was triggered, this file contains state of the root
+# partition *before* ignition-disks.
+saved_partstate=/run/ignition-ostree-rootfs-partstate.sh
+
+# We run before the rootfs is mounted at /sysroot, but we still need to mount it
+# (in a private namespace) since XFS and Btrfs can only do resizing online (EXT4
+# can do either).
+path=/sysroot
+src=/dev/disk/by-label/root
+mount "${src}" "${path}"
+
+if [ ! -f "${saved_partstate}" ]; then
+    partition=$(realpath /dev/disk/by-label/root)
+else
+    # The rootfs was reprovisioned. Our rule in this case is: we only grow if
+    # the partition backing the rootfs is the same and its size didn't change
+    # (IOW, it was an in-place reprovisioning; e.g. LUKS or xfs -> btrfs).
+    source "${saved_partstate}"
+    if [ "${TYPE}" != "part" ]; then
+        # this really should never happen; but play nice
+        echo "$0: original rootfs blockdev not of type 'part'; not auto-growing"
+        exit 0
+    fi
+    partition=$(realpath "${NAME}")
+    if [ "${SIZE}" != "$(lsblk --nodeps -bno SIZE "${partition}")" ]; then
+        echo "$0: original root partition changed size; not auto-growing"
+        exit 0
+    fi
+    if ! lsblk -no MOUNTPOINT "${partition}" | grep -q '^/sysroot$'; then
+        echo "$0: original root partition no longer backing rootfs; not auto-growing"
+        exit 0
+    fi
+fi
+
+# Go through each blockdev in the hierarchy and verify we know how to grow them
+lsblk -no TYPE "${partition}" | while read dev; do
+    case "${dev}" in
+        part|crypt) ;;
+        *) echo "error: Unsupported blockdev type ${dev}" 1>&2; exit 1 ;;
+    esac
+done
+
+# Get the filesystem type before extending the partition.  This matters
+# because the partition, once extended, might include leftover superblocks
+# from the previous contents of the disk (notably ZFS), causing blkid to
+# refuse to return any filesystem type at all.
+eval $(blkid -p -o export "${src}")
+ROOTFS_TYPE=${TYPE:-}
+case "${ROOTFS_TYPE}" in
+    xfs|ext4|btrfs) ;;
+    *) echo "error: Unsupported filesystem for ${path}: '${ROOTFS_TYPE}'" 1>&2; exit 1 ;;
+esac
+
+# Now, go through the hierarchy, growing everything. Note we go one device at a
+# time using --nodeps, because ordering is buggy in el8:
+# https://bugzilla.redhat.com/show_bug.cgi?id=1940607
+current_blkdev=${partition}
+while true; do
+    eval "$(lsblk --paths --nodeps --pairs -o NAME,TYPE,PKNAME "${current_blkdev}")"
+    MAJMIN=$(echo $(lsblk -dno MAJ:MIN "${NAME}"))
+    case "${TYPE}" in
+        part)
+            eval $(udevadm info --query property --export "${current_blkdev}" | grep ^DM_ || :)
+            if [ -n "${DM_MPATH:-}" ]; then
+                PKNAME=/dev/mapper/${DM_MPATH}
+                partnum=${DM_PART}
+                # Since growpart does not understand device mapper, we have to use sfdisk.
+                echo ", +" | sfdisk --no-reread --no-tell-kernel --force -N "${partnum}" "${PKNAME}"
+                udevadm settle || : # Wait for udev-triggered kpartx to update mappings
+            elif [[ "${PKNAME}" = /dev/dasd* ]]; then
+                echo "partition is on DASD device; skipping growpart"
+            else
+                partnum=$(cat "/sys/dev/block/${MAJMIN}/partition")
+                # XXX: ideally this'd be idempotent and we wouldn't `|| :`
+                growpart "${PKNAME}" "${partnum}" || :
+            fi
+            # If this is a 512e disk, then ensure the partition end is 4K
+            # aligned to be compatible with LUKS. If it's a 4Kn disk, `size`
+            # necessarily must be 4K aligned (note the sysfs value is always
+            # reported in 512b sizes). We should be able to drop this once
+            # https://github.com/util-linux/util-linux/issues/2140 is fixed.
+            size=$(cat "/sys/dev/block/${MAJMIN}/size")
+            phy_sec=$(blockdev --getpbsz "${PKNAME}")
+            if [ "$((size % 8))" != 0 ] && [ "${phy_sec:-}" = 4096 ]; then
+                size=$(((size >> 3) << 3))  # round down to nearest 4K boundary
+                echo ", ${size}" | sfdisk --no-reread --force -N "${partnum}" "${PKNAME}"
+                partx --update --nr "${partnum}" "${PKNAME}"
+            fi
+            ;;
+        crypt)
+            # lsblk doesn't print PKNAME of crypt devices with --nodeps
+            PKNAME=/dev/$(ls "/sys/dev/block/${MAJMIN}/slaves")
+            LUKS_DUMP=$(cryptsetup luksDump "$PKNAME" --dump-json-metadata)
+            if jq -e '[.tokens[].type] | index("clevis")' <<< "$LUKS_DUMP"; then
+                # XXX: yuck... we need to expose this sanely in clevis
+                (. /usr/bin/clevis-luks-common-functions
+                 eval $(udevadm info --query=property --export "${NAME}")
+                 clevis_luks_unlock_device "${PKNAME}" | cryptsetup resize -d- "${DM_NAME}"
+                )
+            elif jq -e '.segments["0"].encryption | startswith("paes")' <<< "$LUKS_DUMP"; then
+                # CEX LUKS volume: https://github.com/coreos/ignition/issues/1693
+                cryptsetup resize root --key-file /etc/luks/cex.key
+            else
+                echo "$LUKS_DUMP"
+                echo "error: unknown LUKS device"
+                exit 1
+            fi
+            ;;
+        # already checked
+        *) echo "unreachable" 1>&2; exit 1 ;;
+    esac
+    holders="/sys/dev/block/${MAJMIN}/holders"
+    [ -d "${holders}" ] || break
+    nholders="$(ls "${holders}" | wc -l)"
+    if [ "${nholders}" -eq 0 ]; then
+        break
+    elif [ "${nholders}" -gt 1 ]; then
+        # this shouldn't happen since we've checked the partition types already
+        echo "error: Unsupported block device with multiple children: ${NAME}" 1>&2
+        exit 1
+    fi
+    current_blkdev=/dev/$(ls "${holders}")
+done
+
+# Wipe any filesystem signatures from the extended partition that don't
+# correspond to the FS type we detected earlier.
+wipefs -af -t "no${ROOTFS_TYPE}" "${src}"
+
+# TODO: Add XFS to https://github.com/systemd/systemd/blob/master/src/partition/growfs.c
+# and use it instead.
+case "${ROOTFS_TYPE}" in
+    xfs) xfs_growfs "${path}" ;;
+    ext4) resize2fs "${src}" ;;
+    btrfs) btrfs filesystem resize max ${path} ;;
+esac
+
+# The ignition-ostree-transposefs-xfsauto.service unit needs to know if we
+# actually run. This is also useful for tests.
+touch /run/ignition-ostree-growfs.stamp
diff --git a/overlay.d/05core/usr/lib/dracut/modules.d/40ignition-ostree/ignition-ostree-mount-firstboot-sysroot.service b/overlay.d/05core/usr/lib/dracut/modules.d/40ignition-ostree/ignition-ostree-mount-firstboot-sysroot.service
deleted file mode 100644
index 3ba677d0e3..0000000000
--- a/overlay.d/05core/usr/lib/dracut/modules.d/40ignition-ostree/ignition-ostree-mount-firstboot-sysroot.service
+++ /dev/null
@@ -1,25 +0,0 @@
-[Unit]
-Description=Ignition OSTree: Mount (firstboot) /sysroot
-# These dependencies should match the "other" in
-# ignition-ostree-mount-subsequent-sysroot.service
-DefaultDependencies=false
-# If root is specified, then systemd's generator will win
-ConditionKernelCommandLine=!root
-ConditionKernelCommandLine=ostree
-# This is redundant since we're queued on -diskful.target, but eh.
-ConditionPathExists=!/run/ostree-live
-# There can be only one, Highlander style
-Conflicts=ignition-ostree-mount-subsequent-sysroot.service
-Before=initrd-root-fs.target
-After=ignition-disks.service
-# Note we don't have a Requires: /dev/disk/by-label/root here like
-# the -subsequent service does because ignition-disks may have
-# regenerated it.
-Requires=ignition-disks.service
-# These have an explicit dependency on After=sysroot.mount today
-Before=ostree-prepare-root.service ignition-remount-sysroot.service
-
-[Service]
-Type=oneshot
-RemainAfterExit=yes
-ExecStart=/usr/sbin/ignition-ostree-mount-sysroot
diff --git a/overlay.d/05core/usr/lib/dracut/modules.d/40ignition-ostree/ignition-ostree-mount-state-overlays.service b/overlay.d/05core/usr/lib/dracut/modules.d/40ignition-ostree/ignition-ostree-mount-state-overlays.service
new file mode 100644
index 0000000000..59fac81264
--- /dev/null
+++ b/overlay.d/05core/usr/lib/dracut/modules.d/40ignition-ostree/ignition-ostree-mount-state-overlays.service
@@ -0,0 +1,19 @@
+[Unit]
+Description=Ignition OSTree Mount State Overlays
+DefaultDependencies=false
+ConditionKernelCommandLine=|ostree
+ConditionPathExists=|/run/ostree-live
+
+# Need to do this with all mount points active
+After=ignition-mount.service
+# Not strictly required, but both do /var things
+After=ignition-ostree-populate-var.service
+
+# But *before* we start dumping files in there
+Before=ignition-files.service
+
+[Service]
+Type=oneshot
+RemainAfterExit=yes
+ExecStart=/usr/libexec/ignition-ostree-mount-state-overlays mount
+ExecStop=/usr/libexec/ignition-ostree-mount-state-overlays umount
diff --git a/overlay.d/05core/usr/lib/dracut/modules.d/40ignition-ostree/ignition-ostree-mount-state-overlays.sh b/overlay.d/05core/usr/lib/dracut/modules.d/40ignition-ostree/ignition-ostree-mount-state-overlays.sh
new file mode 100755
index 0000000000..84a987b6b7
--- /dev/null
+++ b/overlay.d/05core/usr/lib/dracut/modules.d/40ignition-ostree/ignition-ostree-mount-state-overlays.sh
@@ -0,0 +1,50 @@
+#!/bin/bash
+set -euo pipefail
+
+fatal() {
+    echo "$@" >&2
+    exit 1
+}
+
+if [ $# -ne 1 ] || { [[ $1 != mount ]] && [[ $1 != umount ]]; }; then
+    fatal "Usage: $0 <mount|umount>"
+fi
+
+# if state overlays are not enabled, there's nothing to do
+if ! ls /sysroot/usr/lib/systemd/system/local-fs.target.requires/ostree-state-overlay@*.service 2>/dev/null; then
+    exit 0
+fi
+
+do_mount() {
+    # be nice to persistent /var; if the top-level state overlay dir exists,
+    # then assume it's properly labeled
+    relabel=1
+    state_overlays_dir=/sysroot/var/ostree/state-overlays
+    if [ -d ${state_overlays_dir} ]; then
+        relabel=0
+    fi
+    for overlay in /usr/lib/opt /usr/local; do
+        escaped=$(systemd-escape --path "${overlay}")
+        overlay_dirs=${state_overlays_dir}/${escaped}
+        mkdir -p "${overlay_dirs}"/{upper,work}
+        # ideally we'd use `ostree admin state-overlay`, but that'd require
+        # pulling in bwrap and chroot which isn't yet in the FCOS initrd
+        mount -t overlay overlay /sysroot/${overlay} -o "lowerdir=/sysroot/${overlay},upperdir=${overlay_dirs}/upper,workdir=${overlay_dirs}/work"
+    done
+    if [ $relabel = 1 ]; then
+        coreos-relabel /var/ostree
+        # the above relabel will have relabeled the upperdir too; relabel that
+        # from the perspective of the mount point so it's not var_t
+        for overlay in /usr/lib/opt /usr/local; do
+            coreos-relabel ${overlay}
+        done
+    fi
+}
+
+do_umount() {
+    for overlay in /usr/lib/opt /usr/local; do
+        umount /sysroot/${overlay}
+    done
+}
+
+"do_$1"
diff --git a/overlay.d/05core/usr/lib/dracut/modules.d/40ignition-ostree/ignition-ostree-mount-subsequent-sysroot.service b/overlay.d/05core/usr/lib/dracut/modules.d/40ignition-ostree/ignition-ostree-mount-subsequent-sysroot.service
deleted file mode 100644
index 92dde886f0..0000000000
--- a/overlay.d/05core/usr/lib/dracut/modules.d/40ignition-ostree/ignition-ostree-mount-subsequent-sysroot.service
+++ /dev/null
@@ -1,24 +0,0 @@
-# Note this unit is conditionally enabled by ignition-ostree-generator
-[Unit]
-Description=CoreOS: Mount (subsequent) /sysroot
-# These dependencies should match the "other" in
-# ignition-ostree-mount-firsboot-sysroot.service
-DefaultDependencies=false
-# If root is specified, then systemd's generator will win
-ConditionKernelCommandLine=!root
-ConditionKernelCommandLine=ostree
-ConditionPathExists=!/run/ostree-live
-# There can be only one, Highlander style
-Conflicts=ignition-ostree-mount-firstboot-sysroot.service
-# And in contrast to the firstboot, we expect
-# the root device to be ready.
-Requires=dev-disk-by\x2dlabel-root.device
-After=dev-disk-by\x2dlabel-root.device
-Before=initrd-root-fs.target
-# This has an explicit dependency on After=sysroot.mount today
-Before=ostree-prepare-root.service
-
-[Service]
-Type=oneshot
-RemainAfterExit=yes
-ExecStart=/usr/sbin/ignition-ostree-mount-sysroot
diff --git a/overlay.d/05core/usr/lib/dracut/modules.d/40ignition-ostree/ignition-ostree-mount-sysroot.sh b/overlay.d/05core/usr/lib/dracut/modules.d/40ignition-ostree/ignition-ostree-mount-sysroot.sh
deleted file mode 100755
index a51c4b26fe..0000000000
--- a/overlay.d/05core/usr/lib/dracut/modules.d/40ignition-ostree/ignition-ostree-mount-sysroot.sh
+++ /dev/null
@@ -1,18 +0,0 @@
-#!/bin/bash
-set -euo pipefail
-
-# Note that on *new machines* this script is now only ever used on firstboot. On
-# subsequent boots, systemd-fstab-generator mounts /sysroot from the
-# root=UUID=... and rootflags=... kargs.
-
-# We may do a migration window at some point where older machines have these
-# kargs injected so that we can simplify the model further.
-
-rootpath=/dev/disk/by-label/root
-if ! [ -b "${rootpath}" ]; then
-  echo "ignition-ostree-mount-sysroot: Failed to find ${rootpath}" 1>&2
-  exit 1
-fi
-
-echo "Mounting ${rootpath} ($(realpath "${rootpath}")) to /sysroot"
-mount -o "$(coreos-rootflags)" "${rootpath}" /sysroot
diff --git a/overlay.d/05core/usr/lib/dracut/modules.d/40ignition-ostree/ignition-ostree-populate-var.sh b/overlay.d/05core/usr/lib/dracut/modules.d/40ignition-ostree/ignition-ostree-populate-var.sh
index 01212db7d5..e630b8753d 100755
--- a/overlay.d/05core/usr/lib/dracut/modules.d/40ignition-ostree/ignition-ostree-populate-var.sh
+++ b/overlay.d/05core/usr/lib/dracut/modules.d/40ignition-ostree/ignition-ostree-populate-var.sh
@@ -35,11 +35,5 @@ for varsubdir in lib log home roothome opt srv usrlocal mnt media; do
         systemd-tmpfiles --create --boot --root=/sysroot --prefix="/var/${varsubdir}"
     fi
 
-    if [[ $varsubdir == roothome ]]; then
-        # TODO move this to tmpfiles.d once systemd-tmpfiles handles C! with --root correctly.
-        # See https://github.com/coreos/fedora-coreos-config/pull/137
-        cp /sysroot/etc/skel/.bash* /sysroot/var/${varsubdir}
-    fi
-
     coreos-relabel "/var/${varsubdir}"
 done
diff --git a/overlay.d/05core/usr/lib/dracut/modules.d/40ignition-ostree/ignition-ostree-transposefs-autosave-xfs.service b/overlay.d/05core/usr/lib/dracut/modules.d/40ignition-ostree/ignition-ostree-transposefs-autosave-xfs.service
new file mode 100644
index 0000000000..e955c4a69a
--- /dev/null
+++ b/overlay.d/05core/usr/lib/dracut/modules.d/40ignition-ostree/ignition-ostree-transposefs-autosave-xfs.service
@@ -0,0 +1,23 @@
+[Unit]
+Description=Ignition OSTree: Autosave XFS Rootfs Partition
+DefaultDependencies=false
+After=ignition-disks.service
+# Avoid racing with UUID regeneration
+After=ignition-ostree-uuid-root.service
+After=ignition-ostree-growfs.service
+# https://issues.redhat.com/browse/OCPBUGS-16157
+# On multipath systems mounting the /sysroot before
+# the ignition-ostree services causes the transpose to fail.
+Before=sysroot.mount
+Before=ignition-ostree-transposefs-restore.service
+OnFailure=emergency.target
+OnFailureJobMode=isolate
+
+ConditionKernelCommandLine=ostree
+# only run if ignition-ostree-growfs ran since that's when pathological cases occur
+ConditionPathExists=/run/ignition-ostree-growfs.stamp
+
+[Service]
+Type=oneshot
+RemainAfterExit=yes
+ExecStart=/usr/libexec/ignition-ostree-transposefs autosave-xfs
diff --git a/overlay.d/05core/usr/lib/dracut/modules.d/40ignition-ostree/ignition-ostree-transposefs-detect.service b/overlay.d/05core/usr/lib/dracut/modules.d/40ignition-ostree/ignition-ostree-transposefs-detect.service
index 495949ed2a..389dc9eedf 100644
--- a/overlay.d/05core/usr/lib/dracut/modules.d/40ignition-ostree/ignition-ostree-transposefs-detect.service
+++ b/overlay.d/05core/usr/lib/dracut/modules.d/40ignition-ostree/ignition-ostree-transposefs-detect.service
@@ -6,6 +6,8 @@ Before=ignition-disks.service
 Before=initrd-root-fs.target
 Before=sysroot.mount
 ConditionKernelCommandLine=ostree
+OnFailure=emergency.target
+OnFailureJobMode=isolate
 
 # This stage requires udevd to detect disks
 Requires=systemd-udevd.service
diff --git a/overlay.d/05core/usr/lib/dracut/modules.d/40ignition-ostree/ignition-ostree-transposefs-restore.service b/overlay.d/05core/usr/lib/dracut/modules.d/40ignition-ostree/ignition-ostree-transposefs-restore.service
index 0b6d1449fc..eef3c064c7 100644
--- a/overlay.d/05core/usr/lib/dracut/modules.d/40ignition-ostree/ignition-ostree-transposefs-restore.service
+++ b/overlay.d/05core/usr/lib/dracut/modules.d/40ignition-ostree/ignition-ostree-transposefs-restore.service
@@ -4,8 +4,13 @@ DefaultDependencies=false
 After=ignition-disks.service
 # Avoid racing with UUID regeneration
 After=ignition-ostree-uuid-root.service
-Before=ignition-ostree-growfs.service
-Before=ignition-ostree-mount-firstboot-sysroot.service
+After=ignition-ostree-growfs.service
+# https://issues.redhat.com/browse/OCPBUGS-16157
+# On multipath systems mounting the /sysroot before
+# the ignition-ostree services causes the transpose to fail.
+Before=sysroot.mount
+OnFailure=emergency.target
+OnFailureJobMode=isolate
 
 ConditionKernelCommandLine=ostree
 ConditionPathIsDirectory=/run/ignition-ostree-transposefs
diff --git a/overlay.d/05core/usr/lib/dracut/modules.d/40ignition-ostree/ignition-ostree-transposefs-save.service b/overlay.d/05core/usr/lib/dracut/modules.d/40ignition-ostree/ignition-ostree-transposefs-save.service
index 591da2d9af..bc03499ecb 100644
--- a/overlay.d/05core/usr/lib/dracut/modules.d/40ignition-ostree/ignition-ostree-transposefs-save.service
+++ b/overlay.d/05core/usr/lib/dracut/modules.d/40ignition-ostree/ignition-ostree-transposefs-save.service
@@ -8,6 +8,8 @@ ConditionPathIsDirectory=/run/ignition-ostree-transposefs
 # Any services looking at mounts need to order after this
 # because it causes device re-probing.
 After=coreos-gpt-setup.service
+OnFailure=emergency.target
+OnFailureJobMode=isolate
 
 [Service]
 Type=oneshot
diff --git a/overlay.d/05core/usr/lib/dracut/modules.d/40ignition-ostree/ignition-ostree-transposefs.sh b/overlay.d/05core/usr/lib/dracut/modules.d/40ignition-ostree/ignition-ostree-transposefs.sh
index 54ebcc134a..7bec45c936 100755
--- a/overlay.d/05core/usr/lib/dracut/modules.d/40ignition-ostree/ignition-ostree-transposefs.sh
+++ b/overlay.d/05core/usr/lib/dracut/modules.d/40ignition-ostree/ignition-ostree-transposefs.sh
@@ -2,6 +2,7 @@
 set -euo pipefail
 
 boot_sector_size=440
+esp_typeguid=c12a7328-f81f-11d2-ba4b-00a0c93ec93b
 bios_typeguid=21686148-6449-6e6f-744e-656564454649
 prep_typeguid=9e1a2d38-c612-4316-aa26-8b49521e5a8b
 
@@ -21,7 +22,12 @@ saved_esp=${saved_data}/esp
 saved_bios=${saved_data}/bios
 saved_prep=${saved_data}/prep
 zram_dev=${saved_data}/zram_dev
-partstate_root=/run/ignition-ostree-rootfs-partstate.json
+partstate_root=/run/ignition-ostree-rootfs-partstate.sh
+
+is_rhcos9() {
+    source /etc/os-release
+    [ "${ID}" == "rhcos" ] && [ "${RHEL_VERSION%%.*}" -eq 9 ]
+}
 
 # Print jq query string for wiped filesystems with label $1
 query_fslabel() {
@@ -30,18 +36,25 @@ query_fslabel() {
 
 # Print jq query string for partitions with type GUID $1
 query_parttype() {
-    echo ".storage?.disks? // [] | map(.partitions?) | flatten | map(select(try .typeGuid catch \"\" | ascii_downcase == \"$1\"))"
+    echo ".storage?.disks? // [] | map(.partitions?) | flatten | map(select(has(\"typeGuid\") and (.typeGuid | ascii_downcase == \"$1\")))"
 }
 
+# Print partition labels for partitions with type GUID $1
+get_partlabels_for_parttype() {
+    jq -r "$(query_parttype $1) | .[].label" "${ignition_cfg}"
+}
 
 # Mounts device to directory, with extra logging of the src device
 mount_verbose() {
     local srcdev=$1; shift
     local destdir=$1; shift
-    echo "Mounting ${srcdev} ($(realpath "$srcdev")) to $destdir"
-    mount "${srcdev}" "${destdir}"
+    local mode=${1:-ro}
+    echo "Mounting ${srcdev} ${mode} ($(realpath "$srcdev")) to $destdir"
+    mkdir -p "${destdir}"
+    mount -o "${mode}" "${srcdev}" "${destdir}"
 }
 
+# A copy of this exists in ignition-ostree-growfs.sh.
 # Sometimes, for some reason the by-label symlinks aren't updated. Detect these
 # cases, and explicitly `udevadm trigger`.
 # See: https://bugzilla.redhat.com/show_bug.cgi?id=1908780
@@ -50,7 +63,10 @@ udev_trigger_on_label_mismatch() {
     local expected_dev=$1; shift
     local actual_dev
     expected_dev=$(realpath "${expected_dev}")
-    actual_dev=$(realpath "/dev/disk/by-label/$label")
+    # We `|| :` here because sometimes /dev/disk/by-label/$label is missing.
+    # We've seen this on Fedora kernels with debug enabled (common in `rawhide`).
+    # See https://github.com/coreos/fedora-coreos-tracker/issues/1092
+    actual_dev=$(realpath "/dev/disk/by-label/$label" || :)
     if [ "$actual_dev" != "$expected_dev" ]; then
         echo "Expected /dev/disk/by-label/$label to point to $expected_dev, but points to $actual_dev; triggering udev"
         udevadm trigger --settle "$expected_dev"
@@ -63,50 +79,161 @@ get_partition_offset() {
     cat "/sys${devpath}/start"
 }
 
-mount_and_restore_filesystem() {
+# copied from generator-lib.sh
+karg() {
+    local name="$1" value="${2:-}"
+    local cmdline=( $(</proc/cmdline) )
+    for arg in "${cmdline[@]}"; do
+        if [[ "${arg%%=*}" == "${name}" ]]; then
+            value="${arg#*=}"
+        fi
+    done
+    echo "${value}"
+}
+
+mount_and_restore_filesystem_by_label() {
     local label=$1; shift
     local mountpoint=$1; shift
     local saved_fs=$1; shift
     local new_dev
-    new_dev=$(jq -r "$(query_fslabel "${label}") | .[0].device" "${ignition_cfg}")
-    udev_trigger_on_label_mismatch "${label}" "${new_dev}"
-    mkdir -p "${mountpoint}"
-    mount_verbose "/dev/disk/by-label/${label}" "${mountpoint}"
-    find "${saved_fs}" -mindepth 1 -maxdepth 1 -exec mv -t "${mountpoint}" {} \;
+    new_dev=$(jq -r "$(query_fslabel "${label}") | .[0].device // \"\"" "${ignition_cfg}")
+    # in the autosave-xfs path, it's not driven by the Ignition config so we
+    # don't expect a new device there
+    if [ -n "${new_dev}" ]; then
+        udev_trigger_on_label_mismatch "${label}" "${new_dev}"
+    fi
+    mount_verbose "/dev/disk/by-label/${label}" "${mountpoint}" rw
+    find "${saved_fs}" -mindepth 1 -maxdepth 1 -exec mv -t "${mountpoint}" {} +
+}
+
+mount_and_save_filesystem_by_label() {
+    local label=$1; shift
+    local saved_fs=$1; shift
+    local fs=/dev/disk/by-label/${label}
+    if [[ -f /run/coreos/secure-execution ]]; then
+        local roothash_karg=${label}fs.roothash
+        local roothash=$(karg "${roothash_karg}")
+        if [ -z "${roothash}" ]; then
+          echo "Missing kernel argument ${roothash_karg}; aborting"
+          exit 1
+        fi
+        local roothash_part=/dev/disk/by-partlabel/${label}hash
+        veritysetup open "${fs}" "${label}" "${roothash_part}" "${roothash}"
+        fs=/dev/mapper/${label}
+    fi
+    mount_verbose "${fs}" /var/tmp/mnt
+    cp -aT /var/tmp/mnt "${saved_fs}"
+    umount /var/tmp/mnt
+    if [[ -f /run/coreos/secure-execution ]]; then
+        veritysetup close "${label}"
+    fi
+}
+
+# This implements https://github.com/coreos/fedora-coreos-tracker/issues/1183.
+should_autosave_rootfs() {
+    local fstype
+    fstype=$(lsblk -no FSTYPE "${root_part}")
+    if [ "$fstype" != xfs ]; then
+        echo "Filesystem is not XFS (found $fstype); skipping" >&2
+        echo 0
+        return
+    fi
+    local agcount
+    # This runs xfs_info on the unmounted filesystem, because mounting an
+    # XFS filesystem that has grown an excessive number of allocation groups
+    # can be very slow.
+    eval $(xfs_info "${root_part}" | grep -o 'agcount=[0-9]*')
+    # This is roughly ~700GiB currently (based on initial ag sizing at build time)
+    # which ensures we grow only on "large" root filesystems.
+    # Specifically for e.g. OpenShift, this ensures we don't reprovision on default
+    # worker node root filesystems.
+    local threshold
+    threshold=400
+    if [ "$agcount" -lt "${threshold}" ]; then
+        echo "autosave-xfs: ${root_part} agcount=$agcount is lower than threshold=${threshold}" >&2
+        echo 0
+        return
+    else
+        echo "autosave-xfs: ${root_part} agcount=$agcount meets threshold=${threshold}" >&2
+        echo 1
+    fi
 }
 
+ensure_zram_dev() {
+    if test -d "${saved_data}"; then
+        return 0
+    fi
+    mem_available=$(grep MemAvailable /proc/meminfo | awk '{print $2}')
+    # Just error out early if we don't even have 1G to work with. This
+    # commonly happens if you `cosa run` but forget to add `--memory`. That
+    # way you get a nicer error instead of the spew of EIO errors from `cp`.
+    # The amount we need is really dependent on a bunch of factors, but just
+    # ballpark it at 3G.
+    if [ "${mem_available}" -lt $((1*1024*1024)) ] && [ "${wipes_root}" != 0 ]; then
+        echo "Root reprovisioning requires at least 3G of RAM" >&2
+        exit 1
+    fi
+    modprobe zram num_devices=0
+    read dev < /sys/class/zram-control/hot_add
+    # disksize is set arbitrarily large, as zram is capped by mem_limit
+    echo 10G > /sys/block/zram"${dev}"/disksize
+    # Limit zram to 90% of available RAM: we want to be greedy since the
+    # boot breaks anyway, but we still want to leave room for everything
+    # else so it hits ENOSPC and doesn't invoke the OOM killer
+    echo $(( mem_available * 90 / 100 ))K > /sys/block/zram"${dev}"/mem_limit
+    mkfs.xfs -q /dev/zram"${dev}"
+    mkdir "${saved_data}"
+    mount -t xfs /dev/zram"${dev}" "${saved_data}"
+    # save the zram device number created for when called to cleanup
+    echo "${dev}" > "${zram_dev}"
+}
+
+print_zram_mm_stat() {
+    echo "zram usage:"
+    read dev < "${zram_dev}"
+    cat /sys/block/zram"${dev}"/mm_stat
+}
+
+# In Secure Execution case user is not allowed to modify partition table
+check_and_set_secex_config() {
+    if [[ -f /run/coreos/secure-execution ]]; then
+        local wr=$(jq "$(query_fslabel root) | length" "${ignition_cfg}")
+        local wb=$(jq "$(query_fslabel boot) | length" "${ignition_cfg}")
+        if [ "${wr}${wb}" != "00" ]; then
+            echo "Modifying bootfs and rootfs is not supported in Secure Execution mode"
+            exit 1
+        fi
+        # Cached config isn't merged, so reset it and recheck again, just to make sure
+        ignition_cfg=/usr/lib/ignition/base.d/01-secex.ign
+    fi
+}
+
+# We could have done this during 'detect' below, but other cases also request
+# info from config, so just check cached one and reset to secex.ign now
+check_and_set_secex_config
+
 case "${1:-}" in
     detect)
         # Mounts are not in a private namespace so we can mount ${saved_data}
         wipes_root=$(jq "$(query_fslabel root) | length" "${ignition_cfg}")
         wipes_boot=$(jq "$(query_fslabel boot) | length" "${ignition_cfg}")
-        wipes_esp=$(jq "$(query_fslabel EFI-SYSTEM) | length" "${ignition_cfg}")
+        creates_esp=$(jq "$(query_parttype ${esp_typeguid}) | length" "${ignition_cfg}")
         creates_bios=$(jq "$(query_parttype ${bios_typeguid}) | length" "${ignition_cfg}")
         creates_prep=$(jq "$(query_parttype ${prep_typeguid}) | length" "${ignition_cfg}")
-        if [ "${wipes_root}${wipes_boot}${wipes_esp}${creates_bios}${creates_prep}" = "00000" ]; then
+        if [ "${wipes_root}${wipes_boot}${creates_esp}${creates_bios}${creates_prep}" = "00000" ]; then
             exit 0
         fi
         echo "Detected partition replacement in fetched Ignition config: /run/ignition.json"
-        # verify all BIOS and PReP partitions have non-null unique labels
+        # verify all ESP, BIOS, and PReP partitions have non-null unique labels
+        unique_esp=$(jq -r "$(query_parttype ${esp_typeguid}) | [.[].label | values] | unique | length" "${ignition_cfg}")
         unique_bios=$(jq -r "$(query_parttype ${bios_typeguid}) | [.[].label | values] | unique | length" "${ignition_cfg}")
         unique_prep=$(jq -r "$(query_parttype ${prep_typeguid}) | [.[].label | values] | unique | length" "${ignition_cfg}")
-        if [ "${creates_bios}" != "${unique_bios}" -o "${creates_prep}" != "${unique_prep}" ]; then
-            echo "Found duplicate or missing BIOS-BOOT or PReP labels in config" >&2
+        if [ "${creates_esp}" != "${unique_esp}" -o "${creates_bios}" != "${unique_bios}" -o "${creates_prep}" != "${unique_prep}" ]; then
+            echo "Found duplicate or missing ESP, BIOS-BOOT, or PReP labels in config" >&2
             exit 1
         fi
-        modprobe zram num_devices=0
-        read dev < /sys/class/zram-control/hot_add
-        # disksize is set arbitrarily large, as zram is capped by mem_limit
-        echo 10G > /sys/block/zram"${dev}"/disksize
-        # Limit zram to 90% of available RAM: we want to be greedy since the
-        # boot breaks anyway, but we still want to leave room for everything
-        # else so it hits ENOSPC and doesn't invoke the OOM killer
-        echo $(( $(grep MemAvailable /proc/meminfo | awk '{print $2}') * 90 / 100 ))K > /sys/block/zram"${dev}"/mem_limit
-        mkfs.xfs /dev/zram"${dev}"
-        mkdir "${saved_data}"
-        mount /dev/zram"${dev}" "${saved_data}"
-        # save the zram device number created for when called to cleanup
-        echo "${dev}" > "${zram_dev}"
+
+        ensure_zram_dev
 
         if [ "${wipes_root}" != "0" ]; then
             mkdir "${saved_root}"
@@ -114,7 +241,7 @@ case "${1:-}" in
         if [ "${wipes_boot}" != "0" ]; then
             mkdir "${saved_boot}"
         fi
-        if [ "${wipes_esp}" != "0" ]; then
+        if [ "${creates_esp}" != "0" ]; then
             mkdir "${saved_esp}"
         fi
         if [ "${creates_bios}" != "0" ]; then
@@ -124,24 +251,37 @@ case "${1:-}" in
             mkdir "${saved_prep}"
         fi
         ;;
+    autosave-xfs)
+        should_autosave=$(should_autosave_rootfs)
+        if [ "${should_autosave}" = "1" ]; then
+            wipes_root=1
+            ensure_zram_dev
+            # in the in-place reprovisioning case, the rootfs was already saved
+            if [ ! -d "${saved_root}" ]; then
+                mkdir "${saved_root}"
+                echo "Moving rootfs to RAM..."
+                mount_and_save_filesystem_by_label root "${saved_root}"
+                print_zram_mm_stat
+            fi
+            mkfs.xfs "${root_part}" -L root -f
+            # for tests
+            touch /run/ignition-ostree-autosaved-xfs.stamp
+        fi
+        ;;
     save)
         # Mounts happen in a private mount namespace since we're not "offically" mounting
         if [ -d "${saved_root}" ]; then
             echo "Moving rootfs to RAM..."
-            mount_verbose "${root_part}" /sysroot
-            cp -aT /sysroot "${saved_root}"
+            mount_and_save_filesystem_by_label root "${saved_root}"
             # also store the state of the partition
-            lsblk "${root_part}" --nodeps --paths --json -b -o NAME,SIZE | jq -c . > "${partstate_root}"
+            lsblk "${root_part}" --nodeps --pairs -b --paths -o NAME,TYPE,SIZE > "${partstate_root}"
         fi
         if [ -d "${saved_boot}" ]; then
             echo "Moving bootfs to RAM..."
-            mkdir -p /sysroot/boot
-            mount_verbose "${boot_part}" /sysroot/boot
-            cp -aT /sysroot/boot "${saved_boot}"
+            mount_and_save_filesystem_by_label boot "${saved_boot}"
         fi
         if [ -d "${saved_esp}" ]; then
             echo "Moving EFI System Partition to RAM..."
-            mkdir -p /sysroot/boot/efi
             mount_verbose "${esp_part}" /sysroot/boot/efi
             cp -aT /sysroot/boot/efi "${saved_esp}"
         fi
@@ -159,32 +299,39 @@ case "${1:-}" in
             echo "Moving PReP partition to RAM..."
             cat "${prep_part}" > "${saved_prep}/partition"
         fi
-        echo "zram usage:"
-        read dev < "${zram_dev}"
-        cat /sys/block/zram"${dev}"/mm_stat
+        print_zram_mm_stat
         ;;
     restore)
         # Mounts happen in a private mount namespace since we're not "offically" mounting
         if [ -d "${saved_root}" ]; then
             echo "Restoring rootfs from RAM..."
-            mount_and_restore_filesystem root /sysroot "${saved_root}"
+            mount_and_restore_filesystem_by_label root /sysroot "${saved_root}"
             chcon -v --reference "${saved_root}" /sysroot  # the root of the fs itself
             chattr +i $(ls -d /sysroot/ostree/deploy/*/deploy/*/)
         fi
         if [ -d "${saved_boot}" ]; then
             echo "Restoring bootfs from RAM..."
-            mount_and_restore_filesystem boot /sysroot/boot "${saved_boot}"
+            mount_and_restore_filesystem_by_label boot /sysroot/boot "${saved_boot}"
             chcon -v --reference "${saved_boot}" /sysroot/boot  # the root of the fs itself
         fi
         if [ -d "${saved_esp}" ]; then
             echo "Restoring EFI System Partition from RAM..."
-            mount_and_restore_filesystem EFI-SYSTEM /sysroot/boot/efi "${saved_esp}"
+            get_partlabels_for_parttype "${esp_typeguid}" | while read label; do
+                # Don't use mount_and_restore_filesystem_by_label because:
+                # 1. We're mounting by partlabel, not FS label
+                # 2. We need to copy the contents to each partition, not move
+                #    them once
+                # 3. We don't need the by-label symlink to be correct and
+                #    nothing later in boot will be mounting the filesystem
+                mountpoint="/mnt/esp-${label}"
+                mount_verbose "/dev/disk/by-partlabel/${label}" "${mountpoint}" rw
+                find "${saved_esp}" -mindepth 1 -maxdepth 1 -exec cp -at "${mountpoint}" {} +
+            done
         fi
         if [ -d "${saved_bios}" ]; then
             echo "Restoring BIOS Boot partition and boot sector from RAM..."
             expected_start=$(cat "${saved_bios}/start")
-            # iterate over each new BIOS Boot partition, by label
-            jq -r "$(query_parttype ${bios_typeguid}) | .[].label" "${ignition_cfg}" | while read label; do
+            get_partlabels_for_parttype "${bios_typeguid}" | while read label; do
                 cur_part="/dev/disk/by-partlabel/${label}"
                 # boot sector hardcodes the partition start; ensure it matches
                 cur_start=$(get_partition_offset "${cur_part}")
@@ -201,8 +348,7 @@ case "${1:-}" in
         fi
         if [ -d "${saved_prep}" ]; then
             echo "Restoring PReP partition from RAM..."
-            # iterate over each new PReP partition, by label
-            jq -r "$(query_parttype ${prep_typeguid}) | .[].label" "${ignition_cfg}" | while read label; do
+            get_partlabels_for_parttype "${prep_typeguid}" | while read label; do
                 cat "${saved_prep}/partition" > "/dev/disk/by-partlabel/${label}"
             done
         fi
@@ -213,7 +359,27 @@ case "${1:-}" in
             read dev < "${zram_dev}"
             umount "${saved_data}"
             rm -rf "${saved_data}" "${partstate_root}"
-            echo "${dev}" > /sys/class/zram-control/hot_remove
+            # After unmounting, make sure zram device state is stable before we remove it.
+            # See https://github.com/openshift/os/issues/1149
+            # Should remove when https://bugzilla.redhat.com/show_bug.cgi?id=2172058 is fixed.
+            # Seems the previous workaround https://github.com/coreos/fedora-coreos-config/pull/2226
+            # can not completely resolve the race issue, try in loop with a small sleep for el9 + !x86_64.
+            if [ $(uname -m) != x86_64 ] && is_rhcos9; then
+                for x in {0..10}; do
+                    if ! echo "${dev}" > /sys/class/zram-control/hot_remove 2>/dev/null; then
+                        sleep 0.1
+                    else
+                        dev=
+                        break
+                    fi
+                done
+                # try it one last time and let it possibly fail
+                if [ -n "${dev}" ]; then
+                    echo "${dev}" > /sys/class/zram-control/hot_remove
+                fi
+            else
+                echo "${dev}" > /sys/class/zram-control/hot_remove
+            fi
         fi
         ;;
     *)
diff --git a/overlay.d/05core/usr/lib/dracut/modules.d/40ignition-ostree/ignition-ostree-uuid-boot.service b/overlay.d/05core/usr/lib/dracut/modules.d/40ignition-ostree/ignition-ostree-uuid-boot.service
index bad9ece0ce..85448c2a67 100644
--- a/overlay.d/05core/usr/lib/dracut/modules.d/40ignition-ostree/ignition-ostree-uuid-boot.service
+++ b/overlay.d/05core/usr/lib/dracut/modules.d/40ignition-ostree/ignition-ostree-uuid-boot.service
@@ -1,17 +1,17 @@
 [Unit]
-Description=Ignition OSTree: Regenerate filesystem UUID (boot)
-DefaultDependencies=false
+Description=Ignition OSTree: Regenerate Filesystem UUID (boot)
 ConditionPathExists=/usr/lib/initrd-release
 ConditionKernelCommandLine=ostree
 ConditionPathExists=!/run/ostree-live
-# We run pretty early
-Before=coreos-copy-firstboot-network.service
-Before=ignition-fetch.service
-Before=ignition-setup-base.service
-Before=ignition-setup-user.service
 # Any services looking at mounts need to order after this
 # because it causes device re-probing.
 After=coreos-gpt-setup.service
+# This could mount the bootfs rw and the ext4 in el9 at least doesn't seem to
+# like doing that in parallel with restamping the UUID
+Before=ignition-kargs.service
+
+# If we're going to reprovision the bootfs, then there's no need to restamp
+ConditionKernelCommandLine=!bootfs.roothash
 
 Before=systemd-fsck@dev-disk-by\x2dlabel-boot.service
 Requires=dev-disk-by\x2dlabel-boot.device
diff --git a/overlay.d/05core/usr/lib/dracut/modules.d/40ignition-ostree/ignition-ostree-uuid-root.service b/overlay.d/05core/usr/lib/dracut/modules.d/40ignition-ostree/ignition-ostree-uuid-root.service
index 8f3ea1c5f5..80ecf9d42a 100644
--- a/overlay.d/05core/usr/lib/dracut/modules.d/40ignition-ostree/ignition-ostree-uuid-root.service
+++ b/overlay.d/05core/usr/lib/dracut/modules.d/40ignition-ostree/ignition-ostree-uuid-root.service
@@ -1,21 +1,24 @@
 [Unit]
-Description=Ignition OSTree: Regenerate filesystem UUID (root)
-# These conditions match mount-firstboot-sysroot.service
+Description=Ignition OSTree: Regenerate Filesystem UUID (root)
+# These conditions match those generated in coreos-diskful-generator
 DefaultDependencies=false
-ConditionKernelCommandLine=!root
 ConditionKernelCommandLine=ostree
 ConditionPathExists=!/run/ostree-live
-Before=initrd-root-fs.target
+Before=sysroot.mount initrd-root-fs.target
 After=ignition-disks.service
+# If we've reprovisioned the rootfs, then there's no need to restamp
+ConditionPathExists=!/run/ignition-ostree-transposefs
+ConditionKernelCommandLine=!rootfs.roothash
 
 After=dev-disk-by\x2dlabel-root.device
 # Avoid racing with fsck
 Before=systemd-fsck@dev-disk-by\x2dlabel-root.service
+Before=systemd-fsck@dev-disk-by\x2dlabel-dm-mpath-root.service
 
 # Note we don't have a Requires: /dev/disk/by-label/root here like
 # the -subsequent service does because ignition-disks may have
 # regenerated it.
-Before=ignition-ostree-mount-firstboot-sysroot.service
+Before=sysroot.mount
 
 [Service]
 Type=oneshot
diff --git a/overlay.d/05core/usr/lib/dracut/modules.d/40ignition-ostree/module-setup.sh b/overlay.d/05core/usr/lib/dracut/modules.d/40ignition-ostree/module-setup.sh
index ba10fed1c4..b73f977939 100755
--- a/overlay.d/05core/usr/lib/dracut/modules.d/40ignition-ostree/module-setup.sh
+++ b/overlay.d/05core/usr/lib/dracut/modules.d/40ignition-ostree/module-setup.sh
@@ -10,7 +10,14 @@ install_ignition_unit() {
     local unit=$1; shift
     local target=${1:-complete}
     inst_simple "$moddir/$unit" "$systemdsystemunitdir/$unit"
-    systemctl -q --root="$initdir" add-requires "ignition-${target}.target" "$unit"
+    # note we `|| exit 1` here so we error out if e.g. the units are missing
+    # see https://github.com/coreos/fedora-coreos-config/issues/799
+    systemctl -q --root="$initdir" add-requires "ignition-${target}.target" "$unit" || exit 1
+}
+
+installkernel() {
+    # Used by ignition-ostree-transposefs
+    instmods -c zram
 }
 
 install() {
@@ -21,12 +28,21 @@ install() {
         systemd-sysusers \
         systemd-tmpfiles \
         sort \
+        xfs_info \
+        xfs_spaceman \
         uniq
 
-    # coreos-growpart deps
+    if [[ $(uname -m) = s390x ]]; then
+        # for Secure Execution
+        inst_multiple \
+            veritysetup
+    fi
+
+    # ignition-ostree-growfs deps
     inst_multiple  \
         basename   \
         blkid      \
+        blockdev   \
         cat        \
         dirname    \
         findmnt    \
@@ -38,6 +54,7 @@ install() {
         touch      \
         xfs_admin  \
         xfs_growfs \
+        wc         \
         wipefs
 
     # growpart deps
@@ -67,25 +84,28 @@ install() {
 
     inst_multiple jq chattr
     inst_script "$moddir/ignition-ostree-transposefs.sh" "/usr/libexec/ignition-ostree-transposefs"
-    for x in detect save restore; do
+    for x in detect save autosave-xfs restore; do
         install_ignition_unit ignition-ostree-transposefs-${x}.service
     done
 
     # Disk support
-    install_ignition_unit ignition-ostree-mount-firstboot-sysroot.service diskful
     for p in boot root; do
         install_ignition_unit ignition-ostree-uuid-${p}.service diskful
     done
     inst_script "$moddir/ignition-ostree-firstboot-uuid" \
         "/usr/sbin/ignition-ostree-firstboot-uuid"
-    install_ignition_unit ignition-ostree-mount-subsequent-sysroot.service diskful-subsequent
-    inst_script "$moddir/ignition-ostree-mount-sysroot.sh" \
-        "/usr/sbin/ignition-ostree-mount-sysroot"
-    inst_script "$moddir/coreos-rootflags.sh" \
-        "/usr/sbin/coreos-rootflags"
 
     install_ignition_unit ignition-ostree-growfs.service
-    inst_script "$moddir/coreos-growpart" /usr/libexec/coreos-growpart
+    inst_script "$moddir/ignition-ostree-growfs.sh" \
+        /usr/sbin/ignition-ostree-growfs
+
+    install_ignition_unit ignition-ostree-check-rootfs-size.service
+    inst_script "$moddir/coreos-check-rootfs-size" \
+        /usr/libexec/coreos-check-rootfs-size
+
+    install_ignition_unit ignition-ostree-mount-state-overlays.service
+    inst_script "$moddir/ignition-ostree-mount-state-overlays.sh" \
+        /usr/libexec/ignition-ostree-mount-state-overlays
 
     inst_script "$moddir/coreos-relabel" /usr/bin/coreos-relabel
 }
diff --git a/overlay.d/05core/usr/lib/dracut/modules.d/50coreos-kernel/coreos-check-kernel.service b/overlay.d/05core/usr/lib/dracut/modules.d/50coreos-kernel/coreos-check-kernel.service
index ce8a0ac524..f325a7278e 100644
--- a/overlay.d/05core/usr/lib/dracut/modules.d/50coreos-kernel/coreos-check-kernel.service
+++ b/overlay.d/05core/usr/lib/dracut/modules.d/50coreos-kernel/coreos-check-kernel.service
@@ -1,5 +1,5 @@
 [Unit]
-Description=Check that initrd matches kernel
+Description=Check That Initrd Matches Kernel
 DefaultDependencies=false
 Before=sysinit.target systemd-modules-load.service
 ConditionPathIsDirectory=!/usr/lib/modules/%v
diff --git a/overlay.d/05core/usr/lib/dracut/modules.d/50coreos-kernel/module-setup.sh b/overlay.d/05core/usr/lib/dracut/modules.d/50coreos-kernel/module-setup.sh
index d5cd1b1e12..cac7b643f4 100644
--- a/overlay.d/05core/usr/lib/dracut/modules.d/50coreos-kernel/module-setup.sh
+++ b/overlay.d/05core/usr/lib/dracut/modules.d/50coreos-kernel/module-setup.sh
@@ -2,7 +2,9 @@ install_unit() {
     unit="$1"; shift
     target="$1"; shift
     inst_simple "$moddir/$unit" "$systemdsystemunitdir/$unit"
-    systemctl -q --root="$initdir" add-requires "$target" "$unit"
+    # note we `|| exit 1` here so we error out if e.g. the units are missing
+    # see https://github.com/coreos/fedora-coreos-config/issues/799
+    systemctl -q --root="$initdir" add-requires "$target" "$unit" || exit 1
 }
 
 install() {
diff --git a/overlay.d/05core/usr/lib/dracut/modules.d/60coreos-agetty-workaround/coreos-touch-run-agetty.service b/overlay.d/05core/usr/lib/dracut/modules.d/60coreos-agetty-workaround/coreos-touch-run-agetty.service
new file mode 100644
index 0000000000..b6984b18e3
--- /dev/null
+++ b/overlay.d/05core/usr/lib/dracut/modules.d/60coreos-agetty-workaround/coreos-touch-run-agetty.service
@@ -0,0 +1,13 @@
+# Temporary hack to work around agetty SELinux denials.
+# https://github.com/coreos/fedora-coreos-config/pull/859#issuecomment-783713383
+# https://bugzilla.redhat.com/show_bug.cgi?id=1932053
+[Unit]
+Description=CoreOS: Touch /run/agetty.reload
+Documentation=https://bugzilla.redhat.com/show_bug.cgi?id=1932053
+DefaultDependencies=false
+Before=initrd.target
+
+[Service]
+Type=oneshot
+RemainAfterExit=yes
+ExecStart=/usr/bin/touch /run/agetty.reload
diff --git a/overlay.d/05core/usr/lib/dracut/modules.d/60coreos-agetty-workaround/module-setup.sh b/overlay.d/05core/usr/lib/dracut/modules.d/60coreos-agetty-workaround/module-setup.sh
new file mode 100755
index 0000000000..1423fd5a42
--- /dev/null
+++ b/overlay.d/05core/usr/lib/dracut/modules.d/60coreos-agetty-workaround/module-setup.sh
@@ -0,0 +1,21 @@
+#!/bin/bash
+# -*- mode: shell-script; indent-tabs-mode: nil; sh-basic-offset: 4; -*-
+# ex: ts=8 sw=4 sts=4 et filetype=sh
+
+# Temporary workaround for https://bugzilla.redhat.com/show_bug.cgi?id=1932053.
+
+install_unit() {
+    local unit=$1; shift
+    inst_simple "$moddir/$unit" "$systemdsystemunitdir/$unit"
+    # note we `|| exit 1` here so we error out if e.g. the units are missing
+    # see https://github.com/coreos/fedora-coreos-config/issues/799
+    systemctl -q --root="$initdir" add-requires initrd.target "$unit" || exit 1
+}
+
+install() {
+    inst_multiple \
+        touch
+
+    # TODO f35: check if we can drop this whole module
+    install_unit coreos-touch-run-agetty.service
+}
diff --git a/overlay.d/05core/usr/lib/dracut/modules.d/99emergency-shell-setup/emergency-shell.sh b/overlay.d/05core/usr/lib/dracut/modules.d/99emergency-shell-setup/emergency-shell.sh
new file mode 100644
index 0000000000..b9b89e671a
--- /dev/null
+++ b/overlay.d/05core/usr/lib/dracut/modules.d/99emergency-shell-setup/emergency-shell.sh
@@ -0,0 +1,85 @@
+# Display relevant errors then enter emergency shell
+
+# _wait_for_journalctl_to_stop will block until either:
+# - no messages have appeared in journalctl for the past 5 seconds
+# - 15 seconds have elapsed
+_wait_for_journalctl_to_stop() {
+    local time_since_last_log=0
+
+    local time_started="$(date '+%s')"
+    local now="$(date '+%s')"
+
+    while [ ${time_since_last_log} -lt 5 -a $((now-time_started)) -lt 15 ]; do
+        sleep 1
+
+        local last_log_timestamp="$(journalctl -e -n 1 -q -o short-unix | cut -d '.' -f 1)"
+        local now="$(date '+%s')"
+
+        local time_since_last_log=$((now-last_log_timestamp))
+    done
+}
+
+_display_relevant_errors() {
+    failed=$(systemctl --failed --no-legend --plain | cut -f 1 -d ' ')
+    if [ -n "${failed}" ]; then
+        # Something failed, suppress kernel logs so that it's more likely
+        # the useful bits from the journal are available.
+        dmesg --console-off
+
+        # There's a couple straggler systemd messages. Wait until it's been 5
+        # seconds since something was written to the journal.
+        _wait_for_journalctl_to_stop
+
+        # Print Ignition logs
+        if echo ${failed} | grep -qFe 'ignition-'; then
+            cat <<EOF
+------
+Ignition has failed. Please ensure your config is valid. Note that only
+Ignition spec v3.0.0+ configs are accepted.
+
+A CLI validation tool to check this called ignition-validate can be
+downloaded from GitHub:
+    https://github.com/coreos/ignition/releases
+------
+
+EOF
+        fi
+
+        # If this is a live boot, check for ENOSPC in initramfs filesystem
+        # Try creating a 64 KiB file, in case a small file was deleted on
+        # service failure
+        # https://github.com/coreos/fedora-coreos-tracker/issues/1055
+        if [ -f /etc/coreos-live-initramfs ] && \
+            ! dd if=/dev/zero of=/tmp/check-space bs=4K count=16 2>/dev/null; then
+            cat <<EOF
+------
+Ran out of memory when unpacking initrd filesystem.  Ensure your system has
+at least 2 GiB RAM if booting with coreos.live.rootfs_url, or 4 GiB otherwise.
+------
+
+EOF
+            # Don't show logs from failed units, since they'll just be
+            # random misleading errors.
+        else
+            echo "Displaying logs from failed units: ${failed}"
+            for unit in ${failed}; do
+                # 10 lines should be enough for everyone
+                SYSTEMD_COLORS=true journalctl -b --no-pager --no-hostname -u ${unit} -n 10
+            done
+        fi
+    fi
+}
+
+# in SE case drop everything before entering shell
+if [ -f /run/coreos/secure-execution ]; then
+    rm -f /run/ignition.json
+    rm -f /usr/lib/ignition/user.ign
+    rm -f /usr/lib/coreos/ignition.asc
+fi
+
+# Print warnings/informational messages to all configured consoles on the
+# machine. Code inspired by https://github.com/dracutdevs/dracut/commit/32f68c1
+MESSAGE="$(_display_relevant_errors)"
+while read -r _tty rest; do
+    echo -e "$MESSAGE" > /dev/"$_tty"
+done < /proc/consoles
diff --git a/overlay.d/05core/usr/lib/dracut/modules.d/99emergency-timeout/ignition-virtio-dump-journal.service b/overlay.d/05core/usr/lib/dracut/modules.d/99emergency-shell-setup/ignition-virtio-dump-journal.service
similarity index 75%
rename from overlay.d/05core/usr/lib/dracut/modules.d/99emergency-timeout/ignition-virtio-dump-journal.service
rename to overlay.d/05core/usr/lib/dracut/modules.d/99emergency-shell-setup/ignition-virtio-dump-journal.service
index 18a964cef9..27160589db 100644
--- a/overlay.d/05core/usr/lib/dracut/modules.d/99emergency-timeout/ignition-virtio-dump-journal.service
+++ b/overlay.d/05core/usr/lib/dracut/modules.d/99emergency-shell-setup/ignition-virtio-dump-journal.service
@@ -1,12 +1,14 @@
 [Unit]
-Description=Dump journal to virtio port
+Description=Dump Journal to Virtio Port
 ConditionPathExists=/etc/initrd-release
+ConditionPathExists=!/run/coreos/secure-execution
 DefaultDependencies=false
 ConditionVirtualization=|kvm
 ConditionVirtualization=|qemu
 Requires=systemd-journald.service
 After=systemd-journald.service
 After=basic.target
+Before=initrd.target
 
 [Service]
 Type=oneshot
diff --git a/overlay.d/05core/usr/lib/dracut/modules.d/99emergency-timeout/ignition-virtio-dump-journal.sh b/overlay.d/05core/usr/lib/dracut/modules.d/99emergency-shell-setup/ignition-virtio-dump-journal.sh
similarity index 100%
rename from overlay.d/05core/usr/lib/dracut/modules.d/99emergency-timeout/ignition-virtio-dump-journal.sh
rename to overlay.d/05core/usr/lib/dracut/modules.d/99emergency-shell-setup/ignition-virtio-dump-journal.sh
diff --git a/overlay.d/05core/usr/lib/dracut/modules.d/99emergency-timeout/module-setup.sh b/overlay.d/05core/usr/lib/dracut/modules.d/99emergency-shell-setup/module-setup.sh
similarity index 70%
rename from overlay.d/05core/usr/lib/dracut/modules.d/99emergency-timeout/module-setup.sh
rename to overlay.d/05core/usr/lib/dracut/modules.d/99emergency-shell-setup/module-setup.sh
index ed4ccc57ba..df52cba573 100755
--- a/overlay.d/05core/usr/lib/dracut/modules.d/99emergency-timeout/module-setup.sh
+++ b/overlay.d/05core/usr/lib/dracut/modules.d/99emergency-shell-setup/module-setup.sh
@@ -7,15 +7,18 @@ install_unit_wants() {
     local target="$1"; shift
     local instantiated="${1:-$unit}"; shift
     inst_simple "$moddir/$unit" "$systemdsystemunitdir/$unit"
-    systemctl -q --root="$initdir" add-wants "$target" "$instantiated"
+    # note we `|| exit 1` here so we error out if e.g. the units are missing
+    # see https://github.com/coreos/fedora-coreos-config/issues/799
+    systemctl -q --root="$initdir" add-wants "$target" "$instantiated" || exit 1
 }
 
 install() {
     inst_multiple \
         cut \
-        date
+        date \
+        dd
 
-    inst_hook emergency 99 "${moddir}/timeout.sh"
+    inst_hook emergency 99 "${moddir}/emergency-shell.sh"
 
     inst_script "$moddir/ignition-virtio-dump-journal.sh" "/usr/bin/ignition-virtio-dump-journal"
     install_unit_wants ignition-virtio-dump-journal.service emergency.target
diff --git a/overlay.d/05core/usr/lib/dracut/modules.d/99emergency-timeout/timeout.sh b/overlay.d/05core/usr/lib/dracut/modules.d/99emergency-timeout/timeout.sh
deleted file mode 100644
index 85fb3d761d..0000000000
--- a/overlay.d/05core/usr/lib/dracut/modules.d/99emergency-timeout/timeout.sh
+++ /dev/null
@@ -1,99 +0,0 @@
-# Before starting the emergency shell, prompt the user to press Enter.
-# If they don't, reboot the system.
-#
-# Assumes /bin/sh is bash.
-
-# _wait_for_journalctl_to_stop will block until either:
-# - no messages have appeared in journalctl for the past 5 seconds
-# - 15 seconds have elapsed
-_wait_for_journalctl_to_stop() {
-    local time_since_last_log=0
-
-    local time_started="$(date '+%s')"
-    local now="$(date '+%s')"
-
-    while [ ${time_since_last_log} -lt 5 -a $((now-time_started)) -lt 15 ]; do
-        sleep 1
-
-        local last_log_timestamp="$(journalctl -e -n 1 -q -o short-unix | cut -d '.' -f 1)"
-        local now="$(date '+%s')"
-
-        local time_since_last_log=$((now-last_log_timestamp))
-    done
-}
-
-_prompt_for_timeout() {
-    local timeout=300
-    local interval=15
-
-    if [[ -e /.emergency-shell-confirmed ]]; then
-        return
-    fi
-    failed=$(systemctl --failed --no-legend --plain | cut -f 1 -d ' ')
-    if [ -n "${failed}" ]; then
-        # Something failed, suppress kernel logs so that it's more likely
-        # the useful bits from the journal are available.
-        dmesg --console-off
-
-        # There's a couple straggler systemd messages. Wait until it's been 5
-        # seconds since something was written to the journal.
-        _wait_for_journalctl_to_stop
-
-        # Print Ignition logs
-        if echo ${failed} | grep -qFe 'ignition-'; then
-        cat <<EOF
-------
-Ignition has failed. Please ensure your config is valid. Note that only
-Ignition spec v3.0.0+ configs are accepted.
-
-A CLI validation tool to check this called ignition-validate can be
-downloaded from GitHub:
-    https://github.com/coreos/ignition/releases
-------
-
-EOF
-        fi
-        echo "Displaying logs from failed units: ${failed}"
-        for unit in ${failed}; do
-            # 10 lines should be enough for everyone
-            journalctl -b --no-pager --no-hostname -u ${unit} -n 10
-        done
-    fi
-
-    # Regularly prompt with time remaining.  This ensures the prompt doesn't
-    # get lost among kernel and systemd messages, and makes it clear what's
-    # going on if the user just connected a serial console.
-    while [[ $timeout > 0 ]]; do
-        local m=$(( $timeout / 60 ))
-        local s=$(( $timeout % 60 ))
-        local m_label="minutes"
-        if [[ $m = 1 ]]; then
-            m_label="minute"
-        fi
-
-        if [[ $s != 0 ]]; then
-            echo -n -e "Press Enter for emergency shell or wait $m $m_label $s seconds for reboot.      \r"
-        else
-            echo -n -e "Press Enter for emergency shell or wait $m $m_label for reboot.                 \r"
-        fi
-
-        local anything
-        if read -t $interval anything; then
-            > /.emergency-shell-confirmed
-            return
-        fi
-        timeout=$(( $timeout - $interval ))
-    done
-
-    echo -e "\nRebooting."
-    # This is not very nice, but since reboot.target likely conflicts with
-    # the existing goal target wrt the desired state of shutdown.target,
-    # there doesn't seem to be a better option.
-    systemctl reboot --force
-    exit 0
-}
-
-# If we're invoked from a dracut breakpoint rather than
-# dracut-emergency.service, we won't have a controlling terminal and stdio
-# won't be connected to it. Explicitly read/write /dev/console.
-_prompt_for_timeout < /dev/console > /dev/console
diff --git a/overlay.d/05core/usr/lib/dracut/modules.d/99journal-conf/00-journal-log-forwarding.conf b/overlay.d/05core/usr/lib/dracut/modules.d/99journal-conf/00-journal-log-forwarding.conf
index 091a114d5c..c7654af70e 100644
--- a/overlay.d/05core/usr/lib/dracut/modules.d/99journal-conf/00-journal-log-forwarding.conf
+++ b/overlay.d/05core/usr/lib/dracut/modules.d/99journal-conf/00-journal-log-forwarding.conf
@@ -1,12 +1,12 @@
 [Journal]
 # For now we are using kmsg for multiplexing output to
 # multiple console devices during early boot.
-# 
+#
 # We do not want to use kmsg in the future as there may be sensitive
 # ignition data that leaks to non-root users (by reading the kernel
 # ring buffer using `dmesg`). In the future we will rely on kernel
 # console multiplexing (link below) for this and will not use kmsg.
-# 
+#
 # https://github.com/coreos/fedora-coreos-tracker/issues/136
 ForwardToKMsg=yes
 MaxLevelKMsg=info
diff --git a/overlay.d/05core/usr/lib/systemd/system-generators/coreos-boot-mount-generator b/overlay.d/05core/usr/lib/systemd/system-generators/coreos-boot-mount-generator
index f790b74671..5724fdcb26 100755
--- a/overlay.d/05core/usr/lib/systemd/system-generators/coreos-boot-mount-generator
+++ b/overlay.d/05core/usr/lib/systemd/system-generators/coreos-boot-mount-generator
@@ -2,11 +2,7 @@
 export PATH="/usr/bin:/usr/sbin:${PATH}"
 set -euo pipefail
 
-# Generators don't have logging right now
-# https://github.com/systemd/systemd/issues/15638
-exec 1>/dev/kmsg; exec 2>&1
-
-UNIT_DIR="${1:-/tmp}"
+. /usr/lib/coreos/generator-lib.sh
 
 # Turn out if you boot with "root=..." $UNIT_DIR is not writable.
 [ -w "${UNIT_DIR}" ] || {
@@ -14,13 +10,6 @@ UNIT_DIR="${1:-/tmp}"
     exit 0
 }
 
-add_wants() {
-    local name="$1"; shift
-    local wants_dir="${UNIT_DIR}/local-fs.target.wants"
-    mkdir -p "${wants_dir}"
-    ln -sf "../${name}" "${wants_dir}/${name}"
-}
-
 # If there's already an /etc/fstab entries for /boot, then this is is a non-FCOS
 # system, likely RHCOS pre-4.3 (which still used Anaconda).  In that case, we
 # don't want to overwrite what the systemd-fstab-generator will do.
@@ -28,26 +17,32 @@ if findmnt --fstab /boot &>/dev/null; then
     exit 0
 fi
 
+# Don't create mount units for /boot on live systems.
+# ConditionPathExists won't work here because conditions don't affect
+# the dependency on the underlying device unit.
+if [ -f /run/ostree-live ]; then
+    exit 0
+fi
+
+add_wants() {
+    local name="$1"; shift
+    local wants_dir="${UNIT_DIR}/local-fs.target.wants"
+    mkdir -p "${wants_dir}"
+    ln -sf "../${name}" "${wants_dir}/${name}"
+}
+
 # Generate mount units that work with device mapper. The traditional
 # device unit (dev-disk-by\x2dlabel...) does not work since it is not the
 # device that systemd will fsck. This code ensures that if the label
 # is backed by a device-mapper target the dev-mapper.*.device is used.
 mk_mount() {
     local mount_pt="${1}"; shift
-    local label="${1}"; shift
-    local path="/dev/disk/by-label/${label}"
+    local path="${1}"; shift
+    local options="${1}"; shift
 
+    local devservice=$(systemd-escape -p ${path} --suffix=service)
     local unit_name=$(systemd-escape -p ${mount_pt} --suffix=mount)
 
-    eval $(udevadm info --query property --export "${path}")
-    device="$(systemd-escape ${path})"
-    if [ "${DM_NAME:-x}" != "x" ]; then
-        path="/dev/mapper/${DM_NAME}"
-        device="$(systemd-escape dev/mapper/${DM_NAME})"
-    fi
-    device="${device//-dev/dev}"
-    echo "coreos-boot-mount-generator: using ${device} for ${label} mount to ${mount_pt}"
-
     cat > "${UNIT_DIR}/${unit_name}" <<EOF
 # Automatically created by coreos-boot-mount-generator
 [Unit]
@@ -55,35 +50,66 @@ Description=CoreOS Dynamic Mount for ${mount_pt}
 Documentation=https://github.com/coreos/fedora-coreos-config
 
 Before=local-fs.target
-Requires=systemd-fsck@${device}.service
-After=systemd-fsck@${device}.service
+Requires=systemd-fsck@${devservice}
+After=systemd-fsck@${devservice}
 
 [Mount]
 What=${path}
 Where=${mount_pt}
+Options=${options}
 EOF
-
     add_wants "${unit_name}"
 }
 
+# Copied from
+# https://github.com/dracutdevs/dracut/blob/9491e599282d0d6bb12063eddbd192c0d2ce8acf/modules.d/99base/dracut-lib.sh#L586
+# rather than sourcing it.
+label_uuid_to_dev() {
+    local _dev
+    _dev="${1#block:}"
+    case "$_dev" in
+        LABEL=*)
+            echo "/dev/disk/by-label/$(echo "${_dev#LABEL=}" | sed 's,/,\\x2f,g;s, ,\\x20,g')"
+            ;;
+        PARTLABEL=*)
+            echo "/dev/disk/by-partlabel/$(echo "${_dev#PARTLABEL=}" | sed 's,/,\\x2f,g;s, ,\\x20,g')"
+            ;;
+        UUID=*)
+            echo "/dev/disk/by-uuid/$(echo "${_dev#UUID=}" | tr "[:upper:]" "[:lower:]")"
+            ;;
+        PARTUUID=*)
+            echo "/dev/disk/by-partuuid/$(echo "${_dev#PARTUUID=}" | tr "[:upper:]" "[:lower:]")"
+            ;;
+    esac
+}
 
-# Don't create mount units for /boot or /boot/efi on live systems.
-# ConditionPathExists won't work here because conditions don't affect
-# the dependency on the underlying device unit.
-if [ ! -f /run/ostree-live ]; then
-    mk_mount /boot boot
-
-    # Only mount the EFI System Partition on machines where it exists,
-    # which are 1) machines actually booted through EFI, and 2) x86_64
-    # when booted through BIOS.
-    if [ "$(uname -m)" = "x86_64" -o -d /sys/firmware/efi ]; then
-        mk_mount /boot/efi EFI-SYSTEM
-        # In the general case the ESP might have per-machine or private
-        # data on it.  Let's not make it world readable on general
-        # principle.
-        # https://github.com/coreos/fedora-coreos-tracker/issues/640
-        cat >>${UNIT_DIR}/boot-efi.mount << EOF
-Options=umask=0077
-EOF
-    fi
+# If the root device is multipath, hook up /boot to use that too,
+# based on our custom udev rules in 90-coreos-device-mapper.rules
+# that creates "label found on mpath" links.
+# Otherwise, use the usual by-label symlink.
+# See discussion in https://github.com/coreos/fedora-coreos-config/pull/1022
+bootdev=/dev/disk/by-label/boot
+bootkarg=$(karg boot)
+mpath=$(karg rd.multipath)
+if [ -n "${mpath}" ] && [ "${mpath}" != 0 ]; then
+    bootdev=/dev/disk/by-label/dm-mpath-boot
+# Newer nodes inject boot=UUID=..., but we support a larger subset of the dracut/fips API
+elif [ -n "${bootkarg}" ]; then
+    # Adapted from https://github.com/dracutdevs/dracut/blob/9491e599282d0d6bb12063eddbd192c0d2ce8acf/modules.d/01fips/fips.sh#L17
+    case "$bootkarg" in
+        LABEL=* | UUID=* | PARTUUID=* | PARTLABEL=*)
+            bootdev="$(label_uuid_to_dev "$bootkarg")";;
+        /dev/*) bootdev=$bootkarg;;
+        *) echo "Unknown boot karg '${bootkarg}'; falling back to ${bootdev}";;
+    esac
+# This is used for the first boot only
+elif [ -f /run/coreos/bootfs_uuid ]; then
+    bootdev=/dev/disk/by-uuid/$(cat /run/coreos/bootfs_uuid)
 fi
+
+# We mount read-only by default mostly to protect
+# against accidental damage.  Only a few things
+# owned by CoreOS should be touching /boot or the ESP.
+# Use nodev,nosuid because some hardening guides want
+# that even though it's of minimal value.
+mk_mount /boot "${bootdev}" ro,nodev,nosuid
diff --git a/overlay.d/05core/usr/lib/systemd/system-generators/coreos-liveiso-autologin-generator b/overlay.d/05core/usr/lib/systemd/system-generators/coreos-liveiso-autologin-generator
index aa8e9f4ffd..987f888960 100755
--- a/overlay.d/05core/usr/lib/systemd/system-generators/coreos-liveiso-autologin-generator
+++ b/overlay.d/05core/usr/lib/systemd/system-generators/coreos-liveiso-autologin-generator
@@ -2,23 +2,7 @@
 
 set -euo pipefail
 
-# Generators don't have logging right now
-# https://github.com/systemd/systemd/issues/15638
-exec 1>/dev/kmsg; exec 2>&1
-
-UNIT_DIR="${1:-/tmp}"
-
-have_karg() {
-    local arg="$1"
-    local cmdline=( $(</proc/cmdline) )
-    local i
-    for i in "${cmdline[@]}"; do
-        if [[ "$i" =~ "$arg=" ]]; then
-            return 0
-        fi
-    done
-    return 1
-}
+. /usr/lib/coreos/generator-lib.sh
 
 write_dropin() {
     local service="$1"; shift
@@ -36,17 +20,6 @@ ExecStart=-/sbin/agetty --autologin core -o '-p -f core' ${args} %I \$TERM
 EOF
 }
 
-quiet_kernel_console_messages() {
-    cat <<'EOF' > /etc/sysctl.d/20-coreos-autologin-kernel-printk.conf
-# Raise console message logging level from DEBUG (7) to WARNING (4)
-# so that kernel debug message don't get interspersed on the console
-# that
-# may frustrate a user trying to interactively do an install with
-# nmtui and coreos-installer.
-kernel.printk=4
-EOF
-}
-
 write_interactive_live_motd() {
     # Write motd to a tmp file and not directly to /etc/motd because
     # SELinux denies write from init_t to etc_t
@@ -101,9 +74,7 @@ fi
 # If the user supplied an Ignition config, they have the ability to enable
 # autologin themselves.  Don't automatically render them insecure, since
 # they might be running in production and booting via e.g. IPMI.
-# See https://github.com/coreos/ignition/pull/958 for the MESSAGE_ID source.
-ign_usercfg_msg=$(journalctl -q MESSAGE_ID=57124006b5c94805b77ce473e92a8aeb IGNITION_CONFIG_TYPE=user)
-if [ -n "${ign_usercfg_msg}" ]; then
+if jq -e .userConfigProvided /etc/.ignition-result.json &>/dev/null; then
     exit 0
 fi
 
@@ -111,13 +82,6 @@ write_dropin "getty@.service"        "--noclear"
 # Also autologin on serial console if someone enables that
 write_dropin "serial-getty@.service" "--keep-baud 115200,38400,9600"
 
-# When the installer runs a lot of things happen on the system (audit
-# messages from running via sudo, re-reading partition table messages,
-# mounting filesystem messages, etc.). Quieting the verbosity of the
-# kernel console will help us keep our sanity.
-quiet_kernel_console_messages
-
-
 # Write an motd that will let the user know about the live environment
 # and what is possible.
 write_interactive_live_motd
diff --git a/overlay.d/05core/usr/lib/systemd/system-generators/coreos-sulogin-force-generator b/overlay.d/05core/usr/lib/systemd/system-generators/coreos-sulogin-force-generator
new file mode 100755
index 0000000000..9edb7a88eb
--- /dev/null
+++ b/overlay.d/05core/usr/lib/systemd/system-generators/coreos-sulogin-force-generator
@@ -0,0 +1,66 @@
+#!/usr/bin/bash
+
+# This systemd.generator(7) detects if rescue or emergency targets were
+# requested from the kernel cmdline; if so, it overrides the respective
+# target to set force sulogin, allowing use of rescue/emergency targets
+# on systems with locked root password (as is Fedora default).
+#
+# This does NOT bypass locked root password on a fsck failure, but WILL
+# bypass when rescue/emergency targets are chosen from kernel cmdline.
+# Since this requires console/grub access, it is assumed to be at least
+# as secure as a user reset of the root password using grub to modify
+# the kernel cmdline with init=/bin/bash .
+#
+# NOTE: the SYSTEMD_SULOGIN_FORCE method used here does not bypass any
+# assigned password; root password is only bypassed when locked/unset.
+
+export PATH="/usr/bin:/usr/sbin:${PATH}"
+if [ -n "$1" ]; then
+    # If invoked with arguments (not testing) log to kmsg
+    # https://github.com/systemd/systemd/issues/15638
+    exec 1>/dev/kmsg; exec 2>&1
+fi
+
+# If invoked with no arguments (for testing) write to /tmp
+UNIT_DIR="${1:-/tmp}"
+
+set -euo pipefail
+
+have_some_karg() {
+    local args=("$@")
+    IFS=" " read -r -a cmdline <<< "$(</proc/cmdline)"
+    local i
+    for i in "${cmdline[@]}"; do
+        for a in "${args[@]}"; do
+        if [[ "$i" == "$a" ]]; then
+            return 0
+        fi
+        done
+    done
+    return 1
+}
+
+write_dropin() {
+    local service="$1"
+
+    local out_dir="${UNIT_DIR}/${service}.service.d"
+    mkdir -p "${out_dir}"
+
+    # /tmp isn't r/w yet, and the shell needs to cache the here-document
+    TMPDIR=/run
+    cat > "${out_dir}/sulogin-force.conf" <<EOF
+# Automatically created by coreos-sulogin-force-generator
+[Service]
+Environment=SYSTEMD_SULOGIN_FORCE=1
+EOF
+    echo "$(basename ${0}): set SYSTEMD_SULOGIN_FORCE=1 for ${service}.service"
+}
+
+# Match kernel command line targets for systemd(1) rescue/emergency
+# Ignores 'rd.' prefixed targets since they enter the dracut ramdisk
+# environment which does not interact with installed system root user.
+if have_some_karg 'systemd.unit=rescue.target' rescue single s S 1; then
+    write_dropin rescue
+elif have_some_karg 'systemd.unit=emergency.target' emergency '-b' ; then
+    write_dropin emergency
+fi
diff --git a/overlay.d/05core/usr/lib/systemd/system-preset/40-coreos-systemd.preset b/overlay.d/05core/usr/lib/systemd/system-preset/40-coreos-systemd.preset
new file mode 100644
index 0000000000..a242ebaaa1
--- /dev/null
+++ b/overlay.d/05core/usr/lib/systemd/system-preset/40-coreos-systemd.preset
@@ -0,0 +1,16 @@
+# This file contains overrides for systemd services that are
+# enabled by default, but conflict with things we ship.
+
+# We don't have swap by default, and systemd-oomd hard requires it.
+disable systemd-oomd.service
+
+# Disable systemd-firstboot because it conflicts with Ignition.
+# In most cases this is handled via the remove-from-packages
+# bits in the manifest (ignition-and-ostree.yaml), but
+# we want to support overlaying builds of systemd from git.
+disable systemd-firstboot.service
+
+# This hasn't been tested with ostree/rpm-ostree and heavily overlaps
+# with the latter.  Preemptively disable the service; it will hopefully
+# be subpackaged though for Fedora.
+disable systemd-sysext.service
diff --git a/overlay.d/05core/usr/lib/systemd/system-preset/40-coreos.preset b/overlay.d/05core/usr/lib/systemd/system-preset/40-coreos.preset
index 183366eb50..f4577d8fe9 100644
--- a/overlay.d/05core/usr/lib/systemd/system-preset/40-coreos.preset
+++ b/overlay.d/05core/usr/lib/systemd/system-preset/40-coreos.preset
@@ -1,16 +1,19 @@
 # Presets here that eventually should live in the generic fedora presets
-enable coreos-growpart.service
+
 # console-login-helper-messages - https://github.com/coreos/console-login-helper-messages
-enable console-login-helper-messages-issuegen.service
-enable console-login-helper-messages-motdgen.service
-enable console-login-helper-messages-issuegen.path
-enable console-login-helper-messages-motdgen.path
 enable console-login-helper-messages-gensnippet-os-release.service
 enable console-login-helper-messages-gensnippet-ssh-keys.service
 # CA certs (probably to add to base fedora eventually)
 enable coreos-update-ca-trust.service
-# This one is from https://github.com/coreos/ignition-dracut
-enable ignition-firstboot-complete.service
+# Set kernel console log level
+enable coreos-printk-quiet.service
+# https://github.com/coreos/ignition/issues/1125
+enable coreos-ignition-firstboot-complete.service
+# Delete Ignition config from provider on platforms where it's possible
+# https://github.com/coreos/ignition/pull/1350
+enable ignition-delete-config.service
+# Delete Ignition config from provider when upgrading existing nodes
+enable coreos-ignition-delete-config.service
 # Boot checkin services for cloud providers.
 enable afterburn-checkin.service
 enable afterburn-firstboot-checkin.service
@@ -26,3 +29,6 @@ enable bootupd.socket
 # The event for the attached device comes as a diag event.
 # Ideally it should have been added as part of base Fedora - but since it was arch specific, it was not added: https://bugzilla.redhat.com/show_bug.cgi?id=1433859
 enable rtas_errd.service
+enable clevis-luks-askpass.path
+# Provide status information about the Ignition run
+enable coreos-ignition-write-issues.service
diff --git a/overlay.d/05core/usr/lib/systemd/system/coreos-ignition-delete-config.service b/overlay.d/05core/usr/lib/systemd/system/coreos-ignition-delete-config.service
new file mode 100644
index 0000000000..cdcea6200a
--- /dev/null
+++ b/overlay.d/05core/usr/lib/systemd/system/coreos-ignition-delete-config.service
@@ -0,0 +1,28 @@
+# Can be removed from FCOS in Fedora 37 or after the next barrier release,
+# whichever comes first.  Can be removed from RHCOS in the first release
+# after every node is guaranteed to have booted at least once with 4.11 or
+# higher.
+
+[Unit]
+Description=CoreOS Delete Ignition Config From Hypervisor
+Documentation=https://coreos.github.io/ignition/
+
+ConditionKernelCommandLine=|ignition.platform.id=virtualbox
+ConditionKernelCommandLine=|ignition.platform.id=vmware
+ConditionPathExists=!/var/lib/coreos-ignition-delete-config.stamp
+# Hack: if the user masked ignition-delete-config.service, we shouldn't run
+# either.
+ConditionPathIsSymbolicLink=!/etc/systemd/system/ignition-delete-config.service
+
+# We check a stamp file written by ignition-delete-config.service.  That
+# service runs Before=sysinit.target, on which we have a default dependency,
+# so this is really just documentation.
+After=ignition-delete-config.service
+
+[Service]
+Type=oneshot
+ExecStart=/usr/libexec/coreos-ignition-delete-config
+RemainAfterExit=yes
+
+[Install]
+WantedBy=multi-user.target
diff --git a/overlay.d/05core/usr/lib/systemd/system/coreos-ignition-firstboot-complete.service b/overlay.d/05core/usr/lib/systemd/system/coreos-ignition-firstboot-complete.service
new file mode 100644
index 0000000000..42adf1e6b0
--- /dev/null
+++ b/overlay.d/05core/usr/lib/systemd/system/coreos-ignition-firstboot-complete.service
@@ -0,0 +1,21 @@
+[Unit]
+Description=CoreOS Mark Ignition Boot Complete
+Documentation=https://docs.fedoraproject.org/en-US/fedora-coreos/
+ConditionKernelCommandLine=ignition.firstboot
+ConditionPathExists=!/run/ostree-live
+RequiresMountsFor=/boot
+
+[Service]
+Type=oneshot
+RemainAfterExit=yes
+# The MountFlags=slave is so we remount /boot temporarily writable;
+# see https://github.com/ostreedev/ostree/issues/1265 for the bigger picture.
+# This option creates a new mount namespace; from the point of view of
+# everything else, /boot stays readonly. We only have a transient writable mount
+# for the lifetime of the unit.
+MountFlags=slave
+ExecStart=/usr/libexec/coreos-ignition-firstboot-complete
+
+[Install]
+# Part of basic.target so this happens early on in firstboot
+WantedBy=basic.target
diff --git a/overlay.d/05core/usr/lib/systemd/system/coreos-ignition-write-issues.service b/overlay.d/05core/usr/lib/systemd/system/coreos-ignition-write-issues.service
new file mode 100644
index 0000000000..a6268b85ac
--- /dev/null
+++ b/overlay.d/05core/usr/lib/systemd/system/coreos-ignition-write-issues.service
@@ -0,0 +1,16 @@
+# This service writes issue files describing status
+# information about the Ignition run, which includes
+# Ignition warnings and information if no Ignition
+# config is provided.
+[Unit]
+Description=Create Ignition Status Issue Files
+Before=systemd-user-sessions.service
+ConditionPathExists=/etc/.ignition-result.json
+
+[Service]
+Type=oneshot
+ExecStart=/usr/libexec/coreos-ignition-write-issues
+RemainAfterExit=yes
+
+[Install]
+WantedBy=multi-user.target
diff --git a/overlay.d/05core/usr/lib/systemd/system/coreos-liveiso-success.service b/overlay.d/05core/usr/lib/systemd/system/coreos-liveiso-success.service
index a3db8673a6..d148d12cb2 100644
--- a/overlay.d/05core/usr/lib/systemd/system/coreos-liveiso-success.service
+++ b/overlay.d/05core/usr/lib/systemd/system/coreos-liveiso-success.service
@@ -15,15 +15,12 @@ ConditionPathExists=/dev/virtio-ports/coreos.liveiso-success
 
 [Service]
 Type=simple
-# https://stackoverflow.com/questions/44358723/systemd-unit-file-problems-with-tr
-IgnoreSIGPIPE=false
-# See https://cgit.freedesktop.org/systemd/systemd/plain/src/systemd/sd-messages.h for the MESSAGE_ID source.
-# The logic here is that we're doing a streaming journalctl query (-f to follow)
-# and the `| head` bit will cause the pipeline to wait until at least one line is
-# emitted, which will happen when a user login starts.  We then just write a static
-# message to the virtio channel, which https://github.com/coreos/coreos-assembler/pull/1330
-# knows how to read.
-ExecStart=/bin/sh -c 'journalctl -b -q -f --no-tail -o cat -u systemd-logind.service MESSAGE_ID=8d45620c1a4348dbb17410da57c60c66 | head -1; echo coreos-liveiso-success > /dev/virtio-ports/coreos.liveiso-success'
+# Wait for a user session to start, then write a static message to the
+# virtio channel, which https://github.com/coreos/coreos-assembler/pull/1330
+# knows how to read.  We previously did "journalctl -f ... | head -1" here,
+# but RHEL 8 has systemd 239, which has
+# https://github.com/systemd/systemd/issues/9374.
+ExecStart=/bin/sh -c 'while [ -z "$(loginctl list-sessions --no-legend)" ]; do sleep 1; done; echo coreos-liveiso-success > /dev/virtio-ports/coreos.liveiso-success'
 
 [Install]
 WantedBy=multi-user.target
diff --git a/overlay.d/05core/usr/lib/systemd/system/coreos-printk-quiet.service b/overlay.d/05core/usr/lib/systemd/system/coreos-printk-quiet.service
new file mode 100644
index 0000000000..d93a32ad28
--- /dev/null
+++ b/overlay.d/05core/usr/lib/systemd/system/coreos-printk-quiet.service
@@ -0,0 +1,27 @@
+[Unit]
+Description=CoreOS: Set printk To Level 4 (warn)
+Documentation=https://github.com/coreos/fedora-coreos-tracker/issues/1244
+# We can run right after `/proc` being mounted at least
+DefaultDependencies=no
+# We run as early as possible; the only dependency we have really
+# is the implicit After=systemd-journald.socket injected by the
+# default of our stdout writing to the journal.
+Conflicts=shutdown.target
+Before=sysinit.target shutdown.target
+# We want this service to read what we wrote
+Before=systemd-sysctl.service
+# Relatedly, we don't want to override an explicitly specified kernel argument
+ConditionKernelCommandLine=!debug
+ConditionKernelCommandLine=!quiet
+ConditionKernelCommandLine=!loglevel
+
+[Service]
+Type=oneshot
+RemainAfterExit=yes
+# We need to make /run/sysctl.d if it doesn't exist and also
+# ensure it has a SELinux label that works for systemd-sysctl.service.
+# Then we just generate a sysctl file which is read by systemd-sysctl.service.
+ExecStart=/bin/bash -euo pipefail -c 'mkdir -p /run/sysctl.d && chcon --reference=/etc/sysctl.d /run/sysctl.d && echo "kernel.printk = 4" > /run/sysctl.d/01-coreos-printk.conf'
+
+[Install]
+WantedBy=sysinit.target
diff --git a/overlay.d/05core/usr/lib/systemd/system/emergency.service.d/coreos-sulogin-force.conf b/overlay.d/05core/usr/lib/systemd/system/emergency.service.d/coreos-sulogin-force.conf
deleted file mode 100644
index 390f72723f..0000000000
--- a/overlay.d/05core/usr/lib/systemd/system/emergency.service.d/coreos-sulogin-force.conf
+++ /dev/null
@@ -1,7 +0,0 @@
-# https://github.com/coreos/coreos-installer/commit/15a79263d0bd5d72056a6080f6687dc10cba2dda
-# https://github.com/systemd/systemd/pull/10397
-# We want things like `systemd.unit=emergency.target` and `single` on the 
-# kernel command line to just work even with our locked root account.
-# This file is used as an override for both emergency.target and rescue.target.
-[Service]
-Environment=SYSTEMD_SULOGIN_FORCE=1
diff --git a/overlay.d/05core/usr/lib/systemd/system/ignition-delete-config.service.d/10-flag-file.conf b/overlay.d/05core/usr/lib/systemd/system/ignition-delete-config.service.d/10-flag-file.conf
new file mode 100644
index 0000000000..e501d74dab
--- /dev/null
+++ b/overlay.d/05core/usr/lib/systemd/system/ignition-delete-config.service.d/10-flag-file.conf
@@ -0,0 +1,7 @@
+# Create a flag file to notify coreos-ignition-delete-config.service that
+# we've run, and put it in /run because /var isn't mounted yet.
+# coreos-ignition-delete-config.service will then avoid trying to delete
+# the config again, and will create a persistent stamp file in /var/lib.
+
+[Service]
+ExecStart=/bin/touch /run/coreos-ignition-delete-config.stamp
diff --git a/overlay.d/05core/usr/lib/systemd/system/rescue.service.d/coreos-sulogin-force.conf b/overlay.d/05core/usr/lib/systemd/system/rescue.service.d/coreos-sulogin-force.conf
deleted file mode 120000
index a8a1f7adb7..0000000000
--- a/overlay.d/05core/usr/lib/systemd/system/rescue.service.d/coreos-sulogin-force.conf
+++ /dev/null
@@ -1 +0,0 @@
-../emergency.service.d/coreos-sulogin-force.conf
\ No newline at end of file
diff --git a/overlay.d/05core/usr/lib/systemd/system/systemd-backlight@.service.d/45-after-ostree-remount.conf b/overlay.d/05core/usr/lib/systemd/system/systemd-backlight@.service.d/45-after-ostree-remount.conf
new file mode 100644
index 0000000000..fc1c8218a4
--- /dev/null
+++ b/overlay.d/05core/usr/lib/systemd/system/systemd-backlight@.service.d/45-after-ostree-remount.conf
@@ -0,0 +1,4 @@
+# Temporary fix for https://github.com/coreos/fedora-coreos-tracker/issues/975
+# until https://github.com/ostreedev/ostree/issues/2115 is fixed.
+[Unit]
+After=ostree-remount.service
diff --git a/overlay.d/05core/usr/lib/systemd/system/systemd-firstboot.service.d/fcos-disable.conf b/overlay.d/05core/usr/lib/systemd/system/systemd-firstboot.service.d/fcos-disable.conf
new file mode 100644
index 0000000000..fc7f00518b
--- /dev/null
+++ b/overlay.d/05core/usr/lib/systemd/system/systemd-firstboot.service.d/fcos-disable.conf
@@ -0,0 +1,5 @@
+# See the comment in 40-coreos-systemd.preset; we're
+# keeping this even stronger disable override for now,
+# but it may not really be necessary.
+[Unit]
+ConditionPathExists=/run/nosuchfile
diff --git a/overlay.d/05core/usr/lib/tmpfiles.d/root-bash.conf b/overlay.d/05core/usr/lib/tmpfiles.d/root-bash.conf
new file mode 100644
index 0000000000..d59075ba29
--- /dev/null
+++ b/overlay.d/05core/usr/lib/tmpfiles.d/root-bash.conf
@@ -0,0 +1,4 @@
+# Use C! according to https://github.com/coreos/fedora-coreos-config/pull/137/files#diff-4c7caeddac3f43acdf59b41ffa4e3533bb846d2e3bfa31f885a65e9598a2c4a1R1-R2
+C!	/var/roothome/.bashrc	-	-	-	-	/usr/etc/skel/.bashrc
+C!	/var/roothome/.bash_profile	-	-	-	-	/usr/etc/skel/.bash_profile
+C!	/var/roothome/.bash_logout	-	-	-	-	/usr/etc/skel/.bash_logout
diff --git a/overlay.d/05core/usr/lib/udev/rules.d/65-gce-disk-naming.rules b/overlay.d/05core/usr/lib/udev/rules.d/65-gce-disk-naming.rules
deleted file mode 100644
index e19c1c5b91..0000000000
--- a/overlay.d/05core/usr/lib/udev/rules.d/65-gce-disk-naming.rules
+++ /dev/null
@@ -1,38 +0,0 @@
-# Copyright 2016 Google Inc. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-#     distributed under the License is distributed on an "AS IS" BASIS,
-#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-# Name the attached disks as the specified by deviceName.
-
-ACTION!="add|change", GOTO="gce_disk_naming_end"
-SUBSYSTEM!="block", GOTO="gce_disk_naming_end"
-
-# SCSI naming
-KERNEL=="sd*|vd*", ENV{ID_VENDOR}=="Google", IMPORT{program}="scsi_id --export --whitelisted -d $tempnode"
-
-# NVME naming
-KERNEL=="nvme0n1*", ENV{ID_VENDOR}=="Google", ENV{ID_SERIAL_SHORT}="local-nvme-ssd-0"
-KERNEL=="nvme0n2*", ENV{ID_VENDOR}=="Google", ENV{ID_SERIAL_SHORT}="local-nvme-ssd-1"
-KERNEL=="nvme0n3*", ENV{ID_VENDOR}=="Google", ENV{ID_SERIAL_SHORT}="local-nvme-ssd-2"
-KERNEL=="nvme0n4*", ENV{ID_VENDOR}=="Google", ENV{ID_SERIAL_SHORT}="local-nvme-ssd-3"
-KERNEL=="nvme0n5*", ENV{ID_VENDOR}=="Google", ENV{ID_SERIAL_SHORT}="local-nvme-ssd-4"
-KERNEL=="nvme0n6*", ENV{ID_VENDOR}=="Google", ENV{ID_SERIAL_SHORT}="local-nvme-ssd-5"
-KERNEL=="nvme0n7*", ENV{ID_VENDOR}=="Google", ENV{ID_SERIAL_SHORT}="local-nvme-ssd-6"
-KERNEL=="nvme0n8*", ENV{ID_VENDOR}=="Google", ENV{ID_SERIAL_SHORT}="local-nvme-ssd-7"
-KERNEL=="nvme*", ENV{ID_VENDOR}=="Google", ENV{ID_SERIAL}="Google_EphemeralDisk_$env{ID_SERIAL_SHORT}"
-
-# Symlinks
-KERNEL=="sd*|vd*|nvme*", ENV{DEVTYPE}=="disk", ENV{ID_VENDOR}=="Google", SYMLINK+="disk/by-id/google-$env{ID_SERIAL_SHORT}"
-KERNEL=="sd*|vd*|nvme*", ENV{DEVTYPE}=="partition", ENV{ID_VENDOR}=="Google", SYMLINK+="disk/by-id/google-$env{ID_SERIAL_SHORT}-part%n"
-
-LABEL="gce_disk_naming_end"
diff --git a/overlay.d/05core/usr/lib/udev/rules.d/66-azure-storage.rules b/overlay.d/05core/usr/lib/udev/rules.d/66-azure-storage.rules
deleted file mode 100644
index 5fb3693031..0000000000
--- a/overlay.d/05core/usr/lib/udev/rules.d/66-azure-storage.rules
+++ /dev/null
@@ -1,28 +0,0 @@
-ACTION=="add|change", SUBSYSTEM=="block", ENV{ID_VENDOR}=="Msft", ENV{ID_MODEL}=="Virtual_Disk", GOTO="azure_disk"
-GOTO="azure_end"
-
-LABEL="azure_disk"
-# Root has a GUID of 0000 as the second value
-# The resource/resource has GUID of 0001 as the second value
-ATTRS{device_id}=="?00000000-0000-*", ENV{fabric_name}="root", GOTO="azure_names"
-ATTRS{device_id}=="?00000000-0001-*", ENV{fabric_name}="resource", GOTO="azure_names"
-ATTRS{device_id}=="?00000001-0001-*", ENV{fabric_name}="BEK", GOTO="azure_names"
-# Wellknown SCSI controllers
-ATTRS{device_id}=="{f8b3781a-1e82-4818-a1c3-63d806ec15bb}", ENV{fabric_scsi_controller}="scsi0", GOTO="azure_datadisk"
-ATTRS{device_id}=="{f8b3781b-1e82-4818-a1c3-63d806ec15bb}", ENV{fabric_scsi_controller}="scsi1", GOTO="azure_datadisk"
-ATTRS{device_id}=="{f8b3781c-1e82-4818-a1c3-63d806ec15bb}", ENV{fabric_scsi_controller}="scsi2", GOTO="azure_datadisk"
-ATTRS{device_id}=="{f8b3781d-1e82-4818-a1c3-63d806ec15bb}", ENV{fabric_scsi_controller}="scsi3", GOTO="azure_datadisk"
-GOTO="azure_end"
-
-# Retrieve LUN number for datadisks
-LABEL="azure_datadisk"
-ENV{DEVTYPE}=="partition", PROGRAM="/bin/sh -c 'readlink /sys/class/block/%k/../device|cut -d: -f4'", ENV{fabric_name}="$env{fabric_scsi_controller}/lun$result", GOTO="azure_names"
-PROGRAM="/bin/sh -c 'readlink /sys/class/block/%k/device|cut -d: -f4'", ENV{fabric_name}="$env{fabric_scsi_controller}/lun$result", GOTO="azure_names"
-GOTO="azure_end"
-
-# Create the symlinks
-LABEL="azure_names"
-ENV{DEVTYPE}=="disk", SYMLINK+="disk/azure/$env{fabric_name}"
-ENV{DEVTYPE}=="partition", SYMLINK+="disk/azure/$env{fabric_name}-part%n"
-
-LABEL="azure_end"
diff --git a/overlay.d/05core/usr/lib/udev/rules.d/90-coreos-device-mapper.rules b/overlay.d/05core/usr/lib/udev/rules.d/90-coreos-device-mapper.rules
index b908be71e9..385f262437 100644
--- a/overlay.d/05core/usr/lib/udev/rules.d/90-coreos-device-mapper.rules
+++ b/overlay.d/05core/usr/lib/udev/rules.d/90-coreos-device-mapper.rules
@@ -1,24 +1,26 @@
+# CoreOS-specific symlinks for dm-multipath filesystem labels,
+# used for `label=boot` and `label=root`.
+
 ACTION=="remove", GOTO="dm_label_end"
 SUBSYSTEM!="block", GOTO="dm_label_end"
 KERNEL!="dm-*", GOTO="dm_label_end"
 
 # Ensure that the device mapper target is active
-# And the required attributes exist.
-ENV{DM_ACTIVATION}!="1", GOTO="dm_label_end"
-ENV{ID_FS_LABEL_ENC}!="?*", GOTO="dm_label_end"
-ENV{ID_FS_UUID_ENC}!="?*", GOTO="dm_label_end"
+ENV{DM_SUSPENDED}=="1", GOTO="dm_label_end"
 
 # Only act on filesystems. This should prevent layered devices
 # such as Raid on Multipath devices from appearing.
 ENV{ID_FS_USAGE}!="filesystem", GOTO="dm_label_end"
 
+# And if the filesystem doesn't have a label+uuid, we're done.
+ENV{ID_FS_LABEL_ENC}!="?*", GOTO="dm_label_end"
+ENV{ID_FS_UUID_ENC}!="?*", GOTO="dm_label_end"
+
 # Setup up Multipath labels and UUID's. Match on DM_UUID which
 # is stable regardless of whether friendly names are used or not.
 # 66-kpartx.rules use DM_UUID to match for linear mappings on multipath
 # targets.
 ENV{DM_UUID}=="*mpath*" \
-  , ENV{DM_SUSPENDED}=="Active" \
-  , ENV{DM_TABLES_LOADED}=="Live" \
   , SYMLINK+="disk/by-label/dm-mpath-$env{ID_FS_LABEL_ENC}" \
   , SYMLINK+="disk/by-uuid/dm-mpath-$env{ID_FS_UUID_ENC}"
 
diff --git a/overlay.d/05core/usr/lib/udev/rules.d/99-azure-product-uuid.rules b/overlay.d/05core/usr/lib/udev/rules.d/99-azure-product-uuid.rules
deleted file mode 100644
index a5af9b1f48..0000000000
--- a/overlay.d/05core/usr/lib/udev/rules.d/99-azure-product-uuid.rules
+++ /dev/null
@@ -1,9 +0,0 @@
-SUBSYSTEM!="dmi", GOTO="product_uuid-exit"
-ATTR{sys_vendor}!="Microsoft Corporation", GOTO="product_uuid-exit"
-ATTR{product_name}!="Virtual Machine", GOTO="product_uuid-exit"
-TEST!="/sys/devices/virtual/dmi/id/product_uuid", GOTO="product_uuid-exit"
-
-RUN+="/bin/chmod 0444 /sys/devices/virtual/dmi/id/product_uuid"
-
-LABEL="product_uuid-exit"
-
diff --git a/overlay.d/05core/usr/libexec/coreos-ignition-delete-config b/overlay.d/05core/usr/libexec/coreos-ignition-delete-config
new file mode 100755
index 0000000000..69d32b709e
--- /dev/null
+++ b/overlay.d/05core/usr/libexec/coreos-ignition-delete-config
@@ -0,0 +1,23 @@
+#!/bin/bash
+
+set -euo pipefail
+
+IFS=" " read -r -a cmdline <<< "$(</proc/cmdline)"
+cmdline_arg() {
+    local name="$1" value
+    for arg in "${cmdline[@]}"; do
+        if [[ "${arg%%=*}" == "${name}" ]]; then
+            value="${arg#*=}"
+        fi
+    done
+    echo "${value}"
+}
+
+# Avoid running again if ignition-delete-config.service has run, but still
+# create our own stamp file now that /var is mounted.
+if [ ! -e /run/coreos-ignition-delete-config.stamp ]; then
+    PLATFORM_ID=$(cmdline_arg ignition.platform.id)
+    /usr/libexec/ignition-rmcfg --platform=${PLATFORM_ID}
+fi
+
+touch /var/lib/coreos-ignition-delete-config.stamp
diff --git a/overlay.d/05core/usr/libexec/coreos-ignition-firstboot-complete b/overlay.d/05core/usr/libexec/coreos-ignition-firstboot-complete
new file mode 100755
index 0000000000..e1963f08d8
--- /dev/null
+++ b/overlay.d/05core/usr/libexec/coreos-ignition-firstboot-complete
@@ -0,0 +1,19 @@
+#!/bin/bash
+set -euo pipefail
+
+mount -o remount,rw /boot
+
+# We're done provisioning. Remove the whole /boot/ignition directory if present,
+# which may include a baked Ignition config. See
+# https://github.com/coreos/fedora-coreos-tracker/issues/889.
+rm -rf /boot/ignition
+
+# Regarding the lack of `-f` for rm ; we should have only run if GRUB detected
+# this file. Fail if we are unable to remove it, rather than risking rerunning
+# Ignition at next boot.
+rm /boot/ignition.firstboot
+
+# rdcore zipl checks for /boot/ignition.firstboot
+if [[ $(uname -m) = s390x ]]; then
+    /usr/lib/dracut/modules.d/50rdcore/rdcore zipl --boot-mount=/boot
+fi
diff --git a/overlay.d/05core/usr/libexec/coreos-ignition-write-issues b/overlay.d/05core/usr/libexec/coreos-ignition-write-issues
new file mode 100755
index 0000000000..958ec31370
--- /dev/null
+++ b/overlay.d/05core/usr/libexec/coreos-ignition-write-issues
@@ -0,0 +1,71 @@
+#!/usr/bin/bash
+set -euo pipefail
+
+IGNITION_RESULT=/etc/.ignition-result.json
+
+WARN='\033[0;33m' # yellow
+RESET='\033[0m' # reset
+
+mkdir -p /run/issue.d
+touch /run/issue.d/30_coreos_ignition_provisioning.issue
+
+d=$(date --date "$(jq -r .provisioningDate "${IGNITION_RESULT}")" +"%Y/%m/%d %H:%M:%S %Z")
+ignitionBoot=$(jq -r .provisioningBootID "${IGNITION_RESULT}")
+if [ $(cat /proc/sys/kernel/random/boot_id) = "${ignitionBoot}" ]; then
+    echo "Ignition: ran on ${d} (this boot)" \
+        > /run/issue.d/30_coreos_ignition_provisioning.issue
+
+    # checking for /run/ostree-live as the live system with persistent storage can run Ignition more than once
+    if ! test -f /run/ostree-live && jq -e .previousReport.provisioningDate "${IGNITION_RESULT}" &>/dev/null; then
+        prevdate=$(date --date "$(jq -r .previousReport.provisioningDate "${IGNITION_RESULT}")" +"%Y/%m/%d %H:%M:%S %Z")
+        cat << EOF > /etc/issue.d/30_coreos_ignition_run_more_than_once.issue
+${WARN}
+############################################################################
+WARNING: Ignition previously ran on ${prevdate}. Unexpected
+behavior may occur. Ignition is not designed to run more than once per system.
+############################################################################
+${RESET}
+EOF
+    fi
+    # In Ignition, we've two config validation checks, the one after
+    # fetching a config and the second after merging configs. Sometimes,
+    # a warning goes away after merging, however, it's possible that a
+    # warning appears in case merging creates a contradiction between
+    # two fields. So this workflow eventually sends duplicate warnings
+    # in journal entries. Hence, we need to avoid displaying duplicate
+    # Ignition warnings on the console.
+    # For e.g. In the journal entries, we might see the following logs:
+    #
+    # warning at $.systemd.units.0.contents, line 1 col 997: unit "echo@.service" is enabled, but has no install section so enable does nothing
+    # warning at $.systemd.units.0.contents: unit "echo@.service" is enabled, but has no install section so enable does nothing
+    #
+    # In order to normalize these logs, we'd need to get rid of the line
+    # and column numbers entirely using the sed command, and then use
+    # `sort -u` to remove duplicate content. After this, we'd see the
+    # following warning on the console:
+    #
+    # warning at $.systemd.units.0.contents: unit "echo@.service" is enabled, but has no install section so enable does nothing
+    #
+    # TODO: find a way to query journal entries recorded before the
+    # system switches to real root
+    journalctl -t ignition -o cat -p warning | sed -r 's/, line [0-9]+ col [0-9]+//g' | sort -u | while read line; do
+        echo -e "${WARN}Ignition: $line${RESET}" >> /etc/issue.d/30_coreos_ignition_warnings.issue
+    done
+else
+    nreboots=$(($(journalctl --list-boots | wc -l) - 1))
+    [ "${nreboots}" -eq 1 ] && boot="boot" || boot="boots"
+    echo "Ignition: ran on ${d} (at least $nreboots $boot ago)" \
+        > /run/issue.d/30_coreos_ignition_provisioning.issue
+fi
+
+if jq -e .userConfigProvided "${IGNITION_RESULT}" &>/dev/null; then
+    echo "Ignition: user-provided config was applied" \
+        >> /run/issue.d/30_coreos_ignition_provisioning.issue
+else
+    echo -e "${WARN}Ignition: no config provided by user${RESET}" \
+        >> /run/issue.d/30_coreos_ignition_provisioning.issue
+fi
+
+# Our makeshift way of getting /run/issue.d semantics. See:
+# https://github.com/coreos/console-login-helper-messages/blob/e06fc88ae8fbcc3a422bc8c686f70c15aebb9d9a/usr/lib/console-login-helper-messages/issue.defs#L8-L17
+ln -sf /run/issue.d/30_coreos_ignition_provisioning.issue /etc/issue.d/
diff --git a/overlay.d/08composefs/README.md b/overlay.d/08composefs/README.md
new file mode 100644
index 0000000000..383ebb17a1
--- /dev/null
+++ b/overlay.d/08composefs/README.md
@@ -0,0 +1 @@
+Enable composefs by default; more in https://ostreedev.github.io/ostree/composefs/
diff --git a/overlay.d/08composefs/usr/lib/ostree/prepare-root.conf b/overlay.d/08composefs/usr/lib/ostree/prepare-root.conf
new file mode 100644
index 0000000000..2faae22bc9
--- /dev/null
+++ b/overlay.d/08composefs/usr/lib/ostree/prepare-root.conf
@@ -0,0 +1,2 @@
+[composefs]
+enabled = true
diff --git a/overlay.d/08nouveau/statoverride b/overlay.d/08nouveau/statoverride
new file mode 100644
index 0000000000..27a95affe2
--- /dev/null
+++ b/overlay.d/08nouveau/statoverride
@@ -0,0 +1,2 @@
+# Config file for overriding permission bits on overlay files/dirs
+# Format: =<file mode in decimal> <absolute path to a file or directory>
diff --git a/overlay.d/09misc/statoverride b/overlay.d/09misc/statoverride
new file mode 100644
index 0000000000..27a95affe2
--- /dev/null
+++ b/overlay.d/09misc/statoverride
@@ -0,0 +1,2 @@
+# Config file for overriding permission bits on overlay files/dirs
+# Format: =<file mode in decimal> <absolute path to a file or directory>
diff --git a/overlay.d/14NetworkManager-plugins/usr/lib/NetworkManager/conf.d/10-disable-default-plugins.conf b/overlay.d/14NetworkManager-plugins/usr/lib/NetworkManager/conf.d/10-disable-default-plugins.conf
deleted file mode 100644
index 3182f67027..0000000000
--- a/overlay.d/14NetworkManager-plugins/usr/lib/NetworkManager/conf.d/10-disable-default-plugins.conf
+++ /dev/null
@@ -1,9 +0,0 @@
-# Stop NetworkManager from trying to load the ifcfg-rh plugin by default,
-# which we don't ship.  This actually disables all default plugins, of which
-# ifcfg-rh is currently the only one.  
-#
-# Note that we must do this for now because `-=` syntax doesn't work
-# with compiled-in defaults. Proposed upstream fix:
-# https://gitlab.freedesktop.org/NetworkManager/NetworkManager/-/merge_requests/491
-[main]
-plugins=
diff --git a/overlay.d/15fcos/statoverride b/overlay.d/15fcos/statoverride
new file mode 100644
index 0000000000..27a95affe2
--- /dev/null
+++ b/overlay.d/15fcos/statoverride
@@ -0,0 +1,2 @@
+# Config file for overriding permission bits on overlay files/dirs
+# Format: =<file mode in decimal> <absolute path to a file or directory>
diff --git a/overlay.d/15fcos/usr/lib/dracut/modules.d/50ignition-conf-fcos/20-aws-nm-cloud-setup.ign b/overlay.d/15fcos/usr/lib/dracut/modules.d/50ignition-conf-fcos/20-aws-nm-cloud-setup.ign
new file mode 100644
index 0000000000..0d39b1686c
--- /dev/null
+++ b/overlay.d/15fcos/usr/lib/dracut/modules.d/50ignition-conf-fcos/20-aws-nm-cloud-setup.ign
@@ -0,0 +1,16 @@
+{
+  "ignition": {
+    "version": "3.0.0"
+  },
+  "storage": {
+    "files": [
+      {
+        "path": "/etc/systemd/system/nm-cloud-setup.service.d/env-aws.conf",
+        "contents": {
+          "source": "data:,%5BService%5D%0AEnvironment%3DNM_CLOUD_SETUP_EC2%3Dyes%0A"
+        },
+        "mode": 420
+      }
+    ]
+  }
+}
diff --git a/overlay.d/15fcos/usr/lib/dracut/modules.d/50ignition-conf-fcos/20-azure-nm-cloud-setup.ign b/overlay.d/15fcos/usr/lib/dracut/modules.d/50ignition-conf-fcos/20-azure-nm-cloud-setup.ign
new file mode 100644
index 0000000000..ed2a5c5ac8
--- /dev/null
+++ b/overlay.d/15fcos/usr/lib/dracut/modules.d/50ignition-conf-fcos/20-azure-nm-cloud-setup.ign
@@ -0,0 +1,16 @@
+{
+  "ignition": {
+    "version": "3.0.0"
+  },
+  "storage": {
+    "files": [
+      {
+        "path": "/etc/systemd/system/nm-cloud-setup.service.d/env-azure.conf",
+        "contents": {
+          "source": "data:,%5BService%5D%0AEnvironment%3DNM_CLOUD_SETUP_AZURE%3Dyes%0A"
+        },
+        "mode": 420
+      }
+    ]
+  }
+}
diff --git a/overlay.d/15fcos/usr/lib/dracut/modules.d/50ignition-conf-fcos/20-gcp-nm-cloud-setup.ign b/overlay.d/15fcos/usr/lib/dracut/modules.d/50ignition-conf-fcos/20-gcp-nm-cloud-setup.ign
new file mode 100644
index 0000000000..22966dd36b
--- /dev/null
+++ b/overlay.d/15fcos/usr/lib/dracut/modules.d/50ignition-conf-fcos/20-gcp-nm-cloud-setup.ign
@@ -0,0 +1,16 @@
+{
+  "ignition": {
+    "version": "3.0.0"
+  },
+  "storage": {
+    "files": [
+      {
+        "path": "/etc/systemd/system/nm-cloud-setup.service.d/env-gcp.conf",
+        "contents": {
+          "source": "data:,%5BService%5D%0AEnvironment%3DNM_CLOUD_SETUP_GCP%3Dyes%0A"
+        },
+        "mode": 420
+      }
+    ]
+  }
+}
diff --git a/overlay.d/15fcos/usr/lib/dracut/modules.d/50ignition-conf-fcos/module-setup.sh b/overlay.d/15fcos/usr/lib/dracut/modules.d/50ignition-conf-fcos/module-setup.sh
index 88463c4b58..8e9f9d923d 100755
--- a/overlay.d/15fcos/usr/lib/dracut/modules.d/50ignition-conf-fcos/module-setup.sh
+++ b/overlay.d/15fcos/usr/lib/dracut/modules.d/50ignition-conf-fcos/module-setup.sh
@@ -8,6 +8,24 @@ depends() {
 
 install() {
     mkdir -p "$initdir/usr/lib/ignition/base.d"
+    mkdir -p "$initdir/usr/lib/ignition/base.platform.d"
+
+    # Common entries
     inst "$moddir/30-afterburn-sshkeys-core.ign" \
         "/usr/lib/ignition/base.d/30-afterburn-sshkeys-core.ign"
+
+    # Platform specific: aws
+    mkdir -p "$initdir/usr/lib/ignition/base.platform.d/aws"
+    inst "$moddir/20-aws-nm-cloud-setup.ign" \
+        "/usr/lib/ignition/base.platform.d/aws/20-aws-nm-cloud-setup.ign"
+
+    # Platform specific: azure
+    mkdir -p "$initdir/usr/lib/ignition/base.platform.d/azure"
+    inst "$moddir/20-azure-nm-cloud-setup.ign" \
+        "/usr/lib/ignition/base.platform.d/azure/20-azure-nm-cloud-setup.ign"
+
+    # Platform specific: gcp
+    mkdir -p "$initdir/usr/lib/ignition/base.platform.d/gcp"
+    inst "$moddir/20-gcp-nm-cloud-setup.ign" \
+        "/usr/lib/ignition/base.platform.d/gcp/20-gcp-nm-cloud-setup.ign"
 }
diff --git a/overlay.d/15fcos/usr/lib/motd.d/tracker.motd b/overlay.d/15fcos/usr/lib/motd.d/tracker.motd
index 837cc57ccf..aff5d6b42f 100644
--- a/overlay.d/15fcos/usr/lib/motd.d/tracker.motd
+++ b/overlay.d/15fcos/usr/lib/motd.d/tracker.motd
@@ -1,3 +1,3 @@
 Tracker: https://github.com/coreos/fedora-coreos-tracker
-Discuss: https://discussion.fedoraproject.org/c/server/coreos/
+Discuss: https://discussion.fedoraproject.org/tag/coreos
 
diff --git a/overlay.d/15fcos/usr/lib/systemd/system-preset/45-fcos.preset b/overlay.d/15fcos/usr/lib/systemd/system-preset/45-fcos.preset
index f54a5805a3..95d8087d2b 100644
--- a/overlay.d/15fcos/usr/lib/systemd/system-preset/45-fcos.preset
+++ b/overlay.d/15fcos/usr/lib/systemd/system-preset/45-fcos.preset
@@ -1,10 +1,8 @@
-# User metrics client
-enable fedora-coreos-pinger.service
-# Provide information if no ignition is provided
-enable coreos-check-ignition-config.service
 enable coreos-check-ssh-keys.service
-# Monitor the stub-resolv.conf SELinux context
-enable coreos-reset-stub-resolv-selinux-context.path
-# Run once on startup to prevent some race conditions with
-# NetworkManager and systemd-resolved starting up.
-enable coreos-reset-stub-resolv-selinux-context.service
+# Check if cgroupsv1 is still being used
+enable coreos-check-cgroups-version.service
+# https://fedoraproject.org/wiki/Changes/EnableFwupdRefreshByDefault
+enable fwupd-refresh.timer
+# Check if wifi firmwares are missing when NetworkManager-wifi is installed
+# https://github.com/coreos/fedora-coreos-tracker/issues/1575
+enable coreos-check-wireless-firmwares.service
diff --git a/overlay.d/15fcos/usr/lib/systemd/system/coreos-check-cgroups-version.service b/overlay.d/15fcos/usr/lib/systemd/system/coreos-check-cgroups-version.service
new file mode 100644
index 0000000000..101392140b
--- /dev/null
+++ b/overlay.d/15fcos/usr/lib/systemd/system/coreos-check-cgroups-version.service
@@ -0,0 +1,12 @@
+# This service is used for printing a message if
+# cgroups v1 is still being used
+[Unit]
+Description=Check if cgroupsv1 Is Still Being Used
+ConditionControlGroupController=v1
+Before=systemd-user-sessions.service
+[Service]
+Type=oneshot
+ExecStart=/usr/libexec/coreos-check-cgroups-version
+RemainAfterExit=yes
+[Install]
+WantedBy=multi-user.target
diff --git a/overlay.d/15fcos/usr/lib/systemd/system/coreos-check-ignition-config.service b/overlay.d/15fcos/usr/lib/systemd/system/coreos-check-ignition-config.service
deleted file mode 100644
index 3b73e3d176..0000000000
--- a/overlay.d/15fcos/usr/lib/systemd/system/coreos-check-ignition-config.service
+++ /dev/null
@@ -1,16 +0,0 @@
-# This service is used for printing a message if 
-# no Ignition config is provided.
-[Unit]
-Description=Check if Ignition config is provided
-# Only perform checks on the first (Ignition) boot as they are
-# mostly useful only on that boot. This ensures systems started
-# before Ignition/Afterburn started logging structured data don't
-# get misleading messages. Also handles the case where the journal
-# gets rotated and no longer has the structured log messages.
-ConditionKernelCommandLine=ignition.firstboot
-[Service]
-Type=oneshot
-ExecStart=/usr/libexec/coreos-check-ignition-config.sh
-RemainAfterExit=yes
-[Install]
-WantedBy=multi-user.target
diff --git a/overlay.d/15fcos/usr/lib/systemd/system/coreos-check-ssh-keys.service b/overlay.d/15fcos/usr/lib/systemd/system/coreos-check-ssh-keys.service
index 3536f52900..0e24ae8aaa 100644
--- a/overlay.d/15fcos/usr/lib/systemd/system/coreos-check-ssh-keys.service
+++ b/overlay.d/15fcos/usr/lib/systemd/system/coreos-check-ssh-keys.service
@@ -1,7 +1,7 @@
-# This service is used for printing a message if no ssh keys were added 
+# This service is used for printing a message if no ssh keys were added
 # by Ignition/Afterburn
 [Unit]
-Description=Check that ssh-keys are added by Afterburn/Ignition
+Description=Check That SSH Keys Are Added by Afterburn/Ignition
 # It allows other units to synchronize around any instance
 # of `afterburn-sshkeys@` and not just the `core` user.
 # See https://github.com/coreos/afterburn/pull/481
@@ -12,11 +12,13 @@ After=afterburn-sshkeys.target
 # get misleading messages. Also handles the case where the journal
 # gets rotated and no longer has the structured log messages.
 ConditionKernelCommandLine=ignition.firstboot
+# Run before user sessions to avoid reloading agetty
+Before=systemd-user-sessions.service
 
 [Service]
 Type=oneshot
 ProtectHome=read-only
-ExecStart=/usr/libexec/coreos-check-ssh-keys.sh
+ExecStart=/usr/libexec/coreos-check-ssh-keys
 RemainAfterExit=yes
 [Install]
 WantedBy=multi-user.target
diff --git a/overlay.d/15fcos/usr/lib/systemd/system/coreos-check-wireless-firmwares.service b/overlay.d/15fcos/usr/lib/systemd/system/coreos-check-wireless-firmwares.service
new file mode 100644
index 0000000000..360bf2ed85
--- /dev/null
+++ b/overlay.d/15fcos/usr/lib/systemd/system/coreos-check-wireless-firmwares.service
@@ -0,0 +1,11 @@
+# This service is used for printing a message if
+# some wireless firmwares are missing
+[Unit]
+Description=Check For Wireless Firmware Packages
+Before=systemd-user-sessions.service
+[Service]
+Type=oneshot
+ExecStart=/usr/libexec/coreos-check-wireless-firmwares
+RemainAfterExit=yes
+[Install]
+WantedBy=multi-user.target
diff --git a/overlay.d/15fcos/usr/lib/systemd/system/coreos-reset-stub-resolv-selinux-context.path b/overlay.d/15fcos/usr/lib/systemd/system/coreos-reset-stub-resolv-selinux-context.path
deleted file mode 100644
index 05bd20fd3f..0000000000
--- a/overlay.d/15fcos/usr/lib/systemd/system/coreos-reset-stub-resolv-selinux-context.path
+++ /dev/null
@@ -1,11 +0,0 @@
-[Unit]
-Description=Monitor /run/systemd/resolve/stub-resolv.conf to fixup SELinux context.
-Documentation=https://github.com/systemd/systemd/pull/17976
-Before=systemd-resolved.service
-
-[Path]
-PathModified=/run/systemd/resolve/stub-resolv.conf
-Unit=coreos-reset-stub-resolv-selinux-context.service
-
-[Install]
-WantedBy=multi-user.target
diff --git a/overlay.d/15fcos/usr/lib/systemd/system/coreos-reset-stub-resolv-selinux-context.service b/overlay.d/15fcos/usr/lib/systemd/system/coreos-reset-stub-resolv-selinux-context.service
deleted file mode 100644
index dab08f9853..0000000000
--- a/overlay.d/15fcos/usr/lib/systemd/system/coreos-reset-stub-resolv-selinux-context.service
+++ /dev/null
@@ -1,18 +0,0 @@
-[Unit]
-Description=Fixup SELinux context on /run/systemd/resolve/stub-resolv.conf
-Documentation=https://github.com/systemd/systemd/pull/17976
-ConditionPathExists=/run/systemd/resolve/stub-resolv.conf
-# Run once on startup in addition to the path unit invocations
-# so that we can order ourselves before NetworkManager to prevent
-# at least a few race condition denials.
-After=systemd-resolved.service
-Before=NetworkManager.service
-
-[Service]
-Type=oneshot
-# This service can be started more than once. If the file changes.
-RemainAfterExit=no
-ExecStart=restorecon -v /run/systemd/resolve/stub-resolv.conf
-
-[Install]
-WantedBy=multi-user.target
diff --git a/overlay.d/15fcos/usr/lib/sysusers.d/00-coreos-nobody.conf b/overlay.d/15fcos/usr/lib/sysusers.d/00-coreos-nobody.conf
new file mode 100644
index 0000000000..86e5d56c39
--- /dev/null
+++ b/overlay.d/15fcos/usr/lib/sysusers.d/00-coreos-nobody.conf
@@ -0,0 +1,9 @@
+# Legacy IDs for 'nobody' user/group. This is a CoreOS mismatched entry
+# which will need to be migrated:
+# https://github.com/coreos/fedora-coreos-tracker/issues/1201
+
+# g nobody 65534
+# u nobody 65534:65534 "Kernel Overflow User" - -
+
+g nobody 99
+u nobody 99:99 "Kernel Overflow User" - -
diff --git a/overlay.d/15fcos/usr/lib/sysusers.d/00-coreos-static.conf b/overlay.d/15fcos/usr/lib/sysusers.d/00-coreos-static.conf
new file mode 100644
index 0000000000..9678fe6fdf
--- /dev/null
+++ b/overlay.d/15fcos/usr/lib/sysusers.d/00-coreos-static.conf
@@ -0,0 +1,32 @@
+# These are pinned users/groups whose static IDs are only used
+# this way on CoreOS nodes.
+
+g cgred 996
+g chrony 992
+g cockpit-ws 987
+g dockerroot 986
+g etcd 997
+g input 104
+g kube 994
+g nfsnobody 65534
+g polkitd 998
+g ssh_keys 999
+g sssd 993
+g sudo 16
+g systemd-bus-proxy 988
+g systemd-network 990
+g systemd-resolve 989
+g systemd-timesync 991
+
+u chrony            994:992     -                              /var/lib/chrony -
+u cockpit-ws        988:987     "User for cockpit-ws"          -               -
+u dockerroot        997:986     "Docker User"                  /var/lib/docker -
+u etcd              998:997     "etcd user"                    /var/lib/etcd   -
+u kube              996:994     "Kubernetes user"              -               -
+u nfsnobody         65534:65534 "Anonymous NFS User"           /var/lib/nfs    -
+u polkitd           999:998     "User for polkitd"             -               -
+u sssd              995:993     "User for sssd"                -               -
+u systemd-bus-proxy 989:988     "systemd Bus Proxy"            -               -
+u systemd-network   991:990     "systemd Network Management"   -               -
+u systemd-resolve   990:989     "systemd Resolver"             -               -
+u systemd-timesync  993:991     "systemd Time Synchronization" -               -
diff --git a/overlay.d/15fcos/usr/lib/sysusers.d/10-static-extra.conf b/overlay.d/15fcos/usr/lib/sysusers.d/10-static-extra.conf
new file mode 100644
index 0000000000..38f65ee6e4
--- /dev/null
+++ b/overlay.d/15fcos/usr/lib/sysusers.d/10-static-extra.conf
@@ -0,0 +1,24 @@
+# These are users/groups with static IDs which follow usual Fedora-wide
+# allocation. They are usually coming from relevant packages, but we also
+# pre-populate them on CoreOS.
+
+g avahi-autoipd 170
+g ceph 167
+g dbus 81
+g dip 40
+g rpc 32
+g rpcuser 29
+g sshd 74
+g systemd-journal 190
+g tcpdump 72
+g utempter 35
+g utmp 22
+
+u avahi-autoipd 170:170     "Avahi IPv4LL Stack"          /var/lib/avahi-autoipd -
+u ceph          167:167     "Ceph daemons"                /var/lib/ceph          -
+u dbus          81:81       "System Message Bus"          -                      -
+u nfsnobody     65534:65534 "Anonymous NFS User"          /var/lib/nfs           -
+u rpc           32:32       "Rpcbind Daemon"              /var/lib/rpcbind       -
+u rpcuser       29:29       "RPC Service User"            /var/lib/nfs           -
+u sshd          74:74       "Privilege-separated SSH"     /var/empty/sshd        -
+u tcpdump       72:72       -                             -                      -
diff --git a/overlay.d/15fcos/usr/libexec/coreos-check-cgroups-version b/overlay.d/15fcos/usr/libexec/coreos-check-cgroups-version
new file mode 100755
index 0000000000..64a9793e0d
--- /dev/null
+++ b/overlay.d/15fcos/usr/libexec/coreos-check-cgroups-version
@@ -0,0 +1,25 @@
+#!/usr/bin/bash
+# This script checks if the system is still using cgroups v1
+# and prints a message to the serial console.
+
+# Change the output color to yellow
+warn=$(echo -e '\033[0;33m')
+# No color
+nc=$(echo -e '\033[0m')
+
+motd_path=/run/motd.d/30_cgroupsv1_warning.motd
+
+cat << EOF > "${motd_path}"
+${warn}
+##########################################################################
+WARNING: This system is using cgroups v1. In Fedora 41 this system will
+no longer continue to boot. It is strongly recommended to migrate this
+system and your workloads to use cgroups v2. For instructions on how to
+adjust kernel arguments to use cgroups v2, see:
+https://docs.fedoraproject.org/en-US/fedora-coreos/kernel-args/
+
+To disable this warning, use:
+sudo systemctl disable coreos-check-cgroups-version.service
+##########################################################################
+${nc}
+EOF
diff --git a/overlay.d/15fcos/usr/libexec/coreos-check-ignition-config.sh b/overlay.d/15fcos/usr/libexec/coreos-check-ignition-config.sh
deleted file mode 100755
index 819a79b8d1..0000000000
--- a/overlay.d/15fcos/usr/libexec/coreos-check-ignition-config.sh
+++ /dev/null
@@ -1,26 +0,0 @@
-#!/usr/bin/bash
-# The logic for the message_id is handled in 
-# https://github.com/coreos/ignition/pull/958
-# In this script, we need to capture the journald 
-# log with the particular message_id and query using 
-#`jq` utility to check if a user config is provided.
-
-# Change the output color to yellow
-warn='\033[0;33m'
-# No color
-nc='\033[0m'
-
-# See https://github.com/coreos/ignition/pull/958 for the MESSAGE_ID source.
-# It will track the journal messages related to an Ignition config provided 
-# by the user.
-output=$(journalctl -o json-pretty MESSAGE_ID=57124006b5c94805b77ce473e92a8aeb | jq -s '.[] | select(.IGNITION_CONFIG_TYPE == "user")'| wc -l)
-
-if [[ $output -gt  0 ]];then
-    echo "Ignition: user provided config was applied" > /etc/issue.d/30_ignition_config_info.issue
-else
-    echo -e "${warn}Ignition: no config provided by user${nc}" > /etc/issue.d/30_ignition_config_info.issue
-fi
-
-# Ask all running agetty instances to reload and update their
-# displayed prompts in case this script was run before agetty.
-/usr/sbin/agetty --reload
diff --git a/overlay.d/15fcos/usr/libexec/coreos-check-ssh-keys.sh b/overlay.d/15fcos/usr/libexec/coreos-check-ssh-keys
similarity index 64%
rename from overlay.d/15fcos/usr/libexec/coreos-check-ssh-keys.sh
rename to overlay.d/15fcos/usr/libexec/coreos-check-ssh-keys
index f59d2c064c..fc8d4b90a0 100755
--- a/overlay.d/15fcos/usr/libexec/coreos-check-ssh-keys.sh
+++ b/overlay.d/15fcos/usr/libexec/coreos-check-ssh-keys
@@ -1,5 +1,5 @@
 #!/usr/bin/bash
-# This script will print a message in the serial console 
+# This script will print a message in the serial console
 # if no ssh keys were added by Ignition/Afterburn.
 main() {
     # Change the output color to yellow
@@ -7,19 +7,22 @@ main() {
     # No color
     nc='\033[0m'
 
-    # See https://github.com/coreos/ignition/pull/964 for the MESSAGE_ID 
-    # source. It will track the authorized-ssh-keys entries in journald 
-    # provided via Ignition.
+    # See https://github.com/coreos/ignition/pull/964 for the MESSAGE_ID
+    # source. It will track the authorized-ssh-keys entries in journald
+    # provided via Ignition. Limit journal output to the most recent boot
+    # so we don't get output from re-used /var/ partitions.
     ignitionusers=$(
-            journalctl -o json-pretty MESSAGE_ID=225067b87bbd4a0cb6ab151f82fa364b | \
+            journalctl -b 0 -o json-pretty MESSAGE_ID=225067b87bbd4a0cb6ab151f82fa364b | \
             jq -r '.MESSAGE' | \
             xargs -I{} echo "Ignition: {}")
 
-    # See https://github.com/coreos/afterburn/pull/397 for the MESSAGE_ID 
-    # source. It will track the authorized-ssh-keys entries in journald  
-    # provided via Afterburn.
+    # See https://github.com/coreos/afterburn/pull/397 for the MESSAGE_ID
+    # source. It will track the authorized-ssh-keys entries in journald
+    # provided via Afterburn.Limit journal output to the most recent boot
+    # so we don't get output from re-used /var/ partitions.
+
     afterburnusers=$(
-            journalctl -o json-pretty MESSAGE_ID=0f7d7a502f2d433caa1323440a6b4190 | \
+            journalctl -b 0 -o json-pretty MESSAGE_ID=0f7d7a502f2d433caa1323440a6b4190 | \
             jq -r '.MESSAGE' | \
             xargs -I{} echo "Afterburn: {}")
 
@@ -38,13 +41,9 @@ main() {
     if [ -n "$output" ]; then
         echo "$output" > /etc/issue.d/30_ssh_authorized_keys.issue
     else
-        echo -e "${warn}No ssh authorized keys provided by Ignition or Afterburn${nc}" \
+        echo -e "${warn}No SSH authorized keys provided by Ignition or Afterburn${nc}" \
             > /etc/issue.d/30_ssh_authorized_keys.issue
     fi
-
-    # Ask all running agetty instances to reload and update their
-    # displayed prompts in case this script was run before agetty.
-    /usr/sbin/agetty --reload
 }
 
 main
diff --git a/overlay.d/15fcos/usr/libexec/coreos-check-wireless-firmwares b/overlay.d/15fcos/usr/libexec/coreos-check-wireless-firmwares
new file mode 100755
index 0000000000..9f93e31353
--- /dev/null
+++ b/overlay.d/15fcos/usr/libexec/coreos-check-wireless-firmwares
@@ -0,0 +1,63 @@
+#!/usr/bin/bash
+# This script checks if
+# and will prints a message to the serial console
+# to warn the user about missing wifi firmware messages
+# and provide remediation steps
+# See https://github.com/coreos/fedora-coreos-tracker/issues/1575
+
+set -euo pipefail
+
+# List of wifi-firmwares
+# SOURCE: https://pagure.io/fedora-comps/blob/main/f/comps-f41.xml.in#_2700
+firmwares=(
+atheros-firmware
+b43-fwcutter
+b43-openfwwf
+brcmfmac-firmware
+iwlegacy-firmware
+iwlwifi-dvm-firmware
+iwlwifi-mvm-firmware
+libertas-firmware
+mt7xxx-firmware
+nxpwireless-firmware
+realtek-firmware
+tiwilink-firmware
+atmel-firmware
+bcm283x-firmware
+zd1211-firmware
+)
+# Get firmware names into `a|b|c|d` regex string
+regex=$(IFS='|'; echo "${firmwares[*]}")
+
+layered_packages="$(rpm-ostree status --json -b | jq -r '.deployments[0]."requested-packages"[]')"
+
+if grep -q "NetworkManager-wifi" <<< "$layered_packages"; then
+    if grep -qP $regex <<< "$layered_packages"; then
+      exit 0
+    fi
+else
+  exit 0
+fi
+
+# Change the output color to yellow
+warn=$(echo -e '\033[0;33m')
+# No color
+nc=$(echo -e '\033[0m')
+
+motd_path=/run/motd.d/30_wireless_firmwares_warning.motd
+
+cat << EOF > "${motd_path}"
+${warn}
+##########################################################################
+WARNING: NetworkManager-wifi is a requested layered package on this
+system, but no Wi-Fi drivers are requested. The Wi-Fi drivers will no
+longer be included by default in the future.
+
+More context and remediation steps are available in the following FAQ entry:
+https://docs.fedoraproject.org/en-US/fedora-coreos/faq/#wifi-fw
+
+To disable this warning, use:
+sudo systemctl disable coreos-check-wireless-firmwares.service
+##########################################################################
+${nc}
+EOF
diff --git a/overlay.d/16disable-zincati/etc/zincati/config.d/90-disable-on-non-production-stream.toml b/overlay.d/16disable-zincati/etc/zincati/config.d/90-disable-on-non-production-stream.toml
new file mode 100644
index 0000000000..795389841f
--- /dev/null
+++ b/overlay.d/16disable-zincati/etc/zincati/config.d/90-disable-on-non-production-stream.toml
@@ -0,0 +1,2 @@
+# https://github.com/coreos/fedora-coreos-tracker/issues/163
+updates.enabled = false
diff --git a/overlay.d/16disable-zincati/statoverride b/overlay.d/16disable-zincati/statoverride
new file mode 100644
index 0000000000..27a95affe2
--- /dev/null
+++ b/overlay.d/16disable-zincati/statoverride
@@ -0,0 +1,2 @@
+# Config file for overriding permission bits on overlay files/dirs
+# Format: =<file mode in decimal> <absolute path to a file or directory>
diff --git a/overlay.d/20platform-chrony/statoverride b/overlay.d/20platform-chrony/statoverride
new file mode 100644
index 0000000000..27a95affe2
--- /dev/null
+++ b/overlay.d/20platform-chrony/statoverride
@@ -0,0 +1,2 @@
+# Config file for overriding permission bits on overlay files/dirs
+# Format: =<file mode in decimal> <absolute path to a file or directory>
diff --git a/overlay.d/20platform-chrony/usr/lib/systemd/system-preset/20-fcos.preset b/overlay.d/20platform-chrony/usr/lib/systemd/system-preset/20-fcos.preset
new file mode 100644
index 0000000000..df54774a6a
--- /dev/null
+++ b/overlay.d/20platform-chrony/usr/lib/systemd/system-preset/20-fcos.preset
@@ -0,0 +1,2 @@
+# To inspect the platform and configure chrony
+enable coreos-platform-chrony-config.service
diff --git a/overlay.d/20platform-chrony/usr/lib/systemd/system/chronyd.service.d/platform-chrony.conf b/overlay.d/20platform-chrony/usr/lib/systemd/system/chronyd.service.d/platform-chrony.conf
new file mode 100644
index 0000000000..996df36b34
--- /dev/null
+++ b/overlay.d/20platform-chrony/usr/lib/systemd/system/chronyd.service.d/platform-chrony.conf
@@ -0,0 +1,4 @@
+# Override chrony configuration with the output of
+# coreos-platform-chrony-config.service.
+[Service]
+EnvironmentFile=-/run/coreos/sysconfig-chrony
diff --git a/overlay.d/20platform-chrony/usr/lib/systemd/system/coreos-platform-chrony-config.service b/overlay.d/20platform-chrony/usr/lib/systemd/system/coreos-platform-chrony-config.service
new file mode 100644
index 0000000000..53c208c58c
--- /dev/null
+++ b/overlay.d/20platform-chrony/usr/lib/systemd/system/coreos-platform-chrony-config.service
@@ -0,0 +1,20 @@
+[Unit]
+Description=CoreOS Configure Chrony Based On The Platform
+ConditionKernelCommandLine=|ignition.platform.id=azurestack
+ConditionKernelCommandLine=|ignition.platform.id=azure
+ConditionKernelCommandLine=|ignition.platform.id=aws
+# RHCOS 4.1/4.2 bootimages
+ConditionKernelCommandLine=|ignition.platform.id=ec2
+# RHCOS 4.1/4.2 bootimages
+ConditionKernelCommandLine=|ignition.platform.id=gce
+ConditionKernelCommandLine=|ignition.platform.id=gcp
+ConditionKernelCommandLine=|ignition.platform.id=qemu
+Before=NetworkManager.service
+Before=chronyd.service
+
+[Service]
+Type=oneshot
+ExecStart=/usr/libexec/coreos-platform-chrony-config
+RemainAfterExit=yes
+[Install]
+WantedBy=multi-user.target
diff --git a/overlay.d/20platform-chrony/usr/lib/systemd/system-generators/coreos-platform-chrony b/overlay.d/20platform-chrony/usr/libexec/coreos-platform-chrony-config
similarity index 54%
rename from overlay.d/20platform-chrony/usr/lib/systemd/system-generators/coreos-platform-chrony
rename to overlay.d/20platform-chrony/usr/libexec/coreos-platform-chrony-config
index 27511e730c..9d7076661c 100755
--- a/overlay.d/20platform-chrony/usr/lib/systemd/system-generators/coreos-platform-chrony
+++ b/overlay.d/20platform-chrony/usr/libexec/coreos-platform-chrony-config
@@ -3,7 +3,7 @@ set -euo pipefail
 # Configuring the timeserver for the platform is often handled
 # by pre-baking a config into a particular image for a platform, but
 # that doesn't work for us because we have a single update stream.  Hence
-# this generator dynamically inspects the platform and reconfigures chrony.
+# this service dynamically inspects the platform and reconfigures chrony.
 #
 # AWS: https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/set-time.html
 # Azure: https://docs.microsoft.com/en-us/azure/virtual-machines/linux/time-sync
@@ -11,20 +11,38 @@ set -euo pipefail
 #
 # Originally spawned from discussion in https://github.com/openshift/installer/pull/3513
 
-# Generators don't have logging right now
-# https://github.com/systemd/systemd/issues/15638
-exec 1>/dev/kmsg; exec 2>&1
+self=$(basename "$0")
 
-self=$(basename $0)
-confpath=/run/coreos-platform-chrony.conf
+# Exit early if chrony configuration has been changed from the image default
+if ! cmp {/usr,}/etc/chrony.conf >/dev/null; then
+    echo "$self: /etc/chrony.conf is modified; not changing the default"
+    exit 0
+fi
+if ! cmp {/usr,}/etc/sysconfig/chronyd >/dev/null; then
+    echo "$self: /etc/sysconfig/chronyd is modified; not changing the default"
+    exit 0
+fi
 
-# Yeah this isn't a completely accurate kernel argument parser but
-# we don't have one shared across shell services at the moment.
-platform="$(grep -Eo ' ignition.platform.id=[a-z]+' /proc/cmdline | cut -f 2 -d =)"
-case "${platform}" in
-    azure|aws|gcp) ;;  # OK, this is a platform we know how to support
-    *) exit 0 ;;
-esac
+IFS=" " read -r -a cmdline <<< "$(</proc/cmdline)"
+cmdline_arg() {
+    local name="$1" value
+    for arg in "${cmdline[@]}"; do
+        if [[ "${arg%%=*}" == "${name}" ]]; then
+            value="${arg#*=}"
+        fi
+    done
+    echo "${value}"
+}
+platform=$(cmdline_arg ignition.platform.id)
+
+# Exit early if the ptp_kvm module can't be loaded for the qemu platform
+if [[ ${platform} == "qemu" ]]; then
+    modprobe -q ptp_kvm || exit 0
+fi
+
+mkdir -p /run/coreos
+confpath=/run/coreos/platform-chrony.conf
+altenvfilepath=/run/coreos/sysconfig-chrony
 
 # If not set already (by host customization or this script), set
 # PEERNTP=no so that DHCP-provided NTP servers are not added to chrony.
@@ -33,24 +51,24 @@ esac
 # TODO: once https://bugzilla.redhat.com/show_bug.cgi?id=1828434 is
 # resolved, this won't be required.
 if [ ! -e /etc/sysconfig/network ] || ! grep -q "PEERNTP" /etc/sysconfig/network; then
-    cat <<EOF >> /etc/sysconfig/network
+    # Historically on QEMU, we haven't been disabling PEERNTP. Let's keep doing
+    # that even if we have ptp_kvm. chrony will just use the NTP servers as
+    # additional sources.
+    if [[ ${platform} != "qemu" ]]; then
+        cat <<EOF >> /etc/sysconfig/network
 # PEERNTP=no is automatically added by default when a platform-provided time
 # source is available, but this behavior may be overridden through an Ignition
 # config specifying PEERNTP=yes. See https://github.com/coreos/fedora-coreos-config/pull/412.
 PEERNTP=no
 EOF
+    fi
 fi
 
-if ! cmp {/usr,}/etc/chrony.conf >/dev/null; then
-    echo "$self: /etc/chrony.conf is modified; not changing the default"
-    exit 0
-fi
-
-(echo "# Generated by $self - do not edit directly" 
- sed -e s,'^makestep,#makestep,' -e s,'^pool,#pool,' < /etc/chrony.conf
+(echo "# Generated by $self - do not edit directly"
+ sed -e s,'^makestep,#makestep,' -e s,'^pool,#pool,' -e s,'^leapsectz,#leapsectz,' < /etc/chrony.conf
 cat <<EOF
 
-# Allow the system clock step on any clock update. 
+# Allow the system clock step on any clock update.
 # It will avoid the time resynchronization issue when VMs are resumed from suspend.
 # See https://bugzilla.redhat.com/show_bug.cgi?id=1780165 for more information.
 makestep 1.0 -1
@@ -58,32 +76,37 @@ makestep 1.0 -1
 EOF
 ) > "${confpath}"
 case "${platform}" in
-    azure) 
+    azure | azurestack)
+        # the /dev/ptp_hyperv symlink is created by:
+        # https://github.com/systemd/systemd/blob/e67a5c14f0345f5ac456cfa109324dd9e70114d3/rules.d/50-udev-default.rules.in#L106
         (echo '# See also https://docs.microsoft.com/en-us/azure/virtual-machines/linux/time-sync'
-         echo 'refclock PHC /dev/ptp0 poll 3 dpoll -2 offset 0'
+         echo 'refclock PHC /dev/ptp_hyperv poll 3 dpoll -2 offset 0'
+         echo 'leapsectz right/UTC'
         ) >> "${confpath}" ;;
-    aws)
+    aws | ec2)
         (echo '# See also https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/set-time.html'
          echo 'server 169.254.169.123 prefer iburst minpoll 4 maxpoll 4'
         ) >> "${confpath}" ;;
-    gcp)
+    gcp | gce)
         (echo '# See also https://cloud.google.com/compute/docs/instances/managing-instances#configure-ntp'
          echo '# and https://cloud.google.com/compute/docs/images/configuring-imported-images'
          echo 'server metadata.google.internal prefer iburst'
         ) >> "${confpath}" ;;
+    qemu)
+        sed -i s,'^#pool,pool,' "${confpath}"
+        (echo '# KVM virtual PHC'
+         echo 'refclock PHC /dev/ptp0 poll 2'
+        ) >> "${confpath}" ;;
     *) echo "should not be reached" 1>&2; exit 1 ;;
 esac
 # Policy doesn't allow chronyd to read run_t
 chcon --reference=/etc/chrony.conf "${confpath}"
 
-UNIT_DIR="${1:-/tmp}"
 
-unitconfpath="${UNIT_DIR}/chronyd.service.d/coreos-platform-chrony.conf"
-mkdir -p $(dirname "${unitconfpath}")
-cat >"${unitconfpath}" << EOF
-[Service]
-ExecStart=
-ExecStart=/usr/sbin/chronyd -f ${confpath} \$OPTIONS
-EOF
+# Read in the existing $OPTIONS variable setting from /etc/sysconfig/chronyd
+# and write out a new $OPTIONS variable (with specified new configuration path)
+# to /etc/sysconfig/chronyd
+source /etc/sysconfig/chronyd
+echo "OPTIONS='${OPTIONS} -f ${confpath}'" > ${altenvfilepath}
 
 echo "$self: Updated chrony to use ${platform} configuration ${confpath}"
diff --git a/overlay.d/25azure-udev-rules/statoverride b/overlay.d/25azure-udev-rules/statoverride
new file mode 100644
index 0000000000..27a95affe2
--- /dev/null
+++ b/overlay.d/25azure-udev-rules/statoverride
@@ -0,0 +1,2 @@
+# Config file for overriding permission bits on overlay files/dirs
+# Format: =<file mode in decimal> <absolute path to a file or directory>
diff --git a/overlay.d/25azure-udev-rules/usr/lib/dracut/modules.d/25azure-udev-rules/module-setup.sh b/overlay.d/25azure-udev-rules/usr/lib/dracut/modules.d/25azure-udev-rules/module-setup.sh
new file mode 100644
index 0000000000..1be75d0e3d
--- /dev/null
+++ b/overlay.d/25azure-udev-rules/usr/lib/dracut/modules.d/25azure-udev-rules/module-setup.sh
@@ -0,0 +1,17 @@
+#!/usr/bin/bash
+# Install 68-azure-sriov-nm-unmanaged.rules into the initramfs
+
+# called by dracut
+check() {
+    return 0
+}
+
+# called by dracut
+depends() {
+    return 0
+}
+
+# called by dracut
+install() {
+    inst_rules 68-azure-sriov-nm-unmanaged.rules
+}
diff --git a/overlay.d/05core/usr/lib/udev/rules.d/68-azure-sriov-nm-unmanaged.rules b/overlay.d/25azure-udev-rules/usr/lib/udev/rules.d/68-azure-sriov-nm-unmanaged.rules
similarity index 70%
rename from overlay.d/05core/usr/lib/udev/rules.d/68-azure-sriov-nm-unmanaged.rules
rename to overlay.d/25azure-udev-rules/usr/lib/udev/rules.d/68-azure-sriov-nm-unmanaged.rules
index 59cf73bb0a..e47ef35466 100644
--- a/overlay.d/05core/usr/lib/udev/rules.d/68-azure-sriov-nm-unmanaged.rules
+++ b/overlay.d/25azure-udev-rules/usr/lib/udev/rules.d/68-azure-sriov-nm-unmanaged.rules
@@ -1,4 +1,4 @@
 # Accelerated Networking on Azure exposes a new SRIOV interface to the VM.
 # This interface is transparently bonded to the synthetic interface,
 # so NetworkManager should just ignore any SRIOV interfaces.
-SUBSYSTEM=="net", DRIVERS=="hv_pci", ACTION=="add", ENV{NM_UNMANAGED}="1"
+SUBSYSTEM=="net", DRIVERS=="hv_pci", ACTION=="add|change|move", ENV{NM_UNMANAGED}="1"
diff --git a/overlay.d/30lvmdevices/etc/lvm/devices/system.devices b/overlay.d/30lvmdevices/etc/lvm/devices/system.devices
new file mode 100644
index 0000000000..94f55ea0d6
--- /dev/null
+++ b/overlay.d/30lvmdevices/etc/lvm/devices/system.devices
@@ -0,0 +1,8 @@
+# LVM uses devices listed in this file.
+#
+# This is an empty lvmdevices(8) file placed here by the CoreOS overlays.
+# The existence of the file prevents LVM from auto-activating any LVM devices
+# that aren't explicitly allowlisted by being added to this file. Any newly
+# added PV/VG devices that get created via pvcreate/vgreate will have an entry
+# added here automatically by the tools. For more information on this see
+# https://github.com/coreos/fedora-coreos-tracker/issues/1517
diff --git a/overlay.d/30lvmdevices/statoverride b/overlay.d/30lvmdevices/statoverride
new file mode 100644
index 0000000000..27a95affe2
--- /dev/null
+++ b/overlay.d/30lvmdevices/statoverride
@@ -0,0 +1,2 @@
+# Config file for overriding permission bits on overlay files/dirs
+# Format: =<file mode in decimal> <absolute path to a file or directory>
diff --git a/overlay.d/30lvmdevices/usr/lib/systemd/system-preset/45-coreos-populate-lvmdevices.preset b/overlay.d/30lvmdevices/usr/lib/systemd/system-preset/45-coreos-populate-lvmdevices.preset
new file mode 100644
index 0000000000..7a4fe5dfc8
--- /dev/null
+++ b/overlay.d/30lvmdevices/usr/lib/systemd/system-preset/45-coreos-populate-lvmdevices.preset
@@ -0,0 +1 @@
+enable coreos-populate-lvmdevices.service
diff --git a/overlay.d/30lvmdevices/usr/lib/systemd/system/coreos-populate-lvmdevices.service b/overlay.d/30lvmdevices/usr/lib/systemd/system/coreos-populate-lvmdevices.service
new file mode 100644
index 0000000000..8349d0253b
--- /dev/null
+++ b/overlay.d/30lvmdevices/usr/lib/systemd/system/coreos-populate-lvmdevices.service
@@ -0,0 +1,25 @@
+[Unit]
+Description=CoreOS Populate LVM Devices File
+Documentation=https://github.com/coreos/fedora-coreos-tracker/issues/1517
+# Only run this import once.
+ConditionPathExists=!/var/lib/coreos-populate-lvmdevices.stamp
+# Don't add default dependencies so we can run early enough to populate
+# the devices file before any LVM devices are used.
+DefaultDependencies=false
+# Since our ConditionPathExists file lives under /var/lib/ let's make
+# sure any filesystems/mounts that exist and are needed to access that
+# directory are set up.
+RequiresMountsFor=/var/lib
+# On OpenShift/Kubernetes we want to ensure we run before kubernetes
+# comes up where storage drivers may be initiated and present more LVM
+# devices to the system that we don't want to be considered.
+Before=kubelet.service
+
+[Service]
+Type=oneshot
+RemainAfterExit=yes
+ExecStart=/usr/libexec/coreos-populate-lvmdevices
+ExecStartPost=touch /var/lib/coreos-populate-lvmdevices.stamp
+
+[Install]
+WantedBy=default.target
diff --git a/overlay.d/30lvmdevices/usr/libexec/coreos-populate-lvmdevices b/overlay.d/30lvmdevices/usr/libexec/coreos-populate-lvmdevices
new file mode 100755
index 0000000000..d479fcc51d
--- /dev/null
+++ b/overlay.d/30lvmdevices/usr/libexec/coreos-populate-lvmdevices
@@ -0,0 +1,46 @@
+#!/bin/bash
+set -euo pipefail
+
+# This script will detect any LVM devices and add them to the lvmdevices
+# file, which will force this host to only consider those devices in
+# the future. This script should be run once to do the population and
+# should not need to be run again. See
+# https://github.com/coreos/fedora-coreos-tracker/issues/1517
+
+LVMDEVICESFILENAME="system.devices"
+LVMDEVICESFILE="/etc/lvm/devices/${LVMDEVICESFILENAME}"
+
+# If the devices file doesn't exist that is a bit odd because we
+# shipped it in the same update this migration script runs but let's
+# just bail out. Someone could have deleted the lvmdevices file and
+# then later accidentally run the migration script again.
+if [ ! -f $LVMDEVICESFILE ]; then
+    echo "$LVMDEVICESFILE does not exist. Exiting."
+    exit 0
+fi
+
+# If the file exists and the file is different than what was shipped
+# then we exit early. In this case the system likely already had an
+# lvmdevices file defined already.
+if ! diff -u "/usr/${LVMDEVICESFILE}" "${LVMDEVICESFILE}"; then
+    echo "Detected modified $LVMDEVICESFILE file. Exiting."
+    exit 0
+fi
+
+# Detect all existing PVs using `pvs` with a blank devicesfile
+# setting, which will un-limit the search.
+PVS=$(pvs --devicesfile="" --noheadings -o pv_name)
+
+if [ -z "$PVS" ]; then
+    echo "No LVM devices detected. Exiting."
+    exit 0
+fi
+
+echo "Populating lvmdevices file with detected devices."
+for pv in $(pvs --devicesfile="" --noheadings -o pv_name); do
+    echo "Adding ${pv} to lvmdevices file $LVMDEVICESFILE"
+    lvmdevices --journal output --adddev "$pv" --devicesfile "$LVMDEVICESFILENAME"
+done
+
+echo "Activating lvmdevices after devices file population"
+pvscan --cache --activate ay
diff --git a/overlay.d/40grub/usr/lib/bootupd/grub2-static/configs.d/40_coreos-ignition.cfg b/overlay.d/40grub/usr/lib/bootupd/grub2-static/configs.d/40_coreos-ignition.cfg
new file mode 100644
index 0000000000..31bda1dcec
--- /dev/null
+++ b/overlay.d/40grub/usr/lib/bootupd/grub2-static/configs.d/40_coreos-ignition.cfg
@@ -0,0 +1,18 @@
+# Remove soon when Ignition is providing this:
+# https://github.com/coreos/fedora-coreos-config/pull/2769#discussion_r1428152480
+#
+# Determine if this is a first boot and set the ${ignition_firstboot} variable
+# which is used in the kernel command line.
+set ignition_firstboot=""
+if [ -f "/ignition.firstboot" ]; then
+    # Default networking parameters to be used with ignition.
+    set ignition_network_kcmdline=''
+
+    # Source in the `ignition.firstboot` file which could override the
+    # above $ignition_network_kcmdline with static networking config.
+    # This override feature is also by coreos-installer to persist static
+    # networking config provided during install to the first boot of the machine.
+    source "/ignition.firstboot"
+
+    set ignition_firstboot="ignition.firstboot ${ignition_network_kcmdline}"
+fi
diff --git a/overlay.d/40grub/usr/lib/bootupd/grub2-static/configs.d/70_coreos-user.cfg b/overlay.d/40grub/usr/lib/bootupd/grub2-static/configs.d/70_coreos-user.cfg
new file mode 100644
index 0000000000..8c4460b127
--- /dev/null
+++ b/overlay.d/40grub/usr/lib/bootupd/grub2-static/configs.d/70_coreos-user.cfg
@@ -0,0 +1,6 @@
+
+# Import user defined configuration
+# tracker: https://github.com/coreos/fedora-coreos-tracker/issues/805
+if [ -f $prefix/user.cfg ]; then
+  source $prefix/user.cfg
+fi
diff --git a/overlay.d/README.md b/overlay.d/README.md
index 1784ca51b9..9597731fdf 100644
--- a/overlay.d/README.md
+++ b/overlay.d/README.md
@@ -1,8 +1,15 @@
+These overlay directories are automatically committed to the build OSTree repo
+by coreos-assembler. They are then explicitly included in our various manifest
+files via `ostree-layers` (this used to be done automatically, but that's no
+longer the case).
+
 05core
------
+------
 
 This overlay matches `fedora-coreos-base.yaml`; core Ignition+ostree bits.
 
+This overlay is shared with RHCOS/SCOS 9.
+
 08nouveau
 ---------
 
@@ -17,19 +24,6 @@ https://bugzilla.redhat.com/show_bug.cgi?id=1700056
 
 Warning about `/etc/sysconfig`.
 
-14NetworkManager-plugins
-------------------------
-
-Disables the Red Hat Linux legacy `ifcfg` format.
-
-20platform-chrony
------------------
-
-Add static chrony configuration for NTP servers provided on platforms
-such as `azure`, `aws`, `gcp`. The chrony config for these NTP servers
-should override other chrony configuration (e.g. DHCP-provided)
-configuration.
-
 15fcos
 ------
 
@@ -38,11 +32,53 @@ Things that are more closely "Fedora CoreOS":
 * disable password logins by default over SSH
 * enable SSH keys written by Ignition and Afterburn
 * branding (MOTD)
-* enable services by default (fedora-coreos-pinger)
+* enable FCOS-specific services by default
 * display warnings on the console if no ignition config was provided or no ssh
   key found.
 
+16disable-zincati
+-----------------
+
+Disable Zincati on non-production streams:
+https://github.com/coreos/fedora-coreos-tracker/issues/163
+
 20platform-chrony
 -----------------
 
-Platform aware timeserver setup for chrony daemon.
+Add static chrony configuration for NTP servers provided on platforms
+such as `azure`, `aws`, `gcp`. The chrony config for these NTP servers
+should override other chrony configuration (e.g. DHCP-provided)
+configuration.
+
+25azure-udev-rules
+-------------------
+
+Add udev rules for SRIOV networking on Azure. The udev rules are also
+needed in the initramfs [1] and are delivered here via a dracut
+module. This may be able to be removed once an upstream PR [2]
+merges, though we need to make sure the RPM [3] includes the dracut
+bits to include the rules in the initramfs too.
+
+[1] https://github.com/coreos/fedora-coreos-tracker/issues/1383
+[2] https://github.com/Azure/WALinuxAgent/pull/1622
+[3] https://src.fedoraproject.org/rpms/WALinuxAgent/pull-request/4
+
+30lvmdevices
+-------------------
+
+Populate an lvmdevices(8) file to limit LVM from autoactivating all
+devices it sees in a system. By default systems will get a "blank"
+configuration file with a comment in it explaining what it is used
+for. There is also a one-time "populate" service that will run and
+add any devices it sees into the devices file. This will serve to
+import existing devices on upgrading systems or new systems with
+pre-existing LVM devices attached. See the tracker issue [1] for more
+information.
+
+[1] https://github.com/coreos/fedora-coreos-tracker/issues/1517
+
+40grub
+------
+
+Add in static grub configs that will be leveraged by bootupd when
+managing bootloaders. See https://github.com/coreos/bootupd/pull/543
diff --git a/platforms.yaml b/platforms.yaml
new file mode 100644
index 0000000000..8e615549bf
--- /dev/null
+++ b/platforms.yaml
@@ -0,0 +1,177 @@
+# This file specifies image customizations that are specific to particular
+# architecture/platform pairs.  It is applied (indirectly, via
+# /boot/coreos/platforms.json) by create_disk.sh and gf-set-platform at
+# build time and coreos-installer at install time (if --platform is
+# specified).
+#
+# Currently this is used to configure the default console.  For any
+# arch/platform pairs not specified, GRUB and the kernel will apply their
+# own defaults.  Note that coreos-installer install --console will
+# completely override any GRUB commands specified here.
+#
+# s390x doesn't use GRUB and requires running zipl after updating kargs,
+# so it can't be added to this file without additional development work.
+#
+# All architectures, platforms, and fields are optional.
+aarch64:
+  aws:
+    # https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/connect-to-serial-console.html#sc-considerations
+    # It appears there's no screenshot support on ARM
+    grub_commands:
+      - serial --speed=115200 efi0
+      - terminal_input serial_efi0
+      - terminal_output serial_efi0
+    kernel_arguments:
+      - console=ttyS0,115200n8
+  azure:
+    # https://docs.microsoft.com/en-us/troubleshoot/azure/virtual-machines/serial-console-linux
+    # https://docs.microsoft.com/en-us/troubleshoot/azure/virtual-machines/boot-diagnostics
+    # Have GRUB write only to console, which gets copied to the serial port.
+    # If we use serial (serial efi0; terminal_output console serial_efi0)
+    # we get doubled output.
+    kernel_arguments:
+      - console=tty0
+      - console=ttyAMA0,115200n8
+  openstack:
+    # Graphical console primary, serial console available for logging
+    # https://docs.openstack.org/diskimage-builder/latest/elements/bootloader/README.html
+    # https://issues.redhat.com/browse/OCPBUGS-2926
+    grub_commands:
+      - serial --speed=115200
+      - terminal_input serial console
+      - terminal_output serial console
+    kernel_arguments:
+      - console=ttyAMA0,115200n8
+      - console=tty0
+  packet:
+    # https://metal.equinix.com/developers/docs/resilience-recovery/serial-over-ssh/#limitations
+    grub_commands:
+      - serial --speed=115200
+      - terminal_input serial
+      - terminal_output serial
+    kernel_arguments:
+      - console=ttyAMA0,115200
+  qemu:
+    # The kernel successfully autodetects a serial console, but we still
+    # want GRUB to use one
+    grub_commands:
+      - serial --speed=115200
+      - terminal_input serial console
+      - terminal_output serial console
+ppc64le:
+  openstack:
+    # Graphical console primary, serial console available for logging
+    # petitboot doesn't understand GRUB console commands
+    # https://docs.openstack.org/diskimage-builder/latest/elements/bootloader/README.html
+    # https://issues.redhat.com/browse/OCPBUGS-2926
+    kernel_arguments:
+      - console=hvc0
+      - console=tty0
+  qemu:
+    # petitboot doesn't understand GRUB console commands, but we need to
+    # pass console kargs
+    # https://github.com/coreos/coreos-assembler/pull/2400#discussion_r701412417
+    kernel_arguments:
+      - console=hvc0
+      - console=tty0
+x86_64:
+  aws:
+    # https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/connect-to-serial-console.html#sc-considerations
+    # https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_GetConsoleScreenshot.html
+    grub_commands:
+      - serial --speed=115200
+      - terminal_input serial console
+      - terminal_output serial console
+    kernel_arguments:
+      - console=tty0
+      - console=ttyS0,115200n8
+  azure:
+    # https://docs.microsoft.com/en-us/troubleshoot/azure/virtual-machines/serial-console-linux
+    # https://docs.microsoft.com/en-us/troubleshoot/azure/virtual-machines/boot-diagnostics
+    # Have GRUB write only to console, which gets copied to the serial port.
+    # If we use serial we get doubled output.
+    kernel_arguments:
+      - console=tty0
+      - console=ttyS0,115200n8
+  gcp:
+    # Four serial ports are available; we use the first one
+    # https://cloud.google.com/compute/docs/troubleshooting/troubleshooting-using-serial-console
+    # https://cloud.google.com/compute/docs/troubleshooting/capturing-vm-screenshots
+    grub_commands:
+      - serial --speed=115200
+      - terminal_input serial console
+      - terminal_output serial console
+    kernel_arguments:
+      - console=tty0
+      - console=ttyS0,115200n8
+  ibmcloud:
+    # Docs suggest 9600 bps, but that doesn't seem reasonable
+    # https://cloud.ibm.com/docs/vpc?topic=vpc-create-linux-custom-image#kernel-args
+    # https://cloud.ibm.com/docs/vpc?topic=vpc-vsi_is_connecting_console&interface=ui
+    grub_commands:
+      - serial --speed=115200
+      - terminal_input serial console
+      - terminal_output serial console
+    kernel_arguments:
+      - console=tty0
+      - console=ttyS0,115200n8
+  kubevirt:
+    # No official docs on this for kubevirt. Requested some:
+    # https://github.com/kubevirt/kubevirt/issues/9400
+    # The web UI displays the VNC console first, so go with
+    # graphical console primary, serial console secondary.
+    grub_commands:
+      - serial --speed=115200
+      - terminal_input serial console
+      - terminal_output serial console
+    kernel_arguments:
+      - console=ttyS0,115200n8
+      - console=tty0
+  openstack:
+    # Graphical console primary, serial console available for logging
+    # https://docs.openstack.org/diskimage-builder/latest/elements/bootloader/README.html
+    # https://issues.redhat.com/browse/OCPBUGS-2926
+    grub_commands:
+      - serial --speed=115200
+      - terminal_input serial console
+      - terminal_output serial console
+    kernel_arguments:
+      - console=ttyS0,115200n8
+      - console=tty0
+  packet:
+    # https://metal.equinix.com/developers/docs/resilience-recovery/serial-over-ssh/#limitations
+    grub_commands:
+      - serial --unit=1 --speed=115200
+      - terminal_input serial
+      - terminal_output serial
+    kernel_arguments:
+      - console=ttyS1,115200n8
+  qemu:
+    # https://github.com/coreos/fedora-coreos-tracker/issues/954
+    grub_commands:
+      - serial --speed=115200
+      - terminal_input serial console
+      - terminal_output serial console
+    kernel_arguments:
+      - console=tty0
+      - console=ttyS0,115200n8
+  virtualbox:
+    # Graphical console primary, serial console available for logging
+    # https://docs.fedoraproject.org/en-US/fedora-coreos/provisioning-virtualbox/#_troubleshooting_first_boot_problems
+    grub_commands:
+      - serial --speed=115200
+      - terminal_input serial console
+      - terminal_output serial console
+    kernel_arguments:
+      - console=ttyS0,115200n8
+      - console=tty0
+  vmware:
+    # Graphical console primary, serial console available for logging
+    # https://docs.fedoraproject.org/en-US/fedora-coreos/provisioning-vmware/#_troubleshooting_first_boot_problems
+    grub_commands:
+      - serial --speed=115200
+      - terminal_input serial console
+      - terminal_output serial console
+    kernel_arguments:
+      - console=ttyS0,115200n8
+      - console=tty0
diff --git a/tests/kola/README.md b/tests/kola/README.md
new file mode 100644
index 0000000000..7bcce15636
--- /dev/null
+++ b/tests/kola/README.md
@@ -0,0 +1,18 @@
+# Kola External Tests
+
+For more information about the `kola` external tests, see the [Integrating External Test Suites](https://coreos.github.io/coreos-assembler/kola/external-tests/#integrating-external-test-suites)
+topic in the `coreos-assembler` documentation.
+
+## Sharing tests between FCOS/RHCOS
+
+These tests are intended to be shared between FCOS and RHCOS. If you are adding
+a new test(s) to this collection, ensure that the test also run on RHCOS.
+Otherwise, use the `kola` JSON header to specify that it only runs on FCOS like
+so:
+
+```bash
+#!/bin/bash
+## kola:
+##   distros: fcos
+...
+```
diff --git a/tests/kola/binfmt/data/commonlib.sh b/tests/kola/binfmt/data/commonlib.sh
new file mode 120000
index 0000000000..1742d51e67
--- /dev/null
+++ b/tests/kola/binfmt/data/commonlib.sh
@@ -0,0 +1 @@
+../../data/commonlib.sh
\ No newline at end of file
diff --git a/tests/kola/binfmt/qemu b/tests/kola/binfmt/qemu
new file mode 100755
index 0000000000..27e8d5e9f5
--- /dev/null
+++ b/tests/kola/binfmt/qemu
@@ -0,0 +1,31 @@
+#!/bin/bash
+## kola:
+##   # This test pulls a container from a registry.
+##   tags: "platform-independent needs-internet"
+##   # This test doesn't make meaningful changes to the system and should
+##   # be able to be combined with other tests.
+##   exclusive: false
+##   # Only run on fcos, as rhel does not support emulation
+##   distros: fcos
+##   # We ship the x86_64 on all non-x86_64 arches.
+##   architectures: "! x86_64"
+##   description: Verify the x86_64 qemu emulator works on non-x86_64 instances.
+
+# See https://github.com/coreos/fedora-coreos-tracker/issues/1237
+
+set -xeuo pipefail
+
+# shellcheck disable=SC1091
+. "$KOLA_EXT_DATA/commonlib.sh"
+
+case "$(arch)" in
+    aarch64|ppc64le|s390x)
+        containerArch=$(podman run --arch=amd64 --rm quay.io/fedora/fedora:40 arch)
+        if [ "$containerArch" != "x86_64" ]; then
+            fatal "Test failed: x86_64 qemu emulator failed to run"
+        fi
+        ok "Test passed: x86_64 qemu emulator works on $(arch)" ;;
+    *)
+    # We shouldn't reach this point
+    fatal "No qemu-user-static support for $(arch)" ;;
+esac
diff --git a/tests/kola/boot/bootupd b/tests/kola/boot/bootupd
new file mode 100755
index 0000000000..d6bd554c5c
--- /dev/null
+++ b/tests/kola/boot/bootupd
@@ -0,0 +1,60 @@
+#!/bin/bash
+## kola:
+##   exclusive: false
+##   description: Verify that bootupd works.
+
+# We generally implement project-owned tests run in the pipeline
+# and be able to run the existing bootupd tests.
+# See https://github.com/coreos/fedora-coreos-config/pull/677
+
+set -xeuo pipefail
+
+# shellcheck disable=SC1091
+. "$KOLA_EXT_DATA/commonlib.sh"
+
+# Not all streams on which this test runs has bootupd on all arches yet. On
+# x86_64 and aarch64, we always expect bootupd to be installed. On ppc64le and
+# s390x, let's just conditionally check that *if* bootupd is installed, then
+# it's functioning as expected. We can harden it more once we've hard cut over
+# to 9.4.
+check_state_file=
+case "$(arch)" in
+    aarch64|x86_64)
+        # on these arches, we always expect state files to exist
+        check_state_file=1
+        # aarch64 (and x86_64) uses uefi firmware, should check grub2-efi
+        evr=$(rpm -q grub2-common --qf '%{EVR}')
+        if ! bootupctl status | grep "grub2-efi-.*-${evr}"; then
+            fatal "bootupctl status output should include grub2-efi package version"
+        fi
+        ;;
+    ppc64le)
+        # ppc64le has it if built by osbuild, otherwise not
+        if [ -e /sysroot/.aleph-version.json ]; then
+            check_state_file=1
+        fi
+        # ppc64le (and x86_64) uses bios firmware, should check grub2-tools
+        grub_version=$(rpm -q grub2-tools --qf '%{NEVRA}')
+        if ! bootupctl status | grep "${grub_version}"; then
+            fatal "bootupctl status output should include grub2-tools package version"
+        fi
+        ;& # fallthrough
+    *)
+        if ! rpm -q bootupd; then
+            exit 0
+        fi
+        ;;
+esac
+
+state_file=/boot/bootupd-state.json
+if [ -n "${check_state_file}" ] && [ ! -f "${state_file}" ]; then
+    fatal "${state_file} not present"
+fi
+
+# Verify `bootupctl status` output contains related content.
+# https://github.com/coreos/bootupd/issues/694
+if ! bootupctl status | grep "CoreOS aleph version"; then
+    fatal "bootupctl status output should include 'CoreOS aleph version'"
+fi
+
+ok bootupctl
diff --git a/tests/kola/boot/data/commonlib.sh b/tests/kola/boot/data/commonlib.sh
new file mode 120000
index 0000000000..1742d51e67
--- /dev/null
+++ b/tests/kola/boot/data/commonlib.sh
@@ -0,0 +1 @@
+../../data/commonlib.sh
\ No newline at end of file
diff --git a/tests/kola/boot/grub2-install b/tests/kola/boot/grub2-install
new file mode 100755
index 0000000000..d9aa01e0fe
--- /dev/null
+++ b/tests/kola/boot/grub2-install
@@ -0,0 +1,73 @@
+#!/bin/bash
+## kola:
+##   tags: "platform-independent"
+##   # The test is not available for aarch64 and s390x,
+##   # as aarch64 is UEFI only and s390x is not using grub2
+##   architectures: "! aarch64 s390x"
+##   description: Verify that we install BIOS/PReP bootloader
+##     using grub2-install from the target system.
+
+# See
+# - https://github.com/coreos/coreos-assembler/pull/3190
+# - https://github.com/coreos/coreos-assembler/issues/3148
+
+set -xeuo pipefail
+
+# shellcheck disable=SC1091
+. "$KOLA_EXT_DATA/commonlib.sh"
+
+boot_dev=/boot
+arch=$(arch)
+grub_install_args=""
+grub_modules=()
+case ${arch} in
+  x86_64)
+    target=i386-pc
+    core="${boot_dev}/grub2/${target}/core.img"
+    partition=$(findmnt -no SOURCE ${boot_dev})
+    device=$(lsblk --paths -no PKNAME ${partition})
+    #
+    # Handle modules for both the create_disk.sh case that only
+    # includes "mdraid1x" and bootupd case (used when building using
+    # OSBuild) that adds "part_gpt". The aleph stage in osbuild is
+    # the only one that creates a file at /sysroot/.aleph-version.json
+    #
+    # sync grub2-install parameters with the bootupd invocation from:
+    # https://github.com/coreos/bootupd/blob/main/src/bios.rs
+    grub_modules+=("mdraid1x")
+    if [ -f /sysroot/.aleph-version.json ]; then
+        # collapse this if in't the previous line when all is moved
+        # over to building using OSBuild.
+        grub_modules+=("part_gpt")
+    fi
+    ;;
+
+  ppc64le)
+    target=powerpc-ieee1275
+    core="${boot_dev}/grub2/${target}/core.elf"
+    device=$(realpath /dev/disk/by-partlabel/PowerPC-PReP-boot)
+    # sync grub2-install parameters with the bootupd invocation from:
+    # https://github.com/coreos/bootupd/blob/main/src/bios.rs
+    grub_install_args="--no-nvram"
+    ;;
+
+  *)
+    fatal "grub2-install testing is not supported on ${arch}"
+    ;;
+esac
+
+[ -e ${core} ] || fatal "file ${core} doesn't exist"
+core_sum=$(sha256sum ${core} | awk '{print $1}')
+
+mount -o remount,rw ${boot_dev}
+grub2-install --target ${target} --boot-directory ${boot_dev} \
+      ${grub_modules:+--modules "${grub_modules[*]}"}         \
+      ${grub_install_args} ${device}
+
+new_core_sum=$(sha256sum ${core} | awk '{print $1}')
+
+# compare core.img checksum before and after run grub2-install
+if [ "${core_sum}" != "${new_core_sum}" ]; then
+  fatal "Error: not using grub2-install from target system"
+fi
+ok "using grub2-install from target system on ${arch}"
diff --git a/tests/kola/butane/grub-users/config.bu b/tests/kola/butane/grub-users/config.bu
new file mode 100644
index 0000000000..b29a490d51
--- /dev/null
+++ b/tests/kola/butane/grub-users/config.bu
@@ -0,0 +1,8 @@
+variant: fcos
+version: 1.5.0
+grub:
+  users:
+    - name: bovik
+      password_hash: grub.pbkdf2.sha512.10000.530F1D13E19326914A29D9899EE215EC3F7FBFC7B92EDAE3ED7A229431DAD23B53A18CC8218E2BAFD35532A6F0D1897B0CA74ED3BA604FCF439FEC13FA44ED6A.58F8D4F336F58970F78E546FAA9E4EA050E539D1A09831A72C9AC795FAEF820F7420331A0B36503255B67E501F9CDAC37D790E68737782FE9906EA42975F1B54
+    - name: core
+      password_hash: grub.pbkdf2.sha512.10000.79294A73DB2345902AB2DA6EAF83699188B6AC2393D23C95EEB234CEA95476834C8567AFF15166E6762C2F4846B99C3C9F173C0C8F7AED6BD76322FAB9491131.A94ED4C213A830266595B170AFE2591F2F19F04BEC82AA2B60A682AFD0DFBDC2A90F8D6204648B0539DEDDAF3E7C90C5DB54722A2FA54BD26D112C983D450BA8
diff --git a/tests/kola/butane/grub-users/data/commonlib.sh b/tests/kola/butane/grub-users/data/commonlib.sh
new file mode 120000
index 0000000000..b8dcbdca1a
--- /dev/null
+++ b/tests/kola/butane/grub-users/data/commonlib.sh
@@ -0,0 +1 @@
+../../../data/commonlib.sh
\ No newline at end of file
diff --git a/tests/kola/butane/grub-users/test.sh b/tests/kola/butane/grub-users/test.sh
new file mode 100755
index 0000000000..f31aacbebe
--- /dev/null
+++ b/tests/kola/butane/grub-users/test.sh
@@ -0,0 +1,58 @@
+#!/bin/bash
+## kola:
+##   tags: "platform-independent"
+##   # GRUB sugar currently only exists in FCOS
+##   distros: fcos
+##   # coreos-post-ignition-checks.service forbids GRUB passwords on
+##   # ppc64le and s390x
+##   architectures: "! ppc64le s390x"
+##   description: Verify that setting GRUB password works.
+
+set -xeuo pipefail
+
+# shellcheck disable=SC1091
+. "$KOLA_EXT_DATA/commonlib.sh"
+
+case "${AUTOPKGTEST_REBOOT_MARK:-}" in
+"")
+    # configuration written directly by Ignition
+    if ! grep -q '^set superusers="bovik core"$' /boot/grub2/user.cfg; then
+        fatal "Missing superusers in GRUB user.cfg"
+    fi
+    for user in bovik core; do
+        if ! grep -q "^password_pbkdf2 $user grub.pbkdf2.sha512.10000." /boot/grub2/user.cfg; then
+            fatal "Missing user $user in GRUB user.cfg"
+        fi
+    done
+    ok "Butane GRUB sugar"
+
+    # force a new deployment
+    rpm-ostree kargs --append test-added-karg
+    /tmp/autopkgtest-reboot rebooted
+    ;;
+rebooted)
+    # check that we booted into the correct deployment
+    if ! grep -q test-added-karg /proc/cmdline; then
+        fatal "Rebooted into old deployment"
+    fi
+    # cross-check karg with BLS configs
+    if grep -q test-added-karg /boot/loader.0/entries/ostree-1*.conf; then
+        fatal "Old BLS config contains new karg"
+    fi
+    if ! grep -q test-added-karg /boot/loader.0/entries/ostree-2*.conf; then
+        fatal "New BLS config doesn't contain new karg"
+    fi
+    # old deployment should require a password to boot
+    if ! grep -q '^grub_users ""$' /boot/loader.0/entries/ostree-1*.conf; then
+        fatal "Missing grub_users setting in old BLS config"
+    fi
+    # new one should not
+    if grep -q grub_users /boot/loader.0/entries/ostree-2*.conf; then
+        fatal "grub_users setting present in new BLS config"
+    fi
+    ok "BLS grub_users setting"
+    ;;
+*)
+    fatal "Unexpected mark: ${AUTOPKGTEST_REBOOT_MARK}"
+    ;;
+esac
diff --git a/tests/kola/chrony/coreos-platform-chrony-generator b/tests/kola/chrony/coreos-platform-chrony-generator
deleted file mode 100755
index b902608bb6..0000000000
--- a/tests/kola/chrony/coreos-platform-chrony-generator
+++ /dev/null
@@ -1,15 +0,0 @@
-#!/bin/bash
-set -euo pipefail
-# Test the coreos-platform-chrony generator.
-
-# kola: { "platforms": "aws,azure,gcp" }
-
-cd $(mktemp -d)
-
-platform="$(grep -Eo ' ignition.platform.id=[a-z]+' /proc/cmdline | cut -f 2 -d =)"
-case "${platform}" in
-    aws) chronyc sources |grep '169.254.169.123'; echo "ok chrony aws" ;;
-    azure) chronyc sources |grep 'PHC0'; echo "ok chrony azure" ;;
-    gcp) chronyc sources | grep metadata.google.internal; echo "ok chrony gcp" ;;
-    *) echo "unhandled platform ${platform} ?"; exit 1 ;;
-esac
diff --git a/tests/kola/chrony/dhcp-propagation b/tests/kola/chrony/dhcp-propagation
deleted file mode 100755
index 018af69677..0000000000
--- a/tests/kola/chrony/dhcp-propagation
+++ /dev/null
@@ -1,75 +0,0 @@
-#!/bin/bash
-
-# This script creates two veth interfaces i.e. one for the host machine 
-# and other for the container(dnsmasq server). This setup will be helpful
-# to verify the DHCP propagation of NTP servers. This will also avoid any 
-# regression that might cause in RHCOS or FCOS when the upstream changes 
-# come down and obsolete the temporary work (https://github.com/coreos/fedora-coreos-config/pull/412)
-
-set -xeuo pipefail
-
-# Just run on QEMU; it should work theoretically on cloud platforms, but many
-# of those have platform-specific sources which take precedence.
-# kola: { "tags": "needs-internet", "platforms": "qemu-unpriv" }
-
-test_setup() {
-
-    # This is needed to run a container with systemd
-    setsebool container_manage_cgroup 1
-
-    # create a network namespace
-    ip netns add container
-
-    # create veth pair and assign a namespace to veth-container
-    ip link add veth-host type veth peer name veth-container
-    ip link set veth-container netns container
-
-    # assign an IP address to the `veth-container` interface and bring it up
-    ip netns exec container ip address add 172.16.0.1/24 dev veth-container
-    ip netns exec container ip link set veth-container up
-
-    # run podman commands to set up dnsmasq server
-    pushd $(mktemp -d)
-    NTPHOSTIP=$(getent hosts time-c-g.nist.gov | cut -d ' ' -f 1)
-    cat <<EOF >Dockerfile
-FROM registry.fedoraproject.org/fedora:33
-RUN dnf -y install systemd dnsmasq iproute iputils \
-&& dnf clean all \
-&& systemctl enable dnsmasq
-RUN echo -e 'dhcp-range=172.16.0.10,172.16.0.20,12h\nbind-interfaces\ninterface=veth-container\ndhcp-option=option:ntp-server,$NTPHOSTIP' > /etc/dnsmasq.d/dhcp
-CMD [ "/sbin/init" ]
-EOF
-    podman build -t dnsmasq .
-    popd
-    podman run -d --rm --name dnsmasq --privileged --network ns:/var/run/netns/container dnsmasq
-
-    # Tell NM to manage the `veth-host` interface and bring it up (will attempt DHCP).
-    # Do this after we start dnsmasq so we don't have to deal with DHCP timeouts.
-    nmcli dev set veth-host managed yes
-    ip link set veth-host up
-}
-
-main() {
-
-    test_setup
-
-    # check for NTP server information
-    # Name: time-c-g.nist.gov  IP address: 129.6.15.30
-    NTPSERVER=""
-    retries=300
-    while [[ $retries -gt 0 && ! $NTPSERVER =~ "time-c-g.nist.gov" ]]; do
-         NTPSERVER=$(chronyc sources)
-         if [[ $NTPSERVER != *"time-c-g.nist.gov"* ]]; then
-             echo "waiting for ntp server to appear"
-             sleep 1
-             retries=$((retries - 1))
-         fi
-    done
-
-    if [ $retries -eq  0 ]; then
-        echo "propagation of ntp server information via dhcp failed"
-        exit 1
-    fi
-}
-
-main
diff --git a/tests/kola/clhm/data/commonlib.sh b/tests/kola/clhm/data/commonlib.sh
new file mode 120000
index 0000000000..1742d51e67
--- /dev/null
+++ b/tests/kola/clhm/data/commonlib.sh
@@ -0,0 +1 @@
+../../data/commonlib.sh
\ No newline at end of file
diff --git a/tests/kola/clhm/ignition-warnings/config.bu b/tests/kola/clhm/ignition-warnings/config.bu
new file mode 100644
index 0000000000..3b0ab78a3f
--- /dev/null
+++ b/tests/kola/clhm/ignition-warnings/config.bu
@@ -0,0 +1,16 @@
+variant: fcos
+version: 1.1.0
+systemd:
+  # This systemd unit doesn't have the Install
+  # section in it, so as part of the validation
+  # step, Ignition will throw the following warning:
+  # 'warning at $.systemd.units.0.contents: unit "echo@.service" is enabled, but has no install section so enable does nothing'
+  units:
+    - name: echo.service
+      enabled: true
+      contents: |
+        [Unit]
+        Description=echo service template
+        [Service]
+        Type=oneshot
+        ExecStart=/bin/echo %i
diff --git a/tests/kola/clhm/ignition-warnings/data/commonlib.sh b/tests/kola/clhm/ignition-warnings/data/commonlib.sh
new file mode 120000
index 0000000000..b8dcbdca1a
--- /dev/null
+++ b/tests/kola/clhm/ignition-warnings/data/commonlib.sh
@@ -0,0 +1 @@
+../../../data/commonlib.sh
\ No newline at end of file
diff --git a/tests/kola/clhm/ignition-warnings/test.sh b/tests/kola/clhm/ignition-warnings/test.sh
new file mode 100755
index 0000000000..74b22eddd1
--- /dev/null
+++ b/tests/kola/clhm/ignition-warnings/test.sh
@@ -0,0 +1,28 @@
+#!/bin/bash
+## kola:
+##   tags: "platform-independent"
+##   # We intentionally exclude the Install section from a systemd unit.
+##   # This is valid but not ideal, so Butane warns about it.
+##   allowConfigWarnings: true
+##   description: Verify the Ignition warnings are displayed on the console.
+
+set -xeuo pipefail
+
+# shellcheck disable=SC1091
+. "$KOLA_EXT_DATA/commonlib.sh"
+
+WARN='\e\[0;33m' # yellow
+RESET='\e\[0m' # reset
+
+warning="${WARN}Ignition: warning at \\$.systemd.units.0.contents: unit \"echo.service\" is enabled, but has no install section so enable does nothing${RESET}"
+warningsfile="/etc/issue.d/30_coreos_ignition_warnings.issue"
+
+# Check for the motd file
+if ! test -f ${warningsfile}; then
+    fatal "not found Ignition warnings issue file"
+fi
+
+if ! grep -P -q "${warning}" "${warningsfile}"; then
+    fatal "Ignition warning did not show up as expected in issue.d"
+fi
+ok "Successfully displayed Ignition warning on the console"
diff --git a/tests/kola/clhm/network-device-info b/tests/kola/clhm/network-device-info
new file mode 100755
index 0000000000..a89ad8346f
--- /dev/null
+++ b/tests/kola/clhm/network-device-info
@@ -0,0 +1,20 @@
+#!/bin/bash
+## kola:
+##   # This is a read-only test that can be run with other tests.
+##   exclusive: false
+##   description: Verify that CLHM wrote a snippet for the NIC IP info.
+
+# See https://github.com/coreos/fedora-coreos-tracker/issues/1153
+
+set -xeuo pipefail
+
+# shellcheck disable=SC1091
+. "$KOLA_EXT_DATA/commonlib.sh"
+
+for file in /etc/issue.d/22*.issue
+do
+    if [ ! -f "$file" ]; then
+    fatal "Network Device info does not exist"
+    fi
+done
+ok "Network Device info exists"
diff --git a/tests/kola/containers/cgroups-v2 b/tests/kola/containers/cgroups-v2
new file mode 100755
index 0000000000..292d0a038b
--- /dev/null
+++ b/tests/kola/containers/cgroups-v2
@@ -0,0 +1,24 @@
+#!/bin/bash
+## kola:
+##   # This test only runs on FCOS because RHCOS does not currently
+##   # support cgroupsv2 by default.
+##   # TODO-RHCOS: drop "fcos" tag when cgroupsv2 lands in RHCOS
+##   distros: fcos
+##   exclusive: false
+##   description: Verify the system supports cgroupsv2 on FCOS.
+
+set -xeuo pipefail
+
+# shellcheck disable=SC1091
+. "$KOLA_EXT_DATA/commonlib.sh"
+
+# make sure the system is on cgroups v2
+has_cgroup_karg=1
+grep -q systemd.unified_cgroup_hierarchy /proc/cmdline || has_cgroup_karg=0
+sys_fs_cgroup_source=$(findmnt -no SOURCE /sys/fs/cgroup)
+if [ $has_cgroup_karg == 1 ]; then
+    fatal "found systemd.unified_cgroup_hierarchy=0"
+fi
+if [[ $sys_fs_cgroup_source != cgroup2 ]]; then
+    fatal "/sys/fs/cgroup is not cgroup2"
+fi
diff --git a/tests/kola/containers/data/commonlib.sh b/tests/kola/containers/data/commonlib.sh
new file mode 120000
index 0000000000..1742d51e67
--- /dev/null
+++ b/tests/kola/containers/data/commonlib.sh
@@ -0,0 +1 @@
+../../data/commonlib.sh
\ No newline at end of file
diff --git a/tests/kola/containers/quadlet/config.bu b/tests/kola/containers/quadlet/config.bu
new file mode 100644
index 0000000000..f137415b0b
--- /dev/null
+++ b/tests/kola/containers/quadlet/config.bu
@@ -0,0 +1,36 @@
+variant: fcos
+version: 1.4.0
+storage:
+  files:
+    - path: /etc/containers/systemd/test.container
+      contents:
+        inline: |
+          [Unit]
+          Description=A minimal container
+
+          [Container]
+          Image=quay.io/fedora/fedora-minimal:40
+          Exec=sleep 60
+          Volume=test.volume:/data
+          Network=test.network
+
+          [Service]
+          Restart=always
+
+          [Install]
+          WantedBy=multi-user.target default.target
+    - path: /etc/containers/systemd/test.volume
+      contents:
+        inline: |
+          [Volume]
+          User=root
+          Group=root
+          Label=org.test.Key=quadlet-test-volume
+    - path: /etc/containers/systemd/test.network
+      contents:
+        inline: |
+          [Network]
+          Subnet=172.16.0.0/24
+          Gateway=172.16.0.1
+          IPRange=172.16.0.0/28
+          Label=org.test.Key=quadlet-test-network
diff --git a/tests/kola/containers/quadlet/data/commonlib.sh b/tests/kola/containers/quadlet/data/commonlib.sh
new file mode 120000
index 0000000000..b8dcbdca1a
--- /dev/null
+++ b/tests/kola/containers/quadlet/data/commonlib.sh
@@ -0,0 +1 @@
+../../../data/commonlib.sh
\ No newline at end of file
diff --git a/tests/kola/containers/quadlet/test.sh b/tests/kola/containers/quadlet/test.sh
new file mode 100755
index 0000000000..3925b5e73c
--- /dev/null
+++ b/tests/kola/containers/quadlet/test.sh
@@ -0,0 +1,45 @@
+#!/bin/bash
+## kola:
+##   # We're pulling a container image from Quay.io
+##   tags: "platform-independent needs-internet"
+##   description: Verify that basic quadlet functionality works.
+
+set -xeuo pipefail
+
+# shellcheck disable=SC1091
+. "$KOLA_EXT_DATA/commonlib.sh"
+
+# Test volume
+if ! is_service_active test-volume.service; then
+  fatal "test-volume.service failed to start"
+fi
+volume_info=$(podman volume inspect systemd-test)
+if [[ "$(jq -r '.[0].Labels."org.test.Key"' <<< "$volume_info")" != "quadlet-test-volume" ]]; then
+    fatal "Volume not correctly created"
+fi
+
+# Test network
+if ! is_service_active test-network.service; then
+  fatal "test-network.service failed to start"
+fi
+network_info=$(podman network inspect systemd-test)
+if [[ "$(jq -r '.[0].labels."org.test.Key"' <<< "$network_info")" != "quadlet-test-network" ]]; then
+    fatal "Network not correctly created"
+fi
+
+# Test container
+if ! is_service_active test.service; then
+  fatal "test-network.service failed to start"
+fi
+container_info=$(podman container inspect systemd-test)
+if [[ "$(jq -r '.[0].ImageName' <<< "$container_info")" != "quay.io/fedora/fedora-minimal:40" ]]; then
+    fatal "Container not using the correct image"
+fi
+if [[ "$(jq -r '.[0].NetworkSettings.Networks[].NetworkID' <<< "$container_info")" != "systemd-test" ]]; then
+    fatal "Container not using the correct network"
+fi
+if [[ "$(jq -r '.[0].HostConfig.Binds[0]' <<< "$container_info")" != "systemd-test:/data:rw,rprivate,nosuid,nodev,rbind" ]]; then
+    fatal "Container not using the correct volume"
+fi
+
+ok "Successfully tested basic quadlet functionality"
diff --git a/tests/kola/content-origins/data/commonlib.sh b/tests/kola/content-origins/data/commonlib.sh
new file mode 120000
index 0000000000..1742d51e67
--- /dev/null
+++ b/tests/kola/content-origins/data/commonlib.sh
@@ -0,0 +1 @@
+../../data/commonlib.sh
\ No newline at end of file
diff --git a/tests/kola/content-origins/test.sh b/tests/kola/content-origins/test.sh
new file mode 100755
index 0000000000..ea3dd9f3c5
--- /dev/null
+++ b/tests/kola/content-origins/test.sh
@@ -0,0 +1,31 @@
+#!/bin/bash
+## kola:
+##   tags: "platform-independent"
+##   # This is a read-only, nondestructive test.
+##   exclusive: false
+##   # May support e.g. centos in the future
+##   distros: "fcos rhcos"
+##   description: Verify the RPM %{vendor} flag for everything installed
+##     matches what we expect.
+
+set -xeuo pipefail
+
+# shellcheck disable=SC1091
+. "$KOLA_EXT_DATA/commonlib.sh"
+
+. /usr/lib/os-release
+
+case "${ID}" in
+    fedora) vendor='Fedora Project';;
+    rhel|rhcos) vendor='Red Hat, Inc.';;
+    *) echo "Unknown operating system ID=${ID}; skipping this test"; exit 0;;
+esac
+
+cd $(mktemp -d)
+rpm -qa --queryformat='%{name},%{vendor}\n' > rpmvendors.txt
+if grep -vF ",${vendor}" rpmvendors.txt > unmatched.txt; then
+    cat unmatched.txt
+    fatal "Expected only vendor ${vendor} for all packages"
+fi
+echo "ok all RPMs produced by Vendor: ${vendor}"
+
diff --git a/tests/kola/data/commonlib.sh b/tests/kola/data/commonlib.sh
new file mode 100644
index 0000000000..70055d0858
--- /dev/null
+++ b/tests/kola/data/commonlib.sh
@@ -0,0 +1,122 @@
+# shellcheck shell=bash
+# This is a common library created for the ok & fatal function and symlinks
+# added to the data/ in each directory
+
+ok() {
+    echo "ok" "$@"
+}
+
+fatal() {
+    echo "$@" >&2
+    exit 1
+}
+
+get_ipv4_for_nic() {
+    local nic_name=$1
+    local ip
+    ip="$(ip -j addr show "${nic_name}" | jq -r '.[0].addr_info | map(select(.family == "inet")) | .[0].local')"
+    if [ -z "$ip" ]; then
+        echo "Error: can not get ip for ${nic_name}"
+        exit 1
+    fi
+    echo "$ip"
+}
+
+get_fcos_stream() {
+    rpm-ostree status -b --json | jq -r '.deployments[0]["base-commit-meta"]["fedora-coreos.stream"]'
+}
+
+is_fcos() {
+    source /etc/os-release
+    [ "${ID}" == "fedora" ] && [ "${VARIANT_ID}" == "coreos" ]
+}
+
+# Note when using this, you probably also want to check `get_rhel_maj_ver`.
+is_rhcos() {
+    source /etc/os-release
+    { [ "${ID}" == "rhel" ] && [ "${VARIANT_ID}" == "coreos" ]; } || \
+    [ "${ID}" == "rhcos" ]
+}
+
+get_fedora_ver() {
+    source /etc/os-release
+    if is_fcos; then
+        echo "${VERSION_ID}"
+    fi
+}
+
+get_rhel_maj_ver() {
+    source /etc/os-release
+    if [ "${ID}" == "rhcos" ]; then
+        echo "${RHEL_VERSION%%.*}"
+    elif [ "${ID}" == "rhel" ]; then
+        echo "${VERSION_ID%%.*}"
+    else
+        fatal "Unknown ID $ID"
+    fi
+}
+
+# rhcos9
+is_rhcos9() {
+    source /etc/os-release
+    { [ "${ID}" == "rhcos" ] && [ "${RHEL_VERSION%%.*}" -eq 9 ]; } || \
+    { [ "${ID}" == "rhel" ] && [ "${VERSION_ID%%.*}" -eq 9 ]; }
+}
+
+# scos
+is_scos() {
+    source /etc/os-release
+    { [ "${ID}" == "scos" ] || [ "${ID}" == "centos" ]; } && [ "${VARIANT_ID}" == "coreos" ]
+}
+
+IFS=" " read -r -a cmdline <<< "$(</proc/cmdline)"
+cmdline_arg() {
+    local name="$1" value=""
+    for arg in "${cmdline[@]}"; do
+        if [[ "${arg%%=*}" == "${name}" ]]; then
+            value="${arg#*=}"
+        fi
+    done
+    echo "${value}"
+}
+
+# wait for ~60s when in activating status
+is_service_active() {
+    local service="$1"; shift
+    local timeout="${1:-60}"; shift
+    for _x in $(seq "${timeout}"); do
+        [ "$(systemctl is-active "${service}")" != "activating" ] && break
+        sleep 1
+    done
+    # return actual result
+    systemctl is-active "${service}"
+}
+
+## Some functions to support version comparison
+# Returns true iff $1 is equal to $2
+vereq() {
+    [ "$1" == "$2" ]
+}
+
+# Returns true iff $1 is less than $2
+verlt() {
+    vereq "$1" "$2" && return 1
+    local lowest
+    lowest="$(echo -e "$1\n$2" | sort -V | head -n 1)"
+    [ "$1" == "$lowest" ]
+}
+
+# Returns true iff $1 is less than or equal to $2
+verlte() {
+    vereq "$1" "$2" || verlt "$1" "$2"
+}
+
+# Returns true iff $1 is greater than to $2
+vergt() {
+    ! verlte "$1" "$2"
+}
+
+# Returns true iff $1 is greater than or equal to $2
+vergte() {
+    vereq "$1" "$2" || vergt "$1" "$2"
+}
diff --git a/tests/kola/disks/data/commonlib.sh b/tests/kola/disks/data/commonlib.sh
new file mode 120000
index 0000000000..1742d51e67
--- /dev/null
+++ b/tests/kola/disks/data/commonlib.sh
@@ -0,0 +1 @@
+../../data/commonlib.sh
\ No newline at end of file
diff --git a/tests/kola/disks/growfs b/tests/kola/disks/growfs
new file mode 100755
index 0000000000..fd88336377
--- /dev/null
+++ b/tests/kola/disks/growfs
@@ -0,0 +1,20 @@
+#!/bin/bash
+## kola:
+##   exclusive: false
+##   description: Verify that the rootfs is automatically grown on first
+##     boot by default and that the autosave-xfs logic didn't kick in.
+
+set -xeuo pipefail
+
+# shellcheck disable=SC1091
+. "$KOLA_EXT_DATA/commonlib.sh"
+
+if [ ! -f /run/ignition-ostree-growfs.stamp ]; then
+    fatal "rootfs was not grown on first boot"
+fi
+ok "rootfs grown on first boot"
+
+if [ -f /run/ignition-ostree-autosaved-xfs.stamp ]; then
+    fatal "unexpected autosaved XFS"
+fi
+ok "rootfs wasn't automatically reprovisioned"
diff --git a/tests/kola/disks/lvmdevices b/tests/kola/disks/lvmdevices
new file mode 100755
index 0000000000..bcea424def
--- /dev/null
+++ b/tests/kola/disks/lvmdevices
@@ -0,0 +1,65 @@
+#!/bin/bash
+## kola:
+##   description: Verify LVM devices file handling works as expected.
+##                This test is meant to cover coreos-populate-lvmdevices.service.
+##   # additionalDisks is only supported on qemu.
+##   platforms: qemu
+##   # A few extra disks to set up LVM on.
+##   additionalDisks: ["1G:serial=disk1", "2G:serial=disk2"]
+
+set -xeuo pipefail
+
+. $KOLA_EXT_DATA/commonlib.sh
+
+LVMDEVICESFILENAME="system.devices"
+LVMDEVICESFILE="/etc/lvm/devices/${LVMDEVICESFILENAME}"
+
+# Check that coreos-populate-lvmdevices did run
+if [ ! -e /var/lib/coreos-populate-lvmdevices.stamp ]; then
+  fatal "coreos-populate-lvmdevices didn't run"
+fi
+
+case "${AUTOPKGTEST_REBOOT_MARK:-}" in
+  "")
+      # Verify the lvmdevices file by default matches what was shipped.
+      if ! diff -u "/usr/${LVMDEVICESFILE}" "${LVMDEVICESFILE}"; then
+          fatal "Detected modified $LVMDEVICESFILE file."
+      fi
+
+      # Set up LVM on the two disks and set up a vg/lv/fs/mount on one
+      # of them. Specify a blank devicesfile so the *create commands
+      # won't touch our devices file.
+      pvcreate --devicesfile="" /dev/disk/by-id/virtio-disk1 /dev/disk/by-id/virtio-disk2
+      vgcreate --devicesfile="" vg1 /dev/disk/by-id/virtio-disk1
+      lvcreate --devicesfile="" vg1 --name=lv1 -l 100%FREE
+      mkfs.ext4 /dev/vg1/lv1
+      echo "/dev/vg1/lv1 /srv/ ext4 defaults 0 2" >> /etc/fstab
+
+      # Remove the stamp file to force the "migration" to happen on
+      # next boot.
+      rm -f /var/lib/coreos-populate-lvmdevices.stamp
+
+      # reboot to simulate running migration for first time on a
+      # system with pre-existing LVM devices
+      /tmp/autopkgtest-reboot rebooted
+      ;;
+
+  rebooted)
+      # Check that the devices are in the devices file.
+      grep -q 'IDTYPE=sys_serial IDNAME=disk1' "$LVMDEVICESFILE" || fatal "Missing disk1 in devices file"
+      grep -q 'IDTYPE=sys_serial IDNAME=disk2' "$LVMDEVICESFILE" || fatal "Missing disk2 in devices file"
+
+      # Check that we can see the PVs
+      if [ "$(pvs --noheadings | wc -l)" != '2' ]; then
+        fatal "Incorrect number of LVM PVs detected"
+      fi
+
+      # Check that /srv/ is a mountpoint
+      if ! mountpoint /srv; then
+        fatal "/srv/ is not mounted, but it should be"
+      fi
+
+      ok "LVM Devices file populated correctly"
+      ;;
+  *) fatal "unexpected mark: ${AUTOPKGTEST_REBOOT_MARK}";;
+esac
diff --git a/tests/kola/disks/no-google-device-links b/tests/kola/disks/no-google-device-links
new file mode 100755
index 0000000000..9ed282ba11
--- /dev/null
+++ b/tests/kola/disks/no-google-device-links
@@ -0,0 +1,18 @@
+#!/bin/bash
+## kola:
+##   platforms: "! gcp"
+##   exclusive: false
+##   description: Verify no /dev/disk/by-id/*google* device links are found on non GCP.
+
+# See https://issues.redhat.com/browse/OCPBUGS-13754
+
+set -xeuo pipefail
+
+# shellcheck disable=SC1091
+. "$KOLA_EXT_DATA/commonlib.sh"
+
+links=$(find /dev/disk/by-id/ -iname "*google*")
+if [[ -n "${links:-}" ]]; then
+    fatal "Error: should not find /dev/disk/by-id/*google* device links on non GCP"
+fi
+ok "No /dev/disk/by-id/*google* device links are found on non GCP"
diff --git a/tests/kola/disks/partition-scheme b/tests/kola/disks/partition-scheme
new file mode 100755
index 0000000000..c20a09f54f
--- /dev/null
+++ b/tests/kola/disks/partition-scheme
@@ -0,0 +1,88 @@
+#!/bin/bash
+## kola:
+##   exclusive: false
+##   description: Verify the partition scheme is what we expect.
+##
+
+set -xeuo pipefail
+
+# shellcheck disable=SC1091
+. "$KOLA_EXT_DATA/commonlib.sh"
+
+root_part=$(findmnt -n -o SOURCE /sysroot)
+disk_name=$(lsblk --json -o PKNAME --path "$root_part" | jq --raw-output '.blockdevices[].pkname')
+diskData=$(sfdisk --json "$disk_name" | jq '.partitiontable' )
+partitionData=$(echo $diskData | jq '.partitions[]')
+totalPartitions=$(echo $diskData | jq '.partitions | length')
+sector_size=$(echo $diskData | jq .sectorsize)
+
+ONE_MiB=$(( 1024 * 1024 ))
+
+# Keep information about the expected partitions in
+# an associative array. Since associative arrays are
+# unordered keep around a sorted variable so we can
+# infer the correct order later.
+case "$(uname -m)" in
+    "aarch64")
+        sorted="reserved EFI-SYSTEM boot root"
+        declare -A expected=(
+            ["reserved"]="1"
+            ["EFI-SYSTEM"]="127"
+            ["boot"]="384"
+            ["root"]=""
+        );;
+    "ppc64le")
+        sorted="PowerPC-PReP-boot reserved boot root"
+        declare -A expected=(
+            ["PowerPC-PReP-boot"]="4"
+            ["reserved"]="1"
+            ["boot"]="384"
+            ["root"]=""
+        );;
+    "x86_64")
+        sorted="BIOS-BOOT EFI-SYSTEM boot root"
+        declare -A expected=(
+            ["BIOS-BOOT"]="1"
+            ["EFI-SYSTEM"]="127"
+            ["boot"]="384"
+            ["root"]=""
+        );;
+    "s390x")
+        sorted="boot root"
+        declare -A expected=(
+            ["boot"]="384"
+            ["root"]=""
+        );;
+esac
+
+# Check if the number of partitions match
+if [[ $totalPartitions -ne "${#expected[@]}" ]]; then
+    fatal "Expected ${#expected[@]} partitions, got $totalPartitions"
+fi
+
+# There is a 1MiB gap at the beginning of the disks
+expected_start=$(( 1 * $ONE_MiB / $sector_size ))
+
+# Iterate over the partitions and check their start and size
+for key in $sorted; do
+    size_MiB="${expected[${key}]}"
+    start=$(echo "$partitionData" | jq "select ( .name == \"$key\") | .start")
+    sectors=$(echo "$partitionData" | jq "select ( .name == \"$key\") | .size")
+    test -z "$start" && fatal "Could not detect start sector for $key"
+    test -z "$sectors" && fatal "Could not detect size in sectors for $key"
+    size=$(( $sectors * $sector_size ))
+    if [[ "$start" -ne "$expected_start" ]]; then
+        fatal "Expected $key partition start sector of $expected_start, got $start"
+    fi
+    if [ ! -z "$size_MiB" ]; then
+        expected_size=$(($size_MiB * $ONE_MiB))
+        if [[ "$expected_size" -ne "$size" ]]; then
+            fatal "Expected $key partition of size $expected_size, got $size"
+        fi
+    fi
+    # The expected start of the next partition will be the start of this partition
+    # plus the size of this partition.
+    expected_start=$(($expected_start + $size / $sector_size))
+done
+
+ok partition scheme
diff --git a/tests/kola/disks/root-boot-ro b/tests/kola/disks/root-boot-ro
new file mode 100755
index 0000000000..c197b804b9
--- /dev/null
+++ b/tests/kola/disks/root-boot-ro
@@ -0,0 +1,19 @@
+#!/bin/bash
+## kola:
+##   exclusive: false
+##   description: Verify /sysroot and /boot are read-only partitions.
+
+set -xeuo pipefail
+
+# shellcheck disable=SC1091
+. "$KOLA_EXT_DATA/commonlib.sh"
+
+for part in /sysroot /boot; do
+    if ! findmnt -n -o options ${part} | grep -q "ro,"; then
+        fatal "${part} is missing ro option"
+    fi
+    if test -w "${part}" || touch "${part}/somefile" 2>/dev/null; then
+        fatal "${part} is writable"
+    fi
+done
+ok read-only partitions
diff --git a/tests/kola/disks/root-prjquota b/tests/kola/disks/root-prjquota
new file mode 100755
index 0000000000..1da66d491e
--- /dev/null
+++ b/tests/kola/disks/root-prjquota
@@ -0,0 +1,15 @@
+#!/bin/bash
+## kola:
+##   exclusive: false
+##   description: Verify /sysroot is mounted with prjquota.
+
+set -xeuo pipefail
+
+# shellcheck disable=SC1091
+. "$KOLA_EXT_DATA/commonlib.sh"
+
+rootflags=$(findmnt /sysroot -no OPTIONS)
+if ! grep prjquota <<< "${rootflags}"; then
+    fatal "missing prjquota in root mount flags: ${rootflags}"
+fi
+ok "root mounted with prjquota"
diff --git a/tests/kola/disks/systemd-repart-service b/tests/kola/disks/systemd-repart-service
new file mode 100755
index 0000000000..527fdd2a81
--- /dev/null
+++ b/tests/kola/disks/systemd-repart-service
@@ -0,0 +1,16 @@
+#!/bin/bash
+## kola:
+##   exclusive: false
+##   description: Verify systemd-repart.service is masked.
+
+# See https://github.com/coreos/fedora-coreos-config/pull/744
+
+set -xeuo pipefail
+
+# shellcheck disable=SC1091
+. "$KOLA_EXT_DATA/commonlib.sh"
+
+if [ "$(systemctl is-enabled systemd-repart.service)" != 'masked' ]; then
+    fatal "systemd-repart.service systemd unit should be masked"
+fi
+ok "systemd-repart.service systemd unit is masked"
diff --git a/tests/kola/disks/tmpfs b/tests/kola/disks/tmpfs
new file mode 100755
index 0000000000..370810660c
--- /dev/null
+++ b/tests/kola/disks/tmpfs
@@ -0,0 +1,15 @@
+#!/bin/bash
+## kola:
+##   exclusive: false
+##   description: Verify that we have tmpfs on /tmp.
+
+set -xeuo pipefail
+
+# shellcheck disable=SC1091
+. "$KOLA_EXT_DATA/commonlib.sh"
+
+tmpfs=$(findmnt -n -o FSTYPE /tmp)
+if [ "${tmpfs}" != "tmpfs" ]; then
+    fatal "Error: Expected tmpfs on /tmp, found: ${tmpfs}"
+fi
+ok "tmpfs on /tmp"
diff --git a/tests/kola/docker/basic b/tests/kola/docker/basic
new file mode 100755
index 0000000000..8a140bb67c
--- /dev/null
+++ b/tests/kola/docker/basic
@@ -0,0 +1,18 @@
+#!/bin/bash
+## kola:
+##   # Must not run this test with another podman test
+##   exclusive: true
+##   # We only ship moby/docker in FCOS
+##   distros: fcos
+##   # This test pulls a container image from remote sources.
+##   tags: "platform-independent needs-internet"
+##   description: Verify that running a basic container with docker works.
+
+set -xeuo pipefail
+
+# shellcheck disable=SC1091
+. "$KOLA_EXT_DATA/commonlib.sh"
+
+docker run --rm quay.io/fedora/fedora:40 true
+
+ok "basic docker run successfully"
diff --git a/tests/kola/docker/data/commonlib.sh b/tests/kola/docker/data/commonlib.sh
new file mode 120000
index 0000000000..1742d51e67
--- /dev/null
+++ b/tests/kola/docker/data/commonlib.sh
@@ -0,0 +1 @@
+../../data/commonlib.sh
\ No newline at end of file
diff --git a/tests/kola/extensions/data/commonlib.sh b/tests/kola/extensions/data/commonlib.sh
new file mode 120000
index 0000000000..1742d51e67
--- /dev/null
+++ b/tests/kola/extensions/data/commonlib.sh
@@ -0,0 +1 @@
+../../data/commonlib.sh
\ No newline at end of file
diff --git a/tests/kola/extensions/package b/tests/kola/extensions/package
new file mode 100755
index 0000000000..92539d8ac5
--- /dev/null
+++ b/tests/kola/extensions/package
@@ -0,0 +1,54 @@
+#!/bin/bash
+## kola:
+##   tags: "platform-independent needs-internet"
+##   # This test only runs on FCOS as OS extensions are implemented differently on RHCOS.
+##   distros: fcos
+##   # This is dependent on network and disk speed but we've seen the
+##   # test take longer than 10 minutes in our aarch64 qemu tests.
+##   timeoutMin: 15
+##   minMemory: 1536
+##   description: Verify that we can install some common tools as OS extensions.
+
+set -xeuo pipefail
+
+# shellcheck disable=SC1091
+. "$KOLA_EXT_DATA/commonlib.sh"
+
+commands=(
+  'gdb'
+  'htop'
+  'strace'
+  'tcpdump'
+  'tree'
+  'crio'
+)
+
+# Also try some OS extensions which have host bindings. These can only
+# work on streams that are sure to have the archive repo available
+# (mostly streams based on Fedora stable releases). Since `next`
+# rebases early to the next major Fedora and the other streams operate
+# continuously on non-stable Fedora we'll just limit this to
+# `stable`/`testing`/`testing-devel`. It's possible that this test will
+# fail on one of those streams if we fast track VIM and it's not
+# availabe in any repo, but we can just snooze the entire test in that
+# case.
+case "$(get_fcos_stream)" in
+    "stable"|"testing"|"testing-devel") commands+=('vim') ;;
+    *) ;;
+esac
+
+rpm-ostree install --apply-live "${commands[@]}"
+
+failed=""
+
+for c in "${commands[@]}"; do
+  if [[ -z "$(command -v "${c}")" ]]; then
+    echo "Could not find: ${c}"
+    failed+=" ${1}"
+  fi
+done
+
+if [[ -n "${failed}" ]]; then
+  fatal "could not install: ${failed}"
+fi
+ok "successfully installed os rpm package extensions"
diff --git a/tests/kola/files/aleph-version b/tests/kola/files/aleph-version
new file mode 100755
index 0000000000..acea887355
--- /dev/null
+++ b/tests/kola/files/aleph-version
@@ -0,0 +1,14 @@
+#!/bin/bash
+## kola:
+##   exclusive: false
+##   description: Verify /sysroot/.coreos-aleph-version.json exists.
+
+# Defined in https://github.com/coreos/fedora-coreos-tracker/blob/master/internals/README-internals.md#aleph-version
+
+set -xeuo pipefail
+
+# shellcheck disable=SC1091
+. "$KOLA_EXT_DATA/commonlib.sh"
+
+jq . < /sysroot/.coreos-aleph-version.json >/dev/null
+ok aleph
diff --git a/tests/kola/files/amd-ucode-firmware b/tests/kola/files/amd-ucode-firmware
new file mode 100755
index 0000000000..64b16151c4
--- /dev/null
+++ b/tests/kola/files/amd-ucode-firmware
@@ -0,0 +1,18 @@
+#!/bin/bash
+## kola:
+##   exclusive: false
+##   description: Verify that the host ships AMD microcode updates.
+##   architectures: x86_64
+
+# This will allow us to detect when the amd-ucode-firmware split happens in RHCOS:
+# https://github.com/coreos/fedora-coreos-tracker/issues/1618
+
+set -xeuo pipefail
+
+# shellcheck disable=SC1091
+. "$KOLA_EXT_DATA/commonlib.sh"
+
+if ! ls /usr/lib/firmware/amd-ucode/microcode*; then
+    fatal "no AMD microcode found on host"
+fi
+ok "found AMD microcode files"
diff --git a/tests/kola/files/check-symlink b/tests/kola/files/check-symlink
new file mode 100755
index 0000000000..abe044180f
--- /dev/null
+++ b/tests/kola/files/check-symlink
@@ -0,0 +1,22 @@
+#!/bin/bash
+## kola:
+##   exclusive: false
+##   description: Verify /etc release symlinks are valid.
+
+# See
+# - https://bugzilla.redhat.com/show_bug.cgi?id=2068148
+# - https://github.com/openshift/os/pull/815
+
+set -xeuo pipefail
+
+# shellcheck disable=SC1091
+. "$KOLA_EXT_DATA/commonlib.sh"
+
+for file_name in /etc/system-release-cpe /etc/system-release /etc/redhat-release /etc/os-release
+do
+    real_path=$(readlink -f ${file_name})
+    if ! test -f "${real_path}"; then
+        fatal "Error: ${file_name} symlink to ${real_path} which not exists"
+    fi
+    ok "${file_name} symlink to valid file ${real_path}"
+done
diff --git a/tests/kola/files/console-config b/tests/kola/files/console-config
new file mode 100755
index 0000000000..327436be37
--- /dev/null
+++ b/tests/kola/files/console-config
@@ -0,0 +1,82 @@
+#!/bin/bash
+## kola:
+##   # We don't need a special Ignition config and we don't modify the VM
+##   exclusive: false
+##   # s390x doesn't have any configuration in platforms.yaml, so
+##   # platforms.json is not included in the image
+##   architectures: "! s390x"
+##   description: Verify that the kargs and grub.cfg commands specified in
+##     platforms.json have been properly applied to the image.
+
+# Also check cosa correctly translated platforms.yaml to platforms.json, by
+# spot-checking certain expected values in platforms.json.
+
+set -xeuo pipefail
+
+# shellcheck disable=SC1091
+. "$KOLA_EXT_DATA/commonlib.sh"
+
+platform_json_kargs() {
+    jq -r ".$1.kernel_arguments // [] | join(\" \")" < /boot/coreos/platforms.json
+}
+
+platform_json_grub_cmds() {
+    jq -r ".$1.grub_commands // [] | join(\"\\\\n\")" < /boot/coreos/platforms.json
+}
+
+check_platforms_json() {
+    local platform="$1" expected_kargs="$2" expected_grub_cmds="$3"
+    found_kargs=$(platform_json_kargs "$platform")
+    found_grub_cmds=$(platform_json_grub_cmds "$platform")
+    if [ "$expected_kargs" != "$found_kargs" ]; then
+        fatal "platforms.json incorrect kargs for $platform"
+    fi
+    if [ "$expected_grub_cmds" != "$found_grub_cmds" ]; then
+        fatal "platforms.json incorrect GRUB commands for $platform"
+    fi
+    ok "platforms.json for $platform"
+}
+
+# Check whether platforms.json exists
+if [ ! -e /boot/coreos/platforms.json ]; then
+    fatal "platforms.json doesn't exist!"
+fi
+ok "platforms.json exists!"
+
+# Check that platforms.json matches grub.cfg and kargs
+platform=$(cmdline_arg ignition.platform.id)
+expected_kargs=$(platform_json_kargs "$platform")
+expected_grub_cmds=$(platform_json_grub_cmds "$platform")
+if [ -f /boot/grub2/console.cfg ]; then
+    grub_console_cfg_file=/boot/grub2/console.cfg
+else
+    grub_console_cfg_file=/boot/grub2/grub.cfg
+fi
+if [ -n "$expected_kargs" ] && ! grep -Eq " ${expected_kargs}( |$)" /proc/cmdline; then
+    fatal "Didn't find $expected_kargs in $(cat /proc/cmdline)"
+fi
+if [ -n "$expected_grub_cmds" ] && ! grep -qzP "\n${expected_grub_cmds}\n" $grub_console_cfg_file; then
+    fatal "Didn't find platform grub commands in ${grub_console_cfg_file}"
+fi
+ok "platforms.json matches grub.cfg and kargs"
+
+# Check that platforms.json has reasonable contents (i.e. that cosa
+# properly forwarded it from platforms.yaml).
+# Check at least one platform on each architecture.
+case $(uname -m) in
+x86_64)
+    # Matches the legacy defaults
+    check_platforms_json qemu "console=tty0 console=ttyS0,115200n8" "serial --speed=115200\nterminal_input serial console\nterminal_output serial console"
+    # Different from legacy defaults
+    check_platforms_json packet "console=ttyS1,115200n8" "serial --unit=1 --speed=115200\nterminal_input serial\nterminal_output serial"
+    ;;
+aarch64)
+    # GRUB commands but no kargs
+    check_platforms_json qemu "" "serial --speed=115200\nterminal_input serial console\nterminal_output serial console"
+    ;;
+ppc64le)
+    # Kargs but no GRUB commands
+    check_platforms_json qemu "console=hvc0 console=tty0" ""
+    ;;
+esac
+ok "platforms.json has expected contents"
diff --git a/tests/kola/files/data/commonlib.sh b/tests/kola/files/data/commonlib.sh
new file mode 120000
index 0000000000..1742d51e67
--- /dev/null
+++ b/tests/kola/files/data/commonlib.sh
@@ -0,0 +1 @@
+../../data/commonlib.sh
\ No newline at end of file
diff --git a/tests/kola/files/etc-permissions b/tests/kola/files/etc-permissions
new file mode 100755
index 0000000000..13430b047c
--- /dev/null
+++ b/tests/kola/files/etc-permissions
@@ -0,0 +1,44 @@
+#!/bin/bash
+## kola:
+##   exclusive: false
+##   description: Verify that /etc/(passwd|group|shadow|gshadow) have correct permissions.
+
+set -xeuo pipefail
+
+# shellcheck disable=SC1091
+. "$KOLA_EXT_DATA/commonlib.sh"
+
+incorrect=""
+for f in '/etc/passwd' '/etc/group'; do
+    if [[ $(stat --format="%a %u %g" "${f}") != "644 0 0" ]]; then
+        incorrect+=" ${f}"
+    fi
+done
+for f in '/etc/passwd-' '/etc/group-'; do
+    if [[ -f "${f}" ]]; then
+        if [[ $(stat --format="%a %u %g" "${f}") != "644 0 0" ]]; then
+            incorrect+=" ${f}"
+        fi
+    fi
+done
+for f in '/etc/shadow' '/etc/gshadow'; do
+    if [[ $(stat --format="%a %u %g" "${f}") != "0 0 0" ]]; then
+        incorrect+=" ${f}"
+    fi
+done
+for f in '/etc/shadow-' '/etc/gshadow-'; do
+    if [[ -f "${f}" ]]; then
+        if [[ $(stat --format="%a %u %g" "${f}") != "0 0 0" ]]; then
+            incorrect+=" ${f}"
+        fi
+    fi
+done
+
+if [[ -n "${incorrect}" ]]; then
+    # We explicitely want to split on whitespace here
+    # shellcheck disable=SC2086
+    ls -al ${incorrect}
+    fatal "found incorrect permissions for: ${incorrect}"
+fi
+
+ok "correct ownership and mode on /etc/passwd, /etc/group, /etc/shadow and /etc/gshadow"
diff --git a/tests/kola/files/fcos_groups b/tests/kola/files/fcos_groups
new file mode 100755
index 0000000000..860c8a8ef1
--- /dev/null
+++ b/tests/kola/files/fcos_groups
@@ -0,0 +1,111 @@
+#!/usr/bin/env bash
+## kola:
+##   distros: fcos
+##   exclusive: false
+##   description: Verify system groups that are shipped as part of the base OS.
+
+# Those groups come in different shapes (with static or dynamic GIDs, from
+# plain files or from scriptlets) and each case is covered by a corresponding
+# check here below.
+
+set -euo pipefail
+
+. "${KOLA_EXT_DATA}/commonlib.sh"
+
+# Check base system groups (from `setup` package).
+declare -A setup_groups=( \
+    ["root"]="0" \
+    ["bin"]="1" \
+    ["daemon"]="2" \
+    ["sys"]="3" \
+    ["adm"]="4" \
+    ["tty"]="5" \
+    ["disk"]="6" \
+    ["lp"]="7" \
+    ["mem"]="8" \
+    ["kmem"]="9" \
+    ["wheel"]="10" \
+    ["cdrom"]="11" \
+    ["mail"]="12" \
+    ["man"]="15" \
+    ["dialout"]="18" \
+    ["floppy"]="19" \
+    ["games"]="20" \
+    ["tape"]="33" \
+    ["video"]="39" \
+    ["ftp"]="50" \
+    ["lock"]="54" \
+    ["audio"]="63" \
+    ["users"]="100" \
+    ["input"]="104" \
+# CoreOS mismatch: https://github.com/coreos/fedora-coreos-tracker/issues/1201
+#   ["nobody"]="65534" \
+    ["nobody"]="99" \
+)
+for groupname in "${!setup_groups[@]}"; do
+    gid="${setup_groups[$groupname]}";
+    if [[ $(getent group "${groupname}") != ${groupname}:x:${gid}:* ]]; then
+        getent group
+        fatal "failure on setup_groups entry ${groupname}"
+    fi
+done
+echo "all expected base groups from 'setup' package are in place"
+
+# Check additional groups with static GIDs.
+declare -A extra_groups_static=( \
+    ["utmp"]="22" \
+    ["rpcuser"]="29" \
+    ["rpc"]="32" \
+    ["utempter"]="35" \
+    ["dip"]="40" \
+    ["tss"]="59" \
+    ["tcpdump"]="72" \
+    ["sshd"]="74" \
+    ["dbus"]="81" \
+    ["ceph"]="167" \
+    ["avahi-autoipd"]="170" \
+    ["systemd-journal"]="190" \
+)
+for groupname in "${!extra_groups_static[@]}"; do
+    gid="${extra_groups_static[$groupname]}";
+    if [[ $(getent group "${groupname}") != ${groupname}:x:${gid}:* ]]; then
+        getent group
+        fatal "failure on extra_group_static entry ${groupname}"
+    fi
+done
+echo "all expected extra static groups are in place"
+
+# Check CoreOS-specific static GIDs.
+declare -A coreos_groups_static=( \
+    ["sudo"]="16" \
+    ["dockerroot"]="986" \
+    ["cockpit-ws"]="987" \
+    ["systemd-bus-proxy"]="988" \
+    ["systemd-resolve"]="989" \
+    ["systemd-network"]="990" \
+    ["systemd-timesync"]="991" \
+    ["chrony"]="992" \
+    ["sssd"]="993" \
+    ["kube"]="994" \
+    ["cgred"]="996" \
+    ["etcd"]="997" \
+    ["polkitd"]="998" \
+    ["ssh_keys"]="999" \
+    ["nfsnobody"]="65534" \
+)
+for groupname in "${!coreos_groups_static[@]}"; do
+    gid="${coreos_groups_static[$groupname]}";
+    if [[ $(getent group "${groupname}") != ${groupname}:x:${gid}:* ]]; then
+        getent group
+        fatal "failure on coreos_group_static entry ${groupname}"
+    fi
+done
+echo "all expected CoreOS static groups are in place"
+
+# Check a dynamic group (from `clevis` package).
+if [[ $(getent group clevis) != clevis:x:* ]]; then
+    getent group
+    fatal "failure on group 'clevis'"
+fi
+echo "group 'clevis' from 'clevis' package is in place"
+
diff --git a/tests/kola/files/fcos_users b/tests/kola/files/fcos_users
new file mode 100755
index 0000000000..68dcec38ff
--- /dev/null
+++ b/tests/kola/files/fcos_users
@@ -0,0 +1,93 @@
+#!/usr/bin/env bash
+## kola:
+##   distros: fcos
+##   exclusive: false
+##   description: Verify system users that are shipped as part of the base OS.
+
+# Those users come in different shapes (with static or dynamic IDs, from
+# plain files or from scriptlets) and each case is covered by a corresponding
+# check here below.
+
+set -euo pipefail
+
+. "${KOLA_EXT_DATA}/commonlib.sh"
+
+# Check base system users (from `setup` package).
+declare -A setup_users=( \
+    ["root"]="0:0" \
+    ["bin"]="1:1" \
+    ["daemon"]="2:2" \
+    ["adm"]="3:4" \
+    ["lp"]="4:7" \
+    ["sync"]="5:0" \
+    ["shutdown"]="6:0" \
+    ["halt"]="7:0" \
+    ["mail"]="8:12" \
+    ["operator"]="11:0" \
+    ["games"]="12:100" \
+    ["ftp"]="14:50" \
+# CoreOS mismatch: https://github.com/coreos/fedora-coreos-tracker/issues/1201
+#   ["nobody"]="65534:65534" \
+    ["nobody"]="99:99" \
+)
+for username in "${!setup_users[@]}"; do
+    ids="${setup_users[$username]}";
+    if [[ $(getent passwd "${username}") != ${username}:x:${ids}:* ]]; then
+        getent passwd
+        fatal "failure on setup_users entry ${username}"
+    fi
+done
+echo "all expected base users from 'setup' package are in place"
+
+# Check additional users with static IDs.
+declare -A extra_users_static=( \
+    ["dbus"]="81:81" \
+    ["tcpdump"]="72:72" \
+    ["sshd"]="74:74" \
+    ["ceph"]="167:167"
+    ["tss"]="59:59" \
+    ["avahi-autoipd"]="170:170" \
+    ["rpc"]="32:32" \
+    ["rpcuser"]="29:29" \
+    ["nfsnobody"]="65534:65534" \
+)
+for username in "${!extra_users_static[@]}"; do
+    ids="${extra_users_static[$username]}";
+    if [[ $(getent passwd "${username}") != ${username}:x:${ids}:* ]]; then
+        getent passwd
+        fatal "failure on extra_user_static entry ${username}"
+    fi
+done
+echo "all expected extra static users are in place"
+
+
+# Check CoreOS-specific static UIDs.
+declare -A coreos_users_static=( \
+    ["dockerroot"]="997:986" \
+    ["kube"]="996:994" \
+    ["sssd"]="995:993" \
+    ["polkitd"]="999:998" \
+    ["etcd"]="998:997" \
+    ["chrony"]="994:992" \
+    ["systemd-timesync"]="993:991" \
+    ["systemd-network"]="991:990" \
+    ["systemd-resolve"]="990:989" \
+    ["systemd-bus-proxy"]="989:988" \
+    ["cockpit-ws"]="988:987" \
+)
+for username in "${!coreos_users_static[@]}"; do
+    ids="${coreos_users_static[$username]}";
+    if [[ $(getent passwd "${username}") != ${username}:x:${ids}:* ]]; then
+        getent passwd
+        fatal "failure on coreos_user_static entry ${username}"
+    fi
+done
+echo "all expected CoreOS static users are in place"
+
+# Check a dynamic user (from `clevis` package).
+if [[ $(getent passwd clevis) != clevis:x:* ]]; then
+    getent passwd
+    fatal "failure on user 'clevis'"
+fi
+echo "user 'clevis' from 'clevis' package is in place"
+
diff --git a/tests/kola/files/file-directory-permissions b/tests/kola/files/file-directory-permissions
new file mode 100755
index 0000000000..b10c8fb329
--- /dev/null
+++ b/tests/kola/files/file-directory-permissions
@@ -0,0 +1,52 @@
+#!/bin/bash
+## kola:
+##   exclusive: false
+##   description: Verify that there are no files and directories
+##     with 'g+w' or 'o+w' permission in /etc, except the known lists.
+
+set -xeuo pipefail
+
+# shellcheck disable=SC1091
+. "$KOLA_EXT_DATA/commonlib.sh"
+
+# List of known files and directories with group write permission
+list_known=()
+
+# List of known files and directories with group write permission (RHCOS only)
+list_known_rhcos=(
+    '/usr/share/licenses/publicsuffix-list-dafsa/COPYING'
+)
+
+is_fcos="false"
+if [[ "$(source /etc/os-release && echo "${ID}")" == "fedora" ]]; then
+    is_fcos="true"
+fi
+
+unknown=""
+while IFS= read -r -d '' e; do
+    found="false"
+    for k in "${list_known[@]}"; do
+        if [[ "${k}" == "${e}" ]]; then
+            found="true"
+            break
+        fi
+    done
+    if [[ "${is_fcos}" == "false" ]]; then
+        for k in "${list_known_rhcos[@]}"; do
+            if [[ "${k}" == "${e}" ]]; then
+                found="true"
+                break
+            fi
+        done
+    fi
+    if [[ "${found}" == "false" ]]; then
+        unknown+=" ${e}"
+    fi
+done< <(find /usr /etc -type f -perm /022 -print0 -o -type d -perm /022 -print0)
+
+if [[ -n "${unknown}" ]]; then
+    find /usr /etc -type f -perm /022 -print0 -o -type d -perm /022 -print0 | xargs -0 ls -al
+    find /usr /etc -type f -perm /022 -print0 -o -type d -perm /022 -print0 | xargs -0 rpm -qf
+    fatal "found files or directories with 'g+w' or 'o+w' permission"
+fi
+ok "no files with 'g+w' or 'o+w' permission found in /etc"
diff --git a/tests/kola/files/fwupd-refresh-timer b/tests/kola/files/fwupd-refresh-timer
new file mode 100755
index 0000000000..3624e539d8
--- /dev/null
+++ b/tests/kola/files/fwupd-refresh-timer
@@ -0,0 +1,23 @@
+#!/bin/bash
+## kola:
+##   exclusive: false
+##   description: Verify fwupd-refresh.timer is enabled.
+##   tags: "platform-independent"
+##   # This test only runs on FCOS as we only enable the timer there
+##   distros: fcos
+
+set -xeuo pipefail
+
+# shellcheck disable=SC1091
+. "$KOLA_EXT_DATA/commonlib.sh"
+
+if verlt "$(get_fedora_ver)" 39; then
+    ok "Skipping fwupd-refresh.timer test"
+    exit 0
+fi
+
+unit="fwupd-refresh.timer"
+if ! systemctl is-enabled ${unit} 1>/dev/null; then
+    fatal "Unit ${unit} should be enabled"
+fi
+ok "Unit ${unit} is enabled as expected"
diff --git a/tests/kola/files/initrd/compression b/tests/kola/files/initrd/compression
new file mode 100755
index 0000000000..3994260cff
--- /dev/null
+++ b/tests/kola/files/initrd/compression
@@ -0,0 +1,18 @@
+#!/bin/bash
+## kola:
+##   exclusive: false
+##   description: Verify initrd is compressed with zstd.
+#
+# If dracut can't find the configured compressor, it warns and falls back to
+# gzip (!).  Fail if the initrd isn't compressed with zstd.
+
+set -xeuo pipefail
+
+# shellcheck disable=SC1091
+. "$KOLA_EXT_DATA/commonlib.sh"
+
+# Check initrd for zstd magic number
+if ! LANG=C grep -aUPq "\x28\xb5\x2f\xfd" /boot/ostree/*/init*; then
+    fatal "Didn't find zstd compression in initrd"
+fi
+ok "Found zstd compression in initrd"
diff --git a/tests/kola/files/initrd/data/commonlib.sh b/tests/kola/files/initrd/data/commonlib.sh
new file mode 120000
index 0000000000..b8dcbdca1a
--- /dev/null
+++ b/tests/kola/files/initrd/data/commonlib.sh
@@ -0,0 +1 @@
+../../../data/commonlib.sh
\ No newline at end of file
diff --git a/tests/kola/files/initrd/executables b/tests/kola/files/initrd/executables
new file mode 100755
index 0000000000..9d4a0ce317
--- /dev/null
+++ b/tests/kola/files/initrd/executables
@@ -0,0 +1,17 @@
+#!/bin/bash
+## kola:
+##   exclusive: false
+##   description: Verify all initrd scripts are executable.
+
+set -xeuo pipefail
+
+# shellcheck disable=SC1091
+. "$KOLA_EXT_DATA/commonlib.sh"
+
+tmpd=$(mktemp -d)
+( cd "${tmpd}" && lsinitrd --unpack /boot/ostree/*/init* )
+if find "${tmpd}/usr/"{bin,sbin,libexec} ! -perm -0111 | grep -v clevis-luks-common-functions; then
+    fatal "Found non-executable scripts in initrd"
+fi
+rm -r "${tmpd}"
+ok "All initrd scripts are executable"
diff --git a/tests/kola/files/initrd/expected-contents b/tests/kola/files/initrd/expected-contents
new file mode 100755
index 0000000000..85470ea4d0
--- /dev/null
+++ b/tests/kola/files/initrd/expected-contents
@@ -0,0 +1,58 @@
+#!/bin/bash
+## kola:
+##   exclusive: false
+##   description: Verify that the initrd includes some specific files.
+##   # Currently the files we check only need to be available on x86_64 and aarch64
+##   architectures: x86_64 aarch64
+
+# This test runs on both FCOS & RHCOS. The initrd includes specific files which,
+# if omitted from the image, will cause some failures with certain Ignition
+# configs. This test doesn't assert the functionality of any files, it
+# simply gives a high level check to see if the files are available.
+# See https://github.com/coreos/fedora-coreos-config/issues/1775
+
+set -xeuo pipefail
+
+# shellcheck disable=SC1091
+. "$KOLA_EXT_DATA/commonlib.sh"
+
+required_initrd_files=(
+    # Files from the 25azure-udev-rules overlay
+    "/usr/lib/udev/rules.d/66-azure-storage.rules"
+    "/usr/lib/udev/rules.d/99-azure-product-uuid.rules"
+    # Files from the google-compute-engine-guest-configs-udev RPM (FCOS)
+    # or 30gcp-udev-rules overlay (RHCOS). We can probably remove these
+    # checks once RHCOS is using the RPM too.
+    "/usr/lib/udev/rules.d/65-gce-disk-naming.rules"
+    "/usr/lib/udev/google_nvme_id"
+)
+
+required_initrd_kmods=()
+
+if grep CONFIG_ISCSI_IBFT=m "/usr/lib/modules/$(uname -r)/config"; then
+    # iSCSI iBFT kernel module
+    # https://issues.redhat.com/browse/OCPBUGS-19811
+    required_initrd_kmods+=("iscsi_ibft")
+fi
+
+tmpd=$(mktemp -d)
+cleanup() {
+    rm -r "${tmpd}"
+}
+trap cleanup EXIT
+( cd "${tmpd}" && lsinitrd --unpack /boot/ostree/*/init* )
+
+for file in "${required_initrd_files[@]}"; do
+    if [ ! -e "${tmpd}/${file}" ]; then
+        fatal "${file} was not found in initrd"
+    fi
+done
+
+for kmod in "${required_initrd_kmods[@]}"; do
+    found=$(find "${tmpd}/usr/lib/modules/$(uname -r)" -name "${kmod}.ko*")
+    if [ -z "${found}" ]; then
+        fatal "kernel module ${kmod} was not found in initrd"
+    fi
+done
+
+ok "Found expected initrd files"
diff --git a/tests/kola/files/kernel-headers b/tests/kola/files/kernel-headers
new file mode 100755
index 0000000000..fb4114c070
--- /dev/null
+++ b/tests/kola/files/kernel-headers
@@ -0,0 +1,18 @@
+#!/bin/bash
+## kola:
+##   exclusive: false
+##   description: Verify that we are not including the kernel headers on the host.
+
+# See
+# - https://bugzilla.redhat.com/show_bug.cgi?id=1814719
+# - https://gitlab.cee.redhat.com/coreos/redhat-coreos/-/merge_requests/1116
+
+set -xeuo pipefail
+
+# shellcheck disable=SC1091
+. "$KOLA_EXT_DATA/commonlib.sh"
+
+if test -d /usr/include/linux; then
+   fatal "Error: should not have kernel headers on host"
+fi
+ok "kernel headers not on host"
diff --git a/tests/kola/files/license b/tests/kola/files/license
new file mode 100755
index 0000000000..4722f541a2
--- /dev/null
+++ b/tests/kola/files/license
@@ -0,0 +1,14 @@
+#!/bin/bash
+## kola:
+##   exclusive: false
+##   description: Verify /usr/share/licenses/fedora-coreos-config/LICENSE exists.
+
+set -xeuo pipefail
+
+# shellcheck disable=SC1091
+. "$KOLA_EXT_DATA/commonlib.sh"
+
+if ! test -f /usr/share/licenses/fedora-coreos-config/LICENSE; then
+    fatal missing LICENSE
+fi
+ok LICENSE
diff --git a/tests/kola/files/logrotate-service b/tests/kola/files/logrotate-service
new file mode 100755
index 0000000000..7f8273be8f
--- /dev/null
+++ b/tests/kola/files/logrotate-service
@@ -0,0 +1,17 @@
+#!/bin/bash
+## kola:
+##   exclusive: false
+##   description: Verify logrotate and logrotate.timer services are enabled.
+##   tags: "platform-independent"
+
+set -xeuo pipefail
+
+# shellcheck disable=SC1091
+. "$KOLA_EXT_DATA/commonlib.sh"
+
+for unit in logrotate logrotate.timer; do
+    if ! systemctl is-enabled ${unit} 1>/dev/null; then
+        fatal "Unit ${unit} should be enabled"
+    fi
+    ok "Unit ${unit} is enabled as expected"
+done
diff --git a/tests/kola/files/remove-manifest-files b/tests/kola/files/remove-manifest-files
new file mode 100755
index 0000000000..c4dc186814
--- /dev/null
+++ b/tests/kola/files/remove-manifest-files
@@ -0,0 +1,18 @@
+#!/bin/bash
+## kola:
+##   # This test only runs on FCOS bceause RHCOS does ship `info` which is a
+##   # Requires on a number of RHCOS packages.
+##   # TODO-RHCOS: modify test to check for RHCOS-specific `remove-files` entry
+##   distros: fcos
+##   exclusive: false
+##   description: Verify remove-files works in the manifest.
+
+set -xeuo pipefail
+
+# shellcheck disable=SC1091
+. "$KOLA_EXT_DATA/commonlib.sh"
+
+# See remove-files in the manifest
+if test -d /usr/share/info; then
+    fatal "found /usr/share/info"
+fi
diff --git a/tests/kola/files/root-bash b/tests/kola/files/root-bash
new file mode 100755
index 0000000000..94caff2e6f
--- /dev/null
+++ b/tests/kola/files/root-bash
@@ -0,0 +1,17 @@
+#!/bin/bash
+## kola:
+##   exclusive: false
+##   description: Verify /var/roothome/.bash* files exist.
+
+# See https://bugzilla.redhat.com/show_bug.cgi?id=1193590
+set -xeuo pipefail
+
+. $KOLA_EXT_DATA/commonlib.sh
+
+for bashfile in .bashrc .bash_profile .bash_logout
+do
+    if ! test -f "/var/roothome/${bashfile}"; then
+        fatal "Error: could not find /var/roothome/${bashfile}"
+    fi
+done
+ok "have /var/roothome/.bash* files"
diff --git a/tests/kola/files/root-immutable-bit b/tests/kola/files/root-immutable-bit
new file mode 100755
index 0000000000..a25a50b9e9
--- /dev/null
+++ b/tests/kola/files/root-immutable-bit
@@ -0,0 +1,17 @@
+#!/bin/bash
+## kola:
+##   exclusive: false
+##   description: Verify / has the immutable bit or we're using composefs.
+
+set -xeuo pipefail
+
+# shellcheck disable=SC1091
+. "$KOLA_EXT_DATA/commonlib.sh"
+
+fstype=$(findmnt -nvr / -o FSTYPE)
+if test $fstype == overlay; then
+    echo "on overlayfs"
+elif ! lsattr -d / | grep -qe '--i--'; then
+    fatal "missing immutable bit on /"
+fi
+ok immutable bit
diff --git a/tests/kola/files/rpmdb-sqlite b/tests/kola/files/rpmdb-sqlite
new file mode 100755
index 0000000000..e05b343e2b
--- /dev/null
+++ b/tests/kola/files/rpmdb-sqlite
@@ -0,0 +1,17 @@
+#!/bin/bash
+## kola:
+##   # Read only test thus safe to run in parallel
+##   exclusive: false
+##   description: Verify that we're using the sqlite rpmdb backend.
+
+# See https://github.com/coreos/fedora-coreos-tracker/issues/623
+
+set -xeuo pipefail
+
+# shellcheck disable=SC1091
+. "$KOLA_EXT_DATA/commonlib.sh"
+
+if [ ! -f /usr/share/rpm/rpmdb.sqlite ]; then
+    fatal "Didn't find file /usr/share/rpm/rpmdb.sqlite"
+fi
+ok rpmdb is sqlite
diff --git a/tests/kola/files/setgid b/tests/kola/files/setgid
new file mode 100755
index 0000000000..0dd9259e1c
--- /dev/null
+++ b/tests/kola/files/setgid
@@ -0,0 +1,40 @@
+#!/bin/bash
+## kola:
+##   exclusive: false
+##   description: Verify that there are no file/directory with
+##     SetGID bit set, except the known files and directories.
+
+set -xeuo pipefail
+
+# shellcheck disable=SC1091
+. "$KOLA_EXT_DATA/commonlib.sh"
+
+# List of known files and directories with SetGID bit set
+# Drop '/usr/libexec/openssh/ssh-keysign' after
+# https://src.fedoraproject.org/rpms/openssh/c/b615362fd0b4da657d624571441cb74983de6e3f?branch=rawhide
+# is in all OS.
+list_setgid_files=(
+    '/usr/bin/write'
+    '/usr/libexec/openssh/ssh-keysign'
+    '/usr/libexec/utempter/utempter'
+)
+
+unknown_setgid_files=""
+while IFS= read -r -d '' e; do
+    found="false"
+    for k in "${list_setgid_files[@]}"; do
+        if [[ "${k}" == "${e}" ]]; then
+            found="true"
+            break
+        fi
+    done
+    if [[ "${found}" == "false" ]]; then
+        unknown_setgid_files+=" ${e}"
+    fi
+done< <(find /usr /etc -type f -perm /2000 -print0 -o -type d -perm /2000 -print0)
+
+if [[ -n "${unknown_setgid_files}" ]]; then
+    echo "SetGID:${unknown_setgid_files}"
+    fatal "found files/directories with SetUID/GID bit set"
+fi
+ok "no unknown file/directory with SetUID/GID bit set"
diff --git a/tests/kola/files/setuid b/tests/kola/files/setuid
new file mode 100755
index 0000000000..1fcbf69c8b
--- /dev/null
+++ b/tests/kola/files/setuid
@@ -0,0 +1,76 @@
+#!/bin/bash
+## kola:
+##   exclusive: false
+##   description: Verify that there are no file/directory with
+##     SetUID bit set, except the known files and directories.
+
+set -xeuo pipefail
+
+# shellcheck disable=SC1091
+. "$KOLA_EXT_DATA/commonlib.sh"
+
+# List of known files and directories with SetUID bit set
+list_setuid_files=(
+    '/usr/bin/chage'
+    '/usr/bin/chfn'
+    '/usr/bin/chsh'
+    '/usr/bin/fusermount'
+    '/usr/bin/fusermount3'
+    '/usr/bin/gpasswd'
+    '/usr/bin/mount'
+    '/usr/bin/newgrp'
+    '/usr/bin/passwd'
+    '/usr/bin/pkexec'
+    '/usr/bin/su'
+    '/usr/bin/sudo'
+    '/usr/bin/umount'
+    '/usr/lib/polkit-1/polkit-agent-helper-1'
+    '/usr/libexec/openssh/ssh-keysign'
+    '/usr/sbin/grub2-set-bootflag'
+    '/usr/sbin/mount.nfs'
+    '/usr/sbin/pam_timestamp_check'
+    '/usr/sbin/unix_chkpwd'
+)
+
+# List of known files and directories with SetUID bit set (RHCOS only)
+list_setuid_files_rhcos=(
+    '/usr/libexec/dbus-1/dbus-daemon-launch-helper'
+    '/usr/libexec/sssd/krb5_child'
+    '/usr/libexec/sssd/ldap_child'
+    '/usr/libexec/sssd/proxy_child'
+    '/usr/libexec/sssd/selinux_child'
+    '/usr/sbin/userhelper'
+)
+
+is_fcos="false"
+if [[ "$(source /etc/os-release && echo "${ID}")" == "fedora" ]]; then
+    is_fcos="true"
+fi
+
+unknown_setuid_files=""
+while IFS= read -r -d '' e; do
+    found="false"
+    for k in "${list_setuid_files[@]}"; do
+        if [[ "${k}" == "${e}" ]]; then
+            found="true"
+            break
+        fi
+    done
+    if [[ "${is_fcos}" == "false" ]]; then
+        for k in "${list_setuid_files_rhcos[@]}"; do
+            if [[ "${k}" == "${e}" ]]; then
+                found="true"
+                break
+            fi
+        done
+    fi
+    if [[ "${found}" == "false" ]]; then
+        unknown_setuid_files+=" ${e}"
+    fi
+done< <(find /usr /etc -type f -perm /4000 -print0 -o -type d -perm /4000 -print0)
+
+if [[ -n "${unknown_setuid_files}" ]]; then
+    echo "SetUID:${unknown_setuid_files}"
+    fatal "found files/directories with SetUID/GID bit set"
+fi
+ok "no unknown file/directory with SetUID/GID bit set"
diff --git a/tests/kola/files/sudoers b/tests/kola/files/sudoers
new file mode 100755
index 0000000000..ac15f14e3e
--- /dev/null
+++ b/tests/kola/files/sudoers
@@ -0,0 +1,15 @@
+#!/bin/bash
+## kola:
+##   exclusive: false
+##   description: Verify the permissions and syntax of /etc/sudoers
+##     and /etc/sudoers.d/* are readable only for root.
+
+# See https://bugzilla.redhat.com/show_bug.cgi?id=1981979
+
+set -xeuo pipefail
+
+# shellcheck disable=SC1091
+. "$KOLA_EXT_DATA/commonlib.sh"
+
+visudo -c
+ok "sudoers files are valid"
diff --git a/tests/kola/files/system-generators b/tests/kola/files/system-generators
new file mode 100755
index 0000000000..a24ab11ebd
--- /dev/null
+++ b/tests/kola/files/system-generators
@@ -0,0 +1,17 @@
+#!/bin/bash
+## kola:
+##   exclusive: false
+##   description: Verify files in /usr/lib/systemd/system-generators are executable.
+
+set -xeuo pipefail
+
+# shellcheck disable=SC1091
+. "$KOLA_EXT_DATA/commonlib.sh"
+
+find /usr/lib/systemd/system-generators -type f | while read -r f; do
+    mode=$(stat -c '%a' "${f}")
+    if [[ "${mode}" != 555 ]] && [[ "${mode}" != 755 ]]; then
+        fatal "Error: generator is not executable: ${f}. Expected mode 555 or 755, found ${mode}"
+    fi
+done
+ok "system-generators are executable"
diff --git a/tests/kola/files/unlabeled-contexts b/tests/kola/files/unlabeled-contexts
new file mode 100755
index 0000000000..08e7b46c70
--- /dev/null
+++ b/tests/kola/files/unlabeled-contexts
@@ -0,0 +1,18 @@
+#!/bin/bash
+## kola:
+##   exclusive: false
+##   description: Verify no files under /var and /etc have unlabeled_t SELinux label.
+
+set -xeuo pipefail
+
+# shellcheck disable=SC1091
+. "$KOLA_EXT_DATA/commonlib.sh"
+
+# check that no files are unlabeled
+unlabeled=$(find /var /etc -context '*:unlabeled_t:*')
+if [ -n "${unlabeled}" ]; then
+    echo "Found unlabeled files:"
+    echo "${unlabeled}"
+    exit 1
+fi
+ok no files with unlabeled_t SELinux label
diff --git a/tests/kola/files/validate-symlinks b/tests/kola/files/validate-symlinks
new file mode 100755
index 0000000000..791d11df30
--- /dev/null
+++ b/tests/kola/files/validate-symlinks
@@ -0,0 +1,64 @@
+#!/bin/bash
+## kola:
+##   tags: "platform-independent"
+##   # This is a read-only test that can be run with other tests.
+##   exclusive: false
+##   description: Verify that there are no broken symlinks in /etc/ and /usr/,
+##     except the known files which require further investigation.
+
+# See https://github.com/coreos/fedora-coreos-config/issues/1782
+
+set -xeuo pipefail
+
+# shellcheck disable=SC1091
+. "$KOLA_EXT_DATA/commonlib.sh"
+
+# List of known broken symlinks that require further investigation.
+#
+# - The symlinks in /usr/lib/firmware are often broken and are typically not an
+#   issue, so let's skip this location altogether.
+# - json-glib -> https://bugzilla.redhat.com/show_bug.cgi?id=2297094
+#
+list_broken_symlinks_skip=(
+    '/etc/mtab'
+    '/etc/ssl/'
+    '/etc/swid/swidtags.d/fedoraproject.org'
+    '/etc/xdg/systemd/user'
+    '/usr/lib/.build-id/'
+    '/usr/lib/bootc/storage'
+    '/usr/lib/firmware'
+    '/usr/lib/modules/'
+    '/usr/share/licenses/json-glib/COPYING'
+    '/usr/share/rhel/secrets/etc-pki-entitlement'
+    '/usr/share/rhel/secrets/redhat.repo'
+    '/usr/share/rhel/secrets/rhsm'
+)
+
+# If RHCOS, update the array of ignored symlinks
+if is_scos || is_rhcos; then
+    rhcos_list=(
+        '/etc/grub2-efi.cfg'
+        '/etc/pki/entitlement-host'
+        '/etc/pki/tls/'
+        '/etc/rhsm-host'
+        '/etc/sysconfig/grub'
+    )
+    list_broken_symlinks_skip+=("${rhcos_list[@]}")
+fi
+
+find /usr/ /etc/ -type l -not -path "/usr/etc*" | while read -r file_name; do
+    real_path=$(realpath -m "${file_name}")
+    if [[ -e "${real_path}" ]]; then
+        continue
+    fi
+    found="false"
+    for search_element in "${list_broken_symlinks_skip[@]}"; do
+        if [[ "${file_name}" == "${search_element}"* || "${file_name}" == "${search_element}" ]]; then
+            found="true"
+            break
+        fi
+    done
+    if [[ "${found}" == "false" ]]; then
+        fatal "Error: ${file_name} symlink to ${real_path} which does not exist"
+    fi
+done
diff --git a/tests/kola/files/yum-repo-dir b/tests/kola/files/yum-repo-dir
new file mode 100755
index 0000000000..8b8a42f187
--- /dev/null
+++ b/tests/kola/files/yum-repo-dir
@@ -0,0 +1,14 @@
+#!/bin/bash
+## kola:
+##   exclusive: false
+##   description: Verify /etc/yum.repos.d exists.
+
+set -xeuo pipefail
+
+# shellcheck disable=SC1091
+. "$KOLA_EXT_DATA/commonlib.sh"
+
+if ! test -d /etc/yum.repos.d; then
+    fatal "Error: not find /etc/yum.repos.d"
+fi
+ok "have /etc/yum.repos.d"
diff --git a/tests/kola/firewall/iptables-legacy/config.bu b/tests/kola/firewall/iptables-legacy/config.bu
new file mode 100644
index 0000000000..c7092c4022
--- /dev/null
+++ b/tests/kola/firewall/iptables-legacy/config.bu
@@ -0,0 +1,28 @@
+variant: fcos
+version: 1.4.0
+storage:
+  links:
+    - path: /etc/alternatives/iptables
+      target: /usr/sbin/iptables-legacy
+      overwrite: true
+      hard: false
+    - path: /etc/alternatives/iptables-restore
+      target: /usr/sbin/iptables-legacy-restore
+      overwrite: true
+      hard: false
+    - path: /etc/alternatives/iptables-save
+      target: /usr/sbin/iptables-legacy-save
+      overwrite: true
+      hard: false
+    - path: /etc/alternatives/ip6tables
+      target: /usr/sbin/ip6tables-legacy
+      overwrite: true
+      hard: false
+    - path: /etc/alternatives/ip6tables-restore
+      target: /usr/sbin/ip6tables-legacy-restore
+      overwrite: true
+      hard: false
+    - path: /etc/alternatives/ip6tables-save
+      target: /usr/sbin/ip6tables-legacy-save
+      overwrite: true
+      hard: false
diff --git a/tests/kola/firewall/iptables-legacy/data/commonlib.sh b/tests/kola/firewall/iptables-legacy/data/commonlib.sh
new file mode 120000
index 0000000000..b8dcbdca1a
--- /dev/null
+++ b/tests/kola/firewall/iptables-legacy/data/commonlib.sh
@@ -0,0 +1 @@
+../../../data/commonlib.sh
\ No newline at end of file
diff --git a/tests/kola/firewall/iptables-legacy/test.sh b/tests/kola/firewall/iptables-legacy/test.sh
new file mode 100755
index 0000000000..1c3055d7b8
--- /dev/null
+++ b/tests/kola/firewall/iptables-legacy/test.sh
@@ -0,0 +1,20 @@
+#!/bin/bash
+## kola:
+##   distros: fcos
+##   exclusive: true
+##   description: Verify that one can configure a node to use the legacy
+##     iptables backend.
+
+# It is scoped to only FCOS because RHCOS only supports nft.
+
+set -xeuo pipefail
+
+# shellcheck disable=SC1091
+. "$KOLA_EXT_DATA/commonlib.sh"
+
+# Make sure we're on legacy iptables
+if ! iptables --version | grep legacy; then
+    iptables --version # output for logs
+    fatal "iptables version is not legacy"
+fi
+ok "iptables in legacy mode"
diff --git a/tests/kola/firewall/iptables/data/commonlib.sh b/tests/kola/firewall/iptables/data/commonlib.sh
new file mode 120000
index 0000000000..b8dcbdca1a
--- /dev/null
+++ b/tests/kola/firewall/iptables/data/commonlib.sh
@@ -0,0 +1 @@
+../../../data/commonlib.sh
\ No newline at end of file
diff --git a/tests/kola/firewall/iptables/test.sh b/tests/kola/firewall/iptables/test.sh
new file mode 100755
index 0000000000..75ebb78551
--- /dev/null
+++ b/tests/kola/firewall/iptables/test.sh
@@ -0,0 +1,17 @@
+#!/bin/bash
+## kola:
+##   exclusive: false
+##   description: Verify that the expected iptables backend is configured.
+
+# https://github.com/coreos/fedora-coreos-tracker/issues/676
+
+set -xeuo pipefail
+
+# shellcheck disable=SC1091
+. "$KOLA_EXT_DATA/commonlib.sh"
+
+if ! iptables --version | grep nf_tables; then
+    iptables --version # output for logs
+    fatal "iptables version is not nft"
+fi
+ok "iptables in nft mode"
diff --git a/tests/kola/gshadow b/tests/kola/gshadow
index 7a2ab2f940..d01a304355 100755
--- a/tests/kola/gshadow
+++ b/tests/kola/gshadow
@@ -1,7 +1,9 @@
 #!/bin/bash
-set -xeuo pipefail
-# Verify that glibc's parsing of /etc/gshadow does not cause systemd-sysusers
-# to segfault on specially constructed lines.
+## kola:
+##   description: Verify that glibc's parsing of /etc/gshadow does
+##     not cause systemd-sysusers to segfault on specially constructed lines.
+
+# See https://github.com/coreos/bugs/issues/1394
 #
 # One line must fit into the character buffer (1024 bytes, unless a previous
 # line was longer) but have enough group members such that
@@ -11,8 +13,9 @@ set -xeuo pipefail
 # The parser would return early to avoid overflow, leaving the static result
 # struct pointing to pointers from the previous line which are now invalid,
 # causing segfaults when those pointers are dereferenced.
-#
-# Tests: https://github.com/coreos/bugs/issues/1394
+
+set -xeuo pipefail
+
 echo 'grp0:*::root' >> /etc/gshadow
 echo 'grp1:*::somebody.a1,somebody.a2,somebody.a3,somebody.a4,somebody.a5,somebody.a6,somebody.a7,somebody.a8,somebody.a9,somebody.a10,somebody.a11,somebody.a12,somebody.a13,somebody.a14,somebody.a15,somebody.a16,somebody.a17,somebody.a18,somebody.a19,somebody.a20,somebody.a21,somebody.a22,somebody.a23,somebody.a24,somebody.a25,somebody.a26,somebody.a27,somebody.a28,somebody.a29,somebody.a30,somebody.a31,somebody.a32,somebody.a33,somebody.a34,somebody.a35,somebody.a36,somebody.a37,somebody.a38,somebody.a39,somebody.a40,somebody.a41,somebody.a42,somebody.a43,somebody.a44,somebody.a45,somebody.a46,somebody.a47,a1234' >> /etc/gshadow
 echo 'grp2:*::root' >> /etc/gshadow
diff --git a/tests/kola/ignition/data/commonlib.sh b/tests/kola/ignition/data/commonlib.sh
new file mode 120000
index 0000000000..1742d51e67
--- /dev/null
+++ b/tests/kola/ignition/data/commonlib.sh
@@ -0,0 +1 @@
+../../data/commonlib.sh
\ No newline at end of file
diff --git a/tests/kola/ignition/delete-config/config.bu b/tests/kola/ignition/delete-config/config.bu
new file mode 100644
index 0000000000..3faa264000
--- /dev/null
+++ b/tests/kola/ignition/delete-config/config.bu
@@ -0,0 +1,36 @@
+variant: fcos
+version: 1.4.0
+storage:
+  files:
+    - path: /etc/fake-ignition-rmcfg
+      mode: 0755
+      contents:
+        inline: |
+          #!/bin/bash
+          # Mocked replacement for ignition-rmcfg that just records that
+          # we ran it, and fails if we run it twice.  We can't run
+          # ignition-rmcfg directly because it doesn't succeed on any
+          # platform we test on.
+          if [ -e /run/ignition-rmcfg-ran ]; then
+              echo "ignition-rmcfg ran twice"
+              exit 1
+          fi
+          touch /run/ignition-rmcfg-ran
+systemd:
+  units:
+    - name: ignition-delete-config.service
+      dropins:
+        - name: 50-kola.conf
+          contents: |
+            [Unit]
+            ConditionKernelCommandLine=|ignition.platform.id=qemu
+            [Service]
+            ExecStartPre=mount --bind /etc/fake-ignition-rmcfg /usr/libexec/ignition-rmcfg
+    - name: coreos-ignition-delete-config.service
+      dropins:
+        - name: 50-kola.conf
+          contents: |
+            [Unit]
+            ConditionKernelCommandLine=|ignition.platform.id=qemu
+            [Service]
+            ExecStartPre=mount --bind /etc/fake-ignition-rmcfg /usr/libexec/ignition-rmcfg
diff --git a/tests/kola/ignition/delete-config/data/commonlib.sh b/tests/kola/ignition/delete-config/data/commonlib.sh
new file mode 120000
index 0000000000..b8dcbdca1a
--- /dev/null
+++ b/tests/kola/ignition/delete-config/data/commonlib.sh
@@ -0,0 +1 @@
+../../../data/commonlib.sh
\ No newline at end of file
diff --git a/tests/kola/ignition/delete-config/test.sh b/tests/kola/ignition/delete-config/test.sh
new file mode 100755
index 0000000000..46b3b29e4a
--- /dev/null
+++ b/tests/kola/ignition/delete-config/test.sh
@@ -0,0 +1,109 @@
+#!/bin/bash
+## kola:
+##   platforms: qemu
+##   description: Verify that we delete userdata from provider after Ignition
+##     completes.
+
+# See https://github.com/coreos/ignition/issues/1315
+# There are 2 services:
+# 1)ignition-delete-config.service, which deletes Ignition
+# configs from VMware and VirtualBox on first boot.
+# 2)coreos-ignition-delete-config.service, do the same thing
+# on existing machines on upgrade, using a stamp file in /var/lib
+# to avoid multiple runs.
+# Ideally we'd test on virtualbox and vmware, but we don't have tests
+# there, so we mock specifically for ignition.platform.id=qemu.
+# Test scenarios:
+# On first boot, verify that both 2 services ran.
+# On upgrade boot, verify that 1) should not run, 2) should run.
+# On normal boot, verify that both 2 services should not run.
+# On upgrade boot with 2) masked, verify that both 2 services
+# should not run.
+
+set -xeuo pipefail
+
+# shellcheck disable=SC1091
+. "$KOLA_EXT_DATA/commonlib.sh"
+
+case "${AUTOPKGTEST_REBOOT_MARK:-}" in
+"")
+    # Ignition boot
+
+    # https://github.com/coreos/ignition/issues/1833
+    if [ $(stat --format="%a" /etc/systemd/system/ignition-delete-config.service.d/50-kola.conf) != "644" ]; then
+        fatal "dropin file permission should be 644"
+    fi
+    if [ ! -e /run/ignition-rmcfg-ran ]; then
+        fatal "mocked ignition-rmcfg did not run on first boot"
+    fi
+
+    if [ $(systemctl is-active ignition-delete-config ||:) != active ]; then
+        fatal "ignition-delete-config didn't succeed on first boot"
+    fi
+    if [ $(systemctl is-active coreos-ignition-delete-config ||:) != active ]; then
+        fatal "coreos-ignition-delete-config didn't succeed on first boot"
+    fi
+    ok "First boot OK"
+
+    # Reset state and reboot
+    rm /var/lib/coreos-ignition-delete-config.stamp
+    /tmp/autopkgtest-reboot upgrade
+    ;;
+
+upgrade)
+    # Simulated upgrade from Ignition < 2.14.0
+
+    if [ ! -e /run/ignition-rmcfg-ran ]; then
+        fatal "mocked ignition-rmcfg did not run on upgrade boot"
+    fi
+
+    if [ $(systemctl is-active ignition-delete-config ||:) != inactive ]; then
+        fatal "ignition-delete-config ran on upgrade boot"
+    fi
+    if [ $(systemctl is-active coreos-ignition-delete-config ||:) != active ]; then
+        fatal "coreos-ignition-delete-config didn't succeed on upgrade boot"
+    fi
+    ok "Upgrade boot OK"
+
+    /tmp/autopkgtest-reboot steady-state
+    ;;
+
+steady-state)
+    # Steady-state boot; nothing should run
+
+    if [ -e /run/ignition-rmcfg-ran ]; then
+        fatal "mocked ignition-rmcfg ran on steady-state boot"
+    fi
+
+    if [ $(systemctl is-active ignition-delete-config ||:) != inactive ]; then
+        fatal "ignition-delete-config ran on steady-state boot"
+    fi
+    if [ $(systemctl is-active coreos-ignition-delete-config ||:) != inactive ]; then
+        fatal "coreos-ignition-delete-config ran on steady-state boot"
+    fi
+    ok "Steady-state boot OK"
+
+    # Reset state for masked unit and reboot
+    rm /var/lib/coreos-ignition-delete-config.stamp
+    systemctl mask ignition-delete-config.service
+    /tmp/autopkgtest-reboot masked
+    ;;
+
+masked)
+    # Simulated upgrade with masked ignition-delete-config.service
+
+    if [ -e /run/ignition-rmcfg-ran ]; then
+        fatal "mocked ignition-rmcfg ran on masked boot"
+    fi
+
+    if [ $(systemctl is-active ignition-delete-config ||:) != inactive ]; then
+        fatal "ignition-delete-config ran on masked boot"
+    fi
+    if [ $(systemctl is-active coreos-ignition-delete-config ||:) != inactive ]; then
+        fatal "coreos-ignition-delete-config ran on masked boot"
+    fi
+    ok "Masked unit OK"
+    ;;
+
+*) fatal "unexpected mark: ${AUTOPKGTEST_REBOOT_MARK}" ;;
+esac
diff --git a/tests/kola/ignition/journald-log b/tests/kola/ignition/journald-log
new file mode 100755
index 0000000000..0a98841229
--- /dev/null
+++ b/tests/kola/ignition/journald-log
@@ -0,0 +1,21 @@
+#!/bin/bash
+## kola:
+##   exclusive: false
+##   description: Verify that we send the journald log entry for a
+##     user-provided config.
+
+# See https://github.com/coreos/ignition/pull/958 for the MESSAGE_ID source.
+
+set -xeuo pipefail
+
+# shellcheck disable=SC1091
+. "$KOLA_EXT_DATA/commonlib.sh"
+
+ignitionJournalMsgId="57124006b5c94805b77ce473e92a8aeb"
+
+num=$(journalctl -o json-pretty MESSAGE_ID=$ignitionJournalMsgId | jq -s ".[] | select(.IGNITION_CONFIG_TYPE == \"user\")" | wc -l)
+
+if [ "$num" -eq 0 ]; then
+  fatal "Ignition didn't write $ignitionJournalMsgId"
+fi
+ok "ignition successfully wrote $ignitionJournalMsgId"
diff --git a/tests/kola/ignition/kargs/config.ign b/tests/kola/ignition/kargs/config.ign
new file mode 100644
index 0000000000..00816dc153
--- /dev/null
+++ b/tests/kola/ignition/kargs/config.ign
@@ -0,0 +1,9 @@
+{
+  "ignition": {
+    "version": "3.3.0"
+  },
+  "kernelArguments": {
+    "shouldExist": ["foobar"],
+    "shouldNotExist": ["mitigations=auto,nosmt"]
+  }
+}
diff --git a/tests/kola/ignition/kargs/data/commonlib.sh b/tests/kola/ignition/kargs/data/commonlib.sh
new file mode 120000
index 0000000000..b8dcbdca1a
--- /dev/null
+++ b/tests/kola/ignition/kargs/data/commonlib.sh
@@ -0,0 +1 @@
+../../../data/commonlib.sh
\ No newline at end of file
diff --git a/tests/kola/ignition/kargs/test.sh b/tests/kola/ignition/kargs/test.sh
new file mode 100755
index 0000000000..c29b1f1c7e
--- /dev/null
+++ b/tests/kola/ignition/kargs/test.sh
@@ -0,0 +1,17 @@
+#!/bin/bash
+## kola:
+##   platforms: qemu
+##   description: Verify Ignition supports kernel arguments.
+
+set -xeuo pipefail
+
+# shellcheck disable=SC1091
+. "$KOLA_EXT_DATA/commonlib.sh"
+
+if ! grep foobar /proc/cmdline; then
+    fatal "missing foobar in kernel cmdline"
+fi
+if grep mitigations /proc/cmdline; then
+    fatal "found mitigations in kernel cmdline"
+fi
+ok "Ignition kargs"
diff --git a/tests/kola/ignition/remote/config.bu b/tests/kola/ignition/remote/config.bu
new file mode 100644
index 0000000000..9750fddd77
--- /dev/null
+++ b/tests/kola/ignition/remote/config.bu
@@ -0,0 +1,8 @@
+variant: fcos
+version: 1.4.0
+ignition:
+  config:
+    merge:
+      - source: https://raw.githubusercontent.com/coreos/fedora-coreos-config/testing-devel/tests/kola/ignition/remote/remote.ign
+        verification:
+          hash: sha512-1c840823419a2eae431356b58d0c498f4ec84ef3d2b9a4fa42f75749a89fe1f413a848d9082d5dc6c243324b57fa7a76b4ef6dde5d023f9bba549b7755836170
diff --git a/tests/kola/ignition/remote/data/commonlib.sh b/tests/kola/ignition/remote/data/commonlib.sh
new file mode 120000
index 0000000000..b8dcbdca1a
--- /dev/null
+++ b/tests/kola/ignition/remote/data/commonlib.sh
@@ -0,0 +1 @@
+../../../data/commonlib.sh
\ No newline at end of file
diff --git a/tests/kola/ignition/remote/remote.ign b/tests/kola/ignition/remote/remote.ign
new file mode 100644
index 0000000000..1323ca2b40
--- /dev/null
+++ b/tests/kola/ignition/remote/remote.ign
@@ -0,0 +1,21 @@
+{
+  "ignition": {
+    "version": "3.3.0"
+  },
+  "kernelArguments": {
+    "shouldExist": [
+      "foobar"
+    ]
+  },
+  "storage": {
+    "files": [
+      {
+        "path": "/etc/testfile",
+        "contents": {
+          "source": "data:,test"
+        },
+        "mode": 420
+      }
+    ]
+  }
+}
diff --git a/tests/kola/ignition/remote/test.sh b/tests/kola/ignition/remote/test.sh
new file mode 100755
index 0000000000..6b2fa6a07c
--- /dev/null
+++ b/tests/kola/ignition/remote/test.sh
@@ -0,0 +1,27 @@
+#!/bin/bash
+## kola:
+##   tags: needs-internet
+##   description: Verify Ignition supports remote config.
+
+# See https://bugzilla.redhat.com/show_bug.cgi?id=1980679
+# remote.ign is on github, inject kernelArguments and write
+# something to /etc/testfile.
+# config.ign is to include remote kargsfile.ign.
+
+
+set -xeuo pipefail
+
+# shellcheck disable=SC1091
+. "$KOLA_EXT_DATA/commonlib.sh"
+
+if ! grep -q foobar /proc/cmdline; then
+    fatal "missing foobar in kernel cmdline"
+else
+    ok "find foobar in kernel cmdline"
+fi
+if ! test -e /etc/testfile; then
+    fatal "not found /etc/testfile"
+else
+    ok "find expected file /etc/testfile"
+fi
+ok "Ignition remote config test"
diff --git a/tests/kola/ignition/resource/authenticated-gs/config.bu b/tests/kola/ignition/resource/authenticated-gs/config.bu
new file mode 100644
index 0000000000..7453a016fb
--- /dev/null
+++ b/tests/kola/ignition/resource/authenticated-gs/config.bu
@@ -0,0 +1,18 @@
+# Objects accessible by any authenticated GS user, such as the credentials
+# associated with the GCE instance
+
+variant: fcos
+version: 1.2.0
+ignition:
+  config:
+    merge:
+      - source: "gs://ignition-test-fixtures/resources/authenticated-var.ign"
+storage:
+  files:
+    # Check that anonymous access works with credentials
+    - path: /var/resource/gs-anon
+      contents:
+        source: "gs://ignition-test-fixtures/resources/anonymous"
+    - path: /var/resource/gs-auth
+      contents:
+        source: "gs://ignition-test-fixtures/resources/authenticated"
diff --git a/tests/kola/ignition/resource/authenticated-gs/data/commonlib.sh b/tests/kola/ignition/resource/authenticated-gs/data/commonlib.sh
new file mode 120000
index 0000000000..7028449b11
--- /dev/null
+++ b/tests/kola/ignition/resource/authenticated-gs/data/commonlib.sh
@@ -0,0 +1 @@
+../../../../data/commonlib.sh
\ No newline at end of file
diff --git a/tests/kola/ignition/resource/authenticated-gs/data/expected/gs-anon b/tests/kola/ignition/resource/authenticated-gs/data/expected/gs-anon
new file mode 100644
index 0000000000..bfebec6131
--- /dev/null
+++ b/tests/kola/ignition/resource/authenticated-gs/data/expected/gs-anon
@@ -0,0 +1 @@
+kola-anonymous
\ No newline at end of file
diff --git a/tests/kola/ignition/resource/authenticated-gs/data/expected/gs-auth b/tests/kola/ignition/resource/authenticated-gs/data/expected/gs-auth
new file mode 100644
index 0000000000..f4253bde46
--- /dev/null
+++ b/tests/kola/ignition/resource/authenticated-gs/data/expected/gs-auth
@@ -0,0 +1 @@
+kola-authenticated
\ No newline at end of file
diff --git a/tests/kola/ignition/resource/authenticated-gs/data/expected/gs-config b/tests/kola/ignition/resource/authenticated-gs/data/expected/gs-config
new file mode 100644
index 0000000000..9433f22dd3
--- /dev/null
+++ b/tests/kola/ignition/resource/authenticated-gs/data/expected/gs-config
@@ -0,0 +1 @@
+kola-config
\ No newline at end of file
diff --git a/tests/kola/ignition/resource/authenticated-gs/test.sh b/tests/kola/ignition/resource/authenticated-gs/test.sh
new file mode 100755
index 0000000000..8f39c30b05
--- /dev/null
+++ b/tests/kola/ignition/resource/authenticated-gs/test.sh
@@ -0,0 +1,31 @@
+#!/bin/bash
+## kola:
+##   tags: needs-internet
+##   # We authenticate to GCS with the GCP instance's credentials.
+##   platforms: gcp
+##   description: Verify that we can fetch resources from GCS.
+
+set -xeuo pipefail
+
+# shellcheck disable=SC1091
+. "$KOLA_EXT_DATA/commonlib.sh"
+
+if ! diff -rZ $KOLA_EXT_DATA/expected /var/resource; then
+    fatal "fetched data mismatch"
+else
+    ok "fetched data ok"
+fi
+
+# verify that the objects are inaccessible anonymously
+for obj in authenticated authenticated-var.ign; do
+    if curl -sf "https://storage.googleapis.com/ignition-test-fixtures/resources/$obj"; then
+        fatal "anonymously fetching authenticated resource should have failed, but did not"
+    fi
+done
+
+# ...but that the anonymous object is accessible
+if ! curl -sf "https://storage.googleapis.com/ignition-test-fixtures/resources/anonymous" > /dev/null; then
+    fatal "anonymous resource is inaccessible"
+fi
+
+ok "resource checks ok"
diff --git a/tests/kola/ignition/resource/authenticated-s3/config.bu b/tests/kola/ignition/resource/authenticated-s3/config.bu
new file mode 100644
index 0000000000..444a258a72
--- /dev/null
+++ b/tests/kola/ignition/resource/authenticated-s3/config.bu
@@ -0,0 +1,35 @@
+# Objects accessible by any authenticated S3 user, such as the IAM role
+# associated with the instance
+
+variant: fcos
+version: 1.5.0
+ignition:
+  config:
+    merge:
+      - source: "s3://ignition-test-fixtures/resources/authenticated-var-v3.ign"
+storage:
+  files:
+    # Check that anonymous access works with credentials
+    - path: /var/resource/s3-anon
+      contents:
+        source: "s3://ignition-test-fixtures/resources/anonymous"
+    - path: /var/resource/s3-auth
+      contents:
+        source: "s3://ignition-test-fixtures/resources/authenticated"
+    - path: /var/resource/arn-auth
+      contents:
+        source: "arn:aws:s3:::ignition-test-fixtures/resources/authenticated"
+    # Publicly-readable object, fetched via an access point.  Access points
+    # don't allow anonymous access.
+    - path: /var/resource/arn-ap-anon
+      contents:
+        source: "arn:aws:s3:us-east-1:460538899914:accesspoint/ignition-test-fixtures-ap/object/resources/anonymous"
+    - path: /var/resource/arn-ap-auth
+      contents:
+        source: "arn:aws:s3:us-east-1:460538899914:accesspoint/ignition-test-fixtures-ap/object/resources/authenticated"
+    - path: /var/resource/arn-ap-versioned-original
+      contents:
+        source: "arn:aws:s3:us-east-1:460538899914:accesspoint/ignition-test-fixtures-ap/object/resources/versioned?versionId=Y9YqVujoLyHHSHJ4DslyXoaLvcilQJnU"
+    - path: /var/resource/arn-ap-versioned-latest
+      contents:
+        source: "arn:aws:s3:us-east-1:460538899914:accesspoint/ignition-test-fixtures-ap/object/resources/versioned"
diff --git a/tests/kola/ignition/resource/authenticated-s3/data/commonlib.sh b/tests/kola/ignition/resource/authenticated-s3/data/commonlib.sh
new file mode 120000
index 0000000000..7028449b11
--- /dev/null
+++ b/tests/kola/ignition/resource/authenticated-s3/data/commonlib.sh
@@ -0,0 +1 @@
+../../../../data/commonlib.sh
\ No newline at end of file
diff --git a/tests/kola/ignition/resource/authenticated-s3/data/expected/arn-ap-anon b/tests/kola/ignition/resource/authenticated-s3/data/expected/arn-ap-anon
new file mode 100644
index 0000000000..bfebec6131
--- /dev/null
+++ b/tests/kola/ignition/resource/authenticated-s3/data/expected/arn-ap-anon
@@ -0,0 +1 @@
+kola-anonymous
\ No newline at end of file
diff --git a/tests/kola/ignition/resource/authenticated-s3/data/expected/arn-ap-auth b/tests/kola/ignition/resource/authenticated-s3/data/expected/arn-ap-auth
new file mode 100644
index 0000000000..f4253bde46
--- /dev/null
+++ b/tests/kola/ignition/resource/authenticated-s3/data/expected/arn-ap-auth
@@ -0,0 +1 @@
+kola-authenticated
\ No newline at end of file
diff --git a/tests/kola/ignition/resource/authenticated-s3/data/expected/arn-ap-versioned-latest b/tests/kola/ignition/resource/authenticated-s3/data/expected/arn-ap-versioned-latest
new file mode 100644
index 0000000000..f55556eed1
--- /dev/null
+++ b/tests/kola/ignition/resource/authenticated-s3/data/expected/arn-ap-versioned-latest
@@ -0,0 +1 @@
+updated
\ No newline at end of file
diff --git a/tests/kola/ignition/resource/authenticated-s3/data/expected/arn-ap-versioned-original b/tests/kola/ignition/resource/authenticated-s3/data/expected/arn-ap-versioned-original
new file mode 100644
index 0000000000..94f3610c08
--- /dev/null
+++ b/tests/kola/ignition/resource/authenticated-s3/data/expected/arn-ap-versioned-original
@@ -0,0 +1 @@
+original
\ No newline at end of file
diff --git a/tests/kola/ignition/resource/authenticated-s3/data/expected/arn-auth b/tests/kola/ignition/resource/authenticated-s3/data/expected/arn-auth
new file mode 100644
index 0000000000..f4253bde46
--- /dev/null
+++ b/tests/kola/ignition/resource/authenticated-s3/data/expected/arn-auth
@@ -0,0 +1 @@
+kola-authenticated
\ No newline at end of file
diff --git a/tests/kola/ignition/resource/authenticated-s3/data/expected/s3-anon b/tests/kola/ignition/resource/authenticated-s3/data/expected/s3-anon
new file mode 100644
index 0000000000..bfebec6131
--- /dev/null
+++ b/tests/kola/ignition/resource/authenticated-s3/data/expected/s3-anon
@@ -0,0 +1 @@
+kola-anonymous
\ No newline at end of file
diff --git a/tests/kola/ignition/resource/authenticated-s3/data/expected/s3-auth b/tests/kola/ignition/resource/authenticated-s3/data/expected/s3-auth
new file mode 100644
index 0000000000..f4253bde46
--- /dev/null
+++ b/tests/kola/ignition/resource/authenticated-s3/data/expected/s3-auth
@@ -0,0 +1 @@
+kola-authenticated
\ No newline at end of file
diff --git a/tests/kola/ignition/resource/authenticated-s3/data/expected/s3-config b/tests/kola/ignition/resource/authenticated-s3/data/expected/s3-config
new file mode 100644
index 0000000000..9433f22dd3
--- /dev/null
+++ b/tests/kola/ignition/resource/authenticated-s3/data/expected/s3-config
@@ -0,0 +1 @@
+kola-config
\ No newline at end of file
diff --git a/tests/kola/ignition/resource/authenticated-s3/test.sh b/tests/kola/ignition/resource/authenticated-s3/test.sh
new file mode 100755
index 0000000000..e7ab36b9d2
--- /dev/null
+++ b/tests/kola/ignition/resource/authenticated-s3/test.sh
@@ -0,0 +1,31 @@
+#!/bin/bash
+## kola:
+##   tags: needs-internet
+##   # We authenticate to S3 with the EC2 instance's IAM role.
+##   platforms: aws
+##   description: Verify that we can fetch resources from S3.
+
+set -xeuo pipefail
+
+# shellcheck disable=SC1091
+. "$KOLA_EXT_DATA/commonlib.sh"
+
+if ! diff -rZ $KOLA_EXT_DATA/expected /var/resource; then
+    fatal "fetched data mismatch"
+else
+    ok "fetched data ok"
+fi
+
+# verify that the objects are inaccessible anonymously
+for obj in authenticated authenticated-var-v3.ign; do
+    if curl -sf "https://ignition-test-fixtures.s3.amazonaws.com/resources/$obj"; then
+        fatal "anonymously fetching authenticated resource should have failed, but did not"
+    fi
+done
+
+# ...but that the anonymous object is accessible
+if ! curl -sf "https://ignition-test-fixtures.s3.amazonaws.com/resources/anonymous" > /dev/null; then
+    fatal "anonymous resource is inaccessible"
+fi
+
+ok "resource checks ok"
diff --git a/tests/kola/ignition/resource/remote/config.bu b/tests/kola/ignition/resource/remote/config.bu
new file mode 100644
index 0000000000..ea45d98587
--- /dev/null
+++ b/tests/kola/ignition/resource/remote/config.bu
@@ -0,0 +1,37 @@
+variant: fcos
+version: 1.5.0
+storage:
+  files:
+    - path: /var/resource/http
+      contents:
+        source: "http://ignition-test-fixtures.s3.amazonaws.com/resources/anonymous"
+    - path: /var/resource/https
+      contents:
+        source: "https://ignition-test-fixtures.s3.amazonaws.com/resources/anonymous"
+    - path: /var/resource/gs-anon
+      contents:
+        source: "gs://ignition-test-fixtures/resources/anonymous"
+    - path: /var/resource/s3-anon
+      contents:
+        source: "s3://ignition-test-fixtures/resources/anonymous"
+    - path: /var/resource/s3-versioned-original
+      contents:
+        source: "s3://ignition-test-fixtures/resources/versioned?versionId=Y9YqVujoLyHHSHJ4DslyXoaLvcilQJnU"
+    - path: /var/resource/s3-versioned-latest
+      contents:
+        source: "s3://ignition-test-fixtures/resources/versioned"
+    - path: /var/resource/s3-versioned-https-original
+      contents:
+        source: "https://ignition-test-fixtures.s3.amazonaws.com/resources/versioned?versionId=Y9YqVujoLyHHSHJ4DslyXoaLvcilQJnU"
+    - path: /var/resource/s3-versioned-https-latest
+      contents:
+        source: "https://ignition-test-fixtures.s3.amazonaws.com/resources/versioned"
+    - path: /var/resource/arn-anon
+      contents:
+        source: "arn:aws:s3:::ignition-test-fixtures/resources/anonymous"
+    - path: /var/resource/arn-versioned-original
+      contents:
+        source: "arn:aws:s3:::ignition-test-fixtures/resources/versioned?versionId=Y9YqVujoLyHHSHJ4DslyXoaLvcilQJnU"
+    - path: /var/resource/arn-versioned-latest
+      contents:
+        source: "arn:aws:s3:::ignition-test-fixtures/resources/versioned"
diff --git a/tests/kola/ignition/resource/remote/data/commonlib.sh b/tests/kola/ignition/resource/remote/data/commonlib.sh
new file mode 120000
index 0000000000..7028449b11
--- /dev/null
+++ b/tests/kola/ignition/resource/remote/data/commonlib.sh
@@ -0,0 +1 @@
+../../../../data/commonlib.sh
\ No newline at end of file
diff --git a/tests/kola/ignition/resource/remote/data/expected/arn-anon b/tests/kola/ignition/resource/remote/data/expected/arn-anon
new file mode 100644
index 0000000000..bfebec6131
--- /dev/null
+++ b/tests/kola/ignition/resource/remote/data/expected/arn-anon
@@ -0,0 +1 @@
+kola-anonymous
\ No newline at end of file
diff --git a/tests/kola/ignition/resource/remote/data/expected/arn-versioned-latest b/tests/kola/ignition/resource/remote/data/expected/arn-versioned-latest
new file mode 100644
index 0000000000..f55556eed1
--- /dev/null
+++ b/tests/kola/ignition/resource/remote/data/expected/arn-versioned-latest
@@ -0,0 +1 @@
+updated
\ No newline at end of file
diff --git a/tests/kola/ignition/resource/remote/data/expected/arn-versioned-original b/tests/kola/ignition/resource/remote/data/expected/arn-versioned-original
new file mode 100644
index 0000000000..94f3610c08
--- /dev/null
+++ b/tests/kola/ignition/resource/remote/data/expected/arn-versioned-original
@@ -0,0 +1 @@
+original
\ No newline at end of file
diff --git a/tests/kola/ignition/resource/remote/data/expected/gs-anon b/tests/kola/ignition/resource/remote/data/expected/gs-anon
new file mode 100644
index 0000000000..bfebec6131
--- /dev/null
+++ b/tests/kola/ignition/resource/remote/data/expected/gs-anon
@@ -0,0 +1 @@
+kola-anonymous
\ No newline at end of file
diff --git a/tests/kola/ignition/resource/remote/data/expected/http b/tests/kola/ignition/resource/remote/data/expected/http
new file mode 100644
index 0000000000..bfebec6131
--- /dev/null
+++ b/tests/kola/ignition/resource/remote/data/expected/http
@@ -0,0 +1 @@
+kola-anonymous
\ No newline at end of file
diff --git a/tests/kola/ignition/resource/remote/data/expected/https b/tests/kola/ignition/resource/remote/data/expected/https
new file mode 100644
index 0000000000..bfebec6131
--- /dev/null
+++ b/tests/kola/ignition/resource/remote/data/expected/https
@@ -0,0 +1 @@
+kola-anonymous
\ No newline at end of file
diff --git a/tests/kola/ignition/resource/remote/data/expected/s3-anon b/tests/kola/ignition/resource/remote/data/expected/s3-anon
new file mode 100644
index 0000000000..bfebec6131
--- /dev/null
+++ b/tests/kola/ignition/resource/remote/data/expected/s3-anon
@@ -0,0 +1 @@
+kola-anonymous
\ No newline at end of file
diff --git a/tests/kola/ignition/resource/remote/data/expected/s3-versioned-https-latest b/tests/kola/ignition/resource/remote/data/expected/s3-versioned-https-latest
new file mode 100644
index 0000000000..f55556eed1
--- /dev/null
+++ b/tests/kola/ignition/resource/remote/data/expected/s3-versioned-https-latest
@@ -0,0 +1 @@
+updated
\ No newline at end of file
diff --git a/tests/kola/ignition/resource/remote/data/expected/s3-versioned-https-original b/tests/kola/ignition/resource/remote/data/expected/s3-versioned-https-original
new file mode 100644
index 0000000000..94f3610c08
--- /dev/null
+++ b/tests/kola/ignition/resource/remote/data/expected/s3-versioned-https-original
@@ -0,0 +1 @@
+original
\ No newline at end of file
diff --git a/tests/kola/ignition/resource/remote/data/expected/s3-versioned-latest b/tests/kola/ignition/resource/remote/data/expected/s3-versioned-latest
new file mode 100644
index 0000000000..f55556eed1
--- /dev/null
+++ b/tests/kola/ignition/resource/remote/data/expected/s3-versioned-latest
@@ -0,0 +1 @@
+updated
\ No newline at end of file
diff --git a/tests/kola/ignition/resource/remote/data/expected/s3-versioned-original b/tests/kola/ignition/resource/remote/data/expected/s3-versioned-original
new file mode 100644
index 0000000000..94f3610c08
--- /dev/null
+++ b/tests/kola/ignition/resource/remote/data/expected/s3-versioned-original
@@ -0,0 +1 @@
+original
\ No newline at end of file
diff --git a/tests/kola/ignition/resource/remote/test.sh b/tests/kola/ignition/resource/remote/test.sh
new file mode 100755
index 0000000000..21df7c9df4
--- /dev/null
+++ b/tests/kola/ignition/resource/remote/test.sh
@@ -0,0 +1,18 @@
+#!/bin/bash
+## kola:
+##   tags: needs-internet
+##   # Don't pass AWS or GCP credentials to instance
+##   noInstanceCreds: true
+##   description: Verify that Ignition can fetch anonymous resources within a
+##     cloud platform (S3 -> AWS, GCS -> GCP) when no credentials are supplied.
+
+set -xeuo pipefail
+
+# shellcheck disable=SC1091
+. "$KOLA_EXT_DATA/commonlib.sh"
+
+if ! diff -rZ $KOLA_EXT_DATA/expected /var/resource; then
+    fatal "fetched data mismatch"
+else
+    ok "fetched data ok"
+fi
diff --git a/tests/kola/ignition/stable-boot/config.bu b/tests/kola/ignition/stable-boot/config.bu
new file mode 100644
index 0000000000..c785d136df
--- /dev/null
+++ b/tests/kola/ignition/stable-boot/config.bu
@@ -0,0 +1,15 @@
+variant: fcos
+version: 1.3.0
+storage:
+  disks:
+  - device: /dev/disk/by-id/coreos-boot-disk
+    wipe_table: false
+    partitions:
+    - number: 5
+      size_mib: 1024
+      label: toor
+  filesystems:
+    - path: /var/lib/toor
+      device: /dev/disk/by-partlabel/toor
+      format: ext4
+      with_mount_unit: true
diff --git a/tests/kola/ignition/stable-boot/data/commonlib.sh b/tests/kola/ignition/stable-boot/data/commonlib.sh
new file mode 120000
index 0000000000..b8dcbdca1a
--- /dev/null
+++ b/tests/kola/ignition/stable-boot/data/commonlib.sh
@@ -0,0 +1 @@
+../../../data/commonlib.sh
\ No newline at end of file
diff --git a/tests/kola/ignition/stable-boot/test.sh b/tests/kola/ignition/stable-boot/test.sh
new file mode 100755
index 0000000000..3916ef99d3
--- /dev/null
+++ b/tests/kola/ignition/stable-boot/test.sh
@@ -0,0 +1,23 @@
+#!/bin/bash
+## kola:
+##   tags: "platform-independent"
+##   description: Verify that Ignition is able to use `coreos-boot-disk` symlink.
+
+set -xeuo pipefail
+
+# shellcheck disable=SC1091
+. "$KOLA_EXT_DATA/commonlib.sh"
+
+# symlink shouldn't be propogated to real-root
+link="/dev/disk/by-id/coreos-boot-disk"
+if [[ -h "${link}" ]]; then
+    fatal "${link} still exists"
+fi
+
+# sanity-check that the root disk has all required partitions
+findmnt -nvr -o SOURCE /boot
+findmnt -nvr -o SOURCE /sysroot
+toor=$(findmnt -nvr -o SOURCE /var/lib/toor)
+if [[ ! "$toor" =~ ^/dev/[a-z0-9]+p?5$ ]]; then
+    fatal "${toor} is not 5th partition"
+fi
diff --git a/tests/kola/ignition/systemd-disable/config.bu b/tests/kola/ignition/systemd-disable/config.bu
new file mode 100644
index 0000000000..5b198c4128
--- /dev/null
+++ b/tests/kola/ignition/systemd-disable/config.bu
@@ -0,0 +1,6 @@
+variant: fcos
+version: 1.4.0
+systemd:
+  units:
+    - name: zincati.service
+      enabled: false
diff --git a/tests/kola/ignition/systemd-disable/data/commonlib.sh b/tests/kola/ignition/systemd-disable/data/commonlib.sh
new file mode 120000
index 0000000000..b8dcbdca1a
--- /dev/null
+++ b/tests/kola/ignition/systemd-disable/data/commonlib.sh
@@ -0,0 +1 @@
+../../../data/commonlib.sh
\ No newline at end of file
diff --git a/tests/kola/ignition/systemd-disable/test.sh b/tests/kola/ignition/systemd-disable/test.sh
new file mode 100755
index 0000000000..7a1c2c4711
--- /dev/null
+++ b/tests/kola/ignition/systemd-disable/test.sh
@@ -0,0 +1,20 @@
+#!/bin/bash
+## kola:
+##   tags: "platform-independent"
+##   # This test is currently scoped to FCOS because `zincati` is only available
+##   # on FCOS.
+##   # TODO-RHCOS: Determine if any services on RHCOS may be disabled and adapt test
+##   distros: fcos
+##   description: Verify that Ignition supports to disable units.
+
+# See https://github.com/coreos/fedora-coreos-tracker/issues/392
+
+set -xeuo pipefail
+
+# shellcheck disable=SC1091
+. "$KOLA_EXT_DATA/commonlib.sh"
+
+if [ "$(systemctl is-enabled zincati.service)" != 'disabled' ]; then
+    fatal "zincati.service systemd unit should be disabled"
+fi
+ok "zincati.service systemd unit is enabled"
diff --git a/tests/kola/ignition/systemd-enable-units/config.bu b/tests/kola/ignition/systemd-enable-units/config.bu
new file mode 100644
index 0000000000..f037808cf2
--- /dev/null
+++ b/tests/kola/ignition/systemd-enable-units/config.bu
@@ -0,0 +1,16 @@
+variant: fcos
+version: 1.3.0
+systemd:
+  units:
+    - name: touch@.service
+      contents: |
+        [Service]
+        Type=oneshot
+        ExecStart=/bin/touch /run/%i
+        RemainAfterExit=yes
+        [Install]
+        WantedBy=multi-user.target
+    - name: touch@foo.service
+      enabled: true
+    - name: podman.socket
+      enabled: true
diff --git a/tests/kola/ignition/systemd-enable-units/data/commonlib.sh b/tests/kola/ignition/systemd-enable-units/data/commonlib.sh
new file mode 120000
index 0000000000..b8dcbdca1a
--- /dev/null
+++ b/tests/kola/ignition/systemd-enable-units/data/commonlib.sh
@@ -0,0 +1 @@
+../../../data/commonlib.sh
\ No newline at end of file
diff --git a/tests/kola/ignition/systemd-enable-units/test.sh b/tests/kola/ignition/systemd-enable-units/test.sh
new file mode 100755
index 0000000000..31c6a63a7b
--- /dev/null
+++ b/tests/kola/ignition/systemd-enable-units/test.sh
@@ -0,0 +1,31 @@
+#!/bin/bash
+## kola:
+##   tags: "platform-independent"
+##   description: Verify that Ignition supports to enable systemd units
+##     of different types.
+
+# See
+# - https://github.com/coreos/ignition/issues/586
+# - https://github.com/systemd/systemd/pull/9901
+
+set -xeuo pipefail
+
+# shellcheck disable=SC1091
+. "$KOLA_EXT_DATA/commonlib.sh"
+
+# make sure the presets worked and the instantiated unit is enabled
+if [ "$(systemctl is-enabled touch@foo.service)" != 'enabled' ]; then
+    fatal "touch@foo.service systemd unit should be enabled"
+fi
+ok "touch@foo.service systemd unit is enabled"
+
+# make sure the unit ran
+if ! test -e /run/foo; then
+    fatal "touch@foo.service didn't run as /run/foo does not exist"
+fi
+ok "touch@foo.service ran as /run/foo exists"
+
+if [ "$(systemctl is-enabled podman.socket)" != 'enabled' ]; then
+    fatal "podman.socket systemd unit should be enabled"
+fi
+ok "podman.socket systemd unit is enabled"
diff --git a/tests/kola/ignition/systemd-unmasking/config.fcc b/tests/kola/ignition/systemd-unmasking/config.bu
similarity index 100%
rename from tests/kola/ignition/systemd-unmasking/config.fcc
rename to tests/kola/ignition/systemd-unmasking/config.bu
diff --git a/tests/kola/ignition/systemd-unmasking/data/commonlib.sh b/tests/kola/ignition/systemd-unmasking/data/commonlib.sh
new file mode 120000
index 0000000000..b8dcbdca1a
--- /dev/null
+++ b/tests/kola/ignition/systemd-unmasking/data/commonlib.sh
@@ -0,0 +1 @@
+../../../data/commonlib.sh
\ No newline at end of file
diff --git a/tests/kola/ignition/systemd-unmasking/test.sh b/tests/kola/ignition/systemd-unmasking/test.sh
index 77c5e8f4be..e600060e10 100755
--- a/tests/kola/ignition/systemd-unmasking/test.sh
+++ b/tests/kola/ignition/systemd-unmasking/test.sh
@@ -1,21 +1,19 @@
 #!/bin/bash
-set -xeuo pipefail
-
-# This test makes sure that ignition is able to unmask units
-# It just so happens we have masked dnsmasq in FCOS so we can
-# test this by unmasking it.
+## kola:
+##   tags: "platform-independent"
+##   # This test is currently scoped to FCOS because `dnsmasq` is not masked on
+##   # RHCOS.
+##   # TODO-RHCOS: determine if any services on RHCOS are masked and adapt test
+##   distros: fcos
+##   description: Verify that Ignition supports to unmask units.
 
-ok() {
-    echo "ok" "$@"
-}
+set -xeuo pipefail
 
-fatal() {
-    echo "$@" >&2
-    exit 1
-}
+# shellcheck disable=SC1091
+. "$KOLA_EXT_DATA/commonlib.sh"
 
-# make sure the systemd unit (dnsmasq) is unmasked and enabled
-if [ $(systemctl is-enabled dnsmasq.service) != 'enabled' ]; then
+# Make sure the systemd unit (dnsmasq) is unmasked and enabled
+if [ "$(systemctl is-enabled dnsmasq.service)" != 'enabled' ]; then
     fatal "dnsmasq.service systemd unit should be unmasked and enabled"
 fi
 ok "dnsmasq.service systemd unit is unmasked and enabled"
diff --git a/tests/kola/k8s/node-e2e/config.bu b/tests/kola/k8s/node-e2e/config.bu
new file mode 100644
index 0000000000..653c7ad32e
--- /dev/null
+++ b/tests/kola/k8s/node-e2e/config.bu
@@ -0,0 +1,8 @@
+variant: fcos
+version: 1.3.0
+ignition:
+  config:
+    merge:
+      # XXX: what this config does should probably be folded into the test
+      # directly instead so that the release-specific cri-o is used
+      - source: https://raw.githubusercontent.com/kubernetes/test-infra/master/jobs/e2e_node/crio/crio_cgrpv2.ign
diff --git a/tests/kola/k8s/node-e2e/data/commonlib.sh b/tests/kola/k8s/node-e2e/data/commonlib.sh
new file mode 120000
index 0000000000..b8dcbdca1a
--- /dev/null
+++ b/tests/kola/k8s/node-e2e/data/commonlib.sh
@@ -0,0 +1 @@
+../../../data/commonlib.sh
\ No newline at end of file
diff --git a/tests/kola/k8s/node-e2e/data/node-e2e b/tests/kola/k8s/node-e2e/data/node-e2e
new file mode 100755
index 0000000000..580c73a57b
--- /dev/null
+++ b/tests/kola/k8s/node-e2e/data/node-e2e
@@ -0,0 +1,35 @@
+#!/bin/bash
+# TODO: Doc
+
+set -xeuo pipefail
+
+# shellcheck disable=SC1091
+. "$KOLA_EXT_DATA/commonlib.sh"
+
+SELF=$(realpath "$0")
+IMAGE=quay.io/projectquay/golang:1.17
+
+branch=$1; shift
+
+if [ -z "${container:-}" ]; then
+    mkdir -m 0600 ~/.ssh
+    ssh-keygen -N "" -t ed25519 -f /srv/kube
+    cat /srv/kube.pub >> ~/.ssh/authorized_keys
+    chmod 0600 ~/.ssh/*
+    exec podman run --net=host --rm -v /srv/kube:/srv/kube:z \
+        -v "${KOLA_EXT_DATA}:/srv/kola_ext_data:z" \
+        -v "${SELF}:/srv/self:z" "${IMAGE}" /srv/self "${branch}"
+    fatal "unreachable"
+fi
+
+# in container now
+dnf install -y rsync diffutils hostname
+git clone -b "${branch}" --depth 1 https://github.com/kubernetes/kubernetes
+make -C kubernetes test-e2e-node REMOTE=true REMOTE_MODE=ssh SSH_USER=root \
+    RUNTIME=remote CONTAINER_RUNTIME_ENDPOINT="unix:///var/run/crio/crio.sock" \
+    SSH_KEY=/srv/kube HOSTS=localhost FOCUS="\[NodeConformance\]" \
+    SSH_OPTIONS='-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' \
+    TEST_ARGS='--kubelet-flags="--cgroup-driver=systemd --cgroups-per-qos=true
+                                --cgroup-root=/ --non-masquerade-cidr=0.0.0.0/0
+                                --runtime-cgroups=/system.slice/crio.service
+                                --kubelet-cgroups=/system.slice/kubelet.service"'
diff --git a/tests/kola/k8s/node-e2e/master b/tests/kola/k8s/node-e2e/master
new file mode 100755
index 0000000000..ec2dd3baa2
--- /dev/null
+++ b/tests/kola/k8s/node-e2e/master
@@ -0,0 +1,15 @@
+#!/bin/bash
+## kola:
+##   minMemory: 4096
+##   timeoutMin: 45
+##   minDisk: 20
+##   # Just run on x86_64 for now. We'd need multi-arch images with go 1.17.
+##   architectures: x86_64
+##   requiredTag: k8s
+##   tags: needs-internet
+##   description: Verify the Kubernetes e2e node tests work, and
+##     ensure that we don't inadvertedly break Kubernetes.
+
+set -xeuo pipefail
+
+exec "${KOLA_EXT_DATA}/node-e2e" "${KOLA_TEST_EXE}"
diff --git a/tests/kola/kdump/crash/config.bu b/tests/kola/kdump/crash/config.bu
new file mode 100644
index 0000000000..1a45c87ec5
--- /dev/null
+++ b/tests/kola/kdump/crash/config.bu
@@ -0,0 +1,19 @@
+variant: fcos
+version: 1.4.0
+kernel_arguments:
+  should_exist:
+    # We need to make sure we have a large enough crashkernel for FCOS
+    # and RHCOS here. Currently the worst case output of `kdumpctl estimate`
+    # is aarch64 RHCOS where the it says "Recommended crashkernel: 448M".
+    # Though for some reason when we set crashkernel=448M ppc64le complains
+    # and wants 512M so let's set it to 512M here.
+    - crashkernel=512M
+systemd:
+  units:
+    - name: kdump.service
+      enabled: true
+      dropins:
+        - name: debug.conf
+          contents: |
+            [Service]
+            Environment="debug=1"
diff --git a/tests/kola/kdump/crash/data/commonlib.sh b/tests/kola/kdump/crash/data/commonlib.sh
new file mode 120000
index 0000000000..b8dcbdca1a
--- /dev/null
+++ b/tests/kola/kdump/crash/data/commonlib.sh
@@ -0,0 +1 @@
+../../../data/commonlib.sh
\ No newline at end of file
diff --git a/tests/kola/kdump/crash/test.sh b/tests/kola/kdump/crash/test.sh
new file mode 100755
index 0000000000..1fa0978712
--- /dev/null
+++ b/tests/kola/kdump/crash/test.sh
@@ -0,0 +1,57 @@
+#!/bin/bash
+## kola:
+##   # Testing kdump requires some reserved memory for the crashkernel.
+##   minMemory: 4096
+##   # Skip checks for things like kernel crashes in the console logs.
+##   # For this test we trigger a kernel crash on purpose.
+##   tags: skip-base-checks
+##   # This test includes a few reboots and the generation of a vmcore,
+##   # which can take longer than the default 10 minute timeout.
+##   timeoutMin: 15
+##   description: Verify that the crashkernel reserved memory is large enough.
+
+# See https://docs.fedoraproject.org/en-US/fedora-coreos/debugging-kernel-crashes/
+
+set -xeuo pipefail
+
+# shellcheck disable=SC1091
+. "$KOLA_EXT_DATA/commonlib.sh"
+
+case "${AUTOPKGTEST_REBOOT_MARK:-}" in
+  "")
+      # use 120s for this since kdump can take a while to build its initramfs,
+      # especially if the system is loaded
+      if ! is_service_active kdump.service 120; then
+          fatal "kdump.service failed to start"
+      fi
+      # Verify that the crashkernel reserved memory is large enough
+      output=$(kdumpctl estimate)
+      if grep -q "WARNING: Current crashkernel size is lower than recommended size" <<< "$output"; then
+          fatal "The reserved crashkernel size is lower than recommended."
+      fi
+      /tmp/autopkgtest-reboot-prepare aftercrash
+      # Add in a sleep to workaround race condition where XFS/kernel errors happen
+      # during crash kernel boot.
+      # https://github.com/coreos/fedora-coreos-tracker/issues/1195
+      sleep 5
+      echo "Triggering sysrq"
+      sync
+      echo 1 > /proc/sys/kernel/sysrq
+      # This one will trigger kdump, which will write the kernel core, then reboot.
+      echo c > /proc/sysrq-trigger
+      # We shouldn't reach this point
+      sleep 5
+      fatal "failed to invoke sysrq"
+      ;;
+  aftercrash)
+      kcore=$(find /var/crash -type f -name vmcore)
+      if test -z "${kcore}"; then
+        fatal "No kcore found in /var/crash"
+      fi
+      info=$(file "${kcore}")
+      if ! [[ "${info}" =~ 'vmcore: Kdump'.*'system Linux' ]]; then
+        fatal "vmcore does not appear to be a Kdump?"
+      fi
+      ;;
+  *) fatal "unexpected mark: ${AUTOPKGTEST_REBOOT_MARK}";;
+esac
diff --git a/tests/kola/kdump/data/commonlib.sh b/tests/kola/kdump/data/commonlib.sh
new file mode 120000
index 0000000000..1742d51e67
--- /dev/null
+++ b/tests/kola/kdump/data/commonlib.sh
@@ -0,0 +1 @@
+../../data/commonlib.sh
\ No newline at end of file
diff --git a/tests/kola/kdump/service b/tests/kola/kdump/service
new file mode 100755
index 0000000000..f6893b06b9
--- /dev/null
+++ b/tests/kola/kdump/service
@@ -0,0 +1,16 @@
+#!/bin/bash
+## kola:
+##   exclusive: false
+##   description: Verify that kdump didn't start by default.
+
+# It's either disabled, or enabled but conditional on crashkernel= karg,
+# which we don't bake.
+
+set -xeuo pipefail
+
+# shellcheck disable=SC1091
+. "$KOLA_EXT_DATA/commonlib.sh"
+
+if ! systemctl show -p ActiveState kdump.service | grep -q ActiveState=inactive; then
+    fatal "Unit kdump.service shouldn't be active"
+fi
diff --git a/tests/kola/kubernetes/kube-watch/config.bu b/tests/kola/kubernetes/kube-watch/config.bu
new file mode 100644
index 0000000000..c139236c7f
--- /dev/null
+++ b/tests/kola/kubernetes/kube-watch/config.bu
@@ -0,0 +1,34 @@
+variant: fcos
+version: 1.2.0
+storage:
+  directories:
+    # This is for verifying that `kubernetes_file_t` labeled files can be
+    # watched by systemd
+    # See: https://github.com/coreos/fedora-coreos-tracker/issues/861
+    # See: https://github.com/containers/container-selinux/issues/135
+    - path: /etc/kubernetes
+systemd:
+  units:
+    - name: kube-watch.service
+      # This is for verifying that `kubernetes_file_t` labeled files can be
+      # watched by systemd
+      # See: https://github.com/coreos/fedora-coreos-tracker/issues/861
+      # See: https://github.com/containers/container-selinux/issues/135
+      contents: |
+        [Service]
+        ExecStart=/usr/bin/touch /var/tmp/kube-watched
+        RemainAfterExit=yes
+        Type=oneshot
+        [Install]
+        WantedBy=multi-user.target
+    - name: kube-watch.path
+      # This is for verifying that `kubernetes_file_t` labeled files can be
+      # watched by systemd
+      # See: https://github.com/coreos/fedora-coreos-tracker/issues/861
+      # See: https://github.com/containers/container-selinux/issues/135
+      enabled: true
+      contents: |
+        [Path]
+        PathExists=/etc/kubernetes/kubeconfig
+        [Install]
+        WantedBy=multi-user.target
diff --git a/tests/kola/kubernetes/kube-watch/data/commonlib.sh b/tests/kola/kubernetes/kube-watch/data/commonlib.sh
new file mode 120000
index 0000000000..b8dcbdca1a
--- /dev/null
+++ b/tests/kola/kubernetes/kube-watch/data/commonlib.sh
@@ -0,0 +1 @@
+../../../data/commonlib.sh
\ No newline at end of file
diff --git a/tests/kola/kubernetes/kube-watch/test.sh b/tests/kola/kubernetes/kube-watch/test.sh
new file mode 100755
index 0000000000..eb29e3b018
--- /dev/null
+++ b/tests/kola/kubernetes/kube-watch/test.sh
@@ -0,0 +1,42 @@
+#!/bin/bash
+## kola:
+##   exclusive: false
+##   description: Verify that `kubernetes_file_t` labeled files can be
+##     watched by systemd.
+
+# See
+# - https://github.com/coreos/fedora-coreos-tracker/issues/861
+# - https://github.com/containers/container-selinux/issues/135
+
+set -xeuo pipefail
+
+# shellcheck disable=SC1091
+. "$KOLA_EXT_DATA/commonlib.sh"
+
+if [ "$(systemctl is-active kube-watch.path)" != "active" ]; then
+    fatal "kube-watch.path did not activate successfully"
+fi
+ok "kube-watch.path successfully activated"
+
+touch /etc/kubernetes/kubeconfig
+ok "successfully created /etc/kubernetes/kubeconfig"
+
+# Give the service 30 seconds to activate
+active=0
+for i in {1..30}; do
+    if systemctl is-active kube-watch.service; then
+        active=1
+        break
+    fi
+    sleep 1
+done
+if [[ $active != 1 ]]; then
+    systemctl status kube-watch.service
+    fatal "kube-watch.service did not successfully activate"
+fi
+ok "kube-watch.service activated successfully"
+
+if ! test -e /var/tmp/kube-watched; then
+    fatal "/var/tmp/kube-watched does not exist"
+fi
+ok "/var/tmp/kube-watched exists"
diff --git a/tests/kola/kubernetes/systemd-env-read/config.bu b/tests/kola/kubernetes/systemd-env-read/config.bu
new file mode 100644
index 0000000000..84f5bd89e7
--- /dev/null
+++ b/tests/kola/kubernetes/systemd-env-read/config.bu
@@ -0,0 +1,27 @@
+variant: fcos
+version: 1.2.0
+storage:
+  files:
+    - path: /etc/kubernetes/envfile
+      # This is for verifying that `kubernetes_file_t` labeled files can be
+      # read by systemd
+      # See: https://bugzilla.redhat.com/show_bug.cgi?id=1973418
+      mode: 0644
+      contents:
+        inline: |
+          KUBE="FCOS"
+systemd:
+  units:
+    - name: kube-env.service
+      # This is for verifying that `kubernetes_file_t` labeled files can be
+      # read by systemd
+      # See: https://bugzilla.redhat.com/show_bug.cgi?id=1973418
+      enabled: true
+      contents: |
+        [Service]
+        EnvironmentFile=/etc/kubernetes/envfile
+        ExecStart=/usr/bin/echo ${KUBE}
+        RemainAfterExit=yes
+        Type=oneshot
+        [Install]
+        WantedBy=multi-user.target
diff --git a/tests/kola/kubernetes/systemd-env-read/data/commonlib.sh b/tests/kola/kubernetes/systemd-env-read/data/commonlib.sh
new file mode 120000
index 0000000000..b8dcbdca1a
--- /dev/null
+++ b/tests/kola/kubernetes/systemd-env-read/data/commonlib.sh
@@ -0,0 +1 @@
+../../../data/commonlib.sh
\ No newline at end of file
diff --git a/tests/kola/kubernetes/systemd-env-read/test.sh b/tests/kola/kubernetes/systemd-env-read/test.sh
new file mode 100755
index 0000000000..849177dd77
--- /dev/null
+++ b/tests/kola/kubernetes/systemd-env-read/test.sh
@@ -0,0 +1,29 @@
+#!/bin/bash
+## kola:
+##   exclusive: false
+##   description: Verify that `kubernetes_file_t` labeled files can be read
+##     by systemd, also verify the `kube-env` service started successfully
+##     and the service wrote to the journal successfully.
+
+# See https://bugzilla.redhat.com/show_bug.cgi?id=1973418
+
+set -xeuo pipefail
+
+# shellcheck disable=SC1091
+. "$KOLA_EXT_DATA/commonlib.sh"
+
+if [ "$( stat -c %C /etc/kubernetes/envfile)" != "system_u:object_r:kubernetes_file_t:s0" ]; then
+    fatal "/etc/kubernetes/envfile is labeled incorrectly"
+fi
+ok "/etc/kubernetes/envfile is labeled correctly"
+
+if [ "$(systemctl is-failed kube-env.service)" != "active" ]; then
+    fatal "kube-env.service failed unexpectedly"
+fi
+ok "kube-env.service successfully started"
+
+# Verify that 'FCOS' was wrtitten to the journal
+if ! journalctl -o cat -u kube-env.service | grep FCOS; then
+    fatal "kube-env.service did not write 'FCOS' to journal"
+fi
+ok "kube-env.service ran and wrote 'FCOS' to the journal"
diff --git a/tests/kola/logging/data/commonlib.sh b/tests/kola/logging/data/commonlib.sh
new file mode 120000
index 0000000000..1742d51e67
--- /dev/null
+++ b/tests/kola/logging/data/commonlib.sh
@@ -0,0 +1 @@
+../../data/commonlib.sh
\ No newline at end of file
diff --git a/tests/kola/logging/printk b/tests/kola/logging/printk
new file mode 100755
index 0000000000..cd9e126d39
--- /dev/null
+++ b/tests/kola/logging/printk
@@ -0,0 +1,19 @@
+#!/bin/bash
+## kola:
+##   exclusive: false
+##   description: Verify default console log level is 4.
+
+# See https://github.com/coreos/fedora-coreos-tracker/issues/1244
+
+set -xeuo pipefail
+
+# shellcheck disable=SC1091
+. "$KOLA_EXT_DATA/commonlib.sh"
+
+printk=$(cat /proc/sys/kernel/printk)
+
+if ! [[ "$printk" =~ ^4 ]]; then
+    fatal "printk: expected console log level 4, found ${printk}"
+fi
+
+ok "Found expected printk value "
diff --git a/tests/kola/misc-ign-ro/test.sh b/tests/kola/misc-ign-ro/test.sh
deleted file mode 100755
index 6321fc7207..0000000000
--- a/tests/kola/misc-ign-ro/test.sh
+++ /dev/null
@@ -1,38 +0,0 @@
-#!/bin/bash
-set -xeuo pipefail
-
-ok() {
-        echo "ok" "$@"
-    }
-
-fatal() {
-        echo "$@" >&2
-            exit 1
-        }
-
-# This test makes sure that swap on zram devices can be set up
-# using the zram-generator as defined in the docs at
-# https://docs.fedoraproject.org/en-US/fedora-coreos/sysconfig-configure-swaponzram/
-
-if ! grep -q 'zram0' /proc/swaps; then
-    fatal "expected zram0 to be set up"
-fi
-ok "swap on zram was set up correctly"
-
-# Make sure that coreos-update-ca-trust kicked in and observe the result.
-if ! systemctl show coreos-update-ca-trust.service -p ActiveState | grep ActiveState=active; then
-    fatal "coreos-update-ca-trust.service not active"
-fi
-if ! grep '^# coreos.com$' /etc/pki/ca-trust/extracted/openssl/ca-bundle.trust.crt; then
-    fatal "expected coreos.com in ca-bundle"
-fi
-ok "coreos-update-ca-trust.service"
-
-# Make sure that the stub-resolv.conf file has the correct selinux context.
-# https://github.com/fedora-selinux/selinux-policy/pull/509#issuecomment-744540382
-# https://github.com/systemd/systemd/pull/17976
-context=$(stat --format "%C" /run/systemd/resolve/stub-resolv.conf)
-if [ "$context" != "system_u:object_r:net_conf_t:s0" ]; then
-    fatal "SELinux context on stub-resolv.conf is wrong"
-fi
-ok "SELinux context on stub-resolv.conf is correct"
diff --git a/tests/kola/misc-ro b/tests/kola/misc-ro
deleted file mode 100755
index 7fcfab8a28..0000000000
--- a/tests/kola/misc-ro
+++ /dev/null
@@ -1,144 +0,0 @@
-#!/bin/bash
-# This is a place to put random quick read-only tests.
-set -xeuo pipefail
-
-ok() {
-    echo "ok" "$@"
-}
-
-fatal() {
-    echo "$@" >&2
-    exit 1
-}
-
-on_platform() {
-    grep -q " ignition.platform.id=$1 " /proc/cmdline
-}
-
-get_journal_msg_timestamp() {
-    journalctl -o json -b 0 --grep "$1" \
-        | jq -r --slurp '.[0]["__MONOTONIC_TIMESTAMP"]'
-}
-
-# Test some services are enabled or disabled appropriately
-for unit in logrotate; do
-    if ! systemctl is-enabled ${unit} 1>/dev/null; then
-        fatal "Unit ${unit} should be enabled"
-    fi
-done
-# Make sure that kdump didn't start (it's either disabled, or enabled but
-# conditional on crashkernel= karg, which we don't bake).
-if ! systemctl show -p ActiveState kdump.service | grep -q ActiveState=inactive; then
-    fatal "Unit kdump.service shouldn't be active"
-fi
-# systemd-resolved should be disabled on f32 but
-# enabled on f33+.
-source /etc/os-release
-if systemctl is-enabled systemd-resolved 1>/dev/null; then
-    if [ "$VERSION_ID" == "32" ]; then
-        fatal "Unit systemd-resolved should not be enabled"
-    fi
-else
-    if [ "$VERSION_ID" != "32" ]; then
-        fatal "Unit systemd-resolved should be enabled"
-    fi
-fi
-ok services
-
-# https://github.com/coreos/fedora-coreos-config/commit/2a5c2abc796ac645d705700bf445b50d4cda8f5f
-if ip link | grep -o -e " eth[0-9]:"; then
-    fatal "detected eth* NIC naming on node"
-fi
-ok nic naming
-
-if test -w /sysroot; then
-    fatal "found writable /sysroot"
-fi
-ok sysroot ro
-
-if ! lsattr -d / | grep -qe '--i--'; then
-    fatal "missing immutable bit on /"
-fi
-ok immutable bit
-
-# See remove-files in the manifest
-if test -d /usr/share/info; then
-    fatal "found /usr/share/info"
-fi
-
-# See https://github.com/coreos/coreos-assembler/pull/1786
-path=/usr/lib/systemd/system-generators/coreos-platform-chrony
-mode=$(stat -c '%a' ${path})
-if test "${mode}" != 555; then
-    fatal "For path ${path} expected mode 555, found ${mode}"
-fi
-
-switchroot_ts=$(get_journal_msg_timestamp 'Switching root.')
-nm_ts=$(get_journal_msg_timestamp 'NetworkManager .* starting')
-# by default, kola on QEMU shouldn't need to bring up networking
-# https://github.com/coreos/fedora-coreos-config/pull/426
-if [[ $nm_ts -lt $switchroot_ts ]] && on_platform qemu; then
-    fatal "NetworkManager started in initramfs!"
-# and as a sanity-check that this test works, verify that on AWS
-# we did bring up networking in the initrd
-elif [[ $nm_ts -gt $switchroot_ts ]] && on_platform aws; then
-    fatal "NetworkManager not started in initramfs!"
-fi
-ok conditional initrd networking
-
-if ! test -f /usr/share/licenses/fedora-coreos-config/LICENSE; then
-    fatal missing LICENSE
-fi
-ok LICENSE
-
-case "$(arch)" in
-    x86_64|aarch64)
-        if runuser -u core -- ls /boot/efi &>/dev/null; then
-            fatal "Was able to access /boot/efi as non-root"
-        fi
-        # This is just a basic sanity check; at some point we
-        # will implement "project-owned tests run in the pipeline"
-        # and be able to run the existing bootupd tests:
-        # https://github.com/coreos/fedora-coreos-config/pull/677
-        bootupctl status
-        ok bootupctl
-        ;;
-esac
-
-# check that no files are unlabeled
-unlabeled=$(find /var /etc -context '*:unlabeled_t:*')
-if [ -n "${unlabeled}" ]; then
-    echo "Found unlabeled files:"
-    echo "${unlabeled}"
-    exit 1
-fi
-ok no files with unlabeled_t SELinux label
-
-# make sure we stick with bdb until we're ready to move to sqlite
-# https://github.com/coreos/fedora-coreos-tracker/issues/623
-if [ ! -f /usr/share/rpm/Packages ]; then
-    fatal "Didn't find bdb file /usr/share/rpm/Packages"
-fi
-ok rpmdb is bdb
-
-# make sure we don't default to having swap on zram
-# https://github.com/coreos/fedora-coreos-tracker/issues/509
-# https://github.com/coreos/fedora-coreos-config/pull/687
-if [ -e /dev/zram0 ]; then
-    fatal "zram0 swap device set up on default install"
-fi
-ok no zram swap by default
-
-# make sure dnsmasq is masked
-# https://github.com/coreos/fedora-coreos-tracker/issues/519#issuecomment-705140528
-if [ $(systemctl is-enabled dnsmasq.service) != 'masked' ]; then
-    fatal "dnsmasq.service systemd unit should be masked"
-fi
-ok "dnsmasq.service systemd unit is masked"
-
-# make sure systemd-repart is masked
-# https://github.com/coreos/fedora-coreos-config/pull/744
-if [ $(systemctl is-enabled systemd-repart.service) != 'masked' ]; then
-    fatal "systemd-repart.service systemd unit should be masked"
-fi
-ok "systemd-repart.service systemd unit is masked"
diff --git a/tests/kola/networking/bridge-static-via-kargs b/tests/kola/networking/bridge-static-via-kargs
new file mode 100755
index 0000000000..478d7639bd
--- /dev/null
+++ b/tests/kola/networking/bridge-static-via-kargs
@@ -0,0 +1,26 @@
+#!/bin/bash
+## kola:
+##   # additionalNics is only supported on QEMU
+##   platforms: qemu
+##   # Add 2 NIC for this test
+##   additionalNics: 2
+##   # Configuration of the 2 NICs for this test
+##   # We use net.ifnames=0 to disable consistent network naming here because on
+##   # different firmwares (BIOS vs UEFI) the NIC names are different.
+##   # See https://github.com/coreos/fedora-coreos-tracker/issues/1060
+##   appendKernelArgs: "ip=10.10.10.10::10.10.10.1:255.255.255.0:mybridge:br0:none bridge=br0:eth1,eth2 net.ifnames=0"
+##   description: Verify bridge networks works via kernel arguments.
+
+set -xeuo pipefail
+
+# shellcheck disable=SC1091
+. "$KOLA_EXT_DATA/commonlib.sh"
+
+bridge="br0"
+
+# Verify "br0" gets ip address by `appendKernelArgs` in `kola` above
+nic_ip=$(get_ipv4_for_nic ${bridge})
+if [ "${nic_ip}" != "10.10.10.10" ]; then
+    fatal "Error: get ${bridge} ip = ${nic_ip}, expected is 10.10.10.10"
+fi
+ok "get ${bridge} ip is 10.10.10.10"
diff --git a/tests/kola/networking/data/commonlib.sh b/tests/kola/networking/data/commonlib.sh
new file mode 120000
index 0000000000..1742d51e67
--- /dev/null
+++ b/tests/kola/networking/data/commonlib.sh
@@ -0,0 +1 @@
+../../data/commonlib.sh
\ No newline at end of file
diff --git a/tests/kola/networking/default-network-behavior-change/data/commonlib.sh b/tests/kola/networking/default-network-behavior-change/data/commonlib.sh
new file mode 120000
index 0000000000..b8dcbdca1a
--- /dev/null
+++ b/tests/kola/networking/default-network-behavior-change/data/commonlib.sh
@@ -0,0 +1 @@
+../../../data/commonlib.sh
\ No newline at end of file
diff --git a/tests/kola/networking/default-network-behavior-change/test.sh b/tests/kola/networking/default-network-behavior-change/test.sh
new file mode 100755
index 0000000000..540ea1ea5f
--- /dev/null
+++ b/tests/kola/networking/default-network-behavior-change/test.sh
@@ -0,0 +1,209 @@
+#!/bin/bash
+## kola:
+##   tags: "platform-independent"
+##   exclusive: false
+##   description: Verify the default networking configurations match expected
+##     results.
+#
+# Since we depend so much on the default networking configurations let's
+# alert ourselves when any default networking configuration changes in
+# NetworkManager. This allows us to react and adjust to the changes
+# (if needed) instead of finding out later that problems were introduced.
+# some context in: https://github.com/coreos/fedora-coreos-tracker/issues/1000
+
+set -xeuo pipefail
+
+# shellcheck disable=SC1091
+. "$KOLA_EXT_DATA/commonlib.sh"
+
+# EXPECTED_INITRD_NETWORK_CFG2
+#   - used on older RHEL 8.4 release
+EXPECTED_INITRD_NETWORK_CFG2="[connection]
+id=Wired Connection
+uuid=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
+type=ethernet
+autoconnect-retries=1
+multi-connect=3
+permissions=
+[ethernet]
+mac-address-blacklist=
+[ipv4]
+dhcp-timeout=90
+dns-search=
+method=auto
+required-timeout=20000
+[ipv6]
+addr-gen-mode=eui64
+dhcp-timeout=90
+dns-search=
+method=auto
+[proxy]"
+# EXPECTED_INITRD_NETWORK_CFG3
+#   - used on Fedora 36+ and RHEL8.6+
+EXPECTED_INITRD_NETWORK_CFG3="[connection]
+id=Wired Connection
+uuid=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
+type=ethernet
+autoconnect-retries=1
+multi-connect=3
+[ethernet]
+[ipv4]
+dhcp-timeout=90
+method=auto
+required-timeout=20000
+[ipv6]
+addr-gen-mode=eui64
+dhcp-timeout=90
+method=auto
+[proxy]
+[user]
+org.freedesktop.NetworkManager.origin=nm-initrd-generator"
+# EXPECTED_INITRD_NETWORK_CFG5
+#   - used on Fedora 37+, scos and RHEL 9.2
+EXPECTED_INITRD_NETWORK_CFG5="# Created by nm-initrd-generator
+[connection]
+id=Wired Connection
+uuid=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
+type=ethernet
+autoconnect-priority=-100
+autoconnect-retries=1
+multi-connect=3
+[ethernet]
+[ipv4]
+dhcp-timeout=90
+method=auto
+required-timeout=20000
+[ipv6]
+dhcp-timeout=90
+method=auto
+[proxy]
+[user]
+org.freedesktop.NetworkManager.origin=nm-initrd-generator"
+
+# EXPECTED_REALROOT_NETWORK_CFG1:
+#   - used on RHEL <= 8.5
+EXPECTED_REALROOT_NETWORK_CFG1="[connection]
+id=Wired connection 1
+uuid=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
+type=ethernet
+autoconnect-priority=-999
+interface-name=xxxx
+permissions=
+timestamp=xxxxxxxxxx
+[ethernet]
+mac-address-blacklist=
+[ipv4]
+dns-search=
+method=auto
+[ipv6]
+addr-gen-mode=stable-privacy
+dns-search=
+method=auto
+[proxy]
+[.nmmeta]
+nm-generated=true"
+# EXPECTED_REALROOT_NETWORK_CFG2:
+#   - used on all Fedora 36+ and RHEL8.6+
+EXPECTED_REALROOT_NETWORK_CFG2="[connection]
+id=Wired connection 1
+uuid=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
+type=ethernet
+autoconnect-priority=-999
+interface-name=xxxx
+timestamp=xxxxxxxxxx
+[ethernet]
+[ipv4]
+method=auto
+[ipv6]
+addr-gen-mode=stable-privacy
+method=auto
+[proxy]
+[.nmmeta]
+nm-generated=true"
+# EXPECTED_REALROOT_NETWORK_CFG3:
+#   - used on all Fedora 37+, scos and RHEL 9.2
+EXPECTED_REALROOT_NETWORK_CFG3="[connection]
+id=Wired connection 1
+uuid=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
+type=ethernet
+autoconnect-priority=-999
+interface-name=xxxx
+timestamp=xxxxxxxxxx
+[ethernet]
+[ipv4]
+method=auto
+[ipv6]
+addr-gen-mode=default
+method=auto
+[proxy]
+[.nmmeta]
+nm-generated=true"
+
+# Function that will remove unique (per-run) data from a connection file
+# and also delete any blank lines.
+normalize_connection_file() {
+    sed -e s/^uuid=.*$/uuid=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx/ \
+        -e s/^timestamp=.*$/timestamp=xxxxxxxxxx/                 \
+        -e s/^interface-name=.*$/interface-name=xxxx/             \
+        -e /^$/d                                                  \
+        "${1}"
+}
+
+source /etc/os-release
+if [ "$ID" == "fedora" ]; then
+    if [ "$VERSION_ID" -ge "37" ]; then
+        EXPECTED_INITRD_NETWORK_CFG=$EXPECTED_INITRD_NETWORK_CFG5
+        EXPECTED_REALROOT_NETWORK_CFG=$EXPECTED_REALROOT_NETWORK_CFG3
+    elif [ "$VERSION_ID" -eq "36" ]; then
+        EXPECTED_INITRD_NETWORK_CFG=$EXPECTED_INITRD_NETWORK_CFG3
+        EXPECTED_REALROOT_NETWORK_CFG=$EXPECTED_REALROOT_NETWORK_CFG2
+    else
+        fatal "fail: not operating on expected OS version"
+    fi
+elif [[ "${ID}" = "rhel" ]] || [[ "${ID_LIKE}" =~ "rhel" ]]; then
+    # For the version comparison use string substitution to remove the
+    # '.` from the version so we can use integer comparison
+
+    if is_scos || is_rhcos9; then
+        EXPECTED_INITRD_NETWORK_CFG=$EXPECTED_INITRD_NETWORK_CFG5
+        EXPECTED_REALROOT_NETWORK_CFG=$EXPECTED_REALROOT_NETWORK_CFG3
+    elif [ "${RHEL_VERSION/\./}" -ge 86 ]; then
+        EXPECTED_INITRD_NETWORK_CFG=$EXPECTED_INITRD_NETWORK_CFG3
+        EXPECTED_REALROOT_NETWORK_CFG=$EXPECTED_REALROOT_NETWORK_CFG2
+    elif [ "${RHEL_VERSION/\./}" -eq 84 ]; then
+        EXPECTED_INITRD_NETWORK_CFG=$EXPECTED_INITRD_NETWORK_CFG2
+        EXPECTED_REALROOT_NETWORK_CFG=$EXPECTED_REALROOT_NETWORK_CFG1
+    else
+        fatal "fail: not operating on expected OS version"
+    fi
+else
+    fatal "fail: not operating on expected OS"
+fi
+
+
+# Execute nm-initrd-generator against our default kargs (defined by
+# afterburn drop in) to get the generated initrd network config.
+DEFAULT_KARGS_FILE=/usr/lib/dracut/modules.d/35coreos-network/50-afterburn-network-kargs-default.conf
+source <(grep -o 'AFTERBURN_NETWORK_KARGS_DEFAULT=.*' $DEFAULT_KARGS_FILE)
+tmpdir=$(mktemp -d)
+/usr/libexec/nm-initrd-generator \
+    -c "${tmpdir}/connections" \
+    -i "${tmpdir}/initrd-data-dir" \
+    -r "${tmpdir}/conf.d" \
+    -- $AFTERBURN_NETWORK_KARGS_DEFAULT
+GENERATED_INITRD_NETWORK_CFG=$(normalize_connection_file \
+                               "${tmpdir}/connections/default_connection.nmconnection")
+
+# Diff the outputs and fail if the expected doesn't match the generated.
+if ! diff -u <(echo "$EXPECTED_INITRD_NETWORK_CFG") <(echo "$GENERATED_INITRD_NETWORK_CFG"); then
+    fatal "fail: the expected initrd network config is not given by the kargs"
+fi
+
+# Check the default NetworkManager runtime generated connection profile in
+# the real root to make sure it matches what we expect.
+GENERATED_REALROOT_NETWORK_CFG=$(normalize_connection_file \
+                                 <(sudo cat "/run/NetworkManager/system-connections/Wired connection 1.nmconnection"))
+if ! diff -u <(echo "$EXPECTED_REALROOT_NETWORK_CFG") <(echo "$GENERATED_REALROOT_NETWORK_CFG"); then
+    fatal "fail: the expected realroot network config is not given by the kargs"
+fi
+ok "success: expected network configs were generated"
diff --git a/tests/kola/networking/dnsmasq-service b/tests/kola/networking/dnsmasq-service
new file mode 100755
index 0000000000..3700592a60
--- /dev/null
+++ b/tests/kola/networking/dnsmasq-service
@@ -0,0 +1,20 @@
+#!/bin/bash
+## kola:
+##   # This test only runs on FCOS because the `dnsmasq` service on RHCOS is simply
+##   # disabled, not masked.
+##   # TODO-RHCOS: adapt test to check that `dnsmasq` is in the proper state on RHCOS
+##   distros: fcos
+##   exclusive: false
+##   description: Verify dnsmasq service is masked.
+
+# See https://github.com/coreos/fedora-coreos-tracker/issues/519#issuecomment-705140528
+
+set -xeuo pipefail
+
+# shellcheck disable=SC1091
+. "$KOLA_EXT_DATA/commonlib.sh"
+
+if [ "$(systemctl is-enabled dnsmasq.service)" != 'masked' ]; then
+    fatal "dnsmasq.service systemd unit should be masked"
+fi
+ok "dnsmasq.service systemd unit is masked"
diff --git a/tests/kola/networking/force-persist-ip/config.bu b/tests/kola/networking/force-persist-ip/config.bu
new file mode 100644
index 0000000000..6897419b2e
--- /dev/null
+++ b/tests/kola/networking/force-persist-ip/config.bu
@@ -0,0 +1,16 @@
+variant: fcos
+version: 1.4.0
+storage:
+  files:
+    - path: /etc/NetworkManager/system-connections/eth1.nmconnection
+      mode: 0600
+      contents:
+        inline: |
+          [connection]
+          id=eth1
+          type=ethernet
+          interface-name=eth1
+          [ipv4]
+          dns-search=
+          may-fail=false
+          method=auto
diff --git a/tests/kola/networking/force-persist-ip/data/commonlib.sh b/tests/kola/networking/force-persist-ip/data/commonlib.sh
new file mode 120000
index 0000000000..b8dcbdca1a
--- /dev/null
+++ b/tests/kola/networking/force-persist-ip/data/commonlib.sh
@@ -0,0 +1 @@
+../../../data/commonlib.sh
\ No newline at end of file
diff --git a/tests/kola/networking/force-persist-ip/test.sh b/tests/kola/networking/force-persist-ip/test.sh
new file mode 100755
index 0000000000..bdecb10cf8
--- /dev/null
+++ b/tests/kola/networking/force-persist-ip/test.sh
@@ -0,0 +1,47 @@
+#!/bin/bash
+## kola:
+##   # additionalNics is only supported on QEMU
+##   platforms: qemu
+##   # Add 1 NIC for this test
+##   additionalNics: 1
+##   # The functionality we're testing here and the configuration for the NIC
+##   # We use net.ifnames=0 to disable consistent network naming here because on
+##   # different firmwares (BIOS vs UEFI) the NIC names are different.
+##   # See https://github.com/coreos/fedora-coreos-tracker/issues/1060
+##   appendKernelArgs: "ip=10.10.10.10::10.10.10.1:255.255.255.0:myhostname:eth1:none:8.8.8.8 net.ifnames=0 coreos.force_persist_ip"
+##   description: Verify that coreos.force_persist_ip will force propagating
+##     kernel argument based networking configuration into the real root.
+
+# Setup configuration for a single NIC with two different ways:
+# - kargs provide static network config for eth1 and also coreos.force_persist_ip
+# - Ignition provides dhcp network config for eth1
+# Expected result:
+# - with coreos.force_persist_ip ip=kargs win, verify that
+#   eth1 has the static IP address via kargs
+# https://bugzilla.redhat.com/show_bug.cgi?id=1958930#c29
+
+set -xeuo pipefail
+
+# shellcheck disable=SC1091
+. "$KOLA_EXT_DATA/commonlib.sh"
+
+# Verify eth1 gets staic ip via kargs
+nic_name="eth1"
+nic_ip=$(get_ipv4_for_nic ${nic_name})
+if [ ${nic_ip} != "10.10.10.10" ]; then
+    fatal "Error: get ${nic_name} ip = ${nic_ip}, expected is 10.10.10.10"
+fi
+ok "get ${nic_name} ip is 10.10.10.10"
+
+syscon="/etc/NetworkManager/system-connections"
+if [ ! -f "${syscon}/${nic_name}.nmconnection" ]; then
+    fatal "Error: can not find ${syscon}/${nic_name}.nmconnection"
+fi
+ok "find ${syscon}/${nic_name}.nmconnection"
+
+# Verify logs
+if ! journalctl -b 0 -u coreos-teardown-initramfs | \
+     grep -q "info: coreos.force_persist_ip detected: will force network config propagation"; then
+  fatal "Error: force network config propagation not work"
+fi
+ok "force network config propagation"
diff --git a/tests/kola/networking/hostname/fallback-hostname/config.bu b/tests/kola/networking/hostname/fallback-hostname/config.bu
new file mode 100644
index 0000000000..de581ea483
--- /dev/null
+++ b/tests/kola/networking/hostname/fallback-hostname/config.bu
@@ -0,0 +1,10 @@
+variant: fcos
+version: 1.4.0
+storage:
+  files:
+    - path: /etc/NetworkManager/conf.d/90-no-dhcp-dns-hostname.conf
+      mode: 0600
+      contents:
+        inline: |
+          [main]
+          hostname-mode=none
diff --git a/tests/kola/networking/hostname/fallback-hostname/data/commonlib.sh b/tests/kola/networking/hostname/fallback-hostname/data/commonlib.sh
new file mode 120000
index 0000000000..b8dcbdca1a
--- /dev/null
+++ b/tests/kola/networking/hostname/fallback-hostname/data/commonlib.sh
@@ -0,0 +1 @@
+../../../data/commonlib.sh
\ No newline at end of file
diff --git a/tests/kola/networking/hostname/fallback-hostname/test.sh b/tests/kola/networking/hostname/fallback-hostname/test.sh
new file mode 100755
index 0000000000..e473fcd7f6
--- /dev/null
+++ b/tests/kola/networking/hostname/fallback-hostname/test.sh
@@ -0,0 +1,75 @@
+#!/bin/bash
+## kola:
+##   tags: "platform-independent"
+##   description: Verify that the fallback hostname is `localhost`.
+
+# This test validates that the fallback hostname is set to `localhost`
+# by first disabling NetworkManager from setting the hostname
+# via DHCP or DNS (see config.bu), also verify that the
+# hostname is set from the fallback hostname and is `localhost`.
+# See https://github.com/coreos/fedora-coreos-tracker/issues/902
+#
+# Use the output of hostnamectl to gather information about how
+# the hostname is/was set. We're expecting something like this:
+#   {
+#       "Hostname" : "localhost",
+#       "StaticHostname" : null,
+#       "PrettyHostname" : null,
+#       "DefaultHostname" : "localhost",
+#       "HostnameSource" : "default",
+#       "IconName" : "computer-vm",
+#       "Chassis" : "vm",
+#       "Deployment" : null,
+#       "Location" : null,
+#       "KernelName" : "Linux",
+#       "KernelRelease" : "5.18.17-200.fc36.x86_64",
+#       "KernelVersion" : "#1 SMP PREEMPT_DYNAMIC Thu Aug 11 14:36:06 UTC 2022",
+#       "OperatingSystemPrettyName" : "Fedora CoreOS 36.20220814.20.0",
+#       "OperatingSystemCPEName" : "cpe:/o:fedoraproject:fedora:36",
+#       "OperatingSystemHomeURL" : "https://getfedora.org/coreos/",
+#       "HardwareVendor" : "QEMU",
+#       "HardwareModel" : "Standard PC _i440FX + PIIX, 1996_",
+#       "ProductUUID" : null
+#   }
+#
+# "hostnamectl --json=pretty" is not supported on rhel8 yet, the
+# expected output like this:
+#    Static hostname: n/a
+# Transient hostname: localhost
+#          Icon name: computer-vm
+#            Chassis: vm
+#         Machine ID: d9d6dbacde8345f2a275e1d6dca81b78
+#            Boot ID: a47ef11ca938496985d5d85a4274c664
+#     Virtualization: kvm
+#   Operating System: Red Hat Enterprise Linux CoreOS 412.86.202209030446-0 (Ootpa)
+#        CPE OS Name: cpe:/o:redhat:enterprise_linux:8::coreos
+#             Kernel: Linux 4.18.0-372.19.1.el8_6.x86_64
+#       Architecture: x86-64
+
+set -xeuo pipefail
+
+# shellcheck disable=SC1091
+. "$KOLA_EXT_DATA/commonlib.sh"
+
+output=$(hostnamectl --json=pretty)
+hostname=$(echo "$output" | jq -r '.Hostname')
+fallback=$(echo "$output" | jq -r '.DefaultHostname')
+static=$(echo "$output" | jq -r '.StaticHostname')
+namesource=$(echo "$output" | jq -r '.HostnameSource')
+
+if [ "$hostname" != 'localhost' ]; then
+    fatal "hostname was not expected"
+fi
+if [ "$fallback" != 'localhost' ]; then
+    fatal "fallback hostname was not expected"
+fi
+if [ "$static" != 'null' ]; then
+    fatal "static hostname not expected to be set"
+fi
+if [ "$namesource" != 'default' ]; then
+    # For this test since we disabled NM setting the hostname we
+    # expect the hostname to have been set via the fallback/default
+    fatal "hostname was set from non-default/fallback source"
+fi
+
+ok "fallback hostname wired up correctly"
diff --git a/tests/kola/networking/ifname-karg/data/commonlib.sh b/tests/kola/networking/ifname-karg/data/commonlib.sh
new file mode 120000
index 0000000000..b8dcbdca1a
--- /dev/null
+++ b/tests/kola/networking/ifname-karg/data/commonlib.sh
@@ -0,0 +1 @@
+../../../data/commonlib.sh
\ No newline at end of file
diff --git a/tests/kola/networking/ifname-karg/data/ifname-karg-lib.sh b/tests/kola/networking/ifname-karg/data/ifname-karg-lib.sh
new file mode 100644
index 0000000000..b372b968f8
--- /dev/null
+++ b/tests/kola/networking/ifname-karg/data/ifname-karg-lib.sh
@@ -0,0 +1,37 @@
+# This is a library created for our ifname-karg tests
+
+# shellcheck disable=SC1091
+. "$KOLA_EXT_DATA/commonlib.sh"
+
+# check IP for given NIC name
+check_ip() {
+    # User provides the NIC name.
+    local nic_name=$1
+    # The expected IP is the first one in the range given out by QEMU
+    # user mode networking: https://www.qemu.org/docs/master/system/devices/net.html#using-the-user-mode-network-stack
+    local expected_ip="10.0.2.15"
+    # Verify the given nic name has the expected IP.
+    local nic_ip=$(get_ipv4_for_nic ${nic_name})
+    if [ "${nic_ip}" != "${expected_ip}" ]; then
+        fatal "Error: get ${nic_name} ip = ${nic_ip}, expected is ${expected_ip}"
+    fi
+    ok "get ${nic_name} ip is ${expected_ip}"
+}
+
+# simple file existence check
+check_file_exists() {
+    local file=$1
+    if [ ! -f $file ]; then
+        fatal "expected file ${file} doesn't exist on disk"
+    fi
+    ok "expected file ${file} exists on disk"
+}
+
+# simple file non-existence check
+check_file_not_exists() {
+    local file=$1
+    if [ -f $file ]; then
+        fatal "expected file ${file} to not exist on disk"
+    fi
+    ok "file ${file} does not exist on disk"
+}
diff --git a/tests/kola/networking/ifname-karg/everyboot-systemd-link-file/config.bu b/tests/kola/networking/ifname-karg/everyboot-systemd-link-file/config.bu
new file mode 100644
index 0000000000..c79d3aa60e
--- /dev/null
+++ b/tests/kola/networking/ifname-karg/everyboot-systemd-link-file/config.bu
@@ -0,0 +1,7 @@
+variant: fcos
+version: 1.4.0
+kernel_arguments:
+  should_exist:
+    # Persistently set the ifname kernel argument to set the given MAC to the
+    # NIC named `kolatest`. The MAC address is the default one QEMU assigns.
+    - ifname=kolatest:52:54:00:12:34:56
diff --git a/tests/kola/networking/ifname-karg/everyboot-systemd-link-file/data/commonlib.sh b/tests/kola/networking/ifname-karg/everyboot-systemd-link-file/data/commonlib.sh
new file mode 120000
index 0000000000..7028449b11
--- /dev/null
+++ b/tests/kola/networking/ifname-karg/everyboot-systemd-link-file/data/commonlib.sh
@@ -0,0 +1 @@
+../../../../data/commonlib.sh
\ No newline at end of file
diff --git a/tests/kola/networking/ifname-karg/everyboot-systemd-link-file/data/ifname-karg-lib.sh b/tests/kola/networking/ifname-karg/everyboot-systemd-link-file/data/ifname-karg-lib.sh
new file mode 120000
index 0000000000..d50acc0fb4
--- /dev/null
+++ b/tests/kola/networking/ifname-karg/everyboot-systemd-link-file/data/ifname-karg-lib.sh
@@ -0,0 +1 @@
+../../data/ifname-karg-lib.sh
\ No newline at end of file
diff --git a/tests/kola/networking/ifname-karg/everyboot-systemd-link-file/test.sh b/tests/kola/networking/ifname-karg/everyboot-systemd-link-file/test.sh
new file mode 100755
index 0000000000..389e04c136
--- /dev/null
+++ b/tests/kola/networking/ifname-karg/everyboot-systemd-link-file/test.sh
@@ -0,0 +1,41 @@
+#!/bin/bash
+## kola:
+##   description: Verify persistent ifname= karg works via systemd-network-generator.
+##   # appendFirstbootKernelArgs is only supported on QEMU
+##   platforms: qemu
+##   # Don't run the propagate code. With this test we want to
+##   # validate that the systemd.link file gets created by
+##   # systemd-network-generator.
+##   appendFirstbootKernelArgs: "coreos.no_persist_ip"
+
+set -xeuo pipefail
+
+# shellcheck disable=SC1091
+. "$KOLA_EXT_DATA/commonlib.sh"
+. $KOLA_EXT_DATA/ifname-karg-lib.sh
+
+nicname='kolatest'
+
+run_tests() {
+    # Make sure nothing was persisted from the initramfs
+    check_file_not_exists '/etc/udev/rules.d/80-ifname.rules'
+    # Make sure systemd-network-generator ran (from the real root)
+    check_file_exists "/run/systemd/network/*-${nicname}.link"
+    # Make sure the NIC is in use and got the expected IP address
+    check_ip "${nicname}"
+}
+
+case "${AUTOPKGTEST_REBOOT_MARK:-}" in
+    "")
+        ok "first boot"
+        run_tests
+        /tmp/autopkgtest-reboot rebooted
+      ;;
+
+    rebooted)
+        ok "second boot"
+        run_tests
+      ;;
+
+  *) fatal "unexpected mark: ${AUTOPKGTEST_REBOOT_MARK}";;
+esac
diff --git a/tests/kola/networking/ifname-karg/udev-rule-firstboot-propagation b/tests/kola/networking/ifname-karg/udev-rule-firstboot-propagation
new file mode 100755
index 0000000000..37489d97c7
--- /dev/null
+++ b/tests/kola/networking/ifname-karg/udev-rule-firstboot-propagation
@@ -0,0 +1,45 @@
+#!/bin/bash
+## kola:
+##   description: Verify firstboot ifname= karg udev rule propoagation works.
+##   # appendFirstbootKernelArgs is only supported on QEMU
+##   platforms: qemu
+##   # Append ifname kernel argument to set the given MAC address to the NIC
+##   # named `kolatest`. The MAC address is the default one QEMU assigns.
+##   appendFirstbootKernelArgs: "ifname=kolatest:52:54:00:12:34:56"
+
+# Part of https://github.com/coreos/fedora-coreos-tracker/issues/553
+
+set -xeuo pipefail
+
+# shellcheck disable=SC1091
+. "$KOLA_EXT_DATA/commonlib.sh"
+. $KOLA_EXT_DATA/ifname-karg-lib.sh
+
+nicname='kolatest'
+
+case "${AUTOPKGTEST_REBOOT_MARK:-}" in
+    "")
+        ok "first boot"
+        # Make sure the rules were persisted from the initramfs
+        check_file_exists '/etc/udev/rules.d/80-ifname.rules'
+        # On first boot we expect systemd-network-generator to run too
+        # because the ifname= karg was present, but only for first boot
+        check_file_exists "/run/systemd/network/*-${nicname}.link"
+        # Make sure the NIC is in use and got the expected IP address
+        check_ip "${nicname}"
+        /tmp/autopkgtest-reboot rebooted
+      ;;
+
+    rebooted)
+        ok "second boot"
+        # Make sure the rules are still there
+        check_file_exists '/etc/udev/rules.d/80-ifname.rules'
+        # On second boot the ifname= karg isn't there so the file
+        # created by systemd-network-generator shouldn't exist.
+        check_file_not_exists "/run/systemd/network/*-${nicname}.link"
+        # Make sure the NIC is in use and got the expected IP address
+        check_ip "${nicname}"
+      ;;
+
+  *) fatal "unexpected mark: ${AUTOPKGTEST_REBOOT_MARK}";;
+esac
diff --git a/tests/kola/networking/kargs-rd-net b/tests/kola/networking/kargs-rd-net
new file mode 100755
index 0000000000..557026c531
--- /dev/null
+++ b/tests/kola/networking/kargs-rd-net
@@ -0,0 +1,38 @@
+#!/bin/bash
+## kola:
+##   # appendFirstbootKernelArgs is only supported on QEMU
+##   platforms: qemu
+##   # The functionality we're testing here.
+##   appendFirstbootKernelArgs: "rd.net.timeout.dhcp=30 rd.net.dhcp.retry=8"
+##   description: Verify rd.net.timeout.dhcp and rd.net.dhcp.retry
+##     are supported by NetworkManager.
+
+# See
+# - https://gitlab.freedesktop.org/NetworkManager/NetworkManager/-/issues/559
+# - https://bugzilla.redhat.com/show_bug.cgi?id=1879094#c10
+# - https://bugzilla.redhat.com/show_bug.cgi?id=1877740
+
+# Append "rd.net.timeout.dhcp=30 rd.net.dhcp.retry=8" to kernel parameter
+# when boot, get total timeout is `timeout * retry`, 30*8(240) seconds
+# in this test scenario.
+
+set -xeuo pipefail
+
+# shellcheck disable=SC1091
+. "$KOLA_EXT_DATA/commonlib.sh"
+
+# check kernel parameter
+cmdline="/proc/cmdline"
+for rd in rd.net.timeout.dhcp rd.net.dhcp.retry;
+do
+    if ! grep -q $rd $cmdline; then
+        fatal "Error: can not find $rd in kernel parameter"
+    fi
+done
+
+log=$(journalctl -b -u NetworkManager | grep "beginning transaction")
+timeout=$(echo $log | awk -F'[( )]' '{print $(NF-2)}')
+if [ $timeout -ne 240 ]; then
+    fatal "Error: actual dhcp timeout is $timeout, expected 240"
+fi
+ok "Total dhcp timeout matches expected value"
diff --git a/tests/kola/networking/mtu-on-bond-ignition/config.bu b/tests/kola/networking/mtu-on-bond-ignition/config.bu
new file mode 100644
index 0000000000..ef3b13fc8b
--- /dev/null
+++ b/tests/kola/networking/mtu-on-bond-ignition/config.bu
@@ -0,0 +1,60 @@
+variant: fcos
+version: 1.4.0
+storage:
+  files:
+    - path: /etc/NetworkManager/system-connections/bond0.100.nmconnection
+      mode: 0600
+      contents:
+        inline: |
+          [connection]
+          id=bond0.100
+          type=vlan
+          interface-name=bond0.100
+          [ethernet]
+          mtu=9000
+          [vlan]
+          flags=1
+          id=100
+          parent=bond0
+          [ipv4]
+          address1=10.10.10.10/24,10.10.10.1
+          dhcp-hostname=staticvlanbond
+          may-fail=false
+          method=manual
+    - path: /etc/NetworkManager/system-connections/bond0.nmconnection
+      mode: 0600
+      contents:
+        inline: |
+          [connection]
+          id=bond0
+          type=bond
+          interface-name=bond0
+          [ethernet]
+          mtu=9000
+          [bond]
+          miimon=100
+          mode=active-backup
+          [ipv4]
+          method=disabled
+          [ipv6]
+          method=disabled
+    - path: /etc/NetworkManager/system-connections/eth1.nmconnection
+      mode: 0600
+      contents:
+        inline: |
+          [connection]
+          id=eth1
+          type=ethernet
+          interface-name=eth1
+          master=bond0
+          slave-type=bond
+    - path: /etc/NetworkManager/system-connections/eth2.nmconnection
+      mode: 0600
+      contents:
+        inline: |
+          [connection]
+          id=eth2
+          type=ethernet
+          interface-name=eth2
+          master=bond0
+          slave-type=bond
diff --git a/tests/kola/networking/mtu-on-bond-ignition/data/commonlib.sh b/tests/kola/networking/mtu-on-bond-ignition/data/commonlib.sh
new file mode 120000
index 0000000000..1742d51e67
--- /dev/null
+++ b/tests/kola/networking/mtu-on-bond-ignition/data/commonlib.sh
@@ -0,0 +1 @@
+../../data/commonlib.sh
\ No newline at end of file
diff --git a/tests/kola/networking/mtu-on-bond-ignition/test.sh b/tests/kola/networking/mtu-on-bond-ignition/test.sh
new file mode 100755
index 0000000000..865c59ca7f
--- /dev/null
+++ b/tests/kola/networking/mtu-on-bond-ignition/test.sh
@@ -0,0 +1,52 @@
+#!/bin/bash
+## kola:
+##   # additionalNics is only supported on QEMU
+##   platforms: qemu
+##   # Add 2 NIC for this test
+##   additionalNics: 2
+##   # We use net.ifnames=0 to disable consistent network naming here because on
+##   # different firmwares (BIOS vs UEFI) the NIC names are different.
+##   # See https://github.com/coreos/fedora-coreos-tracker/issues/1060
+##   appendKernelArgs: net.ifnames=0
+##   description: Verify that configure MTU on a VLAN subinterface for
+##     the bond via Ignition works.
+#
+# Set MTU on a VLAN subinterface for the bond using Ignition config and check
+# - verify MTU on the bond matches config
+# - verify MTU on the VLAN subinterface for the bond matches config
+# - verify ip address on the VLAN subinterface for the bond matches config
+#
+# The Ignition config is generated using nm-initrd-generator according to
+# https://docs.fedoraproject.org/en-US/fedora-coreos/sysconfig-network-configuration/
+# kargs="bond=bond0:eth1,eth2:mode=active-backup,miimon=100:9000 \
+#        ip=10.10.10.10::10.10.10.1:255.255.255.0:staticvlanbond:bond0.100:none:9000: \
+#        vlan=bond0.100:bond0"
+# $/usr/libexec/nm-initrd-generator -s -- $kargs
+#
+# Using kernel args to `configure MTU on a VLAN subinterface for the bond` refer to
+# https://github.com/coreos/fedora-coreos-config/pull/1401
+
+set -xeuo pipefail
+
+# shellcheck disable=SC1091
+. "$KOLA_EXT_DATA/commonlib.sh"
+
+bond="bond0"
+vlan="bond0.100"
+for interface in $bond $vlan
+do
+    mtu=""
+    # MTU is changed to 9000 according to config.bu
+    mtu=$(nmcli -g 802-3-ethernet.mtu connection show ${interface})
+    if [ "${mtu}" != "9000" ]; then
+        fatal "Error: get ${interface} mtu = ${mtu}, expected is 9000"
+    fi
+    ok "${interface} mtu is correct"
+done
+
+# Verify "bond0.100" gets ip address 10.10.10.10 according to config.bu
+nic_ip=$(get_ipv4_for_nic ${vlan})
+if [ "${nic_ip}" != "10.10.10.10" ]; then
+    fatal "Error: get ${vlan} ip = ${nic_ip}, expected is 10.10.10.10"
+fi
+ok "get ${vlan} ip is 10.10.10.10"
diff --git a/tests/kola/networking/mtu-on-bond-kargs b/tests/kola/networking/mtu-on-bond-kargs
new file mode 100755
index 0000000000..c91f6b76f7
--- /dev/null
+++ b/tests/kola/networking/mtu-on-bond-kargs
@@ -0,0 +1,44 @@
+#!/bin/bash
+## kola:
+##   # additionalNics is only supported on QEMU
+##   platforms: qemu
+##   # Add 2 NIC for this test
+##   additionalNics: 2
+##   # Configuration of the 2 NICs for this test
+##   # We use net.ifnames=0 to disable consistent network naming here because on
+##   # different firmwares (BIOS vs UEFI) the NIC names are different.
+##   # See https://github.com/coreos/fedora-coreos-tracker/issues/1060
+##   appendKernelArgs: "bond=bond0:eth1,eth2:mode=active-backup,miimon=100:9000 ip=10.10.10.10::10.10.10.1:255.255.255.0:staticvlanbond:bond0.100:none:9000 vlan=bond0.100:bond0 net.ifnames=0"
+##   description: Verify that configuring MTU on a VLAN subinterface
+##     for the bond via kernel arguements works.
+
+# Configuring MTU on a VLAN subinterface for the bond,
+# - verify MTU on the bond matches config.
+# - verify MTU on the VLAN subinterface for the bond matches config.
+# - verify ip address on the VLAN subinterface for the bond matches config.
+# See https://bugzilla.redhat.com/show_bug.cgi?id=1932502#c9
+
+set -xeuo pipefail
+
+# shellcheck disable=SC1091
+. "$KOLA_EXT_DATA/commonlib.sh"
+
+bond="bond0"
+vlan="bond0.100"
+for interface in $bond $vlan
+do
+    mtu=""
+    # MTU is changed to 9000 by `appendKernelArgs` in `kola` above
+    mtu=$(nmcli -g 802-3-ethernet.mtu connection show ${interface})
+    if [ "${mtu}" != "9000" ]; then
+        fatal "Error: get ${interface} mtu = ${mtu}, expected is 9000"
+    fi
+    ok "${interface} mtu is correct"
+done
+
+# Verify "bond0.100" gets ip address by `appendKernelArgs` in `kola` above
+nic_ip=$(get_ipv4_for_nic ${vlan})
+if [ "${nic_ip}" != "10.10.10.10" ]; then
+    fatal "Error: get ${vlan} ip = ${nic_ip}, expected is 10.10.10.10"
+fi
+ok "get ${vlan} ip is 10.10.10.10"
diff --git a/tests/kola/networking/nameserver b/tests/kola/networking/nameserver
new file mode 100755
index 0000000000..82f73eb633
--- /dev/null
+++ b/tests/kola/networking/nameserver
@@ -0,0 +1,41 @@
+#!/bin/bash
+## kola:
+##   # appendKernelArgs is only supported on QEMU
+##   platforms: qemu
+##   appendKernelArgs: "nameserver=8.8.8.8 nameserver=1.1.1.1"
+##   description: Verify that we config multiple nameservers via kernel
+##     arguments work well.
+
+# RHCOS: need to check /etc/resolv.conf and nmconnection.
+# FCOS:  using systemd-resolved which needs to run resolvectl to check.
+# See https://bugzilla.redhat.com/show_bug.cgi?id=1763341
+
+set -xeuo pipefail
+
+# shellcheck disable=SC1091
+. "$KOLA_EXT_DATA/commonlib.sh"
+
+if is_fcos; then
+    # run resolvectl
+    dns=$(resolvectl dns)
+    if ! ([[ "$dns" =~ "8.8.8.8" ]] && [[ "$dns" =~ "1.1.1.1" ]]); then
+        fatal "Error: can not find nameserver via resolvectl"
+    fi
+elif is_rhcos; then
+    # check nameserver in /etc/resolv.conf
+    resolv=/etc/resolv.conf
+    cat ${resolv}
+    if ! (grep -q "nameserver 8.8.8.8" ${resolv} && \
+        grep -q "nameserver 1.1.1.1" ${resolv}); then
+        fatal "Error: can not find nameserver in ${resolv}"
+    fi
+fi
+
+# check nameserver in config file
+conf=/etc/NetworkManager/system-connections/default_connection.nmconnection
+cat ${conf}
+if ! grep -q "dns=8.8.8.8;1.1.1.1;" ${conf}; then
+    fatal "Error: can not find nameserver in ${conf}"
+fi
+
+ok "multiple nameserver"
diff --git a/tests/kola/networking/network-online-service b/tests/kola/networking/network-online-service
new file mode 100755
index 0000000000..a167350993
--- /dev/null
+++ b/tests/kola/networking/network-online-service
@@ -0,0 +1,18 @@
+#!/bin/bash
+## kola:
+##   exclusive: false
+##   description: Verify network-online.target is inactive by default.
+
+# See https://github.com/coreos/fedora-coreos-config/pull/1088
+
+set -xeuo pipefail
+
+# shellcheck disable=SC1091
+. "$KOLA_EXT_DATA/commonlib.sh"
+
+# We shouldn't pull this into the transaction by default.
+# https://www.freedesktop.org/wiki/Software/systemd/NetworkTarget/
+if ! systemctl show -p ActiveState network-online.target | grep -q ActiveState=inactive; then
+    fatal "Unit network-online.target shouldn't be active"
+fi
+ok "unit network-online.target inactive"
diff --git a/tests/kola/networking/nic-naming b/tests/kola/networking/nic-naming
new file mode 100755
index 0000000000..e1dd12ba6e
--- /dev/null
+++ b/tests/kola/networking/nic-naming
@@ -0,0 +1,23 @@
+#!/bin/bash
+## kola:
+##   exclusive: false
+##   platforms: "! azure"
+##   description: Verify that we detected eth* NIC naming after booted.
+
+# Disable on azure because of a limitation of the hv_netvsc driver
+# there. According to [1] and [2] the driver does not provide sufficient
+# information to systemd-udev for systemd's naming to work. It falls
+# back to basic ethX naming.
+# See
+# - https://access.redhat.com/solutions/3204751
+# - https://github.com/Azure/WALinuxAgent/issues/1877
+# - https://github.com/coreos/fedora-coreos-config/commit/2a5c2abc796ac645d705700bf445b50d4cda8f5f
+set -xeuo pipefail
+
+# shellcheck disable=SC1091
+. "$KOLA_EXT_DATA/commonlib.sh"
+
+if ip link | grep -o -e " eth[0-9]:"; then
+    fatal "detected eth* NIC naming on node"
+fi
+ok nic naming
diff --git a/tests/kola/networking/nm-dhcp-client b/tests/kola/networking/nm-dhcp-client
new file mode 100755
index 0000000000..e6b28e56c6
--- /dev/null
+++ b/tests/kola/networking/nm-dhcp-client
@@ -0,0 +1,14 @@
+#!/bin/bash
+## kola:
+##   exclusive: false
+##   description: Verify NM's internal DHCP client runs by default.
+
+set -xeuo pipefail
+
+# shellcheck disable=SC1091
+. "$KOLA_EXT_DATA/commonlib.sh"
+
+if ! journalctl -b 0 -u NetworkManager --grep=dhcp | grep -q "Using DHCP client 'internal'"; then
+  fatal "Error: NetworkManager's internal DHCP client is not running"
+fi
+ok "NetworkManager's internal DHCP client is running"
diff --git a/tests/kola/networking/nm-ifcfg-rh-plugin b/tests/kola/networking/nm-ifcfg-rh-plugin
new file mode 100755
index 0000000000..1f71433bd2
--- /dev/null
+++ b/tests/kola/networking/nm-ifcfg-rh-plugin
@@ -0,0 +1,28 @@
+#!/bin/bash
+## kola:
+##   # This test should behave the same on every platform.
+##   tags: "platform-independent"
+##   # This is a read-only, nondestructive test.
+##   exclusive: false
+##   description: Verify that the ifcfg-rh plugin is not loaded on FCOS,
+##     and make sure that at least for RHCOS8 it is included.
+
+set -xeuo pipefail
+
+# shellcheck disable=SC1091
+. "$KOLA_EXT_DATA/commonlib.sh"
+
+# Check if it exists or not. The plugin provides a dbus interface
+# so if it is loaded there will be something listening at that name
+exists=0
+busctl status com.redhat.ifcfgrh1 && exists=1
+
+if is_fcos; then
+    [ "$exists" == "0" ] || fatal "ifcfg-rh plugin detected on FCOS"
+elif is_rhcos || is_scos; then
+    [ "$exists" == "1" ] || fatal "ifcfg-rh plugin not detected on RHCOS/SCOS"
+else
+    fatal "nm-ifcfg-rh-plugin does not support this distro/version"
+fi
+
+ok "ifcfg-rh plugin test successful"
diff --git a/tests/kola/networking/nm-start b/tests/kola/networking/nm-start
new file mode 100755
index 0000000000..f78c4489a5
--- /dev/null
+++ b/tests/kola/networking/nm-start
@@ -0,0 +1,34 @@
+#!/bin/bash
+## kola:
+##   exclusive: false
+##   description: Verify kola on QEMU shouldn't bring up networking in the
+##     initrd by default, and on AWS we did bring up networking in the initrd.
+
+# See https://github.com/coreos/fedora-coreos-config/pull/426
+
+set -xeuo pipefail
+
+# shellcheck disable=SC1091
+. "$KOLA_EXT_DATA/commonlib.sh"
+
+get_journal_msg_timestamp() {
+    journalctl -o json -b 0 --grep "$1" \
+        | jq -r --slurp '.[0]["__MONOTONIC_TIMESTAMP"]'
+}
+
+on_platform() {
+    grep -q " ignition.platform.id=$1 " /proc/cmdline
+}
+
+switchroot_ts=$(get_journal_msg_timestamp 'Switching root.')
+nm_ts=$(get_journal_msg_timestamp 'NetworkManager .* starting')
+# by default, kola on QEMU shouldn't need to bring up networking
+# https://github.com/coreos/fedora-coreos-config/pull/426
+if [[ $nm_ts -lt $switchroot_ts ]] && on_platform qemu; then
+    fatal "NetworkManager started in initramfs!"
+# and as a sanity-check that this test works, verify that on AWS
+# we did bring up networking in the initrd
+elif [[ $nm_ts -gt $switchroot_ts ]] && on_platform aws; then
+    fatal "NetworkManager not started in initramfs!"
+fi
+ok conditional initrd networking
diff --git a/tests/kola/networking/nmstate/data/nmstate-common.sh b/tests/kola/networking/nmstate/data/nmstate-common.sh
new file mode 100755
index 0000000000..7cbf83c993
--- /dev/null
+++ b/tests/kola/networking/nmstate/data/nmstate-common.sh
@@ -0,0 +1,24 @@
+# shellcheck disable=SC1091
+. "$KOLA_EXT_DATA/commonlib.sh"
+
+main() {
+    nmstatectl show
+    journalctl -u nmstate
+
+    local prefix="first boot"
+    if [ "${AUTOPKGTEST_REBOOT_MARK:-}" == rebooted ]; then
+        prefix="second boot"
+    fi
+
+    if ! nmcli c show br-ex; then
+        fatal "${prefix}: bridge not configured"
+    fi
+
+    if ! ls /etc/nmstate/*applied; then
+        fatal "${prefix}: nmstate yamls files not marked as applied"
+    fi
+
+    if [ "${AUTOPKGTEST_REBOOT_MARK:-}" == "" ]; then
+        /tmp/autopkgtest-reboot rebooted
+    fi
+}
diff --git a/tests/kola/networking/nmstate/policy/config.bu b/tests/kola/networking/nmstate/policy/config.bu
new file mode 100644
index 0000000000..dd4f8d55fc
--- /dev/null
+++ b/tests/kola/networking/nmstate/policy/config.bu
@@ -0,0 +1,24 @@
+variant: fcos
+version: 1.4.0
+storage:
+  files:
+    - path: /etc/nmstate/br-ex-policy.yml
+      contents:
+        inline: |
+          capture:
+            default-gw-route: routes.running.destination=="0.0.0.0/0"
+            default-gw-iface: interfaces.name==capture.default-gw-route.routes.running.0.next-hop-interface
+          desiredState:
+            interfaces:
+            - name: "{{ capture.default-gw-iface.interfaces.0.name }}"
+              type: ethernet
+              state: up
+            - name: br-ex
+              type: linux-bridge
+              copy-mac-from: "{{ capture.default-gw-iface.interfaces.0.name }}"
+              state: up
+              ipv4: "{{ capture.default-gw-iface.interfaces.0.ipv4 }}"
+              ipv6: "{{ capture.default-gw-iface.interfaces.0.ipv6 }}"
+              bridge:
+                port:
+                - name: "{{ capture.default-gw-iface.interfaces.0.name }}"
diff --git a/tests/kola/networking/nmstate/policy/data/commonlib.sh b/tests/kola/networking/nmstate/policy/data/commonlib.sh
new file mode 120000
index 0000000000..b8dcbdca1a
--- /dev/null
+++ b/tests/kola/networking/nmstate/policy/data/commonlib.sh
@@ -0,0 +1 @@
+../../../data/commonlib.sh
\ No newline at end of file
diff --git a/tests/kola/networking/nmstate/policy/data/nmstate-common.sh b/tests/kola/networking/nmstate/policy/data/nmstate-common.sh
new file mode 120000
index 0000000000..fe39f3bae7
--- /dev/null
+++ b/tests/kola/networking/nmstate/policy/data/nmstate-common.sh
@@ -0,0 +1 @@
+../../data/nmstate-common.sh
\ No newline at end of file
diff --git a/tests/kola/networking/nmstate/policy/test.sh b/tests/kola/networking/nmstate/policy/test.sh
new file mode 100755
index 0000000000..2387a688b5
--- /dev/null
+++ b/tests/kola/networking/nmstate/policy/test.sh
@@ -0,0 +1,13 @@
+#!/bin/bash
+## kola:
+##   tags: "platform-independent needs-internet"
+##   description: Verify that configure a DHCP linux bridge using
+##     butane and nmstate service with policy works.
+
+# See https://github.com/coreos/fedora-coreos-tracker/issues/1175
+
+set -xeuo pipefail
+
+. $KOLA_EXT_DATA/nmstate-common.sh
+
+main
diff --git a/tests/kola/networking/nmstate/state/config.bu b/tests/kola/networking/nmstate/state/config.bu
new file mode 100644
index 0000000000..d2d7e834b5
--- /dev/null
+++ b/tests/kola/networking/nmstate/state/config.bu
@@ -0,0 +1,17 @@
+variant: fcos
+version: 1.4.0
+storage:
+  files:
+    - path: /etc/nmstate/br-ex.yml
+      contents:
+        inline: |
+          interfaces:
+           - name: br-ex
+             type: linux-bridge
+             state: up
+             ipv4:
+               enabled: false
+             ipv6:
+               enabled: false
+             bridge:
+               port: []
diff --git a/tests/kola/networking/nmstate/state/data/commonlib.sh b/tests/kola/networking/nmstate/state/data/commonlib.sh
new file mode 120000
index 0000000000..b8dcbdca1a
--- /dev/null
+++ b/tests/kola/networking/nmstate/state/data/commonlib.sh
@@ -0,0 +1 @@
+../../../data/commonlib.sh
\ No newline at end of file
diff --git a/tests/kola/networking/nmstate/state/data/nmstate-common.sh b/tests/kola/networking/nmstate/state/data/nmstate-common.sh
new file mode 120000
index 0000000000..fe39f3bae7
--- /dev/null
+++ b/tests/kola/networking/nmstate/state/data/nmstate-common.sh
@@ -0,0 +1 @@
+../../data/nmstate-common.sh
\ No newline at end of file
diff --git a/tests/kola/networking/nmstate/state/test.sh b/tests/kola/networking/nmstate/state/test.sh
new file mode 100755
index 0000000000..65bcafb40d
--- /dev/null
+++ b/tests/kola/networking/nmstate/state/test.sh
@@ -0,0 +1,13 @@
+#!/bin/bash
+## kola:
+##   tags: "platform-independent needs-internet"
+##   description: Verify that configure a DHCP linux bridge using
+##     butane and nmstate service with state works.
+
+# See https://github.com/coreos/fedora-coreos-tracker/issues/1175
+
+set -xeuo pipefail
+
+. $KOLA_EXT_DATA/nmstate-common.sh
+
+main
diff --git a/tests/kola/networking/no-default-initramfs-net-propagation/bootif b/tests/kola/networking/no-default-initramfs-net-propagation/bootif
new file mode 100755
index 0000000000..aee002a7c6
--- /dev/null
+++ b/tests/kola/networking/no-default-initramfs-net-propagation/bootif
@@ -0,0 +1,29 @@
+#!/bin/bash
+## kola:
+##   # appendFirstbootKernelArgs is only supported on qemu
+##   platforms: qemu
+##   # Append BOOTIF kernel argument so we can test how nm-initrd-generator
+##   # and the coreos-teardown-initramfs interact. The MAC address is the
+##   # default one QEMU assigns.
+##   # Add rd.neednet=1 so we can force networking to be brought up on
+##   # qemu to test that doing so doesn't materially change things.
+##   appendFirstbootKernelArgs: "BOOTIF=52:54:00:12:34:56 rd.neednet=1"
+#
+# In addition to the pure network defaults case we should also make
+# sure that when BOOTIF= or rd.bootif= are provided on the kernel
+# command line (typically from PXE servers) that we don't propagate
+# networking configs either. See https://github.com/coreos/fedora-coreos-tracker/issues/1048
+
+set -xeuo pipefail
+
+# shellcheck disable=SC1091
+. "$KOLA_EXT_DATA/commonlib.sh"
+
+# Run the test bits (sets the $fail var)
+. $KOLA_EXT_DATA/test-net-propagation.sh
+
+if [ -z "${fail:-}" ]; then
+    ok "success: no initramfs network propagation for default configuration with BOOTIF"
+else
+    fatal "fail: no initramfs network propagation for default configuration with BOOTIF"
+fi
diff --git a/tests/kola/networking/no-default-initramfs-net-propagation/data/commonlib.sh b/tests/kola/networking/no-default-initramfs-net-propagation/data/commonlib.sh
new file mode 120000
index 0000000000..b8dcbdca1a
--- /dev/null
+++ b/tests/kola/networking/no-default-initramfs-net-propagation/data/commonlib.sh
@@ -0,0 +1 @@
+../../../data/commonlib.sh
\ No newline at end of file
diff --git a/tests/kola/networking/no-default-initramfs-net-propagation/data/test-net-propagation.sh b/tests/kola/networking/no-default-initramfs-net-propagation/data/test-net-propagation.sh
new file mode 100644
index 0000000000..5370905969
--- /dev/null
+++ b/tests/kola/networking/no-default-initramfs-net-propagation/data/test-net-propagation.sh
@@ -0,0 +1,19 @@
+# common pieces of each of the no-default-initramfs-net-propagation tests
+if ! journalctl -t coreos-teardown-initramfs | \
+       grep 'info: skipping propagation of default networking configs'; then
+    echo "no log message claiming to skip initramfs network propagation" >&2
+    fail=1
+fi
+
+if [ -n "$(ls -A /etc/NetworkManager/system-connections/)" ]; then
+    echo "configs exist in /etc/NetworkManager/system-connections/, but shouldn't" >&2
+    fail=1
+fi
+
+# Also check /etc/sysconfig/network-scripts/ because ifcfg files are
+# supported downstream. Need to ignore readme-ifcfg-rh.txt here because of
+# https://gitlab.freedesktop.org/NetworkManager/NetworkManager/-/commit/96d7362
+if [ -n "$(ls -A -I readme-ifcfg-rh.txt /etc/sysconfig/network-scripts/)" ]; then
+    echo "configs exist in /etc/sysconfig/network-scripts/, but shouldn't" >&2
+    fail=1
+fi
diff --git a/tests/kola/networking/no-default-initramfs-net-propagation/default b/tests/kola/networking/no-default-initramfs-net-propagation/default
new file mode 100755
index 0000000000..85e722b24c
--- /dev/null
+++ b/tests/kola/networking/no-default-initramfs-net-propagation/default
@@ -0,0 +1,22 @@
+#!/bin/bash
+## kola:
+##   exclusive: false
+##   description: Verify no initramfs network propagation by default.
+#
+# With pure network defaults no networking should have been propagated
+# from the initramfs. This test tries to verify that is the case.
+# https://github.com/coreos/fedora-coreos-tracker/issues/696
+
+set -xeuo pipefail
+
+# shellcheck disable=SC1091
+. "$KOLA_EXT_DATA/commonlib.sh"
+
+# Run the test bits (sets the $fail var)
+. $KOLA_EXT_DATA/test-net-propagation.sh
+
+if [ -z "${fail:-}" ]; then
+    ok "success: no initramfs network propagation for default configuration"
+else
+    fatal "fail: no initramfs network propagation for default configuration"
+fi
diff --git a/tests/kola/networking/no-default-initramfs-net-propagation/test.sh b/tests/kola/networking/no-default-initramfs-net-propagation/test.sh
deleted file mode 100755
index 4a44f2e729..0000000000
--- a/tests/kola/networking/no-default-initramfs-net-propagation/test.sh
+++ /dev/null
@@ -1,32 +0,0 @@
-#!/bin/bash
-set -xeuo pipefail
-
-# With pure network defaults no networking should have been propagated
-# from the initramfs. This test tries to verify that is the case.
-# https://github.com/coreos/fedora-coreos-tracker/issues/696
-
-ok() {
-    echo "ok" "$@"
-}
-
-fatal() {
-    echo "$@" >&2
-    exit 1
-}
-
-if ! journalctl -t coreos-teardown-initramfs | \
-       grep 'info: skipping propagation of default networking configs'; then
-    echo "no log message claiming to skip initramfs network propagation" >&2
-    fail=1
-fi
-
-if [ -n "$(ls -A /etc/NetworkManager/system-connections/)" ]; then
-    echo "configs exist in /etc/NetworkManager/system-connections/, but shouldn't" >&2
-    fail=1
-fi
-
-if [ -z "${fail:-}" ]; then
-    ok "success: no initramfs network propagation for default configuration"
-else
-    fatal "fail: no initramfs network propagation for default configuration"
-fi
diff --git a/tests/kola/networking/no-persist-ip b/tests/kola/networking/no-persist-ip
new file mode 100755
index 0000000000..96822274d4
--- /dev/null
+++ b/tests/kola/networking/no-persist-ip
@@ -0,0 +1,35 @@
+#!/bin/bash
+## kola:
+##   # additionalNics is only supported on QEMU
+##   platforms: qemu
+##   # Add 1 NIC for this test
+##   additionalNics: 1
+##   # The functionality we're testing here and the configuration for the NIC.
+##   # We use net.ifnames=0 to disable consistent network naming here because on
+##   # different firmwares (BIOS vs UEFI) the NIC names are different.
+##   # See https://github.com/coreos/fedora-coreos-tracker/issues/1060
+##   appendKernelArgs: "ip=10.10.10.10::10.10.10.1:255.255.255.0:myhostname:eth1:none net.ifnames=0 coreos.no_persist_ip"
+##   description: Verify that the coreos.no_persist_ip kernel argument will
+##     prevent propagating kernel argument based networking configuration
+##     into the real root.
+
+# It does this by providing karg static networking config for eth1 and
+# then verifying that DHCP is used in the real root.
+
+set -xeuo pipefail
+
+# shellcheck disable=SC1091
+. "$KOLA_EXT_DATA/commonlib.sh"
+
+# Verify eth1 gets ip address via dhcp
+nic_name="eth1"
+nic_ip=$(get_ipv4_for_nic ${nic_name})
+if [ ${nic_ip} != "10.0.2.31" ]; then
+    fatal "Error: get ${nic_name} ip = ${nic_ip}, expected is 10.0.2.31"
+fi
+ok "get ${nic_name} ip is 10.0.2.31"
+
+if ! journalctl -b -u coreos-teardown-initramfs | grep -q "info: skipping propagating initramfs settings"; then
+    fatal "Error: can not get log: (info: skipping propagating initramfs settings)"
+fi
+ok "test coreos.no_persist_ip disable initramfs network propagation"
diff --git a/tests/kola/networking/prefer-ignition-networking/config.bu b/tests/kola/networking/prefer-ignition-networking/config.bu
new file mode 100644
index 0000000000..6897419b2e
--- /dev/null
+++ b/tests/kola/networking/prefer-ignition-networking/config.bu
@@ -0,0 +1,16 @@
+variant: fcos
+version: 1.4.0
+storage:
+  files:
+    - path: /etc/NetworkManager/system-connections/eth1.nmconnection
+      mode: 0600
+      contents:
+        inline: |
+          [connection]
+          id=eth1
+          type=ethernet
+          interface-name=eth1
+          [ipv4]
+          dns-search=
+          may-fail=false
+          method=auto
diff --git a/tests/kola/networking/prefer-ignition-networking/data/commonlib.sh b/tests/kola/networking/prefer-ignition-networking/data/commonlib.sh
new file mode 120000
index 0000000000..1742d51e67
--- /dev/null
+++ b/tests/kola/networking/prefer-ignition-networking/data/commonlib.sh
@@ -0,0 +1 @@
+../../data/commonlib.sh
\ No newline at end of file
diff --git a/tests/kola/networking/prefer-ignition-networking/test.sh b/tests/kola/networking/prefer-ignition-networking/test.sh
new file mode 100755
index 0000000000..025fbd1d37
--- /dev/null
+++ b/tests/kola/networking/prefer-ignition-networking/test.sh
@@ -0,0 +1,40 @@
+#!/bin/bash
+## kola:
+##   # additionalNics is only supported on QEMU
+##   platforms: qemu
+##   # Add 1 additional NIC for this test
+##   additionalNics: 1
+##   # Set the kernel arguments so that we can set the configuration for the NIC.
+##   # We use net.ifnames=0 to disable consistent network naming here because on
+##   # different firmwares (BIOS vs UEFI) the NIC names are different.
+##   # See https://github.com/coreos/fedora-coreos-tracker/issues/1060
+##   appendKernelArgs: "ip=10.10.10.10::10.10.10.1:255.255.255.0:myhostname:eth1:none:8.8.8.8 net.ifnames=0"
+##   description: Verify that networking configuration is propagated
+##     via Ignition by default.
+
+# Setup configuration for a single NIC with two different ways:
+# - kargs provide static network config for eth1 without coreos.force_persist_ip
+# - Ignition provides dhcp network config for eth1
+# Expected result:
+# - without coreos.force_persist_ip Ignition networking
+#   configuration wins, verify that eth1 gets ip via dhcp
+# See https://bugzilla.redhat.com/show_bug.cgi?id=1958930#c29
+
+set -xeuo pipefail
+
+# shellcheck disable=SC1091
+. "$KOLA_EXT_DATA/commonlib.sh"
+
+# Verify eth1 gets ip address via dhcp
+nic_name="eth1"
+nic_ip=$(get_ipv4_for_nic ${nic_name})
+if [ ${nic_ip} != "10.0.2.31" ]; then
+    fatal "Error: get ${nic_name} ip = ${nic_ip}, expected is 10.0.2.31"
+fi
+ok "get ${nic_name} ip is 10.0.2.31"
+
+syscon="/etc/NetworkManager/system-connections"
+if [ ! -f "${syscon}/${nic_name}.nmconnection" ]; then
+    fatal "Error: can not find ${syscon}/${nic_name}.nmconnection"
+fi
+ok "find ${syscon}/${nic_name}.nmconnection"
diff --git a/tests/kola/networking/rd-net-timeout-carrier/config.bu b/tests/kola/networking/rd-net-timeout-carrier/config.bu
new file mode 100644
index 0000000000..f1acb823b0
--- /dev/null
+++ b/tests/kola/networking/rd-net-timeout-carrier/config.bu
@@ -0,0 +1,6 @@
+variant: fcos
+version: 1.4.0
+kernel_arguments:
+  should_exist:
+    - rd.net.timeout.carrier=15
+    - rd.neednet=1
diff --git a/tests/kola/networking/rd-net-timeout-carrier/data/commonlib.sh b/tests/kola/networking/rd-net-timeout-carrier/data/commonlib.sh
new file mode 120000
index 0000000000..1742d51e67
--- /dev/null
+++ b/tests/kola/networking/rd-net-timeout-carrier/data/commonlib.sh
@@ -0,0 +1 @@
+../../data/commonlib.sh
\ No newline at end of file
diff --git a/tests/kola/networking/rd-net-timeout-carrier/test.sh b/tests/kola/networking/rd-net-timeout-carrier/test.sh
new file mode 100755
index 0000000000..163b0c0ce7
--- /dev/null
+++ b/tests/kola/networking/rd-net-timeout-carrier/test.sh
@@ -0,0 +1,44 @@
+#!/bin/bash
+## kola:
+##   tags: "platform-independent"
+##   description: Verify that NetworkManager supports configuring the
+##     carrier timeout via the `rd.net.timeout.carrier=` karg.
+
+# Without recreating an environment that requires this setting to be set
+# (which would be hard to do), we'll test this by just making sure
+# that setting the kernel argument ensures the runtime configuration
+# file /run/NetworkManager/conf.d/15-carrier-timeout.conf gets created.
+#
+# Checking that this file gets created is also tricky because we
+# `rm -rf /run/NetworkManager` in coreos-teardown-initramfs on first boot,
+# so we'll workaround that by setting the karg permanently and then
+# performing a reboot so we can check on the second boot if the file exists.
+#
+# See:
+# - https://bugzilla.redhat.com/show_bug.cgi?id=1917773
+# - https://gitlab.freedesktop.org/NetworkManager/NetworkManager/-/commit/e300138892ee0fc3824d38b527b60103a01758ab#
+
+set -xeuo pipefail
+
+# shellcheck disable=SC1091
+. "$KOLA_EXT_DATA/commonlib.sh"
+
+case "${AUTOPKGTEST_REBOOT_MARK:-}" in
+  "")
+      ok "first boot"
+      /tmp/autopkgtest-reboot rebooted
+      ;;
+
+  rebooted)
+      ok "second boot"
+      grep rd.net.timeout.carrier=15 /proc/cmdline
+
+      generator_conf="/run/NetworkManager/conf.d/15-carrier-timeout.conf"
+      if [ ! -f ${generator_conf} ]; then
+        fatal "Error: can not generate ${generator_conf} with karg rd.net.timeout.carrier"
+      fi
+      ok "NM generated rd.net.timeout.carrier configuration in the initramfs"
+      ;;
+
+  *) fatal "unexpected mark: ${AUTOPKGTEST_REBOOT_MARK}";;
+esac
diff --git a/tests/kola/networking/resolv/data/commonlib.sh b/tests/kola/networking/resolv/data/commonlib.sh
new file mode 120000
index 0000000000..b8dcbdca1a
--- /dev/null
+++ b/tests/kola/networking/resolv/data/commonlib.sh
@@ -0,0 +1 @@
+../../../data/commonlib.sh
\ No newline at end of file
diff --git a/tests/kola/networking/resolv/systemd-resolved b/tests/kola/networking/resolv/systemd-resolved
new file mode 100755
index 0000000000..36de025c2e
--- /dev/null
+++ b/tests/kola/networking/resolv/systemd-resolved
@@ -0,0 +1,23 @@
+#!/bin/bash
+## kola:
+##  # This test only runs on FCOS because `systemd-resolved` is not installed on
+##  # RHCOS
+##   distros:  fcos
+##   exclusive: false
+##   description: Verify systemd-resolved and the stub listener
+##     should be enabled by default.
+
+set -xeuo pipefail
+
+# shellcheck disable=SC1091
+. "$KOLA_EXT_DATA/commonlib.sh"
+
+if ! systemctl is-enabled systemd-resolved 1>/dev/null; then
+    fatal "Unit systemd-resolved should be enabled"
+fi
+
+# systemd-resolved should be fully functional on f35+
+if ! grep 'nameserver 127.0.0.53' /etc/resolv.conf; then
+    fatal "systemd-resolved stub listener isn't enabled"
+fi
+ok "systemd-resolved is enabled and the stub listener is enabled"
diff --git a/tests/kola/networking/team-dhcp-via-ignition/config.bu b/tests/kola/networking/team-dhcp-via-ignition/config.bu
new file mode 100644
index 0000000000..8121382a8a
--- /dev/null
+++ b/tests/kola/networking/team-dhcp-via-ignition/config.bu
@@ -0,0 +1,44 @@
+variant: fcos
+version: 1.4.0
+storage:
+  files:
+    - path: /etc/NetworkManager/system-connections/team0.nmconnection
+      mode: 0600
+      contents:
+        inline: |
+          [connection]
+          id=team0
+          type=team
+          autoconnect-retries=1
+          interface-name=team0
+          multi-connect=1
+          [team]
+          config={"runner": {"name": "activebackup"}, "link_watch": {"name": "ethtool"}}
+          [ipv4]
+          dns-search=
+          may-fail=false
+          method=auto
+    - path: /etc/NetworkManager/system-connections/team0-slave-eth1.nmconnection
+      mode: 0600
+      contents:
+        inline: |
+          [connection]
+          id=team0-slave-eth1
+          type=ethernet
+          interface-name=eth1
+          master=team0
+          slave-type=team
+          [team-port]
+          config={"prio": 100}
+    - path: /etc/NetworkManager/system-connections/team0-slave-eth2.nmconnection
+      mode: 0600
+      contents:
+        inline: |
+          [connection]
+          id=team0-slave-eth2
+          type=ethernet
+          interface-name=eth2
+          master=team0
+          slave-type=team
+          [team-port]
+          config={"prio": 100}
diff --git a/tests/kola/networking/team-dhcp-via-ignition/data/commonlib.sh b/tests/kola/networking/team-dhcp-via-ignition/data/commonlib.sh
new file mode 120000
index 0000000000..1742d51e67
--- /dev/null
+++ b/tests/kola/networking/team-dhcp-via-ignition/data/commonlib.sh
@@ -0,0 +1 @@
+../../data/commonlib.sh
\ No newline at end of file
diff --git a/tests/kola/networking/team-dhcp-via-ignition/test.sh b/tests/kola/networking/team-dhcp-via-ignition/test.sh
new file mode 100755
index 0000000000..6d506c23c9
--- /dev/null
+++ b/tests/kola/networking/team-dhcp-via-ignition/test.sh
@@ -0,0 +1,54 @@
+#!/bin/bash
+## kola:
+##   # additionalNics is only supported on QEMU
+##   platforms: qemu
+##   # Add 2 NIC for this test
+##   additionalNics: 2
+##   # We use net.ifnames=0 to disable consistent network naming here because on
+##   # different firmwares (BIOS vs UEFI) the NIC names are different.
+##   # See https://github.com/coreos/fedora-coreos-tracker/issues/1060
+##   appendKernelArgs: "net.ifnames=0"
+##   description: Verify team networking works via Ignition config.
+
+# The Ignition config refers to
+# https://docs.fedoraproject.org/en-US/fedora-coreos/sysconfig-network-configuration/
+
+set -xeuo pipefail
+
+# shellcheck disable=SC1091
+. "$KOLA_EXT_DATA/commonlib.sh"
+
+team="team0"
+
+# Verify team0 gets dhcp according to config.bu
+nic_ip=$(get_ipv4_for_nic ${team})
+if [ "${nic_ip}" != "10.0.2.31" ]; then
+    fatal "Error: get ${team} ip = ${nic_ip}, expected is 10.0.2.31"
+fi
+
+expected_state="setup:
+  runner: activebackup
+ports:
+  eth1
+    link watches:
+      link summary: up
+      instance[link_watch_0]:
+        name: ethtool
+        link: up
+        down count: 0
+  eth2
+    link watches:
+      link summary: up
+      instance[link_watch_0]:
+        name: ethtool
+        link: up
+        down count: 0
+runner:
+  active port: eth1"
+
+state=`teamdctl team0 state`
+if ! diff -u <(echo "$expected_state") <(echo "$state"); then
+    fatal "Error: the expected team0 network is not the same as expected"
+fi
+
+ok "networking ${team} tests"
diff --git a/tests/kola/networking/tls b/tests/kola/networking/tls
new file mode 100755
index 0000000000..ebaff45f1c
--- /dev/null
+++ b/tests/kola/networking/tls
@@ -0,0 +1,22 @@
+#!/bin/bash
+## kola:
+##   exclusive: false
+##   tags: "platform-independent needs-internet"
+##   description: Verify we can fetch from various popular hosts over TLS.
+
+set -xeuo pipefail
+
+# shellcheck disable=SC1091
+. "$KOLA_EXT_DATA/commonlib.sh"
+
+urls_to_fetch=(
+    "https://cloud.google.com"
+    "https://aws.amazon.com/"
+    "https://azure.microsoft.com"
+    "https://start.fedoraproject.org/"
+)
+
+for url in "${urls_to_fetch[@]}"; do
+    curl -I -s -S -m 30 --retry 5 "$url"
+done
+ok "tls"
diff --git a/tests/kola/ntp/chrony/coreos-platform-chrony-config b/tests/kola/ntp/chrony/coreos-platform-chrony-config
new file mode 100755
index 0000000000..d1dc0294eb
--- /dev/null
+++ b/tests/kola/ntp/chrony/coreos-platform-chrony-config
@@ -0,0 +1,24 @@
+#!/bin/bash
+## kola:
+##   exclusive: false
+##   platforms: "aws azure gcp qemu"
+##   description: Verify the coreos-platform-chrony generator works on cloud.
+
+set -xeuo pipefail
+
+# shellcheck disable=SC1091
+. "$KOLA_EXT_DATA/commonlib.sh"
+
+platform=$(cmdline_arg ignition.platform.id)
+case "${platform}" in
+    aws) chronyc -n sources |grep '169.254.169.123'; echo "ok chrony aws" ;;
+    azure) chronyc -n sources |grep 'PHC'; echo "ok chrony azure" ;;
+    gcp) chronyc -n sources | grep '169.254.169.254'; echo "ok chrony gcp" ;;
+    qemu)
+        # ptp_kvm isn't available on all arches nor all hosts, so don't assume it's always there; see
+        # https://github.com/coreos/fedora-coreos-config/pull/2263#discussion_r1157694192
+        if lsmod | grep -q ptp_kvm; then
+            chronyc -n sources | grep 'PHC0'; echo "ok chrony qemu"
+        fi ;;
+    *) echo "unhandled platform ${platform} ?"; exit 1 ;;
+esac
diff --git a/tests/kola/ntp/chrony/data/commonlib.sh b/tests/kola/ntp/chrony/data/commonlib.sh
new file mode 120000
index 0000000000..b8dcbdca1a
--- /dev/null
+++ b/tests/kola/ntp/chrony/data/commonlib.sh
@@ -0,0 +1 @@
+../../../data/commonlib.sh
\ No newline at end of file
diff --git a/tests/kola/ntp/chrony/data/ntplib.sh b/tests/kola/ntp/chrony/data/ntplib.sh
new file mode 120000
index 0000000000..b2285dcab1
--- /dev/null
+++ b/tests/kola/ntp/chrony/data/ntplib.sh
@@ -0,0 +1 @@
+../../data/ntplib.sh
\ No newline at end of file
diff --git a/tests/kola/ntp/chrony/dhcp-propagation b/tests/kola/ntp/chrony/dhcp-propagation
new file mode 100755
index 0000000000..fb3d6059aa
--- /dev/null
+++ b/tests/kola/ntp/chrony/dhcp-propagation
@@ -0,0 +1,44 @@
+#!/bin/bash
+## kola:
+##   # Add the needs-internet tag. This test builds a container from remote
+##   # sources and uses a remote NTP server.
+##   tags: "needs-internet"
+##   # Limit to qemu because some cloud providers have their own special
+##   # sauce for NTP/chrony and we don't want to interfere with that.
+##   platforms: qemu
+##   # Pulling and building the container can take a long time if a
+##   # slow mirror gets chosen. Bump the timeout to 15 minutes
+##   timeoutMin: 15
+##   # There's a bug in dnf that is causing OOM on low memory systems:
+##   # https://bugzilla.redhat.com/show_bug.cgi?id=1907030
+##   # https://pagure.io/releng/issue/10935#comment-808601
+##   minMemory: 1536
+##   description: Verify that chronyd service got ntp servers from DHCP.
+#
+# This script creates two veth interfaces i.e. one for the host machine
+# and other for the container(dnsmasq server). This setup will be helpful
+# to verify the DHCP propagation of NTP servers. This will also avoid any
+# regression that might cause in RHCOS or FCOS when the upstream changes
+# come down and obsolete the temporary work (https://github.com/coreos/fedora-coreos-config/pull/412)
+
+set -xeuo pipefail
+
+# shellcheck disable=SC1091
+. "$KOLA_EXT_DATA/commonlib.sh"
+. $KOLA_EXT_DATA/ntplib.sh
+
+main() {
+
+    # Choose a host from https://tf.nist.gov/tf-cgi/servers.cgi
+    # that can get DNS over DNSSEC since the environment our OpenShift
+    # runs in requires DNSSEC validation. Test with https://dnsviz.net/
+    ntp_host_ip=$(getent hosts utcnist.colorado.edu | cut -d ' ' -f 1)
+
+    ntp_test_setup $ntp_host_ip
+
+    check_for_ntp_server $ntp_host_ip "chronyc -n sources"
+
+    ok "chronyd got ntp servers from DHCP"
+}
+
+main
diff --git a/tests/kola/ntp/data/ntplib.sh b/tests/kola/ntp/data/ntplib.sh
new file mode 100644
index 0000000000..f4e933ddce
--- /dev/null
+++ b/tests/kola/ntp/data/ntplib.sh
@@ -0,0 +1,60 @@
+# This is a library created for our NTP tests
+
+# shellcheck disable=SC1091
+. "$KOLA_EXT_DATA/commonlib.sh"
+
+ntp_test_setup() {
+    ntp_host_ip=$1
+
+    # create a network namespace
+    ip netns add container
+
+    # create veth pair and assign a namespace to veth-container
+    ip link add veth-host type veth peer name veth-container
+    ip link set veth-container netns container
+
+    # assign an IP address to the `veth-container` interface and bring it up
+    ip netns exec container ip address add 172.16.0.1/24 dev veth-container
+    ip netns exec container ip link set veth-container up
+
+    # run podman commands to set up dnsmasq server
+    pushd "$(mktemp -d)"
+    cat <<EOF >Dockerfile
+FROM quay.io/fedora/fedora:40
+RUN rm -f /etc/yum.repos.d/*.repo \
+&& curl -L https://raw.githubusercontent.com/coreos/fedora-coreos-config/testing-devel/fedora-archive.repo -o /etc/yum.repos.d/fedora-archive.repo
+RUN dnf -y install systemd dnsmasq iproute iputils \
+&& dnf clean all \
+&& systemctl enable dnsmasq
+RUN echo -e 'dhcp-range=172.16.0.10,172.16.0.20,12h\nbind-interfaces\ninterface=veth-container\ndhcp-option=option:ntp-server,$ntp_host_ip' > /etc/dnsmasq.d/dhcp
+CMD [ "/sbin/init" ]
+EOF
+
+    podman build -t dnsmasq .
+    popd
+    podman run -d --rm --name dnsmasq --privileged --network ns:/var/run/netns/container dnsmasq
+
+    # Tell NM to manage the `veth-host` interface and bring it up (will attempt DHCP).
+    # Do this after we start dnsmasq so we don't have to deal with DHCP timeouts.
+    nmcli dev set veth-host managed yes
+    ip link set veth-host up
+}
+
+check_for_ntp_server() {
+    ntp_host_ip=$1
+    ntp_sources_cmd=$2
+
+    ntp_server=""
+    retries=300
+    while [[ $retries -gt 0 ]]; do
+         ntp_sources=$($ntp_sources_cmd)
+         [[ "$ntp_sources" =~ "$ntp_host_ip" ]] && break
+         echo "waiting for ntp server to appear"
+         sleep 1
+         retries=$((retries - 1))
+    done
+
+    if [ $retries -eq  0 ]; then
+        fatal "propagation of ntp server information via dhcp failed"
+    fi
+}
diff --git a/tests/kola/ntp/timesyncd/dhcp-propagation/config.bu b/tests/kola/ntp/timesyncd/dhcp-propagation/config.bu
new file mode 100644
index 0000000000..bce15939ea
--- /dev/null
+++ b/tests/kola/ntp/timesyncd/dhcp-propagation/config.bu
@@ -0,0 +1,17 @@
+variant: fcos
+version: 1.4.0
+systemd:
+  units:
+    - name: chronyd.service
+      enabled: false
+    - name: systemd-timesyncd.service
+      enabled: true
+# There currently isn't any NetworkManager dispatcher that
+# propagates DHCP info to timesyncd in Fedora so we'll just
+# pull one that we can use from the internet.
+storage:
+  files:
+    - path: /etc/NetworkManager/dispatcher.d/30-timesyncd
+      contents:
+        source: "https://github.com/eworm-de/networkmanager-dispatcher-timesyncd/raw/main/30-timesyncd"
+      mode: 0755
diff --git a/tests/kola/ntp/timesyncd/dhcp-propagation/data/commonlib.sh b/tests/kola/ntp/timesyncd/dhcp-propagation/data/commonlib.sh
new file mode 120000
index 0000000000..7028449b11
--- /dev/null
+++ b/tests/kola/ntp/timesyncd/dhcp-propagation/data/commonlib.sh
@@ -0,0 +1 @@
+../../../../data/commonlib.sh
\ No newline at end of file
diff --git a/tests/kola/ntp/timesyncd/dhcp-propagation/data/ntplib.sh b/tests/kola/ntp/timesyncd/dhcp-propagation/data/ntplib.sh
new file mode 120000
index 0000000000..07da484d88
--- /dev/null
+++ b/tests/kola/ntp/timesyncd/dhcp-propagation/data/ntplib.sh
@@ -0,0 +1 @@
+../../../data/ntplib.sh
\ No newline at end of file
diff --git a/tests/kola/ntp/timesyncd/dhcp-propagation/test.sh b/tests/kola/ntp/timesyncd/dhcp-propagation/test.sh
new file mode 100755
index 0000000000..f0baf6b46b
--- /dev/null
+++ b/tests/kola/ntp/timesyncd/dhcp-propagation/test.sh
@@ -0,0 +1,46 @@
+#!/bin/bash
+## kola:
+##   # Add the needs-internet tag. This test builds a container from remote
+##   # sources and uses a remote NTP server.
+##   tags: "needs-internet"
+##   # Limit to qemu because some cloud providers have their own special
+##   # sauce for NTP/chrony and we don't want to interfere with that.
+##   platforms: qemu
+##   # Pulling and building the container can take a long time if a
+##   # slow mirror gets chosen. Bump the timeout to 15 minutes
+##   timeoutMin: 15
+##   # There's a bug in dnf that is causing OOM on low memory systems:
+##   # https://bugzilla.redhat.com/show_bug.cgi?id=1907030
+##   # https://pagure.io/releng/issue/10935#comment-808601
+##   minMemory: 1536
+##   # We only care about timesyncd in Fedora. It's not available elsewhere.
+##   distros: fcos
+##   description: Verify that timesyncd service got ntp servers from DHCP.
+#
+# This script creates two veth interfaces i.e. one for the host machine
+# and other for the container(dnsmasq server). This setup will be helpful
+# to verify the DHCP propagation of NTP servers. This will also avoid any
+# regression that might cause in RHCOS or FCOS when the upstream changes
+# come down and obsolete the temporary work (https://github.com/coreos/fedora-coreos-config/pull/412)
+
+set -xeuo pipefail
+
+# shellcheck disable=SC1091
+. "$KOLA_EXT_DATA/commonlib.sh"
+. $KOLA_EXT_DATA/ntplib.sh
+
+main() {
+
+    # Choose a host from https://tf.nist.gov/tf-cgi/servers.cgi
+    # that can get DNS over DNSSEC since the environment our OpenShift
+    # runs in requires DNSSEC validation. Test with https://dnsviz.net/
+    ntp_host_ip=$(getent hosts utcnist.colorado.edu | cut -d ' ' -f 1)
+
+    ntp_test_setup $ntp_host_ip
+
+    check_for_ntp_server $ntp_host_ip "timedatectl show-timesync -p ServerAddress --value"
+
+    ok "timesyncd got ntp servers from DHCP"
+}
+
+main
diff --git a/tests/kola/platforms/aws/assert-xen b/tests/kola/platforms/aws/assert-xen
new file mode 100755
index 0000000000..cb2bdba084
--- /dev/null
+++ b/tests/kola/platforms/aws/assert-xen
@@ -0,0 +1,25 @@
+#!/bin/bash
+## kola:
+##   # This is a read-only test and can be run with other tests
+##   exclusive: false
+##   # This test is targeted at AWS
+##   platforms: aws
+##   # Force this test to not run by default unless named specifically
+##   # or `--tag aws-xen-test` is passed to `kola run`. i.e. this test
+##   # should only run on Xen instances and the caller should request
+##   # the test.
+##   requiredTag: aws-xen-test
+##   description: Verify that the booted AWS instance is XEN based.
+
+set -xeuo pipefail
+
+# shellcheck disable=SC1091
+. "$KOLA_EXT_DATA/commonlib.sh"
+
+token=$(curl -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 21600")
+hypervisor=$(curl -H "X-aws-ec2-metadata-token: $token" http://169.254.169.254/2022-09-24/meta-data/system || true)
+if [ "${hypervisor}" != "xen" ]; then
+    fatal "expected xen instance type"
+fi
+
+ok xen instance type
diff --git a/tests/kola/platforms/aws/data/commonlib.sh b/tests/kola/platforms/aws/data/commonlib.sh
new file mode 120000
index 0000000000..b8dcbdca1a
--- /dev/null
+++ b/tests/kola/platforms/aws/data/commonlib.sh
@@ -0,0 +1 @@
+../../../data/commonlib.sh
\ No newline at end of file
diff --git a/tests/kola/platforms/aws/nvme b/tests/kola/platforms/aws/nvme
new file mode 100755
index 0000000000..3415544c63
--- /dev/null
+++ b/tests/kola/platforms/aws/nvme
@@ -0,0 +1,51 @@
+#!/bin/bash
+## kola:
+##   # This is a read-only test and can be run with other tests
+##   exclusive: false
+##   # This test is targeted at AWS
+##   platforms: aws
+##   description: Verify that we crate an AWS instances with nvme storage.
+
+# See https://github.com/coreos/fedora-coreos-tracker/issues/1306
+#
+# We'll check to see if anything is listed in `nvme list-subsys`
+# and then run the test based on that. Example output:
+#
+# [core@ip-10-0-1-155 ~]$ nvme list-subsys -o json
+# [
+#   {
+#     "HostNQN":"nqn.2014-08.org.nvmexpress:uuid:c2c16640-0d5c-472a-846c-2f981c5914db",
+#     "HostID":"c37c43ca-31a3-4015-8704-1f493081da36",
+#     "Subsystems":[
+#       {
+#         "Name":"nvme-subsys0",
+#         "NQN":"nqn.2014.08.org.nvmexpress:1d0f0000AWS2710D45D72D3C9D4AAmazon EC2 NVMe Instance Storage",
+#         "Paths":[
+#           {
+#             "Name":"nvme0",
+#             "Transport":"pcie",
+#             "Address":"0000:00:1e.0",
+#             "State":"live"
+#           }
+#         ]
+#       }
+#     ]
+#   }
+# ]
+
+set -xeuo pipefail
+
+# shellcheck disable=SC1091
+. "$KOLA_EXT_DATA/commonlib.sh"
+
+nvme_info=$(nvme list-subsys -o json || true)
+has_nvme=$(jq -r ".[].Subsystems[].Paths[] | select(.Name == \"nvme0\").Name" <<< "$nvme_info")
+if [ -n "${has_nvme}" ]; then
+    if [ ! -e '/dev/nvme0n1' ]; then
+        fatal "instance has nvme device but no nvme0n1 accessible"
+    fi
+else
+    echo "it appears this system has no nvme devices. skipping"
+fi
+
+ok aws nvme device
diff --git a/tests/kola/platforms/gcp/confidential-vm-nvme-symlink b/tests/kola/platforms/gcp/confidential-vm-nvme-symlink
new file mode 100755
index 0000000000..5f340345b5
--- /dev/null
+++ b/tests/kola/platforms/gcp/confidential-vm-nvme-symlink
@@ -0,0 +1,81 @@
+#!/bin/bash
+## kola:
+##   description: Verify new GCP udev rules work well on confidential instance.
+##   # Note: each local SSD is 375 GB in size, refer to https://cloud.google.com/compute/docs/disks/local-ssd
+##   additionalDisks: ["375G:channel=nvme"]
+##   platforms: gcp
+##   requiredTag: confidential
+
+# See https://issues.redhat.com/browse/OCPBUGS-7582
+# https://github.com/coreos/fedora-coreos-tracker/issues/1457
+# https://github.com/coreos/coreos-assembler/issues/3556
+#
+# Force this test to not run by default unless named specifically
+# or `--tag confidential` is passed to `kola run`, also requires
+# `--gcp-machinetype n2d-standard-2 --gcp-confidential-type sev_snp`
+#
+# It will create confidential instance on GCP with 1 nvme persistent disk
+# and 1 local ssd disk, then check the new udev rules make effect.
+
+set -xeuo pipefail
+
+# shellcheck disable=SC1091
+. "$KOLA_EXT_DATA/commonlib.sh"
+
+# Set global variable with NVME json info
+NVME_INFO=$(nvme list-subsys -o json)
+
+# verify the instance is Confidential VM
+assert_confidential_vm() {
+    local sevlog=$(dmesg | grep SEV-SNP | head)
+    if [ -n "${sevlog}" ] && echo "${sevlog}" | grep "Memory Encryption Features active: AMD SEV SEV-ES SEV-SNP"; then
+        ok "instance is Confidential VM"
+    else
+        fatal "instance should be Confidential VM"
+    fi
+}
+
+# check instance has 2 disks
+assert_two_nvme_disks() {
+    local nvme_count=$(jq -r ".[].Subsystems | length" <<< "${NVME_INFO}")
+    if [ $nvme_count -ne 2 ]; then
+        fatal "instance does not have 2 disks"
+    fi
+}
+
+# check nvme device
+assert_nvme_disk_accessible() {
+    local disk=$1
+    local nvme_disk=$(jq -r ".[].Subsystems[].Paths[] | select(.Name == \"${disk}\").Name" <<< "${NVME_INFO}")
+    if [ -n "${nvme_disk}" ]; then
+        if [ ! -e "/dev/${disk}n1" ]; then
+            fatal "instance has nvme device but no ${disk} accessible"
+        fi
+    else
+        fatal "can not find ${disk} on the instance"
+    fi
+}
+
+# check symlink
+assert_expected_symlink_exists() {
+    local device=$1
+    # Run google_nvme_id to populate ID_SERIAL_SHORT env var
+    eval $(/usr/lib/udev/google_nvme_id -d "${device}")
+    if [ ! -n "${ID_SERIAL_SHORT:-}" ]; then
+        fatal "can not get nvme ${device} ID_SERIAL_SHORT"
+    fi
+
+    local link="/dev/disk/by-id/google-${ID_SERIAL_SHORT}"
+    if ! ls -l "${link}"; then
+        fatal "can not find ${device} symlink ${link}"
+    fi
+}
+
+assert_confidential_vm
+assert_two_nvme_disks
+
+for disk in nvme0 nvme1; do
+    assert_nvme_disk_accessible $disk
+    assert_expected_symlink_exists "/dev/${disk}n1"
+    ok "Found /dev/${disk}n1 symlink"
+done
diff --git a/tests/kola/platforms/gcp/data/commonlib.sh b/tests/kola/platforms/gcp/data/commonlib.sh
new file mode 120000
index 0000000000..b8dcbdca1a
--- /dev/null
+++ b/tests/kola/platforms/gcp/data/commonlib.sh
@@ -0,0 +1 @@
+../../../data/commonlib.sh
\ No newline at end of file
diff --git a/tests/kola/podman/data/commonlib.sh b/tests/kola/podman/data/commonlib.sh
new file mode 120000
index 0000000000..1742d51e67
--- /dev/null
+++ b/tests/kola/podman/data/commonlib.sh
@@ -0,0 +1 @@
+../../data/commonlib.sh
\ No newline at end of file
diff --git a/tests/kola/podman/dns/data/commonlib.sh b/tests/kola/podman/dns/data/commonlib.sh
new file mode 120000
index 0000000000..b8dcbdca1a
--- /dev/null
+++ b/tests/kola/podman/dns/data/commonlib.sh
@@ -0,0 +1 @@
+../../../data/commonlib.sh
\ No newline at end of file
diff --git a/tests/kola/podman/dns/test.sh b/tests/kola/podman/dns/test.sh
new file mode 100755
index 0000000000..e77836d192
--- /dev/null
+++ b/tests/kola/podman/dns/test.sh
@@ -0,0 +1,46 @@
+#!/bin/bash
+## kola:
+##   # This test pulls a container from a registry.
+##   tags: "platform-independent needs-internet"
+##   # This test doesn't make meaningful changes to the system and
+##   # should be able to be combined with other tests.
+##   exclusive: false
+##   # This test reaches out to the internet and it could take more
+##   # time to pull down the container.
+##   timeoutMin: 3
+##   description: Verify that DNS in rootless podman containers can
+##     resolve external domains.
+
+# See https://github.com/coreos/fedora-coreos-tracker/issues/923
+
+set -xeuo pipefail
+
+# shellcheck disable=SC1091
+. "$KOLA_EXT_DATA/commonlib.sh"
+
+runascoreuserscript='
+#!/bin/bash
+set -euxo pipefail
+
+podman network create testnetwork
+podman run --rm -t --network=testnetwork quay.io/fedora/fedora:40 getent hosts google.com
+podman network rm testnetwork
+'
+
+runascoreuser() {
+    # NOTE: If we don't use `| cat` the output won't get copied
+    # and won't show up in the output of the ext test.
+    sudo -u core "$@" | cat
+}
+
+main() {
+    echo "$runascoreuserscript" > /tmp/runascoreuserscript
+    chmod +x /tmp/runascoreuserscript
+    if ! runascoreuser /tmp/runascoreuserscript ; then
+        fatal "DNS in rootless podman testnetwork failed. Test Fails"
+    else
+        ok "DNS in rootless podman testnetwork Suceeded. Test Passes"
+    fi
+}
+
+main
diff --git a/tests/kola/podman/rootless-pasta-networking b/tests/kola/podman/rootless-pasta-networking
new file mode 100755
index 0000000000..594743a6ee
--- /dev/null
+++ b/tests/kola/podman/rootless-pasta-networking
@@ -0,0 +1,55 @@
+#!/bin/bash
+## kola:
+##   description: Verify that rootless pasta networking passt works.
+##   # This test downloads containers and curls from the net.
+##   tags: "platform-independent needs-internet"
+##   # This test doesn't make meaningful changes to the system and
+##   # should be able to be combined with other tests.
+##   exclusive: false
+##   # This test reaches out to the internet and it could take more
+##   # time to pull down the container.
+##   timeoutMin: 3
+
+# See https://github.com/coreos/fedora-coreos-tracker/issues/1436
+
+set -xeuo pipefail
+
+# shellcheck disable=SC1091
+. "$KOLA_EXT_DATA/commonlib.sh"
+
+runascoreuserscript='#!/bin/bash
+set -euxo pipefail
+# Just a basic test that uses pasta network and sets the gateway
+podman run -i --net=pasta:--mtu,1500 quay.io/fedora/fedora:40 bash <<"EOF"
+set -euxo pipefail
+# Verify the mtu got set to 1500. No /sbin/ip so just use /sys/class/net/<nic>/mtu
+cat /sys/class/net/e*/mtu | grep 1500
+# Download something from the internet. Here we use one of the test
+# fixtures from the ignition.resource.remote test.
+result=$(curl https://ignition-test-fixtures.s3.amazonaws.com/resources/anonymous)
+[ "$result" == "kola-anonymous" ] || exit 1
+EOF
+'
+
+runascoreuser() {
+    # NOTE: If we don't use `| cat` the output won't get copied
+    # to our unit and won't show up in the `systemctl status` output
+    # of the ext test.
+    sudo -u core "$@" | cat
+}
+
+main() {
+
+    # Execute script as the core user to exercise rootless podman
+    runascoreuserscriptpath=$(mktemp --suffix=runascoreuser)
+    echo "$runascoreuserscript" > $runascoreuserscriptpath
+    chmod +x $runascoreuserscriptpath
+    chown core $runascoreuserscriptpath
+    if runascoreuser $runascoreuserscriptpath; then
+        ok "Podman with pasta networking succeeded!"
+    else
+        fatal "Podman with pasta networking failed"
+    fi
+}
+
+main
diff --git a/tests/kola/podman/rootless-systemd b/tests/kola/podman/rootless-systemd
index 13874a5e76..5f0e08d928 100755
--- a/tests/kola/podman/rootless-systemd
+++ b/tests/kola/podman/rootless-systemd
@@ -1,20 +1,30 @@
 #!/bin/bash
+## kola:
+##   # This test builds a container from remote sources.
+##   # This test uses a remote NTP server.
+##   tags: "platform-independent needs-internet"
+##   # Pulling and building the container can take a long time if a
+##   # slow mirror gets chosen.
+##   timeoutMin: 15
+##   # There's a bug in dnf that is causing OOM on low memory systems:
+##   # https://bugzilla.redhat.com/show_bug.cgi?id=1907030
+##   # https://pagure.io/releng/issue/10935#comment-808601
+##   minMemory: 1536
+##   description: Verify that rootless+systemd works.
 
-# This script runs a rootless podman container with systemd inside
-# that brings up httpd. Tests that rootless+systemd works. See issue:
-# https://github.com/containers/podman/issues/7441
+# See https://github.com/containers/podman/issues/7441
+#
+# This script runs a rootless podman container (rootless because it's
+# run as the `core` user) with systemd inside that brings up httpd.
 #
 # If it gets easy to change the kargs in the future we should try this
 # both on cgroups v1 and cgroups v2.
 
-set -euxo pipefail
+set -xeuo pipefail
 
-# Just run on QEMU for now. If this test works in one place it should
-# work everywhere.
-# kola: { "tags": "needs-internet", "platforms": "qemu-unpriv" }
+# shellcheck disable=SC1091
+. "$KOLA_EXT_DATA/commonlib.sh"
 
-# Script to be executed as the `core` user to build and
-# run a rootless podman container that uses systemd inside.
 runascoreuserscript='
 #!/bin/bash
 set -euxo pipefail
@@ -25,8 +35,11 @@ set -euxo pipefail
 #       https://github.com/coreos/coreos-assembler/issues/1645
 cd $(mktemp -d)
 cat <<EOF > Containerfile
-FROM registry.fedoraproject.org/fedora:33
-RUN dnf -y install systemd httpd \
+FROM quay.io/fedora/fedora:40
+RUN rm -f /etc/yum.repos.d/*.repo \
+&& curl -L https://raw.githubusercontent.com/coreos/fedora-coreos-config/testing-devel/fedora-archive.repo -o /etc/yum.repos.d/fedora-archive.repo
+RUN dnf -y update \
+&& dnf -y install systemd httpd \
 && dnf clean all \
 && systemctl enable httpd
 ENTRYPOINT [ "/sbin/init" ]
@@ -52,15 +65,16 @@ main() {
     chmod +x /tmp/runascoreuserscript
     runascoreuser /tmp/runascoreuserscript
 
-    # Let it come up
     sleep 5
 
-    if ! curl http://localhost:8080 1>/dev/null; then
-        echo TEST FAILED 1>&2
+    retryflag="--retry-all-errors"
+    # Try to grab the web page. Retry as it might not be up fully yet.
+    if ! curl --silent --show-error --retry 5 ${retryflag} http://localhost:8080 >/dev/null; then
         runascoreuser podman logs httpd
-        return 1
+        fatal "setup http server container failed"
     fi
+
+    ok "setup http server container successfully"
 }
 
 main
-exit $?
diff --git a/tests/kola/reboot/data/commonlib.sh b/tests/kola/reboot/data/commonlib.sh
new file mode 120000
index 0000000000..1742d51e67
--- /dev/null
+++ b/tests/kola/reboot/data/commonlib.sh
@@ -0,0 +1 @@
+../../data/commonlib.sh
\ No newline at end of file
diff --git a/tests/kola/reboot/test.sh b/tests/kola/reboot/test.sh
new file mode 100755
index 0000000000..35b6d24a68
--- /dev/null
+++ b/tests/kola/reboot/test.sh
@@ -0,0 +1,67 @@
+#!/bin/bash
+## kola:
+##   platforms: qemu
+##   description: Verify default system configuration are both on first and
+##     subsequent boots.
+
+set -xeuo pipefail
+
+# shellcheck disable=SC1091
+. "$KOLA_EXT_DATA/commonlib.sh"
+
+# /var
+varsrc=$(findmnt -nvr /var -o SOURCE)
+rootsrc=$(findmnt -nvr /sysroot -o SOURCE)
+[[ $(realpath "$varsrc") == $(realpath "$rootsrc") ]]
+ok "/var is backed by rootfs"
+
+# sanity-check that boot is mounted by UUID
+if ! systemctl cat boot.mount | grep -q What=/dev/disk/by-uuid; then
+  systemctl cat boot.mount
+  fatal "boot mounted not by UUID"
+fi
+ok "boot mounted by UUID"
+
+# check that we took ownership of the bootfs
+[ -f /boot/.root_uuid ]
+
+# s390x does not have grub, skip this part
+
+if [ "$(arch)" == "ppc64le" ] || [ "$(arch)" == "s390x" ]; then
+  echo "skipping EFI verification on arch $(arch)"
+else
+  # check for the UUID dropins
+  [ -f /boot/grub2/bootuuid.cfg ]
+  mount -o ro /dev/disk/by-label/EFI-SYSTEM /boot/efi
+  found_bootuuid="false"
+  for f in /boot/efi/EFI/*/bootuuid.cfg; do
+      if [ -f "$f" ]; then
+          found_bootuuid="true"
+      fi
+  done
+  if [[ "${found_bootuuid}" == "false" ]]; then
+      fatal "No /boot/efi/EFI/*/bootuuid.cfg found"
+  fi
+  umount /boot/efi
+fi
+
+case "${AUTOPKGTEST_REBOOT_MARK:-}" in
+  "")
+      ok "first boot"
+      /tmp/autopkgtest-reboot rebooted
+      ;;
+
+  rebooted)
+      # check for expected default kargs
+      grep root=UUID="$(cat /boot/.root_uuid)" /proc/cmdline
+      ok "found root karg"
+
+      bootsrc=$(findmnt -nvr /boot -o SOURCE)
+      eval $(blkid -p -o export "${bootsrc}")
+      grep boot=UUID="${UUID}" /proc/cmdline
+      ok "found boot karg"
+
+      ok "second boot"
+      ;;
+  *) fatal "unexpected mark: ${AUTOPKGTEST_REBOOT_MARK}";;
+esac
diff --git a/tests/kola/root-reprovision/autosave-xfs/data/commonlib.sh b/tests/kola/root-reprovision/autosave-xfs/data/commonlib.sh
new file mode 120000
index 0000000000..b8dcbdca1a
--- /dev/null
+++ b/tests/kola/root-reprovision/autosave-xfs/data/commonlib.sh
@@ -0,0 +1 @@
+../../../data/commonlib.sh
\ No newline at end of file
diff --git a/tests/kola/root-reprovision/autosave-xfs/test.sh b/tests/kola/root-reprovision/autosave-xfs/test.sh
new file mode 100755
index 0000000000..8c6fb6e045
--- /dev/null
+++ b/tests/kola/root-reprovision/autosave-xfs/test.sh
@@ -0,0 +1,33 @@
+#!/bin/bash
+## kola:
+##   # This test reprovisions the rootfs automatically.
+##   tags: "platform-independent reprovision"
+##   # Trigger automatic XFS reprovisioning (heuristic)
+##   minDisk: 1000
+##   # Root reprovisioning requires at least 4GiB of memory.
+##   minMemory: 4096
+##   # This test includes a lot of disk I/O and needs a higher
+##   # timeout value than the default.
+##   timeoutMin: 15
+##   description: Verify the root reprovision with XFS
+##     on large disk triggers autosaved.
+##     This test is meant to cover ignition-ostree-transposefs-autosave-xfs.service
+
+set -xeuo pipefail
+
+# shellcheck disable=SC1091
+. "$KOLA_EXT_DATA/commonlib.sh"
+
+if [ ! -f /run/ignition-ostree-autosaved-xfs.stamp ]; then
+    fatal "expected autosaved XFS"
+fi
+# Verify we printed something about the agcount
+journalctl -u ignition-ostree-transposefs-autosave-xfs.service --grep=agcount
+ok "autosaved XFS on large disk"
+
+eval $(xfs_info /sysroot | grep -o 'agcount=[0-9]*')
+expected=4
+if [ "$agcount" -gt "$expected" ]; then
+    fatal "expected agcount of at most ${expected}, got ${agcount}"
+fi
+ok "low agcount on large disk"
diff --git a/tests/kola/root-reprovision/filesystem-only/config.ign b/tests/kola/root-reprovision/filesystem-only/config.ign
index 3784c8cbfe..28d6ca57f5 100644
--- a/tests/kola/root-reprovision/filesystem-only/config.ign
+++ b/tests/kola/root-reprovision/filesystem-only/config.ign
@@ -1,6 +1,6 @@
 {
     "ignition": {
-        "version": "3.0.0"
+        "version": "3.2.0"
     },
     "storage": {
         "filesystems": [
@@ -8,7 +8,8 @@
                 "device": "/dev/disk/by-label/root",
                 "wipeFilesystem": true,
                 "format": "ext4",
-                "label": "root"
+                "label": "root",
+                "mountOptions": ["debug"]
             }
         ]
     }
diff --git a/tests/kola/root-reprovision/filesystem-only/data/commonlib.sh b/tests/kola/root-reprovision/filesystem-only/data/commonlib.sh
new file mode 120000
index 0000000000..b8dcbdca1a
--- /dev/null
+++ b/tests/kola/root-reprovision/filesystem-only/data/commonlib.sh
@@ -0,0 +1 @@
+../../../data/commonlib.sh
\ No newline at end of file
diff --git a/tests/kola/root-reprovision/filesystem-only/test.sh b/tests/kola/root-reprovision/filesystem-only/test.sh
index 10b2faf8b8..6dd520d5bd 100755
--- a/tests/kola/root-reprovision/filesystem-only/test.sh
+++ b/tests/kola/root-reprovision/filesystem-only/test.sh
@@ -1,16 +1,34 @@
 #!/bin/bash
-# kola: {"platforms": "qemu", "minMemory": 4096}
+## kola:
+##   # This test reprovisions the rootfs.
+##   tags: "platform-independent reprovision"
+##   # Root reprovisioning requires at least 4GiB of memory.
+##   minMemory: 4096
+##   # This test includes a lot of disk I/O and needs a higher
+##   # timeout value than the default.
+##   timeoutMin: 15
+##   description: Verify the root reprovisioning with specified file system works.
+
 set -xeuo pipefail
 
-fstype=$(findmnt -nvr / -o FSTYPE)
+# shellcheck disable=SC1091
+. "$KOLA_EXT_DATA/commonlib.sh"
+
+fstype=$(findmnt -nvr /sysroot -o FSTYPE)
 [[ $fstype == ext4 ]]
+ok "source is ext4"
+
+rootflags=$(findmnt /sysroot -no OPTIONS)
+if ! grep debug <<< "${rootflags}"; then
+    fatal "missing debug in root mount flags: ${rootflags}"
+fi
+ok "root mounted with debug"
 
 case "${AUTOPKGTEST_REBOOT_MARK:-}" in
   "")
       # check that the partition was grown
-      if [ ! -e /run/coreos-growpart.stamp ]; then
-          echo "coreos-growpart did not run"
-          exit 1
+      if [ ! -e /run/ignition-ostree-growfs.stamp ]; then
+          fatal "ignition-ostree-growfs did not run"
       fi
 
       # reboot once to sanity-check we can find root on second boot
@@ -19,6 +37,7 @@ case "${AUTOPKGTEST_REBOOT_MARK:-}" in
 
   rebooted)
       grep root=UUID= /proc/cmdline
+      ok "found root karg"
       ;;
-  *) echo "unexpected mark: ${AUTOPKGTEST_REBOOT_MARK}"; exit 1;;
+  *) fatal "unexpected mark: ${AUTOPKGTEST_REBOOT_MARK}";;
 esac
diff --git a/tests/kola/root-reprovision/luks/512e/config.ign b/tests/kola/root-reprovision/luks/512e/config.ign
new file mode 120000
index 0000000000..f72ce41f73
--- /dev/null
+++ b/tests/kola/root-reprovision/luks/512e/config.ign
@@ -0,0 +1 @@
+../config.ign
\ No newline at end of file
diff --git a/tests/kola/root-reprovision/luks/512e/data b/tests/kola/root-reprovision/luks/512e/data
new file mode 120000
index 0000000000..4909e06efb
--- /dev/null
+++ b/tests/kola/root-reprovision/luks/512e/data
@@ -0,0 +1 @@
+../data
\ No newline at end of file
diff --git a/tests/kola/root-reprovision/luks/512e/test.sh b/tests/kola/root-reprovision/luks/512e/test.sh
new file mode 100755
index 0000000000..54e9b9a077
--- /dev/null
+++ b/tests/kola/root-reprovision/luks/512e/test.sh
@@ -0,0 +1,36 @@
+#!/bin/bash
+## kola:
+##   # This test reprovisions the rootfs.
+##   tags: "reprovision"
+##   # This uses additionalDisks, which is QEMU only
+##   platforms: qemu
+##   # Root reprovisioning requires at least 4GiB of memory.
+##   minMemory: 4096
+##   # A TPM backend device is not available on s390x to suport TPM.
+##   architectures: "! s390x"
+##   # This test includes a lot of disk I/O and needs a higher
+##   # timeout value than the default.
+##   timeoutMin: 15
+##   description: Verify that LUKS on a 512e disks works.
+##   primaryDisk: ":512e"
+
+set -xeuo pipefail
+
+# shellcheck disable=SC1091
+. "$KOLA_EXT_DATA/commonlib.sh"
+
+# sanity-check that it's a 512e disk
+phy_sec=$(blockdev --getpbsz /dev/disk/by-id/virtio-primary-disk)
+log_sec=$(blockdev --getss /dev/disk/by-id/virtio-primary-disk)
+if [ "${phy_sec}" != 4096 ] || [ "${log_sec}" != 512 ]; then
+    fatal "root device isn't 512e"
+fi
+
+# sanity-check that LUKS chose a 4096 sector size
+luks_sec=$(blockdev --getss /dev/mapper/myluksdev)
+if [ "${luks_sec}" != 4096 ]; then
+    fatal "root LUKS device isn't 4k"
+fi
+
+# run the rest of the tests
+. $KOLA_EXT_DATA/luks-test.sh
diff --git a/tests/kola/root-reprovision/luks/autosave-xfs/config.ign b/tests/kola/root-reprovision/luks/autosave-xfs/config.ign
new file mode 120000
index 0000000000..f72ce41f73
--- /dev/null
+++ b/tests/kola/root-reprovision/luks/autosave-xfs/config.ign
@@ -0,0 +1 @@
+../config.ign
\ No newline at end of file
diff --git a/tests/kola/root-reprovision/luks/autosave-xfs/data b/tests/kola/root-reprovision/luks/autosave-xfs/data
new file mode 120000
index 0000000000..4909e06efb
--- /dev/null
+++ b/tests/kola/root-reprovision/luks/autosave-xfs/data
@@ -0,0 +1 @@
+../data
\ No newline at end of file
diff --git a/tests/kola/root-reprovision/luks/autosave-xfs/test.sh b/tests/kola/root-reprovision/luks/autosave-xfs/test.sh
new file mode 100755
index 0000000000..cde6748231
--- /dev/null
+++ b/tests/kola/root-reprovision/luks/autosave-xfs/test.sh
@@ -0,0 +1,39 @@
+#!/bin/bash
+## kola:
+##   # This test reprovisions the rootfs.
+##   tags: "platform-independent reprovision"
+##   # Root reprovisioning requires at least 4GiB of memory.
+##   minMemory: 4096
+##   # A TPM backend device is not available on s390x to suport TPM.
+##   architectures: "! s390x"
+##   # This test includes a lot of disk I/O and needs a higher
+##   # timeout value than the default.
+##   timeoutMin: 15
+##   # Trigger automatic XFS reprovisioning
+##   minDisk: 1000
+##   description: Verify the root reprovision with XFS and TPM
+##     on large disk triggers autosaved.
+##     This test is meant to cover ignition-ostree-transposefs-autosave-xfs.service
+
+set -xeuo pipefail
+
+# shellcheck disable=SC1091
+. "$KOLA_EXT_DATA/commonlib.sh"
+
+# check that we ran automatic XFS reprovisioning
+if [ -z "${AUTOPKGTEST_REBOOT_MARK:-}" ]; then
+    if [ ! -f /run/ignition-ostree-autosaved-xfs.stamp ]; then
+        fatal "expected autosaved XFS"
+    fi
+    ok "autosaved XFS on large disk"
+
+    eval $(xfs_info /sysroot | grep -o 'agcount=[0-9]*')
+    expected=4
+    if [ "$agcount" -gt "${expected}" ]; then
+        fatal "expected agcount of at most ${expected}, got ${agcount}"
+    fi
+    ok "low agcount on large disk"
+fi
+
+# run the rest of the tests
+. $KOLA_EXT_DATA/luks-test.sh
diff --git a/tests/kola/root-reprovision/luks/config.ign b/tests/kola/root-reprovision/luks/config.ign
index 950aa670d8..07317d10b4 100644
--- a/tests/kola/root-reprovision/luks/config.ign
+++ b/tests/kola/root-reprovision/luks/config.ign
@@ -1,6 +1,6 @@
 {
   "ignition": {
-    "version": "3.2.0"
+    "version": "3.4.0"
   },
   "storage": {
     "luks": [
@@ -10,6 +10,8 @@
         "clevis": {
           "tpm2": true
         },
+        "discard": true,
+        "openOptions": ["--perf-no_read_workqueue"],
         "label": "root",
         "wipeVolume": true
       }
diff --git a/tests/kola/root-reprovision/luks/data/commonlib.sh b/tests/kola/root-reprovision/luks/data/commonlib.sh
new file mode 120000
index 0000000000..b8dcbdca1a
--- /dev/null
+++ b/tests/kola/root-reprovision/luks/data/commonlib.sh
@@ -0,0 +1 @@
+../../../data/commonlib.sh
\ No newline at end of file
diff --git a/tests/kola/root-reprovision/luks/data/luks-test.sh b/tests/kola/root-reprovision/luks/data/luks-test.sh
new file mode 100755
index 0000000000..615b4fd175
--- /dev/null
+++ b/tests/kola/root-reprovision/luks/data/luks-test.sh
@@ -0,0 +1,60 @@
+# This file is sourced by both `ext.config.root-reprovision.luks`
+# and `ext.config.root-reprovision.luks.autosave-xfs`.
+
+# shellcheck disable=SC1091
+. "$KOLA_EXT_DATA/commonlib.sh"
+
+srcdev=$(findmnt -nvr /sysroot -o SOURCE)
+[[ ${srcdev} == /dev/mapper/myluksdev ]]
+
+blktype=$(lsblk -o TYPE "${srcdev}" --noheadings)
+[[ ${blktype} == crypt ]]
+
+fstype=$(findmnt -nvr /sysroot -o FSTYPE)
+[[ ${fstype} == xfs ]]
+ok "source is XFS on LUKS device"
+
+rootflags=$(findmnt /sysroot -no OPTIONS)
+if ! grep prjquota <<< "${rootflags}"; then
+    fatal "missing prjquota in root mount flags: ${rootflags}"
+fi
+ok "root mounted with prjquota"
+
+table=$(dmsetup table myluksdev)
+if ! grep -q allow_discards <<< "${table}"; then
+    fatal "missing allow_discards in root DM table: ${table}"
+fi
+if ! grep -q no_read_workqueue <<< "${table}"; then
+    fatal "missing no_read_workqueue in root DM table: ${table}"
+fi
+ok "discard and custom option enabled for root LUKS"
+
+# while we're here, sanity-check that boot is mounted by UUID
+if ! systemctl cat boot.mount | grep -q What=/dev/disk/by-uuid; then
+  systemctl cat boot.mount
+  fatal "boot mounted not by UUID"
+fi
+ok "boot mounted by UUID"
+
+case "${AUTOPKGTEST_REBOOT_MARK:-}" in
+  "")
+      # check that ignition-ostree-growfs ran
+      if [ ! -e /run/ignition-ostree-growfs.stamp ]; then
+          fatal "ignition-ostree-growfs did not run"
+      fi
+
+      # reboot once to sanity-check we can find root on second boot
+      /tmp/autopkgtest-reboot rebooted
+      ;;
+
+  rebooted)
+      grep root=UUID= /proc/cmdline
+      grep rd.luks.name= /proc/cmdline
+      ok "found root kargs"
+
+      # while we're here, sanity-check that we have a boot=UUID karg too
+      grep boot=UUID= /proc/cmdline
+      ok "found boot karg"
+      ;;
+  *) fatal "unexpected mark: ${AUTOPKGTEST_REBOOT_MARK}";;
+esac
diff --git a/tests/kola/root-reprovision/luks/test.sh b/tests/kola/root-reprovision/luks/test.sh
index c27672deb4..60b1585e02 100755
--- a/tests/kola/root-reprovision/luks/test.sh
+++ b/tests/kola/root-reprovision/luks/test.sh
@@ -1,31 +1,29 @@
 #!/bin/bash
-# kola: {"platforms": "qemu", "minMemory": 4096, "architectures": "x86_64"}
-set -xeuo pipefail
-
-srcdev=$(findmnt -nvr / -o SOURCE)
-[[ ${srcdev} == /dev/mapper/myluksdev ]]
+## kola:
+##   # This test reprovisions the rootfs.
+##   tags: "platform-independent reprovision"
+##   # Root reprovisioning requires at least 4GiB of memory.
+##   minMemory: 4096
+##   # A TPM backend device is not available on s390x to suport TPM.
+##   architectures: "! s390x"
+##   # This test includes a lot of disk I/O and needs a higher
+##   # timeout value than the default.
+##   timeoutMin: 15
+##   description: Verify the root reprovision with XFS and TPM
+##     does not trigger autosaved.
 
-blktype=$(lsblk -o TYPE "${srcdev}" --noheadings)
-[[ ${blktype} == crypt ]]
-
-fstype=$(findmnt -nvr / -o FSTYPE)
-[[ ${fstype} == xfs ]]
+set -xeuo pipefail
 
-case "${AUTOPKGTEST_REBOOT_MARK:-}" in
-  "")
-      # check that growpart didn't run
-      if [ -e /run/coreos-growpart.stamp ]; then
-          echo "coreos-growpart ran"
-          exit 1
-      fi
+# shellcheck disable=SC1091
+. "$KOLA_EXT_DATA/commonlib.sh"
 
-      # reboot once to sanity-check we can find root on second boot
-      /tmp/autopkgtest-reboot rebooted
-      ;;
+# check that we didn't run automatic XFS reprovisioning
+if [ -z "${AUTOPKGTEST_REBOOT_MARK:-}" ]; then
+    if [ -f /run/ignition-ostree-autosaved-xfs.stamp ]; then
+        fatal "unexpected autosaved XFS"
+    fi
+    ok "no autosaved XFS on large disk"
+fi
 
-  rebooted)
-      grep root=UUID= /proc/cmdline
-      grep rd.luks.name= /proc/cmdline
-      ;;
-  *) echo "unexpected mark: ${AUTOPKGTEST_REBOOT_MARK}"; exit 1;;
-esac
+# run the rest of the tests
+. $KOLA_EXT_DATA/luks-test.sh
diff --git a/tests/kola/root-reprovision/raid1/data/commonlib.sh b/tests/kola/root-reprovision/raid1/data/commonlib.sh
new file mode 120000
index 0000000000..b8dcbdca1a
--- /dev/null
+++ b/tests/kola/root-reprovision/raid1/data/commonlib.sh
@@ -0,0 +1 @@
+../../../data/commonlib.sh
\ No newline at end of file
diff --git a/tests/kola/root-reprovision/raid1/test.sh b/tests/kola/root-reprovision/raid1/test.sh
index 7ecc340e60..0abc7deecb 100755
--- a/tests/kola/root-reprovision/raid1/test.sh
+++ b/tests/kola/root-reprovision/raid1/test.sh
@@ -1,22 +1,49 @@
 #!/bin/bash
-# kola: {"platforms": "qemu", "minMemory": 4096, "additionalDisks": ["5G", "5G"]}
+## kola:
+##   # additionalDisks is only supported on qemu.
+##   platforms: qemu
+##   # Root reprovisioning requires at least 4GiB of memory.
+##   minMemory: 4096
+##   # Linear RAID is setup on these disks.
+##   additionalDisks: ["5G", "5G"]
+##   # This test includes a lot of disk I/O and needs a higher
+##   # timeout value than the default.
+##   timeoutMin: 15
+##   # This test reprovisions the rootfs.
+##   tags: reprovision
+##   description: Verify the root reprovision with RAID 1 works.
+
 set -xeuo pipefail
 
-srcdev=$(findmnt -nvr / -o SOURCE)
+# shellcheck disable=SC1091
+. "$KOLA_EXT_DATA/commonlib.sh"
+
+srcdev=$(findmnt -nvr /sysroot -o SOURCE)
 [[ ${srcdev} == $(realpath /dev/md/foobar) ]]
 
 blktype=$(lsblk -o TYPE "${srcdev}" --noheadings)
 [[ ${blktype} == raid1 ]]
 
-fstype=$(findmnt -nvr / -o FSTYPE)
+fstype=$(findmnt -nvr /sysroot -o FSTYPE)
 [[ ${fstype} == xfs ]]
+ok "source is XFS on RAID1 device"
+
+rootflags=$(findmnt /sysroot -no OPTIONS)
+if ! grep prjquota <<< "${rootflags}"; then
+    fatal "missing prjquota in root mount flags: ${rootflags}"
+fi
+ok "root mounted with prjquota"
 
 case "${AUTOPKGTEST_REBOOT_MARK:-}" in
   "")
-      # check that growpart didn't run
-      if [ -e /run/coreos-growpart.stamp ]; then
-          echo "coreos-growpart ran"
-          exit 1
+      # check that ignition-ostree-growfs didn't run
+      if [ -e /run/ignition-ostree-growfs.stamp ]; then
+          fatal "ignition-ostree-growfs ran"
+      fi
+
+      # check that autosave-xfs didn't run
+      if [ -e /run/ignition-ostree-autosaved-xfs.stamp ]; then
+          fatal "unexpected autosaved XFS"
       fi
 
       # reboot once to sanity-check we can find root on second boot
@@ -26,6 +53,7 @@ case "${AUTOPKGTEST_REBOOT_MARK:-}" in
   rebooted)
       grep root=UUID= /proc/cmdline
       grep rd.md.uuid= /proc/cmdline
+      ok "found root kargs"
       ;;
-  *) echo "unexpected mark: ${AUTOPKGTEST_REBOOT_MARK}"; exit 1;;
+  *) fatal "unexpected mark: ${AUTOPKGTEST_REBOOT_MARK}";;
 esac
diff --git a/tests/kola/root-reprovision/swap-before-root/config.bu b/tests/kola/root-reprovision/swap-before-root/config.bu
new file mode 100644
index 0000000000..c67161402b
--- /dev/null
+++ b/tests/kola/root-reprovision/swap-before-root/config.bu
@@ -0,0 +1,23 @@
+variant: fcos
+version: 1.4.0
+
+storage:
+  disks:
+    - device: /dev/disk/by-id/coreos-boot-disk
+      partitions:
+        - number: 4
+          label: swap
+          size_mib: 512
+          wipe_partition_entry: true
+        - number: 5
+          label: root
+  filesystems:
+    - device: /dev/disk/by-partlabel/swap
+      label: swap
+      format: swap
+      wipe_filesystem: true
+      with_mount_unit: true
+    - device: /dev/disk/by-partlabel/root
+      label: root
+      format: xfs
+      wipe_filesystem: true
diff --git a/tests/kola/root-reprovision/swap-before-root/data/commonlib.sh b/tests/kola/root-reprovision/swap-before-root/data/commonlib.sh
new file mode 120000
index 0000000000..b8dcbdca1a
--- /dev/null
+++ b/tests/kola/root-reprovision/swap-before-root/data/commonlib.sh
@@ -0,0 +1 @@
+../../../data/commonlib.sh
\ No newline at end of file
diff --git a/tests/kola/root-reprovision/swap-before-root/test.sh b/tests/kola/root-reprovision/swap-before-root/test.sh
new file mode 100755
index 0000000000..e88c1cdb03
--- /dev/null
+++ b/tests/kola/root-reprovision/swap-before-root/test.sh
@@ -0,0 +1,39 @@
+#!/bin/bash
+## kola:
+##   # Root reprovisioning requires at least 4GiB of memory.
+##   minMemory: 4096
+##   # This test includes a lot of disk I/O and needs a higher
+##   # timeout value than the default.
+##   timeoutMin: 15
+##   # We intentionally put the root filesystem on partition 5.  This is
+##   # legal but usually not intended, so Butane warns about it.
+##   allowConfigWarnings: true
+##   # This test reprovisions the rootfs.
+##   tags: reprovision platform-independent
+##   description: Verify the root reprovision and swap enabled are supported.
+
+set -xeuo pipefail
+
+# shellcheck disable=SC1091
+. "$KOLA_EXT_DATA/commonlib.sh"
+
+swapstatus=$(systemctl is-active dev-disk-by\\x2dpartlabel-swap.swap)
+[[ ${swapstatus} == active ]]
+ok "swap is active"
+
+fstype=$(findmnt -nvr /sysroot -o FSTYPE)
+[[ ${fstype} == xfs ]]
+ok "source is xfs"
+
+case "${AUTOPKGTEST_REBOOT_MARK:-}" in
+  "")
+      # reboot once to sanity-check we can find root on second boot
+      /tmp/autopkgtest-reboot rebooted
+      ;;
+
+  rebooted)
+      grep root=UUID= /proc/cmdline
+      ok "found root karg"
+      ;;
+  *) fatal "unexpected mark: ${AUTOPKGTEST_REBOOT_MARK}";;
+esac
diff --git a/tests/kola/rpm-ostree-countme/data/commonlib.sh b/tests/kola/rpm-ostree-countme/data/commonlib.sh
new file mode 120000
index 0000000000..1742d51e67
--- /dev/null
+++ b/tests/kola/rpm-ostree-countme/data/commonlib.sh
@@ -0,0 +1 @@
+../../data/commonlib.sh
\ No newline at end of file
diff --git a/tests/kola/rpm-ostree-countme/test.sh b/tests/kola/rpm-ostree-countme/test.sh
new file mode 100755
index 0000000000..8f41159d9c
--- /dev/null
+++ b/tests/kola/rpm-ostree-countme/test.sh
@@ -0,0 +1,73 @@
+#!/bin/bash
+## kola:
+##   tags: "platform-independent needs-internet"
+##   # This test only runs on FCOS because countme support is not available in RHCOS
+##   distros: fcos
+##   description: Verify rpm-ostree-countme service works well.
+
+set -xeuo pipefail
+
+# shellcheck disable=SC1091
+. "$KOLA_EXT_DATA/commonlib.sh"
+
+journal_cursor() {
+    journalctl --output json --lines 1 \
+        | jq --raw-output '.["__CURSOR"]' > /tmp/countme.cursor
+}
+
+journal_after_cursor() {
+    journalctl --output=json \
+        --after-cursor "$(cat /tmp/countme.cursor)" \
+        --output=json --unit=rpm-ostree-countme.service \
+        --grep "Successful requests:" \
+        | jq --raw-output '.MESSAGE'
+}
+
+# Check that the timer has been enabled
+if [[ $(systemctl show -p ActiveState rpm-ostree-countme.timer) != "ActiveState=active" ]] && \
+   [[ $(systemctl show -p SubState rpm-ostree-countme.timer) != "SubState=waiting" ]]; then
+    fatal "rpm-ostree-countme timer has not been started"
+fi
+
+# Try five times to avoid Fedora infra flakes
+for i in $(seq 1 5); do
+    # Remove status file so that we retry every time we flake
+    rm -f /var/lib/private/rpm-ostree-countme/countme
+    # Update the journal cursor
+    journal_cursor
+
+    # Check that running the service manually is successful
+    systemctl start rpm-ostree-countme.service
+    if [[ $(systemctl show -p ActiveState rpm-ostree-countme.service) != "ActiveState=inactive" ]] && \
+       [[ $(systemctl show -p SubState rpm-ostree-countme.service) != "SubState=dead" ]] && \
+       [[ $(systemctl show -p Result rpm-ostree-countme.service) != "Result=success" ]] && \
+       [[ $(systemctl show -p ExecMainStatus rpm-ostree-countme.service) != "ExecMainStatus=0" ]]; then
+        echo "rpm-ostree-countme exited with an error (try: $i):"
+        systemctl status rpm-ostree-countme.service
+        sleep 10
+        continue
+    fi
+
+    # Check rpm-ostree count me output
+    output="$(journal_after_cursor)"
+    trimmed=${output##Successful requests: }
+    if [[ ! $trimmed =~ ^[0-9]+/[0-9]+$ ]]; then
+        echo "rpm-ostree-countme service output does not match expected success output (try: $i):"
+        echo "${output}"
+        sleep 10
+        continue
+    fi
+    tries=${trimmed%%/*}
+    total=${trimmed##*/}
+    if [ "${tries}" != "${total}" ]; then
+        echo "rpm-ostree-countme service output shows failed requests (try: $i):"
+        echo "${output}"
+        sleep 10
+        continue
+    fi
+
+    ok countme
+    exit 0
+done
+
+fatal "rpm-ostree-countme service failed or only partially completed five times"
diff --git a/tests/kola/rpm-ostree/container-deps b/tests/kola/rpm-ostree/container-deps
new file mode 100755
index 0000000000..5b8d3fce95
--- /dev/null
+++ b/tests/kola/rpm-ostree/container-deps
@@ -0,0 +1,18 @@
+#!/bin/bash
+## kola:
+##   exclusive: false
+##   description: Verify some dependencies of the rpm-ostree container stack.
+
+set -xeuo pipefail
+
+# shellcheck disable=SC1091
+. "$KOLA_EXT_DATA/commonlib.sh"
+
+# Verify this command exists since it's a hard dependency of ostree's container bits.
+skopeo experimental-image-proxy --help
+
+# Also this one verifies the linkage between ostree and rpm-ostree via extensions
+h=$(ostree container encapsulate --help)
+[[ "$h" =~ --copymeta ]] || fatal "missing ostree container"
+
+ok skopeo
diff --git a/tests/kola/rpm-ostree/data/commonlib.sh b/tests/kola/rpm-ostree/data/commonlib.sh
new file mode 120000
index 0000000000..1742d51e67
--- /dev/null
+++ b/tests/kola/rpm-ostree/data/commonlib.sh
@@ -0,0 +1 @@
+../../data/commonlib.sh
\ No newline at end of file
diff --git a/tests/kola/rpm-ostree/kernel-replace b/tests/kola/rpm-ostree/kernel-replace
new file mode 100755
index 0000000000..2f484ca7a0
--- /dev/null
+++ b/tests/kola/rpm-ostree/kernel-replace
@@ -0,0 +1,160 @@
+#!/bin/bash
+## kola:
+##   # Increase timeout since this test has a lot of I/O and involves rebasing
+##   timeoutMin: 20
+##   # We've seen some OOM when 1024M is used:
+##   # https://github.com/coreos/fedora-coreos-tracker/issues/1506
+##   minMemory: 2048
+##   # Needs internet access as we fetch files from koji
+##   # We add the "reprovision" tag here even though we aren't
+##   # reprovisioning as a hack so that in our pipeline the test
+##   # will execute with no other tests. We were seeing a lot of
+##   # timeouts on ppc64le.
+##   tags: "needs-internet platform-independent reprovision"
+##   description: Verify that build of a container image with a new kernel
+##     and reboot into it succeeds.
+
+#
+# Copyright (C) 2023 Red Hat, Inc.
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the
+# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+# Boston, MA 02111-1307, USA.
+
+set -euxo pipefail
+
+# shellcheck disable=SC1091
+. "$KOLA_EXT_DATA/commonlib.sh"
+
+cd $(mktemp -d)
+
+# TODO: It'd be much better to test this via a registry
+
+# define OS ID in order to assign appropriate kernel later
+OS_ID=$(. /etc/os-release; echo $ID)
+image_dir=/var/tmp/coreos
+image=oci:${image_dir}
+image_pull=ostree-unverified-image:$image
+tmp_imagedir=${image_dir}-tmp
+arch=$(arch)
+
+# we only want to query repos once but need the info on multiple boots, so write out state to /var
+if [ ! -f /var/kola-kernel.evr ]; then
+  case "$OS_ID" in
+    rhcos|scos|rhel|centos)
+      c9s_mirror="https://mirror.stream.centos.org/9-stream/BaseOS/${arch}/os"
+      # we're probably already on CentOS and running the latest kernel. so
+      # here we need to instead pick whatever the latest that's *not* the same
+      kver=$(dnf repoquery kernel --qf '%{EVR}' --repofrompath=tmp,"${c9s_mirror}" --disablerepo '*' --enablerepo tmp | \
+                grep -v "$(rpm -q kernel --qf '%{EVR}')" | tail -n1)
+      kver="${kver}.${arch}"
+      url="${c9s_mirror}/Packages/kernel"
+      ;;
+    fedora)
+      # to get the version to use: koji latest-pkg f40 kernel
+      # XXX: once we can rely on dnf in FCOS, use repoquery also here
+      kver="6.8.5-301.fc40.${arch}"
+      url="https://kojipkgs.fedoraproject.org//packages/kernel/6.8.5/301.fc40/${arch}/kernel"
+      ;;
+    *)
+      echo "Unknown OS_ID: ${OS_ID}"
+      exit 1
+      ;;
+  esac
+  echo "$kver" > /var/kola-kernel.evr
+  echo "$url" > /var/kola-kernel.url
+else
+  kver=$(cat /var/kola-kernel.evr)
+  url=$(cat /var/kola-kernel.url)
+fi
+
+case "${AUTOPKGTEST_REBOOT_MARK:-}" in
+  "")
+    # Take the existing ostree commit, and export it to a container image, then rebase to it.
+    rpm-ostree status --json > status.json
+    checksum=$(jq -r '.deployments[0].checksum' < status.json)
+    v0=$(jq -r '.deployments[0].version' < status.json)
+    imgref=$(jq -r '.deployments[0]["container-image-reference"]' < status.json)
+    rm ${image_dir} -rf
+    encapsulate_args=()
+    # A hack...if we're booted into a container, then we need to fake things out
+    # for the merge commit to turn it back into an image.  What we *really* want
+    # here obviously is seamless support for re-serializing a container from
+    # the ostree storage, but right now we're not doing the tar-split stuff.
+    if [[ "$imgref" != null ]]; then
+      encapsulate_args+=("--label" "ostree.bootable=true")
+    fi
+    # Since we're switching OS update stream, turn off zincati
+    systemctl mask --now zincati
+    ostree container encapsulate "${encapsulate_args[@]}" --repo=/ostree/repo ${checksum} "${image}"
+    # This one keeps --experimental, but we also test without it below
+    rpm-ostree rebase --experimental "$image_pull"
+    ostree container image list --repo=/ostree/repo | tee imglist.txt
+    # Test rebasing back to ostree https://github.com/coreos/rpm-ostree/issues/3677
+    rpm-ostree rebase "$checksum"
+    rpm-ostree rebase "$image_pull"
+    /tmp/autopkgtest-reboot 1
+    ;;
+  1)
+    # Setup
+    # copy the OCI dir to containers-storage for a local build
+    skopeo copy $image containers-storage:localhost/coreos
+    rm "${image_dir}" -rf
+    td=$(mktemp -d)
+    cd ${td}
+    version=$(rpm-ostree --version | grep Version)
+    cat > Containerfile << EOF
+FROM localhost/coreos
+RUN rpm-ostree override replace \
+    $url-{,core-,modules-,modules-core-,modules-extra-}$kver.rpm && \
+    rpm-ostree cleanup -m && \
+    ostree container commit
+EOF
+    # Older podman found in RHEL8 blows up without /etc/resolv.conf
+    # which happens in our qemu path.
+    touched_resolv_conf=0
+    if test '!' -f /etc/resolv.conf; then
+      podmanv=$(podman --version)
+      case "${podmanv#podman version }" in
+        3.*) touched_resolv_conf=1; touch /etc/resolv.conf;;
+      esac
+    fi
+    podman build --net=host -t localhost/coreos-derived --squash .
+    if test "${touched_resolv_conf}" -eq 1; then
+      rm -vf /etc/resolv.conf
+    fi
+    derived=oci:$image_dir:derived
+    skopeo copy containers-storage:localhost/coreos-derived $derived
+    rpm-ostree --version
+    rpm-ostree rebase ostree-unverified-image:$derived
+    ostree container image list --repo=/ostree/repo
+    rm $image_dir -rf
+    /tmp/autopkgtest-reboot 2
+    ;;
+  2)
+    un=$(uname -r)
+    if test "$un" != "$kver"; then
+      echo "Expected kernel $kver but found $un"
+      exit 1
+    else
+      echo "Kernel switch to $un was successful"
+    fi
+    test -f /usr/lib/modules/$kver/initramfs.img
+    test -f /usr/lib/modules/$kver/vmlinuz
+    ;;
+  *)
+    echo "Unknown AUTOPKGTEST_REBOOT_MARK: ${AUTOPKGTEST_REBOOT_MARK:-}"
+    exit 1
+    ;;
+esac
diff --git a/tests/kola/secex/ensure/test.sh b/tests/kola/secex/ensure/test.sh
new file mode 100755
index 0000000000..42fb391421
--- /dev/null
+++ b/tests/kola/secex/ensure/test.sh
@@ -0,0 +1,34 @@
+#!/bin/bash
+## kola:
+##   architectures: s390x
+##   platforms: qemu
+##   requiredTag: secex
+##   timeoutMin: 5
+##   description: Verify the s390x Secure Execution QEMU image works. It also
+##     implicitly tests Ignition config decryption.
+
+# We don't run it by default because it requires running with `--qemu-secex`.
+
+set -xeuo pipefail
+
+check_luks() {
+    local mnt dev type
+    mnt=${1}
+    dev=$(findmnt -nvr ${mnt} -o SOURCE)
+    type=$(lsblk -o TYPE "${dev}" --noheadings)
+    [[ ${type} == crypt ]]
+}
+
+# 1 means system runs with Secure Execution
+grep -q 1 /sys/firmware/uv/prot_virt_guest
+
+# Check firstboot kargs have dm-verity hashes
+grep -q rootfs.roothash /proc/cmdline
+grep -q bootfs.roothash /proc/cmdline
+
+# Check we have SE partition with sdboot image
+mount /dev/disk/by-label/se /sysroot/se
+[[ -f /sysroot/se/sdboot ]]
+
+check_luks /sysroot
+check_luks /boot
diff --git a/tests/kola/secex/reboot/test.sh b/tests/kola/secex/reboot/test.sh
new file mode 100755
index 0000000000..a4e64c03de
--- /dev/null
+++ b/tests/kola/secex/reboot/test.sh
@@ -0,0 +1,30 @@
+#!/bin/bash
+## kola:
+##   architectures: s390x
+##   platforms: qemu
+##   requiredTag: secex
+##   timeoutMin: 5
+##   description: Verify the qemu-secex image reboots with SE enabled. It also
+##     implicitly tests Ignition config decryption.
+
+# We don't run it by default because it requires running with
+# `--qemu-secex --qemu-secex-hostkey HKD-<serial>.crt`.
+
+set -xeuo pipefail
+
+grep -q 1 /sys/firmware/uv/prot_virt_guest
+
+case "${AUTOPKGTEST_REBOOT_MARK:-}" in
+"")
+    rpm-ostree kargs --append secex_test
+    /tmp/autopkgtest-reboot rebooted
+    ;;
+rebooted)
+    grep -q rd.luks.name=$(cryptsetup luksUUID /dev/disk/by-label/crypt_rootfs)=root /proc/cmdline
+    grep -q secex_test /proc/cmdline
+    ;;
+*)
+    echo "unexpected mark: ${AUTOPKGTEST_REBOOT_MARK}";
+    exit 1
+    ;;
+esac
diff --git a/tests/kola/misc-ign-ro/config.fcc b/tests/kola/security/coreos-update-ca-trust/config.bu
similarity index 88%
rename from tests/kola/misc-ign-ro/config.fcc
rename to tests/kola/security/coreos-update-ca-trust/config.bu
index 7dcb62bf93..def29be580 100644
--- a/tests/kola/misc-ign-ro/config.fcc
+++ b/tests/kola/security/coreos-update-ca-trust/config.bu
@@ -1,13 +1,7 @@
 variant: fcos
-version: 1.1.0
+version: 1.2.0
 storage:
   files:
-    - path: /etc/systemd/zram-generator.conf
-      mode: 0644
-      contents:
-        inline: |
-          # This config file enables a /dev/zram0 device with the default settings
-          [zram0]
     - path: /etc/pki/ca-trust/source/anchors/coreos.crt
       mode: 0644
       contents:
diff --git a/tests/kola/security/coreos-update-ca-trust/data/commonlib.sh b/tests/kola/security/coreos-update-ca-trust/data/commonlib.sh
new file mode 120000
index 0000000000..b8dcbdca1a
--- /dev/null
+++ b/tests/kola/security/coreos-update-ca-trust/data/commonlib.sh
@@ -0,0 +1 @@
+../../../data/commonlib.sh
\ No newline at end of file
diff --git a/tests/kola/security/coreos-update-ca-trust/test.sh b/tests/kola/security/coreos-update-ca-trust/test.sh
new file mode 100755
index 0000000000..93f63fe86a
--- /dev/null
+++ b/tests/kola/security/coreos-update-ca-trust/test.sh
@@ -0,0 +1,18 @@
+#!/bin/bash
+## kola:
+##   exclusive: false
+##   description: Verify that coreos-update-ca-trust service works.
+
+set -xeuo pipefail
+
+# shellcheck disable=SC1091
+. "$KOLA_EXT_DATA/commonlib.sh"
+
+# Make sure that coreos-update-ca-trust kicked in and observe the result.
+if ! systemctl show coreos-update-ca-trust.service -p ActiveState | grep ActiveState=active; then
+    fatal "coreos-update-ca-trust.service not active"
+fi
+if ! grep '^# coreos.com$' /etc/pki/ca-trust/extracted/openssl/ca-bundle.trust.crt; then
+    fatal "expected coreos.com in ca-bundle"
+fi
+ok "coreos-update-ca-trust.service"
diff --git a/tests/kola/security/passwd/config.bu b/tests/kola/security/passwd/config.bu
new file mode 100644
index 0000000000..d9c0b5e109
--- /dev/null
+++ b/tests/kola/security/passwd/config.bu
@@ -0,0 +1,7 @@
+variant: fcos
+version: 1.4.0
+passwd:
+  users:
+    - name: tester
+      # encrypted version of 'foobar'. generated with `mkpasswd --method=yescrypt`
+      password_hash: $y$j9T$0HA8ReLTqRTccLKT0gzVY.$/e.OCrjePrh2tOm8CAoLqCMlZWS9q/WSAPBaZuopRs4
diff --git a/tests/kola/security/passwd/data/commonlib.sh b/tests/kola/security/passwd/data/commonlib.sh
new file mode 120000
index 0000000000..b8dcbdca1a
--- /dev/null
+++ b/tests/kola/security/passwd/data/commonlib.sh
@@ -0,0 +1 @@
+../../../data/commonlib.sh
\ No newline at end of file
diff --git a/tests/kola/security/passwd/test.sh b/tests/kola/security/passwd/test.sh
new file mode 100755
index 0000000000..a3c5637a21
--- /dev/null
+++ b/tests/kola/security/passwd/test.sh
@@ -0,0 +1,27 @@
+#!/bin/bash
+## kola:
+##   # This test only runs on FCOS because RHCOS does not support `yescrypt`
+##   # TODO-RHCOS: adapt to use different `crypt` scheme for RHCOS
+##   distros:  fcos
+##   exclusive: false
+##   description: Verify that a user password provisioned by Ignition works.
+
+set -xeuo pipefail
+
+# shellcheck disable=SC1091
+. "$KOLA_EXT_DATA/commonlib.sh"
+
+OUTPUT=$(echo 'foobar' | setsid su - tester -c id)
+
+if [[ $OUTPUT != "uid=1001(tester) gid=1001(tester) groups=1001(tester) context=system_u:system_r:unconfined_service_t:s0" ]]; then
+    fatal "Failure when checking command output running with specified username and password"
+fi
+# https://fedoraproject.org/wiki/Changes/yescrypt_as_default_hashing_method_for_shadow
+# Testing that passwd command creates a yescrypt password hash(starting with '$y$')
+sudo useradd tester2
+echo "42abcdef" | sudo passwd tester2 --stdin
+PASSWD_CONFIRMATION=$(sudo grep tester2 /etc/shadow)
+if [[ ${PASSWD_CONFIRMATION:0:11} != 'tester2:$y$' ]]; then
+    fatal "passwd did not create a yescrypt password hash"
+fi
+ok "User-password provisioned and passwd command successfully tested"
diff --git a/tests/kola/selinux/data/commonlib.sh b/tests/kola/selinux/data/commonlib.sh
new file mode 120000
index 0000000000..1742d51e67
--- /dev/null
+++ b/tests/kola/selinux/data/commonlib.sh
@@ -0,0 +1 @@
+../../data/commonlib.sh
\ No newline at end of file
diff --git a/tests/kola/selinux/default b/tests/kola/selinux/default
new file mode 100755
index 0000000000..e5bc702c2d
--- /dev/null
+++ b/tests/kola/selinux/default
@@ -0,0 +1,17 @@
+#!/bin/bash
+## kola:
+##   exclusive: false
+##   tags: "platform-independent"
+##   description: Verify that the SELinux policy isn't marked as modified.
+
+# See https://github.com/openshift/os/issues/1036
+
+set -xeuo pipefail
+
+# shellcheck disable=SC1091
+. "$KOLA_EXT_DATA/commonlib.sh"
+
+if ostree admin config-diff | grep 'selinux/targeted/policy'; then
+    fatal "SELinux policy is marked as modified"
+fi
+ok "SELinux policy not marked as modified"
diff --git a/tests/kola/selinux/enforcing b/tests/kola/selinux/enforcing
new file mode 100755
index 0000000000..2c163e3d50
--- /dev/null
+++ b/tests/kola/selinux/enforcing
@@ -0,0 +1,54 @@
+#!/bin/bash
+## kola:
+##   exclusive: true
+##   description: Verify selinux should be Enforcing by default.
+##     Also can switch to Permissive mode and switch back.
+
+set -xeuo pipefail
+
+# shellcheck disable=SC1091
+. "$KOLA_EXT_DATA/commonlib.sh"
+
+case "${AUTOPKGTEST_REBOOT_MARK:-}" in
+  "")
+    # SELinux should be on
+    enforce=$(getenforce)
+    if [ "${enforce}" != "Enforcing" ]; then
+      fatal "Error: Expected SELinux Enforcing, found ${enforce}"
+    fi
+    ok "SELinux is Enforcing"
+
+    # Check SELinux switches to Permissive mode
+    setenforce 0
+    enforce=$(getenforce)
+    if [ "${enforce}" != "Permissive" ]; then
+      fatal "Error: Expected SELinux Permissive, found ${enforce}"
+    fi
+    ok "SELinux is Permissive"
+
+    # Check SELinux switches back to Enforcing mode
+    setenforce 1
+    enforce=$(getenforce)
+    if [ "${enforce}" != "Enforcing" ]; then
+      fatal "Error: Expected SELinux Enforcing, found ${enforce}"
+    fi
+    ok "SELinux is Enforcing"
+
+    # Modify config to permanently switch to permissive
+    sed -i 's/SELINUX=enforcing/SELINUX=permissive/' /etc/selinux/config
+    ok "First boot"
+    /tmp/autopkgtest-reboot rebooted
+    ;;
+  rebooted)
+    # check SELinux is in permissive mode
+    enforce=$(getenforce)
+    if [ "${enforce}" != "Permissive" ]; then
+      fatal "Error: Expected SELinux Permissive, found ${enforce}"
+    fi
+    ok "SELinux is Permissive"
+    ok "Second boot"
+    ;;
+  *)
+    fatal "unexpected mark: ${AUTOPKGTEST_REBOOT_MARK}"
+    ;;
+esac
diff --git a/tests/kola/selinux/podman-tmpfs-context b/tests/kola/selinux/podman-tmpfs-context
new file mode 100755
index 0000000000..d87e1c3f77
--- /dev/null
+++ b/tests/kola/selinux/podman-tmpfs-context
@@ -0,0 +1,23 @@
+#!/bin/bash
+## kola:
+##   # This test doesn't meaningfully change the system and
+##   # can be run with other tests.
+##   exclusive: false
+##   # This test pulls a container image from the network.
+##   tags: "needs-internet platform-independent"
+##   description: Verify that tmpfs mounts inside a container
+##     have the `container_file_t` label.
+
+# See https://github.com/coreos/fedora-coreos-tracker/issues/1366
+
+set -xeuo pipefail
+
+# shellcheck disable=SC1091
+. "$KOLA_EXT_DATA/commonlib.sh"
+
+context=$(podman run --rm --privileged quay.io/fedora/fedora:40 \
+            bash -c "mount -t tmpfs tmpfs /mnt/ && stat --format '%C' /mnt/")
+if [ "$context" != "system_u:object_r:container_file_t:s0" ]; then
+    fatal "SELinux context for files on a tmpfs inside a container is wrong"
+fi
+ok "SELinux context for files on a tmpfs inside a container is correct"
diff --git a/tests/kola/selinux/stub-resolve-context b/tests/kola/selinux/stub-resolve-context
new file mode 100755
index 0000000000..9708af2c09
--- /dev/null
+++ b/tests/kola/selinux/stub-resolve-context
@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+## kola:
+##   # This test only runs on FCOS because `systemd-resolved` is not installed on
+##   # RHCOS
+##   distros:  fcos
+##   exclusive: false
+##   description: Verify that the stub-resolv.conf file has the correct
+##     selinux context.
+
+# See
+# - https://github.com/fedora-selinux/selinux-policy/pull/509#issuecomment-744540382
+# - https://github.com/systemd/systemd/pull/17976
+
+set -xeuo pipefail
+
+# shellcheck disable=SC1091
+. "$KOLA_EXT_DATA/commonlib.sh"
+
+context=$(stat --format "%C" /run/systemd/resolve/stub-resolv.conf)
+if [ "$context" != "system_u:object_r:net_conf_t:s0" ]; then
+    fatal "SELinux context on stub-resolv.conf is wrong"
+fi
+ok "SELinux context on stub-resolv.conf is correct"
diff --git a/tests/kola/selinux/usrlocal-context b/tests/kola/selinux/usrlocal-context
new file mode 100755
index 0000000000..2e7a61499f
--- /dev/null
+++ b/tests/kola/selinux/usrlocal-context
@@ -0,0 +1,17 @@
+#!/bin/bash
+## kola:
+##   exclusive: false
+##   description: Verify /usr/sbin and /var/usrlocal/sbin have
+##     the same SELinux security context.
+
+set -xeuo pipefail
+
+# shellcheck disable=SC1091
+. "$KOLA_EXT_DATA/commonlib.sh"
+
+bin_ctx=$(stat -c %C /usr/sbin)
+usrlocal_sbin_ctx=$(stat -c %C /var/usrlocal/sbin)
+if test "${bin_ctx}" != "${usrlocal_sbin_ctx}"; then
+    fatal "Error: /usr/sbin is ${bin_ctx} but /var/usrlocal/sbin is ${usrlocal_sbin_ctx}"
+fi
+ok "/usr/sbin /var/usrlocal/sbin have the same SELinux security context"
diff --git a/tests/kola/ssh/custom-host-key-permissions/config.bu b/tests/kola/ssh/custom-host-key-permissions/config.bu
new file mode 100644
index 0000000000..edb2977afe
--- /dev/null
+++ b/tests/kola/ssh/custom-host-key-permissions/config.bu
@@ -0,0 +1,56 @@
+variant: fcos
+version: 1.4.0
+storage:
+  files:
+    - path: /etc/ssh-host-key
+      mode: 0640
+      group:
+        name: ssh_keys
+      contents:
+        inline: |
+          -----BEGIN OPENSSH PRIVATE KEY-----
+          b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAABlwAAAAdzc2gtcn
+          NhAAAAAwEAAQAAAYEAkJuYAEhET69RcKmN45holIk3HbzNs3hM4tBofA9R/dw5EMoUd6BY
+          SXHV0QqMGYjS2UqICG/zKrYMWY6uBjL/Xq7aiezZ4AaVNNDyFNHuY9Or0/W1j50JhEBDJc
+          e7rjRPtb7jGtHNdioTftdvb2UQQWm0Ko8krndPZqNd/zNXDEYtBoMjxOLYgt88BDdIKXD1
+          w0/3GdLysQdrzIsnomYhgQlVmxE3VRhTAYUgTuKVDPO33OmuA8LE8wUH9PwV3yfB4x470E
+          bT/8c9y/tgFtGtrTT0IERrU9rXmA9ZPKVBHMbpCghGcMmdatlzibOZInQ3FlUvNFn4HDVx
+          r6ZZS8ZCOPwjVrx50LbVvUHw6/tv8jrojpkW3HL9RkU49RiHpG72/Y1umIPWaWjGtLqRCj
+          iWGkl0Pjxs/OkkizRL39hzbRszypBMSeKU7/QsPLuiSj8mI5s6fm7tKp8sCQydy2k4BBiq
+          o801KWo3V+q5X2Ly1g20T1WsHPl6ohISu15ln1KrAAAFiPA9CfnwPQn5AAAAB3NzaC1yc2
+          EAAAGBAJCbmABIRE+vUXCpjeOYaJSJNx28zbN4TOLQaHwPUf3cORDKFHegWElx1dEKjBmI
+          0tlKiAhv8yq2DFmOrgYy/16u2ons2eAGlTTQ8hTR7mPTq9P1tY+dCYRAQyXHu640T7W+4x
+          rRzXYqE37Xb29lEEFptCqPJK53T2ajXf8zVwxGLQaDI8Ti2ILfPAQ3SClw9cNP9xnS8rEH
+          a8yLJ6JmIYEJVZsRN1UYUwGFIE7ilQzzt9zprgPCxPMFB/T8Fd8nweMeO9BG0//HPcv7YB
+          bRra009CBEa1Pa15gPWTylQRzG6QoIRnDJnWrZc4mzmSJ0NxZVLzRZ+Bw1ca+mWUvGQjj8
+          I1a8edC21b1B8Ov7b/I66I6ZFtxy/UZFOPUYh6Ru9v2NbpiD1mloxrS6kQo4lhpJdD48bP
+          zpJIs0S9/Yc20bM8qQTEnilO/0LDy7oko/JiObOn5u7SqfLAkMnctpOAQYqqPNNSlqN1fq
+          uV9i8tYNtE9VrBz5eqISErteZZ9SqwAAAAMBAAEAAAGANMdNNYEqyYib6UpBGrnkJZ5lKu
+          nfi+rS6Q+WqvzueICZpVqUGMtBneC54NeAJcut5RfSSX4Omt6h6EfulR2k3fpkkeWL6buN
+          Vp8SU+4BG5dEhhKOZzGyKP5JY68f/WdjVlqqyf2cB045Gljn95jD05QQaV4gTbsHFFd49a
+          1XzoeIZHGVqwT+b9mpLoK8yD9Nu7DiZ5757Ang2uFJIHk1LkLpZzTj5J+BoDLmBVSmgPks
+          +KijVgUO3AHQkyY0l2OLUGlTqdT2vOSehu+a2+oCEMt38NeoSz5B2vMxPYwC1fbCyxELso
+          J8z6NgJGdGdJ2TNz+nTOycKp8Wm0Lq5ze3UnGOBRfJigfhrLSPVkq2Q6zR8CR85h3wvWjU
+          Kp6BLkoX6wO3KRCkpU5m63JWpFz8LVUUUT3/afxl31Qe0y6tmq1MlZ2WwdZgPGau2xm6G7
+          jWxK27ZJ1PR2UacFFXeHFAZyGmvpLArQQ2sFZUXhrujWhtc9UUXaieCx7AktobkZ3FAAAA
+          wCvP6W7p7M87EsoamwOZ9hChjJr9bhHquc11nm1CCk1WoquCJwBzLY+einj/Sjk6QxKu2C
+          AxqJrKAu1B93EBU5ZXNnqax0jcbO8xBeiiqlP05o4RpM6iw/kIwQ9GINjxBh0/u16IRmJx
+          /50TCu9iTxmtMOTRq+qFlNZqA//3qnpFBRCa/lKwhkn4nNOa9d+XZbD7arVDpR6omAkbnc
+          1skzzwVDuvECHnij4c+fOIMOmub0KShBLB2A5Ig+d8jU8PzgAAAMEAxZ956C+NO0O452Ax
+          Rx4jDAdOQR8Z/zj79oU2WWWrjpkjQPq82XLX3S4ZJJieW6+olSr43+6JGOeIRJRGOqvsAd
+          xnKfapFQyb45Jitgj3lnkezaauEbbjRdJze5q+wXZfOiq/xbn0racw1d7AS0USsWeljNb1
+          BET+XXv/dEth0/SPzDxfWigd0sJW/xhVvxj1O+STg0IvWSda4l1FgQSbNiUMKSt5NFx/S5
+          xtIqLL7+1Vwvzji47MEze4Bjfw3gTlAAAAwQC7UwljKIn3GHkQ6dPW3j1X2A6Xymy5kU4s
+          SYG98R/87Cx/7eiXhjZ1cS0i235VYm6XZUCdNXGOwYuTDL4q0eKltwLHM7wRgBfXja5yzS
+          kQeAkXhU/zPYd62lK/vbwT7VDBgtec+8p6L7kdnv9Ejio6KELCQp6R+pcExv8xSUfubhUw
+          p6BVaBu3biDjiZbWAOSIDHF5rqk6yxGU1oz3MFvg7psyQBeQHm8kQ2qcpHJZL0Czi2uoEa
+          JluaUsyXI/kE8AAAARYmdpbGJlcnRAc2lyeXN0ZXMBAg==
+          -----END OPENSSH PRIVATE KEY-----
+    - path: /etc/ssh-host-key.pub
+      mode: 0644
+      contents:
+        inline: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCQm5gASERPr1FwqY3jmGiUiTcdvM2zeEzi0Gh8D1H93DkQyhR3oFhJcdXRCowZiNLZSogIb/MqtgxZjq4GMv9ertqJ7NngBpU00PIU0e5j06vT9bWPnQmEQEMlx7uuNE+1vuMa0c12KhN+129vZRBBabQqjySud09mo13/M1cMRi0GgyPE4tiC3zwEN0gpcPXDT/cZ0vKxB2vMiyeiZiGBCVWbETdVGFMBhSBO4pUM87fc6a4DwsTzBQf0/BXfJ8HjHjvQRtP/xz3L+2AW0a2tNPQgRGtT2teYD1k8pUEcxukKCEZwyZ1q2XOJs5kidDcWVS80WfgcNXGvpllLxkI4/CNWvHnQttW9QfDr+2/yOuiOmRbccv1GRTj1GIekbvb9jW6Yg9ZpaMa0upEKOJYaSXQ+PGz86SSLNEvf2HNtGzPKkExJ4pTv9Cw8u6JKPyYjmzp+bu0qnywJDJ3LaTgEGKqjzTUpajdX6rlfYvLWDbRPVawc+XqiEhK7XmWfUqs=
+    - path: /etc/ssh/sshd_config.d/80-hostkey.conf
+      mode: 0600
+      contents:
+        inline: HostKey /etc/ssh-host-key
diff --git a/tests/kola/ssh/custom-host-key-permissions/data/commonlib.sh b/tests/kola/ssh/custom-host-key-permissions/data/commonlib.sh
new file mode 120000
index 0000000000..b8dcbdca1a
--- /dev/null
+++ b/tests/kola/ssh/custom-host-key-permissions/data/commonlib.sh
@@ -0,0 +1 @@
+../../../data/commonlib.sh
\ No newline at end of file
diff --git a/tests/kola/ssh/custom-host-key-permissions/test.sh b/tests/kola/ssh/custom-host-key-permissions/test.sh
new file mode 100755
index 0000000000..a1abfc5f9c
--- /dev/null
+++ b/tests/kola/ssh/custom-host-key-permissions/test.sh
@@ -0,0 +1,35 @@
+#!/bin/bash
+## kola:
+##   tags: platform-independent
+##   description: Verify that sshd still works with a custom
+##     host key with mode 640 and group ssh_keys.
+
+# See
+# - https://github.com/coreos/fedora-coreos-tracker/issues/1394
+# - https://src.fedoraproject.org/rpms/openssh/pull-request/37
+
+set -xeuo pipefail
+
+# shellcheck disable=SC1091
+. "$KOLA_EXT_DATA/commonlib.sh"
+
+# recent Fedora sshd binaries will fail to start if all configured host keys
+# have mode > 600 and their modes haven't automatically been fixed
+# that part is implicitly tested by kola, which needs sshd to connect
+
+# make sure our key was actually used
+# grep -q causes sshd -T to fail on SIGPIPE
+if ! sshd -T | grep "^hostkey /etc/ssh-host-key$" >/dev/null; then
+    sshd -T
+    fatal "configured host key not used by sshd"
+fi
+ok "configured host key used by sshd"
+
+# sshd starts successfully if any keys are mode 600, which would invalidate
+# the test except that our HostKey directive should replace the default
+# keys.  verify this.
+if [[ $(sshd -T | grep "^hostkey " | wc -l) != 1 ]]; then
+    sshd -T | grep "^hostkey "
+    fatal "sshd uses multiple host keys"
+fi
+ok "sshd uses only the configured host key"
diff --git a/tests/kola/swap/data/commonlib.sh b/tests/kola/swap/data/commonlib.sh
new file mode 120000
index 0000000000..1742d51e67
--- /dev/null
+++ b/tests/kola/swap/data/commonlib.sh
@@ -0,0 +1 @@
+../../data/commonlib.sh
\ No newline at end of file
diff --git a/tests/kola/swap/zram-default b/tests/kola/swap/zram-default
new file mode 100755
index 0000000000..6e9a11a88f
--- /dev/null
+++ b/tests/kola/swap/zram-default
@@ -0,0 +1,20 @@
+#!/bin/bash
+## kola:
+##   exclusive: false
+##   description: Verify that we don't have swap on zram by default.
+
+# https://github.com/coreos/fedora-coreos-tracker/issues/509
+# https://github.com/coreos/fedora-coreos-config/pull/687
+#
+# We can run this on both FCOS and RHCOS as neither should have a zram device
+# enabled by default. (In RHCOS, there is no zram support at all)
+
+set -xeuo pipefail
+
+# shellcheck disable=SC1091
+. "$KOLA_EXT_DATA/commonlib.sh"
+
+if [ -e /dev/zram0 ]; then
+    fatal "zram0 swap device set up on default install"
+fi
+ok no zram swap by default
diff --git a/tests/kola/swap/zram-generator/config.bu b/tests/kola/swap/zram-generator/config.bu
new file mode 100644
index 0000000000..09208a5fce
--- /dev/null
+++ b/tests/kola/swap/zram-generator/config.bu
@@ -0,0 +1,10 @@
+variant: fcos
+version: 1.2.0
+storage:
+  files:
+    - path: /etc/systemd/zram-generator.conf
+      mode: 0644
+      contents:
+        inline: |
+          # This config file enables a /dev/zram0 device with the default settings
+          [zram0]
diff --git a/tests/kola/swap/zram-generator/data/commonlib.sh b/tests/kola/swap/zram-generator/data/commonlib.sh
new file mode 120000
index 0000000000..b8dcbdca1a
--- /dev/null
+++ b/tests/kola/swap/zram-generator/data/commonlib.sh
@@ -0,0 +1 @@
+../../../data/commonlib.sh
\ No newline at end of file
diff --git a/tests/kola/swap/zram-generator/test.sh b/tests/kola/swap/zram-generator/test.sh
new file mode 100755
index 0000000000..e2ad3ea6ac
--- /dev/null
+++ b/tests/kola/swap/zram-generator/test.sh
@@ -0,0 +1,20 @@
+#!/bin/bash
+## kola:
+##   # This test only runs on FCOS because RHCOS does not have zram support.
+##   distros: fcos
+##   # This test conflicts with swap/zram-default so we cannot set this to non-exclusive
+##   exclusive: true
+##   description: Verify that swap on zram devices can be set up using the
+##     zram-generator as defined.
+
+# See docs at https://docs.fedoraproject.org/en-US/fedora-coreos/sysconfig-configure-swaponzram/
+
+set -xeuo pipefail
+
+# shellcheck disable=SC1091
+. "$KOLA_EXT_DATA/commonlib.sh"
+
+if ! grep -q 'zram0' /proc/swaps; then
+    fatal "expected zram0 to be set up"
+fi
+ok "swap on zram was set up correctly"
diff --git a/tests/kola/systemd/condition-needs-update b/tests/kola/systemd/condition-needs-update
new file mode 100755
index 0000000000..c99f275b06
--- /dev/null
+++ b/tests/kola/systemd/condition-needs-update
@@ -0,0 +1,64 @@
+#!/bin/bash
+## kola:
+##   tags: "platform-independent"
+##   description: Verify systemd-sysuser.service with `ConditionNeedsUpdate=`
+##     will be triggered with new deployment at boot time.
+
+# See https://github.com/ostreedev/ostree/issues/3069#issuecomment-1798115799
+
+# Systemd units using ConditionNeedsUpdate= run if the mtime of .updated in
+# the specified directory is newer than /usr. Since /usr has an mtime of
+# 0, there's no way to have an older .updated file refer to
+# https://ostreedev.github.io/ostree/repo/#content-objects. Systemd units
+# typically specify ConditionNeedsUpdate=/etc or ConditionNeedsUpdate=/var
+# to support stateless systems like ostree.
+
+# Remove the file .updated from the new deployment's /etc and the OS's /var
+# regardless of where they came from to ensure that these systemd units
+# run when booting new deployments, see
+# https://github.com/ostreedev/ostree/commit/19d18842cf2df944c7e9536494353aefa2916743
+
+set -xeuo pipefail
+
+. "$KOLA_EXT_DATA/commonlib.sh"
+
+username=footest
+kargs="somedummykarg=1"
+
+case "${AUTOPKGTEST_REBOOT_MARK:-}" in
+  "")
+    ok "first boot"
+    if getent passwd ${username}; then
+      fatal "should not get user ${username} before testing"
+    fi
+    # create sysusers config
+    mkdir /etc/sysusers.d
+    echo "u      ${username}  -   ${username}" > /etc/sysusers.d/30-${username}.conf
+    /tmp/autopkgtest-reboot second-boot
+    ;;
+
+  second-boot)
+    # reboot to check user footest not created
+    ok "second boot"
+    if getent passwd ${username}; then
+      fatal "should not get user ${username} after second boot"
+    fi
+    # create a new deployment
+    rpm-ostree kargs --append=${kargs}
+    /tmp/autopkgtest-reboot third-boot
+    ;;
+
+  third-boot)
+    ok "third boot"
+    # check user footest is created
+    if ! getent passwd ${username}; then
+      fatal "should get user ${username} with new deployment after third boot"
+    fi
+    # check appended kargs
+    if ! grep "${kargs}" /proc/cmdline; then
+      fatal "can not get appended kargs ${kargs}"
+    fi
+    ;;
+
+    *) fatal "unexpected mark: ${AUTOPKGTEST_REBOOT_MARK}";;
+esac
diff --git a/tests/kola/systemd/data/commonlib.sh b/tests/kola/systemd/data/commonlib.sh
new file mode 120000
index 0000000000..1742d51e67
--- /dev/null
+++ b/tests/kola/systemd/data/commonlib.sh
@@ -0,0 +1 @@
+../../data/commonlib.sh
\ No newline at end of file
diff --git a/tests/kola/systemd/default-unit-timeouts b/tests/kola/systemd/default-unit-timeouts
new file mode 100755
index 0000000000..18570f3e70
--- /dev/null
+++ b/tests/kola/systemd/default-unit-timeouts
@@ -0,0 +1,56 @@
+#!/bin/bash
+## kola:
+##   exclusive: false
+##   tags: "platform-independent"
+##   description: Verify systemd default shutdown time is 1m 30s.
+
+# See https://github.com/coreos/fedora-coreos-tracker/issues/1404
+
+set -xeuo pipefail
+
+# shellcheck disable=SC1091
+. "$KOLA_EXT_DATA/commonlib.sh"
+
+DefaultTimeoutStartUSec=$(systemctl show --value --property=DefaultTimeoutStartUSec)
+DefaultTimeoutStopUSec=$(systemctl show --value --property=DefaultTimeoutStopUSec)
+DefaultTimeoutAbortUSec=$(systemctl show --value --property=DefaultTimeoutAbortUSec)
+DefaultDeviceTimeoutUSec=$(systemctl show --value --property=DefaultDeviceTimeoutUSec)
+
+error="DefaultTimeoutStartUsec=$DefaultTimeoutStartUSec "
+error+="DefaultTimeoutStopUSec=$DefaultTimeoutStopUSec "
+error+="DefaultTimeoutAbortUSec=$DefaultTimeoutAbortUSec "
+error+="DefaultDeviceTimeoutUSec=$DefaultDeviceTimeoutUSec"
+timeout="1min 30s"
+
+systemd_version=$(rpm -q --queryformat '%{VERSION}' systemd)
+required_version="252.4"
+
+# Verify that the timeouts are 1 minute and 30 seconds (90 seconds)
+
+if [[ "$DefaultTimeoutStartUSec" == "$timeout" ]]; then
+    ok "The default start timeout is 1m 30 seconds."
+else
+    fatal $error
+fi
+
+if [[ $DefaultTimeoutStopUSec == "$timeout" ]]; then
+    ok "The default stop timeout is 1m 30 seconds."
+else
+    fatal $error
+fi
+
+if [[ $DefaultTimeoutAbortUSec == "$timeout" ]]; then
+    ok "The default abort timeout time is 1m 30 seconds."
+else
+    fatal $error
+fi
+
+if vergte "${systemd_version}" "${required_version}"; then
+    if [[ $DefaultDeviceTimeoutUSec == "$timeout" ]]; then
+        ok "The default device timeout is 1m 30 seconds."
+    else
+        fatal $error
+    fi
+else
+    ok "This variable is not available in the current systemd version"
+fi
diff --git a/tests/kola/systemd/network-online/config.bu b/tests/kola/systemd/network-online/config.bu
new file mode 100644
index 0000000000..09246d1f1f
--- /dev/null
+++ b/tests/kola/systemd/network-online/config.bu
@@ -0,0 +1,19 @@
+variant: fcos
+version: 1.4.0
+systemd:
+  units:
+    - name: block-network-online.service
+      enabled: true
+      contents: |
+        [Unit]
+        After=NetworkManager-wait-online.service
+        Before=network-online.target
+        Wants=network-online.target
+
+        [Service]
+        Type=oneshot
+        RemainAfterExit=yes
+        ExecStart=sleep infinity
+
+        [Install]
+        WantedBy=multi-user.target
diff --git a/tests/kola/systemd/network-online/data/commonlib.sh b/tests/kola/systemd/network-online/data/commonlib.sh
new file mode 120000
index 0000000000..b8dcbdca1a
--- /dev/null
+++ b/tests/kola/systemd/network-online/data/commonlib.sh
@@ -0,0 +1 @@
+../../../data/commonlib.sh
\ No newline at end of file
diff --git a/tests/kola/systemd/network-online/test.sh b/tests/kola/systemd/network-online/test.sh
new file mode 100755
index 0000000000..442ef34ae0
--- /dev/null
+++ b/tests/kola/systemd/network-online/test.sh
@@ -0,0 +1,42 @@
+#!/bin/bash
+## kola:
+##   description: Verify that network-online.target doesn't block login
+##   tags: platform-independent
+##   # this really shouldn't take long; if it does, it's that we're hitting the
+##   # very issue we're testing for
+##   timeoutMin: 3
+
+# If the user provides a systemd unit which pulls in `network-online.target`,
+# we want to make sure that logins don't block on `network-online.target` being
+# reached. `block-network-online.service` verifies this by pulling in `network-
+# online.target` and preventing it from being reached by running `Before=` it
+# and sleeping forever.
+
+# We hit this in RHCOS with iscsi.service causing network-online.target to block
+# remote-fs-pre.target and hence systemd-user-sessions.service from running:
+#
+# https://github.com/openshift/os/pull/1279
+# https://issues.redhat.com/browse/OCPBUGS-11124
+
+set -euo pipefail
+
+# shellcheck disable=SC1091
+. "$KOLA_EXT_DATA/commonlib.sh"
+
+# The fact that we're here means that `systemd-user-sessions.service` was
+# reached and logins work since kola was able to SSH to start us. But let's do
+# some sanity-checks to verify that the test was valid.
+
+# verify that block-network-online.service is still activating since it's stuck sleeping
+if [[ $(systemctl show block-network-online -p ActiveState) != "ActiveState=activating" ]]; then
+  systemctl status block-network-online.service
+  fatal "block-network-online.service isn't activating"
+fi
+
+# verify that network-online.target is not yet active since it's still blocked on block-network-online.service
+if [[ $(systemctl show network-online.target -p ActiveState) != "ActiveState=inactive" ]]; then
+  systemctl status network-online.target
+  fatal "network-online.target isn't inactive"
+fi
+
+echo "ok network-online.target does not block login"
diff --git a/tests/kola/toolbox/data/commonlib.sh b/tests/kola/toolbox/data/commonlib.sh
new file mode 120000
index 0000000000..1742d51e67
--- /dev/null
+++ b/tests/kola/toolbox/data/commonlib.sh
@@ -0,0 +1 @@
+../../data/commonlib.sh
\ No newline at end of file
diff --git a/tests/kola/toolbox/test.sh b/tests/kola/toolbox/test.sh
new file mode 100755
index 0000000000..d8db6db43b
--- /dev/null
+++ b/tests/kola/toolbox/test.sh
@@ -0,0 +1,52 @@
+#!/bin/bash
+## kola:
+##   tags: "platform-independent needs-internet"
+##   # This test only runs on FCOS because RHCOS is missing the `machinectl` command.
+##   # Additionally, there are some distro specific choices made for this test that
+##   # should/could be adapted for RHCOS.
+##   # TODO-RHCOS: adapt test for RHCOS specifics or create separate RHCOS toolbox test
+##   distros: fcos
+##   # Toolbox container is currently available only for x86_64 and aarch64 in Fedora
+##   architectures: x86_64 aarch64
+##   description: Make sure that basic toolbox functionality works (creating,
+##     running commands, and removing).
+
+# Important note: Commands are run indirectly via calls to `machinectl shell`
+# to re-create the user environment needed for unprivileged podman
+# functionality. However, machinectl shell does not propagate the exit
+# code/status of the invoked shell process thus we need additionnal checks to
+# ensure that previous commands were successful.
+
+set -xeuo pipefail
+
+# shellcheck disable=SC1091
+. "$KOLA_EXT_DATA/commonlib.sh"
+
+# Try five times to create the toolbox to avoid container registry infra flakes
+for i in $(seq 1 5); do
+    machinectl shell core@ /bin/toolbox create --assumeyes 1>/dev/null
+    if [[ $(machinectl shell core@ /bin/toolbox list --containers | grep --count fedora-toolbox-) -ne 1 ]]; then
+        echo "Could not create toolbox on try: $i"
+        sleep 10
+    else
+        break
+    fi
+done
+if [[ $(machinectl shell core@ /bin/toolbox list --containers | grep --count fedora-toolbox-) -ne 1 ]]; then
+    fatal "Could not create toolbox"
+fi
+ok toolbox create
+
+machinectl shell core@ /bin/toolbox run touch ok_toolbox
+if [[ ! -f '/home/core/ok_toolbox' ]]; then
+    fatal "Could not run a simple command inside a toolbox"
+fi
+ok toolbox run
+
+toolbox="$(machinectl shell core@ /bin/toolbox list --containers | grep fedora-toolbox- | awk '{print $2}')"
+machinectl shell core@ /bin/podman stop "${toolbox}"
+machinectl shell core@ /bin/toolbox rm "${toolbox}"
+if [[ -n "$(machinectl shell core@ /bin/toolbox list --containers)" ]]; then
+    fatal "Could not remove the toolbox container"
+fi
+ok toolbox rm
diff --git a/tests/kola/upgrade/extended/config.bu b/tests/kola/upgrade/extended/config.bu
new file mode 100644
index 0000000000..671e295a4e
--- /dev/null
+++ b/tests/kola/upgrade/extended/config.bu
@@ -0,0 +1,24 @@
+variant: fcos
+# Must use version 1.0.0 here to use Ignition spec 3.0.0 for
+# our oldest supported starting points (i.e. 31.20200108.3.0)
+version: 1.0.0
+storage:
+  files:
+    - path: /etc/systemd/journald.conf.d/forward-to-console.conf
+      mode: 0644
+      contents:
+        inline: |
+          # Send journal messages to the console. This is so we can see the
+          # progress even after autopkgtest-reboot-prepare has been called.
+          [Journal]
+          ForwardToConsole=yes
+    - path: /etc/zincati/config.d/99-config.toml
+      mode: 0644
+      contents:
+        inline: |
+          # Don't wait for any rollout window, update ASAP
+          [identity]
+          rollout_wariness = 0.0000
+          # Increase the frequency at which we check for updates
+          [agent.timing]
+          steady_interval_secs = 20
diff --git a/tests/kola/upgrade/extended/data/commonlib.sh b/tests/kola/upgrade/extended/data/commonlib.sh
new file mode 120000
index 0000000000..b8dcbdca1a
--- /dev/null
+++ b/tests/kola/upgrade/extended/data/commonlib.sh
@@ -0,0 +1 @@
+../../../data/commonlib.sh
\ No newline at end of file
diff --git a/tests/kola/upgrade/extended/test.sh b/tests/kola/upgrade/extended/test.sh
new file mode 100755
index 0000000000..988acaaee1
--- /dev/null
+++ b/tests/kola/upgrade/extended/test.sh
@@ -0,0 +1,245 @@
+#!/bin/bash
+## kola:
+##   # - needs-internet: to pull updates
+##   tags: "needs-internet"
+##   # Extend the timeout since a lot of updates/reboots can happen.
+##   timeoutMin: 45
+##   # Only run this test when specifically requested.
+##   requiredTag: extended-upgrade
+##   description: Verify upgrade works.
+
+set -eux -o pipefail
+
+# shellcheck disable=SC1091
+. "$KOLA_EXT_DATA/commonlib.sh"
+
+# This test will attempt to test an upgrade from a given starting
+# point (assumed by the caller passing in a specific
+# `cosa kola run --build=x.y.z`) all the way to the latest build
+# that is staged to be released. The test is basic in that it
+# essentially tests 1) updates work 2) boot works.
+#
+# An example invocation for this test would look like:
+
+# ```
+# cosa buildfetch --stream=next --build=34.20210904.1.0 --artifact=qemu
+# cosa decompress --build=34.20210904.1.0
+# cosa kola run --build=34.20210904.1.0 --tag extended-upgrade
+# ```
+#
+# You can monitor the progress from the console and journal:
+#   - everything:
+#       - tail -f tmp/kola/ext.config.upgrade.extended/*/console.txt
+#   - major events:
+#       - tail -f tmp/kola/ext.config.upgrade.extended/*/journal.txt | grep --color -i 'ok reached version'
+#
+# For convenience, here is a list of the earliest releases on each
+# stream/architecture. x86_64 minimum version has to be 32.x because
+# of https://github.com/coreos/fedora-coreos-tracker/issues/1448
+#
+# stable
+#   - x86_64  31.20200108.3.0 -> works for BIOS, not UEFI
+#             32.20200601.3.0
+#   - aarch64 34.20210821.3.0
+#   - s390x   36.20220618.3.1
+# testing
+#   - x86_64  32.20200601.2.1
+#   - aarch64 34.20210904.2.0
+#   - s390x   36.20220618.2.0
+# next
+#   - x86_64  32.20200416.1.0
+#   - aarch64 34.20210904.1.0
+#   - s390x   36.20220618.1.1
+
+. /etc/os-release # for $VERSION_ID
+
+need_restart='false'
+
+# delete the disabling of updates that was done by the test framework
+if [ -f /etc/zincati/config.d/90-disable-auto-updates.toml ]; then
+    rm -f /etc/zincati/config.d/90-disable-auto-updates.toml
+    need_restart='true'
+fi
+
+# Early `next` releases before [1] had auto-updates disabled too. Let's
+# drop that config if it exists.
+# [1] https://github.com/coreos/fedora-coreos-config/commit/99eab318998441760cca224544fc713651f7a16d
+if [ -f /etc/zincati/config.d/90-disable-on-non-production-stream.toml ]; then
+    rm -f /etc/zincati/config.d/90-disable-on-non-production-stream.toml
+    need_restart='true'
+fi
+
+get_booted_deployment_json() {
+    rpm-ostree status  --json | jq -r '.deployments[] | select(.booted == true)'
+}
+version=$(get_booted_deployment_json | jq -r '.version')
+stream=$(get_booted_deployment_json | jq -r '.["base-commit-meta"]["fedora-coreos.stream"]')
+
+# Pick up the last release for the current stream from the update server
+test -f /srv/updateinfo.json || \
+    curl -L "https://updates.coreos.fedoraproject.org/v1/graph?basearch=$(arch)&stream=${stream}&rollout_wariness=0" > /srv/updateinfo.json
+last_release=$(jq -r .nodes[-1].version /srv/updateinfo.json)
+last_release_index=$(jq '.nodes | length-1' /srv/updateinfo.json)
+latest_edge=$(jq -r .edges[0][1] /srv/updateinfo.json)
+
+# Now that we have the release from update json, let's check if it has an edge pointing to it
+# The latest_edge would ideally have the value of last_release_index if the release has rolled out
+# If the edge does not exist, we would pick the second last release as our last_release
+if [ $last_release_index != $latest_edge ]; then
+    last_release=$(jq -r .nodes[-2].version /srv/updateinfo.json)
+fi
+
+# If the user dropped down a /etc/target_stream file then we'll
+# pick up the info from there.
+target_stream=$stream
+test -f /etc/target_stream && target_stream=$(< /etc/target_stream)
+test -f /srv/builds.json || \
+    curl -L "https://builds.coreos.fedoraproject.org/prod/streams/${target_stream}/builds/builds.json" > /srv/builds.json
+target_version=$(jq -r .builds[0].id /srv/builds.json)
+
+
+grab-gpg-keys() {
+    # For older FCOS we had an issue where when we tried to pull the
+    # commits from the repo it would fail if we were on N-2 because
+    # the newer commits would be signed with a key the old OS didn't
+    # know anything about. We applied a workaround in newer releases,
+    # so this workaround should be limited to zincati older than v0.0.24
+    # https://github.com/coreos/fedora-coreos-tracker/issues/749
+    max_version=${target_version:0:2} # i.e. 36, 37, 38, etc..
+    for ver in $(seq $VERSION_ID $max_version); do
+        file="/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-${ver}-primary"
+        if [ ! -e $file ]; then
+            need_restart='true'
+            curl -L "https://src.fedoraproject.org/rpms/fedora-repos/raw/rawhide/f/RPM-GPG-KEY-fedora-${ver}-primary" | \
+                sudo tee $file
+            sudo chcon -v --reference="/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-${VERSION_ID}-primary" $file
+        fi
+    done
+}
+
+fix-update-url() {
+    # We switched to non stg URL in zincati v0.0.10 [1]. For older clients
+    # we need to update the runtime configuration of zincati to get past the problem.
+    # [1] https://github.com/coreos/zincati/commit/1d73801ccd015cdce89f082cb1eeb9b4b8335760
+    file='/etc/zincati/config.d/50-fedora-coreos-cincinnati.toml'
+    if [ ! -e $file ]; then
+        need_restart='true'
+        cat > $file <<'EOF'
+[cincinnati]
+base_url= "https://updates.coreos.fedoraproject.org"
+EOF
+    fi
+}
+
+fix-allow-downgrade() {
+    # Older FCOS will complain about an upgrade target being 'chronologically older than current'
+    # This is documented in https://github.com/coreos/fedora-coreos-tracker/issues/481
+    # We can workaround the problem via a config dropin:
+    file='/etc/zincati/config.d/99-fedora-coreos-allow-downgrade.toml'
+    if [ ! -e $file ]; then
+        need_restart='true'
+        cat > $file <<'EOF'
+updates.allow_downgrade = true
+EOF
+    fi
+}
+
+move-to-cgroups-v2() {
+    # When upgrading to latest F41+ the system won't even boot on cgroups v1
+    if grep -q unified_cgroup_hierarchy /proc/cmdline; then
+        systemctl stop zincati
+        rpm-ostree cancel
+        rpm-ostree kargs --delete=systemd.unified_cgroup_hierarchy
+        need_restart='true'
+    fi
+}
+
+ok "Reached version: $version"
+
+# Are we all the way at the desired target version?
+# If so then we can exit with success!
+if vereq $version $target_version; then
+    ok "Fully upgraded to $target_version"
+    # log bootupctl information for inspection and check the status output
+    state=$(/usr/bin/bootupctl status 2>&1)
+    echo "$state"
+    if ! echo "$state" | grep -q "CoreOS aleph version"; then
+        fatal "check bootupctl status output"
+    fi
+    exit 0
+fi
+
+# Apply workarounds based on the current version of the system.
+#
+# First release on each stream with new enough zincati for updates stg.fedoraproject.org
+# - 31.20200505.3.0
+# - 31.20200505.2.0
+# - 32.20200505.1.0
+#
+# First release with new enough zincati with workaround for N-2 gpg key issue
+# - 35.20211119.3.0
+# - 35.20211119.2.0
+# - 35.20211119.1.0
+#
+# First release with new enough rpm-ostree with fix for allow-downgrade issue
+# - 31.20200517.3.0
+# - 31.20200517.2.0
+# - 32.20200517.1.0
+#
+case "$stream" in
+    'next')
+        verlt $version '35.20211119.1.0' && grab-gpg-keys
+        verlt $version '34.20210413.1.0' && move-to-cgroups-v2
+        verlt $version '32.20200517.1.0' && fix-allow-downgrade
+        verlt $version '32.20200505.1.0' && fix-update-url
+        ;;
+    'testing')
+        verlt $version '35.20211119.2.0' && grab-gpg-keys
+        verlt $version '34.20210529.2.0' && move-to-cgroups-v2
+        verlt $version '31.20200517.2.0' && fix-allow-downgrade
+        verlt $version '31.20200505.2.0' && fix-update-url
+        ;;
+    'stable')
+        verlt $version '35.20211119.3.0' && grab-gpg-keys
+        verlt $version '34.20210529.3.0' && move-to-cgroups-v2
+        verlt $version '31.20200517.3.0' && fix-allow-downgrade
+        verlt $version '31.20200505.3.0' && fix-update-url
+        ;;
+    *) fatal "unexpected stream: $stream";;
+esac
+
+# If we have made it all the way to the last release then
+# we have one more test. We'll now rebase to the target
+# version, which should be in the compose OSTree repo.
+if vereq $version $last_release; then
+    systemctl stop zincati
+    rpm-ostree rebase "fedora-compose:fedora/$(arch)/coreos/${target_stream}" $target_version
+    /tmp/autopkgtest-reboot $version # execute the reboot
+    sleep infinity
+fi
+
+# Restart if configuration was changed
+if [ "${need_restart}" == "true" ]; then
+    /tmp/autopkgtest-reboot setup
+    sleep infinity
+fi
+
+# Watch the Zincati logs to see if it got a lead on a new update.
+# Timeout after some time if no update. Unset pipefail since the
+# journalctl -f will give a bad exit code when grep exits early.
+set +o pipefail
+cmd="journalctl -b 0 -f --no-tail -u zincati.service"
+if ! timeout 90s $cmd | grep --max-count=1 'proceeding to stage it'; then
+    # No update initiated within timeout; let's error.
+    fatal "Updating the system stalled out on version: $version"
+fi
+set -o pipefail
+
+
+# OK update has been initiated, prepare for reboot and loop to show
+# status of zincati and rpm-ostreed
+/tmp/autopkgtest-reboot-prepare $version
+while true; do
+    sleep 20
+    systemctl status rpm-ostreed zincati --lines=0
+done
diff --git a/tests/kola/var-mount/config.ign b/tests/kola/var-mount/config.ign
deleted file mode 100644
index 3408b0cc43..0000000000
--- a/tests/kola/var-mount/config.ign
+++ /dev/null
@@ -1,36 +0,0 @@
-{
-  "ignition": {
-    "version": "3.0.0"
-  },
-  "storage": {
-    "disks": [
-      {
-        "device": "/dev/vda",
-        "partitions": [
-          {
-            "label": "var",
-            "sizeMiB": 0,
-            "startMiB": 5000
-          }
-        ],
-        "wipeTable": false
-      }
-    ],
-    "filesystems": [
-      {
-        "device": "/dev/disk/by-partlabel/var",
-        "format": "xfs",
-        "path": "/var"
-      }
-    ]
-  },
-  "systemd": {
-    "units": [
-      {
-        "contents": "[Unit]\nBefore=local-fs.target\nRequires=systemd-fsck@/dev/disk/by-partlabel/var\nAfter=systemd-fsck@/dev/disk/by-partlabel/var\n\n[Mount]\nWhere=/var\nWhat=/dev/disk/by-partlabel/var\nType=xfs\n\n[Install]\nRequiredBy=local-fs.target",
-        "enabled": true,
-        "name": "var.mount"
-      }
-    ]
-  }
-}
diff --git a/tests/kola/var-mount/luks/config.bu b/tests/kola/var-mount/luks/config.bu
new file mode 100644
index 0000000000..50b5612cf2
--- /dev/null
+++ b/tests/kola/var-mount/luks/config.bu
@@ -0,0 +1,25 @@
+variant: fcos
+version: 1.3.0
+storage:
+  disks:
+    - device: /dev/disk/by-id/coreos-boot-disk
+      partitions:
+        - label: var
+          size_mib: 1000
+          start_mib: 5000
+        - label: varlog
+      wipe_table: false
+  luks:
+    - name: varlog
+      device: /dev/disk/by-partlabel/varlog
+      clevis:
+        tpm2: true
+  filesystems:
+    - device: /dev/disk/by-partlabel/var
+      format: xfs
+      path: /var
+      with_mount_unit: true
+    - device: /dev/mapper/varlog
+      format: ext4
+      path: /var/log
+      with_mount_unit: true
diff --git a/tests/kola/var-mount/luks/data/commonlib.sh b/tests/kola/var-mount/luks/data/commonlib.sh
new file mode 120000
index 0000000000..b8dcbdca1a
--- /dev/null
+++ b/tests/kola/var-mount/luks/data/commonlib.sh
@@ -0,0 +1 @@
+../../../data/commonlib.sh
\ No newline at end of file
diff --git a/tests/kola/var-mount/luks/test.sh b/tests/kola/var-mount/luks/test.sh
new file mode 100755
index 0000000000..00e9455771
--- /dev/null
+++ b/tests/kola/var-mount/luks/test.sh
@@ -0,0 +1,52 @@
+#!/bin/bash
+## kola:
+##   architectures: "! s390x"
+##   description: Verify that reprovision disk with luks works.
+##   tags: platform-independent
+
+set -xeuo pipefail
+
+# shellcheck disable=SC1091
+. "$KOLA_EXT_DATA/commonlib.sh"
+
+# /var
+
+src=$(findmnt -nvr /var -o SOURCE)
+[[ $(realpath "$src") == $(realpath /dev/disk/by-partlabel/var) ]]
+
+fstype=$(findmnt -nvr /var -o FSTYPE)
+[[ $fstype == xfs ]]
+
+# /var/log
+
+src=$(findmnt -nvr /var/log -o SOURCE)
+[[ $(realpath "$src") == $(realpath /dev/mapper/varlog) ]]
+
+blktype=$(lsblk -o TYPE "${src}" --noheadings)
+[[ ${blktype} == crypt ]]
+
+fstype=$(findmnt -nvr /var/log -o FSTYPE)
+[[ $fstype == ext4 ]]
+
+table=$(dmsetup table varlog)
+if grep -q allow_discards <<< "${table}"; then
+    fatal "found allow_discards in /var/log DM table: ${table}"
+fi
+if grep -q no_read_workqueue <<< "${table}"; then
+    fatal "found no_read_workqueue in /var/log DM table: ${table}"
+fi
+ok "discard and custom option not enabled for /var/log"
+
+case "${AUTOPKGTEST_REBOOT_MARK:-}" in
+  "")
+      ok "mounted on first boot"
+
+      # reboot once to sanity-check we can mount on second boot
+      /tmp/autopkgtest-reboot rebooted
+      ;;
+
+  rebooted)
+      ok "mounted on reboot"
+      ;;
+  *) fatal "unexpected mark: ${AUTOPKGTEST_REBOOT_MARK}";;
+esac
diff --git a/tests/kola/var-mount/scsi-id/config.bu b/tests/kola/var-mount/scsi-id/config.bu
new file mode 100644
index 0000000000..118f4215f8
--- /dev/null
+++ b/tests/kola/var-mount/scsi-id/config.bu
@@ -0,0 +1,17 @@
+variant: fcos
+version: 1.4.0
+storage:
+  disks:
+    # the symlink is from wwn=123456789 converted to hexadecimal (base16)
+    - device: /dev/disk/by-id/scsi-300000000075bcd15
+      wipe_table: true
+      partitions:
+        - number: 1
+          label: var
+  filesystems:
+    - path: /var
+      device: /dev/disk/by-partlabel/var
+      format: xfs
+      wipe_filesystem: true
+      label: var
+      with_mount_unit: true
diff --git a/tests/kola/var-mount/scsi-id/data/commonlib.sh b/tests/kola/var-mount/scsi-id/data/commonlib.sh
new file mode 120000
index 0000000000..b8dcbdca1a
--- /dev/null
+++ b/tests/kola/var-mount/scsi-id/data/commonlib.sh
@@ -0,0 +1 @@
+../../../data/commonlib.sh
\ No newline at end of file
diff --git a/tests/kola/var-mount/scsi-id/test.sh b/tests/kola/var-mount/scsi-id/test.sh
new file mode 100755
index 0000000000..405f7a0f55
--- /dev/null
+++ b/tests/kola/var-mount/scsi-id/test.sh
@@ -0,0 +1,34 @@
+#!/bin/bash
+## kola:
+##   # additionalDisks is only supported on QEMU
+##   platforms: qemu
+##   additionalDisks: ["5G:channel=scsi,wwn=123456789"]
+##   description: Verify udev rules /dev/disk/by-id/scsi-* symlinks exist
+##     in initramfs.
+
+# See https://bugzilla.redhat.com/show_bug.cgi?id=1990506
+
+set -xeuo pipefail
+
+# shellcheck disable=SC1091
+. "$KOLA_EXT_DATA/commonlib.sh"
+
+fstype=$(findmnt -nvr /var -o FSTYPE)
+if [ $fstype != xfs ]; then
+    fatal "Error: /var fstype is $fstype, expected is xfs"
+fi
+
+ostree_conf=$(ls /boot/loader/entries/*.conf)
+
+initramfs=/boot$(grep initrd ${ostree_conf} | sed 's/initrd //g')
+tempfile=$(mktemp)
+lsinitrd $initramfs > $tempfile
+if ! grep -q "61-scsi-sg3_id.rules" $tempfile; then
+    fatal "Error: can not find 61-scsi-sg3_id.rules in $initramfs"
+fi
+
+if ! grep -q "63-scsi-sg3_symlink.rules" $tempfile; then
+    fatal "Error: can not find 63-scsi-sg3_symlink.rules in $initramfs"
+fi
+
+rm -f $tempfile
diff --git a/tests/kola/var-mount/simple/config.bu b/tests/kola/var-mount/simple/config.bu
new file mode 100644
index 0000000000..cb7584e459
--- /dev/null
+++ b/tests/kola/var-mount/simple/config.bu
@@ -0,0 +1,22 @@
+variant: fcos
+version: 1.3.0
+storage:
+  disks:
+    - device: /dev/disk/by-id/coreos-boot-disk
+      partitions:
+        - label: var
+          guid: 63194b49-e4b7-43f9-9a8b-df0fd8279bb7
+          size_mib: 1000
+          start_mib: 5000
+        - label: varlog
+          guid: 6385b84e-2c7b-4488-a870-667c565e01a8
+      wipe_table: false
+  filesystems:
+    - device: /dev/disk/by-partuuid/63194b49-e4b7-43f9-9a8b-df0fd8279bb7
+      format: xfs
+      path: /var
+      with_mount_unit: true
+    - device: /dev/disk/by-partuuid/6385b84e-2c7b-4488-a870-667c565e01a8
+      format: ext4
+      path: /var/log
+      with_mount_unit: true
diff --git a/tests/kola/var-mount/simple/data/commonlib.sh b/tests/kola/var-mount/simple/data/commonlib.sh
new file mode 120000
index 0000000000..b8dcbdca1a
--- /dev/null
+++ b/tests/kola/var-mount/simple/data/commonlib.sh
@@ -0,0 +1 @@
+../../../data/commonlib.sh
\ No newline at end of file
diff --git a/tests/kola/var-mount/simple/test.sh b/tests/kola/var-mount/simple/test.sh
new file mode 100755
index 0000000000..c18e23c5a1
--- /dev/null
+++ b/tests/kola/var-mount/simple/test.sh
@@ -0,0 +1,39 @@
+#!/bin/bash
+## kola:
+##   description: Verify that provision disk with guid works.
+##   tags: platform-independent
+
+set -xeuo pipefail
+
+# shellcheck disable=SC1091
+. "$KOLA_EXT_DATA/commonlib.sh"
+
+# /var
+
+src=$(findmnt -nvr /var -o SOURCE)
+[[ $(realpath "$src") == $(realpath /dev/disk/by-partuuid/63194b49-e4b7-43f9-9a8b-df0fd8279bb7) ]]
+
+fstype=$(findmnt -nvr /var -o FSTYPE)
+[[ $fstype == xfs ]]
+
+# /var/log
+
+src=$(findmnt -nvr /var/log -o SOURCE)
+[[ $(realpath "$src") == $(realpath /dev/disk/by-partuuid/6385b84e-2c7b-4488-a870-667c565e01a8) ]]
+
+fstype=$(findmnt -nvr /var/log -o FSTYPE)
+[[ $fstype == ext4 ]]
+
+case "${AUTOPKGTEST_REBOOT_MARK:-}" in
+  "")
+      ok "mounted on first boot"
+
+      # reboot once to sanity-check we can mount on second boot
+      /tmp/autopkgtest-reboot rebooted
+      ;;
+
+  rebooted)
+      ok "mounted on reboot"
+      ;;
+  *) fatal "unexpected mark: ${AUTOPKGTEST_REBOOT_MARK}";;
+esac
diff --git a/tests/kola/var-mount/test.sh b/tests/kola/var-mount/test.sh
deleted file mode 100755
index ec5bd84c68..0000000000
--- a/tests/kola/var-mount/test.sh
+++ /dev/null
@@ -1,11 +0,0 @@
-#!/bin/bash
-set -xeuo pipefail
-
-# restrict to qemu for now because the primary disk path is platform-dependent
-# kola: {"platforms": "qemu"}
-
-src=$(findmnt -nvr /var -o SOURCE)
-[[ $(realpath "$src") == $(realpath /dev/disk/by-partlabel/var) ]]
-
-fstype=$(findmnt -nvr /var -o FSTYPE)
-[[ $fstype == xfs ]]
diff --git a/tests/manual/coreos-builds-bisect.py b/tests/manual/coreos-builds-bisect.py
new file mode 100755
index 0000000000..49a27a9f00
--- /dev/null
+++ b/tests/manual/coreos-builds-bisect.py
@@ -0,0 +1,303 @@
+#!/usr/bin/python3
+#
+# Copyright 2024 Dusty Mabe <dusty@dustymabe.com>
+#
+#  This program is free software; you can redistribute it and/or
+#  modify it under the terms of the GNU Lesser General Public
+#  License as published by the Free Software Foundation; either
+#  version 2 of the License, or (at your option) any later version.
+#
+#  This library is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+#  Lesser General Public License for more details.
+#
+#  You should have received a copy of the GNU Lesser General Public
+#  License along with this library. If not, see <http://www.gnu.org/licenses/>.
+#
+#
+# This program will bisect a list of CoreOS builds, like what are
+# stored in a builds.json file. Example:
+#
+# It was heavily inspired by rpm-ostree-bisect:
+# https://github.com/ostreedev/ostree-releng-scripts/blob/master/rpm-ostree-bisect
+#
+# Here is some high level representation of what it does:
+#
+#    grab info on every build in builds.json
+#    A -> B -> C -> D -> E -> F -> G
+#
+#    user provided good/bad builds
+#
+#    run test script
+#    returns 0 for pass and 1 for failure
+#
+#    known good is A, known bad is G
+#
+#    start bisect:
+#    deploy D, test --> bad
+#    mark D, E, F bad
+#
+#    deploy B, test --> good
+#    mark B good
+#
+#    deploy C, test --> bad
+#
+#    Failure introduced in B -> C
+#
+# An example invocation from a cosa build dir looks like:
+#
+#   cosa buildfetch --force --stream=rawhide # to populate builds/builds.json
+#   ./coreos-builds-bisect.py --bad 41.20240404.91.0  \
+#                             --good 39.20230813.91.0 \
+#                             --testscript=./test.sh
+#
+# If the testing gets interrupted for some reason you should
+# be able to continue it with:
+#
+#   ./coreos-builds-bisect.py --resume                \
+#                             --bad 41.20240404.91.0  \
+#                             --good 39.20230813.91.0 \
+#                             --testscript=./test.sh
+#
+# The testcript that gets called from this will get passed one
+# argument, which is the ID of the build that should be tested.
+# It is up to the test script to pull anything necessary to test
+# that build, run the test, and return success or failure.
+#
+# For example:
+# ```
+#     cat ./test.sh
+#     #!/bin/bash
+#     set -eux -o pipefail
+#     build=$1
+#     cosa buildfetch --force --stream=rawhide --build=$build --artifact=qemu
+#     cosa decompress --build=$build
+#     cosa kola run --build=$build ext.config.mynewtestiwrote
+
+import argparse
+import json
+import os
+import os.path
+import subprocess
+import sys
+import tempfile
+from collections import OrderedDict
+
+DATA_FILE = './coreos-builds-bisect-data.json'
+BUILDS_JSON_FILE = './builds/builds.json'
+
+# Inspired by https://stackoverflow.com/a/287944
+class colors:
+    CLEAR = '\033[0m'
+    CYAN = '\033[96m'
+    GREEN = '\033[92m'
+    YELLOW = '\033[93m'
+    RED = '\033[91m'
+
+def fatal(msg):
+    print(colors.RED + msg + colors.CLEAR, file=sys.stderr)
+    sys.exit(1)
+
+def log_color(msg, color):
+    print(color + msg + colors.CLEAR)
+    sys.stdout.flush()
+
+def log(msg):
+    log_color(msg, colors.CYAN)
+
+def log_success(msg):
+    log_color(msg, colors.GREEN)
+
+def log_warn(msg):
+    log_color(msg, colors.YELLOW)
+
+"""
+    Initialize build ID order dict. The array will be a list of
+    commits in descending order. Each entry will be a dict with
+    key of commitid and value = a dict of version, heuristic
+    (TESTED, GIVEN, ASSUMED), and status (GOOD/BAD/UNKNOWN)
+
+    builds = {
+        '39.20230813.91.0' => {
+                      'heuristic', 'TESTED',
+                      'status': 'GOOD',
+                    },
+    }
+"""
+def initialize_builds_info(buildsjson, arch, badbuild, goodbuild):
+    with open(buildsjson, 'r') as f:
+        builds = json.load(f, object_pairs_hook=OrderedDict)
+    # Further narrow in on just the list of build dicts
+    builds = builds['builds']
+
+    # An ordered dictionary of builds info
+    info = OrderedDict()
+
+    # Populate the info dict
+    for build in builds:
+        # If this build has an entry for this architecture add it to
+        # our builds info dict.
+        if arch in build['arches']:
+            info.update({ build['id']: { 'status': 'UNKNOWN',
+                                         'heuristic': 'ASSUMED'}})
+    # Mark the bad commit bad and the good commit good
+    info[badbuild]['status'] = 'BAD'
+    info[badbuild]['heuristic'] = 'GIVEN'
+    info[goodbuild]['status'] = 'GOOD'
+    info[goodbuild]['heuristic'] = 'GIVEN'
+    return info
+
+
+def load_data(datafile):
+    with open(datafile, 'r') as f:
+        data = json.load(f, object_pairs_hook=OrderedDict)
+    return data
+
+
+def write_data(datafile, data):
+    dirname = os.path.dirname(datafile)
+    (_, tmpfile) = tempfile.mkstemp(
+                    dir=dirname,
+                    prefix="coreos-builds-bisect")
+
+    with open(tmpfile, 'w') as f:
+        json.dump(data, f, indent=4)
+    os.rename(tmpfile, datafile)
+
+
+def verify_script(testscript):
+    # Verify test script exists and is executable
+    if not testscript:
+        fatal("Must provide a --testscript to run")
+    if not (os.path.isfile(testscript)
+         and os.access(testscript, os.X_OK)):
+        fatal(f"provided test script: {testscript} is not an executable file")
+
+
+def bisect(args):
+    badbuild  = args.badbuild
+    goodbuild = args.goodbuild
+    testscript = args.testscript
+    datafile = args.datafile
+
+    builds_info = load_data(datafile)
+
+    # Loop until we're done bisecting
+    while True:
+        # Find list of unknown status builds
+        unknowns = []
+        lastbad = None
+        firstgood = None
+        for buildid in builds_info.keys():
+            status = builds_info[buildid]['status']
+            if status == 'BAD':
+                lastbad = buildid
+            elif status == 'UNKNOWN':
+                unknowns.append(buildid)
+            elif status == 'GOOD':
+                firstgood = buildid
+                break
+
+        # If we have no unknowns then we're done!
+        if len(unknowns) == 0:
+            log("BISECT TEST RESULTS:")
+            if firstgood is None:
+                log("No good builds were found in the history!")
+                return 0
+            # Do a sanity check to see if the good commit was actually tested.
+            if builds_info[firstgood]['heuristic'] == 'GIVEN':
+                log_warn("WARNING: The good build detected was the one given by the user.")
+                log_warn("WARNING: Are you sure this build is good?")
+            log(f"Last known good build: {firstgood}")
+            log(f"First known bad build: {lastbad}")
+            return 0
+
+        # Bisect to find new build id to test and then run the test
+        # //2 makes sure to give us an integer rather than a float
+        newbuildid = unknowns[len(unknowns)//2]
+        revisions = len(unknowns)
+        steps = 0
+        while revisions > 0:
+            revisions //= 2
+            steps += 1
+        log(f"Executing test for new build: {newbuildid}. ~{steps} steps left")
+        ec = subprocess.call([testscript, newbuildid])
+        if ec == 0:
+            log_success(f"{newbuildid} passed")
+            success = True
+        else:
+            log_warn(f"{newbuildid} failed")
+            success = False
+
+
+        # update the data with the results
+        if success:
+            for b in reversed(builds_info.keys()):
+                builds_info[b]['status'] = 'GOOD'
+                if b == newbuildid:
+                    break
+        else:
+            for b in builds_info.keys():
+                builds_info[b]['status'] = 'BAD'
+                if b == newbuildid:
+                    break
+        builds_info[b]['heuristic'] = 'TESTED'
+
+        # Save the state in case the script gets interrupted
+        write_data(datafile, builds_info)
+
+
+def main():
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--arch", dest='arch',
+                        help="What architecture to target", action='store')
+    parser.add_argument("--buildsjson", dest='buildsjson',
+                        help="Path to builds.json file",
+                        action='store', default=BUILDS_JSON_FILE)
+    parser.add_argument("--bad", dest='badbuild',
+                        help="Known Bad Build", action='store')
+    parser.add_argument("--good", dest='goodbuild',
+                        help="Known Good Build", action='store')
+    parser.add_argument("--testscript", help="A test script to run",
+                        action='store')
+    parser.add_argument("--resume", help="Resume a running bisection",
+                        action='store_true')
+    parser.add_argument("--datafile", help="data file to use for state",
+                        action='store', default=DATA_FILE)
+    args = parser.parse_args()
+    log(f"Using data file at: {args.datafile}")
+
+    verify_script(args.testscript)
+
+    if not args.arch:
+        cp = subprocess.run(['arch'], capture_output=True)
+        args.arch = cp.stdout.decode('utf-8').strip()
+    log(f"Targetting architecture: {args.arch}")
+
+    if args.resume:
+        if not os.path.exists(args.datafile):
+            fatal(f"Datafile at {args.datafile} must pre-exist with --resume")
+    else:
+        if not args.badbuild or not args.goodbuild:
+            fatal("Must specify both a bad and a good build via --bad and --good")
+        if not os.path.exists(args.buildsjson):
+            fatal(f"A builds.json file does not exist at {args.buildsjson}")
+        if os.path.exists(args.datafile) and not args.resume:
+            log(f"A datafile exists at {args.datafile} but --resume not specified")
+            log("If you want to resume a bisect pass --resume")
+            fatal("If you want to start a new bisect delete the datafile")
+
+        # initialize data
+        builds_info = initialize_builds_info(args.buildsjson,
+                                             args.arch,
+                                             args.badbuild,
+                                             args.goodbuild)
+        # Write data to file
+        write_data(args.datafile, builds_info)
+
+    bisect(args)
+
+if __name__ == '__main__':
+    main()
diff --git a/tests/manual/coreos-docs-net-testing.sh b/tests/manual/coreos-docs-net-testing.sh
index 5e0482a53a..b1426c879f 100755
--- a/tests/manual/coreos-docs-net-testing.sh
+++ b/tests/manual/coreos-docs-net-testing.sh
@@ -1,6 +1,4 @@
 #!/usr/bin/bash
-set -eu -o pipefail
-
 # This script attempts to test the configurations in our documentation
 # at https://docs.fedoraproject.org/en-US/fedora-coreos/sysconfig-network-configuration/.
 # All it does is test the various documented scenarios via both the
@@ -20,20 +18,24 @@ set -eu -o pipefail
 # I test this way I usually stand up a separate VM on the same bridge
 # and run dnsmasq on a tagged network like:
 #
+#     interface=eth1
 #     cat <<EOF > /etc/dnsmasq.d/vlandhcp
-#     interface=eth1.100
+#     interface=${interface}.100
 #     bind-interfaces
 #     dhcp-range=192.168.200.150,192.168.200.160,12h
-#     ip link add link eth0 name eth0.100 type vlan id 100
-#     ip addr add 192.168.200.1/24 dev eth0.100
-#     ip link set eth0.100 up
+#     EOF
+#     ip link add link $interface name "${interface}.100" type vlan id 100
+#     ip addr add 192.168.200.1/24 dev "${interface}.100"
+#     ip link set "${interface}.100" up
 #     systemctl enable dnsmasq --now
 #
 # - Dusty Mabe - dusty@dustymabe.com
 
+set -eu -o pipefail
+
 vmname="coreos-docs-nettest"
 
-fcct_common=\
+butane_common=\
 'variant: fcos
 version: 1.0.0
 passwd:
@@ -54,16 +56,21 @@ systemd:
           ExecStart=-/usr/sbin/agetty --autologin core --noclear %I $TERM
           TTYVTDisallocate=no
 storage:
-  files:'
+  files:
+    - path: /etc/sysctl.d/20-silence-audit.conf
+      contents:
+        inline: |
+          # Raise console message logging level from DEBUG (7) to WARNING (4)
+          kernel.printk=4'
 
-fcct_hostname='
+butane_hostname='
     - path: /etc/hostname
       mode: 0644
       contents:
         inline: |
           ${hostname}'
 
-fcct_disable_subnic2='
+butane_disable_subnic2='
     - path: /etc/NetworkManager/system-connections/${subnic2}.nmconnection
       mode: 0600
       contents:
@@ -77,7 +84,7 @@ fcct_disable_subnic2='
           [ipv6]
           method=disabled'
 
-fcct_staticip='
+butane_staticip='
     - path: /etc/NetworkManager/system-connections/${interface}.nmconnection
       mode: 0600
       contents:
@@ -94,7 +101,7 @@ fcct_staticip='
           may-fail=false
           method=manual'
 
-fcct_staticbond='
+butane_staticbond='
     - path: /etc/NetworkManager/system-connections/${bondname}.nmconnection
       mode: 0600
       contents:
@@ -134,7 +141,7 @@ fcct_staticbond='
           master=${bondname}
           slave-type=bond'
 
-fcct_dhcpbridge='
+butane_dhcpbridge='
     - path: /etc/NetworkManager/system-connections/${bridgename}.nmconnection
       mode: 0600
       contents:
@@ -171,7 +178,7 @@ fcct_dhcpbridge='
           slave-type=bridge
           [bridge-port]'
 
-fcct_dhcpteam='
+butane_dhcpteam='
     - path: /etc/NetworkManager/system-connections/${teamname}.nmconnection
       mode: 0600
       contents:
@@ -211,7 +218,7 @@ fcct_dhcpteam='
           [team-port]
           config={"prio": 100}'
 
-fcct_staticvlan='
+butane_staticvlan='
     - path: /etc/NetworkManager/system-connections/${interface}.${vlanid}.nmconnection
       mode: 0600
       contents:
@@ -249,7 +256,7 @@ fcct_staticvlan='
           dns-search=
           method=disabled'
 
-fcct_dhcpvlanbond='
+butane_dhcpvlanbond='
     - path: /etc/NetworkManager/system-connections/${bondname}.${vlanid}.nmconnection
       mode: 0600
       contents:
@@ -281,6 +288,8 @@ fcct_dhcpvlanbond='
           mode=active-backup
           [ipv4]
           method=disabled
+          [ipv6]
+          method=disabled
     - path: /etc/NetworkManager/system-connections/${bondname}-slave-${subnic1}.nmconnection
       mode: 0600
       contents:
@@ -304,7 +313,7 @@ fcct_dhcpvlanbond='
 
 check_requirement() {
     req=$1
-    if ! which $req &>/dev/null; then
+    if ! which "$req" &>/dev/null; then
         echo "No $req. Can't continue" 1>&2
         return 1
     fi
@@ -314,7 +323,7 @@ check_requirements() {
     reqs=(
         chcon
         envsubst
-        fcct
+        butane
         jq
         ssh
         ssh-keygen
@@ -323,8 +332,8 @@ check_requirements() {
         virt-install
         virt-ls
     )
-    for req in ${reqs[@]}; do
-        check_requirement $req
+    for req in "${reqs[@]}"; do
+        check_requirement "$req"
     done
 }
 
@@ -334,11 +343,11 @@ start_vm() {
     local ignitionfile=$1; shift
     local kernel=$1; shift
     local initramfs=$1; shift
-    local kernel_args=$@
+    local kernel_args=$*
     virt-install --name $vmname --ram 3096 --vcpus 2 --graphics=none \
                    --quiet --network bridge=virbr0 --network bridge=virbr0 \
-                   --disk size=20,backing_store=${disk} \
-                   --install kernel=${kernel},initrd=${initramfs},kernel_args_overwrite=yes,kernel_args="${kernel_args}" \
+                   --disk size=20,backing_store="${disk}" \
+                   --install kernel="${kernel}",initrd="${initramfs}",kernel_args_overwrite=yes,kernel_args="${kernel_args}" \
                    --qemu-commandline="-fw_cfg name=opt/com.coreos/config,file=$ignitionfile"
 }
 
@@ -352,12 +361,12 @@ destroy_vm() {
 }
 
 create_ignition_file() {
-    local fcctconfig=$1
+    local butaneconfig=$1
     local ignitionfile=$2
     # uncomment and use ign-converter instead if on rhcos less than 4.6
-    #echo "$fcctconfig" | fcct --strict | ign-converter -downtranslate -output $ignitionfile
-    echo "$fcctconfig" | fcct --strict --output $ignitionfile
-    chcon --verbose unconfined_u:object_r:svirt_home_t:s0 $ignitionfile &>/dev/null
+    #echo "$butaneconfig" | butane --strict | ign-converter -downtranslate -output $ignitionfile
+    echo "$butaneconfig" | butane --strict --output "$ignitionfile"
+    chcon --verbose unconfined_u:object_r:svirt_home_t:s0 "$ignitionfile" &>/dev/null
 }
 
 main() {
@@ -382,12 +391,12 @@ main() {
     local sshpubkeyfile="${PWD}/coreos-nettest-sshkey.pub"
     local ignitionfile="${PWD}/coreos-nettest-config.ign"
     local sshpubkey
-    local fcct
-     
+    local butane
+
     check_requirements
 
     # Find out which partition is the boot partition
-    partition=$(guestfish --ro -a $qcow <<EOF
+    partition=$(guestfish --ro -a "$qcow" <<EOF
     run
     findfs-label boot
     exit
@@ -395,17 +404,17 @@ EOF
     )
 
     # Grab kernel/initramfs from the disk
-    files=$(virt-ls -a $qcow -m $partition -R /ostree/)
+    files=$(virt-ls -a "$qcow" -m "$partition" -R /ostree/)
     for f in $files; do
         if [[ "${f}" =~ hmac$ ]]; then
             # ignore .vmlinuz-5.5.9-200.fc31.x86_64.hmac
             true
         elif [[ "${f}" =~ img$ ]]; then
             # grab initramfs in the form initramfs-5.5.9-200.fc31.x86_64.img
-            virt-cat -a $qcow -m $partition "/ostree/${f}" > $initramfs
+            virt-cat -a "$qcow" -m "$partition" "/ostree/${f}" > "$initramfs"
         elif [[ "${f}" =~ '/vmlinuz' ]]; then
             # grab kernel in the form vmlinuz-5.5.9-200.fc31.x86_64
-            virt-cat -a $qcow -m $partition "/ostree/${f}" > $kernel
+            virt-cat -a "$qcow" -m "$partition" "/ostree/${f}" > "$kernel"
         fi
     done
 
@@ -426,38 +435,34 @@ EOF
     # Grab kernel arguments from the disk and use them
     # - strip `options ` from the front of the line
     # - strip `$ignition_firstboot`
-    common_args=$(virt-cat -a $qcow -m $partition "/loader.1/entries/${bls_file}" | \
+    common_args=$(virt-cat -a "$qcow" -m "$partition" "/loader.1/entries/${bls_file}" | \
                   grep -P '^options' | \
                   sed -e 's/options //' | \
                   sed -e 's/$ignition_firstboot//')
     common_args+=' ignition.firstboot' # manually set ignition.firstboot
    #common_args+=' rd.break=pre-mount'
 
-    # Have to add ipv6.disable=1 for Fedora 33+ because of
-    # https://gitlab.freedesktop.org/NetworkManager/NetworkManager/-/issues/539
-    common_args+=' ipv6.disable=1'
-
     # export these values so we can substitute the values
     # in using the envsubst command
     export ip gateway netmask prefix interface nameserver bondname teamname bridgename subnic1 subnic2 vlanid
 
-    fcct_none=$(echo "${fcct_common}" | envsubst)
+    butane_none=$(echo "${butane_common}" | envsubst)
 
     export hostname="staticip"
     x="${common_args} rd.neednet=1"
     x+=" ip=${ip}::${gateway}:${netmask}:${hostname}:${interface}:none:${nameserver}"
     x+=" ip=${subnic2}:off"
     initramfs_staticip=$x
-    fcct_initramfs_staticip="${fcct_none}"
-    fcct_staticip=$(echo "${fcct_common}${fcct_hostname}${fcct_staticip}${fcct_disable_subnic2}" | envsubst)
+    butane_initramfs_staticip="${butane_none}"
+    butane_staticip=$(echo "${butane_common}${butane_hostname}${butane_staticip}${butane_disable_subnic2}" | envsubst)
 
     export hostname="staticbond"
     x="${common_args} rd.neednet=1"
     x+=" ip=${ip}::${gateway}:${netmask}:${hostname}:${bondname}:none:${nameserver}"
     x+=" bond=${bondname}:${subnic1},${subnic2}:mode=active-backup,miimon=100"
     initramfs_staticbond=$x
-    fcct_initramfs_staticbond="${fcct_none}"
-    fcct_staticbond=$(echo "${fcct_common}${fcct_hostname}${fcct_staticbond}" | envsubst)
+    butane_initramfs_staticbond="${butane_none}"
+    butane_staticbond=$(echo "${butane_common}${butane_hostname}${butane_staticbond}" | envsubst)
 
     export hostname="dhcpbridge"
     x="${common_args} rd.neednet=1"
@@ -465,8 +470,8 @@ EOF
     x+=" bridge=${bridgename}:${subnic1},${subnic2}"
     x+=" nameserver=${nameserver}"
     initramfs_dhcpbridge=$x
-    fcct_initramfs_dhcpbridge=$(echo "${fcct_common}${fcct_hostname}" | envsubst)
-    fcct_dhcpbridge=$(echo "${fcct_common}${fcct_hostname}${fcct_dhcpbridge}" | envsubst)
+    butane_initramfs_dhcpbridge=$(echo "${butane_common}${butane_hostname}" | envsubst)
+    butane_dhcpbridge=$(echo "${butane_common}${butane_hostname}${butane_dhcpbridge}" | envsubst)
 
     export hostname="dhcpteam"
     x="${common_args} rd.neednet=1"
@@ -474,28 +479,26 @@ EOF
     x+=" team=${teamname}:${subnic1},${subnic2}"
     x+=" nameserver=${nameserver}"
     initramfs_dhcpteam=$x
-    fcct_initramfs_dhcpteam=$(echo "${fcct_common}${fcct_hostname}" | envsubst)
-    fcct_dhcpteam=$(echo "${fcct_common}${fcct_hostname}${fcct_dhcpteam}" | envsubst)
+    butane_initramfs_dhcpteam=$(echo "${butane_common}${butane_hostname}" | envsubst)
+    butane_dhcpteam=$(echo "${butane_common}${butane_hostname}${butane_dhcpteam}" | envsubst)
 
     export hostname="staticvlan"
     x="${common_args} rd.neednet=1"
     x+=" ip=${ip}::${gateway}:${netmask}:${hostname}:${interface}.${vlanid}:none:${nameserver}"
-    x+=" ip=${interface}:off"
     x+=" vlan=${interface}.${vlanid}:${interface}"
     x+=" ip=${subnic2}:off"
     initramfs_staticvlan=$x
-    fcct_initramfs_staticvlan="${fcct_none}"
-    fcct_staticvlan=$(echo "${fcct_common}${fcct_hostname}${fcct_staticvlan}${fcct_disable_subnic2}" | envsubst)
+    butane_initramfs_staticvlan="${butane_none}"
+    butane_staticvlan=$(echo "${butane_common}${butane_hostname}${butane_staticvlan}${butane_disable_subnic2}" | envsubst)
 
     export hostname="dhcpvlanbond"
     x="${common_args} rd.neednet=1"
-    x+=" ip=vlan${vlanid}:dhcp"
-    x+=" ip=${bondname}:off"
+    x+=" ip=${bondname}.${vlanid}:dhcp"
     x+=" bond=${bondname}:${subnic1},${subnic2}:mode=active-backup,miimon=100"
-    x+=" vlan=vlan${vlanid}:${bondname}"
+    x+=" vlan=${bondname}.${vlanid}:${bondname}"
     initramfs_dhcpvlanbond=$x
-    fcct_initramfs_dhcpvlanbond=$(echo "${fcct_common}${fcct_hostname}" | envsubst)
-    fcct_dhcpvlanbond=$(echo "${fcct_common}${fcct_hostname}${fcct_dhcpvlanbond}" | envsubst)
+    butane_initramfs_dhcpvlanbond=$(echo "${butane_common}${butane_hostname}" | envsubst)
+    butane_dhcpvlanbond=$(echo "${butane_common}${butane_hostname}${butane_dhcpvlanbond}" | envsubst)
 
     destroy_vm || true
 
@@ -508,22 +511,22 @@ EOF
        #dhcpvlanbond  # Requires special setup, see top of file comment
     )
 
-    create_ignition_file "$fcct_none" $ignitionfile
-    for net in ${loopitems[@]}; do
+    create_ignition_file "$butane_none" $ignitionfile
+    for net in "${loopitems[@]}"; do
         var="initramfs_${net}"
         kernel_args=${!var}
-        var="fcct_initramfs_${net}"
-        fcctconfig=${!var}
-        create_ignition_file "$fcctconfig" $ignitionfile
+        var="butane_initramfs_${net}"
+        butaneconfig=${!var}
+        create_ignition_file "$butaneconfig" $ignitionfile
         start_vm $qcow $ignitionfile $kernel $initramfs "${kernel_args}"
         destroy_vm
     done
 
-    for net in ${loopitems[@]}; do
-        var="fcct_${net}"
-        fcctconfig=${!var}
+    for net in "${loopitems[@]}"; do
+        var="butane_${net}"
+        butaneconfig=${!var}
         kernel_args=${common_args}
-        create_ignition_file "$fcctconfig" $ignitionfile
+        create_ignition_file "$butaneconfig" $ignitionfile
         start_vm $qcow $ignitionfile $kernel $initramfs "${kernel_args}"
         destroy_vm
     done
@@ -536,5 +539,5 @@ EOF
 }
 
 
-main $@
+main "$@"
 
diff --git a/tests/manual/coreos-network-testing.sh b/tests/manual/coreos-network-testing.sh
index 505a543a0f..f276c69a63 100755
--- a/tests/manual/coreos-network-testing.sh
+++ b/tests/manual/coreos-network-testing.sh
@@ -1,7 +1,4 @@
 #!/usr/bin/bash
-set -eu -o pipefail
-#set -x
-
 # This script attempts to test networking configuration in various
 # scenarios/configurations. It tries to test what should happen
 # when initramfs networking is passed and/or networking config is
@@ -9,11 +6,15 @@ set -eu -o pipefail
 # configured network is passed properly to the real root when it
 # should be and not when it shouldn't be. See the following issue
 # for more details: https://github.com/coreos/fedora-coreos-tracker/issues/394
+#
 # - Dusty Mabe - dusty@dustymabe.com
 
+set -eu -o pipefail
+#set -x
+
 vmname="coreos-nettest"
 
-fcct_common=\
+butane_common=\
 'variant: fcos
 version: 1.0.0
 passwd:
@@ -45,17 +46,22 @@ storage:
       contents:
         source: https://raw.githubusercontent.com/coreos/fedora-coreos-config/8b08bd030ef3968d00d4fea9a0fa3ca3fbabf852/COPYING
         verification:
-          hash: sha512-d904690e4fc5defb804c2151e397cbe2aeeea821639995610aa377bb2446214c3433616a8708163776941df585b657648f20955e50d4b011ea2a96e7d8e08c66'
+          hash: sha512-d904690e4fc5defb804c2151e397cbe2aeeea821639995610aa377bb2446214c3433616a8708163776941df585b657648f20955e50d4b011ea2a96e7d8e08c66
+    - path: /etc/sysctl.d/20-silence-audit.conf
+      contents:
+        inline: |
+          # Raise console message logging level from DEBUG (7) to WARNING (4)
+          kernel.printk=4'
 
 ignitionhostname='ignitionhost'
-fcct_hostname='
+butane_hostname='
     - path: /etc/hostname
       mode: 0644
       contents:
         inline: |
           ${ignitionhostname}'
 
-fcct_static_nic0_ifcfg='
+butane_static_nic0_ifcfg='
     - path: /etc/sysconfig/network-scripts/ifcfg-${nic0}
       mode: 0600
       contents:
@@ -81,7 +87,7 @@ fcct_static_nic0_ifcfg='
           DEVICE=${nic1}
           ONBOOT=no'
 
-fcct_static_nic0='
+butane_static_nic0='
     - path: /etc/NetworkManager/system-connections/${nic0}.nmconnection
       mode: 0600
       contents:
@@ -111,7 +117,7 @@ fcct_static_nic0='
           [ipv6]
           method=disabled'
 
-fcct_static_team0='
+butane_static_team0='
     - path: /etc/NetworkManager/system-connections/team0.nmconnection
       mode: 0600
       contents:
@@ -153,7 +159,7 @@ fcct_static_team0='
           [team-port]
           config={"prio": 100}'
 
-fcct_static_bond0='
+butane_static_bond0='
     - path: /etc/NetworkManager/system-connections/bond0.nmconnection
       mode: 0600
       contents:
@@ -192,7 +198,7 @@ fcct_static_bond0='
           master=bond0
           slave-type=bond'
 
-fcct_static_br0='
+butane_static_br0='
     - path: /etc/NetworkManager/system-connections/br0.nmconnection
       mode: 0600
       contents:
@@ -233,7 +239,7 @@ fcct_static_br0='
 
 check_requirement() {
     req=$1
-    if ! which $req &>/dev/null; then
+    if ! which "$req" &>/dev/null; then
         echo "No $req. Can't continue" 1>&2
         return 1
     fi
@@ -243,7 +249,7 @@ check_requirements() {
     reqs=(
         chcon
         envsubst
-        fcct
+        butane
         jq
         ssh
         ssh-keygen
@@ -252,8 +258,8 @@ check_requirements() {
         virt-install
         virt-ls
     )
-    for req in ${reqs[@]}; do
-        check_requirement $req
+    for req in "${reqs[@]}"; do
+        check_requirement "$req"
     done
 }
 
@@ -263,11 +269,11 @@ start_vm() {
     local ignitionfile=$1; shift
     local kernel=$1; shift
     local initramfs=$1; shift
-    local kernel_args=$@
+    local kernel_args=$*
     virt-install --name $vmname --ram 3096 --vcpus 2 --graphics=none --noautoconsole \
                    --quiet --network bridge=virbr0 --network bridge=virbr0 \
-                   --disk size=20,backing_store=${disk} \
-                   --install kernel=${kernel},initrd=${initramfs},kernel_args_overwrite=yes,kernel_args="${kernel_args}" \
+                   --disk size=20,backing_store="${disk}" \
+                   --install kernel="${kernel}",initrd="${initramfs}",kernel_args_overwrite=yes,kernel_args="${kernel_args}" \
                    --qemu-commandline="-fw_cfg name=opt/com.coreos/config,file=$ignitionfile"
 }
 
@@ -285,7 +291,7 @@ check_vm() {
     ssh_config+=' -o StrictHostKeyChecking=no'
     ssh_config+=" -i $sshkeyfile"
 
-    if [ $dhcp == 'dhcp' ]; then
+    if [ "$dhcp" == 'dhcp' ]; then
         macinfo=$(virsh dumpxml $vmname | grep 'mac address' | head -n 1)
         macregex='(..:..:..:..:..:..)'
         if ! [[ $macinfo =~ $macregex ]]; then
@@ -374,7 +380,7 @@ check_vm() {
     # verify that the right number of NetworkManager keyfiles got created
     # use `echo -n | wc -l` so we can properly detect 0. Wasn't working
     # with `wc -l <<< $keyfiles`.
-    if [ ${detectedkeyfiles} != ${numkeyfiles} ]; then
+    if [ "${detectedkeyfiles}" != "${numkeyfiles}" ]; then
         rc=1
         echo "ERROR: Expected ${numkeyfiles} NM keyfiles, found ${detectedkeyfiles}" 1>&2
     fi
@@ -385,38 +391,38 @@ check_vm() {
     else
         nameserverinfo="$resolvedotconf"
     fi
-    if ! grep $nameserver &>/dev/null <<< "$nameserverinfo"; then
+    if ! grep "$nameserver" &>/dev/null <<< "$nameserverinfo"; then
         rc=1
         echo "ERROR: Nameserver information was not what was expected" 1>&2
     fi
 
     # verify that there are the right number of ipv4 devices "up"
-    if [ $(jq length <<< $ipinfo) != "$((interfaces+1))" ]; then
+    if [ "$(jq length <<< "$ipinfo")" != "$((interfaces+1))" ]; then
         rc=1
-        echo "ERROR: More interfaces up than expected" 1>&2 
+        echo "ERROR: More interfaces up than expected" 1>&2
     fi
     # verify that the first one in loopback
-    if [ $(jq -r .[0].addr_info[0].dev <<< $ipinfo) != 'lo' ]; then
+    if [ "$(jq -r .[0].addr_info[0].dev <<< "$ipinfo")" != 'lo' ]; then
         rc=1
-        echo "ERROR: The first active interface is not 'lo'" 1>&2 
+        echo "ERROR: The first active interface is not 'lo'" 1>&2
     fi
     # verify that the second one is the expected device
-    if [ $(jq -r .[1].addr_info[0].dev <<< $ipinfo) != "${dev}" ]; then
+    if [ "$(jq -r .[1].addr_info[0].dev <<< "$ipinfo")" != "${dev}" ]; then
         rc=1
-        echo "ERROR: The second active interface is not ${dev}" 1>&2 
+        echo "ERROR: The second active interface is not ${dev}" 1>&2
     fi
     # verify that the second one has the IP we assigned
-    if [ $(jq -r .[1].addr_info[0].local <<< $ipinfo) != "${ip}" ]; then
+    if [ "$(jq -r .[1].addr_info[0].local <<< "$ipinfo")" != "${ip}" ]; then
         rc=1
-        echo "ERROR: The second active interface does not have expected ip" 1>&2 
+        echo "ERROR: The second active interface does not have expected ip" 1>&2
     fi
 
     if [ "$rc" != '0' ]; then
         echo "$hostnameinfo"
         echo "$nameserverinfo"
         echo "$keyfiles"
-        jq -r .[].addr_info[].dev 1>&2 <<< $ipinfo
-        jq -r .[].addr_info[].local 1>&2 <<< $ipinfo
+        jq -r .[].addr_info[].dev 1>&2 <<< "$ipinfo"
+        jq -r .[].addr_info[].local 1>&2 <<< "$ipinfo"
         true
     else
         echo "Check for ${hostname} + dns:${nameserver} + ${dev}/${ip} passed!"
@@ -443,12 +449,12 @@ destroy_vm() {
 }
 
 create_ignition_file() {
-    local fcctconfig=$1
+    local butaneconfig=$1
     local ignitionfile=$2
     # uncomment and use ign-converter instead if on rhcos less than 4.6
-    #echo "$fcctconfig" | fcct --strict | ign-converter -downtranslate -output $ignitionfile
-    echo "$fcctconfig" | fcct --strict --output $ignitionfile
-    chcon --verbose unconfined_u:object_r:svirt_home_t:s0 $ignitionfile &>/dev/null
+    #echo "$butaneconfig" | butane --strict | ign-converter -downtranslate -output $ignitionfile
+    echo "$butaneconfig" | butane --strict --output "$ignitionfile"
+    chcon --verbose unconfined_u:object_r:svirt_home_t:s0 "$ignitionfile" &>/dev/null
 }
 
 
@@ -467,17 +473,17 @@ main() {
     local sshpubkeyfile="${PWD}/coreos-nettest-sshkey.pub"
     local ignitionfile="${PWD}/coreos-nettest-config.ign"
     local sshpubkey
-    local fcct
-     
+    local butane
+
     check_requirements
 
     # generate an ssh key to use:
-    rm -f $sshkeyfile $sshpubkeyfile
-    ssh-keygen -N '' -C '' -f $sshkeyfile &>/dev/null
-    sshpubkey=$(cat $sshpubkeyfile)
+    rm -f "$sshkeyfile" "$sshpubkeyfile"
+    ssh-keygen -N '' -C '' -f "$sshkeyfile" &>/dev/null
+    sshpubkey="$(cat "$sshpubkeyfile")"
 
     # Find out which partition is the boot partition
-    partition=$(guestfish --ro -a $qcow <<EOF
+    partition=$(guestfish --ro -a "$qcow" <<EOF
     run
     findfs-label boot
     exit
@@ -485,17 +491,17 @@ EOF
     )
 
     # Grab kernel/initramfs from the disk
-    files=$(virt-ls -a $qcow -m $partition -R /ostree/)
+    files=$(virt-ls -a "$qcow" -m "$partition" -R /ostree/)
     for f in $files; do
         if [[ "${f}" =~ hmac$ ]]; then
             # ignore .vmlinuz-5.5.9-200.fc31.x86_64.hmac
             true
         elif [[ "${f}" =~ img$ ]]; then
             # grab initramfs in the form initramfs-5.5.9-200.fc31.x86_64.img
-            virt-cat -a $qcow -m $partition "/ostree/${f}" > $initramfs
+            virt-cat -a "$qcow" -m "$partition" "/ostree/${f}" > "$initramfs"
         elif [[ "${f}" =~ '/vmlinuz' ]]; then
             # grab kernel in the form vmlinuz-5.5.9-200.fc31.x86_64
-            virt-cat -a $qcow -m $partition "/ostree/${f}" > $kernel
+            virt-cat -a "$qcow" -m "$partition" "/ostree/${f}" > "$kernel"
         fi
     done
 
@@ -525,7 +531,7 @@ EOF
     # Grab kernel arguments from the disk and use them
     # - strip `options ` from the front of the line
     # - strip `$ignition_firstboot`
-    common_args=$(virt-cat -a $qcow -m $partition "/loader.1/entries/${bls_file}" | \
+    common_args=$(virt-cat -a "$qcow" -m "$partition" "/loader.1/entries/${bls_file}" | \
                   grep -P '^options' | \
                   sed -e 's/options //' | \
                   sed -e 's/$ignition_firstboot//')
@@ -546,10 +552,8 @@ EOF
     x="${common_args} rd.neednet=1 ip=${nic0}:dhcp ip=${nic1}:dhcp"
     initramfs_dhcp_nic0nic1=$x
 
-    # Have to add ipv6.disable=1 for Fedora 33+ because of
-    # https://gitlab.freedesktop.org/NetworkManager/NetworkManager/-/issues/539
     devname=$nic0
-    x="${common_args} rd.neednet=1 ip=${nic1}:off ipv6.disable=1"
+    x="${common_args} rd.neednet=1 ip=${nic1}:off"
     x+=" ip=${ip}::${gateway}:${netmask}:${initramfshostname}:${devname}:none:${nameserverstatic}"
     initramfs_static_nic0=$x
 
@@ -575,12 +579,12 @@ EOF
     # in using the envsubst command
     export ip prefix nameserverstatic gateway sshpubkey ignitionhostname nic0 nic1
 
-    fcct_none=$(echo "${fcct_common}" | envsubst)
-    fcct_static_nic0=$(echo "${fcct_common}${fcct_hostname}${fcct_static_nic0}" | envsubst)
-    fcct_static_bond0=$(echo "${fcct_common}${fcct_hostname}${fcct_static_bond0}" | envsubst)
-    fcct_static_team0=$(echo "${fcct_common}${fcct_hostname}${fcct_static_team0}" | envsubst)
-    fcct_static_br0=$(echo "${fcct_common}${fcct_hostname}${fcct_static_br0}" | envsubst)
-    fcct_static_nic0_ifcfg=$(echo "${fcct_common}${fcct_hostname}${fcct_static_nic0_ifcfg}" | envsubst)
+    butane_none=$(echo "${butane_common}" | envsubst)
+    butane_static_nic0=$(echo "${butane_common}${butane_hostname}${butane_static_nic0}" | envsubst)
+    butane_static_bond0=$(echo "${butane_common}${butane_hostname}${butane_static_bond0}" | envsubst)
+    butane_static_team0=$(echo "${butane_common}${butane_hostname}${butane_static_team0}" | envsubst)
+    butane_static_br0=$(echo "${butane_common}${butane_hostname}${butane_static_br0}" | envsubst)
+    butane_static_nic0_ifcfg=$(echo "${butane_common}${butane_hostname}${butane_static_nic0_ifcfg}" | envsubst)
 
     # If the VM is still around for whatever reason, destroy it
     destroy_vm || true
@@ -590,7 +594,7 @@ EOF
     # networking. Do a ifcfg check to make sure.
     if [ "$rhcos" == 1 ]; then
         echo -e "\n###### Testing ifcfg file via Ignition disables initramfs propagation\n"
-        create_ignition_file "$fcct_static_nic0_ifcfg" $ignitionfile
+        create_ignition_file "$butane_static_nic0_ifcfg" $ignitionfile
         start_vm $qcow $ignitionfile $kernel $initramfs "$initramfs_static_bond0"
         check_vm 'none' 1 0 $ip $nic0 $ignitionhostname $nameserverstatic $sshkeyfile
         reboot_vm
@@ -602,13 +606,24 @@ EOF
     # configuration via Ignition either, so we'll just end up with DHCP and a
     # static hostname that is unset (`n/a`).
     echo -e "\n###### Testing coreos.no_persist_ip disables initramfs propagation\n"
-    create_ignition_file "$fcct_none" $ignitionfile
+    create_ignition_file "$butane_none" $ignitionfile
     start_vm $qcow $ignitionfile $kernel $initramfs "${initramfs_static_nic0} coreos.no_persist_ip"
     check_vm 'dhcp' 2 0 $ip $nic0 'n/a' $nameserverdhcp $sshkeyfile
     reboot_vm
     check_vm 'dhcp' 2 0 $ip $nic0 'n/a' $nameserverdhcp $sshkeyfile
     destroy_vm
 
+    # Do a `coreos.force_persist_ip` check. In this case we won't pass any networking
+    # configuration via Ignition either, so we'll just end up with DHCP and a
+    # static hostname that is unset (`n/a`).
+    echo -e "\n###### Testing coreos.force_persist_ip forces initramfs propagation\n"
+    create_ignition_file "$butane_static_nic0" $ignitionfile
+    start_vm $qcow $ignitionfile $kernel $initramfs "${initramfs_static_bond0} coreos.force_persist_ip"
+    check_vm 'none' 1 3 $ip bond0 $ignitionhostname $nameserverstatic $sshkeyfile
+    reboot_vm
+    check_vm 'none' 1 3 $ip bond0 $ignitionhostname $nameserverstatic $sshkeyfile
+    destroy_vm
+
     # Do a check for the `nameserver=` initramfs arg. Need to test along with
     # the $initramfs_dhcp_nic0nic1 because that brings up more than one
     # interface and is one that doesn't specify the nameserver as part of the
@@ -619,16 +634,16 @@ EOF
     # namesever= before ip= kargs doesn't yield an extra default.nm_connection
     # file. The second is to verify that the nameserver entry gets placed into
     # all connections that get created (i.e. ens2.nm_connection and ens3.nm_connection).
-    # 
+    #
     # We'll perform the first check automatically in check_vm by verifying the
     # number of keyfiles is 2, along with checking that the dns server did make
     # it into the resolv.conf or resolvectl (systemd-resolvd). We won't
     # automatically check that each file has the dns entry for now, but anyone
     # can manually run this and grab a console to the VM and verify that.
-    # 
+    #
     # [1] https://gitlab.freedesktop.org/NetworkManager/NetworkManager/issues/391
     echo -e "\n###### Testing initramfs nameserver= option\n"
-    create_ignition_file "$fcct_none" $ignitionfile
+    create_ignition_file "$butane_none" $ignitionfile
     start_vm $qcow $ignitionfile $kernel $initramfs "nameserver=${nameserverstatic} ${initramfs_dhcp_nic0nic1}"
     check_vm 'dhcp' 2 2 $ip $nic0 'n/a' $nameserverstatic $sshkeyfile
     reboot_vm
@@ -638,7 +653,7 @@ EOF
     # Do a `net.ifnames=0` check and make sure eth0 is the interface name.
     # We don't pass any hostname information so it will just be (`n/a`).
     echo -e "\n###### Testing net.ifnames=0 gives us legacy NIC naming\n"
-    create_ignition_file "$fcct_none" $ignitionfile
+    create_ignition_file "$butane_none" $ignitionfile
     start_vm $qcow $ignitionfile $kernel $initramfs "${initramfs_dhcp_eth0} net.ifnames=0"
     check_vm 'dhcp' 2 1 $ip 'eth0' 'n/a' $nameserverdhcp $sshkeyfile
     # Don't reboot and do another check because we didn't persist the net.ifnames=0 karg
@@ -653,24 +668,24 @@ EOF
         static_br0
     )
 
-    fcctloop=(
+    butaneloop=(
         none
         static_nic0
         static_bond0
         static_team0
         static_br0
     )
-        
-    for initramfsnet in ${initramfsloop[@]}; do
-        for fcctnet in ${fcctloop[@]}; do
+
+    for initramfsnet in "${initramfsloop[@]}"; do
+        for butanenet in "${butaneloop[@]}"; do
             method='none'; interfaces=1;
             nameserver=${nameserverstatic}
             numkeyfiles=3
-            if [ "${fcctnet}" == 'none' ]; then
-                # because we propagate initramfs networking if no real root networking 
+            if [ "${butanenet}" == 'none' ]; then
+                # because we propagate initramfs networking if no real root networking
                 devname=${initramfsnet##*_}
                 hostname=${initramfshostname}
-                # If we're using dhcp for initramfs and not providing any real root 
+                # If we're using dhcp for initramfs and not providing any real root
                 # networking then we need to tell check_vm we're using DHCP and set
                 # a few other values.
                 if [ "${initramfsnet}" == 'dhcp_nic0' ]; then
@@ -686,24 +701,24 @@ EOF
                     numkeyfiles=2
                 fi
             else
-                devname=${fcctnet##*_}
+                devname=${butanenet##*_}
                 hostname=${ignitionhostname}
                 # If we're not using a virtual NIC (bond, bridge, team, etc)
                 # then only two keyfiles will be created.
-                if [ "${fcctnet}" == 'static_nic0' ]; then
+                if [ "${butanenet}" == 'static_nic0' ]; then
                     numkeyfiles=2
                 fi
             fi
             # If devname=nic0 then replace with ${nic0} variable
             [ $devname == "nic0" ] && devname=${nic0}
-            fcctvar="fcct_${fcctnet}"
-            fcctconfig=${!fcctvar}
+            butanevar="butane_${butanenet}"
+            butaneconfig=${!butanevar}
             initramfsvar="initramfs_${initramfsnet}"
             kernel_args=${!initramfsvar}
 
-            echo -e "\n###### Testing initramfs: ${initramfsnet} + ignition/fcct: ${fcctnet}\n"
+            echo -e "\n###### Testing initramfs: ${initramfsnet} + ignition/butane: ${butanenet}\n"
 
-            create_ignition_file "$fcctconfig" $ignitionfile
+            create_ignition_file "$butaneconfig" $ignitionfile
             start_vm $qcow $ignitionfile $kernel $initramfs "${kernel_args}"
             check_vm $method $interfaces $numkeyfiles $ip $devname $hostname $nameserver $sshkeyfile
             reboot_vm
@@ -720,5 +735,5 @@ EOF
 }
 
 
-main $@
+main "$@"
 
diff --git a/tests/manual/create-hyperv-environment.sh b/tests/manual/create-hyperv-environment.sh
new file mode 100755
index 0000000000..25ddfff32b
--- /dev/null
+++ b/tests/manual/create-hyperv-environment.sh
@@ -0,0 +1,131 @@
+#!/bin/bash
+# This script launches an AWS Windows instance suitable for running Hyper-V.
+# Ideally kola would support running automated Hyper-V tests by launching
+# such an instance and driving it over SSH.
+
+set -euo pipefail
+
+REGION="us-east-2"
+AZ="us-east-2a"
+AMI_DESC="Windows_Server-2022-English-Full-Base"
+# least expensive Windows metal option
+INSTANCE_TYPE="m5zn.metal"
+VPC="coreos-hyperv"
+SG="coreos-hyperv"
+INSTANCE="coreos-hyperv"
+KEY_PREFIX="coreos-hyperv"
+DISK_GB=100
+DISP_W=1280
+DISP_H=950
+
+if [ $# != 1 ]; then
+    echo "Usage: $0 <workdir>"
+    exit 1
+fi
+
+dir="$1"
+mkdir -p "$dir"
+
+export AWS_DEFAULT_REGION="$REGION"
+export AWS_DEFAULT_OUTPUT="json"
+
+for req in aws jq python3 xfreerdp ; do
+    if ! which "$req" &>/dev/null; then
+        echo "No $req command. Can't continue." >&2
+        exit 1
+    fi
+done
+
+set -x
+
+# find AMI
+ami=$(aws ssm get-parameters --names "/aws/service/ami-windows-latest/$AMI_DESC" | jq -r ".Parameters[].Value")
+
+# get or create VPC
+vpc=$(aws ec2 describe-vpcs --filters "Name=tag:Name,Values=$VPC" | jq -r ".Vpcs[].VpcId")
+if [ -z "$vpc" ]; then
+    vpc=$(aws ec2 create-vpc --cidr-block 10.0.0.0/16 --tag-specifications "ResourceType=vpc,Tags=[{Key=Name,Value=$VPC}]" | jq -r ".Vpc.VpcId")
+fi
+
+# get or create Internet gateway
+igw=$(aws ec2 describe-internet-gateways --filters "Name=attachment.vpc-id,Values=$vpc" | jq -r ".InternetGateways[].InternetGatewayId")
+if [ -z "$igw" ]; then
+    igw=$(aws ec2 create-internet-gateway | jq -r .InternetGateway.InternetGatewayId)
+    aws ec2 attach-internet-gateway --internet-gateway-id "$igw" --vpc-id "$vpc"
+fi
+
+# get or create subnet
+subnet=$(aws ec2 describe-subnets --filters "Name=vpc-id,Values=$vpc" "Name=availability-zone,Values=$AZ" | jq -r ".Subnets[].SubnetId")
+if [ -z "$subnet" ]; then
+    octet=$(python3 -c "print(ord(\"$AZ\"[-1]) - ord('a'))")
+    subnet=$(aws ec2 create-subnet --vpc-id "$vpc" --availability-zone "$AZ" --cidr-block "10.0.$octet.0/24" | jq -r ".Subnet.SubnetId")
+    aws ec2 modify-subnet-attribute --subnet-id "$subnet" --map-public-ip-on-launch
+fi
+
+# get or create route table
+rt=$(aws ec2 describe-route-tables --filters "Name=association.subnet-id,Values=$subnet" | jq -r ".RouteTables[].RouteTableId")
+if [ -z "$rt" ]; then
+    rt=$(aws ec2 create-route-table --vpc-id "$vpc" | jq -r ".RouteTable.RouteTableId")
+    aws ec2 associate-route-table --route-table-id "$rt" --subnet-id "$subnet"
+    aws ec2 create-route --route-table-id "$rt" --destination-cidr-block 0.0.0.0/0 --gateway-id "$igw"
+fi
+
+# get or create security group
+sg=$(aws ec2 describe-security-groups --filters "Name=vpc-id,Values=$vpc" "Name=tag:Name,Values=$SG" | jq -r ".SecurityGroups[].GroupId")
+if [ -z "$sg" ]; then
+    sg=$(aws ec2 create-security-group --group-name "$SG" --tag-specifications "ResourceType=security-group,Tags=[{Key=Name,Value=$SG}]" --vpc-id "$vpc" --description "Hyper-V" | jq -r ".GroupId")
+    aws ec2 authorize-security-group-ingress --group-id "$sg" --protocol tcp --cidr 0.0.0.0/0 --port 3389
+fi
+
+# create temporary key pair
+key_name="$KEY_PREFIX-$RANDOM"
+ret=$(aws ec2 create-key-pair --key-name "$key_name")
+key_id=$(jq -r ".KeyPairId" <<< $ret)
+trap "aws ec2 delete-key-pair --key-pair-id $key_id" EXIT
+jq -r ".KeyMaterial" <<< $ret > "$dir/key.pem"
+
+# create userdata
+cat > "$dir/userdata" <<EOF
+<powershell>
+Install-WindowsFeature -Name Hyper-V -IncludeManagementTools -Restart
+</powershell>
+EOF
+
+# start instance
+instance=$(aws ec2 run-instances --instance-type "$INSTANCE_TYPE" --image-id "$ami" --subnet-id "$subnet" --security-group-ids "$sg" --key-name "$key_name" --tag-specifications --tag-specifications "ResourceType=instance,Tags=[{Key=Name,Value=$INSTANCE}]" --block-device-mappings '[{"DeviceName": "/dev/sda1", "Ebs": {"VolumeSize": '$DISK_GB'}}]' --user-data "file://$dir/userdata" | jq -r ".Instances[].InstanceId")
+
+# get Windows password
+while true; do
+    passwd=$(aws ec2 get-password-data --instance-id "$instance" --priv-launch-key "$dir/key.pem"  | jq -r .PasswordData)
+    if [ -n "$passwd" ]; then
+        break
+    fi
+    sleep 15
+done
+rm "$dir/key.pem"
+
+# get IP
+instance_ip=$(aws ec2 describe-instances --instance-id "$instance" | jq -r ".Reservations[].Instances[].NetworkInterfaces[].Association.PublicIp")
+
+# generate output
+set +x
+echo "$instance" > "$dir/instance-id"
+echo "$instance_ip" > "$dir/ip"
+mkdir -p "$dir/shared"
+cat > "$dir/connect.sh" <<EOF
+#!/bin/bash
+exec xfreerdp /cert:tofu /v:$instance_ip /u:Administrator /p:'$passwd' /w:$DISP_W /h:$DISP_H /drive:shared,"\$(dirname \$0)/shared"
+EOF
+cat > "$dir/terminate.sh" <<EOF
+#!/bin/bash
+set -xeuo pipefail
+aws --region=$REGION ec2 terminate-instances --instance-id $instance
+EOF
+chmod +x "$dir/connect.sh" "$dir/terminate.sh"
+
+# explain
+cat <<EOF
+
+Wait ~20 minutes for the machine to reboot, then use $dir/connect.sh to connect.
+Files in $dir/shared will be accessible via the "shared" drive under "This PC".
+EOF
diff --git a/versionary b/versionary
new file mode 100755
index 0000000000..09d49f735f
--- /dev/null
+++ b/versionary
@@ -0,0 +1,173 @@
+#!/usr/bin/python3 -u
+
+# This file originally lived in
+# https://github.com/coreos/fedora-coreos-releng-automation. See that repo for
+# archeological git research.
+
+'''
+    Implements the Fedora CoreOS versioning scheme as per:
+        https://github.com/coreos/fedora-coreos-tracker/issues/81
+        https://github.com/coreos/fedora-coreos-tracker/issues/211
+'''
+
+import argparse
+import json
+import os
+import re
+import subprocess
+import sys
+import time
+import yaml
+
+from datetime import datetime
+
+# streams which don't use lockfiles
+UNLOCKED_STREAMS = [
+    'rawhide',
+    'branched',
+    'bodhi-updates-testing',
+    'bodhi-updates',
+]
+
+# https://github.com/coreos/fedora-coreos-tracker/issues/211#issuecomment-543547587
+STREAM_TO_NUM = {
+    'next': 1,
+    'testing': 2,
+    'stable': 3,
+    'next-devel': 10,
+    'testing-devel': 20,
+    'rawhide': 91,
+    'branched': 92,
+    'bodhi-updates-testing': 93,
+    'bodhi-updates': 94,
+}
+
+
+def main():
+    args = parse_args()
+    if args.workdir is not None:
+        os.chdir(args.workdir)
+    assert os.path.isdir('builds'), 'Missing builds/ dir'
+
+    manifest = get_flattened_manifest()
+    x, y, z = (get_x(manifest), get_y(manifest), get_z(manifest))
+    n = get_next_iteration(x, y, z)
+    new_version = f'{x}.{y}.{z}.{n}'
+
+    # sanity check the new version by trying to re-parse it
+    assert parse_version(new_version) is not None
+    print(new_version)
+
+
+def parse_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--workdir', help="path to cosa workdir")
+    return parser.parse_args()
+
+
+def get_x(manifest):
+    """
+        X is the Fedora release version on which we're based.
+    """
+    releasever = manifest['releasever']
+    eprint(f"x: {releasever} (from manifest)")
+    return int(releasever)
+
+
+def get_y(manifest):
+    """
+        Y is the base snapshot date in YYYYMMDD format of Fedora. We derive
+        this using the timestamp in the base lockfile.
+    """
+
+    stream = manifest['add-commit-metadata']['fedora-coreos.stream']
+
+    # XXX: should sanity check that the lockfiles for all the basearches have
+    # matching timestamps
+    exts = ['json', 'yaml']
+    for ext in exts:
+        try:
+            with open(f"src/config/manifest-lock.x86_64.{ext}") as f:
+                lockfile = yaml.safe_load(f)
+                generated = lockfile.get('metadata', {}).get('generated')
+                if not generated:
+                    raise Exception("Missing 'metadata.generated' key "
+                                    f"from {lockfile}")
+                dt = datetime.strptime(generated, '%Y-%m-%dT%H:%M:%SZ')
+                assert stream not in UNLOCKED_STREAMS
+                msg_src = "from lockfile"
+                break
+        except FileNotFoundError:
+            continue
+    else:
+        # must be an unlocked stream
+        assert stream in UNLOCKED_STREAMS
+        msg_src = "unlocked stream"
+        dt = datetime.now()
+
+    ymd = dt.strftime('%Y%m%d')
+    eprint(f"y: {ymd} ({msg_src})")
+    return int(ymd)
+
+
+def get_z(manifest):
+    """
+        Z is the stream indicator.
+    """
+    stream = manifest['add-commit-metadata']['fedora-coreos.stream']
+    assert stream in STREAM_TO_NUM, f"Unknown stream: {stream}"
+    mapped = STREAM_TO_NUM[stream]
+    eprint(f"z: {mapped} (mapped from stream {stream})")
+    return mapped
+
+
+def get_next_iteration(x, y, z):
+    try:
+        with open('builds/builds.json') as f:
+            builds = json.load(f)
+    except FileNotFoundError:
+        builds = {'builds': []}
+
+    if len(builds['builds']) == 0:
+        eprint("n: 0 (no previous builds)")
+        return 0
+
+    last_buildid = builds['builds'][0]['id']
+    last_version = parse_version(last_buildid)
+    if not last_version:
+        eprint(f"n: 0 (previous version {last_buildid} does not match scheme)")
+        return 0
+
+    if (x, y, z) != last_version[:3]:
+        eprint(f"n: 0 (previous version {last_buildid} x.y.z does not match)")
+        return 0
+
+    n = last_version[3] + 1
+    eprint(f"n: {n} (incremented from previous version {last_buildid})")
+    return n
+
+
+def get_flattened_manifest():
+    return yaml.safe_load(
+        subprocess.check_output(['rpm-ostree', 'compose', 'tree',
+                                 '--print-only', 'src/config/manifest.yaml']))
+
+
+def parse_version(version):
+    m = re.match(r'^([0-9]{2})\.([0-9]{8})\.([0-9]+)\.([0-9]+)$', version)
+    if m is None:
+        return None
+    # sanity-check date
+    try:
+        time.strptime(m.group(2), '%Y%m%d')
+    except ValueError:
+        return None
+    return tuple(map(int, m.groups()))
+
+
+def eprint(*args):
+    print(*args, file=sys.stderr)
+
+
+if __name__ == "__main__":
+    sys.exit(main())