From 06a0cd3a68291128f98bf859d7eb649ebd86d4c2 Mon Sep 17 00:00:00 2001
From: harishsurf <hgovinda@redhat.com>
Date: Tue, 23 Apr 2024 20:01:53 +0000
Subject: [PATCH 01/12] Add support for sqlite storage in installer binary
 (PROJQUAY-6286)

Signed-off-by: harishsurf <hgovinda@redhat.com>
---
 .env                                          |  1 +
 Dockerfile                                    | 15 +++++-
 Dockerfile.online                             | 13 ++++-
 .../mirror_appliance/tasks/expand-vars.yaml   |  1 +
 .../tasks/install-quay-service.yaml           |  5 ++
 .../roles/mirror_appliance/tasks/migrate.yaml | 53 +++++++++++++++++++
 .../templates/quay.service.j2                 |  1 +
 cmd/install.go                                |  5 ++
 cmd/upgrade.go                                |  6 +++
 cmd/utils.go                                  | 38 +++++++++++++
 requirements.txt                              |  1 +
 11 files changed, 136 insertions(+), 3 deletions(-)
 create mode 100644 ansible-runner/context/app/project/roles/mirror_appliance/tasks/migrate.yaml
 create mode 100644 requirements.txt

diff --git a/.env b/.env
index 5ad3fac..6cef6ae 100644
--- a/.env
+++ b/.env
@@ -4,3 +4,4 @@ EE_BUILDER_IMAGE=registry.redhat.io/ansible-automation-platform-22/ansible-build
 QUAY_IMAGE=quay.io/projectquay/quay:3.12.0-nightly.20240626
 REDIS_IMAGE=registry.redhat.io/rhel8/redis-6:1-92.1669834635
 PAUSE_IMAGE=registry.access.redhat.com/ubi8/pause:8.7-6
+DB_TO_SQLITE_IMAGE=quay.io/quay/db-to-sqlite:latest
diff --git a/Dockerfile b/Dockerfile
index 6c355c7..09966a3 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -35,7 +35,11 @@ ENV REDIS_IMAGE=${REDIS_IMAGE}
 ENV PAUSE_IMAGE=${PAUSE_IMAGE}
 
 RUN go build -v \
+<<<<<<< HEAD
 	-ldflags "-X github.com/quay/mirror-registry/cmd.releaseVersion=${RELEASE_VERSION} -X github.com/quay/mirror-registry/cmd.eeImage=${EE_IMAGE} -X github.com/quay/mirror-registry/cmd.pauseImage=${PAUSE_IMAGE} -X github.com/quay/mirror-registry/cmd.quayImage=${QUAY_IMAGE} -X github.com/quay/mirror-registry/cmd.redisImage=${REDIS_IMAGE}" \
+=======
+	-ldflags "-X github.com/quay/mirror-registry/cmd.releaseVersion=${RELEASE_VERSION} -X github.com/quay/mirror-registry/cmd.eeImage=${EE_IMAGE} -X github.com/quay/mirror-registry/cmd.pauseImage=${PAUSE_IMAGE} -X github.com/quay/mirror-registry/cmd.quayImage=${QUAY_IMAGE} -X github.com/quay/mirror-registry/cmd.redisImage=${REDIS_IMAGE} -X github.com/quay/mirror-registry/cmd.postgresImage=${POSTGRES_IMAGE} -X github.com/quay/mirror-registry/cmd.sqliteImage=${DB_TO_SQLITE_IMAGE}" \
+>>>>>>> 7261fe9 (Add support for sqlite storage in installer binary (PROJQUAY-6286))
 	-o mirror-registry
 
 # Create Ansible Execution Environment
@@ -70,6 +74,12 @@ FROM $QUAY_IMAGE as quay
 FROM $REDIS_IMAGE as redis
 FROM $PAUSE_IMAGE as pause
 
+# Install db sqlite migration cli
+FROM registry.access.redhat.com/ubi8/python-36 AS db-cli
+
+COPY requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt
+
 # Create mirror registry archive
 FROM registry.access.redhat.com/ubi8:latest AS build
 
@@ -88,11 +98,14 @@ RUN tar -cvf quay.tar -C /quay .
 
 COPY --from=cli /cli/mirror-registry .
 
+COPY --from=db-cli / /db-cli
+RUN tar -cvf db-cli.tar -C /db-cli .
+
 # Bundle quay, redis and pause into a single archive
 RUN tar -cvf image-archive.tar quay.tar redis.tar pause.tar
 
 # Bundle mirror registry archive
-RUN tar -czvf mirror-registry.tar.gz image-archive.tar execution-environment.tar mirror-registry
+RUN tar -czvf mirror-registry.tar.gz image-archive.tar execution-environment.tar mirror-registry db-cli.tar
 
 # Extract bundle to final release image
 FROM registry.access.redhat.com/ubi8:latest AS release
diff --git a/Dockerfile.online b/Dockerfile.online
index 722dcca..d16664d 100644
--- a/Dockerfile.online
+++ b/Dockerfile.online
@@ -31,7 +31,7 @@ ENV REDIS_IMAGE=${REDIS_IMAGE}
 ENV PAUSE_IMAGE=${PAUSE_IMAGE}
 
 RUN go build -v \
-    -ldflags "-X github.com/quay/mirror-registry/cmd.releaseVersion=${RELEASE_VERSION} -X github.com/quay/mirror-registry/cmd.eeImage=${EE_IMAGE} -X github.com/quay/mirror-registry/cmd.pauseImage=${PAUSE_IMAGE} -X github.com/quay/mirror-registry/cmd.quayImage=${QUAY_IMAGE} -X github.com/quay/mirror-registry/cmd.redisImage=${REDIS_IMAGE}" \
+    -ldflags "-X github.com/quay/mirror-registry/cmd.releaseVersion=${RELEASE_VERSION} -X github.com/quay/mirror-registry/cmd.eeImage=${EE_IMAGE} -X github.com/quay/mirror-registry/cmd.pauseImage=${PAUSE_IMAGE} -X github.com/quay/mirror-registry/cmd.quayImage=${QUAY_IMAGE} -X github.com/quay/mirror-registry/cmd.redisImage=${REDIS_IMAGE} -X github.com/quay/mirror-registry/cmd.sqliteImage=${DB_TO_SQLITE_IMAGE}" \
     -o mirror-registry
 
 # Create Ansible Execution Environment
@@ -61,6 +61,12 @@ COPY --from=builder /output/ /output/
 RUN /output/install-from-bindep && rm -rf /output/wheels
 COPY ansible-runner/context/app /runner
 
+# Install db sqlite migration cli
+FROM registry.access.redhat.com/ubi8/python-36 AS db-cli
+
+COPY requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt
+
 # Create mirror registry archive
 FROM registry.redhat.io/ubi8:latest AS build
 
@@ -70,8 +76,11 @@ RUN tar -cvf execution-environment.tar -C /ansible .
 
 COPY --from=cli /cli/mirror-registry .
 
+COPY --from=db-cli / /db-cli
+RUN tar -cvf db-cli.tar -C /db-cli .
+
 # Bundle mirror registry archive
-RUN tar -czvf mirror-registry.tar.gz execution-environment.tar mirror-registry
+RUN tar -czvf mirror-registry.tar.gz execution-environment.tar mirror-registry db-cli.tar
 
 # Extract bundle to final release image
 FROM registry.redhat.io/ubi8:latest AS release
diff --git a/ansible-runner/context/app/project/roles/mirror_appliance/tasks/expand-vars.yaml b/ansible-runner/context/app/project/roles/mirror_appliance/tasks/expand-vars.yaml
index 1abe1f2..06372d8 100644
--- a/ansible-runner/context/app/project/roles/mirror_appliance/tasks/expand-vars.yaml
+++ b/ansible-runner/context/app/project/roles/mirror_appliance/tasks/expand-vars.yaml
@@ -15,3 +15,4 @@
     expanded_sqlite_storage: "{{ expanded_sqlite_storage_output.stdout }}"
     expanded_quay_root: "{{ expanded_quay_root_output.stdout }}"
     expanded_quay_storage: "{{ expanded_quay_storage_output.stdout }}"
+    expanded_sqlite_storage: "{{ expanded_sqlite_storage_output.stdout }}"
diff --git a/ansible-runner/context/app/project/roles/mirror_appliance/tasks/install-quay-service.yaml b/ansible-runner/context/app/project/roles/mirror_appliance/tasks/install-quay-service.yaml
index 0057435..b8b0075 100755
--- a/ansible-runner/context/app/project/roles/mirror_appliance/tasks/install-quay-service.yaml
+++ b/ansible-runner/context/app/project/roles/mirror_appliance/tasks/install-quay-service.yaml
@@ -139,6 +139,11 @@
       name: "{{ sqlite_storage }}"
   when: "not sqlite_storage.startswith('/')"
 
+- name: Create Quay sqlite storage named volume
+  containers.podman.podman_volume:
+      state: present
+      name: sqlite-storage
+
 - name: Start Quay service
   systemd:
     name: quay-app.service
diff --git a/ansible-runner/context/app/project/roles/mirror_appliance/tasks/migrate.yaml b/ansible-runner/context/app/project/roles/mirror_appliance/tasks/migrate.yaml
new file mode 100644
index 0000000..15f3e8c
--- /dev/null
+++ b/ansible-runner/context/app/project/roles/mirror_appliance/tasks/migrate.yaml
@@ -0,0 +1,53 @@
+- name: Install db-to-sqlite package
+  pip:
+    name: db-to-sqlite
+
+- name: Get connection string of running PostgreSQL container
+  containers.podman.podman_container_info:
+    name: quay-postgres
+  register: postgres_container_info
+
+- set_fact:
+    postgres_connection_string: "postgresql://{{ postgres_container_info.containers[0].network_settings.ip_address }}:5432/quay"    
+
+- name: Convert PostgreSQL database to SQLite
+  ansible.builtin.command: >
+    db-to-sqlite "{{ postgres_connection_string }}" pg_quay_dump.db --all
+  environment:
+    POSTGRES_PASSWORD: "{{ postgres_password }}"
+
+- name: Stop Quay service
+  systemd:
+    name: quay-app.service
+    enabled: no
+    daemon_reload: yes
+    state: stopped
+    force: yes
+    scope: "{{ systemd_scope }}"
+
+- name: Create Sqlite storage named volume
+  containers.podman.podman_volume:
+      state: present
+      name: sqlite-storage
+
+- name: Copy sqlite database file to sqlite storage volume
+  ansible.builtin.copy:
+    src: pg_quay_dump.db
+    dest: "{{ quay_storage }}:/pg_quay_dump.db"
+
+- name: Stop Postgres service
+  systemd:
+    name: quay-postgres.service
+    enabled: no
+    daemon_reload: yes
+    state: stopped
+    force: yes
+    scope: "{{ systemd_scope }}"
+
+- name: Start Quay service
+  systemd:
+    name: quay-app.service
+    enabled: yes
+    daemon_reload: yes
+    state: restarted
+    scope: "{{ systemd_scope }}"
diff --git a/ansible-runner/context/app/project/roles/mirror_appliance/templates/quay.service.j2 b/ansible-runner/context/app/project/roles/mirror_appliance/templates/quay.service.j2
index 839fc29..5c1002a 100755
--- a/ansible-runner/context/app/project/roles/mirror_appliance/templates/quay.service.j2
+++ b/ansible-runner/context/app/project/roles/mirror_appliance/templates/quay.service.j2
@@ -13,6 +13,7 @@ ExecStart=/usr/bin/podman run \
     -v {{ expanded_quay_root }}/quay-config:/quay-registry/conf/stack:Z \
     -v {{ expanded_sqlite_storage }}:/sqlite:Z \
     -v {{ expanded_quay_storage }}:/datastorage:Z \
+    -v {{ expanded_sqlite_storage }}:/quay-registry/sqlite/data:Z \
     --image-volume=ignore \
     --pod=quay-pod \
     --conmon-pidfile %t/%n-pid \
diff --git a/cmd/install.go b/cmd/install.go
index 6709cd8..f0eb26f 100644
--- a/cmd/install.go
+++ b/cmd/install.go
@@ -66,6 +66,9 @@ var quayStorage string
 // sqliteStorage is the directory where all the Quay sqlite data is stored
 var sqliteStorage string
 
+// sqliteStorage is the directory where all the Sqlite data is stored
+var sqliteStorage string
+
 // additionalArgs are arguments that you would like to append to the end of the ansible-playbook call (used mostly for development)
 var additionalArgs string
 
@@ -102,6 +105,8 @@ func init() {
 	installCmd.Flags().StringVarP(&sqliteStorage, "sqliteStorage", "", "sqlite-storage", "The folder where quay sqlite data is saved. This defaults to a Podman named volume 'sqlite-storage'. Root is required to uninstall.")
 	installCmd.Flags().StringVarP(&additionalArgs, "additionalArgs", "", "", "Additional arguments you would like to append to the ansible-playbook call. Used mostly for development.")
 
+	installCmd.Flags().StringVarP(&sqliteStorage, "sqliteStorage", "", "sqlite-storage", "The volume where sqlite persistent storage data is saved. This defaults to a Podman named volume 'sqlite-storage'. Root is required to uninstall.")
+
 }
 
 func install() {
diff --git a/cmd/upgrade.go b/cmd/upgrade.go
index 3ecd9f2..0b3d208 100644
--- a/cmd/upgrade.go
+++ b/cmd/upgrade.go
@@ -40,6 +40,8 @@ func init() {
 	upgradeCmd.Flags().StringVarP(&sqliteStorage, "sqliteStorage", "", "sqlite-storage", "The folder where quay sqlite data is saved. This defaults to a Podman named volume 'sqlite-storage'. Root is required to uninstall.")
 	upgradeCmd.Flags().StringVarP(&additionalArgs, "additionalArgs", "", "", "Additional arguments you would like to append to the ansible-playbook call. Used mostly for development.")
 
+	upgradeCmd.Flags().StringVarP(&sqliteStorage, "sqliteStorage", "", "sqlite-storage", "The volume where sqlite persistent storage data is saved. This defaults to a Podman named volume 'sqlite-storage'. Root is required to uninstall.")
+
 }
 
 func upgrade() {
@@ -80,6 +82,10 @@ func upgrade() {
 		}
 	}
 
+	// Load python db migration tool for sqlite
+	err = loadSqliteDBMigrationTool()
+	check(err)
+
 	if imageArchivePath != "" {
 		imageArchiveMountFlag = fmt.Sprintf("-v %s:/runner/image-archive.tar", imageArchivePath)
 		log.Info("Found image archive at " + imageArchivePath)
diff --git a/cmd/utils.go b/cmd/utils.go
index a635308..4e5542b 100644
--- a/cmd/utils.go
+++ b/cmd/utils.go
@@ -12,6 +12,9 @@ import (
 	"strings"
 )
 
+// This variable is set at build time via ldflags
+var sqliteImage string
+
 func loadExecutionEnvironment() error {
 
 	// Ensure execution environment is present
@@ -180,6 +183,35 @@ func check(err error) {
 	}
 }
 
+func loadSqliteDBMigrationTool() error {
+	// Ensure execution environment is present
+	executableDir, err := os.Executable()
+	if err != nil {
+		return err
+	}
+	sqliteMigrationToolPath := path.Join(path.Dir(executableDir), "db-cli.tar")
+	if !pathExists(sqliteMigrationToolPath) {
+		return errors.New("Could not find db-cli.tar at " + sqliteMigrationToolPath)
+	}
+	log.Info("Found db-cli tool at " + sqliteMigrationToolPath)
+
+	// Load db-to-sqlite python dep into podman
+	log.Printf("Loading db-to-sqlite migration tool from db-cli.tar")
+	statement := getImageMetadata("sqlite", sqliteImage, sqliteMigrationToolPath)
+	cmd := exec.Command("/bin/bash", "-c", statement)
+	if verbose {
+		cmd.Stderr = os.Stderr
+		cmd.Stdout = os.Stdout
+	}
+	log.Debug("Importing db-to-sqlite migration tool with command: ", cmd)
+
+	err = cmd.Run()
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
 // getImageMetadata provides the metadata needed for a corresponding image
 func getImageMetadata(app, imageName, archivePath string) string {
 	var statement string
@@ -192,6 +224,12 @@ func getImageMetadata(app, imageName, archivePath string) string {
 					--change 'ENTRYPOINT=["sleep"]' \
 					--change 'CMD=["infinity"]' \
 					- ` + imageName + ` < ` + archivePath
+	case "sqlite":
+		statement = `/usr/bin/podman image import \
+					--change 'ENV PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin' \
+					--change 'ENV container=oci' \
+					--change 'CMD=["db-to-sqlite"]' \
+					- ` + imageName + ` < ` + archivePath
 	case "ansible":
 		statement = `/usr/bin/podman image import \
 					--change 'ENV PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin' \
diff --git a/requirements.txt b/requirements.txt
new file mode 100644
index 0000000..40c3b35
--- /dev/null
+++ b/requirements.txt
@@ -0,0 +1 @@
+db-to-sqlite
\ No newline at end of file

From 5e6681b0f2f0179646ff8fbdf0f2c562a237d23c Mon Sep 17 00:00:00 2001
From: harishsurf <hgovinda@redhat.com>
Date: Tue, 23 Apr 2024 20:47:09 +0000
Subject: [PATCH 02/12] Change base image for python to reduce .tar size

---
 Dockerfile        | 10 ++++++++--
 Dockerfile.online | 16 ++++++++++++++--
 2 files changed, 22 insertions(+), 4 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 09966a3..26580fa 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -75,10 +75,16 @@ FROM $REDIS_IMAGE as redis
 FROM $PAUSE_IMAGE as pause
 
 # Install db sqlite migration cli
-FROM registry.access.redhat.com/ubi8/python-36 AS db-cli
+FROM registry.access.redhat.com/ubi8-minimal AS db-cli
+
+# Install Python 3 and pip
+RUN microdnf update -y && \
+    microdnf install python3 python3-pip -y && \
+    microdnf install gcc-c++ python3-devel -y && \
+    microdnf clean all
 
 COPY requirements.txt .
-RUN pip install --no-cache-dir -r requirements.txt
+RUN pip3 install --no-cache-dir -r requirements.txt
 
 # Create mirror registry archive
 FROM registry.access.redhat.com/ubi8:latest AS build
diff --git a/Dockerfile.online b/Dockerfile.online
index d16664d..93afeea 100644
--- a/Dockerfile.online
+++ b/Dockerfile.online
@@ -61,11 +61,23 @@ COPY --from=builder /output/ /output/
 RUN /output/install-from-bindep && rm -rf /output/wheels
 COPY ansible-runner/context/app /runner
 
+# # Install db sqlite migration cli
+# FROM registry.access.redhat.com/ubi8/python-36 AS db-cli
+
+# COPY requirements.txt .
+# RUN pip install --no-cache-dir -r requirements.txt
+
 # Install db sqlite migration cli
-FROM registry.access.redhat.com/ubi8/python-36 AS db-cli
+FROM registry.access.redhat.com/ubi8-minimal AS db-cli
+
+# Install Python 3 and pip
+RUN microdnf update -y && \
+    microdnf install python3 python3-pip -y && \
+    microdnf install gcc-c++ python3-devel -y && \
+    microdnf clean all
 
 COPY requirements.txt .
-RUN pip install --no-cache-dir -r requirements.txt
+RUN pip3 install --no-cache-dir -r requirements.txt
 
 # Create mirror registry archive
 FROM registry.redhat.io/ubi8:latest AS build

From e2d6fecd2948d6ccc56c3481fd9a57c63e7a6740 Mon Sep 17 00:00:00 2001
From: harishsurf <hgovinda@redhat.com>
Date: Wed, 24 Apr 2024 18:08:36 +0000
Subject: [PATCH 03/12] Add ansible tasks to migrate data from postgres to
 sqlite

---
 .../roles/mirror_appliance/tasks/migrate.yaml | 34 ++++++++++++-------
 .../roles/mirror_appliance/tasks/upgrade.yaml |  3 ++
 cmd/upgrade.go                                |  5 +++
 3 files changed, 29 insertions(+), 13 deletions(-)

diff --git a/ansible-runner/context/app/project/roles/mirror_appliance/tasks/migrate.yaml b/ansible-runner/context/app/project/roles/mirror_appliance/tasks/migrate.yaml
index 15f3e8c..1a5a670 100644
--- a/ansible-runner/context/app/project/roles/mirror_appliance/tasks/migrate.yaml
+++ b/ansible-runner/context/app/project/roles/mirror_appliance/tasks/migrate.yaml
@@ -1,6 +1,14 @@
-- name: Install db-to-sqlite package
-  pip:
-    name: db-to-sqlite
+- name: Check if sqlite migration tool image is loaded
+  command: podman inspect --type=image {{ sqlite_image }}
+  register: db_sqlite
+  ignore_errors: yes
+
+- name: Create necessary directory for storing quay postgres db snapshot
+  ansible.builtin.file:
+    path: "{{ quay_root }}/quay-postgres-backup"
+    mode: 0750
+    state: directory
+    recurse: yes
 
 - name: Get connection string of running PostgreSQL container
   containers.podman.podman_container_info:
@@ -8,13 +16,15 @@
   register: postgres_container_info
 
 - set_fact:
-    postgres_connection_string: "postgresql://{{ postgres_container_info.containers[0].network_settings.ip_address }}:5432/quay"    
+    postgres_connection_string: "postgresql://{{ postgres_container_info.containers[0].network_settings.ip_address }}:5432/quay"
 
-- name: Convert PostgreSQL database to SQLite
-  ansible.builtin.command: >
-    db-to-sqlite "{{ postgres_connection_string }}" pg_quay_dump.db --all
-  environment:
-    POSTGRES_PASSWORD: "{{ postgres_password }}"
+- name: Run db-to-sqlite command
+  command: >
+    podman run --rm
+    -v {{ quayRoot }}/quay-postgres-backup:/app
+    -e CONNECTION_STRING=" {{ postgres_connection_string}}"
+    {{ sqlite_image }}
+    quay_sqlite.db --all
 
 - name: Stop Quay service
   systemd:
@@ -30,10 +40,8 @@
       state: present
       name: sqlite-storage
 
-- name: Copy sqlite database file to sqlite storage volume
-  ansible.builtin.copy:
-    src: pg_quay_dump.db
-    dest: "{{ quay_storage }}:/pg_quay_dump.db"
+- name: Copy data to pg-storage volume
+  command: podman cp { quayRoot }}/quay-postgres-backup/quay_sqlite.db quay-app:/quay-registry/sqlite/data/quay_sqlite.db
 
 - name: Stop Postgres service
   systemd:
diff --git a/ansible-runner/context/app/project/roles/mirror_appliance/tasks/upgrade.yaml b/ansible-runner/context/app/project/roles/mirror_appliance/tasks/upgrade.yaml
index 2af203f..1137e3d 100755
--- a/ansible-runner/context/app/project/roles/mirror_appliance/tasks/upgrade.yaml
+++ b/ansible-runner/context/app/project/roles/mirror_appliance/tasks/upgrade.yaml
@@ -7,6 +7,9 @@
 - name: Set SELinux Rules
   include_tasks: set-selinux-rules.yaml
 
+- name: Migrate postgres db to sqlite for Quay
+  include_tasks: migrate.yaml
+
 - name: Autodetect Image Archive
   include_tasks: autodetect-image-archive.yaml
 
diff --git a/cmd/upgrade.go b/cmd/upgrade.go
index 0b3d208..eff4896 100644
--- a/cmd/upgrade.go
+++ b/cmd/upgrade.go
@@ -182,8 +182,13 @@ func upgrade() {
 		`--quiet `+
 		`--name ansible_runner_instance `+
 		fmt.Sprintf("%s ", eeImage)+
+<<<<<<< HEAD
 		`ansible-playbook -i %s@%s, --private-key /runner/env/ssh_key -e "quay_image=%s quay_version=%s redis_image=%s pause_image=%s quay_hostname=%s local_install=%s quay_root=%s quay_storage=%s sqlite_storage=%s" upgrade_mirror_appliance.yml %s %s`,
 		sshKey, targetUsername, targetHostname, quayImage, quayVersion, redisImage, pauseImage, quayHostname, strconv.FormatBool(isLocalInstall()), quayRoot, quayStorage, sqliteStorage, askBecomePassFlag, additionalArgs)
+=======
+		`ansible-playbook -i %s@%s, --private-key /runner/env/ssh_key -e "quay_image=%s quay_version=%s redis_image=%s postgres_image=%s sqlite_image=%s pause_image=%s quay_hostname=%s local_install=%s quay_root=%s quay_storage=%s pg_storage=%s sqlite_storage=%s" upgrade_mirror_appliance.yml %s %s`,
+		sshKey, targetUsername, targetHostname, quayImage, quayVersion, redisImage, postgresImage, sqliteImage, pauseImage, quayHostname, strconv.FormatBool(isLocalInstall()), quayRoot, quayStorage, pgStorage, sqliteStorage, askBecomePassFlag, additionalArgs)
+>>>>>>> d0f8065 (Add ansible tasks to migrate data from postgres to sqlite)
 
 	log.Debug("Running command: " + podmanCmd)
 	cmd := exec.Command("bash", "-c", podmanCmd)

From a6a981e7f6d8bc3c31eeebcbae0f8b16b5a6f3b9 Mon Sep 17 00:00:00 2001
From: harishsurf <hgovinda@redhat.com>
Date: Thu, 25 Apr 2024 06:06:35 +0000
Subject: [PATCH 04/12] Fix playbook tasks for sqlite db upgrade path

---
 Dockerfile                                    |  5 +-
 Dockerfile.online                             | 10 +--
 Makefile                                      |  4 +-
 .../tasks/install-quay-service.yaml           |  5 --
 .../roles/mirror_appliance/tasks/migrate.yaml | 76 ++++++++++++-------
 .../roles/mirror_appliance/tasks/upgrade.yaml |  6 +-
 cmd/install.go                                |  3 -
 cmd/utils.go                                  |  5 +-
 requirements.txt                              |  3 +-
 9 files changed, 67 insertions(+), 50 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 26580fa..68c1dcb 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -5,6 +5,7 @@ ARG EE_BASE_IMAGE=${EE_BASE_IMAGE}
 ARG EE_BUILDER_IMAGE=${EE_BUILDER_IMAGE}
 ARG REDIS_IMAGE=${REDIS_IMAGE}
 ARG PAUSE_IMAGE=${PAUSE_IMAGE}
+ARG DB_TO_SQLITE_IMAGE=${DB_TO_SQLITE_IMAGE}
 
 # Create Go CLI
 FROM registry.access.redhat.com/ubi8:latest AS cli
@@ -15,6 +16,7 @@ ARG QUAY_IMAGE=${QUAY_IMAGE}
 ARG EE_IMAGE=${EE_IMAGE}
 ARG REDIS_IMAGE=${REDIS_IMAGE}
 ARG PAUSE_IMAGE=${PAUSE_IMAGE}
+ARG DB_TO_SQLITE_IMAGE=${DB_TO_SQLITE_IMAGE}
 
 ENV GOROOT=/usr/local/go
 ENV PATH=$GOPATH/bin:$GOROOT/bin:$PATH 
@@ -33,6 +35,7 @@ ENV EE_IMAGE=${EE_IMAGE}
 ENV QUAY_IMAGE=${QUAY_IMAGE}
 ENV REDIS_IMAGE=${REDIS_IMAGE}
 ENV PAUSE_IMAGE=${PAUSE_IMAGE}
+ENV DB_TO_SQLITE_IMAGE=${DB_TO_SQLITE_IMAGE}
 
 RUN go build -v \
 <<<<<<< HEAD
@@ -79,7 +82,7 @@ FROM registry.access.redhat.com/ubi8-minimal AS db-cli
 
 # Install Python 3 and pip
 RUN microdnf update -y && \
-    microdnf install python3 python3-pip -y && \
+    microdnf install libpq-devel python3 python3-pip -y && \
     microdnf install gcc-c++ python3-devel -y && \
     microdnf clean all
 
diff --git a/Dockerfile.online b/Dockerfile.online
index 93afeea..5de0fe2 100644
--- a/Dockerfile.online
+++ b/Dockerfile.online
@@ -11,6 +11,7 @@ ARG QUAY_IMAGE=${QUAY_IMAGE}
 ARG EE_IMAGE=${EE_IMAGE}
 ARG REDIS_IMAGE=${REDIS_IMAGE}
 ARG PAUSE_IMAGE=${PAUSE_IMAGE}
+ARG DB_TO_SQLITE_IMAGE=${DB_TO_SQLITE_IMAGE}
 
 ENV GOROOT=/usr/local/go
 ENV PATH=$GOPATH/bin:$GOROOT/bin:$PATH 
@@ -29,6 +30,7 @@ ENV EE_IMAGE=${EE_IMAGE}
 ENV QUAY_IMAGE=${QUAY_IMAGE}
 ENV REDIS_IMAGE=${REDIS_IMAGE}
 ENV PAUSE_IMAGE=${PAUSE_IMAGE}
+ENV DB_TO_SQLITE_IMAGE=${DB_TO_SQLITE_IMAGE}
 
 RUN go build -v \
     -ldflags "-X github.com/quay/mirror-registry/cmd.releaseVersion=${RELEASE_VERSION} -X github.com/quay/mirror-registry/cmd.eeImage=${EE_IMAGE} -X github.com/quay/mirror-registry/cmd.pauseImage=${PAUSE_IMAGE} -X github.com/quay/mirror-registry/cmd.quayImage=${QUAY_IMAGE} -X github.com/quay/mirror-registry/cmd.redisImage=${REDIS_IMAGE} -X github.com/quay/mirror-registry/cmd.sqliteImage=${DB_TO_SQLITE_IMAGE}" \
@@ -61,18 +63,12 @@ COPY --from=builder /output/ /output/
 RUN /output/install-from-bindep && rm -rf /output/wheels
 COPY ansible-runner/context/app /runner
 
-# # Install db sqlite migration cli
-# FROM registry.access.redhat.com/ubi8/python-36 AS db-cli
-
-# COPY requirements.txt .
-# RUN pip install --no-cache-dir -r requirements.txt
-
 # Install db sqlite migration cli
 FROM registry.access.redhat.com/ubi8-minimal AS db-cli
 
 # Install Python 3 and pip
 RUN microdnf update -y && \
-    microdnf install python3 python3-pip -y && \
+    microdnf install libpq-devel python3 python3-pip -y && \
     microdnf install gcc-c++ python3-devel -y && \
     microdnf clean all
 
diff --git a/Makefile b/Makefile
index 3562d09..9786ca9 100644
--- a/Makefile
+++ b/Makefile
@@ -21,7 +21,8 @@ build-online-zip:
 		--build-arg EE_BUILDER_IMAGE=${EE_BUILDER_IMAGE} \
 		--build-arg REDIS_IMAGE=${REDIS_IMAGE} \
 		--build-arg PAUSE_IMAGE=${PAUSE_IMAGE} \
-		--file Dockerfile.online . 
+		--build-arg DB_TO_SQLITE_IMAGE=${DB_TO_SQLITE_IMAGE} \
+		--file Dockerfile.online .
 	$(CLIENT) run --name mirror-registry-online-${RELEASE_VERSION} mirror-registry-online:${RELEASE_VERSION}
 	$(CLIENT) cp mirror-registry-online-${RELEASE_VERSION}:/mirror-registry.tar.gz .
 	$(CLIENT) rm mirror-registry-online-${RELEASE_VERSION}
@@ -36,6 +37,7 @@ build-offline-zip:
 		--build-arg EE_BUILDER_IMAGE=${EE_BUILDER_IMAGE} \
 		--build-arg REDIS_IMAGE=${REDIS_IMAGE} \
 		--build-arg PAUSE_IMAGE=${PAUSE_IMAGE} \
+		--build-arg DB_TO_SQLITE_IMAGE=${DB_TO_SQLITE_IMAGE} \
 		--file Dockerfile .
 	$(CLIENT) run --name mirror-registry-offline-${RELEASE_VERSION} mirror-registry-offline:${RELEASE_VERSION}
 	$(CLIENT) cp mirror-registry-offline-${RELEASE_VERSION}:/mirror-registry.tar.gz .
diff --git a/ansible-runner/context/app/project/roles/mirror_appliance/tasks/install-quay-service.yaml b/ansible-runner/context/app/project/roles/mirror_appliance/tasks/install-quay-service.yaml
index b8b0075..0057435 100755
--- a/ansible-runner/context/app/project/roles/mirror_appliance/tasks/install-quay-service.yaml
+++ b/ansible-runner/context/app/project/roles/mirror_appliance/tasks/install-quay-service.yaml
@@ -139,11 +139,6 @@
       name: "{{ sqlite_storage }}"
   when: "not sqlite_storage.startswith('/')"
 
-- name: Create Quay sqlite storage named volume
-  containers.podman.podman_volume:
-      state: present
-      name: sqlite-storage
-
 - name: Start Quay service
   systemd:
     name: quay-app.service
diff --git a/ansible-runner/context/app/project/roles/mirror_appliance/tasks/migrate.yaml b/ansible-runner/context/app/project/roles/mirror_appliance/tasks/migrate.yaml
index 1a5a670..715cf23 100644
--- a/ansible-runner/context/app/project/roles/mirror_appliance/tasks/migrate.yaml
+++ b/ansible-runner/context/app/project/roles/mirror_appliance/tasks/migrate.yaml
@@ -3,6 +3,13 @@
   register: db_sqlite
   ignore_errors: yes
 
+- name: Pull Sqlite image
+  containers.podman.podman_image:
+    name: "{{ sqlite_image }}"
+  when: db_sqlite.rc != 0
+  retries: 5
+  delay: 5
+
 - name: Create necessary directory for storing quay postgres db snapshot
   ansible.builtin.file:
     path: "{{ quay_root }}/quay-postgres-backup"
@@ -10,21 +17,48 @@
     state: directory
     recurse: yes
 
-- name: Get connection string of running PostgreSQL container
-  containers.podman.podman_container_info:
-    name: quay-postgres
-  register: postgres_container_info
-
-- set_fact:
-    postgres_connection_string: "postgresql://{{ postgres_container_info.containers[0].network_settings.ip_address }}:5432/quay"
+- name: Create Sqlite storage named volume
+  containers.podman.podman_volume:
+      state: present
+      name: sqlite-storage
 
-- name: Run db-to-sqlite command
+- name: Run db-to-sqlite command via podman container
   command: >
-    podman run --rm
-    -v {{ quayRoot }}/quay-postgres-backup:/app
-    -e CONNECTION_STRING=" {{ postgres_connection_string}}"
+    podman run -d --name quay-db-migration --pod quay-pod
+    -v {{ sqlite_storage }}:/data:Z
+    --umask=013
     {{ sqlite_image }}
-    quay_sqlite.db --all
+    postgresql://user:{{ PGDB_PASSWORD }}@localhost/quay
+    quay_sqlite.db --all -p
+  register: container_result
+  ignore_errors: yes
+
+- name: Display migration error when sqlite migration fails
+  debug:
+    msg: "{{ container_result.stderr }}"
+  when: container_result.rc != 0
+
+- name: Back up postgres data from mounted volume to host machine
+  command: podman cp quay-db-migration:/data {{ quay_root }}/quay-postgres-backup/
+  when: container_result.rc == 0
+
+- name: Delete temporary migration container
+  containers.podman.podman_container:
+    name: quay-db-migration
+    state: absent
+  when: container_result.rc == 0
+
+# for local dev only
+- name: Update DB_URI in config.yaml
+  replace:
+    path: "{{ quay_root }}/quay-config/config.yaml"
+    regexp: '^DB_URI: postgresql://.*$'
+    replace: 'DB_URI: sqlite:////quay-registry/sqlite/data/quay_sqlite.db'
+  register: db_uri_update
+
+- debug:
+    msg: "DB_URI has been updated."
+  when: db_uri_update.changed
 
 - name: Stop Quay service
   systemd:
@@ -34,14 +68,7 @@
     state: stopped
     force: yes
     scope: "{{ systemd_scope }}"
-
-- name: Create Sqlite storage named volume
-  containers.podman.podman_volume:
-      state: present
-      name: sqlite-storage
-
-- name: Copy data to pg-storage volume
-  command: podman cp { quayRoot }}/quay-postgres-backup/quay_sqlite.db quay-app:/quay-registry/sqlite/data/quay_sqlite.db
+  when: container_result.rc == 0
 
 - name: Stop Postgres service
   systemd:
@@ -51,11 +78,4 @@
     state: stopped
     force: yes
     scope: "{{ systemd_scope }}"
-
-- name: Start Quay service
-  systemd:
-    name: quay-app.service
-    enabled: yes
-    daemon_reload: yes
-    state: restarted
-    scope: "{{ systemd_scope }}"
+  when: container_result.rc == 0
diff --git a/ansible-runner/context/app/project/roles/mirror_appliance/tasks/upgrade.yaml b/ansible-runner/context/app/project/roles/mirror_appliance/tasks/upgrade.yaml
index 1137e3d..2e4c742 100755
--- a/ansible-runner/context/app/project/roles/mirror_appliance/tasks/upgrade.yaml
+++ b/ansible-runner/context/app/project/roles/mirror_appliance/tasks/upgrade.yaml
@@ -7,15 +7,15 @@
 - name: Set SELinux Rules
   include_tasks: set-selinux-rules.yaml
 
+- name: Autodetect existing Secrets in config.yaml
+  include_tasks: upgrade-config-vars.yaml
+
 - name: Migrate postgres db to sqlite for Quay
   include_tasks: migrate.yaml
 
 - name: Autodetect Image Archive
   include_tasks: autodetect-image-archive.yaml
 
-- name: Autodetect existing Secrets in config.yaml
-  include_tasks: upgrade-config-vars.yaml
-
 - name: Upgrade Quay Pod Service
   include_tasks: upgrade-pod-service.yaml
 
diff --git a/cmd/install.go b/cmd/install.go
index f0eb26f..7255da7 100644
--- a/cmd/install.go
+++ b/cmd/install.go
@@ -66,9 +66,6 @@ var quayStorage string
 // sqliteStorage is the directory where all the Quay sqlite data is stored
 var sqliteStorage string
 
-// sqliteStorage is the directory where all the Sqlite data is stored
-var sqliteStorage string
-
 // additionalArgs are arguments that you would like to append to the end of the ansible-playbook call (used mostly for development)
 var additionalArgs string
 
diff --git a/cmd/utils.go b/cmd/utils.go
index 4e5542b..4e1b61f 100644
--- a/cmd/utils.go
+++ b/cmd/utils.go
@@ -228,7 +228,10 @@ func getImageMetadata(app, imageName, archivePath string) string {
 		statement = `/usr/bin/podman image import \
 					--change 'ENV PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin' \
 					--change 'ENV container=oci' \
-					--change 'CMD=["db-to-sqlite"]' \
+					--change 'WORKDIR=/data' \
+					--change 'VOLUME=/data' \
+					--change 'USER=1001' \
+					--change 'ENTRYPOINT ["db-to-sqlite"]' \
 					- ` + imageName + ` < ` + archivePath
 	case "ansible":
 		statement = `/usr/bin/podman image import \
diff --git a/requirements.txt b/requirements.txt
index 40c3b35..9e37515 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1 +1,2 @@
-db-to-sqlite
\ No newline at end of file
+db-to-sqlite
+psycopg2
\ No newline at end of file

From 027923228f631d822cdf33e85ba896c29a48ce40 Mon Sep 17 00:00:00 2001
From: harishsurf <hgovinda@redhat.com>
Date: Wed, 15 May 2024 14:53:06 +0000
Subject: [PATCH 05/12] Remove postgres and old db migration tasks from the
 upgrade cmd

This removes the ansible tasks related to postgres image upgrade
and OMR migration from 1.2.9 to 1.3.0.
---
 .../roles/mirror_appliance/tasks/migrate.yaml | 28 +++++++++++++++++--
 1 file changed, 26 insertions(+), 2 deletions(-)

diff --git a/ansible-runner/context/app/project/roles/mirror_appliance/tasks/migrate.yaml b/ansible-runner/context/app/project/roles/mirror_appliance/tasks/migrate.yaml
index 715cf23..39cd88a 100644
--- a/ansible-runner/context/app/project/roles/mirror_appliance/tasks/migrate.yaml
+++ b/ansible-runner/context/app/project/roles/mirror_appliance/tasks/migrate.yaml
@@ -24,7 +24,7 @@
 
 - name: Run db-to-sqlite command via podman container
   command: >
-    podman run -d --name quay-db-migration --pod quay-pod
+    podman run --name quay-db-migration --pod quay-pod
     -v {{ sqlite_storage }}:/data:Z
     --umask=013
     {{ sqlite_image }}
@@ -48,7 +48,6 @@
     state: absent
   when: container_result.rc == 0
 
-# for local dev only
 - name: Update DB_URI in config.yaml
   replace:
     path: "{{ quay_root }}/quay-config/config.yaml"
@@ -79,3 +78,28 @@
     force: yes
     scope: "{{ systemd_scope }}"
   when: container_result.rc == 0
+
+- name: Delete Postgres Storage named volume
+  containers.podman.podman_volume:
+      state: absent
+      name: pg-storage
+  when: auto_approve|bool == true and pg_storage == "pg-storage" and container_result.rc == 0
+
+- name: Delete Postgres Password Secret
+  containers.podman.podman_secret:
+    state: absent
+    name: pgdb_pass
+  when: container_result.rc == 0
+
+- name: Delete necessary directory for Postgres persistent data
+  ansible.builtin.file:
+    path: "{{ pg_storage }}"
+    state: absent
+  become: yes
+  when: auto_approve|bool == true and pg_storage.startswith('/') and container_result.rc == 0
+
+- name: Cleanup quay-postgres systemd unit file
+  file:
+    state: absent
+    path: "{{ systemd_unit_dir }}/quay-postgres.service"
+  when: container_result.rc == 0
\ No newline at end of file

From ff57778258afe968f0102639c274cfaf8cd7710a Mon Sep 17 00:00:00 2001
From: harishsurf <hgovinda@redhat.com>
Date: Fri, 14 Jun 2024 21:05:53 +0000
Subject: [PATCH 06/12] Add conditional when creating volume based on user
 input

---
 .../app/project/roles/mirror_appliance/tasks/migrate.yaml      | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/ansible-runner/context/app/project/roles/mirror_appliance/tasks/migrate.yaml b/ansible-runner/context/app/project/roles/mirror_appliance/tasks/migrate.yaml
index 39cd88a..4d16ed0 100644
--- a/ansible-runner/context/app/project/roles/mirror_appliance/tasks/migrate.yaml
+++ b/ansible-runner/context/app/project/roles/mirror_appliance/tasks/migrate.yaml
@@ -20,7 +20,8 @@
 - name: Create Sqlite storage named volume
   containers.podman.podman_volume:
       state: present
-      name: sqlite-storage
+      name: "{{ sqlite_storage }}"
+  when: "not sqlite_storage.startswith('/')"
 
 - name: Run db-to-sqlite command via podman container
   command: >

From e36b4dd3a8462ee484feeda32825350cc1638e15 Mon Sep 17 00:00:00 2001
From: harishsurf <hgovinda@redhat.com>
Date: Thu, 27 Jun 2024 22:28:21 +0000
Subject: [PATCH 07/12] Add a check to fail if sqlite file is not created post
 db migration + code cleanup

---
 .env                                          |  2 +-
 Dockerfile                                    |  6 +-----
 .../mirror_appliance/tasks/expand-vars.yaml   |  1 -
 .../tasks/install-redis-service.yaml          |  2 +-
 .../roles/mirror_appliance/tasks/migrate.yaml | 20 +++++++++----------
 .../mirror_appliance/tasks/wait-for-quay.yaml |  1 -
 .../templates/quay.service.j2                 |  1 -
 cmd/upgrade.go                                |  9 ++-------
 8 files changed, 14 insertions(+), 28 deletions(-)

diff --git a/.env b/.env
index 6cef6ae..d24c380 100644
--- a/.env
+++ b/.env
@@ -4,4 +4,4 @@ EE_BUILDER_IMAGE=registry.redhat.io/ansible-automation-platform-22/ansible-build
 QUAY_IMAGE=quay.io/projectquay/quay:3.12.0-nightly.20240626
 REDIS_IMAGE=registry.redhat.io/rhel8/redis-6:1-92.1669834635
 PAUSE_IMAGE=registry.access.redhat.com/ubi8/pause:8.7-6
-DB_TO_SQLITE_IMAGE=quay.io/quay/db-to-sqlite:latest
+DB_TO_SQLITE_IMAGE=quay.io/quay/db-to-sqlite:latest
\ No newline at end of file
diff --git a/Dockerfile b/Dockerfile
index 68c1dcb..469453e 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -38,11 +38,7 @@ ENV PAUSE_IMAGE=${PAUSE_IMAGE}
 ENV DB_TO_SQLITE_IMAGE=${DB_TO_SQLITE_IMAGE}
 
 RUN go build -v \
-<<<<<<< HEAD
-	-ldflags "-X github.com/quay/mirror-registry/cmd.releaseVersion=${RELEASE_VERSION} -X github.com/quay/mirror-registry/cmd.eeImage=${EE_IMAGE} -X github.com/quay/mirror-registry/cmd.pauseImage=${PAUSE_IMAGE} -X github.com/quay/mirror-registry/cmd.quayImage=${QUAY_IMAGE} -X github.com/quay/mirror-registry/cmd.redisImage=${REDIS_IMAGE}" \
-=======
-	-ldflags "-X github.com/quay/mirror-registry/cmd.releaseVersion=${RELEASE_VERSION} -X github.com/quay/mirror-registry/cmd.eeImage=${EE_IMAGE} -X github.com/quay/mirror-registry/cmd.pauseImage=${PAUSE_IMAGE} -X github.com/quay/mirror-registry/cmd.quayImage=${QUAY_IMAGE} -X github.com/quay/mirror-registry/cmd.redisImage=${REDIS_IMAGE} -X github.com/quay/mirror-registry/cmd.postgresImage=${POSTGRES_IMAGE} -X github.com/quay/mirror-registry/cmd.sqliteImage=${DB_TO_SQLITE_IMAGE}" \
->>>>>>> 7261fe9 (Add support for sqlite storage in installer binary (PROJQUAY-6286))
+	-ldflags "-X github.com/quay/mirror-registry/cmd.releaseVersion=${RELEASE_VERSION} -X github.com/quay/mirror-registry/cmd.eeImage=${EE_IMAGE} -X github.com/quay/mirror-registry/cmd.pauseImage=${PAUSE_IMAGE} -X github.com/quay/mirror-registry/cmd.quayImage=${QUAY_IMAGE} -X github.com/quay/mirror-registry/cmd.redisImage=${REDIS_IMAGE} -X github.com/quay/mirror-registry/cmd.sqliteImage=${DB_TO_SQLITE_IMAGE}" \
 	-o mirror-registry
 
 # Create Ansible Execution Environment
diff --git a/ansible-runner/context/app/project/roles/mirror_appliance/tasks/expand-vars.yaml b/ansible-runner/context/app/project/roles/mirror_appliance/tasks/expand-vars.yaml
index 06372d8..1abe1f2 100644
--- a/ansible-runner/context/app/project/roles/mirror_appliance/tasks/expand-vars.yaml
+++ b/ansible-runner/context/app/project/roles/mirror_appliance/tasks/expand-vars.yaml
@@ -15,4 +15,3 @@
     expanded_sqlite_storage: "{{ expanded_sqlite_storage_output.stdout }}"
     expanded_quay_root: "{{ expanded_quay_root_output.stdout }}"
     expanded_quay_storage: "{{ expanded_quay_storage_output.stdout }}"
-    expanded_sqlite_storage: "{{ expanded_sqlite_storage_output.stdout }}"
diff --git a/ansible-runner/context/app/project/roles/mirror_appliance/tasks/install-redis-service.yaml b/ansible-runner/context/app/project/roles/mirror_appliance/tasks/install-redis-service.yaml
index e020c08..2966e98 100755
--- a/ansible-runner/context/app/project/roles/mirror_appliance/tasks/install-redis-service.yaml
+++ b/ansible-runner/context/app/project/roles/mirror_appliance/tasks/install-redis-service.yaml
@@ -20,7 +20,7 @@
     state: present
     name: redis_pass
     data: "{{ redis_password }}"
-    skip_existing: false
+    skip_existing: true
 
 - name: Start Redis service
   systemd:
diff --git a/ansible-runner/context/app/project/roles/mirror_appliance/tasks/migrate.yaml b/ansible-runner/context/app/project/roles/mirror_appliance/tasks/migrate.yaml
index 4d16ed0..48ae2a7 100644
--- a/ansible-runner/context/app/project/roles/mirror_appliance/tasks/migrate.yaml
+++ b/ansible-runner/context/app/project/roles/mirror_appliance/tasks/migrate.yaml
@@ -43,23 +43,21 @@
   command: podman cp quay-db-migration:/data {{ quay_root }}/quay-postgres-backup/
   when: container_result.rc == 0
 
+- name: Check if sqlite file exists
+  stat:
+    path: "{{ quay_root }}/quay-postgres-backup/quay_sqlite.db"
+  register: file_check
+
+- fail:
+    msg: "Sqlite db file '{{ quay_root }}/quay-postgres-backup/quay_sqlite.db' does not exist."
+  when: file_check.stat.exists == false
+
 - name: Delete temporary migration container
   containers.podman.podman_container:
     name: quay-db-migration
     state: absent
   when: container_result.rc == 0
 
-- name: Update DB_URI in config.yaml
-  replace:
-    path: "{{ quay_root }}/quay-config/config.yaml"
-    regexp: '^DB_URI: postgresql://.*$'
-    replace: 'DB_URI: sqlite:////quay-registry/sqlite/data/quay_sqlite.db'
-  register: db_uri_update
-
-- debug:
-    msg: "DB_URI has been updated."
-  when: db_uri_update.changed
-
 - name: Stop Quay service
   systemd:
     name: quay-app.service
diff --git a/ansible-runner/context/app/project/roles/mirror_appliance/tasks/wait-for-quay.yaml b/ansible-runner/context/app/project/roles/mirror_appliance/tasks/wait-for-quay.yaml
index 306db36..755840e 100644
--- a/ansible-runner/context/app/project/roles/mirror_appliance/tasks/wait-for-quay.yaml
+++ b/ansible-runner/context/app/project/roles/mirror_appliance/tasks/wait-for-quay.yaml
@@ -22,4 +22,3 @@
     - name: Fail the playbook due to Quay not becoming alive
       fail:
         msg: "Quay did not become alive. Check debug logs above for details."
-
diff --git a/ansible-runner/context/app/project/roles/mirror_appliance/templates/quay.service.j2 b/ansible-runner/context/app/project/roles/mirror_appliance/templates/quay.service.j2
index 5c1002a..839fc29 100755
--- a/ansible-runner/context/app/project/roles/mirror_appliance/templates/quay.service.j2
+++ b/ansible-runner/context/app/project/roles/mirror_appliance/templates/quay.service.j2
@@ -13,7 +13,6 @@ ExecStart=/usr/bin/podman run \
     -v {{ expanded_quay_root }}/quay-config:/quay-registry/conf/stack:Z \
     -v {{ expanded_sqlite_storage }}:/sqlite:Z \
     -v {{ expanded_quay_storage }}:/datastorage:Z \
-    -v {{ expanded_sqlite_storage }}:/quay-registry/sqlite/data:Z \
     --image-volume=ignore \
     --pod=quay-pod \
     --conmon-pidfile %t/%n-pid \
diff --git a/cmd/upgrade.go b/cmd/upgrade.go
index eff4896..045e218 100644
--- a/cmd/upgrade.go
+++ b/cmd/upgrade.go
@@ -182,13 +182,8 @@ func upgrade() {
 		`--quiet `+
 		`--name ansible_runner_instance `+
 		fmt.Sprintf("%s ", eeImage)+
-<<<<<<< HEAD
-		`ansible-playbook -i %s@%s, --private-key /runner/env/ssh_key -e "quay_image=%s quay_version=%s redis_image=%s pause_image=%s quay_hostname=%s local_install=%s quay_root=%s quay_storage=%s sqlite_storage=%s" upgrade_mirror_appliance.yml %s %s`,
-		sshKey, targetUsername, targetHostname, quayImage, quayVersion, redisImage, pauseImage, quayHostname, strconv.FormatBool(isLocalInstall()), quayRoot, quayStorage, sqliteStorage, askBecomePassFlag, additionalArgs)
-=======
-		`ansible-playbook -i %s@%s, --private-key /runner/env/ssh_key -e "quay_image=%s quay_version=%s redis_image=%s postgres_image=%s sqlite_image=%s pause_image=%s quay_hostname=%s local_install=%s quay_root=%s quay_storage=%s pg_storage=%s sqlite_storage=%s" upgrade_mirror_appliance.yml %s %s`,
-		sshKey, targetUsername, targetHostname, quayImage, quayVersion, redisImage, postgresImage, sqliteImage, pauseImage, quayHostname, strconv.FormatBool(isLocalInstall()), quayRoot, quayStorage, pgStorage, sqliteStorage, askBecomePassFlag, additionalArgs)
->>>>>>> d0f8065 (Add ansible tasks to migrate data from postgres to sqlite)
+		`ansible-playbook -i %s@%s, --private-key /runner/env/ssh_key -e "quay_image=%s quay_version=%s redis_image=%s sqlite_image=%s pause_image=%s quay_hostname=%s local_install=%s quay_root=%s quay_storage=%s sqlite_storage=%s" upgrade_mirror_appliance.yml %s %s`,
+		sshKey, targetUsername, targetHostname, quayImage, quayVersion, redisImage, sqliteImage, pauseImage, quayHostname, strconv.FormatBool(isLocalInstall()), quayRoot, quayStorage, sqliteStorage, askBecomePassFlag, additionalArgs)
 
 	log.Debug("Running command: " + podmanCmd)
 	cmd := exec.Command("bash", "-c", podmanCmd)

From 5f64d83ddfeb7bf7c44b741f18100763410c60ae Mon Sep 17 00:00:00 2001
From: harishsurf <hgovinda@redhat.com>
Date: Fri, 28 Jun 2024 13:48:12 +0000
Subject: [PATCH 08/12] Remove duplicate code + add missing ldflags to Makefile

---
 Makefile                                                       | 2 +-
 .../roles/mirror_appliance/tasks/upgrade-config-vars.yaml      | 1 +
 cmd/install.go                                                 | 3 ---
 cmd/upgrade.go                                                 | 3 ---
 4 files changed, 2 insertions(+), 7 deletions(-)

diff --git a/Makefile b/Makefile
index 9786ca9..c376213 100644
--- a/Makefile
+++ b/Makefile
@@ -8,7 +8,7 @@ all:
 
 build-golang-executable:
 	$(CLIENT) run --rm -v ${PWD}:/usr/src:Z -w /usr/src docker.io/golang:1.16 go build -v \
-	-ldflags "-X github.com/quay/mirror-registry/cmd.eeImage=${EE_IMAGE} -X 'github.com/quay/mirror-registry/cmd.quayImage=${QUAY_IMAGE}' -X 'github.com/quay/mirror-registry/cmd.redisImage=${REDIS_IMAGE}'" \
+	-ldflags "-X 'github.com/quay/mirror-registry/cmd.releaseVersion=${RELEASE_VERSION}' -X 'github.com/quay/mirror-registry/cmd.eeImage=${EE_IMAGE}' -X 'github.com/quay/mirror-registry/cmd.pauseImage=${PAUSE_IMAGE}' -X 'github.com/quay/mirror-registry/cmd.quayImage=${QUAY_IMAGE}' -X 'github.com/quay/mirror-registry/cmd.redisImage=${REDIS_IMAGE}' -X 'github.com/quay/mirror-registry/cmd.sqliteImage=${DB_TO_SQLITE_IMAGE}'" \
 	-o mirror-registry;
 
 build-online-zip: 
diff --git a/ansible-runner/context/app/project/roles/mirror_appliance/tasks/upgrade-config-vars.yaml b/ansible-runner/context/app/project/roles/mirror_appliance/tasks/upgrade-config-vars.yaml
index bc4e4e4..9dfff3d 100644
--- a/ansible-runner/context/app/project/roles/mirror_appliance/tasks/upgrade-config-vars.yaml
+++ b/ansible-runner/context/app/project/roles/mirror_appliance/tasks/upgrade-config-vars.yaml
@@ -14,4 +14,5 @@
 - name: Set facts for the pre-existing secrets only if they are a string and not a jinja2 variable in the config.yaml.
   ansible.builtin.set_fact:
     REDIS_PASSWORD : "{{ quay_config_file['USER_EVENTS_REDIS']['password'] }}"
+    PGDB_PASSWORD : "{{ quay_config_file['DB_URI'].split('@')[0].split(':')[2] }}"
   when: quay_config_file['DATABASE_SECRET_KEY'] is string and quay_config_file['USER_EVENTS_REDIS']['password'] is string and quay_config_file['DB_URI'] is string
\ No newline at end of file
diff --git a/cmd/install.go b/cmd/install.go
index 7255da7..7870600 100644
--- a/cmd/install.go
+++ b/cmd/install.go
@@ -101,9 +101,6 @@ func init() {
 	installCmd.Flags().StringVarP(&quayStorage, "quayStorage", "", "quay-storage", "The folder where quay persistent storage data is saved. This defaults to a Podman named volume 'quay-storage'. Root is required to uninstall.")
 	installCmd.Flags().StringVarP(&sqliteStorage, "sqliteStorage", "", "sqlite-storage", "The folder where quay sqlite data is saved. This defaults to a Podman named volume 'sqlite-storage'. Root is required to uninstall.")
 	installCmd.Flags().StringVarP(&additionalArgs, "additionalArgs", "", "", "Additional arguments you would like to append to the ansible-playbook call. Used mostly for development.")
-
-	installCmd.Flags().StringVarP(&sqliteStorage, "sqliteStorage", "", "sqlite-storage", "The volume where sqlite persistent storage data is saved. This defaults to a Podman named volume 'sqlite-storage'. Root is required to uninstall.")
-
 }
 
 func install() {
diff --git a/cmd/upgrade.go b/cmd/upgrade.go
index 045e218..eb7294b 100644
--- a/cmd/upgrade.go
+++ b/cmd/upgrade.go
@@ -39,9 +39,6 @@ func init() {
 	upgradeCmd.Flags().StringVarP(&quayStorage, "quayStorage", "", "quay-storage", "The folder where quay persistent storage data is saved. This defaults to a Podman named volume 'quay-storage'. Root is required to uninstall.")
 	upgradeCmd.Flags().StringVarP(&sqliteStorage, "sqliteStorage", "", "sqlite-storage", "The folder where quay sqlite data is saved. This defaults to a Podman named volume 'sqlite-storage'. Root is required to uninstall.")
 	upgradeCmd.Flags().StringVarP(&additionalArgs, "additionalArgs", "", "", "Additional arguments you would like to append to the ansible-playbook call. Used mostly for development.")
-
-	upgradeCmd.Flags().StringVarP(&sqliteStorage, "sqliteStorage", "", "sqlite-storage", "The volume where sqlite persistent storage data is saved. This defaults to a Podman named volume 'sqlite-storage'. Root is required to uninstall.")
-
 }
 
 func upgrade() {

From fd850a5e013d1169a7fc1f8df5b54a47e1392902 Mon Sep 17 00:00:00 2001
From: harishsurf <hgovinda@redhat.com>
Date: Mon, 1 Jul 2024 20:30:20 +0000
Subject: [PATCH 09/12] Remove postgres service jinja template + fix DB_URI for
 migration

---
 .../roles/mirror_appliance/tasks/migrate.yaml | 14 ++++++++
 .../templates/postgres.service.j2             | 33 -------------------
 2 files changed, 14 insertions(+), 33 deletions(-)
 delete mode 100755 ansible-runner/context/app/project/roles/mirror_appliance/templates/postgres.service.j2

diff --git a/ansible-runner/context/app/project/roles/mirror_appliance/tasks/migrate.yaml b/ansible-runner/context/app/project/roles/mirror_appliance/tasks/migrate.yaml
index 48ae2a7..dd07e7d 100644
--- a/ansible-runner/context/app/project/roles/mirror_appliance/tasks/migrate.yaml
+++ b/ansible-runner/context/app/project/roles/mirror_appliance/tasks/migrate.yaml
@@ -58,6 +58,20 @@
     state: absent
   when: container_result.rc == 0
 
+- name: Update DB_URI in config.yaml
+  replace:
+    path: "{{ quay_root }}/quay-config/config.yaml"
+    regexp: '^DB_URI: postgresql://.*$'
+    replace: 'DB_URI: sqlite:////sqlite/quay_sqlite.db'
+  register: db_uri_update
+
+- name: Ensure DB_URI was updated successfully
+  assert:
+    that:
+      - db_uri_update.changed
+    fail_msg: "Failed to update DB_URI in quay's config"
+    success_msg: "DB_URI has been updated successfully"
+
 - name: Stop Quay service
   systemd:
     name: quay-app.service
diff --git a/ansible-runner/context/app/project/roles/mirror_appliance/templates/postgres.service.j2 b/ansible-runner/context/app/project/roles/mirror_appliance/templates/postgres.service.j2
deleted file mode 100755
index f001b22..0000000
--- a/ansible-runner/context/app/project/roles/mirror_appliance/templates/postgres.service.j2
+++ /dev/null
@@ -1,33 +0,0 @@
-[Unit]
-Description=PostgreSQL Podman Container for Quay
-Wants=network.target
-After=network-online.target quay-pod.service
-Requires=quay-pod.service
-
-[Service]
-Type=simple
-TimeoutStartSec=5m
-ExecStartPre=-/bin/rm -f %t/%n-pid %t/%n-cid
-ExecStart=/usr/bin/podman run \
-    --name quay-postgres \
-    -v {{ expanded_pg_storage }}:/var/lib/pgsql/data:Z \
-    --image-volume=ignore \
-    -e POSTGRESQL_USER=user \
-    -e POSTGRESQL_DATABASE=quay \
-    --pod=quay-pod \
-    --conmon-pidfile %t/%n-pid \
-    --cidfile %t/%n-cid \
-    --cgroups=no-conmon \
-    --secret=pgdb_pass,type=env,target=POSTGRESQL_PASSWORD \
-    --replace \
-    {{ postgres_image }}
-
-ExecStop=/usr/bin/podman stop --ignore --cidfile %t/%n-cid -t 10
-ExecStopPost=/usr/bin/podman rm --ignore -f --cidfile %t/%n-cid
-PIDFile=%t/%n-pid
-KillMode=none
-Restart=always
-RestartSec=30
-
-[Install]
-WantedBy=multi-user.target default.target

From 1d6dec2dcfb623da35923481f7c775b3c42d998e Mon Sep 17 00:00:00 2001
From: harishsurf <hgovinda@redhat.com>
Date: Tue, 9 Jul 2024 17:19:18 +0000
Subject: [PATCH 10/12] Add CI job for testing db upgrade from postgres to
 sqlite

---
 .github/workflows/jobs.yml                    | 75 +++++++++++++++++++
 .../tasks/upgrade-config-vars.yaml            | 13 +++-
 .../roles/mirror_appliance/tasks/upgrade.yaml |  6 ++
 3 files changed, 92 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/jobs.yml b/.github/workflows/jobs.yml
index f6d9299..1c73aa1 100644
--- a/.github/workflows/jobs.yml
+++ b/.github/workflows/jobs.yml
@@ -184,6 +184,42 @@ jobs:
       - name: Uninstall Registry
         run: ./mirror-registry uninstall -u jonathan -H quay --autoApprove -v -k /home/runner/.ssh/id_rsa
 
+      - name: Download old mirror-registry tarball that runs quay with postgres
+        run: wget -O mirror-registry-postgres.tar.gz https://github.com/quay/mirror-registry/releases/download/v1.3.10/mirror-registry-online.tar.gz 
+
+      - name: Create extraction directory for old mirror-registry binary
+        run: mkdir -p ./mirror-registry-postgres
+
+      - name: Extract tarball into the folder
+        run: tar -xzf mirror-registry-postgres.tar.gz -C ./mirror-registry-postgres
+
+      - name: Install postgres backed quay registry
+        run: ./mirror-registry-postgres/mirror-registry install -u jonathan -r /home/jonathan/quay-install -H quay -v --initPassword password -k /home/runner/.ssh/id_rsa
+
+      - name: Podman login to quay registry
+        run: podman login -u init -p password quay:8443 --tls-verify=false
+
+      - name: Pull busybox image from Docker Hub
+        run: podman pull docker.io/library/busybox
+
+      - name: Tag busybox image for Quay
+        run: podman tag docker.io/library/busybox quay:8443/init/busybox:latest
+
+      - name: Push busybox image to Quay
+        run: podman push quay:8443/init/busybox:latest --tls-verify=false
+
+      - name: Use latest binary to test upgrade cmd to ensure db migration from old postgres to sqlite
+        run: ./mirror-registry upgrade -u jonathan -r /home/jonathan/quay-install -H quay -v --initPassword password -k /home/runner/.ssh/id_rsa
+
+      - name: Pull already pushed busybox image from quay registry
+        run: podman pull quay:8443/init/busybox:latest
+
+      - name: Verify busybox image is pulled successfully
+        run: podman images | grep -q quay:8443/init/busybox:latest
+
+      - name: Uninstall Registry
+        run: ./mirror-registry uninstall -u jonathan -H quay --autoApprove -v -k /home/runner/.ssh/id_rsa
+
       - name: Terraform Destroy
         run: terraform destroy --auto-approve
         shell: bash
@@ -292,6 +328,45 @@ jobs:
       - name: Uninstall Quay
         run: ssh jonathan@quay './mirror-registry uninstall --autoApprove -v'
 
+      - name: Download old mirror-registry tarball that runs quay with postgres
+        run: wget -O mirror-registry-postgres.tar.gz https://github.com/quay/mirror-registry/releases/download/v1.3.10/mirror-registry-online.tar.gz
+
+      - name: SCP old tarball to VM
+        run: scp mirror-registry-postgres.tar.gz jonathan@quay:~/mirror-registry-postgres.tar.gz
+
+      - name: Create extraction directory for old mirror-registry binary
+        run: ssh jonathan@quay 'mkdir -p ./mirror-registry-postgres'
+
+      - name: Extract tarball into the folder
+        run: ssh jonathan@quay 'tar -xzf mirror-registry-postgres.tar.gz -C ./mirror-registry-postgres'
+
+      - name: Install postgres backed quay registry
+        run: ssh jonathan@quay './mirror-registry-postgres/mirror-registry install -u jonathan -r /home/jonathan/quay-install -H quay -v --initPassword password -k /home/runner/.ssh/id_rsa'
+
+      - name: Podman login to quay registry
+        run: ssh jonathan@quay 'podman login -u init -p password quay:8443 --tls-verify=false'
+
+      - name: Pull busybox image from Docker Hub
+        run: ssh jonathan@quay 'podman pull docker.io/library/busybox'
+
+      - name: Tag busybox image for Quay
+        run: ssh jonathan@quay 'podman tag docker.io/library/busybox quay:8443/init/busybox:latest'
+
+      - name: Push busybox image to Quay
+        run: ssh jonathan@quay 'podman push quay:8443/init/busybox:latest --tls-verify=false'
+
+      - name: Use latest binary to test upgrade cmd to ensure db migration from old postgres to sqlite
+        run: ssh jonathan@quay './mirror-registry upgrade -u jonathan -r /home/jonathan/quay-install -H quay -v --initPassword password -k /home/runner/.ssh/id_rsa'
+
+      - name: Pull already pushed busybox image from quay registry
+        run: ssh jonathan@quay 'podman pull quay:8443/init/busybox:latest'
+
+      - name: Verify busybox image was pulled successfully
+        run: ssh jonathan@quay 'podman images | grep -q quay:8443/init/busybox:latest'
+
+      - name: Uninstall Quay
+        run: ssh jonathan@quay './mirror-registry uninstall --autoApprove -v'
+
       - name: Terraform Destroy
         run: terraform destroy --auto-approve
         shell: bash
diff --git a/ansible-runner/context/app/project/roles/mirror_appliance/tasks/upgrade-config-vars.yaml b/ansible-runner/context/app/project/roles/mirror_appliance/tasks/upgrade-config-vars.yaml
index 9dfff3d..596c58c 100644
--- a/ansible-runner/context/app/project/roles/mirror_appliance/tasks/upgrade-config-vars.yaml
+++ b/ansible-runner/context/app/project/roles/mirror_appliance/tasks/upgrade-config-vars.yaml
@@ -11,8 +11,17 @@
   ansible.builtin.set_fact:
     quay_config_file: "{{ remote_yaml_file['content'] | b64decode | from_yaml }}"
 
-- name: Set facts for the pre-existing secrets only if they are a string and not a jinja2 variable in the config.yaml.
+- name: Set facts for the existing redis secrets only if they are a string and not a jinja2 variable in the config.yaml.
   ansible.builtin.set_fact:
     REDIS_PASSWORD : "{{ quay_config_file['USER_EVENTS_REDIS']['password'] }}"
+  when: quay_config_file['DATABASE_SECRET_KEY'] is string and quay_config_file['USER_EVENTS_REDIS']['password'] is string
+
+- name: Check if quay-postgres container is running
+  command: podman ps -q -f name=quay-postgres
+  register: postgres_container_status
+  changed_when: false
+
+- name: Set facts for existing postgres secrets only if they are a string and not a jinja2 variable in the config.yaml.
+  ansible.builtin.set_fact:
     PGDB_PASSWORD : "{{ quay_config_file['DB_URI'].split('@')[0].split(':')[2] }}"
-  when: quay_config_file['DATABASE_SECRET_KEY'] is string and quay_config_file['USER_EVENTS_REDIS']['password'] is string and quay_config_file['DB_URI'] is string
\ No newline at end of file
+  when: postgres_container_status.stdout != "" and quay_config_file['DATABASE_SECRET_KEY'] is string and quay_config_file['DB_URI'] is string
\ No newline at end of file
diff --git a/ansible-runner/context/app/project/roles/mirror_appliance/tasks/upgrade.yaml b/ansible-runner/context/app/project/roles/mirror_appliance/tasks/upgrade.yaml
index 2e4c742..cd65231 100755
--- a/ansible-runner/context/app/project/roles/mirror_appliance/tasks/upgrade.yaml
+++ b/ansible-runner/context/app/project/roles/mirror_appliance/tasks/upgrade.yaml
@@ -10,8 +10,14 @@
 - name: Autodetect existing Secrets in config.yaml
   include_tasks: upgrade-config-vars.yaml
 
+- name: Check if quay-postgres container is running
+  command: podman ps -q -f name=quay-postgres
+  register: postgres_container_status
+  changed_when: false
+
 - name: Migrate postgres db to sqlite for Quay
   include_tasks: migrate.yaml
+  when: postgres_container_status.stdout != ""
 
 - name: Autodetect Image Archive
   include_tasks: autodetect-image-archive.yaml

From 4597a3cea22ef15bb278538f684bebe2e315ce54 Mon Sep 17 00:00:00 2001
From: harishsurf <hgovinda@redhat.com>
Date: Thu, 11 Jul 2024 17:17:27 +0000
Subject: [PATCH 11/12] Remove CI change + move postgres clean up to later
 after quay is up

---
 .github/workflows/jobs.yml                    | 75 -------------------
 .../tasks/cleanup-postgres.yaml               | 31 ++++++++
 .../roles/mirror_appliance/tasks/migrate.yaml | 35 ---------
 .../roles/mirror_appliance/tasks/upgrade.yaml |  4 +
 4 files changed, 35 insertions(+), 110 deletions(-)
 create mode 100644 ansible-runner/context/app/project/roles/mirror_appliance/tasks/cleanup-postgres.yaml

diff --git a/.github/workflows/jobs.yml b/.github/workflows/jobs.yml
index 1c73aa1..f6d9299 100644
--- a/.github/workflows/jobs.yml
+++ b/.github/workflows/jobs.yml
@@ -184,42 +184,6 @@ jobs:
       - name: Uninstall Registry
         run: ./mirror-registry uninstall -u jonathan -H quay --autoApprove -v -k /home/runner/.ssh/id_rsa
 
-      - name: Download old mirror-registry tarball that runs quay with postgres
-        run: wget -O mirror-registry-postgres.tar.gz https://github.com/quay/mirror-registry/releases/download/v1.3.10/mirror-registry-online.tar.gz 
-
-      - name: Create extraction directory for old mirror-registry binary
-        run: mkdir -p ./mirror-registry-postgres
-
-      - name: Extract tarball into the folder
-        run: tar -xzf mirror-registry-postgres.tar.gz -C ./mirror-registry-postgres
-
-      - name: Install postgres backed quay registry
-        run: ./mirror-registry-postgres/mirror-registry install -u jonathan -r /home/jonathan/quay-install -H quay -v --initPassword password -k /home/runner/.ssh/id_rsa
-
-      - name: Podman login to quay registry
-        run: podman login -u init -p password quay:8443 --tls-verify=false
-
-      - name: Pull busybox image from Docker Hub
-        run: podman pull docker.io/library/busybox
-
-      - name: Tag busybox image for Quay
-        run: podman tag docker.io/library/busybox quay:8443/init/busybox:latest
-
-      - name: Push busybox image to Quay
-        run: podman push quay:8443/init/busybox:latest --tls-verify=false
-
-      - name: Use latest binary to test upgrade cmd to ensure db migration from old postgres to sqlite
-        run: ./mirror-registry upgrade -u jonathan -r /home/jonathan/quay-install -H quay -v --initPassword password -k /home/runner/.ssh/id_rsa
-
-      - name: Pull already pushed busybox image from quay registry
-        run: podman pull quay:8443/init/busybox:latest
-
-      - name: Verify busybox image is pulled successfully
-        run: podman images | grep -q quay:8443/init/busybox:latest
-
-      - name: Uninstall Registry
-        run: ./mirror-registry uninstall -u jonathan -H quay --autoApprove -v -k /home/runner/.ssh/id_rsa
-
       - name: Terraform Destroy
         run: terraform destroy --auto-approve
         shell: bash
@@ -328,45 +292,6 @@ jobs:
       - name: Uninstall Quay
         run: ssh jonathan@quay './mirror-registry uninstall --autoApprove -v'
 
-      - name: Download old mirror-registry tarball that runs quay with postgres
-        run: wget -O mirror-registry-postgres.tar.gz https://github.com/quay/mirror-registry/releases/download/v1.3.10/mirror-registry-online.tar.gz
-
-      - name: SCP old tarball to VM
-        run: scp mirror-registry-postgres.tar.gz jonathan@quay:~/mirror-registry-postgres.tar.gz
-
-      - name: Create extraction directory for old mirror-registry binary
-        run: ssh jonathan@quay 'mkdir -p ./mirror-registry-postgres'
-
-      - name: Extract tarball into the folder
-        run: ssh jonathan@quay 'tar -xzf mirror-registry-postgres.tar.gz -C ./mirror-registry-postgres'
-
-      - name: Install postgres backed quay registry
-        run: ssh jonathan@quay './mirror-registry-postgres/mirror-registry install -u jonathan -r /home/jonathan/quay-install -H quay -v --initPassword password -k /home/runner/.ssh/id_rsa'
-
-      - name: Podman login to quay registry
-        run: ssh jonathan@quay 'podman login -u init -p password quay:8443 --tls-verify=false'
-
-      - name: Pull busybox image from Docker Hub
-        run: ssh jonathan@quay 'podman pull docker.io/library/busybox'
-
-      - name: Tag busybox image for Quay
-        run: ssh jonathan@quay 'podman tag docker.io/library/busybox quay:8443/init/busybox:latest'
-
-      - name: Push busybox image to Quay
-        run: ssh jonathan@quay 'podman push quay:8443/init/busybox:latest --tls-verify=false'
-
-      - name: Use latest binary to test upgrade cmd to ensure db migration from old postgres to sqlite
-        run: ssh jonathan@quay './mirror-registry upgrade -u jonathan -r /home/jonathan/quay-install -H quay -v --initPassword password -k /home/runner/.ssh/id_rsa'
-
-      - name: Pull already pushed busybox image from quay registry
-        run: ssh jonathan@quay 'podman pull quay:8443/init/busybox:latest'
-
-      - name: Verify busybox image was pulled successfully
-        run: ssh jonathan@quay 'podman images | grep -q quay:8443/init/busybox:latest'
-
-      - name: Uninstall Quay
-        run: ssh jonathan@quay './mirror-registry uninstall --autoApprove -v'
-
       - name: Terraform Destroy
         run: terraform destroy --auto-approve
         shell: bash
diff --git a/ansible-runner/context/app/project/roles/mirror_appliance/tasks/cleanup-postgres.yaml b/ansible-runner/context/app/project/roles/mirror_appliance/tasks/cleanup-postgres.yaml
new file mode 100644
index 0000000..39cd8ca
--- /dev/null
+++ b/ansible-runner/context/app/project/roles/mirror_appliance/tasks/cleanup-postgres.yaml
@@ -0,0 +1,31 @@
+- name: Stop Postgres service
+  systemd:
+    name: quay-postgres.service
+    enabled: no
+    daemon_reload: yes
+    state: stopped
+    force: yes
+    scope: "{{ systemd_scope }}"
+
+- name: Delete Postgres Storage named volume
+  containers.podman.podman_volume:
+      state: absent
+      name: pg-storage
+  when: auto_approve|bool == true and pg_storage == "pg-storage"
+
+- name: Delete Postgres Password Secret
+  containers.podman.podman_secret:
+    state: absent
+    name: pgdb_pass
+
+- name: Delete necessary directory for Postgres persistent data
+  ansible.builtin.file:
+    path: "{{ pg_storage }}"
+    state: absent
+  become: yes
+  when: auto_approve|bool == true and pg_storage.startswith('/')
+
+- name: Cleanup quay-postgres systemd unit file
+  file:
+    state: absent
+    path: "{{ systemd_unit_dir }}/quay-postgres.service"
diff --git a/ansible-runner/context/app/project/roles/mirror_appliance/tasks/migrate.yaml b/ansible-runner/context/app/project/roles/mirror_appliance/tasks/migrate.yaml
index dd07e7d..8233931 100644
--- a/ansible-runner/context/app/project/roles/mirror_appliance/tasks/migrate.yaml
+++ b/ansible-runner/context/app/project/roles/mirror_appliance/tasks/migrate.yaml
@@ -81,38 +81,3 @@
     force: yes
     scope: "{{ systemd_scope }}"
   when: container_result.rc == 0
-
-- name: Stop Postgres service
-  systemd:
-    name: quay-postgres.service
-    enabled: no
-    daemon_reload: yes
-    state: stopped
-    force: yes
-    scope: "{{ systemd_scope }}"
-  when: container_result.rc == 0
-
-- name: Delete Postgres Storage named volume
-  containers.podman.podman_volume:
-      state: absent
-      name: pg-storage
-  when: auto_approve|bool == true and pg_storage == "pg-storage" and container_result.rc == 0
-
-- name: Delete Postgres Password Secret
-  containers.podman.podman_secret:
-    state: absent
-    name: pgdb_pass
-  when: container_result.rc == 0
-
-- name: Delete necessary directory for Postgres persistent data
-  ansible.builtin.file:
-    path: "{{ pg_storage }}"
-    state: absent
-  become: yes
-  when: auto_approve|bool == true and pg_storage.startswith('/') and container_result.rc == 0
-
-- name: Cleanup quay-postgres systemd unit file
-  file:
-    state: absent
-    path: "{{ systemd_unit_dir }}/quay-postgres.service"
-  when: container_result.rc == 0
\ No newline at end of file
diff --git a/ansible-runner/context/app/project/roles/mirror_appliance/tasks/upgrade.yaml b/ansible-runner/context/app/project/roles/mirror_appliance/tasks/upgrade.yaml
index cd65231..489e044 100755
--- a/ansible-runner/context/app/project/roles/mirror_appliance/tasks/upgrade.yaml
+++ b/ansible-runner/context/app/project/roles/mirror_appliance/tasks/upgrade.yaml
@@ -33,3 +33,7 @@
 
 - name: Wait for Quay
   include_tasks: wait-for-quay.yaml
+
+- name: Clean up old postgres service
+  include_tasks: cleanup-postgres.yaml
+  when: postgres_container_status.stdout != ""

From 6116d2a2e79643e738ff10ea680bb7461930cc10 Mon Sep 17 00:00:00 2001
From: harishsurf <hgovinda@redhat.com>
Date: Mon, 15 Jul 2024 16:11:38 +0000
Subject: [PATCH 12/12] Move upgrading quay image before db migration

quay needs to be on latest version with all alembic migration
changes before db is migrated from postgres to sqlite, else
alembic migration fails
---
 .../roles/mirror_appliance/tasks/migrate.yaml |  8 +++++++
 .../roles/mirror_appliance/tasks/upgrade.yaml | 21 +++++++++++--------
 2 files changed, 20 insertions(+), 9 deletions(-)

diff --git a/ansible-runner/context/app/project/roles/mirror_appliance/tasks/migrate.yaml b/ansible-runner/context/app/project/roles/mirror_appliance/tasks/migrate.yaml
index 8233931..3638dc5 100644
--- a/ansible-runner/context/app/project/roles/mirror_appliance/tasks/migrate.yaml
+++ b/ansible-runner/context/app/project/roles/mirror_appliance/tasks/migrate.yaml
@@ -81,3 +81,11 @@
     force: yes
     scope: "{{ systemd_scope }}"
   when: container_result.rc == 0
+
+- name: Start Quay service
+  systemd:
+    name: quay-app.service
+    enabled: yes
+    daemon_reload: yes
+    state: restarted
+    scope: "{{ systemd_scope }}"
diff --git a/ansible-runner/context/app/project/roles/mirror_appliance/tasks/upgrade.yaml b/ansible-runner/context/app/project/roles/mirror_appliance/tasks/upgrade.yaml
index 489e044..1f40c0e 100755
--- a/ansible-runner/context/app/project/roles/mirror_appliance/tasks/upgrade.yaml
+++ b/ansible-runner/context/app/project/roles/mirror_appliance/tasks/upgrade.yaml
@@ -10,15 +10,6 @@
 - name: Autodetect existing Secrets in config.yaml
   include_tasks: upgrade-config-vars.yaml
 
-- name: Check if quay-postgres container is running
-  command: podman ps -q -f name=quay-postgres
-  register: postgres_container_status
-  changed_when: false
-
-- name: Migrate postgres db to sqlite for Quay
-  include_tasks: migrate.yaml
-  when: postgres_container_status.stdout != ""
-
 - name: Autodetect Image Archive
   include_tasks: autodetect-image-archive.yaml
 
@@ -34,6 +25,18 @@
 - name: Wait for Quay
   include_tasks: wait-for-quay.yaml
 
+- name: Check if quay-postgres container is running
+  command: podman ps -q -f name=quay-postgres
+  register: postgres_container_status
+  changed_when: false
+
+- name: Migrate postgres db to sqlite for Quay
+  include_tasks: migrate.yaml
+  when: postgres_container_status.stdout != ""
+
+- name: Wait for Quay
+  include_tasks: wait-for-quay.yaml
+
 - name: Clean up old postgres service
   include_tasks: cleanup-postgres.yaml
   when: postgres_container_status.stdout != ""