db: Modify upgrade cmd to migrate database from Postgres to Sqlite

.env

@@ -4,3 +4,4 @@ EE_BUILDER_IMAGE=registry.redhat.io/ansible-automation-platform-22/ansible-build
 QUAY_IMAGE=quay.io/projectquay/quay:3.12.0-nightly.20240626
 REDIS_IMAGE=registry.redhat.io/rhel8/redis-6:1-92.1669834635
 PAUSE_IMAGE=registry.access.redhat.com/ubi8/pause:8.7-6
+DB_TO_SQLITE_IMAGE=quay.io/quay/db-to-sqlite:latest

.github/workflows/jobs.yml

@@ -184,6 +184,42 @@ jobs:
       - name: Uninstall Registry
         run: ./mirror-registry uninstall -u jonathan -H quay --autoApprove -v -k /home/runner/.ssh/id_rsa
 
+      - name: Download old mirror-registry tarball that runs quay with postgres
+        run: wget -O mirror-registry-postgres.tar.gz https://github.com/quay/mirror-registry/releases/download/v1.3.10/mirror-registry-online.tar.gz 
+
+      - name: Create extraction directory for old mirror-registry binary
+        run: mkdir -p ./mirror-registry-postgres
+
+      - name: Extract tarball into the folder
+        run: tar -xzf mirror-registry-postgres.tar.gz -C ./mirror-registry-postgres
+
+      - name: Install postgres backed quay registry
+        run: ./mirror-registry-postgres/mirror-registry install -u jonathan -r /home/jonathan/quay-install -H quay -v --initPassword password -k /home/runner/.ssh/id_rsa
+
+      - name: Podman login to quay registry
+        run: podman login -u init -p password quay:8443 --tls-verify=false
+
+      - name: Pull busybox image from Docker Hub
+        run: podman pull docker.io/library/busybox
+
+      - name: Tag busybox image for Quay
+        run: podman tag docker.io/library/busybox quay:8443/init/busybox:latest
+
+      - name: Push busybox image to Quay
+        run: podman push quay:8443/init/busybox:latest --tls-verify=false
+
+      - name: Use latest binary to test upgrade cmd to ensure db migration from old postgres to sqlite
+        run: ./mirror-registry upgrade -u jonathan -r /home/jonathan/quay-install -H quay -v --initPassword password -k /home/runner/.ssh/id_rsa
+
+      - name: Pull already pushed busybox image from quay registry
+        run: podman pull quay:8443/init/busybox:latest
+
+      - name: Verify busybox image is pulled successfully
+        run: podman images | grep -q quay:8443/init/busybox:latest
+
+      - name: Uninstall Registry
+        run: ./mirror-registry uninstall -u jonathan -H quay --autoApprove -v -k /home/runner/.ssh/id_rsa
+
       - name: Terraform Destroy
         run: terraform destroy --auto-approve
         shell: bash
@@ -292,6 +328,45 @@ jobs:
       - name: Uninstall Quay
         run: ssh jonathan@quay './mirror-registry uninstall --autoApprove -v'
 
+      - name: Download old mirror-registry tarball that runs quay with postgres
+        run: wget -O mirror-registry-postgres.tar.gz https://github.com/quay/mirror-registry/releases/download/v1.3.10/mirror-registry-online.tar.gz
+
+      - name: SCP old tarball to VM
+        run: scp mirror-registry-postgres.tar.gz jonathan@quay:~/mirror-registry-postgres.tar.gz
+
+      - name: Create extraction directory for old mirror-registry binary
+        run: ssh jonathan@quay 'mkdir -p ./mirror-registry-postgres'
+
+      - name: Extract tarball into the folder
+        run: ssh jonathan@quay 'tar -xzf mirror-registry-postgres.tar.gz -C ./mirror-registry-postgres'
+
+      - name: Install postgres backed quay registry
+        run: ssh jonathan@quay './mirror-registry-postgres/mirror-registry install -u jonathan -r /home/jonathan/quay-install -H quay -v --initPassword password -k /home/runner/.ssh/id_rsa'
+
+      - name: Podman login to quay registry
+        run: ssh jonathan@quay 'podman login -u init -p password quay:8443 --tls-verify=false'
+
+      - name: Pull busybox image from Docker Hub
+        run: ssh jonathan@quay 'podman pull docker.io/library/busybox'
+
+      - name: Tag busybox image for Quay
+        run: ssh jonathan@quay 'podman tag docker.io/library/busybox quay:8443/init/busybox:latest'
+
+      - name: Push busybox image to Quay
+        run: ssh jonathan@quay 'podman push quay:8443/init/busybox:latest --tls-verify=false'
+
+      - name: Use latest binary to test upgrade cmd to ensure db migration from old postgres to sqlite
+        run: ssh jonathan@quay './mirror-registry upgrade -u jonathan -r /home/jonathan/quay-install -H quay -v --initPassword password -k /home/runner/.ssh/id_rsa'
+
+      - name: Pull already pushed busybox image from quay registry
+        run: ssh jonathan@quay 'podman pull quay:8443/init/busybox:latest'
+
+      - name: Verify busybox image was pulled successfully
+        run: ssh jonathan@quay 'podman images | grep -q quay:8443/init/busybox:latest'
+
+      - name: Uninstall Quay
+        run: ssh jonathan@quay './mirror-registry uninstall --autoApprove -v'
+
       - name: Terraform Destroy
         run: terraform destroy --auto-approve
         shell: bash

Dockerfile

@@ -5,6 +5,7 @@ ARG EE_BASE_IMAGE=${EE_BASE_IMAGE}
 ARG EE_BUILDER_IMAGE=${EE_BUILDER_IMAGE}
 ARG REDIS_IMAGE=${REDIS_IMAGE}
 ARG PAUSE_IMAGE=${PAUSE_IMAGE}
+ARG DB_TO_SQLITE_IMAGE=${DB_TO_SQLITE_IMAGE}
 
 # Create Go CLI
 FROM registry.access.redhat.com/ubi8:latest AS cli
@@ -15,6 +16,7 @@ ARG QUAY_IMAGE=${QUAY_IMAGE}
 ARG EE_IMAGE=${EE_IMAGE}
 ARG REDIS_IMAGE=${REDIS_IMAGE}
 ARG PAUSE_IMAGE=${PAUSE_IMAGE}
+ARG DB_TO_SQLITE_IMAGE=${DB_TO_SQLITE_IMAGE}
 
 ENV GOROOT=/usr/local/go
 ENV PATH=$GOPATH/bin:$GOROOT/bin:$PATH 
@@ -33,9 +35,10 @@ ENV EE_IMAGE=${EE_IMAGE}
 ENV QUAY_IMAGE=${QUAY_IMAGE}
 ENV REDIS_IMAGE=${REDIS_IMAGE}
 ENV PAUSE_IMAGE=${PAUSE_IMAGE}
+ENV DB_TO_SQLITE_IMAGE=${DB_TO_SQLITE_IMAGE}
 
 RUN go build -v \
-	-ldflags "-X github.com/quay/mirror-registry/cmd.releaseVersion=${RELEASE_VERSION} -X github.com/quay/mirror-registry/cmd.eeImage=${EE_IMAGE} -X github.com/quay/mirror-registry/cmd.pauseImage=${PAUSE_IMAGE} -X github.com/quay/mirror-registry/cmd.quayImage=${QUAY_IMAGE} -X github.com/quay/mirror-registry/cmd.redisImage=${REDIS_IMAGE}" \
+	-ldflags "-X github.com/quay/mirror-registry/cmd.releaseVersion=${RELEASE_VERSION} -X github.com/quay/mirror-registry/cmd.eeImage=${EE_IMAGE} -X github.com/quay/mirror-registry/cmd.pauseImage=${PAUSE_IMAGE} -X github.com/quay/mirror-registry/cmd.quayImage=${QUAY_IMAGE} -X github.com/quay/mirror-registry/cmd.redisImage=${REDIS_IMAGE} -X github.com/quay/mirror-registry/cmd.sqliteImage=${DB_TO_SQLITE_IMAGE}" \
 	-o mirror-registry
 
 # Create Ansible Execution Environment
@@ -70,6 +73,18 @@ FROM $QUAY_IMAGE as quay
 FROM $REDIS_IMAGE as redis
 FROM $PAUSE_IMAGE as pause
 
+# Install db sqlite migration cli
+FROM registry.access.redhat.com/ubi8-minimal AS db-cli
+
+# Install Python 3 and pip
+RUN microdnf update -y && \
+    microdnf install libpq-devel python3 python3-pip -y && \
+    microdnf install gcc-c++ python3-devel -y && \
+    microdnf clean all
+
+COPY requirements.txt .
+RUN pip3 install --no-cache-dir -r requirements.txt
+
 # Create mirror registry archive
 FROM registry.access.redhat.com/ubi8:latest AS build
 
@@ -88,11 +103,14 @@ RUN tar -cvf quay.tar -C /quay .
 
 COPY --from=cli /cli/mirror-registry .
 
+COPY --from=db-cli / /db-cli
+RUN tar -cvf db-cli.tar -C /db-cli .
+
 # Bundle quay, redis and pause into a single archive
 RUN tar -cvf image-archive.tar quay.tar redis.tar pause.tar
 
 # Bundle mirror registry archive
-RUN tar -czvf mirror-registry.tar.gz image-archive.tar execution-environment.tar mirror-registry
+RUN tar -czvf mirror-registry.tar.gz image-archive.tar execution-environment.tar mirror-registry db-cli.tar
 
 # Extract bundle to final release image
 FROM registry.access.redhat.com/ubi8:latest AS release

Dockerfile.online

@@ -11,6 +11,7 @@ ARG QUAY_IMAGE=${QUAY_IMAGE}
 ARG EE_IMAGE=${EE_IMAGE}
 ARG REDIS_IMAGE=${REDIS_IMAGE}
 ARG PAUSE_IMAGE=${PAUSE_IMAGE}
+ARG DB_TO_SQLITE_IMAGE=${DB_TO_SQLITE_IMAGE}
 
 ENV GOROOT=/usr/local/go
 ENV PATH=$GOPATH/bin:$GOROOT/bin:$PATH 
@@ -29,9 +30,10 @@ ENV EE_IMAGE=${EE_IMAGE}
 ENV QUAY_IMAGE=${QUAY_IMAGE}
 ENV REDIS_IMAGE=${REDIS_IMAGE}
 ENV PAUSE_IMAGE=${PAUSE_IMAGE}
+ENV DB_TO_SQLITE_IMAGE=${DB_TO_SQLITE_IMAGE}
 
 RUN go build -v \
-    -ldflags "-X github.com/quay/mirror-registry/cmd.releaseVersion=${RELEASE_VERSION} -X github.com/quay/mirror-registry/cmd.eeImage=${EE_IMAGE} -X github.com/quay/mirror-registry/cmd.pauseImage=${PAUSE_IMAGE} -X github.com/quay/mirror-registry/cmd.quayImage=${QUAY_IMAGE} -X github.com/quay/mirror-registry/cmd.redisImage=${REDIS_IMAGE}" \
+    -ldflags "-X github.com/quay/mirror-registry/cmd.releaseVersion=${RELEASE_VERSION} -X github.com/quay/mirror-registry/cmd.eeImage=${EE_IMAGE} -X github.com/quay/mirror-registry/cmd.pauseImage=${PAUSE_IMAGE} -X github.com/quay/mirror-registry/cmd.quayImage=${QUAY_IMAGE} -X github.com/quay/mirror-registry/cmd.redisImage=${REDIS_IMAGE} -X github.com/quay/mirror-registry/cmd.sqliteImage=${DB_TO_SQLITE_IMAGE}" \
     -o mirror-registry
 
 # Create Ansible Execution Environment
@@ -61,6 +63,18 @@ COPY --from=builder /output/ /output/
 RUN /output/install-from-bindep && rm -rf /output/wheels
 COPY ansible-runner/context/app /runner
 
+# Install db sqlite migration cli
+FROM registry.access.redhat.com/ubi8-minimal AS db-cli
+
+# Install Python 3 and pip
+RUN microdnf update -y && \
+    microdnf install libpq-devel python3 python3-pip -y && \
+    microdnf install gcc-c++ python3-devel -y && \
+    microdnf clean all
+
+COPY requirements.txt .
+RUN pip3 install --no-cache-dir -r requirements.txt
+
 # Create mirror registry archive
 FROM registry.redhat.io/ubi8:latest AS build
 
@@ -70,8 +84,11 @@ RUN tar -cvf execution-environment.tar -C /ansible .
 
 COPY --from=cli /cli/mirror-registry .
 
+COPY --from=db-cli / /db-cli
+RUN tar -cvf db-cli.tar -C /db-cli .
+
 # Bundle mirror registry archive
-RUN tar -czvf mirror-registry.tar.gz execution-environment.tar mirror-registry
+RUN tar -czvf mirror-registry.tar.gz execution-environment.tar mirror-registry db-cli.tar
 
 # Extract bundle to final release image
 FROM registry.redhat.io/ubi8:latest AS release

Makefile

@@ -8,7 +8,7 @@ all:
 
 build-golang-executable:
 	$(CLIENT) run --rm -v ${PWD}:/usr/src:Z -w /usr/src docker.io/golang:1.16 go build -v \
-	-ldflags "-X github.com/quay/mirror-registry/cmd.eeImage=${EE_IMAGE} -X 'github.com/quay/mirror-registry/cmd.quayImage=${QUAY_IMAGE}' -X 'github.com/quay/mirror-registry/cmd.redisImage=${REDIS_IMAGE}'" \
+	-ldflags "-X 'github.com/quay/mirror-registry/cmd.releaseVersion=${RELEASE_VERSION}' -X 'github.com/quay/mirror-registry/cmd.eeImage=${EE_IMAGE}' -X 'github.com/quay/mirror-registry/cmd.pauseImage=${PAUSE_IMAGE}' -X 'github.com/quay/mirror-registry/cmd.quayImage=${QUAY_IMAGE}' -X 'github.com/quay/mirror-registry/cmd.redisImage=${REDIS_IMAGE}' -X 'github.com/quay/mirror-registry/cmd.sqliteImage=${DB_TO_SQLITE_IMAGE}'" \
 	-o mirror-registry;
 
 build-online-zip: 
@@ -21,7 +21,8 @@ build-online-zip:
 		--build-arg EE_BUILDER_IMAGE=${EE_BUILDER_IMAGE} \
 		--build-arg REDIS_IMAGE=${REDIS_IMAGE} \
 		--build-arg PAUSE_IMAGE=${PAUSE_IMAGE} \
-		--file Dockerfile.online . 
+		--build-arg DB_TO_SQLITE_IMAGE=${DB_TO_SQLITE_IMAGE} \
+		--file Dockerfile.online .
 	$(CLIENT) run --name mirror-registry-online-${RELEASE_VERSION} mirror-registry-online:${RELEASE_VERSION}
 	$(CLIENT) cp mirror-registry-online-${RELEASE_VERSION}:/mirror-registry.tar.gz .
 	$(CLIENT) rm mirror-registry-online-${RELEASE_VERSION}
@@ -36,6 +37,7 @@ build-offline-zip:
 		--build-arg EE_BUILDER_IMAGE=${EE_BUILDER_IMAGE} \
 		--build-arg REDIS_IMAGE=${REDIS_IMAGE} \
 		--build-arg PAUSE_IMAGE=${PAUSE_IMAGE} \
+		--build-arg DB_TO_SQLITE_IMAGE=${DB_TO_SQLITE_IMAGE} \
 		--file Dockerfile .
 	$(CLIENT) run --name mirror-registry-offline-${RELEASE_VERSION} mirror-registry-offline:${RELEASE_VERSION}
 	$(CLIENT) cp mirror-registry-offline-${RELEASE_VERSION}:/mirror-registry.tar.gz .

ansible-runner/context/app/project/roles/mirror_appliance/tasks/install-redis-service.yaml

@@ -20,7 +20,7 @@
     state: present
     name: redis_pass
     data: "{{ redis_password }}"
-    skip_existing: false
+    skip_existing: true
 
 - name: Start Redis service
   systemd:

ansible-runner/context/app/project/roles/mirror_appliance/tasks/migrate.yaml

@@ -0,0 +1,118 @@
+- name: Check if sqlite migration tool image is loaded
+  command: podman inspect --type=image {{ sqlite_image }}
+  register: db_sqlite
+  ignore_errors: yes
+
+- name: Pull Sqlite image
+  containers.podman.podman_image:
+    name: "{{ sqlite_image }}"
+  when: db_sqlite.rc != 0
+  retries: 5
+  delay: 5
+
+- name: Create necessary directory for storing quay postgres db snapshot
+  ansible.builtin.file:
+    path: "{{ quay_root }}/quay-postgres-backup"
+    mode: 0750
+    state: directory
+    recurse: yes
+
+- name: Create Sqlite storage named volume
+  containers.podman.podman_volume:
+      state: present
+      name: "{{ sqlite_storage }}"
+  when: "not sqlite_storage.startswith('/')"
+
+- name: Run db-to-sqlite command via podman container
+  command: >
+    podman run --name quay-db-migration --pod quay-pod
+    -v {{ sqlite_storage }}:/data:Z
+    --umask=013
+    {{ sqlite_image }}
+    postgresql://user:{{ PGDB_PASSWORD }}@localhost/quay
+    quay_sqlite.db --all -p
+  register: container_result
+  ignore_errors: yes
+
+- name: Display migration error when sqlite migration fails
+  debug:
+    msg: "{{ container_result.stderr }}"
+  when: container_result.rc != 0
+
+- name: Back up postgres data from mounted volume to host machine
+  command: podman cp quay-db-migration:/data {{ quay_root }}/quay-postgres-backup/
+  when: container_result.rc == 0
+
+- name: Check if sqlite file exists
+  stat:
+    path: "{{ quay_root }}/quay-postgres-backup/quay_sqlite.db"
+  register: file_check
+
+- fail:
+    msg: "Sqlite db file '{{ quay_root }}/quay-postgres-backup/quay_sqlite.db' does not exist."
+  when: file_check.stat.exists == false
+
+- name: Delete temporary migration container
+  containers.podman.podman_container:
+    name: quay-db-migration
+    state: absent
+  when: container_result.rc == 0
+
+- name: Update DB_URI in config.yaml
+  replace:
+    path: "{{ quay_root }}/quay-config/config.yaml"
+    regexp: '^DB_URI: postgresql://.*$'
+    replace: 'DB_URI: sqlite:////sqlite/quay_sqlite.db'
+  register: db_uri_update
+
+- name: Ensure DB_URI was updated successfully
+  assert:
+    that:
+      - db_uri_update.changed
+    fail_msg: "Failed to update DB_URI in quay's config"
+    success_msg: "DB_URI has been updated successfully"
+
+- name: Stop Quay service
+  systemd:
+    name: quay-app.service
+    enabled: no
+    daemon_reload: yes
+    state: stopped
+    force: yes
+    scope: "{{ systemd_scope }}"
+  when: container_result.rc == 0
+
+- name: Stop Postgres service
+  systemd:
+    name: quay-postgres.service
+    enabled: no
+    daemon_reload: yes
+    state: stopped
+    force: yes
+    scope: "{{ systemd_scope }}"
+  when: container_result.rc == 0
+
+- name: Delete Postgres Storage named volume
+  containers.podman.podman_volume:
+      state: absent
+      name: pg-storage
+  when: auto_approve|bool == true and pg_storage == "pg-storage" and container_result.rc == 0
+
+- name: Delete Postgres Password Secret
+  containers.podman.podman_secret:
+    state: absent
+    name: pgdb_pass
+  when: container_result.rc == 0
+
+- name: Delete necessary directory for Postgres persistent data
+  ansible.builtin.file:
+    path: "{{ pg_storage }}"
+    state: absent
+  become: yes
+  when: auto_approve|bool == true and pg_storage.startswith('/') and container_result.rc == 0
+
+- name: Cleanup quay-postgres systemd unit file
+  file:
+    state: absent
+    path: "{{ systemd_unit_dir }}/quay-postgres.service"
+  when: container_result.rc == 0

ansible-runner/context/app/project/roles/mirror_appliance/tasks/upgrade-config-vars.yaml

@@ -11,7 +11,17 @@
   ansible.builtin.set_fact:
     quay_config_file: "{{ remote_yaml_file['content'] | b64decode | from_yaml }}"
 
-- name: Set facts for the pre-existing secrets only if they are a string and not a jinja2 variable in the config.yaml.
+- name: Set facts for the existing redis secrets only if they are a string and not a jinja2 variable in the config.yaml.
   ansible.builtin.set_fact:
     REDIS_PASSWORD : "{{ quay_config_file['USER_EVENTS_REDIS']['password'] }}"
-  when: quay_config_file['DATABASE_SECRET_KEY'] is string and quay_config_file['USER_EVENTS_REDIS']['password'] is string and quay_config_file['DB_URI'] is string
+  when: quay_config_file['DATABASE_SECRET_KEY'] is string and quay_config_file['USER_EVENTS_REDIS']['password'] is string
+
+- name: Check if quay-postgres container is running
+  command: podman ps -q -f name=quay-postgres
+  register: postgres_container_status
+  changed_when: false
+
+- name: Set facts for existing postgres secrets only if they are a string and not a jinja2 variable in the config.yaml.
+  ansible.builtin.set_fact:
+    PGDB_PASSWORD : "{{ quay_config_file['DB_URI'].split('@')[0].split(':')[2] }}"
+  when: postgres_container_status.stdout != "" and quay_config_file['DATABASE_SECRET_KEY'] is string and quay_config_file['DB_URI'] is string