db: Replace Postgres database with an in-app Sqlite db for Quay durin…

…g fresh install of OMR (#152)

* Remove postgres references from ansible tasks

Signed-off-by: harishsurf <[email protected]>

* Update ansible tasks to install quay with sqlite

* Clean up remaining postgres references

* Add debug logs when quay-app systemd service fails

* change dir path for sqlite db file to live outside of /quay-registry path

* set WORKER_COUNT=1 when starting quay with Sqlite

without this, multiple workers tend to write simultaneously
causing concurrent write issues

* Add conditional when creating volume based on user input

* use podman unshare to change ownership + bump quay version

When podman runs containers in rootless mode, it remaps the container's
UIDs to the host UIDs assigned to the user. For example, the container's
UID 1001 is no longer 1001 on the host; it will be remapped to something
like 101000. Therefore, we need to use podman unshare to assign the
correct owner UID to the directory.

---------

Signed-off-by: harishsurf <[email protected]>

.env

@@ -1,7 +1,6 @@
 EE_IMAGE=quay.io/quay/mirror-registry-ee:latest
 EE_BASE_IMAGE=registry.redhat.io/ansible-automation-platform-22/ee-minimal-rhel8:1.0.0-249
 EE_BUILDER_IMAGE=registry.redhat.io/ansible-automation-platform-22/ansible-builder-rhel8:1.1.0-103
-POSTGRES_IMAGE=registry.redhat.io/rhel8/postgresql-10:1-203.1669834630
-QUAY_IMAGE=registry.redhat.io/quay/quay-rhel8:v3.8.15
+QUAY_IMAGE=quay.io/projectquay/quay:3.12.0-nightly.20240626
 REDIS_IMAGE=registry.redhat.io/rhel8/redis-6:1-92.1669834635
 PAUSE_IMAGE=registry.access.redhat.com/ubi8/pause:8.7-6

Dockerfile

@@ -3,7 +3,6 @@ ARG QUAY_IMAGE=${QUAY_IMAGE}
 ARG EE_IMAGE=${EE_IMAGE}
 ARG EE_BASE_IMAGE=${EE_BASE_IMAGE}
 ARG EE_BUILDER_IMAGE=${EE_BUILDER_IMAGE}
-ARG POSTGRES_IMAGE=${POSTGRES_IMAGE}
 ARG REDIS_IMAGE=${REDIS_IMAGE}
 ARG PAUSE_IMAGE=${PAUSE_IMAGE}
 
@@ -14,7 +13,6 @@ FROM registry.access.redhat.com/ubi8:latest AS cli
 ARG RELEASE_VERSION=${RELEASE_VERSION}
 ARG QUAY_IMAGE=${QUAY_IMAGE}
 ARG EE_IMAGE=${EE_IMAGE}
-ARG POSTGRES_IMAGE=${POSTGRES_IMAGE}
 ARG REDIS_IMAGE=${REDIS_IMAGE}
 ARG PAUSE_IMAGE=${PAUSE_IMAGE}
 
@@ -34,11 +32,10 @@ ENV RELEASE_VERSION=${RELEASE_VERSION}
 ENV EE_IMAGE=${EE_IMAGE}
 ENV QUAY_IMAGE=${QUAY_IMAGE}
 ENV REDIS_IMAGE=${REDIS_IMAGE}
-ENV POSTGRES_IMAGE=${POSTGRES_IMAGE}
 ENV PAUSE_IMAGE=${PAUSE_IMAGE}
 
 RUN go build -v \
-	-ldflags "-X github.com/quay/mirror-registry/cmd.releaseVersion=${RELEASE_VERSION} -X github.com/quay/mirror-registry/cmd.eeImage=${EE_IMAGE} -X github.com/quay/mirror-registry/cmd.pauseImage=${PAUSE_IMAGE} -X github.com/quay/mirror-registry/cmd.quayImage=${QUAY_IMAGE} -X github.com/quay/mirror-registry/cmd.redisImage=${REDIS_IMAGE} -X github.com/quay/mirror-registry/cmd.postgresImage=${POSTGRES_IMAGE}" \
+	-ldflags "-X github.com/quay/mirror-registry/cmd.releaseVersion=${RELEASE_VERSION} -X github.com/quay/mirror-registry/cmd.eeImage=${EE_IMAGE} -X github.com/quay/mirror-registry/cmd.pauseImage=${PAUSE_IMAGE} -X github.com/quay/mirror-registry/cmd.quayImage=${QUAY_IMAGE} -X github.com/quay/mirror-registry/cmd.redisImage=${REDIS_IMAGE}" \
 	-o mirror-registry
 
 # Create Ansible Execution Environment
@@ -71,7 +68,6 @@ COPY ansible-runner/context/app /runner
 # Pull in Quay dependencies
 FROM $QUAY_IMAGE as quay
 FROM $REDIS_IMAGE as redis
-FROM $POSTGRES_IMAGE as postgres
 FROM $PAUSE_IMAGE as pause
 
 # Create mirror registry archive
@@ -87,16 +83,13 @@ RUN tar -cvf execution-environment.tar -C /ansible .
 COPY --from=redis / /redis
 RUN tar -cvf redis.tar -C /redis .
 
-COPY --from=postgres / /postgres
-RUN tar -cvf postgres.tar -C /postgres .
-
 COPY --from=quay / /quay
 RUN tar -cvf quay.tar -C /quay .
 
 COPY --from=cli /cli/mirror-registry .
 
-# Bundle quay, redis, postgres, and pause into a single archive
-RUN tar -cvf image-archive.tar quay.tar redis.tar postgres.tar pause.tar
+# Bundle quay, redis and pause into a single archive
+RUN tar -cvf image-archive.tar quay.tar redis.tar pause.tar
 
 # Bundle mirror registry archive
 RUN tar -czvf mirror-registry.tar.gz image-archive.tar execution-environment.tar mirror-registry

Dockerfile.online

@@ -9,7 +9,6 @@ FROM registry.redhat.io/ubi8:latest AS cli
 ARG RELEASE_VERSION=${RELEASE_VERSION}
 ARG QUAY_IMAGE=${QUAY_IMAGE}
 ARG EE_IMAGE=${EE_IMAGE}
-ARG POSTGRES_IMAGE=${POSTGRES_IMAGE}
 ARG REDIS_IMAGE=${REDIS_IMAGE}
 ARG PAUSE_IMAGE=${PAUSE_IMAGE}
 
@@ -29,11 +28,10 @@ ENV RELEASE_VERSION=${RELEASE_VERSION}
 ENV EE_IMAGE=${EE_IMAGE}
 ENV QUAY_IMAGE=${QUAY_IMAGE}
 ENV REDIS_IMAGE=${REDIS_IMAGE}
-ENV POSTGRES_IMAGE=${POSTGRES_IMAGE}
 ENV PAUSE_IMAGE=${PAUSE_IMAGE}
 
 RUN go build -v \
-    -ldflags "-X github.com/quay/mirror-registry/cmd.releaseVersion=${RELEASE_VERSION} -X github.com/quay/mirror-registry/cmd.eeImage=${EE_IMAGE} -X github.com/quay/mirror-registry/cmd.pauseImage=${PAUSE_IMAGE} -X github.com/quay/mirror-registry/cmd.quayImage=${QUAY_IMAGE} -X github.com/quay/mirror-registry/cmd.redisImage=${REDIS_IMAGE} -X github.com/quay/mirror-registry/cmd.postgresImage=${POSTGRES_IMAGE}" \
+    -ldflags "-X github.com/quay/mirror-registry/cmd.releaseVersion=${RELEASE_VERSION} -X github.com/quay/mirror-registry/cmd.eeImage=${EE_IMAGE} -X github.com/quay/mirror-registry/cmd.pauseImage=${PAUSE_IMAGE} -X github.com/quay/mirror-registry/cmd.quayImage=${QUAY_IMAGE} -X github.com/quay/mirror-registry/cmd.redisImage=${REDIS_IMAGE}" \
     -o mirror-registry
 
 # Create Ansible Execution Environment

Makefile

@@ -8,7 +8,7 @@ all:
 
 build-golang-executable:
 	$(CLIENT) run --rm -v ${PWD}:/usr/src:Z -w /usr/src docker.io/golang:1.16 go build -v \
-	-ldflags "-X github.com/quay/mirror-registry/cmd.eeImage=${EE_IMAGE} -X 'github.com/quay/mirror-registry/cmd.quayImage=${QUAY_IMAGE}' -X 'github.com/quay/mirror-registry/cmd.redisImage=${REDIS_IMAGE}' -X 'github.com/quay/mirror-registry/cmd.postgresImage=${POSTGRES_IMAGE}'" \
+	-ldflags "-X github.com/quay/mirror-registry/cmd.eeImage=${EE_IMAGE} -X 'github.com/quay/mirror-registry/cmd.quayImage=${QUAY_IMAGE}' -X 'github.com/quay/mirror-registry/cmd.redisImage=${REDIS_IMAGE}'" \
 	-o mirror-registry;
 
 build-online-zip: 
@@ -19,7 +19,6 @@ build-online-zip:
 		--build-arg EE_IMAGE=${EE_IMAGE} \
 		--build-arg EE_BASE_IMAGE=${EE_BASE_IMAGE} \
 		--build-arg EE_BUILDER_IMAGE=${EE_BUILDER_IMAGE} \
-		--build-arg POSTGRES_IMAGE=${POSTGRES_IMAGE} \
 		--build-arg REDIS_IMAGE=${REDIS_IMAGE} \
 		--build-arg PAUSE_IMAGE=${PAUSE_IMAGE} \
 		--file Dockerfile.online . 
@@ -35,7 +34,6 @@ build-offline-zip:
 		--build-arg EE_IMAGE=${EE_IMAGE} \
 		--build-arg EE_BASE_IMAGE=${EE_BASE_IMAGE} \
 		--build-arg EE_BUILDER_IMAGE=${EE_BUILDER_IMAGE} \
-		--build-arg POSTGRES_IMAGE=${POSTGRES_IMAGE} \
 		--build-arg REDIS_IMAGE=${REDIS_IMAGE} \
 		--build-arg PAUSE_IMAGE=${PAUSE_IMAGE} \
 		--file Dockerfile .

README.md

@@ -14,7 +14,7 @@ This application will allow user to easily install Quay and its required compone
 Download one of the installer package from our [releases](https://github.com/quay/mirror-registry/releases) page:
 
 - offline version (contains all required images to run Quay)
-- online version (additional container images to run Quay and Postgres will be downloaded by the installer)
+- online version (additional container images to run Quay and Redis will be downloaded by the installer)
 
 ### Running the installer
 
@@ -32,7 +32,7 @@ The following flags are also available:
 --quayHostname          The value to set SERVER_HOSTNAME in the Quay config.yaml. This defaults to <targetHostname>:8443.
 --quayRoot          -r  The folder where quay persistent quay config data is saved. This defaults to $HOME/quay-install.
 --quayStorage           The folder where quay persistent storage data is saved. This defaults to a Podman named volume 'quay-storage'. Root is required to uninstall.
---pgStorage             The folder where postgres persistent storage data is saved. This defaults to a Podman named volume 'pg-storage'. Root is required to uninstall.
+--sqliteStorage         The folder where quay sqlite db data is saved. This defaults to a Podman named volume 'sqlite-storage'. Root is required to uninstall.
 --ssh-key           -k  The path of your ssh identity key. This defaults to ~/.ssh/quay_installer.
 --sslCert               The path to the SSL certificate Quay should use.
 --sslCheckSkip          Whether or not to check the certificate hostname against the SERVER_HOSTNAME in config.yaml.
@@ -68,7 +68,7 @@ Behind the scenes, Ansible is using `ssh -i ~/.ssh/my_ssh_key [email protected]
 This command will make the following changes to your machine
 
 - Generate trusted SSH keys, if not supplied, in case the deployment target is the local host (required since the installer is ansible-based)
-- Pulls Quay, Redis, and Postgres images from `registry.redhat.io` (if using online installer)
+- Pulls Quay and Redis images from `registry.redhat.io` (if using online installer)
 - Sets up systemd files on host machine to ensure that container runtimes are persistent
 - Creates the folder defined by `--quayRoot` (default: `$HOME/quay-install`) contains install files, local storage, and config bundle.
 - Installs Quay and creates an initial user called `init` with an auto-generated password

ansible-runner/context/app/project/roles/mirror_appliance/tasks/autodetect-image-archive.yaml

@@ -26,10 +26,5 @@
 
 - name: Loading Quay if quay.tar exists
   shell: 
-    cmd: podman image import --change 'ENV container=oci' --change 'ENV PATH=/app/bin/:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin' --change 'ENV PYTHONUNBUFFERED=1' --change 'ENV PYTHONIOENCODING=UTF-8' --change 'ENV LC_ALL=C.UTF-8' --change 'ENV LANG=C.UTF-8' --change 'ENV QUAYDIR=/quay-registry' --change 'ENV QUAYCONF=/quay-registry/conf' --change 'ENV QUAYRUN=/quay-registry/conf' --change 'ENV QUAYPATH=/quay-registry' --change 'ENV PYTHONUSERBASE=/app' --change 'ENV PYTHONPATH=/quay-registry' --change 'ENV TZ=UTC' --change 'ENV RED_HAT_QUAY=true' --change 'ENTRYPOINT=["dumb-init","--","/quay-registry/quay-entrypoint.sh"]' --change 'WORKDIR=/quay-registry' --change 'EXPOSE=7443' --change 'EXPOSE=8080' --change 'EXPOSE=8443' --change 'VOLUME=/conf/stack' --change 'VOLUME=/datastorage' --change 'VOLUME=/tmp' --change 'VOLUME=/var/log' --change 'USER=1001' --change 'CMD ["registry"]' - {{ quay_image }} < {{ quay_root }}/quay.tar
-  when: p.stat.exists and local_install == "false"
-
-- name: Loading Postgres if postgres.tar exists
-  shell: 
-    cmd: podman image import --change 'ENV PATH=/opt/app-root/src/bin:/opt/app-root/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin' --change 'ENV STI_SCRIPTS_URL=image:///usr/libexec/s2i' --change 'ENV STI_SCRIPTS_PATH=/usr/libexec/s2i' --change 'ENV APP_ROOT=/opt/app-root' --change 'ENV APP_DATA=/opt/app-root' --change 'ENV HOME=/var/lib/pgsql' --change 'ENV PLATFORM=el8' --change 'ENV POSTGRESQL_VERSION=10' --change 'ENV POSTGRESQL_PREV_VERSION=9.6' --change 'ENV PGUSER=postgres' --change 'ENV CONTAINER_SCRIPTS_PATH=/usr/share/container-scripts/postgresql' --change 'ENTRYPOINT=["container-entrypoint"]' --change 'WORKDIR=/opt/app-root/src' --change 'EXPOSE=5432' --change 'USER=26' --change 'CMD ["run-postgresql"]' - {{ postgres_image }} < {{ quay_root }}/postgres.tar
+    cmd: podman image import --change 'ENV container=oci' --change 'ENV PATH=/app/bin/:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin' --change 'ENV PYTHONUNBUFFERED=1' --change 'ENV PYTHONIOENCODING=UTF-8' --change 'ENV LC_ALL=C.UTF-8' --change 'ENV LANG=C.UTF-8' --change 'ENV QUAYDIR=/quay-registry' --change 'ENV QUAYCONF=/quay-registry/conf' --change 'ENV QUAYRUN=/quay-registry/conf' --change 'ENV QUAYPATH=/quay-registry' --change 'ENV PYTHONUSERBASE=/app' --change 'ENV PYTHONPATH=/quay-registry' --change 'ENV TZ=UTC' --change 'ENV RED_HAT_QUAY=true' --change 'ENTRYPOINT=["dumb-init","--","/quay-registry/quay-entrypoint.sh"]' --change 'WORKDIR=/quay-registry' --change 'EXPOSE=7443' --change 'EXPOSE=8080' --change 'EXPOSE=8443' --change 'VOLUME=/conf/stack' --change 'VOLUME=/datastorage' --change 'VOLUME=/sqlite' --change 'VOLUME=/tmp' --change 'VOLUME=/var/log' --change 'USER=1001' --change 'CMD ["registry"]' - {{ quay_image }} < {{ quay_root }}/quay.tar
   when: p.stat.exists and local_install == "false"

ansible-runner/context/app/project/roles/mirror_appliance/tasks/expand-vars.yaml

@@ -1,7 +1,3 @@
-- name: Expand pg_storage
-  shell: 'echo {{ pg_storage }}'
-  register: expanded_pg_storage_output
-
 - name: Expand quay_root
   shell: 'echo {{ quay_root }}'
   register: expanded_quay_root_output
@@ -10,8 +6,12 @@
   shell: 'echo {{ quay_storage }}'
   register: expanded_quay_storage_output
 
+- name: Expand sqlite_storage
+  shell: 'echo {{ sqlite_storage }}'
+  register: expanded_sqlite_storage_output
+
 - name: Set expanded variables
   set_fact:
-    expanded_pg_storage: "{{ expanded_pg_storage_output.stdout }}"
+    expanded_sqlite_storage: "{{ expanded_sqlite_storage_output.stdout }}"
     expanded_quay_root: "{{ expanded_quay_root_output.stdout }}"
     expanded_quay_storage: "{{ expanded_quay_storage_output.stdout }}"

ansible-runner/context/app/project/roles/mirror_appliance/tasks/install-quay-service.yaml

@@ -14,6 +14,16 @@
     state: present
   when: "quay_storage.startswith('/')"
 
+- name: Create necessary directory for sqlite data
+  become_method: containers.podman.podman_unshare
+  become: true
+  ansible.builtin.file:
+    mode: u+rwx
+    owner: 1001
+    path: "{{ sqlite_storage }}"
+    state: directory
+  when: "sqlite_storage.startswith('/')"
+
 - name: Create necessary directory for Quay config bundle
   ansible.builtin.file:
     path: "{{ quay_root }}/quay-config"
@@ -120,7 +130,14 @@
 - name: Create Quay Storage named volume
   containers.podman.podman_volume:
       state: present
-      name: quay-storage
+      name: "{{ quay_storage }}"
+  when: "not quay_storage.startswith('/')"
+
+- name: Create Sqlite Storage named volume
+  containers.podman.podman_volume:
+      state: present
+      name: "{{ sqlite_storage }}"
+  when: "not sqlite_storage.startswith('/')"
 
 - name: Start Quay service
   systemd:

ansible-runner/context/app/project/roles/mirror_appliance/tasks/install-redis-service.yaml

@@ -20,7 +20,7 @@
     state: present
     name: redis_pass
     data: "{{ redis_password }}"
-    skip_existing: true
+    skip_existing: false
 
 - name: Start Redis service
   systemd:

ansible-runner/context/app/project/roles/mirror_appliance/tasks/main.yaml

@@ -16,9 +16,6 @@
 - name: Autodetect Image Archive
   include_tasks: autodetect-image-archive.yaml
 
-- name: Install Postgres Service
-  include_tasks: install-postgres-service.yaml
-
 - name: Install Redis Service
   include_tasks: install-redis-service.yaml
 

ansible-runner/context/app/project/roles/mirror_appliance/tasks/secret-vars.yaml

@@ -2,5 +2,4 @@
   set_fact:
     secret_key: "{{ lookup('community.general.random_string', length=48, base64=True) }}"
     database_secret_key: "{{ lookup('community.general.random_string', length=48, base64=True) }}"
-    pgdb_password: "{{ lookup('community.general.random_string', length=24, special=False) }}"
     redis_password: "{{ lookup('community.general.random_string', length=24, special=False) }}"