Update deploy scripts (#2609)

* change nuc_test to nuc_qa profile; add nuc qa targets
* Allow specification of container engine through an environment variable
* Improve help text of build script
* Remove blank entry from list of backups
* Create script to generate and install the semantic domain data (primarily for development use)
* Set imagePullPolicy fo Always for production, QA servers and NUCs emulating the QA server.
* Update documentation for build.py
* Fix bare URL in README.md

README.md

@@ -395,8 +395,8 @@ Auto-format frontend code in the `src` folder.
 
 ### Import Semantic Domains
 
-Imports Semantic Domains from the XML files in `./deploy/scripts/semantic_domains/xml`. Run from within a Python virtual
-environment.
+To import Semantic Domains from the XML files in `./deploy/scripts/semantic_domains/xml`. Run from within a Python
+virtual environment.
 
 1. Generate the files for import into the Mongo database:
 
@@ -458,9 +458,9 @@ sake of devices with limited bandwidth. There are scripts for generating these f
 files in this directory should _not_ be manually edited.
 
 The bash script `scripts/fetch_wordlists.sh` is used to fetch dictionary files for a given language (e.g., `es`) from
-https://cgit.freedesktop.org/libreoffice/dictionaries/ and convert them to raw wordlists (e.g.,
-`src/resources/dictionaries/es.txt`). Execute the script with no arguments for its usage details. Any language not
-currently supported can be manually added as a case in this script.
+the [LibreOffice dictionaries](https://cgit.freedesktop.org/libreoffice/dictionaries/) and convert them to raw wordlists
+(e.g., `src/resources/dictionaries/es.txt`). Execute the script with no arguments for its usage details. Any language
+not currently supported can be manually added as a case in this script.
 
 ```bash
 ./scripts/fetch_wordlist.sh
@@ -611,13 +611,21 @@ Build _The Combine_ containers by running the build script in an activated Pytho
 _TheCombine_'s project directory. (See the [Python](#python) section to create the virtual environment.)
 
 ```bash
-python deploy/scripts/build.py [--nerdctl]
+python deploy/scripts/build.py
 ```
 
 Notes:
 
-- Use the `--nerdctl` option if you are using _Rancher Desktop_ with the `containerd` for the container runtime. If you
-  are using _Docker Desktop_ or _Rancher Desktop_ with the `dockerd` container runtime, omit this option.
+- If you are using _Rancher Desktop_ with `containerd` for the container runtime, set the following environment variable
+  in your user profile:
+
+  ```bash
+  export CONTAINER_CLI="nerdctl"
+  ```
+
+  If you are using _Docker Desktop_ or _Rancher Desktop_ with the `dockerd` container runtime, clear this variable or
+  set its value to `docker`.
+
 - Run with the `--help` option to see all available options.
 - If you see errors like:
 

deploy/helm/cert-proxy-client/templates/_helpers.tpl

@@ -11,19 +11,6 @@
   {{- end }}
 {{- end }}
 
-{{/* Get the Image Pull Policy */}}
-{{- define "cert-proxy-client.imagePullPolicy" }}
-  {{- if .Values.imagePullPolicy }}
-    {{- print .Values.imagePullPolicy }}
-  {{- else }}
-    {{- if eq .Values.global.imageTag "latest" }}
-      {{- print "Always" }}
-    {{- else }}
-      {{- print "IfNotPresent" }}
-    {{- end }}
-  {{- end }}
-{{- end }}
-
 {{/* Build the SSL Certificate secret name */}}
 {{- define "cert-proxy-client.certSecretName" -}}
   {{- $hostString := replace "." "-" .Values.global.serverName }}

deploy/helm/cert-proxy-client/templates/update-cert-cronjob.yaml

@@ -18,7 +18,7 @@ spec:
           serviceAccountName: {{ .Values.serviceAccount.name }}
           containers:
             - image: {{ template "cert-proxy-client.containerImage" . }}
-              imagePullPolicy: {{ template "cert-proxy-client.imagePullPolicy" . }}
+              imagePullPolicy: {{ .Values.global.imagePullPolicy }}
               name: update-cert-cronjob
               command:
                 - update_cert.py

deploy/helm/cert-proxy-client/templates/update-cert-oneshot.yaml

@@ -12,7 +12,7 @@ spec:
       serviceAccountName: {{ .Values.serviceAccount.name }}
       containers:
       - image: {{ template "cert-proxy-client.containerImage" . }}
-        imagePullPolicy: {{ template "cert-proxy-client.imagePullPolicy" . }}
+        imagePullPolicy: {{ .Values.global.imagePullPolicy }}
         name: update-cert-oneshot
         command:
           - update_cert.py

deploy/helm/cert-proxy-server/templates/_helpers.tpl

@@ -10,16 +10,3 @@
     {{- printf "%s:%s" .Values.imageName .Values.global.imageTag }}
   {{- end }}
 {{- end }}
-
-{{/* Get the Image Pull Policy */}}
-{{- define "cert-proxy-server.imagePullPolicy" }}
-  {{- if .Values.imagePullPolicy }}
-    {{- print .Values.imagePullPolicy }}
-  {{- else }}
-    {{- if eq .Values.global.imageTag "latest" }}
-      {{- print "Always" }}
-    {{- else }}
-      {{- print "IfNotPresent" }}
-    {{- end }}
-  {{- end }}
-{{- end }}

deploy/helm/cert-proxy-server/templates/deployment-cert-proxy-server.yaml

@@ -61,7 +61,7 @@ spec:
             - name: CERT_PROXY_NAMESPACE
               value: {{ .Release.Namespace }}
           image: {{ template "cert-proxy-server.containerImage" . }}
-          imagePullPolicy: {{ template "cert-proxy-server.imagePullPolicy" . }}
+          imagePullPolicy: {{ .Values.global.imagePullPolicy }}
           name: combine-cert-proxy
           resources: {}
       restartPolicy: Always

deploy/helm/create-admin-user/templates/_helpers.tpl

@@ -10,16 +10,3 @@
     {{- printf "%s:%s" .Values.imageName .Values.global.imageTag }}
   {{- end }}
 {{- end }}
-
-{{/* Get the Image Pull Policy */}}
-{{- define "create-admin-user.imagePullPolicy" }}
-  {{- if .Values.global.imagePullPolicy }}
-    {{- print .Values.global.imagePullPolicy }}
-  {{- else }}
-    {{- if eq .Values.global.imageTag "latest" }}
-      {{- print "Always" }}
-    {{- else }}
-      {{- print "IfNotPresent" }}
-    {{- end }}
-  {{- end }}
-{{- end }}

deploy/helm/create-admin-user/templates/job-create-admin-user.yaml

@@ -41,7 +41,7 @@ spec:
                   key: COMBINE_ADMIN_EMAIL
                   name: admin-user-secrets
           image: {{ template "create-admin-user.containerImage" . }}
-          imagePullPolicy: {{ template "create-admin-user.imagePullPolicy" . }}
+          imagePullPolicy: {{ .Values.global.imagePullPolicy }}
           name: create-admin-user
           resources: {}
           volumeMounts:

deploy/helm/thecombine/charts/backend/templates/_helpers.tpl

@@ -10,16 +10,3 @@
     {{- printf "%s:%s" .Values.imageName .Values.global.imageTag }}
   {{- end }}
 {{- end }}
-
-{{/* Get the Image Pull Policy */}}
-{{- define "backend.imagePullPolicy" }}
-  {{- if .Values.global.imagePullPolicy }}
-    {{- print .Values.global.imagePullPolicy }}
-  {{- else }}
-    {{- if eq .Values.global.imageTag "latest" }}
-      {{- print "Always" }}
-    {{- else }}
-      {{- print "IfNotPresent" }}
-    {{- end }}
-  {{- end }}
-{{- end }}

deploy/helm/thecombine/charts/backend/templates/deployment-backend.yaml

@@ -35,7 +35,7 @@ spec:
       containers:
         - name: backend
           image: {{ template "backend.containerImage" . }}
-          imagePullPolicy: {{ template "backend.imagePullPolicy" . }}
+          imagePullPolicy: {{ .Values.global.imagePullPolicy }}
           env:
             - name: COMBINE_JWT_SECRET_KEY
               valueFrom:

deploy/helm/thecombine/charts/database/templates/_helpers.tpl

@@ -10,16 +10,3 @@
     {{- printf "%s:%s" .Values.imageName .Values.global.imageTag }}
   {{- end }}
 {{- end }}
-
-{{/* Get the Image Pull Policy */}}
-{{- define "database.imagePullPolicy" }}
-  {{- if .Values.global.imagePullPolicy }}
-    {{- print .Values.global.imagePullPolicy }}
-  {{- else }}
-    {{- if eq .Values.global.imageTag "latest" }}
-      {{- print "Always" }}
-    {{- else }}
-      {{- print "IfNotPresent" }}
-    {{- end }}
-  {{- end }}
-{{- end }}

deploy/helm/thecombine/charts/database/templates/database.yaml

@@ -44,7 +44,7 @@ spec:
     spec:
       containers:
         - image: {{ template "database.containerImage" . }}
-          imagePullPolicy: {{ template "database.imagePullPolicy" . }}
+          imagePullPolicy: {{ .Values.global.imagePullPolicy }}
           name: database
           ports:
             - containerPort: 27017

deploy/helm/thecombine/charts/frontend/templates/_helpers.tpl

@@ -10,16 +10,3 @@
     {{- printf "%s:%s" .Values.imageName .Values.global.imageTag }}
   {{- end }}
 {{- end }}
-
-{{/* Get the Image Pull Policy */}}
-{{- define "frontend.imagePullPolicy" }}
-  {{- if .Values.global.imagePullPolicy }}
-    {{- print .Values.global.imagePullPolicy }}
-  {{- else }}
-    {{- if eq .Values.global.imageTag "latest" }}
-      {{- print "Always" }}
-    {{- else }}
-      {{- print "IfNotPresent" }}
-    {{- end }}
-  {{- end }}
-{{- end }}

deploy/helm/thecombine/charts/frontend/templates/deployment-frontend.yaml

@@ -27,7 +27,7 @@ spec:
       containers:
         - name: frontend
           image: {{ template "frontend.containerImage" . }}
-          imagePullPolicy: {{ template "frontend.imagePullPolicy" . }}
+          imagePullPolicy: {{ .Values.global.imagePullPolicy }}
           env:
             - name: CERT_ADDL_DOMAINS
               valueFrom:

deploy/helm/thecombine/charts/maintenance/templates/_helpers.tpl

@@ -11,19 +11,6 @@
   {{- end }}
 {{- end }}
 
-{{/* Get the Image Pull Policy */}}
-{{- define "maintenance.imagePullPolicy" }}
-  {{- if .Values.global.imagePullPolicy }}
-    {{- print .Values.global.imagePullPolicy }}
-  {{- else }}
-    {{- if eq .Values.global.imageTag "latest" }}
-      {{- print "Always" }}
-    {{- else }}
-      {{- print "IfNotPresent" }}
-    {{- end }}
-  {{- end }}
-{{- end }}
-
 {{/* Build the backup location string */}}
 {{- define "maintenance.backupNameFilter" -}}
   {{- $hostString := replace "." "-" .Values.global.serverName }}

deploy/helm/thecombine/charts/maintenance/templates/deployment-maintenance.yaml

@@ -30,7 +30,7 @@ spec:
           command: [ "/bin/bash", "-c", "--" ]
           args: [ 'sleep infinity & PID=$! ; trap "kill $PID" INT TERM ; wait' ]
           image: {{ template "maintenance.containerImage" . }}
-          imagePullPolicy: {{ template "maintenance.imagePullPolicy" . }}
+          imagePullPolicy: {{ .Values.global.imagePullPolicy }}
           securityContext:
             runAsUser: 999
             runAsGroup: 999

deploy/helm/thecombine/values.yaml

@@ -11,28 +11,33 @@
 
 global:
   serverName: localhost
-  pullSecretName: aws-login-credentials
-  awsS3Access: aws-s3-credentials
-  # Update strategy should be "Recreate" or "Rolling Update"
-  updateStrategy: Recreate
-  adminUsername: "Override"
-  adminPassword: "Override"
-  adminEmail: "Override"
+  # AWS Credentials
   awsAccount: "Override"
   awsDefaultRegion: "Override"
   awsAccessKeyId: "Override"
   awsSecretAccessKey: "Override"
+  # AWS S3 access
+  awsS3Access: aws-s3-credentials
+  awsS3Location: "thecombine.app"
+  # Combine variables
+  adminUsername: "Override"
+  adminPassword: "Override"
+  adminEmail: "Override"
   combineJwtSecretKey: "Override"
   combineSmtpUsername: "Override"
   combineSmtpPassword: "Override"
+  offline: false
+  # Local Storage for fonts
   fontStorageAccessMode: "ReadWriteOnce"
   fontStorageSize: 1Gi
-  offline: false
-  imageTag: "latest"
+  # Values for pulling container image from image registry
+  imagePullPolicy: "Override"
   # Define the image registry to use (may be blank for local images)
   imageRegistry: awsEcr
-  # Default AWS S3 location
-  awsS3Location: "thecombine.app"
+  imageTag: "latest"
+  pullSecretName: aws-login-credentials
+  # Update strategy should be "Recreate" or "Rolling Update"
+  updateStrategy: Recreate
 
 aws-login:
   enabled: true

deploy/scripts/build.py

@@ -13,12 +13,14 @@
 
 from __future__ import annotations
 
-import argparse
+from argparse import ArgumentParser, HelpFormatter, Namespace
 from dataclasses import dataclass
 import logging
+import os
 from pathlib import Path
 import subprocess
 import sys
+import textwrap
 import time
 from typing import Callable, Dict, List, Optional
 
@@ -167,11 +169,33 @@ def get_image_name(repo: Optional[str], component: str, tag: Optional[str]) -> s
     return f"combine_{component}{tag_str}"
 
 
-def parse_args() -> argparse.Namespace:
+class RawFormatter(HelpFormatter):
+    """
+    Allow new lines in help text.
+
+    see https://stackoverflow.com/questions/3853722/how-to-insert-newlines-on-argparse-help-text
+    """
+
+    def _fill_text(self, text, width, indent):  # type: ignore
+        """Strip the indent from the original python definition that plagues most of us."""
+        text = textwrap.dedent(text)
+        text = textwrap.indent(text, indent)  # Apply any requested indent.
+        text = text.splitlines()  # Make a list of lines
+        text = [textwrap.fill(line, width) for line in text]  # Wrap each line
+        text = "\n".join(text)  # Join the lines again
+        return str(text)
+
+
+def parse_args() -> Namespace:
     """Parse user command line arguments."""
-    parser = argparse.ArgumentParser(
+    parser = ArgumentParser(
         description="Build containerd container images for project.",
-        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+        epilog="""
+         NOTE:
+         The '--nerdctl' option is DEPRECATED and will be removed in future versions.
+         Set the environment variable CONTAINER_CLI to 'nerdctl' or 'docker' instead.
+        """,
+        formatter_class=RawFormatter,
     )
     parser.add_argument(
         "--build-args", nargs="*", help="Build arguments to pass to the docker build."
@@ -240,12 +264,17 @@ def main() -> None:
     logging.basicConfig(format="%(levelname)s:%(message)s", level=log_level)
 
     # Setup required build engine - docker or nerdctl
-    if args.nerdctl:
-        build_cmd = ["nerdctl", "-n", args.namespace, "build"]
-        push_cmd = ["nerdctl", "-n", args.namespace, "push"]
-    else:
-        build_cmd = ["docker", "buildx", "build"]
-        push_cmd = ["docker", "push"]
+    container_cli = os.getenv("CONTAINER_CLI", "nerdctl" if args.nerdctl else "docker")
+    match container_cli:
+        case "nerdctl":
+            build_cmd = [container_cli, "-n", args.namespace, "build"]
+            push_cmd = [container_cli, "-n", args.namespace, "push"]
+        case "docker":
+            build_cmd = [container_cli, "buildx", "build"]
+            push_cmd = [container_cli, "push"]
+        case _:
+            logging.critical(f"Container CLI '{container_cli}' is not supported.")
+            sys.exit(1)
 
     # Setup build options
     build_opts: List[str] = []