From 3ccc1595b550394155646a5da5525b394ede99b4 Mon Sep 17 00:00:00 2001 From: Jim Grady Date: Thu, 1 Aug 2024 10:11:35 -0400 Subject: [PATCH] Package container images with the self-extracting installer for The Combine (#3240) * Add container images to install package - add container images for helm charts to the install package - cert-manager - NGINX ingress controller - The Combine - add a `--net-install` to create installer without the images * Update Python dependencies * Update installer README.md * Restore installation of helm in non-airgap installations * Remove cert-manager from standard charts cert-manager is not needed for NUCs or the offline development and it is installed by LTOps for the QA and Production environments. It is only used for development clusters, e.g. on Rancher Desktop or Docker Desktop --- .gitignore | 2 +- README.md | 9 +- deploy/ansible/group_vars/nuc/main.yml | 6 +- deploy/ansible/group_vars/server/main.yml | 10 +- deploy/ansible/host_vars/localhost/main.yml | 10 +- deploy/ansible/playbook_desktop_setup.yaml | 18 +- .../ansible/playbook_k3s_airgapped_files.yml | 58 ++++++ deploy/ansible/playbook_nuc_setup.yaml | 1 + .../roles/container_engine/defaults/main.yml | 2 + .../roles/container_engine/tasks/main.yml | 8 +- .../roles/container_images/defaults/main.yml | 6 + .../roles/container_images/tasks/main.yml | 59 ++++++ .../roles/helm_install/defaults/main.yml | 2 +- .../roles/k8s_install/defaults/main.yml | 5 +- .../ansible/roles/k8s_install/tasks/k3s.yml | 67 +++---- .../roles/k8s_install/tasks/k3s_airgap.yml | 22 +++ .../ansible/roles/k8s_install/tasks/main.yml | 81 +++++--- .../roles/support_tools/defaults/main.yml | 2 - .../roles/support_tools/tasks/main.yml | 9 - .../ansible/roles/wifi_ap/tasks/install.yml | 2 +- deploy/ansible/vars/config_common.yml | 3 + deploy/ansible/vars/k3s_versions.yml | 4 + deploy/helm/cert-proxy-client/values.yaml | 4 +- .../maintenance/templates/get-fonts-hook.yaml | 1 + deploy/requirements.txt | 14 +- deploy/scripts/helm_utils.py | 111 +++++++++++ deploy/scripts/install-combine.sh | 131 +++++++++--- deploy/scripts/kube_env.py | 43 +++- deploy/scripts/package_images.py | 187 ++++++++++++++++++ deploy/scripts/setup_cluster.py | 115 +++++++---- deploy/scripts/setup_combine.py | 162 ++------------- .../scripts/setup_files/cluster_config.yaml | 6 + deploy/scripts/setup_files/profiles/dev.yaml | 4 +- deploy/scripts/utils.py | 14 ++ dev-requirements.txt | 36 ++-- installer/README.md | 44 ++++- installer/make-combine-installer.sh | 80 +++++++- maintenance/requirements.txt | 10 +- 38 files changed, 961 insertions(+), 387 deletions(-) create mode 100644 deploy/ansible/playbook_k3s_airgapped_files.yml create mode 100644 deploy/ansible/roles/container_images/defaults/main.yml create mode 100644 deploy/ansible/roles/container_images/tasks/main.yml create mode 100644 deploy/ansible/roles/k8s_install/tasks/k3s_airgap.yml create mode 100644 deploy/ansible/vars/k3s_versions.yml create mode 100644 deploy/scripts/helm_utils.py create mode 100755 deploy/scripts/package_images.py diff --git a/.gitignore b/.gitignore index 0c2992bdc4..5b13342ff2 100644 --- a/.gitignore +++ b/.gitignore @@ -83,7 +83,7 @@ deploy/scripts/semantic_domains/json/*.json database/semantic_domains/* # Combine installer -installer/combine-installer.run +installer/*.run installer/makeself-* installer/README.pdf diff --git a/README.md b/README.md index 707af1ee32..92a14bc9f7 100644 --- a/README.md +++ b/README.md @@ -531,7 +531,12 @@ cd installer ./make-combine-installer.sh combine-release-number ``` -where `combine-release-number` is the Combine release to be installed, e.g. `v1.2.0`. +where `combine-release-number` is the Combine release to be installed, e.g. `v2.1.0`. + +Options: + +- `--net-install` - build an installer that will download the required images at installation time. The default is to + package the images in the installation script. To update the PDF copy of the installer README.md file, run the following from the `installer` directory: @@ -636,7 +641,7 @@ environment. (See the [Python](#python) section to create the virtual environmen Install the required charts by running: ```bash -python deploy/scripts/setup_cluster.py +python deploy/scripts/setup_cluster.py --type development ``` `deploy/scripts/setup_cluster.py` assumes that the `kubectl` configuration file is setup to manage the desired diff --git a/deploy/ansible/group_vars/nuc/main.yml b/deploy/ansible/group_vars/nuc/main.yml index 1c85395b17..f9084dc30b 100644 --- a/deploy/ansible/group_vars/nuc/main.yml +++ b/deploy/ansible/group_vars/nuc/main.yml @@ -14,17 +14,13 @@ k8s_engine: k3s image_pull_secret: aws-login-credentials +use_airgap_images: false # k8s namespaces app_namespace: thecombine k8s_user: sillsdev -################################################ -# Helm Installation -################################################ -install_helm: no - ################################################ # Support Tool Settings ################################################ diff --git a/deploy/ansible/group_vars/server/main.yml b/deploy/ansible/group_vars/server/main.yml index 8bc8a5189f..1d883ad2e5 100644 --- a/deploy/ansible/group_vars/server/main.yml +++ b/deploy/ansible/group_vars/server/main.yml @@ -9,21 +9,13 @@ # Configure Kubernetes cluster ################################################ -# Specify which Kubernetes engine to install - -# one of k3s, or none. -k8s_engine: none - image_pull_secret: aws-login-credentials +use_airgap_images: false create_namespaces: [] # k8s namespaces app_namespace: thecombine -################################################ -# Helm Installation -################################################ -install_helm: no - ################################################ # Support Tool Settings ################################################ diff --git a/deploy/ansible/host_vars/localhost/main.yml b/deploy/ansible/host_vars/localhost/main.yml index 7df72dd2a2..cf6b4bcc85 100644 --- a/deploy/ansible/host_vars/localhost/main.yml +++ b/deploy/ansible/host_vars/localhost/main.yml @@ -7,22 +7,14 @@ # Configure Kubernetes cluster ################################################ -# Specify which Kubernetes engine to install - -# one of k3s or none. -k8s_engine: k3s - image_pull_secret: aws-login-credentials +use_airgap_images: true # k8s namespaces app_namespace: thecombine k8s_user: "{{ ansible_user_id }}" -################################################ -# Helm Installation -################################################ -install_helm: yes - ################################################ # Support Tool Settings ################################################ diff --git a/deploy/ansible/playbook_desktop_setup.yaml b/deploy/ansible/playbook_desktop_setup.yaml index caedfb6a3d..836152df35 100644 --- a/deploy/ansible/playbook_desktop_setup.yaml +++ b/deploy/ansible/playbook_desktop_setup.yaml @@ -15,27 +15,23 @@ vars_files: - "vars/config_common.yml" + - "vars/k3s_versions.yml" tasks: - - name: Update packages - apt: - update_cache: yes - upgrade: "yes" - - name: Setup WiFi Access Point import_role: name: wifi_ap when: has_wifi - - name: Enable hardware monitoring - import_role: - name: monitor_hardware - when: include_hw_monitoring - - name: Configure Network Interfaces import_role: name: network_config + - name: Install Preloaded Images + import_role: + name: container_images + when: install_airgap_images + - name: Install Container Engine import_role: name: container_engine @@ -47,7 +43,7 @@ - name: Install Helm import_role: name: helm_install - when: install_helm + when: not install_airgap_images - name: Setup Support Tool import_role: diff --git a/deploy/ansible/playbook_k3s_airgapped_files.yml b/deploy/ansible/playbook_k3s_airgapped_files.yml new file mode 100644 index 0000000000..7911849fef --- /dev/null +++ b/deploy/ansible/playbook_k3s_airgapped_files.yml @@ -0,0 +1,58 @@ +--- +############################################################## +# Playbook: playbook_k3s_airgapped_files.yml +# +# playbook_k3s_airgapped_files.yml downloads and packages the +# files necessary to install k3s on an airgapped system. This +# includes: +# - the k3s airgap images +# - k3s executable +# - k3s installation script +# - kubectl +# - helm +# +############################################################## + +- name: Build package for k3s airgap installation + hosts: localhost + gather_facts: yes + become: no + + vars_files: + - "vars/k3s_versions.yml" + + tasks: + - name: Create package directory if necessary + file: + path: "{{ package_dir }}" + state: directory + + - name: Download k3s assets + get_url: + dest: "{{ package_dir }}/{{ item }}" + url: "https://github.com/k3s-io/k3s/releases/download/{{ k3s_version }}/{{ item }}" + loop: + - k3s-airgap-images-amd64.tar.zst + - k3s + - sha256sum-amd64.txt + + - name: Verify k3s downloads + shell: + cmd: sha256sum --check --ignore-missing sha256sum-amd64.txt + chdir: "{{ package_dir }}" + changed_when: false + + - name: Download k3s install script + get_url: + dest: "{{ package_dir }}/install.sh" + url: https://get.k3s.io/ + + - name: Download kubectl + get_url: + dest: "{{ package_dir }}/kubectl" + url: "https://dl.k8s.io/release/{{ kubectl_version }}/bin/linux/amd64/kubectl" + + - name: Download helm + get_url: + dest: "{{ package_dir }}/helm.tar.gz" + url: "https://get.helm.sh/helm-{{ helm_version }}-linux-amd64.tar.gz" diff --git a/deploy/ansible/playbook_nuc_setup.yaml b/deploy/ansible/playbook_nuc_setup.yaml index a2bd3ce5c6..1c6bbb78a7 100644 --- a/deploy/ansible/playbook_nuc_setup.yaml +++ b/deploy/ansible/playbook_nuc_setup.yaml @@ -16,6 +16,7 @@ vars_files: - "vars/config_common.yml" + - "vars/k3s_versions.yml" tasks: - name: Update packages diff --git a/deploy/ansible/roles/container_engine/defaults/main.yml b/deploy/ansible/roles/container_engine/defaults/main.yml index 802a83f543..1276e993b3 100644 --- a/deploy/ansible/roles/container_engine/defaults/main.yml +++ b/deploy/ansible/roles/container_engine/defaults/main.yml @@ -1,3 +1,5 @@ --- container_packages: - containerd.io + +keyring_location: /etc/apt/keyrings diff --git a/deploy/ansible/roles/container_engine/tasks/main.yml b/deploy/ansible/roles/container_engine/tasks/main.yml index 3f19a7bf00..8508c4bc46 100644 --- a/deploy/ansible/roles/container_engine/tasks/main.yml +++ b/deploy/ansible/roles/container_engine/tasks/main.yml @@ -27,7 +27,7 @@ - name: Create keyring directory file: - path: /etc/apt/keyrings + path: "{{ keyring_location }}" state: directory owner: root group: root @@ -35,12 +35,12 @@ - name: Install Docker apt key shell: - cmd: "curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg" - creates: /etc/apt/keyrings/docker.gpg + cmd: "curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o {{ keyring_location }}/docker.gpg" + creates: "{{ keyring_location }}/docker.gpg" - name: Add Docker repository apt_repository: - repo: "deb [arch=amd64 signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu {{ ansible_distribution_release }} stable" + repo: "deb [arch=amd64 signed-by={{ keyring_location }}/docker.gpg] https://download.docker.com/linux/ubuntu {{ ansible_distribution_release }} stable" state: present filename: docker diff --git a/deploy/ansible/roles/container_images/defaults/main.yml b/deploy/ansible/roles/container_images/defaults/main.yml new file mode 100644 index 0000000000..02994e8520 --- /dev/null +++ b/deploy/ansible/roles/container_images/defaults/main.yml @@ -0,0 +1,6 @@ +--- +# Default values for setting up the container images for +# installing pre-downloaded images + +source_image_dir: ../airgap-images +airgap_image_dir: /var/lib/rancher/k3s/agent/images diff --git a/deploy/ansible/roles/container_images/tasks/main.yml b/deploy/ansible/roles/container_images/tasks/main.yml new file mode 100644 index 0000000000..d5edea306d --- /dev/null +++ b/deploy/ansible/roles/container_images/tasks/main.yml @@ -0,0 +1,59 @@ +--- +# Setup airgap images in {{ airgap_image_dir }} to be +# available when k3s and subsequent helm charts are installed. + +- name: Create airgap image directory + file: + path: "{{ airgap_image_dir }}" + state: directory + owner: root + group: root + mode: 0755 + +- name: Copy image files + copy: + src: "{{ source_image_dir }}/{{ item }}" + dest: "{{ airgap_image_dir }}/{{ item }}" + owner: root + group: root + mode: 0644 + loop: + - k3s-airgap-images-amd64.tar.zst + - middleware-airgap-images-amd64.tar.zst + - combine-airgap-images-amd64.tar.zst + +# Add k3s, kubectl and the k3s installation script to +# /usr/local/bin +- name: Copy k3s & utility programes + copy: + src: "{{ source_image_dir }}/{{ item }}" + dest: /usr/local/bin/{{ item }} + owner: root + group: root + mode: 0755 + loop: + - k3s + - kubectl + - install.sh + +# Install helm +- name: Create directory for helm installation + file: + path: /opt/helm/{{ helm_version }} + state: directory + owner: root + group: root + mode: 0755 + +- name: Unpack helm + shell: + cmd: tar xzvf "{{ source_image_dir }}/helm.tar.gz" -C /opt/helm/{{ helm_version }} + +- name: Create link to helm binary + file: + src: /opt/helm/{{ helm_version }}/linux-amd64/helm + dest: /usr/local/bin/helm + state: link + owner: root + group: root + mode: 0755 diff --git a/deploy/ansible/roles/helm_install/defaults/main.yml b/deploy/ansible/roles/helm_install/defaults/main.yml index 5e6d43b831..54401b3cdc 100644 --- a/deploy/ansible/roles/helm_install/defaults/main.yml +++ b/deploy/ansible/roles/helm_install/defaults/main.yml @@ -1,5 +1,5 @@ --- -helm_version: v3.13.2 +helm_version: v3.15.2 helm_arch: linux-amd64 helm_download_dir: /opt/helm-{{ helm_version }}-{{ helm_arch }} diff --git a/deploy/ansible/roles/k8s_install/defaults/main.yml b/deploy/ansible/roles/k8s_install/defaults/main.yml index 450ab1fade..a30962b4af 100644 --- a/deploy/ansible/roles/k8s_install/defaults/main.yml +++ b/deploy/ansible/roles/k8s_install/defaults/main.yml @@ -3,6 +3,8 @@ # Can be overridden by specific groups/hosts k8s_dns_name: "{{ combine_server_name }}" +keyring_location: /etc/apt/keyrings + k8s_required_pkgs: - apt-transport-https - ca-certificates @@ -16,6 +18,3 @@ k3s_options: - traefik - --tls-san - "{{ k8s_dns_name }}" - -k3s_version: "v1.25.14+k3s1" -kubectl_version: "v1.29" diff --git a/deploy/ansible/roles/k8s_install/tasks/k3s.yml b/deploy/ansible/roles/k8s_install/tasks/k3s.yml index b605f045fa..800309c317 100644 --- a/deploy/ansible/roles/k8s_install/tasks/k3s.yml +++ b/deploy/ansible/roles/k8s_install/tasks/k3s.yml @@ -21,50 +21,33 @@ notify: - Reload k3s -- name: Get home directory for {{ k8s_user }} - shell: > - getent passwd {{ k8s_user }} | awk -F: '{ print $6 }' - register: k8s_user_home - changed_when: false - -- name: Get user group id for {{ k8s_user }} - shell: > - getent passwd {{ k8s_user }} | awk -F: '{ print $4 }' - register: k8s_user_group_id - changed_when: false - -- name: Create .kube directories +- name: Create keyring directory if necessary file: - path: "{{ item.home }}/.kube" + path: "{{ keyring_location }}" state: directory - owner: "{{ item.owner }}" - group: "{{ item.group }}" - mode: 0700 - loop: - - home: "{{ k8s_user_home.stdout }}" - owner: "{{ k8s_user }}" - group: "{{ k8s_user_group_id.stdout }}" - - home: /root - owner: root - group: root + owner: root + group: root + mode: 0755 -- name: Copy /etc/rancher/k3s/k3s.yaml to .kube/config - shell: | - cp /etc/rancher/k3s/k3s.yaml {{ item.home }}/.kube/config - chown {{ item.owner }}:{{ item.group }} {{ item.home }}/.kube/config - chmod 600 {{ item.home }}/.kube/config - loop: - - home: "{{ k8s_user_home.stdout }}" - owner: "{{ k8s_user }}" - group: "{{ k8s_user_group_id.stdout }}" - - home: /root - owner: root - group: root +- name: Download the Kubernetes public signing key + shell: + cmd: > + curl -fsSL https://pkgs.k8s.io/core:/stable:/v1.29/deb/Release.key + | gpg --dearmor -o {{ keyring_location }}/kubernetes-apt-keyring.gpg + creates: "{{ keyring_location }}/kubernetes-apt-keyring.gpg" + +- name: Set signing key permissions + file: + name: "{{ keyring_location }}/kubernetes-apt-keyring.gpg" + mode: 0644 + state: file -- name: List contexts - command: kubectl --kubeconfig=/etc/rancher/k3s/k3s.yaml config get-contexts - register: k3s_contexts +- name: Add repository + apt_repository: + repo: "deb [signed-by={{ keyring_location }}/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/v1.29/deb/ /" + filename: kubernetes + mode: 0644 -- name: Change context name from 'default' - command: kubectl --kubeconfig=/etc/rancher/k3s/k3s.yaml config rename-context default {{ kubecfgdir }} - when: k3s_contexts.stdout is regex("^\*? +default.*") +- name: Install kubectl + apt: + name: kubectl diff --git a/deploy/ansible/roles/k8s_install/tasks/k3s_airgap.yml b/deploy/ansible/roles/k8s_install/tasks/k3s_airgap.yml new file mode 100644 index 0000000000..cd7e997569 --- /dev/null +++ b/deploy/ansible/roles/k8s_install/tasks/k3s_airgap.yml @@ -0,0 +1,22 @@ +--- +################################################ +# Install the k3s Lightweight Kubernetes Engine +# from Rancher. +# https://k3s.io/ +################################################ +- name: Install k3s + shell: + cmd: INSTALL_K3S_SKIP_DOWNLOAD=true /usr/local/bin/install.sh {{ k3s_options | join(' ') }} + creates: /etc/systemd/system/k3s.service + +# Change KillMode from "process" to "mixed" to eliminate 90s wait for k3s containers +# to exit. This limits the ability to upgrade k3s in-place without stopping the +# current containers but that is not needed for the Combine use case. +- name: Patch k3s service + lineinfile: + path: /etc/systemd/system/k3s.service + regexp: ^KillMode= + state: present + line: KillMode=mixed + notify: + - Reload k3s diff --git a/deploy/ansible/roles/k8s_install/tasks/main.yml b/deploy/ansible/roles/k8s_install/tasks/main.yml index 533e1f6899..13d7b3566a 100644 --- a/deploy/ansible/roles/k8s_install/tasks/main.yml +++ b/deploy/ansible/roles/k8s_install/tasks/main.yml @@ -3,42 +3,65 @@ apt: name: "{{ k8s_required_pkgs }}" -# configure kubernetes user +# Install k3s Kubernetes engine - name: Install Kubernetes Engine include_tasks: - file: "{{ k8s_engine }}.yml" - when: k8s_engine != "none" + file: k3s.yml + when: not install_airgap_images -- name: Create keyring directory if necessary +# Install from airgap images +- name: Install Kubernetes from Airgap Images + include_tasks: + file: k3s_airgap.yml + when: install_airgap_images + +- name: Get home directory for {{ k8s_user }} + shell: > + getent passwd {{ k8s_user }} | awk -F: '{ print $6 }' + register: k8s_user_home + changed_when: false + +- name: Get user group id for {{ k8s_user }} + shell: > + getent passwd {{ k8s_user }} | awk -F: '{ print $4 }' + register: k8s_user_group_id + changed_when: false + +- name: Create .kube directories file: - path: /etc/apt/keyrings + path: "{{ item.home }}/.kube" state: directory - owner: root - group: root - mode: "0755" - -- name: Download the Kubernetes public signing key - shell: - cmd: > - curl -fsSL https://pkgs.k8s.io/core:/stable:/v1.29/deb/Release.key - | gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg - creates: /etc/apt/keyrings/kubernetes-apt-keyring.gpg - -- name: Set signing key permissions - file: - name: /etc/apt/keyrings/kubernetes-apt-keyring.gpg - mode: 0644 - state: file + owner: "{{ item.owner }}" + group: "{{ item.group }}" + mode: 0700 + loop: + - home: "{{ k8s_user_home.stdout }}" + owner: "{{ k8s_user }}" + group: "{{ k8s_user_group_id.stdout }}" + - home: /root + owner: root + group: root -- name: Add repository - apt_repository: - repo: "deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/v1.29/deb/ /" - filename: kubernetes - mode: 0644 +- name: Copy /etc/rancher/k3s/k3s.yaml to .kube/config + shell: | + cp /etc/rancher/k3s/k3s.yaml {{ item.home }}/.kube/config + chown {{ item.owner }}:{{ item.group }} {{ item.home }}/.kube/config + chmod 600 {{ item.home }}/.kube/config + loop: + - home: "{{ k8s_user_home.stdout }}" + owner: "{{ k8s_user }}" + group: "{{ k8s_user_group_id.stdout }}" + - home: /root + owner: root + group: root -- name: Install kubectl - apt: - name: kubectl +- name: List contexts + command: kubectl --kubeconfig=/etc/rancher/k3s/k3s.yaml config get-contexts + register: k3s_contexts + +- name: Change context name from 'default' + command: kubectl --kubeconfig=/etc/rancher/k3s/k3s.yaml config rename-context default {{ kubecfgdir }} + when: k3s_contexts.stdout is regex("^\*? +default.*") - name: Get home directory for {{ k8s_user }} shell: > diff --git a/deploy/ansible/roles/support_tools/defaults/main.yml b/deploy/ansible/roles/support_tools/defaults/main.yml index 76e1a42aca..030da5ade0 100644 --- a/deploy/ansible/roles/support_tools/defaults/main.yml +++ b/deploy/ansible/roles/support_tools/defaults/main.yml @@ -4,5 +4,3 @@ eth_update_program: /usr/local/bin/display-eth-addr install_ip_viewer: no install_combinectl: no - -wifi_interfaces: "{{ ansible_facts.interfaces | select('search', '^wl[op][0-9]+[a-z][a-z0-9]+') }}" diff --git a/deploy/ansible/roles/support_tools/tasks/main.yml b/deploy/ansible/roles/support_tools/tasks/main.yml index 28c4a0aaa9..896e6153f0 100644 --- a/deploy/ansible/roles/support_tools/tasks/main.yml +++ b/deploy/ansible/roles/support_tools/tasks/main.yml @@ -21,15 +21,6 @@ notify: start display eth when: install_ip_viewer -- name: Verify that there is a single WiFi interface - assert: - that: wifi_interfaces|length == 1 - success_msg: "Setup WiFi Interface: {{ wifi_interfaces }}" - fail_msg: | - Only a single WiFi interface is supported. - Found the following interfaces: - {{ ansible_facts.interfaces }} - - name: Install combinectl tool copy: src: combinectl.sh diff --git a/deploy/ansible/roles/wifi_ap/tasks/install.yml b/deploy/ansible/roles/wifi_ap/tasks/install.yml index 1db948b4c7..6880c28d59 100644 --- a/deploy/ansible/roles/wifi_ap/tasks/install.yml +++ b/deploy/ansible/roles/wifi_ap/tasks/install.yml @@ -59,7 +59,7 @@ line: 127.0.0.1 localhost {{ ansible_hostname }} owner: root group: root - mode: "0644" + mode: 0644 - name: Redirect traffic for The Combine to the AP gateway lineinfile: diff --git a/deploy/ansible/vars/config_common.yml b/deploy/ansible/vars/config_common.yml index 41fb44d589..ff7ecfbe5b 100644 --- a/deploy/ansible/vars/config_common.yml +++ b/deploy/ansible/vars/config_common.yml @@ -2,6 +2,9 @@ # Configure logging combine_use_syslog: true +# Configure whether airgap images should be installed on target +install_airgap_images: false + # Kubernetes local Working directories k8s_working_dir: "{{ lookup('env', 'HOME') }}/.kube/{{ kubecfgdir }}" k8s_admin_cfg: "{{ k8s_working_dir }}/admin_user" diff --git a/deploy/ansible/vars/k3s_versions.yml b/deploy/ansible/vars/k3s_versions.yml new file mode 100644 index 0000000000..e3a79b990f --- /dev/null +++ b/deploy/ansible/vars/k3s_versions.yml @@ -0,0 +1,4 @@ +--- +k3s_version: "v1.30.1%2Bk3s1" +kubectl_version: "v1.30.2" +helm_version: "v3.15.2" diff --git a/deploy/helm/cert-proxy-client/values.yaml b/deploy/helm/cert-proxy-client/values.yaml index ea2e8e63de..3cd7fc9de4 100644 --- a/deploy/helm/cert-proxy-client/values.yaml +++ b/deploy/helm/cert-proxy-client/values.yaml @@ -31,7 +31,9 @@ certRenewBefore: "60" imageName: combine_maint envName: env-cert-proxy -schedule: "*/30 * * * *" +# Run once a minute. If the cert is not up for renewal, The Combine +# will not try to reach AWS S3 +schedule: "* * * * *" cert_renew_before: 60 diff --git a/deploy/helm/thecombine/charts/maintenance/templates/get-fonts-hook.yaml b/deploy/helm/thecombine/charts/maintenance/templates/get-fonts-hook.yaml index 0c049fd392..8ca7f66f48 100644 --- a/deploy/helm/thecombine/charts/maintenance/templates/get-fonts-hook.yaml +++ b/deploy/helm/thecombine/charts/maintenance/templates/get-fonts-hook.yaml @@ -14,6 +14,7 @@ metadata: "helm.sh/hook": post-install, post-upgrade "helm.sh/hook-delete-policy": before-hook-creation spec: + backoffLimit: 11 template: metadata: creationTimestamp: null diff --git a/deploy/requirements.txt b/deploy/requirements.txt index e716281504..6f945ef76e 100644 --- a/deploy/requirements.txt +++ b/deploy/requirements.txt @@ -10,7 +10,7 @@ ansible-core==2.17.1 # via ansible cachetools==5.3.3 # via google-auth -certifi==2024.6.2 +certifi==2024.7.4 # via # kubernetes # requests @@ -18,17 +18,17 @@ cffi==1.16.0 # via cryptography charset-normalizer==3.3.2 # via requests -cryptography==42.0.7 +cryptography==42.0.8 # via # ansible-core # pyopenssl -google-auth==2.29.0 +google-auth==2.32.0 # via kubernetes idna==3.7 # via requests jinja2==3.1.4 # via - # -r deploy/requirements.in + # -r requirements.in # ansible-core # jinja2-base64-filters jinja2-base64-filters==0.1.4 @@ -41,7 +41,7 @@ oauthlib==3.2.2 # via # kubernetes # requests-oauthlib -packaging==24.0 +packaging==24.1 # via ansible-core pyasn1==0.6.0 # via @@ -52,12 +52,12 @@ pyasn1-modules==0.4.0 pycparser==2.22 # via cffi pyopenssl==24.1.0 - # via -r deploy/requirements.in + # via -r requirements.in python-dateutil==2.9.0.post0 # via kubernetes pyyaml==6.0.1 # via - # -r deploy/requirements.in + # -r requirements.in # ansible-core # kubernetes requests==2.32.3 diff --git a/deploy/scripts/helm_utils.py b/deploy/scripts/helm_utils.py new file mode 100644 index 0000000000..4a6a8af18d --- /dev/null +++ b/deploy/scripts/helm_utils.py @@ -0,0 +1,111 @@ +"""Utility functions for building a helm command line for maintaining The Combine.""" + +import logging +import os +from pathlib import Path +import sys +from typing import Any, Dict, List, Optional + +from enum_types import ExitStatus +from utils import run_cmd +import yaml + +scripts_dir = Path(__file__).resolve().parent + + +def create_secrets( + secrets: List[Dict[str, str]], *, output_file: Path, env_vars_req: bool +) -> bool: + """ + Create a YAML file that contains the secrets for the specified chart. + + Returns true if one or more secrets were written to output_file. + """ + secrets_written = False + missing_env_vars: List[str] = [] + with open(output_file, "w") as secret_file: + secret_file.write("---\n") + secret_file.write("global:\n") + for item in secrets: + secret_value = os.getenv(item["env_var"]) + if secret_value: + secret_file.write(f' {item["config_item"]}: "{secret_value}"\n') + secrets_written = True + else: + missing_env_vars.append(item["env_var"]) + if len(missing_env_vars) > 0: + logging.debug("The following environment variables are not defined:") + logging.debug(", ".join(missing_env_vars)) + if not env_vars_req: + return secrets_written + sys.exit(ExitStatus.FAILURE.value) + + return secrets_written + + +def get_installed_charts(helm_opts: List[str], helm_namespace: str) -> List[str]: + """Create a list of the helm charts that are already installed on the target.""" + lookup_results = run_cmd(["helm"] + helm_opts + ["list", "-n", helm_namespace, "-o", "yaml"]) + chart_info: List[Dict[str, str]] = yaml.safe_load(lookup_results.stdout) + chart_list: List[str] = [] + for chart in chart_info: + chart_list.append(chart["name"]) + return chart_list + + +def get_target(config: Dict[str, Any]) -> str: + """List available targets and get selection from the user.""" + print("Available targets:") + for key in config["targets"]: + print(f" {key}") + try: + return input("Enter the target name (Ctrl-C to cancel):") + except KeyboardInterrupt: + logging.info("Exiting.") + sys.exit(ExitStatus.FAILURE.value) + + +def add_override_values( + config: Dict[str, Any], *, chart: str, temp_dir: Path, helm_cmd: List[str] +) -> None: + """Add value overrides specified in the script configuration file.""" + if "override" in config and chart in config["override"]: + override_file = temp_dir / f"config_{chart}.yaml" + with open(override_file, "w") as file: + yaml.dump(config["override"][chart], file) + helm_cmd.extend(["-f", str(override_file)]) + + +def add_language_overrides( + config: Dict[str, Any], + *, + chart: str, + langs: Optional[List[str]], +) -> None: + """Update override configuration with any languages specified on the command line.""" + override_config = config["override"][chart] + if langs: + if "maintenance" not in override_config: + override_config["maintenance"] = {"localLangList": langs} + else: + override_config["maintenance"]["localLangList"] = langs + + +def add_profile_values( + config: Dict[str, Any], *, profile_name: str, chart: str, temp_dir: Path, helm_cmd: List[str] +) -> None: + """Add profile specific values for the chart.""" + # lookup the configuration values for the profile of the selected target + # get the path for the profile configuration file + if profile_name in config["profiles"]: + profile_def = scripts_dir / "setup_files" / "profiles" / f"{profile_name}.yaml" + if profile_def.exists(): + with open(profile_def) as file: + profile_values = yaml.safe_load(file) + if chart in profile_values["charts"]: + profile_file = temp_dir / f"profile_{profile_name}_{chart}.yaml" + with open(profile_file, "w") as file: + yaml.dump(profile_values["charts"][chart], file) + helm_cmd.extend(["-f", str(profile_file)]) + else: + print(f"Warning: cannot find profile {profile_name}", file=sys.stderr) diff --git a/deploy/scripts/install-combine.sh b/deploy/scripts/install-combine.sh index 55aed50a26..6237790930 100755 --- a/deploy/scripts/install-combine.sh +++ b/deploy/scripts/install-combine.sh @@ -1,6 +1,29 @@ #! /usr/bin/env bash set -eo pipefail +################################################################################ +# +# install-combine.sh is intended to install the Combine on an Ubuntu-based Linux +# Laptop. Its usage is defined in the readme file that accompanies the packaged +# installer, ./installer/README.md (or ./installer/README.pdf). +# +# Note that 2 additional options are available that are not documented in the +# readme file. These are intended to only be used for debugging and under the +# guidance of a support engineer. They are: +# - single-step - run the next "step" in the installation process and stop. +# - start-at - start at the step named and run to +# completion. +################################################################################# + +# Warning and Error reporting functions +warning () { + echo "WARNING: $1" >&2 +} +error () { + echo "ERROR: $1" >&2 + exit 1 +} + # Set the environment variables that are required by The Combine. # In addition, the values are stored in a file so that they do not # need to be re-entered on subsequent installations. @@ -26,16 +49,25 @@ set-combine-env () { # Create the virtual environment needed by the Python installation # scripts create-python-venv () { - cd $INSTALL_DIR + cd $DEPLOY_DIR # Install required packages sudo apt install -y python3-pip python3-venv ##### - # Setup Python to run ansible - python3 -m venv venv - source venv/bin/activate - python -m pip install --upgrade pip pip-tools - python -m piptools sync requirements.txt + # Setup Python virtual environment + echo "Setting up venv in ${DEPLOY_DIR}" + if [ -f "./venv.tar.gz" ] ; then + tar xzf ./venv.tar.gz + sed -i "s|%%VENV_DIR%%|${DEPLOY_DIR}/venv|g" ${DEPLOY_DIR}/venv/bin/* + source venv/bin/activate + else + python3 -m venv venv + source venv/bin/activate + echo "Install pip and pip-tools" + python -m pip install --upgrade pip pip-tools + echo "Install dependencies" + python -m piptools sync requirements.txt + fi } # Install Kubernetes engine and other supporting @@ -52,9 +84,13 @@ install-kubernetes () { .EOM ##### # Setup Kubernetes environment and WiFi Access Point - cd ${INSTALL_DIR}/ansible + cd ${DEPLOY_DIR}/ansible - ansible-playbook playbook_desktop_setup.yaml -K -e k8s_user=`whoami` + if [ -d "${DEPLOY_DIR}/airgap-images" ] ; then + ansible-playbook playbook_desktop_setup.yaml -K -e k8s_user=`whoami` -e install_airgap_images=true + else + ansible-playbook playbook_desktop_setup.yaml -K -e k8s_user=`whoami` + fi } # Set the KUBECONFIG environment variable so that the cluster can @@ -65,8 +101,7 @@ set-k3s-env () { # Setup kubectl configuration file K3S_CONFIG_FILE=${HOME}/.kube/config if [ ! -e ${K3S_CONFIG_FILE} ] ; then - echo "Kubernetes (k3s) configuration file is missing." >&2 - exit 1 + error "Kubernetes (k3s) configuration file is missing." fi export KUBECONFIG=${K3S_CONFIG_FILE} ##### @@ -78,7 +113,7 @@ set-k3s-env () { # Install the public charts used by The Combine, specifically, cert-manager # and nginx-ingress-controller -install-required-charts () { +install-base-charts () { set-k3s-env ##### # Install base helm charts @@ -87,10 +122,19 @@ install-required-charts () { ##### # Setup required cluster services - cd ${INSTALL_DIR} + cd ${DEPLOY_DIR} . venv/bin/activate - cd ${INSTALL_DIR}/scripts - ./setup_cluster.py + cd ${DEPLOY_DIR}/scripts + if [ -z "${HELM_TIMEOUT}" ] ; then + SETUP_OPTS="" + else + SETUP_OPTS="--timeout ${HELM_TIMEOUT}" + fi + if [ -d "${DEPLOY_DIR}/airgap-charts" ] ; then + ./setup_cluster.py ${SETUP_OPTS} --chart-dir ${DEPLOY_DIR}/airgap-charts + else + ./setup_cluster.py ${SETUP_OPTS} + fi deactivate } @@ -98,12 +142,12 @@ install-required-charts () { install-the-combine () { ##### # Setup The Combine - cd ${INSTALL_DIR} + cd ${DEPLOY_DIR} . venv/bin/activate - cd ${INSTALL_DIR}/scripts + cd ${DEPLOY_DIR}/scripts set-combine-env set-k3s-env - ./setup_combine.py --tag ${COMBINE_VERSION} --repo public.ecr.aws/thecombine --target desktop + ./setup_combine.py --tag ${COMBINE_VERSION} --repo public.ecr.aws/thecombine --target desktop ${SETUP_OPTS} --debug deactivate } @@ -141,12 +185,25 @@ next-state () { fi } +# Verify that the required network devices have been setup +# for Kubernetes cluster +wait-for-k8s-interfaces () { + echo "Waiting for k8s interfaces" + for interface in $@ ; do + while ! ip link show $interface > /dev/null 2>&1 ; do + sleep 1 + done + done + echo "Interfaces ready" +} + ##### # Setup initial variables -INSTALL_DIR=`pwd` +DEPLOY_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )/.." &> /dev/null && pwd ) # Create directory for configuration files CONFIG_DIR=${HOME}/.config/combine mkdir -p ${CONFIG_DIR} +SINGLE_STEP=0 # See if we need to continue from a previous install STATE_FILE=${CONFIG_DIR}/install-state @@ -169,6 +226,17 @@ while (( "$#" )) ; do restart) next-state "Pre-reqs" ;; + single-step) + SINGLE_STEP=1 + ;; + start-at) + next-state $2 + shift + ;; + timeout) + HELM_TIMEOUT=$2 + shift + ;; uninstall) next-state "Uninstall-combine" ;; @@ -179,12 +247,11 @@ while (( "$#" )) ; do if [[ $OPT =~ ^v[0-9]+\.[0-9]+\.[0-9]+(-[a-zA-Z0-9-]+\.[0-9]+)?$ ]] ; then COMBINE_VERSION="$OPT" else - echo "Invalid version number, $OPT" - exit 1 + error "Invalid version number, $OPT" fi ;; *) - echo "Unrecognized option: $OPT" >&2 + warning "Unrecognized option: $OPT" ;; esac shift @@ -192,8 +259,7 @@ done # Check that we have a COMBINE_VERSION if [ -z "${COMBINE_VERSION}" ] ; then - echo "Combine version is not specified." - exit 1 + error "Combine version is not specified." fi create-python-venv @@ -202,13 +268,14 @@ while [ "$STATE" != "Done" ] ; do case $STATE in Pre-reqs) install-kubernetes + wait-for-k8s-interfaces flannel.1 cni0 next-state "Restart" ;; Restart) next-state "Base-charts" if [ -f /var/run/reboot-required ] ; then echo -e "***** Restart required *****\n" - echo -e "Rerun combine-installer.run after the system has been restarted.\n" + echo -e "Rerun combine installer after the system has been restarted.\n" read -p "Restart now? (Y/n) " RESTART if [[ -z $RESTART || $RESTART =~ ^[yY].* ]] ; then sudo reboot @@ -219,14 +286,23 @@ while [ "$STATE" != "Done" ] ; do STATE=Done fi fi + if [ "$SINGLE_STEP" == "1" ] ; then + STATE=Done + fi ;; Base-charts) - install-required-charts + install-base-charts next-state "Install-combine" + if [ "$SINGLE_STEP" == "1" ] ; then + STATE=Done + fi ;; Install-combine) install-the-combine next-state "Wait-for-combine" + if [ "$SINGLE_STEP" == "1" ] ; then + STATE=Done + fi ;; Wait-for-combine) # Wait until all the combine deployments are up @@ -247,13 +323,12 @@ while [ "$STATE" != "Done" ] ; do next-state "Done" ;; Uninstall-combine) - ${INSTALL_DIR}/scripts/uninstall-combine + ${DEPLOY_DIR}/scripts/uninstall-combine next-state "Done" ;; *) - echo "Unrecognized STATE: ${STATE}" rm ${STATE_FILE} - exit 1 + error "Unrecognized STATE: ${STATE}" ;; esac done diff --git a/deploy/scripts/kube_env.py b/deploy/scripts/kube_env.py index e1c79985a6..2e70824f56 100755 --- a/deploy/scripts/kube_env.py +++ b/deploy/scripts/kube_env.py @@ -10,7 +10,7 @@ class KubernetesEnvironment: - def __init__(self, args: argparse.Namespace) -> None: + def __init__(self, args: argparse.Namespace, *, prompt_for_context: bool = True) -> None: if "kubeconfig" in args and args.kubeconfig is not None: self.kubeconfig = args.kubeconfig else: @@ -18,7 +18,7 @@ def __init__(self, args: argparse.Namespace) -> None: if "context" in args and args.context is not None: # if the user specified a context, use that one. self.kubecontext = args.context - else: + elif prompt_for_context: context_list: List[str] = [] result = run_cmd( @@ -40,6 +40,8 @@ def __init__(self, args: argparse.Namespace) -> None: self.kubecontext = curr_context else: self.kubecontext = None + else: + self.kubecontext = None if "debug" in args: self.debug = args.debug else: @@ -73,6 +75,43 @@ def get_kubectl_opts(self) -> List[str]: return kubectl_opts +def add_helm_opts(parser: argparse.ArgumentParser) -> None: + """Add command line arguments for Helm.""" + parser.add_argument( + "--dry-run", + action="store_true", + help="Invoke the 'helm install' command with the '--dry-run' option.", + dest="dry_run", + ) + parser.add_argument( + "--wait", + action="store_true", + help="Invoke the 'helm install' command with the '--wait' option.", + ) + parser.add_argument( + "--timeout", + help=""" + Maximum time to wait for the helm commands. + Adds the '--wait' option if not specified in configuration. + TIMEOUT is specified as a Go Time Duration. See https://pkg.go.dev/time#ParseDuration. + """, + ) + # Arguments passed to the helm install command + parser.add_argument( + "--set", # matches a 'helm install' option + nargs="+", + help="Specify additional Helm configuration variable to override default values." + " See `helm install --help`", + ) + parser.add_argument( + "--values", + "-f", # matches a 'helm install' option + nargs="+", + help="Specify additional Helm configuration file to override default values." + " See `helm install --help`", + ) + + def add_kube_opts(parser: argparse.ArgumentParser) -> None: """Add commandline arguments for Kubernetes tools.""" parser.add_argument( diff --git a/deploy/scripts/package_images.py b/deploy/scripts/package_images.py new file mode 100755 index 0000000000..5df10cac8f --- /dev/null +++ b/deploy/scripts/package_images.py @@ -0,0 +1,187 @@ +#! /usr/bin/env python3 + +""" +Package the container images used for The Combine to support air-gapped installation. + +The package_images.py script uses the `helm template` command to print the rendered +helm templates for the middleware used by The Combine and for The Combine itself. The +image names are extracted from the templates and then pulled from the repo and stored +in ../images as compressed tarballs; zstd compression is used. +""" +import argparse +import logging +import os +from pathlib import Path +import re +from typing import Any, Dict, List + +from utils import init_logging, run_cmd +import yaml + +# Define configuration and output directories' +scripts_dir = Path(__file__).resolve().parent +ansible_dir = scripts_dir.parent / "ansible" +helm_dir = scripts_dir.parent / "helm" + + +def parse_args() -> argparse.Namespace: + """Parse user command line arguments.""" + parser = argparse.ArgumentParser( + description="Package container images for The Combine.", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + # Add Required arguments + parser.add_argument( + "tag", + help="Tag for the container images to be installed for The Combine.", + ) + parser.add_argument("output_dir", help="Directory for the collected image files.") + # Add Optional arguments + parser.add_argument( + "--config", + "-c", + help="Configuration file for the cluster type(s).", + default=str(scripts_dir / "setup_files" / "cluster_config.yaml"), + ) + parser.add_argument( + "--debug", + action="store_true", + help="Enable debugging output.", + ) + parser.add_argument( + "--quiet", + "-q", + action="store_true", + help="Print less output information.", + ) + return parser.parse_args() + + +def package_k3s(dest_dir: Path) -> None: + logging.info("Packaging k3s images.") + run_cmd( + [ + "ansible-playbook", + "playbook_k3s_airgapped_files.yml", + "--extra-vars", + f"package_dir={dest_dir}", + ], + cwd=str(ansible_dir), + ) + + +def package_images(image_list: List[str], tar_file: Path) -> None: + container_cli = [os.getenv("CONTAINER_CLI", "docker")] + if container_cli[0] == "nerdctl": + container_cli.extend(["--namespace", "k8s.io"]) + # Pull each image + for image in image_list: + pull_cmd = container_cli + ["pull", image] + logging.debug(f"Running {pull_cmd}") + run_cmd(pull_cmd) + # Save pulled images into a .tar archive + run_cmd(container_cli + ["save"] + image_list + ["-o", str(tar_file)]) + # Compress the tarball + run_cmd(["zstd", "--rm", "--force", "--quiet", str(tar_file)]) + + +def package_middleware( + config_file: str, *, cluster_type: str, image_dir: Path, chart_dir: Path +) -> None: + logging.info("Packaging middleware images.") + # read in cluster configuration + with open(config_file) as file: + config: Dict[str, Any] = yaml.safe_load(file) + # get current repos + curr_repo_list: List[str] = [] + middleware_images: List[str] = [] + helm_cmd_results = run_cmd( + ["helm", "repo", "list", "-o", "yaml"], print_cmd=False, check_results=False + ) + if helm_cmd_results.returncode == 0: + curr_helm_repos = yaml.safe_load(helm_cmd_results.stdout) + for repo in curr_helm_repos: + curr_repo_list.append(repo["name"]) + + for chart_descr in config["clusters"][cluster_type]: + # add the chart's repo if we don't already have it + repo = config[chart_descr]["repo"] + if repo["name"] not in curr_helm_repos: + run_cmd(["helm", "repo", "add", repo["name"], repo["url"]]) + curr_repo_list.append(repo["name"]) + + # pull the middleware chart + chart = config[chart_descr]["chart"] + dest_dir = chart_dir / chart["name"] + dest_dir.mkdir(mode=0o755, parents=True, exist_ok=True) + helm_cmd = ["helm", "pull", chart["reference"], "--destination", str(dest_dir)] + if "version" in chart: + helm_cmd.extend(["--version", chart["version"]]) + run_cmd(helm_cmd) + # render chart templates and extract images + for chart_file in dest_dir.glob("*.tgz"): + results = run_cmd(["helm", "template", chart_file]) + for line in results.stdout.splitlines(): + match = re.match(r'[-\s]+image:\s+"*([^"\n]*)"*', line) + if match: + logging.debug(f" - Found image {match.group(1)}") + middleware_images.append(match.group(1)) + logging.debug(f"Middleware images: {middleware_images}") + package_images(middleware_images, image_dir / "middleware-airgap-images-amd64.tar") + + +def package_thecombine(tag: str, image_dir: Path) -> None: + logging.info(f"Packaging The Combine version {tag}.") + logging.debug(" - Get template for The Combine.") + results = run_cmd( + [ + "helm", + "template", + "thecombine", + str(helm_dir / "thecombine"), + "--set", + "global.imageRegistry=public.ecr.aws/thecombine", + "--set", + f"global.imageTag={tag}", + ] + ) + combine_images: List[str] = [] + for line in results.stdout.splitlines(): + match = re.match(r'^[-\s]+image:\s+"*([^"\n]*)"*', line) + if match: + image = match.group(1) + logging.debug(f" - Found image {image}") + if image not in combine_images: + combine_images.append(image) + logging.debug(f"Combine images: {combine_images}") + # Logout of AWS to allow pulling the images + package_images(combine_images, image_dir / "combine-airgap-images-amd64.tar") + + +def main() -> None: + args = parse_args() + + init_logging(args) + + output_dir = Path(args.output_dir).resolve() + image_dir = output_dir / "airgap-images" + image_dir.mkdir(mode=0o755, parents=True, exist_ok=True) + chart_dir = output_dir / "airgap-charts" + chart_dir.mkdir(mode=0o755, parents=True, exist_ok=True) + + # Clear the AWS variables so that they don't end up in the installer + os.environ["AWS_ACCESS_KEY_ID"] = "" + os.environ["AWS_SECRET_ACCESS_KEY"] = "" + os.environ["AWS_ACCOUNT"] = "" + os.environ["AWS_DEFAULT_REGION"] = "" + + # Update helm repos + package_k3s(image_dir) + package_middleware( + args.config, cluster_type="standard", image_dir=image_dir, chart_dir=chart_dir + ) + package_thecombine(args.tag, image_dir) + + +if __name__ == "__main__": + main() diff --git a/deploy/scripts/setup_cluster.py b/deploy/scripts/setup_cluster.py index 38817d2541..aee6ef1af4 100755 --- a/deploy/scripts/setup_cluster.py +++ b/deploy/scripts/setup_cluster.py @@ -4,6 +4,7 @@ from __future__ import annotations import argparse +import logging import os from pathlib import Path import sys @@ -11,8 +12,8 @@ from typing import Any, Dict, List from enum_types import ExitStatus, HelmAction -from kube_env import KubernetesEnvironment, add_kube_opts -from utils import add_namespace, run_cmd +from kube_env import KubernetesEnvironment, add_helm_opts, add_kube_opts +from utils import init_logging, run_cmd import yaml scripts_dir = Path(__file__).resolve().parent @@ -26,11 +27,9 @@ def parse_args() -> argparse.Namespace: formatter_class=argparse.ArgumentDefaultsHelpFormatter, ) add_kube_opts(parser) + add_helm_opts(parser) parser.add_argument( - "--type", - "-t", - default="standard", - help="Type of Kubernetes cluster to be setup as defined in the config file.", + "--chart-dir", help="Directory for the chart files when doing an airgap installation." ) parser.add_argument( "--config", @@ -44,46 +43,57 @@ def parse_args() -> argparse.Namespace: action="store_true", help="Print less output information.", ) + parser.add_argument( + "--type", + "-t", + default="standard", + help="Type of Kubernetes cluster to be setup as defined in the config file.", + ) return parser.parse_args() def main() -> None: """Install pre-requisite helm charts.""" args = parse_args() + init_logging(args) + with open(args.config) as file: config: Dict[str, Any] = yaml.safe_load(file) # Verify that the requested type is in the configuration if args.type not in config["clusters"]: - print( - f"Cluster type '{args.type}' is not in the configuration file, {args.config}", - file=sys.stderr, + logging.error( + f"Cluster type '{args.type}' is not in the configuration file, {args.config}" ) sys.exit(ExitStatus.FAILURE.value) # Note that the helm repo commands affect the helm client and therefore # are not affected by the helm options this_cluster: List[str] = config["clusters"][args.type] - curr_repo_list: List[str] = [] - helm_cmd_results = run_cmd( - ["helm", "repo", "list", "-o", "yaml"], print_cmd=not args.quiet, check_results=False - ) - if helm_cmd_results.returncode == 0: - curr_helm_repos = yaml.safe_load(helm_cmd_results.stdout) - for repo in curr_helm_repos: - curr_repo_list.append(repo["name"]) - # Check for repos that need to be added - for chart_descr in this_cluster: - repo_spec = config[chart_descr]["repo"] - if repo_spec["name"] not in curr_repo_list: - run_cmd( - ["helm", "repo", "add", repo_spec["name"], repo_spec["url"]], - print_cmd=not args.quiet, - print_output=not args.quiet, - ) - # Update the helm repos with added repos and to update chart versions in - # existing repos. - run_cmd(["helm", "repo", "update"], print_cmd=not args.quiet, print_output=not args.quiet) + + # if the chart is to be installed from a file, we don't need to + # add the repo + if args.chart_dir is None: + curr_repo_list: List[str] = [] + helm_cmd_results = run_cmd( + ["helm", "repo", "list", "-o", "yaml"], print_cmd=not args.quiet, check_results=False + ) + if helm_cmd_results.returncode == 0: + curr_helm_repos = yaml.safe_load(helm_cmd_results.stdout) + for repo in curr_helm_repos: + curr_repo_list.append(repo["name"]) + # Check for repos that need to be added + for chart_descr in this_cluster: + repo_spec = config[chart_descr]["repo"] + if repo_spec["name"] not in curr_repo_list: + run_cmd( + ["helm", "repo", "add", repo_spec["name"], repo_spec["url"]], + print_cmd=not args.quiet, + print_output=not args.quiet, + ) + # Update the helm repos with added repos and to update chart versions in + # existing repos. + run_cmd(["helm", "repo", "update"], print_cmd=not args.quiet, print_output=not args.quiet) # List current charts chart_list_results = run_cmd(["helm", "list", "-A", "-o", "yaml"]) @@ -93,11 +103,9 @@ def main() -> None: # Verify the Kubernetes/Helm environment kube_env = KubernetesEnvironment(args) - # Install the required charts + # Install/upgrade the required charts for chart_descr in this_cluster: chart_spec = config[chart_descr]["chart"] - # add namespace if needed - add_namespace(chart_spec["namespace"], kube_env.get_kubectl_opts()) # install the chart helm_cmd = ["helm"] + kube_env.get_helm_opts() if chart_spec["name"] in curr_charts: @@ -110,13 +118,37 @@ def main() -> None: chart_spec["namespace"], helm_action.value, chart_spec["name"], - chart_spec["reference"], ] ) - if "version" in chart_spec: - helm_cmd.extend(["--version", chart_spec["version"]]) - if "wait" in chart_spec and chart_spec["wait"]: + if args.chart_dir is None: + # chart is found in the repo + helm_cmd.extend( + [ + chart_spec["reference"], + ] + ) + if "version" in chart_spec: + helm_cmd.extend(["--version", chart_spec["version"]]) + else: + # chart is a *.tgz file + chart_files = list((Path(args.chart_dir).resolve() / chart_spec["name"]).glob("*.tgz")) + if not chart_files: + logging.error(f"No chart file for {chart['name']} in {args.chart_dir}.") + sys.exit(1) + if len(chart_files) > 1: + logging.warning( + f"Expecting 1 chart file for {chart['name']}, found {len(chart_files)}" + ) + helm_cmd.append(str(chart_files[0])) + + if helm_action == HelmAction.INSTALL: + helm_cmd.append("--create-namespace") + if ("wait" in chart_spec and chart_spec["wait"]) or args.timeout is not None: helm_cmd.append("--wait") + if args.timeout is not None: + helm_cmd.extend(["--timeout", args.timeout]) + if args.dry_run: + helm_cmd.append("--dry-run") with tempfile.TemporaryDirectory() as temp_dir: if "override" in chart_spec: override_file = Path(temp_dir).resolve() / "overrides.yaml" @@ -124,11 +156,16 @@ def main() -> None: yaml.dump(chart_spec["override"], file) helm_cmd.extend(["-f", str(override_file)]) helm_cmd_str = " ".join(helm_cmd) - if not args.quiet: - print(f"Running: {helm_cmd_str}") + logging.info(f"Running: {helm_cmd_str}") # Run with os.system so that there is feedback on stdout/stderr while the # command is running - os.system(helm_cmd_str) + exit_status = os.waitstatus_to_exitcode(os.system(helm_cmd_str)) + logging.info( + f'helm {helm_action.value} of {chart_spec["name"]} ' + + f"returned exit status {hex(exit_status)}" + ) + if exit_status != 0: + sys.exit(exit_status) if __name__ == "__main__": diff --git a/deploy/scripts/setup_combine.py b/deploy/scripts/setup_combine.py index bf3d532377..efb8c2c1f2 100755 --- a/deploy/scripts/setup_combine.py +++ b/deploy/scripts/setup_combine.py @@ -19,19 +19,25 @@ The script also adds value definitions from a profile specific configuration file if it exists. """ import argparse -import logging -import os from pathlib import Path import sys import tempfile -from typing import Any, Dict, List, Optional +from typing import Any, Dict, List from app_release import get_release from aws_env import init_aws_environment import combine_charts from enum_types import ExitStatus, HelmAction -from kube_env import KubernetesEnvironment, add_kube_opts -from utils import add_namespace, run_cmd +from helm_utils import ( + add_language_overrides, + add_override_values, + add_profile_values, + create_secrets, + get_installed_charts, + get_target, +) +from kube_env import KubernetesEnvironment, add_helm_opts, add_kube_opts +from utils import add_namespace, init_logging, run_cmd import yaml scripts_dir = Path(__file__).resolve().parent @@ -46,6 +52,8 @@ def parse_args() -> argparse.Namespace: ) # Arguments used by the Kubernetes tools add_kube_opts(parser) + # Arguments used by Helm + add_helm_opts(parser) # Arguments specific to setting up The Combine parser.add_argument( "--clean", action="store_true", help="Delete chart, if it exists, before installing." @@ -56,12 +64,6 @@ def parse_args() -> argparse.Namespace: help="Configuration file for the target(s).", default=str(scripts_dir / "setup_files" / "combine_config.yaml"), ) - parser.add_argument( - "--dry-run", - action="store_true", - help="Invoke the 'helm install' command with the '--dry-run' option.", - dest="dry_run", - ) parser.add_argument( "--langs", "-l", @@ -74,11 +76,6 @@ def parse_args() -> argparse.Namespace: action="store_true", help="List the available targets and exit.", ) - parser.add_argument( - "--wait", - action="store_true", - help="Invoke the 'helm install' command with the '--wait' option.", - ) parser.add_argument( "--profile", "-p", @@ -91,9 +88,7 @@ def parse_args() -> argparse.Namespace: action="store_true", help="Print less output information.", ) - parser.add_argument( - "--repo", "-r", help="Pull images from the specified Docker image repository." - ) + parser.add_argument("--repo", "-r", help="Pull images from the specified image repository.") parser.add_argument( "--tag", "-t", @@ -106,133 +101,12 @@ def parse_args() -> argparse.Namespace: default="localhost", help="Target system where The Combine is to be installed.", ) - # Arguments passed to the helm install command - parser.add_argument( - "--set", # matches a 'helm install' option - nargs="+", - help="Specify additional Helm configuration variable to override default values." - " See `helm install --help`", - ) - parser.add_argument( - "--values", - "-f", # matches a 'helm install' option - nargs="+", - help="Specify additional Helm configuration file to override default values." - " See `helm install --help`", - ) return parser.parse_args() -def create_secrets( - secrets: List[Dict[str, str]], *, output_file: Path, env_vars_req: bool -) -> bool: - """ - Create a YAML file that contains the secrets for the specified chart. - - Returns true if one or more secrets were written to output_file. - """ - secrets_written = False - missing_env_vars: List[str] = [] - with open(output_file, "w") as secret_file: - secret_file.write("---\n") - secret_file.write("global:\n") - for item in secrets: - secret_value = os.getenv(item["env_var"]) - if secret_value: - secret_file.write(f' {item["config_item"]}: "{secret_value}"\n') - secrets_written = True - else: - missing_env_vars.append(item["env_var"]) - if len(missing_env_vars) > 0: - logging.debug("The following environment variables are not defined:") - logging.debug(", ".join(missing_env_vars)) - if not env_vars_req: - return secrets_written - sys.exit(ExitStatus.FAILURE.value) - - return secrets_written - - -def get_installed_charts(helm_opts: List[str], helm_namespace: str) -> List[str]: - """Create a list of the helm charts that are already installed on the target.""" - lookup_results = run_cmd(["helm"] + helm_opts + ["list", "-n", helm_namespace, "-o", "yaml"]) - chart_info: List[Dict[str, str]] = yaml.safe_load(lookup_results.stdout) - chart_list: List[str] = [] - for chart in chart_info: - chart_list.append(chart["name"]) - return chart_list - - -def get_target(config: Dict[str, Any]) -> str: - """List available targets and get selection from the user.""" - print("Available targets:") - for key in config["targets"]: - print(f" {key}") - try: - return input("Enter the target name (Ctrl-C to cancel):") - except KeyboardInterrupt: - logging.info("Exiting.") - sys.exit(ExitStatus.FAILURE.value) - - -def add_override_values( - config: Dict[str, Any], *, chart: str, temp_dir: Path, helm_cmd: List[str] -) -> None: - """Add value overrides specified in the script configuration file.""" - if "override" in config and chart in config["override"]: - override_file = temp_dir / f"config_{chart}.yaml" - with open(override_file, "w") as file: - yaml.dump(config["override"][chart], file) - helm_cmd.extend(["-f", str(override_file)]) - - -def add_language_overrides( - config: Dict[str, Any], - *, - chart: str, - langs: Optional[List[str]], -) -> None: - """Update override configuration with any languages specified on the command line.""" - override_config = config["override"][chart] - if langs: - if "maintenance" not in override_config: - override_config["maintenance"] = {"localLangList": langs} - else: - override_config["maintenance"]["localLangList"] = langs - - -def add_profile_values( - config: Dict[str, Any], *, profile_name: str, chart: str, temp_dir: Path, helm_cmd: List[str] -) -> None: - """Add profile specific values for the chart.""" - # lookup the configuration values for the profile of the selected target - # get the path for the profile configuration file - if profile_name in config["profiles"]: - profile_def = scripts_dir / "setup_files" / "profiles" / f"{profile_name}.yaml" - if profile_def.exists(): - with open(profile_def) as file: - profile_values = yaml.safe_load(file) - if chart in profile_values["charts"]: - profile_file = temp_dir / f"profile_{profile_name}_{chart}.yaml" - with open(profile_file, "w") as file: - yaml.dump(profile_values["charts"][chart], file) - helm_cmd.extend(["-f", str(profile_file)]) - else: - print(f"Warning: cannot find profile {profile_name}", file=sys.stderr) - - def main() -> None: args = parse_args() - - # Setup the logging level. The command output will be printed on stdout/stderr - # independent of the logging facility - if args.debug: - log_level = logging.DEBUG - elif args.quiet: - log_level = logging.WARNING - else: - log_level = logging.INFO - logging.basicConfig(format="%(levelname)s:%(message)s", level=log_level) + init_logging(args) # Lookup the cluster configuration with open(args.config) as file: @@ -337,8 +211,12 @@ def main() -> None: # set the dry-run option if desired if args.dry_run: helm_install_cmd.append("--dry-run") - if args.wait: + + # set wait and timeout options + if args.wait or args.timeout is not None: helm_install_cmd.append("--wait") + if args.timeout is not None: + helm_install_cmd.extend(["--timeout", args.timeout]) # add the profile specific configuration add_profile_values( diff --git a/deploy/scripts/setup_files/cluster_config.yaml b/deploy/scripts/setup_files/cluster_config.yaml index 13797df35b..6f383e1103 100644 --- a/deploy/scripts/setup_files/cluster_config.yaml +++ b/deploy/scripts/setup_files/cluster_config.yaml @@ -1,10 +1,16 @@ # Define the charts that need to be installed for each cluster type clusters: standard: + - nginx-ingress-controller + development: - cert-manager - nginx-ingress-controller rancher: - rancher-ui + cert-manager: + - cert-manager + ingress: + - nginx-ingress-controller # Specify how each chart is to be installed. The "repo" key specified which # helm repository needs to be added and the "chart" key specifies how to diff --git a/deploy/scripts/setup_files/profiles/dev.yaml b/deploy/scripts/setup_files/profiles/dev.yaml index ae4b78a03f..d965ecd1e9 100644 --- a/deploy/scripts/setup_files/profiles/dev.yaml +++ b/deploy/scripts/setup_files/profiles/dev.yaml @@ -18,14 +18,14 @@ charts: global: imageRegistry: "" - imagePullPolicy: Never + imagePullPolicy: IfNotPresent includeResourceLimits: false awsS3Location: dev.thecombine.app captchaRequired: true emailEnabled: true ingressClass: nginx - imagePullPolicy: Never + imagePullPolicy: IfNotPresent certManager: enabled: true diff --git a/deploy/scripts/utils.py b/deploy/scripts/utils.py index e316b3f13f..fed4be3582 100644 --- a/deploy/scripts/utils.py +++ b/deploy/scripts/utils.py @@ -4,6 +4,8 @@ from __future__ import annotations +import argparse +import logging import subprocess import sys from typing import List, Optional @@ -101,3 +103,15 @@ def choose_from_list( curr_selection = None print(f"{reply} is not in the list. Please re-enter.") return curr_selection + + +def init_logging(args: argparse.Namespace) -> None: + # Setup the logging level. The command output will be printed on stdout/stderr + # independent of the logging facility + if args.debug: + log_level = logging.DEBUG + elif args.quiet: + log_level = logging.WARNING + else: + log_level = logging.INFO + logging.basicConfig(format="%(levelname)s:%(message)s", level=log_level) diff --git a/dev-requirements.txt b/dev-requirements.txt index caf287fb13..aa594ce16b 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -18,7 +18,7 @@ cachetools==5.3.3 # via # google-auth # tox -certifi==2024.6.2 +certifi==2024.7.4 # via # kubernetes # requests @@ -37,7 +37,7 @@ colorama==0.4.6 # -r dev-requirements.in # mkdocs-material # tox -cryptography==42.0.7 +cryptography==42.0.8 # via # pyopenssl # types-pyopenssl @@ -47,11 +47,11 @@ dnspython==2.6.1 # via pymongo eradicate==2.3.0 # via flake8-eradicate -filelock==3.14.0 +filelock==3.15.4 # via # tox # virtualenv -flake8==7.0.0 +flake8==7.1.0 # via # -r dev-requirements.in # flake8-broken-line @@ -63,13 +63,13 @@ flake8-broken-line==1.0.0 # via -r dev-requirements.in flake8-bugbear==24.4.26 # via -r dev-requirements.in -flake8-comprehensions==3.14.0 +flake8-comprehensions==3.15.0 # via -r dev-requirements.in flake8-eradicate==1.5.0 # via -r dev-requirements.in ghp-import==2.1.0 # via mkdocs -google-auth==2.29.0 +google-auth==2.32.0 # via kubernetes humanfriendly==10.0 # via -r dev-requirements.in @@ -85,7 +85,7 @@ jinja2==3.1.4 # mkdocs-material jinja2-base64-filters==0.1.4 # via -r dev-requirements.in -kubernetes==29.0.0 +kubernetes==30.1.0 # via -r dev-requirements.in markdown==3.6 # via @@ -112,13 +112,13 @@ mkdocs-get-deps==0.2.0 # via mkdocs mkdocs-htmlproofer-plugin==1.2.1 # via -r dev-requirements.in -mkdocs-material==9.5.25 +mkdocs-material==9.5.28 # via -r dev-requirements.in mkdocs-material-extensions==1.3.1 # via mkdocs-material mkdocs-static-i18n==1.2.3 # via -r dev-requirements.in -mypy==1.10.0 +mypy==1.10.1 # via -r dev-requirements.in mypy-extensions==1.0.0 # via @@ -128,7 +128,7 @@ oauthlib==3.2.2 # via # kubernetes # requests-oauthlib -packaging==24.0 +packaging==24.1 # via # black # mkdocs @@ -156,7 +156,7 @@ pyasn1==0.6.0 # rsa pyasn1-modules==0.4.0 # via google-auth -pycodestyle==2.11.1 +pycodestyle==2.12.0 # via flake8 pycparser==2.22 # via cffi @@ -166,11 +166,11 @@ pygments==2.18.0 # via mkdocs-material pymdown-extensions==10.8.1 # via mkdocs-material -pymongo==4.7.2 +pymongo==4.8.0 # via -r dev-requirements.in pyopenssl==24.1.0 # via -r dev-requirements.in -pyproject-api==1.6.1 +pyproject-api==1.7.1 # via tox pyreadline3==3.4.1 # via -r dev-requirements.in @@ -212,7 +212,7 @@ tomli==2.0.1 # mypy # pyproject-api # tox -tox==4.15.0 +tox==4.16.0 # via -r dev-requirements.in types-cffi==1.16.0.20240331 # via types-pyopenssl @@ -222,11 +222,11 @@ types-python-dateutil==2.9.0.20240316 # via -r dev-requirements.in types-pyyaml==6.0.12.20240311 # via -r dev-requirements.in -types-requests==2.32.0.20240602 +types-requests==2.32.0.20240622 # via -r dev-requirements.in -types-setuptools==70.0.0.20240524 +types-setuptools==70.2.0.20240704 # via types-cffi -typing-extensions==4.12.1 +typing-extensions==4.12.2 # via # black # mypy @@ -235,7 +235,7 @@ urllib3==2.2.2 # kubernetes # requests # types-requests -virtualenv==20.26.2 +virtualenv==20.26.3 # via tox watchdog==4.0.1 # via mkdocs diff --git a/installer/README.md b/installer/README.md index 8096138605..5e94689c23 100644 --- a/installer/README.md +++ b/installer/README.md @@ -27,11 +27,36 @@ The installation script has been tested on _Ubuntu 22.04_ and _Wasta Linux 22.04 2. Make sure WiFi is "on"; it does not need to be connected to a network. 3. Update all of the existing software packages through your OS's _Software Updater_ application or by running: - ```bash + ```console sudo apt update && sudo apt upgrade -y ``` - This step is optional but will make the installation process go more smoothly. Restart the PC if requested. + This step is optional but will make the installation process go more smoothly. Restart the PC. + + _Note for Wasta Linux users_ + + _Wasta Linux_ includes Skype in its list of available software. Skype no longer supports installing it on Linux from + an `apt` software repository. As a result, when the installation script, or a user, updates the list of available + software, the process fails. To address this issue, you can either: + + 1. Remove the file directly: + + ```console + sudo rm /etc/apt/sources.list.d/skype-stable.list + sudo apt update + ``` + + or + + 2. Deselect _Skype_ in the Software Updater settings + + 1. Open the _Software Settings_ application + 2. Click the _Other Software_ tab + 3. Uncheck the entry for Skype (`https://repo.skype.com/deb stable`) + 4. Click the "Close" button + 5. Click the "Reload" button in the dialog window that is displayed + + Skype is available on _Wasta Linux_ or _Ubuntu_ as a Snap package. 4. Download the installation script from [https://s3.amazonaws.com/software.thecombine.app/combine-installer.run](https://s3.amazonaws.com/software.thecombine.app/combine-installer.run) @@ -144,13 +169,14 @@ To run `combine-installer.run` with options, the option list must be started wit `combine-installer.run` supports the following options: -| option | description | -| -------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -| clean | Remove the previously saved environment (AWS Access Key, admin user info) before performing the installation. | -| restart | Run the installation from the beginning; do not resume a previous installation. | -| uninstall | Remove software installed by this script. | -| update | Update _The Combine_ to the version number provided. This skips installing the support software that was installed previously. | -| version-number | Specify a version to install instead of the current version. A version number will have the form `vn.n.n` where `n` represents an integer value, for example, `v1.20.0`. | +| option | description | +| --------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| clean | Remove the previously saved environment (AWS Access Key, admin user info) before performing the installation. | +| restart | Run the installation from the beginning; do not resume a previous installation. | +| timeout TIMEOUT | Use a different timeout when installing. The default timeout is 5 minutes. With slow internet connections, it is helpful to extend the timeout. See for timeout formats. | +| uninstall | Remove software installed by this script. | +| update | Update _The Combine_ to the version number provided. This skips installing the support software that was installed previously. | +| version-number | Specify a version to install instead of the current version. A version number will have the form `vn.n.n` where `n` represents an integer value, for example, `v1.20.0`. | ### Examples diff --git a/installer/make-combine-installer.sh b/installer/make-combine-installer.sh index 81ad3fdfdd..382d8add94 100755 --- a/installer/make-combine-installer.sh +++ b/installer/make-combine-installer.sh @@ -1,10 +1,78 @@ #! /usr/bin/env bash -if [[ $# -gt 0 ]] ; then - COMBINE_VERSION=$1 -fi -if [ -z "${COMBINE_VERSION}" ] ; then - echo "COMBINE_VERSION is not set." +# Warning and Error reporting functions +warning () { + echo "WARNING: $1" >&2 +} +error () { + echo "ERROR: $1" >&2 exit 1 +} + +# cd to the directory where the script is installed +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) + +NET_INSTALL=0 +# Parse arguments to customize installation +while (( "$#" )) ; do + OPT=$1 + case $OPT in + --net-install) + NET_INSTALL=1 + ;; + v*) + if [[ $OPT =~ ^v[0-9]+\.[0-9]+\.[0-9]+(-[a-zA-Z0-9-]+\.[0-9]+)?$ ]] ; then + COMBINE_VERSION="$OPT" + else + error "Invalid version number, $OPT" + fi + ;; + *) + warning "Unrecognized option: $OPT" + ;; + esac + shift +done + +if [ -z "${COMBINE_VERSION}" ] ; then + error "COMBINE_VERSION is not set." +fi +# setup Python virtual environment +cd ../deploy + +if [[ $NET_INSTALL == 0 ]] ; then + if [ ! -f venv/bin/activate ] ; then + # virtual environment does not exist - create it + python3 -m venv venv + fi + source venv/bin/activate + # update the environment if necessary + python -m pip install --upgrade pip pip-tools + python -m piptools sync requirements.txt + + # Package the Combine for "offline" installation + TEMP_DIR=/tmp/images-$$ + pushd scripts + ./package_images.py ${COMBINE_VERSION} ${TEMP_DIR} + INSTALLER_NAME="combine-installer.run" + popd + # create tarball for venv + # + # replace the current directory in the venv files with a string + # that can be used to relocate the venv + VENV_DIR=`pwd`/venv + echo "VENV_DIR == ${VENV_DIR}" + sed -i "s|${VENV_DIR}|%%VENV_DIR%%|g" venv/bin/* + tar czf ${TEMP_DIR}/venv.tar.gz venv + rm -rf venv +else + # Package the Combine for network installation + INSTALLER_NAME="combine-net-installer.run" +fi + +cd ${SCRIPT_DIR} +makeself --tar-quietly ../deploy ${INSTALLER_NAME} "Combine Installer" scripts/install-combine.sh ${COMBINE_VERSION} +if [[ $NET_INSTALL == 0 ]] ; then + makeself --append ${TEMP_DIR} ${INSTALLER_NAME} + rm -rf ${TEMP_DIR} fi -makeself --tar-quietly ../deploy ./combine-installer.run "Combine Installer" scripts/install-combine.sh ${COMBINE_VERSION} diff --git a/maintenance/requirements.txt b/maintenance/requirements.txt index 9b06468bbe..cc150ca193 100644 --- a/maintenance/requirements.txt +++ b/maintenance/requirements.txt @@ -6,7 +6,7 @@ # cachetools==5.3.3 # via google-auth -certifi==2024.6.2 +certifi==2024.7.4 # via # kubernetes # requests @@ -14,14 +14,14 @@ cffi==1.16.0 # via cryptography charset-normalizer==3.3.2 # via requests -cryptography==42.0.7 +cryptography==42.0.8 # via pyopenssl dnspython==2.6.1 # via pymongo -google-auth==2.29.0 +google-auth==2.32.0 # via kubernetes humanfriendly==10.0 - # via -r maintenance/requirements.in + # via -r requirements.in idna==3.7 # via requests kubernetes==30.1.0 @@ -41,7 +41,7 @@ pycparser==2.22 pymongo==4.8.0 # via -r requirements.in pyopenssl==24.1.0 - # via -r maintenance/requirements.in + # via -r requirements.in python-dateutil==2.9.0.post0 # via kubernetes pyyaml==6.0.1