diff --git a/dev-shell-e2e-test.sh b/dev-shell-e2e-test.sh new file mode 100755 index 00000000..cb3c5a63 --- /dev/null +++ b/dev-shell-e2e-test.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +docker_dir=docker-e2e-test +docker_path=${PWD}/${docker_dir} + +# Function to execute custom commands before exiting +down() { + docker compose --env-file=${docker_path}/.env.dev -f ${docker_path}/docker-compose.yml down --remove-orphans + # remove the docker runtime part + docker volume rm ${docker_dir}_docker-runtime +} + +# Register the cleanup function to be called on EXIT +trap down EXIT + +mkdir -p $PWD/.device/sysroot +docker compose --env-file=${docker_path}/.env.dev -f ${docker_path}/docker-compose.yml run -e DEV_USER=$(id -u) -e DEV_GROUP=$(id -g) -e OSF_TOKEN=${OSF_TOKEN} -e USER_TOKEN=${USER_TOKEN} aklite-e2e-test $@ diff --git a/docker-e2e-test/.env.dev b/docker-e2e-test/.env.dev new file mode 100644 index 00000000..fa7bad19 --- /dev/null +++ b/docker-e2e-test/.env.dev @@ -0,0 +1,19 @@ +FACTORY=$FACTORY +AUTH_TOKEN=$USER_TOKEN +DEVICE_TAG=main + +DEV_DIR=$PWD/.device +SOTA_DIR=$DEV_DIR/sota +USR_SOTA_DIR=$DEV_DIR/usr.sota +ETC_SOTA_DIR=$DEV_DIR/etc.sota +DOCKER_DIR=$DEV_DIR/docker + +# /sysroot dir containing ostree repo in /sysroot/ostree/repo +SYSROOT=$DEV_DIR/sysroot +BOOTDIR=$DEV_DIR/boot + +# /var/lib/docker +DOCKER_DATA_ROOT=$DOCKER_DIR/data + +# Dir containing Dockerfile and build context to build the "aklite-test" image +AKLITE_E2E_TEST_DOCKER_DIR=$PWD/docker-e2e-test diff --git a/docker-e2e-test/Dockerfile b/docker-e2e-test/Dockerfile new file mode 100644 index 00000000..7cc2c8da --- /dev/null +++ b/docker-e2e-test/Dockerfile @@ -0,0 +1,73 @@ +FROM golang:1.22.2-bookworm AS composeapp +# Build composeapp +WORKDIR /build +RUN git clone https://github.com/foundriesio/composeapp.git && cd composeapp \ + && STOREROOT=/var/sota/reset-apps COMPOSEROOT=/var/sota/compose-apps BASESYSTEMCONFIG=/usr/lib/docker make \ + && cp ./bin/composectl /usr/bin/ + +# We may add fioctl and fioconfig to the test sequence. For now, we don't use them +# WORKDIR /build +# RUN git clone https://github.com/foundriesio/fioconfig.git && cd fioconfig \ +# && make bin/fioconfig-linux-amd64 \ +# && cp ./bin/fioconfig-linux-amd64 /usr/bin/fioconfig + +# WORKDIR /build +# RUN git clone https://github.com/foundriesio/fioctl.git && cd fioctl \ +# && make fioctl-linux-amd64 \ +# && cp ./bin/fioctl-linux-amd64 /usr/bin/fioctl + + +FROM foundries/aklite-dev AS aklite + +# Install composectl +COPY --from=composeapp /build/composeapp/bin/composectl /usr/bin/ + +# # Install fioconfig +# COPY --from=composeapp /build/fioconfig/bin/fioconfig-linux-amd64 /usr/bin/fioconfig + +# # Install fioctl +# COPY --from=composeapp /build/fioctl/bin/fioctl-linux-amd64 /usr/bin/fioctl + + +# Install lmp-device-register +RUN apt-get install -y libboost-iostreams-dev + +RUN git clone https://github.com/foundriesio/lmp-device-register \ + && cd lmp-device-register && git checkout mp-90 \ + && cmake -S . -B ./build -DDOCKER_COMPOSE_APP=ON -DHARDWARE_ID=intel-corei7-64 && cmake --build ./build --target install + + +# Add Docker's official GPG key: +RUN apt-get update && apt-get install -y ca-certificates curl +RUN install -m 0755 -d /etc/apt/keyrings +RUN curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc +RUN chmod a+r /etc/apt/keyrings/docker.asc + +# Add the repository to Apt sources: +RUN echo \ + "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu \ + $(. /etc/os-release && echo "$VERSION_CODENAME") stable" | \ + tee /etc/apt/sources.list.d/docker.list > /dev/null + +RUN apt-get update && apt-get install -y docker-ce-cli docker-compose-plugin + +# Install docker credential helper and auth configuration +COPY config.json /usr/lib/docker/config.json +COPY docker-credential-fio-helper /usr/bin/docker-credential-fio-helper + +# Install gosu required for the entry/startup script to add a user and group in the container +RUN wget -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/1.14/gosu-amd64" && \ + chmod +x /usr/local/bin/gosu && \ + gosu nobody true + +# Install pytest +RUN pip install --no-cache-dir --upgrade pip && \ + pip install --no-cache-dir pytest + + # Copy the entrypoint script +COPY entrypoint.sh /usr/local/bin/entrypoint.sh +RUN chmod +x /usr/local/bin/entrypoint.sh + +# Set entrypoint +ENTRYPOINT ["/usr/local/bin/entrypoint.sh"] +CMD ["bash"] diff --git a/docker-e2e-test/config.json b/docker-e2e-test/config.json new file mode 100644 index 00000000..55afd413 --- /dev/null +++ b/docker-e2e-test/config.json @@ -0,0 +1,6 @@ +{ + "credHelpers": { + "hub.foundries.io": "fio-helper" + } +} + diff --git a/docker-e2e-test/docker-compose.yml b/docker-e2e-test/docker-compose.yml new file mode 100644 index 00000000..8a386d65 --- /dev/null +++ b/docker-e2e-test/docker-compose.yml @@ -0,0 +1,45 @@ +version: '3.8' + +services: + dockerd: + image: docker:25.0-dind + command: ["dockerd", "-H", "unix:///var/run/docker/docker.sock"] + volumes: + - ${DOCKER_DATA_ROOT}:/var/lib/docker + - docker-runtime:/var/run/docker + privileged: true + + aklite-e2e-test: + build: + context: ${AKLITE_E2E_TEST_DOCKER_DIR} + args: + AKLITE_VER: master + dockerfile: Dockerfile + + image: aklite-e2e-test + volumes: + - "${PWD}:${PWD}" + - ${SYSROOT}:/sysroot + - ${SYSROOT}/ostree:/ostree + - ${BOOTDIR}:/boot + - ${SOTA_DIR}:/var/sota + - ${USR_SOTA_DIR}:/usr/lib/sota/conf.d + - ${ETC_SOTA_DIR}:/etc/sota/conf.d + - ${DOCKER_DATA_ROOT}:/var/lib/docker + - docker-runtime:/var/run/docker + working_dir: "${PWD}" + hostname: device + user: "root" + environment: + - FACTORY=${FACTORY} + - AUTH_TOKEN=${AUTH_TOKEN} + - DEVICE_TAG=${DEVICE_TAG} + - DOCKER_HOST=unix:///var/run/docker/docker.sock + - DOCKER_CONFIG=/usr/lib/docker + - CXX=clang++ + - CC=clang + depends_on: + - dockerd + +volumes: + docker-runtime: diff --git a/docker-e2e-test/docker-credential-fio-helper b/docker-e2e-test/docker-credential-fio-helper new file mode 100755 index 00000000..8973e1e3 --- /dev/null +++ b/docker-e2e-test/docker-credential-fio-helper @@ -0,0 +1,19 @@ +#!/bin/sh -e + +# Use stderr for logging err output in libaktualizr +export LOG_STDERR=1 +SOTA_DIR="${SOTA_DIR-/var/sota}" + +LOGLEVEL="${CREDS_LOGLEVEL-4}" + +if [ "$1" = "get" ] ; then + if [ ! -f ${SOTA_DIR}/sota.toml ] ; then + echo "ERROR: Device does not appear to be registered under $SOTA_DIR" + exit 1 + fi + server=$(grep -m1 '^[[:space:]]*server' ${SOTA_DIR}/sota.toml | cut -d\" -f2) + if [ -z $server ] ; then + server="https://ota-lite.foundries.io:8443" + fi + exec /usr/local/bin/aktualizr-get --loglevel $LOGLEVEL -u ${server}/hub-creds/ +fi diff --git a/docker-e2e-test/entrypoint.sh b/docker-e2e-test/entrypoint.sh new file mode 100644 index 00000000..bd5fb042 --- /dev/null +++ b/docker-e2e-test/entrypoint.sh @@ -0,0 +1,68 @@ +#!/bin/sh -e + +if [ -z $DEV_USER ] || [ -z $DEV_GROUP ]; then + echo "DEV_USER and DEV_GROUP environment variables must be set." + exit 1 +fi + +# Create a group with the specified GID if it doesn't already exist +if ! getent group $DEV_GROUP >/dev/null; then + groupadd -g $DEV_GROUP devgrp +fi + +# Create a user with the specified UID and GID if it doesn't already exist +if ! getent passwd $DEV_USER >/dev/null; then + useradd -u $DEV_USER -g $DEV_GROUP -m dev +fi + +# Change ownership of the home directory to the appuser +chown -R dev:devgrp /home/dev + +chown dev:devgrp /var/run/docker/docker.sock + +chown -R dev:devgrp /var/sota +chown -R dev:devgrp /usr/lib/sota/conf.d +chown -R dev:devgrp /etc/sota/conf.d +chown -R dev:devgrp /var/lib/docker + +# Initialize ostree +if [ ! -d /sysroot/ostree/repo ]; then + echo "Initializing sysroot ostree..." + ostree admin init-fs /sysroot + ostree admin os-init lmp + ostree config set core.mode bare-user + ${PWD}/tests/make_sys_rootfs.sh initfs lmp intel-corei7-64 lmp + commit=$(ostree commit initfs --branch lmp) + ostree admin deploy --os=lmp $commit + rm -rf initfs + chown -R dev:devgrp /ostree + ostree config set core.mode bare-user-only +fi +if [ ! -d /etc/ostree ]; then + mkdir /etc/ostree + chown -R dev:devgrp /etc/ostree + chown -R dev:devgrp /boot +fi + +ln -sfn ${PWD}/aktualizr/build/src/aktualizr_get/aktualizr-get /usr/local/bin/aktualizr-get + +# Initialize default toml config +sysroot_cfg=/usr/lib/sota/conf.d/z-90-sysroot.toml +if [ ! -f $sysroot_cfg ]; then + echo "[pacman]\nbooted = 0\nos = lmp" > $sysroot_cfg +fi + +bootloader_cfg=/usr/lib/sota/conf.d/z-91-bootloader.toml +if [ ! -f $bootloader_cfg ]; then + echo "[bootloader]\nreboot_command = /usr/bin/true" > $bootloader_cfg +fi + +# Set directory used for reboot indication, the `need_reboot` file is created under this dir +if [ ! -d /var/run/aktualizr-session ]; then + mkdir -p /var/run/aktualizr-session + chown dev:devgrp /var/run/aktualizr-session + chmod 700 /var/run/aktualizr-session +fi + +# Run the command as the created user +exec gosu $DEV_USER:$DEV_GROUP "$@" diff --git a/e2e-test.py b/e2e-test.py new file mode 100644 index 00000000..349c3375 --- /dev/null +++ b/e2e-test.py @@ -0,0 +1,568 @@ +import requests, json, datetime, sys + +import pytest +import subprocess +import json +import os +import logging +import time +import stat + +class ReturnCodes: + UnknownError = 1 + Ok = 0 + CheckinOkCached = 3 + CheckinFailure = 4 + OkNeedsRebootForBootFw = 5 + CheckinNoMatchingTargets = 6 + CheckinNoTargetContent = 8 + InstallAppsNeedFinalization = 10 + CheckinSecurityError = 11 + CheckinExpiredMetadata = 12 + CheckinMetadataFetchFailure = 13 + CheckinMetadataNotFound = 14 + CheckinInvalidBundleMetadata = 15 + CheckinUpdateNewVersion = 16 + CheckinUpdateSyncApps = 17 + CheckinUpdateRollback = 18 + TufTargetNotFound = 20 + RollbackTargetNotFound = 21 + InstallationInProgress = 30 + NoPendingInstallation = 40 + DownloadFailure = 50 + DownloadFailureNoSpace = 60 + DownloadFailureVerificationFailed = 70 + InstallAlreadyInstalled = 75 + InstallAppPullFailure = 80 + InstallNeedsRebootForBootFw = 90 + InstallOfflineRollbackOk = 99 + InstallNeedsReboot = 100 + InstallDowngradeAttempt = 102 + InstallRollbackOk = 110 + InstallRollbackNeedsReboot = 120 + InstallRollbackFailed = 130 + +logger = logging.getLogger(__name__) + +osf_token = os.getenv("USER_TOKEN") +if not osf_token: + logger.error("USER_TOKEN environment variable not set") + sys.exit() + + +aklite_path = "./build/src/aktualizr-lite" +composectl_path = "/usr/bin/composectl" +callback_log_path = "/var/sota/callback_log.txt" + +# Test modes +offline = True +single_step = True +prune = True + +factory_name = "detsch-aklite-test" +base_target_version = 54 + +def get_target_version(offset): + return base_target_version + offset + +# X - Pre-built, no apps (won't be used by tests) +# 0 - Small working ostree, no apps, starting point - $OSTREE_HASH_1 +# 1 - Incomplete ostree (causes rollback) - $OSTREE_HASH_2 +# 2 - Small working ostree with change, no apps - $OSTREE_HASH_3 +# 3 - Add 1 app, keep ostree +# 4 - Add more apps +# 5 - Break 1 existing app (causes rollback) +# 6 - Change app, but it is still broken (causes rollback) +# 7 - Break build, non existing target +# 8 - Fixes build and app, success +# 9 - Update some app version +# 10 - Additional change to ostree - $OSTREE_HASH_4 +# 11 - Broken ostree - $OSTREE_HASH_5 + +class Targets: + First = 0 + BrokenOstree = 1 + WorkingOstree = 2 + AddFirstApp = 3 + AddMoreApps = 4 + BreakApp = 5 + UpdateBrokenApp = 6 + BrokenBuild = 7 + FixApp = 8 + UpdateWorkingApp = 9 + UpdateOstreeWithApps = 10 + BrokenOstreeWithApps = 11 + + def __init__(self, version_offset, install_rollback, run_rollback, build_error, ostree_image_version): + self.version_offset = version_offset + self.install_rollback = install_rollback + self.run_rollback = run_rollback + self.build_error = build_error + self.ostree_image_version = ostree_image_version + +all_targets = { + Targets.First: Targets(Targets.First, False, False, False, 1), + Targets.BrokenOstree: Targets(Targets.BrokenOstree, True, False, False, 2), + Targets.WorkingOstree: Targets(Targets.WorkingOstree, False, False, False, 3), + Targets.AddFirstApp: Targets(Targets.AddFirstApp, False, False, False, 3), + Targets.AddMoreApps: Targets(Targets.AddMoreApps, False, False, False, 3), + Targets.BreakApp: Targets(Targets.BreakApp, False, True, False, 3), + Targets.UpdateBrokenApp: Targets(Targets.UpdateBrokenApp, False, True, False, 3), + Targets.BrokenBuild: Targets(Targets.BrokenBuild, False, False, True, 3), + Targets.FixApp: Targets(Targets.FixApp, False, False, False, 3), + Targets.UpdateWorkingApp: Targets(Targets.UpdateWorkingApp, False, False, False, 3), + Targets.UpdateOstreeWithApps: Targets(Targets.UpdateOstreeWithApps, False, False, False, 4), + Targets.BrokenOstreeWithApps: Targets(Targets.BrokenOstreeWithApps, True, False, False, 5), +} + +def register_if_required(): + if not os.path.exists("/var/sota/client.pem"): + user_token = os.getenv("USER_TOKEN") + cmd = f'DEVICE_FACTORY={factory_name} lmp-device-register --api-token "{user_token}" --start-daemon 0 --tags main' + logger.info(f"Registering device... TOKEN={user_token}") + output = os.popen(cmd).read().strip() + logger.info(output) + else: + logger.info("Device already registered") + +def get_device_name(): + cmd = "openssl x509 -noout -subject -nameopt multiline -in /var/sota/client.pem | grep commonName | sed -n 's/ *commonName *= //p'" + device_uuid = os.popen(cmd).read().strip() + assert len(device_uuid) == 36 + # Assuming device name == uuid + logger.info(f"Device UUID is {device_uuid}") + return device_uuid + +register_if_required() +device_name = get_device_name() +# os.getenv("DEVICE_NAME", "aklite-test-device") + +def verify_events(target_version, expected_events = None, second_to_last_corr_id = False): + headers = {'OSF-TOKEN': osf_token} + r = requests.get(f'https://api.foundries.io/ota/devices/{device_name}/updates/', headers=headers) + d = json.loads(r.text) + # print(json.dumps(d, indent=2)) + + if second_to_last_corr_id: + latest_update = d["updates"][1] + else: + latest_update = d["updates"][0] + corr_id = latest_update["correlation-id"] + assert int(latest_update["version"]) == target_version + r = requests.get(f'https://api.foundries.io/ota/devices/{device_name}/updates/{corr_id}/', headers=headers) + + d_update = json.loads(r.text) + # print(json.dumps(d_update, indent=2)) + event_list = set([ (x["eventType"]["id"], x["event"]["success"]) for x in d_update ]) + if expected_events is None: + expected_events = { + ('EcuDownloadStarted', None), + ('EcuDownloadCompleted', True), + ('EcuInstallationStarted', None), + ('EcuInstallationApplied', None), + ('EcuInstallationCompleted', True) + } + assert set(event_list) == set(expected_events) + +def sys_reboot(): + need_reboot_path = "/var/run/aktualizr-session/need_reboot" + if os.path.isfile(need_reboot_path): + os.remove(need_reboot_path) + +def clear_callbacks_log(): + if os.path.isfile(callback_log_path): + os.remove(callback_log_path) + +# TODO: verify additional callback variables +def verify_callback(expected_calls): + calls = [] + if os.path.isfile(callback_log_path): + with open(callback_log_path) as f: + for l in f.readlines(): + j = json.loads(l) + calls.append((j["MESSAGE"], j["RESULT"])) + clear_callbacks_log() + assert expected_calls == calls + +def aklite_current_version(): + sp = invoke_aklite(['status']) + + # Last line should be like this: + # info: Active image is: 42 sha256:1f5c2258e6e493741c719394bd267b2e163609f1cb3457ccb71fcaf770c5c116 + lines = [ x for x in sp.stdout.decode('utf-8').splitlines() if "Active image is:" in x ] + assert len(lines) == 1 + version = int(lines[0].split()[3]) + return version + +def aklite_current_version_based_on_list(): + sp = invoke_aklite(['list', '--json', '1']) + out_json = json.loads(sp.stdout) + count = sum(target.get("current", False) for target in out_json) + # Make sure there is only 1 "current" version + assert count == 1 + curr_target = next(target for target in out_json if target.get("current", False)) + return curr_target["version"] + +def invoke_aklite(options): + if offline: + options = options + [ "--src-dir", os.path.abspath("./offline-bundles/unified/") ] + return subprocess.run([aklite_path] + options, capture_output=True) + +def write_settings(apps=None, prune=True): + callback_file = "/var/sota/callback.sh" + + callback_content = \ +"""#!/bin/sh +echo { \\"MESSAGE\\": \\"$MESSAGE\\", \\"CURRENT_TARGET\\": \\"$CURRENT_TARGET\\", \\"CURRENT_TARGET_NAME\\": \\"$CURRENT_TARGET_NAME\\", \\"INSTALL_TARGET_NAME\\": \\"$INSTALL_TARGET_NAME\\", \\"RESULT\\": \\"$RESULT\\" } >> /var/sota/callback_log.txt +""" + with open(callback_file, "w") as f: + f.write(callback_content) + st = os.stat(callback_file) + os.chmod(callback_file, st.st_mode | stat.S_IEXEC) + + content = \ +f""" +[pacman] +tags = "main" +""" + if apps is not None: + apps_str = ",".join(apps) + content += f""" +compose_apps = "{apps_str}" +docker_apps = "{apps_str}" +""" + # callback_program = "/var/sota/callback.sh" + + if not prune: + content += "\ndocker_prune = 0\n" + + with open("/etc/sota/conf.d/z-50-fioctl.toml", "w") as f: + f.write(content) + + sota_toml_content = "" + with open("/var/sota/sota.toml") as f: + sota_toml_content = f.read() + + if not "callback_program" in sota_toml_content: + sota_toml_content = sota_toml_content.replace("[pacman]", '[pacman]\ncallback_program = "/var/sota/callback.sh"') + with open("/var/sota/sota.toml", "w") as f: + f.write(sota_toml_content) + +def get_all_current_apps(): + sp = invoke_aklite(['list', '--json', '1']) + out_json = json.loads(sp.stdout) + target_apps = [ target["apps"] for target in out_json if target.get("current", False) ] + # there should be only 1 current target + assert len(target_apps) == 1 + if target_apps[0] is None: + return [] + return [ app["name"] for app in target_apps[0] ] + +def get_running_apps(): + sp = subprocess.run([composectl_path, "ps"], capture_output=True) + output_lines = sp.stdout.decode('utf-8').splitlines() + # print(output_lines) + running_app_names = [ l.split()[0] for l in output_lines if l.split()[1] == "(running)" ] + return running_app_names + +def check_running_apps(expected_apps=None): + # if no apps list is specified, all apps should be running + if expected_apps is None: + expected_apps = get_all_current_apps() + running_apps = get_running_apps() + assert set(expected_apps) == set(running_apps) + +def cleanup_tuf_metadata(): + os.system("""sqlite3 /var/sota/sql.db "delete from meta where meta_type <> 0 or version >= 3;" ".exit" """) + +def cleanup_installed_data(): + os.system("""sqlite3 /var/sota/sql.db "delete from installed_versions;" ".exit" """) + +def install_with_separate_steps(version, requires_reboot=False, install_rollback=False, run_rollback=False, explicit_version=True): + cp = invoke_aklite(['check', '--json', '1']) + assert cp.returncode == ReturnCodes.CheckinUpdateNewVersion + verify_callback([("check-for-update-pre", ""), ("check-for-update-post", "OK")]) + + if install_rollback or run_rollback: + final_version = aklite_current_version() + else: + final_version = version + + if explicit_version: + cp = invoke_aklite(['pull', str(version)]) + else: + cp = invoke_aklite(['pull']) + assert cp.returncode == ReturnCodes.Ok + verify_callback([("download-pre", ""), ("download-post", "OK")]) + verify_events(version, { + ('EcuDownloadStarted', None), + ('EcuDownloadCompleted', True), + }) + + # cp = invoke_aklite(['install', str(get_target_version(Targets.BrokenBuild))]) # not existing target + # assert cp.returncode == ReturnCodes.TufTargetNotFound + # verify_callback([]) + + # version = 159 + # cp = invoke_aklite(['install', str(version)]) # not downloaded target + # assert cp.returncode == ReturnCodes.InstallAppPullFailure + # verify_callback([]) + + + if explicit_version: + cp = invoke_aklite(['install', str(version)]) # OK + else: + cp = invoke_aklite(['install']) # OK + if install_rollback: + assert cp.returncode == ReturnCodes.InstallRollbackOk + verify_callback([("install-pre", ""), ("install-post", "FAILED"), ("install-pre", ""), ("install-post", "OK")]) + verify_events(version, { + ('EcuInstallationStarted', None), + ('EcuInstallationCompleted', False), + }, True) + verify_events(final_version, { + ('EcuInstallationStarted', None), + ('EcuInstallationCompleted', True), + }, False) + + elif requires_reboot: + assert cp.returncode == ReturnCodes.InstallNeedsReboot + verify_callback([("install-pre", ""), ("install-post", "NEEDS_COMPLETION")]) + sys_reboot() + verify_events(version, { + ('EcuInstallationStarted', None), + ('EcuInstallationApplied', None), + }) + cp = invoke_aklite(['run']) + verify_callback([("install-final-pre", ""), ("install-post", "OK")]) + verify_events(version, { + ('EcuInstallationStarted', None), + ('EcuInstallationApplied', None), + ('EcuInstallationCompleted', True), + }) + else: + if run_rollback: + assert cp.returncode == ReturnCodes.InstallRollbackOk + verify_callback([ + ("install-pre", ""), ("install-post", "FAILED"), + ("install-pre", ""), ("install-post", "OK") + ]) + verify_events(version, { + ('EcuInstallationStarted', None), + ('EcuInstallationCompleted', False), + }, True) + + verify_events(final_version, { + ('EcuInstallationStarted', None), + ('EcuInstallationCompleted', True), + }, False) + + else: + assert cp.returncode == ReturnCodes.Ok + verify_callback([("install-pre", ""), ("install-post", "OK")]) + verify_events(version, { + ('EcuInstallationStarted', None), + ('EcuInstallationCompleted', True), + }) + + assert aklite_current_version() == final_version + if not explicit_version: + # Make sure we would not try a new install, after trying to install the latest one + cp = invoke_aklite(['check', '--json', '1']) + assert cp.returncode == ReturnCodes.Ok + verify_callback([("check-for-update-pre", ""), ("check-for-update-post", "OK")]) + +def install_with_single_step(version, requires_reboot=False, install_rollback=False, run_rollback=False, explicit_version=True): + if install_rollback or run_rollback: + final_version = aklite_current_version() + else: + final_version = version + + if explicit_version: + cp = invoke_aklite(['update', str(version)]) + else: + cp = invoke_aklite(['update']) + + if install_rollback: + assert cp.returncode == ReturnCodes.InstallRollbackOk + verify_callback([ + ("check-for-update-pre", ""), ("check-for-update-post", "OK"), + ("download-pre", ""), ("download-post", "OK"), + ("install-pre", ""), ("install-post", "FAILED"), + ("install-pre", ""), ("install-post", "OK")]) + verify_events(version, { + ('EcuDownloadStarted', None), + ('EcuDownloadCompleted', True), + ('EcuInstallationStarted', None), + ('EcuInstallationCompleted', False), + }, True) + verify_events(final_version, { + ('EcuInstallationStarted', None), + ('EcuInstallationCompleted', True), + }, False) + + elif requires_reboot: + assert cp.returncode == ReturnCodes.InstallNeedsReboot + verify_callback([ + ("check-for-update-pre", ""), ("check-for-update-post", "OK"), + ("download-pre", ""), ("download-post", "OK"), + ("install-pre", ""), ("install-post", "NEEDS_COMPLETION"), + ]) + sys_reboot() + verify_events(version, { + ('EcuDownloadStarted', None), + ('EcuDownloadCompleted', True), + ('EcuInstallationStarted', None), + ('EcuInstallationApplied', None), + }) + cp = invoke_aklite(['run']) + # TODO: handle run_rollback + verify_callback([("install-final-pre", ""), ("install-post", "OK")]) + verify_events(version, { + ('EcuDownloadStarted', None), + ('EcuDownloadCompleted', True), + ('EcuInstallationStarted', None), + ('EcuInstallationApplied', None), + ('EcuInstallationCompleted', True), + }) + else: + if run_rollback: + assert cp.returncode == ReturnCodes.InstallRollbackOk + verify_callback([ + ("check-for-update-pre", ""), ("check-for-update-post", "OK"), + ("download-pre", ""), ("download-post", "OK"), + ("install-pre", ""), ("install-post", "FAILED"), + ("install-pre", ""), ("install-post", "OK") + ]) + verify_events(version, { + ('EcuDownloadStarted', None), + ('EcuDownloadCompleted', True), + ('EcuInstallationStarted', None), + ('EcuInstallationCompleted', False), + }, True) + + verify_events(final_version, { + ('EcuInstallationStarted', None), + ('EcuInstallationCompleted', True), + }, False) + + else: + assert cp.returncode == ReturnCodes.Ok + verify_callback([ + ("check-for-update-pre", ""), ("check-for-update-post", "OK"), + ("download-pre", ""), ("download-post", "OK"), + ("install-pre", ""), ("install-post", "OK") + ]) + verify_events(version, { + ('EcuDownloadStarted', None), + ('EcuDownloadCompleted', True), + ('EcuInstallationStarted', None), + ('EcuInstallationCompleted', True), + }) + assert aklite_current_version() == final_version + + if not explicit_version: + # Make sure we would not try a new install, after trying to install the latest one + cp = invoke_aklite(['check', '--json', '1']) + assert cp.returncode == ReturnCodes.Ok + verify_callback([("check-for-update-pre", ""), ("check-for-update-post", "OK")]) + +def install_version(version, requires_reboot=False, install_rollback=False, run_rollback=False, explicit_version=True): + if single_step: + install_with_single_step(version, requires_reboot, install_rollback, run_rollback, explicit_version) + else: + install_with_separate_steps(version, requires_reboot, install_rollback, run_rollback, explicit_version) + +def restore_system_state(): + ######################################################################### + logger.info(f"Restoring base environment. Offline={offline} SingleStep={single_step} Prune={prune}...") + ######################################################################### + # Get to the starting point + write_settings() + sys_reboot() + cp = invoke_aklite(['run']) + version = get_target_version(Targets.First) + cleanup_installed_data() + cp = invoke_aklite(['update', str(version)]) + print(cp.stdout) + sys_reboot() + cp = invoke_aklite(['run']) + assert aklite_current_version() == version + clear_callbacks_log() + cleanup_tuf_metadata() + + ######################################################################### + logger.info("Making sure there are no targets in current DB...") + ######################################################################### + cp = invoke_aklite(['list', '--json', '1']) + assert cp.returncode == ReturnCodes.CheckinSecurityError + +# Incremental install order +install_sequence_incremental = [ + Targets.BrokenOstree, + Targets.WorkingOstree, + Targets.AddFirstApp, + Targets.AddMoreApps, + Targets.BreakApp, + Targets.UpdateBrokenApp, + Targets.BrokenBuild, + Targets.FixApp, + Targets.UpdateWorkingApp, + Targets.UpdateOstreeWithApps, + Targets.BrokenOstreeWithApps, +] + +def run_test_sequence_incremental(): + restore_system_state() + apps = None # All apps, for now + prev_ostree_image_version = 1 + for target_version in install_sequence_incremental: + target = all_targets[target_version] + if target.build_error: # skip this one for now + continue + + version = get_target_version(target.version_offset) + logger.info(f"Updating to {target.version_offset} (target version={version}, {target.install_rollback=}, {target.run_rollback=}, {target.build_error=}, {target.ostree_image_version=}. {single_step=} {offline=}") + requires_reboot = target.ostree_image_version != prev_ostree_image_version + write_settings(apps, prune) + install_version(version, requires_reboot, target.install_rollback, target.run_rollback) + check_running_apps(apps) + if not target.install_rollback and not target.run_rollback: + prev_ostree_image_version = target.ostree_image_version + +def run_test_sequence_update_to_latest(): + restore_system_state() + apps = None # All apps, for now + prev_ostree_image_version = 1 + + last_target = all_targets[Targets.BrokenOstreeWithApps] + target = last_target + + # Try to install latest version, which will lead to a rollback + write_settings(apps, prune) + version = get_target_version(target.version_offset) + requires_reboot = target.ostree_image_version != prev_ostree_image_version + logger.info(f"Updating to latest target {target.version_offset} (target version={version}, {target.install_rollback=}, {target.run_rollback=}, {target.build_error=}, {target.ostree_image_version=}. {single_step=} {offline=}") + install_version(version, requires_reboot, target.install_rollback, target.run_rollback, False) + check_running_apps(apps) + + +# @pytest.mark.parametrize('offline_', [False, True]) +@pytest.mark.parametrize('offline_', [False]) +@pytest.mark.parametrize('single_step_', [True, False]) +def test_incremental_updates(offline_, single_step_): + global offline, single_step + offline = offline_ + single_step = single_step_ + run_test_sequence_incremental() + +@pytest.mark.parametrize('offline_', [False]) +@pytest.mark.parametrize('single_step_', [True, False]) +def test_update_to_latest(offline_, single_step_): + global offline, single_step + offline = offline_ + single_step = single_step_ + run_test_sequence_update_to_latest() + +# TODO: Re-add apps shortlist testing +# TODO: Re-enable offline tests