diff --git a/actions.yaml b/actions.yaml index 132331de..1dcb8c7c 100644 --- a/actions.yaml +++ b/actions.yaml @@ -41,3 +41,6 @@ promote-user-admin: description: | User name to be promoted to admin. type: string +create-backup: + description: | + Creates a backup to s3 storage. diff --git a/config.yaml b/config.yaml index aad862bc..c1d1fd57 100644 --- a/config.yaml +++ b/config.yaml @@ -8,6 +8,9 @@ options: description: | Allows any other homeserver to fetch the server's public rooms directory via federation. + backup_passphrase: + type: string + description: Passphrase used to encrypt a backup using gpg with symmetric key. enable_mjolnir: type: boolean default: false diff --git a/docs/how-to/backup.md b/docs/how-to/backup.md new file mode 100644 index 00000000..beeff1e3 --- /dev/null +++ b/docs/how-to/backup.md @@ -0,0 +1,36 @@ +# How to back up Synapse + +This document shows how to back up Synapse. + +## Deploy s3-integrator charm + +Synapse gets backed up to a S3 compatible object storage. To get the credentials, the `s3-integrator` is used. Refer to +[s3-integrator](https://charmhub.io/s3-integrator/) for specific configuration options. + +``` +juju deploy s3-integrator --channel edge +juju config s3-integrator endpoint= bucket= path= region= s3-uri-style= +juju run s3-integrator/leader sync-s3-credentials access-key= secret-key= +``` + +Integrate with Synapse with: + +`juju integrate synapse:backup s3-integrator` + +## Configure the passphrase + +The backup will be encrypted before being sent using symmetric encryption. You need +to set the desired password with: +``` +juju config synapse backup_passphrase= +``` + +## Run the backup + +Run the backup with the next command: +``` +juju run synapse/leader create-backup +``` + +A new object should be placed in the S3 compatible object storage. This file is a tar +file encrypted with the `gpg` command. \ No newline at end of file diff --git a/src-docs/backup.py.md b/src-docs/backup.py.md index 2b21492c..30226b26 100644 --- a/src-docs/backup.py.md +++ b/src-docs/backup.py.md @@ -5,6 +5,141 @@ # module `backup.py` Provides backup functionality for Synapse. +**Global Variables** +--------------- +- **AWS_COMMAND** +- **BACKUP_FILE_PATTERNS** +- **MEDIA_DIR** +- **LOCAL_DIR_PATTERN** +- **S3_MAX_CONCURRENT_REQUESTS** +- **PASSPHRASE_FILE** +- **BASH_COMMAND** +- **BACKUP_ID_FORMAT** + +--- + + + +## function `paths_to_args` + +```python +paths_to_args(paths: Iterable[str]) → str +``` + +Given a list of paths, quote and concatenate them for use as cli arguments. + + + +**Args:** + + - `paths`: List of paths + + + +**Returns:** + paths concatenated and quoted + + +--- + + + +## function `get_paths_to_backup` + +```python +get_paths_to_backup(container: Container) → Iterable[str] +``` + +Get the list of paths that should be in a backup for Synapse. + + + +**Args:** + + - `container`: Synapse Container. + + + +**Returns:** + Iterable with the list of paths to backup. + + +--- + + + +## function `calculate_size` + +```python +calculate_size(container: Container, paths: Iterable[str]) → int +``` + +Return the combined size of all the paths given. + + + +**Args:** + + - `container`: Container where to check the size of the paths. + - `paths`: Paths to check. + + + +**Returns:** + Total size in bytes. + + + +**Raises:** + + - `BackupError`: If there was a problem calculating the size. + + +--- + + + +## function `create_backup` + +```python +create_backup( + container: Container, + s3_parameters: S3Parameters, + passphrase: str, + backup_id: Optional[str] = None +) → str +``` + +Create a backup for Synapse running it in the workload. + + + +**Args:** + + - `container`: Synapse Container + - `s3_parameters`: S3 parameters for the backup. + - `passphrase`: Passphrase use to encrypt the backup. + - `backup_id`: Name of the object in the backup. It will be autogenerated if it is not set. + + + +**Returns:** + The backup key used for the backup. + + + +**Raises:** + + - `BackupError`: If there was an error creating the backup. + + +--- + +## class `BackupError` +Generic backup Exception. + + + --- @@ -12,7 +147,7 @@ Provides backup functionality for Synapse. ## class `S3Client` S3 Client Wrapper around boto3 library. - + ### function `__init__` @@ -33,7 +168,7 @@ Initialize the S3 client. --- - + ### function `can_use_bucket` @@ -87,7 +222,7 @@ Translates s3_uri_style to AWS addressing_style. --- - + ### classmethod `check_endpoint_or_region_set` diff --git a/src-docs/backup_observer.py.md b/src-docs/backup_observer.py.md index db9ff2de..797002e0 100644 --- a/src-docs/backup_observer.py.md +++ b/src-docs/backup_observer.py.md @@ -16,7 +16,7 @@ S3 Backup relation observer for Synapse. ## class `BackupObserver` The S3 backup relation observer. - + ### function `__init__` diff --git a/src/backup.py b/src/backup.py index 777914f9..a92d7a0c 100644 --- a/src/backup.py +++ b/src/backup.py @@ -3,18 +3,47 @@ """Provides backup functionality for Synapse.""" +import datetime import logging -from typing import Any, Optional +import os +from typing import Any, Dict, Iterable, List, Optional import boto3 +import ops from botocore.config import Config from botocore.exceptions import BotoCoreError, ClientError from botocore.exceptions import ConnectionError as BotoConnectionError +from ops.pebble import APIError, ExecError from pydantic import BaseModel, Field, validator +import synapse + +AWS_COMMAND = "/aws/dist/aws" + +# The configuration files to back up consist in the signing keys +# plus the sqlite db if it exists. +BACKUP_FILE_PATTERNS = ["*.key", "homeserver.db*"] + +# For the data directory, inside the "media" directory, all directories starting +# with local_ will be backed up. The directories starting with "remote_" are from +# other server and is it not necessary to back them up. +MEDIA_DIR = "media" +LOCAL_DIR_PATTERN = "local_*" + +# A smaller value will minimise memory requirements. A bigger value can make the transfer faster. +S3_MAX_CONCURRENT_REQUESTS = 1 +PASSPHRASE_FILE = os.path.join(synapse.SYNAPSE_CONFIG_DIR, ".gpg_backup_passphrase") # nosec +BASH_COMMAND = "bash" +BACKUP_ID_FORMAT = "%Y%m%d%H%M%S" + + logger = logging.getLogger(__name__) +class BackupError(Exception): + """Generic backup Exception.""" + + class S3Error(Exception): """Generic S3 Exception.""" @@ -128,3 +157,203 @@ def can_use_bucket(self) -> bool: ) return False return True + + +def paths_to_args(paths: Iterable[str]) -> str: + """Given a list of paths, quote and concatenate them for use as cli arguments. + + Args: + paths: List of paths + + Returns: + paths concatenated and quoted + """ + return " ".join(f"'{path}'" for path in paths) + + +def get_paths_to_backup(container: ops.Container) -> Iterable[str]: + """Get the list of paths that should be in a backup for Synapse. + + Args: + container: Synapse Container. + + Returns: + Iterable with the list of paths to backup. + """ + paths = [] + for pattern in BACKUP_FILE_PATTERNS: + paths += container.list_files(synapse.SYNAPSE_CONFIG_DIR, pattern=pattern) + # Local media if it exists + media_dir = os.path.join(synapse.SYNAPSE_DATA_DIR, MEDIA_DIR) + if container.exists(media_dir): + paths += container.list_files(media_dir, pattern=LOCAL_DIR_PATTERN) + return [path.path for path in paths] + + +def calculate_size(container: ops.Container, paths: Iterable[str]) -> int: + """Return the combined size of all the paths given. + + Args: + container: Container where to check the size of the paths. + paths: Paths to check. + + Returns: + Total size in bytes. + + Raises: + BackupError: If there was a problem calculating the size. + """ + command = "set -euxo pipefail; du -bsc " + paths_to_args(paths) + " | tail -n1 | cut -f 1" + exec_process = container.exec([BASH_COMMAND, "-c", command]) + stdout, stderr = exec_process.wait_output() + + logger.info( + "Calculating size of paths. Command: %s. stdout: %s. stderr: %s", command, stdout, stderr + ) + + try: + return int(stdout) + except ValueError as exc: + raise BackupError("Cannot calculate size of paths. Wrong stdout.") from exc + + +def _prepare_container( + container: ops.Container, s3_parameters: S3Parameters, passphrase: str +) -> None: + """Prepare container for create or restore backup. + + This means preparing the required aws configuration and the gpg passphrase file. + + Args: + container: Synapse Container. + s3_parameters: S3 parameters for the backup. + passphrase: Passphrase for the backup (used for gpg). + + Raises: + BackupError: if there was an error preparing the configuration. + """ + aws_set_addressing_style = [ + AWS_COMMAND, + "configure", + "set", + "default.s3.addressing_style", + s3_parameters.addressing_style, + ] + + aws_set_concurrent_requests = [ + AWS_COMMAND, + "configure", + "set", + "default.s3.max_concurrent_requests", + str(S3_MAX_CONCURRENT_REQUESTS), + ] + + try: + process = container.exec(aws_set_addressing_style) + process.wait() + process = container.exec(aws_set_concurrent_requests) + process.wait() + except (APIError, ExecError) as exc: + raise BackupError("Backup Failed. Error configuring AWS.") from exc + + try: + container.push(PASSPHRASE_FILE, passphrase) + except ops.pebble.PathError as exc: + raise BackupError("Backup Failed. Error configuring GPG passphrase.") from exc + + +def _get_environment(s3_parameters: S3Parameters) -> Dict[str, str]: + """Get the environment variables for backup that configure aws S3 cli. + + Args: + s3_parameters: S3 parameters. + + Returns: + A dictionary with aws s3 configuration variables. + """ + environment = { + "AWS_ACCESS_KEY_ID": s3_parameters.access_key, + "AWS_SECRET_ACCESS_KEY": s3_parameters.secret_key, + } + if s3_parameters.endpoint: + environment["AWS_ENDPOINT_URL"] = s3_parameters.endpoint + if s3_parameters.region: + environment["AWS_DEFAULT_REGION"] = s3_parameters.region + return environment + + +def _build_backup_command( + s3_parameters: S3Parameters, + backup_id: str, + backup_paths: Iterable[str], + passphrase_file: str, + expected_size: int = int(1e10), +) -> List[str]: + """Build the command to execute the backup. + + Args: + s3_parameters: S3 parameters. + backup_id: The name of the object to back up. + backup_paths: List of paths to back up. + passphrase_file: Passphrase to use to encrypt the backup file. + expected_size: expected size of the backup, so AWS S3 Client can calculate + a reasonable size for the upload parts. + + Returns: + The backup command to execute. + """ + bash_strict_command = "set -euxo pipefail; " + paths = paths_to_args(backup_paths) + tar_command = f"tar -c {paths}" + gpg_command = f"gpg --batch --no-symkey-cache --passphrase-file {passphrase_file} --symmetric" + aws_command = f"{AWS_COMMAND} s3 cp --expected-size={expected_size} - " + aws_command += f"'s3://{s3_parameters.bucket}/{s3_parameters.path}/{backup_id}'" + full_command = bash_strict_command + " | ".join((tar_command, gpg_command, aws_command)) + return [BASH_COMMAND, "-c", full_command] + + +def create_backup( + container: ops.Container, + s3_parameters: S3Parameters, + passphrase: str, + backup_id: Optional[str] = None, +) -> str: + """Create a backup for Synapse running it in the workload. + + Args: + container: Synapse Container + s3_parameters: S3 parameters for the backup. + passphrase: Passphrase use to encrypt the backup. + backup_id: Name of the object in the backup. + It will be autogenerated if it is not set. + + Returns: + The backup key used for the backup. + + Raises: + BackupError: If there was an error creating the backup. + """ + if backup_id is None: + backup_id = datetime.datetime.now().strftime(BACKUP_ID_FORMAT) + + _prepare_container(container, s3_parameters, passphrase) + paths_to_backup = get_paths_to_backup(container) + logger.info("Paths to back up: %s.", list(paths_to_backup)) + if not paths_to_backup: + raise BackupError("Backup Failed. No paths to back up.") + + expected_size = calculate_size(container, paths_to_backup) + backup_command = _build_backup_command( + s3_parameters, backup_id, paths_to_backup, PASSPHRASE_FILE, expected_size + ) + + logger.info("Backup command: %s", backup_command) + environment = _get_environment(s3_parameters) + try: + exec_process = container.exec(backup_command, environment=environment) + stdout, stderr = exec_process.wait_output() + except (APIError, ExecError) as exc: + raise BackupError("Backup Command Failed.") from exc + + logger.info("Backup command output: %s. %s.", stdout, stderr) + return backup_id diff --git a/src/backup_observer.py b/src/backup_observer.py index 331f5176..8ad1d439 100644 --- a/src/backup_observer.py +++ b/src/backup_observer.py @@ -7,9 +7,12 @@ import ops from charms.data_platform_libs.v0.s3 import CredentialsChangedEvent, S3Requirer +from ops.charm import ActionEvent from ops.framework import Object +from ops.pebble import APIError, ExecError import backup +import synapse logger = logging.getLogger(__name__) @@ -36,6 +39,7 @@ def __init__(self, charm: ops.CharmBase): self._s3_client.on.credentials_changed, self._on_s3_credential_changed ) self.framework.observe(self._s3_client.on.credentials_gone, self._on_s3_credential_gone) + self.framework.observe(self._charm.on.create_backup_action, self._on_create_backup_action) def _on_s3_credential_changed(self, _: CredentialsChangedEvent) -> None: """Check new S3 credentials set the unit to blocked if they are wrong.""" @@ -61,3 +65,32 @@ def _on_s3_credential_changed(self, _: CredentialsChangedEvent) -> None: def _on_s3_credential_gone(self, _: CredentialsChangedEvent) -> None: """Handle s3 credentials gone. Set unit status to active.""" self._charm.unit.status = ops.ActiveStatus() + + def _on_create_backup_action(self, event: ActionEvent) -> None: + """Create new backup of Synapse data. + + Args: + event: Event triggering the create backup action. + """ + try: + s3_parameters = backup.S3Parameters(**self._s3_client.get_s3_connection_info()) + except ValueError: + logger.exception("Wrong S3 configuration in backup action") + event.fail("Wrong S3 configuration on create backup action. Check S3 integration.") + return + + backup_passphrase = self._charm.config.get("backup_passphrase") + if not backup_passphrase: + event.fail("Missing backup_passphrase config option.") + return + + container = self._charm.unit.get_container(synapse.SYNAPSE_CONTAINER_NAME) + + try: + backup_id = backup.create_backup(container, s3_parameters, backup_passphrase) + except (backup.BackupError, APIError, ExecError): + logger.exception("Error Creating Backup.") + event.fail("Error Creating Backup.") + return + + event.set_results({"result": "correct", "backup-id": backup_id}) diff --git a/synapse_rock/rockcraft.yaml b/synapse_rock/rockcraft.yaml index 78e1db71..68cf28c1 100644 --- a/synapse_rock/rockcraft.yaml +++ b/synapse_rock/rockcraft.yaml @@ -37,6 +37,8 @@ parts: - coreutils - curl - gosu + - gpg + - gpg-agent - libffi-dev - libicu70 - libjemalloc2 @@ -48,6 +50,7 @@ parts: - python3 - xmlsec1 stage-snaps: + - aws-cli - mjolnir/latest/edge - matrix-appservice-irc/latest/edge plugin: nil diff --git a/tests/integration/test_charm.py b/tests/integration/test_charm.py index 3d259a98..cb13d0f7 100644 --- a/tests/integration/test_charm.py +++ b/tests/integration/test_charm.py @@ -9,6 +9,7 @@ import typing from secrets import token_hex +import magic import pytest import requests from juju.action import Action @@ -582,3 +583,67 @@ async def test_synapse_enable_s3_backup_integration_no_bucket( await model.wait_for_idle(apps=[synapse_app.name], idle_period=5, status="blocked") assert synapse_app.units[0].workload_status == "blocked" assert "bucket does not exist" in synapse_app.units[0].workload_status_message + + +@pytest.mark.usefixtures("s3_backup_bucket") +async def test_synapse_create_backup_correct( + model: Model, + synapse_app: Application, + s3_integrator_app_backup: Application, + s3_backup_configuration: dict, + boto_s3_client: typing.Any, +): + """ + arrange: Synapse app with s3_integrator. Set backup_passphrase + act: Run create-backup action + assert: Correct response from the action that includes the backup-id. + An encrypted object was created in S3 with the correct name. + """ + await model.add_relation(s3_integrator_app_backup.name, f"{synapse_app.name}:backup") + passphrase = token_hex(16) + await synapse_app.set_config({"backup_passphrase": passphrase}) + await model.wait_for_idle( + idle_period=30, + apps=[synapse_app.name, s3_integrator_app_backup.name], + status=ACTIVE_STATUS_NAME, + ) + + synapse_unit: Unit = next(iter(synapse_app.units)) + backup_action: Action = await synapse_unit.run_action("create-backup") + await backup_action.wait() + + assert backup_action.status == "completed" + assert "backup-id" in backup_action.results + bucket_name = s3_backup_configuration["bucket"] + object_key = s3_backup_configuration["path"] + "/" + backup_action.results["backup-id"] + s3objresp = boto_s3_client.get_object(Bucket=bucket_name, Key=object_key) + objbuf = s3objresp["Body"].read() + assert "GPG symmetrically encrypted data (AES256 cipher)" in magic.from_buffer(objbuf) + + +@pytest.mark.usefixtures("s3_backup_bucket") +async def test_synapse_create_backup_no_passphrase( + model: Model, + synapse_app: Application, + s3_integrator_app_backup: Application, +): + """ + arrange: Synapse app with s3_integrator. + act: Run create-backup action + assert: The action fails because there is no passphrase. + """ + await synapse_app.reset_config(["backup_passphrase"]) + await model.add_relation(s3_integrator_app_backup.name, f"{synapse_app.name}:backup") + await model.wait_for_idle( + idle_period=30, + apps=[synapse_app.name, s3_integrator_app_backup.name], + status=ACTIVE_STATUS_NAME, + ) + + synapse_unit: Unit = next(iter(synapse_app.units)) + backup_action: Action = await synapse_unit.run_action("create-backup") + await backup_action.wait() + + assert backup_action.status == "failed" + assert "backup-id" not in backup_action.results + assert "Missing backup_passphrase" in backup_action.message diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index 23fad051..61b2cb90 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -68,6 +68,10 @@ def wait_output(self): stderr=self._stderr, ) + def wait(self): + """Simulate the wait method of the container object.""" + self.wait_output() + def exec_stub(command: list[str], **_kwargs): """A mock implementation of the `exec` method of the container object. diff --git a/tests/unit/test_backup.py b/tests/unit/test_backup.py index d52ce271..63c65001 100644 --- a/tests/unit/test_backup.py +++ b/tests/unit/test_backup.py @@ -5,13 +5,17 @@ # pylint: disable=protected-access +import os +import pathlib from secrets import token_hex from unittest.mock import MagicMock import pytest from botocore.exceptions import ClientError +from ops.testing import Harness import backup +import synapse def test_s3_relation_validation_fails_when_region_and_endpoint_not_set(): @@ -163,3 +167,276 @@ def test_can_use_bucket_bucket_error(s3_parameters_backup, monkeypatch: pytest.M ) assert not s3_client.can_use_bucket() + + +def test_get_paths_to_backup_correct(harness: Harness): + """ + arrange: Create a container filesystem like the one in Synapse, with data and config. + act: Run get_paths_to_backup. + assert: Check that sqlite homeserver db, the signing key and the local media paths are + in the paths to backup, and nothing else. + """ + container = harness.model.unit.get_container(synapse.SYNAPSE_CONTAINER_NAME) + synapse_root = harness.get_filesystem_root(container) + config_dir = synapse_root / pathlib.Path(synapse.SYNAPSE_CONFIG_DIR).relative_to("/") + (config_dir / "example.com.signing.key").open("w").write("backup") + (config_dir / "log.config").open("w").write("do not backup") + (config_dir / "homeserver.yaml").open("w").write("do not backup") + data_dir = synapse_root / pathlib.Path(synapse.SYNAPSE_DATA_DIR).relative_to("/") + (data_dir / "homeserver.db").open("w").write("backup") + media_dir = data_dir / "media" + media_dir.mkdir() + local_content_dir = media_dir / "local_content" + local_content_dir.mkdir() + (local_content_dir / "onefile").open("w").write("backup") + remote_content_dir = media_dir / "remote_content" + remote_content_dir.mkdir() + (remote_content_dir / "onefile").open("w").write("do not backup") + + paths_to_backup = list(backup.get_paths_to_backup(container)) + + assert len(paths_to_backup) == 3 + assert os.path.join(synapse.SYNAPSE_CONFIG_DIR, "example.com.signing.key") in paths_to_backup + assert os.path.join(synapse.SYNAPSE_DATA_DIR, "homeserver.db") in paths_to_backup + assert os.path.join(synapse.SYNAPSE_DATA_DIR, "media", "local_content") in paths_to_backup + + +def test_get_paths_to_backup_empty(harness: Harness): + """ + arrange: Create an empty container filesystem. + act: Call get_paths_to_backup + assert: The paths to backup should be empty. + """ + container = harness.model.unit.get_container(synapse.SYNAPSE_CONTAINER_NAME) + + paths_to_backup = list(backup.get_paths_to_backup(container)) + + assert len(paths_to_backup) == 0 + + +def test_calculate_size(harness: Harness): + """ + arrange: given a container and a list of paths + act: call backup.calculate_size + assert: exec is run in the container, with the correct command and the stdout is parsed to int. + """ + container = harness.model.unit.get_container(synapse.SYNAPSE_CONTAINER_NAME) + paths = ["path1", "path2"] + + def du_handler(argv: list[str]) -> synapse.ExecResult: + """Handler for the exec of bash. + + Args: + argv: argument given to the container.exec. + + Returns: + tuple with status_code, stdout and stderr. + """ + assert argv == [ + "bash", + "-c", + "set -euxo pipefail; du -bsc 'path1' 'path2' | tail -n1 | cut -f 1", + ] + return synapse.ExecResult(0, "1000", "") + + # A better option would be to use run harness.handle_exec, + # but the harness is monkey patched in conftest.py + harness.register_command_handler( # type: ignore # pylint: disable=no-member + container=container, + executable="bash", + handler=du_handler, + ) + + size = backup.calculate_size(container, paths) + + assert size == 1000 + + +def test_prepare_container_correct(harness: Harness, s3_parameters_backup): + """ + arrange: Given the Synapse container, s3parameters, passphrase and its location + act: Call _prepare_container + assert: The file with the passphare is in the container. AWS commands did not fail. + """ + passphrase = token_hex(16) + container = harness.model.unit.get_container(synapse.SYNAPSE_CONTAINER_NAME) + synapse_root = harness.get_filesystem_root(container) + passphrase_relative_dir = pathlib.Path(backup.PASSPHRASE_FILE).relative_to("/").parent + passphrase_dir = synapse_root / passphrase_relative_dir + passphrase_dir.mkdir(exist_ok=True) + + def aws_command_handler(argv: list[str]) -> synapse.ExecResult: + """Handler for the exec of the aws command. + + Args: + argv: argument given to the container.exec. + + Returns: + tuple with status_code, stdout and stderr. + """ + assert argv[0:3] == [ + backup.AWS_COMMAND, + "configure", + "set", + ] + # let the rest of checks for the integration tests. + return synapse.ExecResult(0, "", "") + + harness.register_command_handler( # type: ignore # pylint: disable=no-member + container=container, + executable=backup.AWS_COMMAND, + handler=aws_command_handler, + ) + + backup._prepare_container(container, s3_parameters_backup, passphrase) + + assert container.pull(backup.PASSPHRASE_FILE).read() == passphrase + + +def test_prepare_container_error_aws(harness: Harness, s3_parameters_backup): + """ + arrange: Given the Synapse container, s3parameters, passphrase and its location + mock container.exec when aws commands are called. + act: Call prepare_container + assert: BackupError exception is raised. + """ + passphrase = token_hex(16) + container = harness.model.unit.get_container(synapse.SYNAPSE_CONTAINER_NAME) + synapse_root = harness.get_filesystem_root(container) + passphrase_relative_dir = pathlib.Path(backup.PASSPHRASE_FILE).relative_to("/").parent + passphrase_dir = synapse_root / passphrase_relative_dir + passphrase_dir.mkdir(exist_ok=True) + + def aws_command_handler(_: list[str]) -> synapse.ExecResult: + """Handler for the exec of the aws command. + + Returns: + tuple with status_code, stdout and stderr. + """ + # let the rest of checks for the integration tests. + return synapse.ExecResult(1, "", "error") + + harness.register_command_handler( # type: ignore # pylint: disable=no-member + container=container, + executable=backup.AWS_COMMAND, + handler=aws_command_handler, + ) + + with pytest.raises(backup.BackupError) as err: + backup._prepare_container(container, s3_parameters_backup, passphrase) + assert "Error configuring AWS" in str(err.value) + + +def test_build_backup_command_correct(s3_parameters_backup): + """ + arrange: Given some s3 parameters for backup, a name for the key in the bucket, + paths, passphrase file location and passphrase file + act: run _build_backup_command + assert: the command is the correct calling bash with pipes. + """ + # pylint: disable=line-too-long + paths_to_backup = ["/data/homeserver.db", "/data/example.com.signing.key"] + + command = backup._build_backup_command( + s3_parameters_backup, "20230101231200", paths_to_backup, "/root/.gpg_passphrase" + ) + + assert list(command) == [ + backup.BASH_COMMAND, + "-c", + f"set -euxo pipefail; tar -c '/data/homeserver.db' '/data/example.com.signing.key' | gpg --batch --no-symkey-cache --passphrase-file /root/.gpg_passphrase --symmetric | {backup.AWS_COMMAND} s3 cp --expected-size=10000000000 - 's3://synapse-backup-bucket//synapse-backups/20230101231200'", # noqa: E501 + ] + + +def test_create_backup_correct( + harness: Harness, s3_parameters_backup, monkeypatch: pytest.MonkeyPatch +): + """ + arrange: Given the Synapse container, s3parameters, passphrase, the backup key and its location + mock prepare_container, calculate_size and get paths + act: Call prepare_container + assert: A command is executed to backup and has at least the paths. + """ + container = harness.model.unit.get_container(synapse.SYNAPSE_CONTAINER_NAME) + passphrase = token_hex(16) + backup_id = token_hex(16) + monkeypatch.setattr(backup, "_prepare_container", MagicMock()) + monkeypatch.setattr(backup, "calculate_size", MagicMock(return_value=1000)) + monkeypatch.setattr(backup, "get_paths_to_backup", MagicMock(return_value=["file1", "dir1"])) + + def backup_command_handler(args: list[str]) -> synapse.ExecResult: + """Handler for the exec of the backup command. + + Args: + args: argument given to the container.exec. + + Returns: + tuple with status_code, stdout and stderr. + """ + # simple check to see that at least the files are in the command + assert any(("'file1'" in arg for arg in args)) + assert any(("'dir1'" in arg for arg in args)) + return synapse.ExecResult(0, "", "") + + harness.register_command_handler( # type: ignore # pylint: disable=no-member + container=container, + executable=backup.BASH_COMMAND, + handler=backup_command_handler, + ) + + backup.create_backup(container, s3_parameters_backup, backup_id, passphrase) + + +def test_create_backup_no_files( + harness: Harness, s3_parameters_backup, monkeypatch: pytest.MonkeyPatch +): + """ + arrange: Given the Synapse container, s3parameters, passphrase, the backup key and its location + mock _prepare_container, calculate_size and get paths. get paths is empty + act: Call create_backup + assert: BackupError exception because there is nothing to back up + """ + container = harness.model.unit.get_container(synapse.SYNAPSE_CONTAINER_NAME) + passphrase = token_hex(16) + backup_id = token_hex(16) + monkeypatch.setattr(backup, "_prepare_container", MagicMock()) + monkeypatch.setattr(backup, "calculate_size", MagicMock(return_value=1000)) + monkeypatch.setattr(backup, "get_paths_to_backup", MagicMock(return_value=[])) + with pytest.raises(backup.BackupError) as err: + backup.create_backup(container, s3_parameters_backup, backup_id, passphrase) + assert "No paths to back up" in str(err.value) + + +def test_create_backup_failure( + harness: Harness, s3_parameters_backup, monkeypatch: pytest.MonkeyPatch +): + """ + arrange: Given the Synapse container, s3parameters, passphrase, the backup key and its location + mock prepare_container, calculate_size and get paths. Mock the backup command to fail + act: Call create_backup + assert: BackupError exception because the back up failed + """ + container = harness.model.unit.get_container(synapse.SYNAPSE_CONTAINER_NAME) + passphrase = token_hex(16) + backup_id = token_hex(16) + monkeypatch.setattr(backup, "_prepare_container", MagicMock()) + monkeypatch.setattr(backup, "calculate_size", MagicMock(return_value=1000)) + monkeypatch.setattr(backup, "get_paths_to_backup", MagicMock(return_value=["file1", "dir1"])) + + def backup_command_handler(_: list[str]) -> synapse.ExecResult: + """Handler for the exec of the backup command. + + Returns: + tuple with status_code, stdout and stderr. + """ + # simple check to see that at least the files are in the command + return synapse.ExecResult(1, "", "") + + harness.register_command_handler( # type: ignore # pylint: disable=no-member + container=container, + executable=backup.BASH_COMMAND, + handler=backup_command_handler, + ) + with pytest.raises(backup.BackupError) as err: + backup.create_backup(container, s3_parameters_backup, backup_id, passphrase) + assert "Backup Command Failed" in str(err.value) diff --git a/tests/unit/test_backup_observer.py b/tests/unit/test_backup_observer.py index 9c3f49e5..0a82519a 100644 --- a/tests/unit/test_backup_observer.py +++ b/tests/unit/test_backup_observer.py @@ -9,7 +9,7 @@ import ops import pytest -from ops.testing import Harness +from ops.testing import ActionFailed, Harness import backup @@ -112,3 +112,83 @@ def test_on_s3_credentials_gone_set_active(harness: Harness): harness.remove_relation(relation_id) assert harness.model.unit.status == ops.ActiveStatus() + + +def test_create_backup_correct( + s3_relation_data_backup, harness: Harness, monkeypatch: pytest.MonkeyPatch +): + """ + arrange: start the Synapse charm. Integrate with s3-integrator. + Mock can_use_bucket and create_backup. + act: Run the backup action. + assert: Backup should end correctly, returning correct and the backup name. + """ + monkeypatch.setattr(backup.S3Client, "can_use_bucket", MagicMock(return_value=True)) + monkeypatch.setattr(backup, "create_backup", MagicMock()) + + harness.update_config({"backup_passphrase": token_hex(16)}) + harness.add_relation("backup", "s3-integrator", app_data=s3_relation_data_backup) + harness.begin_with_initial_hooks() + + output = harness.run_action("create-backup") + assert "backup-id" in output.results + assert output.results["result"] == "correct" + + +def test_create_backup_no_passphrase( + s3_relation_data_backup, harness: Harness, monkeypatch: pytest.MonkeyPatch +): + """ + arrange: start the Synapse charm. Integrate with s3-integrator. + Mock can_use_bucket to True and do not set backup_password. + act: Run the backup action. + assert: Backup should fail because of missing backup_passphrase. + """ + monkeypatch.setattr(backup.S3Client, "can_use_bucket", MagicMock(return_value=True)) + monkeypatch.setattr( + backup, "create_backup", MagicMock(side_effect=backup.S3Error("Generic Error")) + ) + + harness.add_relation("backup", "s3-integrator", app_data=s3_relation_data_backup) + harness.begin_with_initial_hooks() + + with pytest.raises(ActionFailed) as err: + harness.run_action("create-backup") + assert "Missing backup_passphrase" in str(err.value.message) + + +def test_create_backup_wrong_backup_failure( + s3_relation_data_backup, harness: Harness, monkeypatch: pytest.MonkeyPatch +): + """ + arrange: start the Synapse charm. Integrate with s3-integrator. Mock create_backup + to fail. + act: Run the backup action. + assert: Backup should fail with error + """ + monkeypatch.setattr(backup.S3Client, "can_use_bucket", MagicMock(return_value=True)) + monkeypatch.setattr( + backup, "create_backup", MagicMock(side_effect=backup.BackupError("Generic Error")) + ) + + harness.add_relation("backup", "s3-integrator", app_data=s3_relation_data_backup) + harness.begin_with_initial_hooks() + harness.update_config({"backup_passphrase": token_hex(16)}) + + with pytest.raises(ActionFailed) as err: + harness.run_action("create-backup") + assert "Error Creating Backup" in str(err.value.message) + + +def test_create_backup_wrong_s3_parameters(harness: Harness): + """ + arrange: start the Synapse charm. Do not integrate with S3. + act: Run the backup action. + assert: Backup should fail with error because there is no S3 integration + """ + harness.begin_with_initial_hooks() + harness.update_config({"backup_passphrase": token_hex(16)}) + + with pytest.raises(ActionFailed) as err: + harness.run_action("create-backup") + assert "Wrong S3 configuration" in str(err.value.message) diff --git a/tox.ini b/tox.ini index 986513ed..256385e3 100644 --- a/tox.ini +++ b/tox.ini @@ -63,6 +63,7 @@ deps = pytest pytest-asyncio pytest-operator + python-magic requests types-PyYAML types-requests @@ -120,6 +121,7 @@ deps = pytest pytest-asyncio pytest-operator + python-magic boto3 # Type error problem with newer version of macaroonbakery macaroonbakery==1.3.2