From 6f9e463452bb98132c469db460fa67c2607b4daa Mon Sep 17 00:00:00 2001 From: cccs-rs <62077998+cccs-rs@users.noreply.github.com> Date: Fri, 25 Oct 2024 12:40:35 +0000 Subject: [PATCH 1/4] Update code for backing up all indices from Docker --- docs/installation/deployment_migration.md | 36 +++++++---------------- 1 file changed, 10 insertions(+), 26 deletions(-) diff --git a/docs/installation/deployment_migration.md b/docs/installation/deployment_migration.md index e92a95c58..86ac6ee1f 100644 --- a/docs/installation/deployment_migration.md +++ b/docs/installation/deployment_migration.md @@ -1,10 +1,10 @@ # Docker-Compose → Kubernetes System Migration -This guide assumes you're moving from a vanilla Docker-Compose-based deployment to a Kubernetes deployment. +This guide assumes you're moving from a vanilla Docker-based deployment to a Kubernetes deployment. -!!! caution "No processing on either system should be occurring during this migration process" +!!! warning "No processing on either system should be occurring during this migration process" -## Docker-Compose System +## Docker System ### Create a Docker service for backups Create a `backup` container that has access to the datastore's API and the filestore volume. @@ -46,7 +46,7 @@ Shell into your `backup` container, open the AL CLI tool, and backup the datasto `heuristic, service, service_delta, signature, user, user_avatar, user_favorites, user_settings, workflow` ```bash - python -m assemblyline.run.cli + python -m assemblyline.run.cli <> backup /mount/backup/al_system ls /mount/backup/ ``` @@ -54,28 +54,12 @@ Shell into your `backup` container, open the AL CLI tool, and backup the datasto !!! info "All indices will be backed up" ```bash - python -m assemblyline.run.cli - backup /mount/backup/al_alert alert force * - backup /mount/backup/al_cached_file cached_file force * - backup /mount/backup/al_emptyresult emptyresult force * - backup /mount/backup/al_error error force * - backup /mount/backup/al_file file force * - backup /mount/backup/al_filescore filescore force * - backup /mount/backup/al_heuristic heuristic force * - backup /mount/backup/al_result result force * - backup /mount/backup/al_safelist safelist force * - backup /mount/backup/al_service_delta service_delta force * - backup /mount/backup/al_service service force * - backup /mount/backup/al_signature signature force * - backup /mount/backup/al_submission submission force * - backup /mount/backup/al_submission_summary submission_summary force * - backup /mount/backup/al_submission_tree submission_tree force * - backup /mount/backup/al_user_avatar user_avatar force * - backup /mount/backup/al_user user force * - backup /mount/backup/al_user_settings user_settings force * - backup /mount/backup/al_workflow workflow force * - exit - ls /mount/backup/ + python -c "from assemblyline.run.cli import ALCommandLineInterface + cli = ALCommandLineInterface() + for index in cli.datastore.ds.get_models().keys(): + # This will each index as a separate directory within the mounted backup + cli.do_backup(f'/mount/backup/al_{index} {index} force *') + " ``` ### Backup Filestore directories Copy `al-cache` and `al-storage` to the backup volume From f129b866ba7ab763415e6af8d98cc543722c5125 Mon Sep 17 00:00:00 2001 From: cccs-rs <62077998+cccs-rs@users.noreply.github.com> Date: Fri, 25 Oct 2024 12:52:29 +0000 Subject: [PATCH 2/4] Correct typos --- docs/installation/deployment_migration.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/installation/deployment_migration.md b/docs/installation/deployment_migration.md index 86ac6ee1f..292ea13d4 100644 --- a/docs/installation/deployment_migration.md +++ b/docs/installation/deployment_migration.md @@ -46,7 +46,7 @@ Shell into your `backup` container, open the AL CLI tool, and backup the datasto `heuristic, service, service_delta, signature, user, user_avatar, user_favorites, user_settings, workflow` ```bash - python -m assemblyline.run.cli <> + python -m assemblyline.run.cli backup /mount/backup/al_system ls /mount/backup/ ``` @@ -58,7 +58,7 @@ Shell into your `backup` container, open the AL CLI tool, and backup the datasto cli = ALCommandLineInterface() for index in cli.datastore.ds.get_models().keys(): # This will each index as a separate directory within the mounted backup - cli.do_backup(f'/mount/backup/al_{index} {index} force *') + cli.do_backup(f'/mount/backup/al_{index} {index} force *:*') " ``` ### Backup Filestore directories From 0e14bd052008539e6403b471d209297a084d0366 Mon Sep 17 00:00:00 2001 From: cccs-rs <62077998+cccs-rs@users.noreply.github.com> Date: Fri, 25 Oct 2024 13:00:22 +0000 Subject: [PATCH 3/4] Correct comment --- docs/installation/deployment_migration.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/installation/deployment_migration.md b/docs/installation/deployment_migration.md index 292ea13d4..f577557af 100644 --- a/docs/installation/deployment_migration.md +++ b/docs/installation/deployment_migration.md @@ -57,7 +57,7 @@ Shell into your `backup` container, open the AL CLI tool, and backup the datasto python -c "from assemblyline.run.cli import ALCommandLineInterface cli = ALCommandLineInterface() for index in cli.datastore.ds.get_models().keys(): - # This will each index as a separate directory within the mounted backup + # This will backup each index as a separate directory within the mounted backup folder cli.do_backup(f'/mount/backup/al_{index} {index} force *:*') " ``` From 80fb76a58783ef4627e42f516b90c77b58923fba Mon Sep 17 00:00:00 2001 From: cccs-rs <62077998+cccs-rs@users.noreply.github.com> Date: Fri, 25 Oct 2024 14:36:34 +0000 Subject: [PATCH 4/4] Update restore script --- docs/installation/deployment_migration.md | 30 ++++++----------------- 1 file changed, 8 insertions(+), 22 deletions(-) diff --git a/docs/installation/deployment_migration.md b/docs/installation/deployment_migration.md index f577557af..acf579273 100644 --- a/docs/installation/deployment_migration.md +++ b/docs/installation/deployment_migration.md @@ -97,29 +97,15 @@ Shell into one of the core containers (ie. scaler) and perform the following: have data in every index (ie. alerts). Performing an `ls /mount/backup` will confirm what you have backed up. - ```bash - python -m assemblyline.run.cli - restore /mount/backup/al_alert alert force * - restore /mount/backup/al_cached_file cached_file force * - restore /mount/backup/al_emptyresult emptyresult force * - restore /mount/backup/al_error error force * - restore /mount/backup/al_file file force * - restore /mount/backup/al_filescore filescore force * - restore /mount/backup/al_heuristic heuristic force * - restore /mount/backup/al_result result force * - restore /mount/backup/al_safelist safelist force * - restore /mount/backup/al_service_delta service_delta force * - restore /mount/backup/al_service service force * - restore /mount/backup/al_signature signature force * - restore /mount/backup/al_submission submission force * - restore /mount/backup/al_submission_summary submission_summary force * - restore /mount/backup/al_submission_tree submission_tree force * - restore /mount/backup/al_user_avatar user_avatar force * - restore /mount/backup/al_user user force * - restore /mount/backup/al_user_settings user_settings force * - restore /mount/backup/al_workflow workflow force * - exit + python -c "from assemblyline.run.cli import ALCommandLineInterface + import os + cli = ALCommandLineInterface() + dir = '/mount/backup' + for index in os.listdir(dir): + if os.path.isdir(f'{dir}/{index}'): + cli.do_restore(f'{dir}/{index}') + " ``` ### Migrating the Filestore