From 3168ad0c2ec1a9f205c257e5db6cba092d056b0f Mon Sep 17 00:00:00 2001 From: cccs-rs <62077998+cccs-rs@users.noreply.github.com> Date: Fri, 25 Oct 2024 18:39:03 +0000 Subject: [PATCH 1/4] Update README --- README.md | 246 +++++++++--------------------------------------------- 1 file changed, 39 insertions(+), 207 deletions(-) diff --git a/README.md b/README.md index aad01862..161f5da2 100644 --- a/README.md +++ b/README.md @@ -1,229 +1,61 @@ +[![Discord](https://img.shields.io/badge/chat-on%20discord-7289da.svg?sanitize=true)](https://discord.gg/GUAy9wErNu) +[![](https://img.shields.io/discord/908084610158714900)](https://discord.gg/GUAy9wErNu) +[![Static Badge](https://img.shields.io/badge/github-assemblyline-blue?logo=github)](https://github.com/CybercentreCanada/assemblyline) +[![Static Badge](https://img.shields.io/badge/github-assemblyline--v4--service-blue?logo=github)](https://github.com/CybercentreCanada/assemblyline-v4-service) +[![GitHub Issues or Pull Requests by label](https://img.shields.io/github/issues/CybercentreCanada/assemblyline/service-base)](https://github.com/CybercentreCanada/assemblyline/issues?q=is:issue+is:open+label:service-base) +[![License](https://img.shields.io/github/license/CybercentreCanada/assemblyline-v4-service)](./LICENSE.md) + # Assemblyline 4 - Service Base This repository provides the base service functionality for Assemblyline 4 services. -## Creating a new Assemblyline service - -### Service file structure - -An Assemblyline service has the following file structure: - -```text -assemblyline-service- -│ -├── Dockerfile -├── .py -└── service_manifest.yml -``` - -This is overview of what each of these does: - -- `Dockerfile` ─ Build file for the service container, see _Dockerfile_ section below for more details -- `.py` ─ Contains main service code -- `service_manifest.yml` ─ Service definition file, see _Service manifest_ section below for more details - - -### Service manifest - -Every service must have a `service_manifest.yml` file in its root directory. The manifest file presents essential information about the service to the Assemblyline core system, information the system must have before it can run the service. - -The diagram below shows all the elements that the manifest file can contain, including a brief description of each. - -```yaml -# Name of the service -name: ResultSample -# Version of the service -version: 1 -# Description of the service -description: > - ALv4 Result example service - - This service provides examples of how to: - - define your service manifest - - use the different section types - - use tags - - use heuristics to score sections - - use the att&ck matrix - - use the updater framework - - define submission parameters - - define service configuration parameters - -# Regex defining the types of files the service accepts and rejects -accepts: .* -rejects: empty|metadata/.* - -# At which stage the service should run (one of: FILTER, EXTRACT, CORE, SECONDARY, POST) -# NOTE: Stages are executed in the order defined in the list -stage: CORE -# Which category the service is part of (one of: Antivirus, Dynamic Analysis, External, Extraction, Filtering, Networking, Static Analysis) -category: Static Analysis - -# Does the service require access to the file to perform its task -# If set to false, the service will only have access to the file metadata (e.g. Hashes, size, type, ...) -file_required: true -# Maximum execution time the service has before it's considered to be timed out -timeout: 60 -# Does the service force the caching of results to be disabled -# (only use for service that will always provided different results each run) -disable_cache: false - -# is the service enabled by default -enabled: true -# does the service make APIs call to other product not part of the assemblyline infrastructure (e.g. VirusTotal, ...) -is_external: false -# Number of concurrent services allowed to run at the same time -licence_count: 0 - -# service configuration block (dictionary of config variables) -# NOTE: The key names can be anything and the value can be of any types -config: - str_config: value1 - int_config: 1 - list_config: [1, 2, 3, 4] - bool_config: false - -# submission params block: a list of submission param object that define parameters -# that the user can change about the service for each of its scans -# supported types: bool, int, str, list -submission_params: - - default: "" - name: password - type: str - value: "" - - default: false - name: extra_work - type: bool - value: false - -# Service heuristic blocks: List of heuristics object that define the different heuristics used in the service -heuristics: - - description: This the first Heuristic for ResultSample service. - filetype: pdf - heur_id: AL_RESULTSAMPLE_1 - name: Masks has PDF - score: 100 - attack_id: T1001 - - description: This is second Heuristic for ResultSample service. - filetype: exe - heur_id: AL_RESULTSAMPLE_2 - name: Drops an exe - score: 1000 - - description: This is third Heuristic for ResultSample service. - filetype: exe - heur_id: AL_RESULTSAMPLE_3 - name: Extraction information - score: 0 - -# Docker configuration block which defines: -# - the name of the docker container that will be created -# - cpu and ram allocation by the container -docker_config: - image: cccs/assemblyline-service-resultsample:latest - cpu_cores: 1.0 - ram_mb_min: 128 - ram_mb: 256 - -# Update configuration block -update_config: - # list of source object from where to fetch files for update and what will be the name of those files on disk - sources: - - uri: https://file-examples.com/wp-content/uploads/2017/02/zip_2MB.zip - name: sample_2mb_file - - uri: https://file-examples.com/wp-content/uploads/2017/02/zip_5MB.zip - name: sample_5mb_file - # intervale in seconds at which the updater runs - update_interval_seconds: 300 - # Should the downloaded files be used to create signatures in the system - generates_signatures: false -``` - -### Dockerfile - -A Dockerfile is required to build the service container that will be executed in the system. - -The following items must be set for all services: - -- All services must be based on the `cccs/assemblyline-v4-service-base:latest` image -- An environment variable must be set for the service path -- Install any service requirements -- Copy the service code into `/opt/al/al_service/` - -```dockerfile -FROM cccs/assemblyline-v4-service-base:latest - -# Set the service path -ENV SERVICE_PATH result_sample.ResultSample +## Image variants and tags -# By default, the base service container as the assemblyline user as the running user -# switch to root to perform installation of dependancies -USER root +| **Tag Type** | **Description** | **Example Tag** | +| :----------: | :----------------------------------------------------------------------------------------------- | :------------------------: | +| latest | The most recent build (can be unstable). | `latest` | +| build_type | The type of build used. `dev` is the latest unstable build. `stable` is the latest stable build. | `stable` or `dev` | +| series | Complete build details, including version and build type: `version.buildType`. | `4.5.stable`, `4.5.1.dev3` | -# See that we all these operations in one line to reduce -# the number of container layers and size of the container -RUN apt-get update && apt-get install -y my_debian_apt_dependency_package && rm -rf /var/lib/apt/lists/* -RUN pip install --no-cache-dir --user my_pip_dependency && rm -rf ~/.cache/pip +## Creating a new Assemblyline service -# Change to the assemblyline user to make sure your service does not run as root -USER assemblyline +You can create a new Assemblyline service by using this [template](https://github.com/CybercentreCanada/assemblyline-service-template): -# Copy the service code in the service directory -WORKDIR /opt/al_service -COPY assemblyline_result_sample_service . +```bash +apt install jq +pip install git+https://github.com/CybercentreCanada/assemblyline-service-template.git +cruft create https://github.com/CybercentreCanada/assemblyline-service-template.git ``` -## Testing an Assemblyline service - -To test an Assemblyline service in standalone mode, the [run_service_once.py](https://github.com/CybercentreCanada/assemblyline-v4-service/src/master/dev/run_service_once.py) script can be used to run a single task through the service for testing. That script does not require that you have a working version of Assemblyline installed, all you need are the Assemblyline python libraries. - -### Setting up dev environment - -**NOTE:** The following environment setup has only been tested on Ubuntu 20.04. - -1. Install required packages - - ``` - sudo apt-get install build-essential libffi-dev python3.7 python3.7-dev python3-pip automake autoconf libtool - ``` - -2. Install Assemblyline v4 service package +## Documentation - ``` - pip install --no-cache-dir --user assemblyline-v4-service - ``` +For more information about service development for Assemblyline, follow this [guide](https://cybercentrecanada.github.io/assemblyline4_docs/developer_manual/services/developing_an_assemblyline_service/). -3. Add your service development directory path (ie. `/home/ubuntu/assemblyline-v4-service`) to the PYTHONPATH environment variable +--- -### Using the `run_service_once.py` script - -#### Steps - -1. Ensure the current working directory is the root of the service directory of the service to be run - - ```shell - cd assemblyline-service- - ``` - -2. From a terminal, run the `run_service_once` script, where `` is the path to the service module and `` is the path of the file to be processed +# Assemblyline 4 - Service Base - ```shell - python3.11 -m assemblyline_v4_service.dev.run_service_once - ``` +Ce référentiel fournit les fonctionnalités de base des services Assemblyline 4. +## Créer un nouveau service Assemblyline -3. The output of the service (`result.json` and extracted/supplementary files) will be located in a directory where the input file is located +Vous pouvez créer un nouveau service Assemblyline en utilisant ce [template](https://github.com/CybercentreCanada/assemblyline-service-template). -#### Example of running the ResultSample service +## Variantes d'images et balises -1. Change working directory to root of the service: +| **Type d'étiquette** | **Description** | **Exemple d'étiquette** | +| :------------------: | :--------------------------------------------------------------------------------------------------------------- | :------------------------: | +| dernière | La version la plus récente (peut être instable). | `latest` | +| build_type | Le type de compilation utilisé. `dev` est la dernière version instable. `stable` est la dernière version stable. | `stable` or `dev` | +| series | Détails complets de la construction, y compris la version et le type de construction : `version.buildType`. | `4.5.stable`, `4.5.1.dev3` | - ```shell - cd assemblyline_result_sample_service - ``` -2. From a terminal, run the `run_service_once` script +```bash +apt install jq +pip install git+https://github.com/CybercentreCanada/assemblyline-service-template.git +cruft create https://github.com/CybercentreCanada/assemblyline-service-template.git +``` - ```shell - python3.11 -m assemblyline_v4_service.dev.run_service_once assemblyline_result_sample_service.result_sample.ResultSample /home/ubuntu/testfile.doc - ``` +## Documentation -3. The `results.json` and any extracted/supplementary files will be outputted to `/home/ubuntu/testfile_resultsample` +Pour plus d'informations sur le développement de services pour Assemblyline, suivez ce [guide](https://cybercentrecanada.github.io/assemblyline4_docs/developer_manual/services/developing_an_assemblyline_service/). From f40af750a2a209f21737dafa3f25bd86c354a816 Mon Sep 17 00:00:00 2001 From: cccs-rs <62077998+cccs-rs@users.noreply.github.com> Date: Mon, 28 Oct 2024 17:44:17 +0000 Subject: [PATCH 2/4] Update documentation --- README.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 161f5da2..fa55a97a 100644 --- a/README.md +++ b/README.md @@ -41,14 +41,13 @@ Ce référentiel fournit les fonctionnalités de base des services Assemblyline Vous pouvez créer un nouveau service Assemblyline en utilisant ce [template](https://github.com/CybercentreCanada/assemblyline-service-template). -## Variantes d'images et balises +## Variantes et étiquettes d'image | **Type d'étiquette** | **Description** | **Exemple d'étiquette** | | :------------------: | :--------------------------------------------------------------------------------------------------------------- | :------------------------: | | dernière | La version la plus récente (peut être instable). | `latest` | | build_type | Le type de compilation utilisé. `dev` est la dernière version instable. `stable` est la dernière version stable. | `stable` or `dev` | -| series | Détails complets de la construction, y compris la version et le type de construction : `version.buildType`. | `4.5.stable`, `4.5.1.dev3` | - +| séries | Le détail de compilation utilisé, incluant la version et le type de compilation : `version.buildType`. | `4.5.stable`, `4.5.1.dev3` | ```bash apt install jq From 576358adf70999329749abe3778027962cce1aed Mon Sep 17 00:00:00 2001 From: cccs-rs <62077998+cccs-rs@users.noreply.github.com> Date: Wed, 18 Dec 2024 16:00:53 +0000 Subject: [PATCH 3/4] Update README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index fa55a97a..dcdb8eaf 100644 --- a/README.md +++ b/README.md @@ -57,4 +57,4 @@ cruft create https://github.com/CybercentreCanada/assemblyline-service-template. ## Documentation -Pour plus d'informations sur le développement de services pour Assemblyline, suivez ce [guide](https://cybercentrecanada.github.io/assemblyline4_docs/developer_manual/services/developing_an_assemblyline_service/). +Pour plus d'informations sur le développement des services pour Assemblyline, suivez ce [guide](https://cybercentrecanada.github.io/assemblyline4_docs/developer_manual/services/developing_an_assemblyline_service/). From 9020d9daf6c9fc226cdc5c678ea0675f27fbc202 Mon Sep 17 00:00:00 2001 From: cccs-rs <62077998+cccs-rs@users.noreply.github.com> Date: Wed, 18 Dec 2024 16:54:50 +0000 Subject: [PATCH 4/4] Fix link to licence --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index dcdb8eaf..6983f3d3 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ [![Static Badge](https://img.shields.io/badge/github-assemblyline-blue?logo=github)](https://github.com/CybercentreCanada/assemblyline) [![Static Badge](https://img.shields.io/badge/github-assemblyline--v4--service-blue?logo=github)](https://github.com/CybercentreCanada/assemblyline-v4-service) [![GitHub Issues or Pull Requests by label](https://img.shields.io/github/issues/CybercentreCanada/assemblyline/service-base)](https://github.com/CybercentreCanada/assemblyline/issues?q=is:issue+is:open+label:service-base) -[![License](https://img.shields.io/github/license/CybercentreCanada/assemblyline-v4-service)](./LICENSE.md) +[![License](https://img.shields.io/github/license/CybercentreCanada/assemblyline-v4-service)](./LICENCE.md) # Assemblyline 4 - Service Base