Merge pull request #113 from CybercentreCanada/persistent-service-update

Persistent service update

assemblyline_result_sample_service/Dockerfile

@@ -3,6 +3,11 @@ FROM cccs/assemblyline-v4-service-base:$branch
 
 ENV SERVICE_PATH result_sample.ResultSample
 
+RUN apt-get update && apt-get install -y git && rm -rf /var/lib/apt/lists/*
+
+# Install packages for update-server
+RUN pip install gunicorn flask gitpython && rm -rf ~/.cache/pip
+
 # Copy ResultSample service code
 WORKDIR /opt/al_service
 COPY assemblyline_result_sample_service .
@@ -13,4 +18,4 @@ USER root
 RUN sed -i -e "s/\$SERVICE_TAG/$version/g" service_manifest.yml
 
 # Switch to assemblyline user
-USER assemblyline
+USER assemblyline

assemblyline_result_sample_service/service_manifest.yml

@@ -104,28 +104,26 @@ docker_config:
   ram_mb: 256
 
 # Dependencies configuration block
-# dependencies:
-#   updates:
-#     container:
-#       allow_internet_access: true
-#       command: ["python", "-m", "update_server"]
-#       image: ${REGISTRY}cccs/assemblyline-service-resultsample:$SERVICE_TAG
-#       ports: ["5003"]
-#       # Upper-bound, adjust based on sources
-#       # cpu_cores: 2
-#       # ram_mb: 4096
-#     run_as_core: True
+dependencies:
+  updates:
+    container:
+      allow_internet_access: true
+      command: ["python", "-m", "update_server"]
+      image: ${REGISTRY}cccs/assemblyline-service-resultsample:$SERVICE_TAG
+      ports: ["5003"]
+      # Defines resource limits, adjust based on sources to avoid OOM during runtime
+      # cpu_cores: 2 (default: 1)
+      # ram_mb: 4096 (default: 512)
+    run_as_core: True
 
 # Update configuration block
 update_config:
   # list of source object from where to fetch files for update and what will be the name of those files on disk
   sources:
-    - name: NSRL_modernm
-      pattern: rds_modernm/NSRLFile.txt
-      uri: https://s3.amazonaws.com/rds.nsrl.nist.gov/RDS/current/rds_modernm.zip
-    - name: NSRL_android
-      pattern: NSRLFile.txt
-      uri: https://s3.amazonaws.com/rds.nsrl.nist.gov/RDS/current/RDS_android.iso
+    - uri: https://file-examples.com/wp-content/uploads/2017/02/zip_2MB.zip
+      name: sample_2mb_file
+    - uri: https://file-examples.com/wp-content/uploads/2017/02/zip_5MB.zip
+      name: sample_5mb_file
   # interval in seconds at which the updater dependency runs
   update_interval_seconds: 300
   # Should the downloaded files be used to create signatures in the system

assemblyline_v4_service/updater/helper.py

@@ -18,14 +18,14 @@ class SkipSource(RuntimeError):
     pass
 
 
-def add_cacert(cert: str):
+def add_cacert(cert: str) -> None:
     # Add certificate to requests
     cafile = certifi.where()
     with open(cafile, 'a') as ca_editor:
         ca_editor.write(f"\n{cert}")
 
 
-def filter_downloads(update_directory, pattern, default_pattern="*"):
+def filter_downloads(update_directory, pattern, default_pattern="*") -> List[Tuple[str, str]]:
     f_files = []
     if not pattern:
         # Regex will either match on the filename or the filepath, either with default or given pattern for source
@@ -135,7 +135,7 @@ def url_download(source: Dict[str, Any], previous_update: int = None,
 
 
 def git_clone_repo(source: Dict[str, Any], previous_update: int = None, default_pattern: str = "*",
-                   logger=None, output_dir: str = None) -> List:
+                   logger=None, output_dir: str = None) -> List[Tuple[str, str]]:
     name = source['name']
     url = source['uri']
     pattern = source.get('pattern', None)