Merge branch 'dev' into add_tag_efficiency

assemblyline_result_sample_service/Dockerfile

@@ -10,9 +10,6 @@ RUN apt-get update && apt-get install -y git && rm -rf /var/lib/apt/lists/*
 
 USER assemblyline
 
-# Install packages for update-server
-RUN pip install gunicorn flask gitpython && rm -rf ~/.cache/pip
-
 # Copy ResultSample service code
 WORKDIR /opt/al_service
 COPY assemblyline_result_sample_service .

assemblyline_result_sample_service/result_sample.py

@@ -145,6 +145,7 @@ def execute(self, request):
             section_color_map = ResultGraphSection(
                 "Example of colormap result section", classification=cl_engine.RESTRICTED)
             section_color_map.set_colormap(cmap_min, cmap_max, cmap_values)
+            section_color_map.promote_as_entropy()
             result.add_section(section_color_map)
 
             # ==================================================================
@@ -345,7 +346,8 @@ def execute(self, request):
                 }}))
             # Optional: Set custom column ordering for table.
             # Column order is automatically inferred/updated on `ResultTableSection.add_row()`.
-            # Passing an empty list ([]) to `ResultTableSection.set_column_order()` will display the columns in alphabetical order
+            # Passing an empty list ([]) to `ResultTableSection.set_column_order()`
+            # will display the columns in alphabetical order
             table_section.set_column_order(['a_str', 'a_bool', 'an_int', 'extra_column_there', 'nested_key_value_pair'])
             result.add_section(table_section)
 
@@ -408,11 +410,11 @@ def execute(self, request):
             # Image Section
             #     This type of section allows the service writer to display images to the user
             image_section = ResultImageSection(request, 'Example of Image section')
-            for x in range(6):
-                image_section.add_image(
-                    os.path.join(os.path.dirname(__file__),
-                                 'data', f'000{x+1}.jpg'),
-                    f'000{x+1}.jpg', f'ResultSample screenshot 000{x+1}', ocr_heuristic_id=6)
+            img_dir = os.path.join(os.path.dirname(__file__), 'data')
+            for fname in sorted(os.listdir(img_dir)):
+                image_section.add_image(os.path.join(img_dir, fname), fname,
+                                        f'ResultSample screenshot {fname.split(".")[0]}', ocr_heuristic_id=6)
+            image_section.promote_as_screenshot()
             result.add_section(image_section)
 
             # ==================================================================

assemblyline_result_sample_service/service_manifest.yml

@@ -42,27 +42,26 @@ is_external: false
 # Number of concurrent services allowed to run at the same time
 licence_count: 0
 
-# Service configuration block (dictionary of config variables)
-#     NOTE: The key names can be anything and the value can be of any types
-# config:
-#   str_config: value1
-#   int_config: 1
-#   list_config: [1, 2, 3, 4]
-#   bool_config: false
+# service configuration block (dictionary of config variables)
+# NOTE: The key names can be anything and the value can be of any types
+config:
+  str_config: value1
+  int_config: 1
+  list_config: [1, 2, 3, 4]
+  bool_config: false
 
-# Submission params block:
-#     A list of submission param object that define parameters
-#     that the user can change about the service for each of its scans
-#     SUPPORTED TYPES: bool, int, str, list
-# submission_params:
-#   - default: ""
-#     name: password
-#     type: str
-#     value: ""
-#   - default: false
-#     name: extra_work
-#     type: bool
-#     value: false
+# submission params block: a list of submission param object that define parameters
+#                          that the user can change about the service for each of its scans
+# supported types: bool, int, str, list
+submission_params:
+  - default: ""
+    name: password
+    type: str
+    value: ""
+  - default: false
+    name: extra_work
+    type: bool
+    value: false
 
 # Service heuristic blocks: List of heuristics object that define the different heuristics used in the service
 heuristics:
@@ -118,30 +117,29 @@ docker_config:
   cpu_cores: 1.0
   ram_mb_min: 128
   ram_mb: 256
-# Dependencies configuration block (Example)
-#
-# dependencies:
-#   updates:
-#     container:
-#       allow_internet_access: true
-#       command: ["python", "-m", "update_server"]
-#       image: ${REGISTRY}cccs/assemblyline-service-resultsample:$SERVICE_TAG
-#       ports: ["5003"]
-#       # Defines resource limits, adjust based on sources to avoid OOM during runtime
-#       # cpu_cores: 2 (default: 1)
-#       # ram_mb: 4096 (default: 512)
-#     run_as_core: True
 
-# Update configuration block (Example)
-#
-# update_config:
-#   # list of source object from where to fetch files for update and what will be the name of those files on disk
-#   sources:
-#     - uri: https://file-examples.com/wp-content/uploads/2017/02/zip_2MB.zip
-#       name: sample_2mb_file
-#     - uri: https://file-examples.com/wp-content/uploads/2017/02/zip_5MB.zip
-#       name: sample_5mb_file
-#   # interval in seconds at which the updater dependency runs
-#   update_interval_seconds: 300
-#   # Should the downloaded files be used to create signatures in the system
-#   generates_signatures: false
+# Dependencies configuration block
+dependencies:
+  updates:
+    container:
+      allow_internet_access: true
+      command: ["python", "-m", "update_server"]
+      image: ${REGISTRY}cccs/assemblyline-service-resultsample:$SERVICE_TAG
+      ports: ["5003"]
+      # Defines resource limits, adjust based on sources to avoid OOM during runtime
+      # cpu_cores: 2 (default: 1)
+      # ram_mb: 4096 (default: 512)
+    run_as_core: True
+
+# Update configuration block
+update_config:
+  # list of source object from where to fetch files for update and what will be the name of those files on disk
+  sources:
+    - uri: https://file-examples.com/wp-content/uploads/2017/02/zip_2MB.zip
+      name: sample_2mb_file
+    - uri: https://file-examples.com/wp-content/uploads/2017/02/zip_5MB.zip
+      name: sample_5mb_file
+  # interval in seconds at which the updater dependency runs
+  update_interval_seconds: 300
+  # Should the downloaded files be used to create signatures in the system
+  generates_signatures: false

assemblyline_v4_service/common/request.py

@@ -1,6 +1,6 @@
 import logging
 import tempfile
-from typing import Any, Dict, Optional, TextIO, Union
+from typing import Any, Dict, List, Optional, TextIO, Union
 
 from assemblyline.common import forge
 from assemblyline.common import log as al_log
@@ -24,17 +24,7 @@ def __init__(self, task: Task) -> None:
         self.log = logging.getLogger(f'assemblyline.service.{task.service_name.lower()}')
 
         self._working_directory = task.working_directory
-        self.deep_scan = task.deep_scan
-        self.extracted = task.extracted
-        self.file_name = task.file_name
-        self.file_type = task.file_type
-        self.file_size = task.file_size
         self._file_path = None
-        self.max_extracted = task.max_extracted
-        self.md5 = task.md5
-        self.sha1 = task.sha1
-        self.sha256 = task.sha256
-        self.sid = task.sid
         self.task = task
 
     def add_extracted(self, path: str, name: str, description: str,
@@ -238,3 +228,43 @@ def temp_submission_data(self, data: Dict[str, Any]) -> None:
         :param data: Temporary submission data
         """
         self.task.temp_submission_data = data
+
+    @property
+    def deep_scan(self) -> bool:
+        return self.task.deep_scan
+
+    @property
+    def extracted(self) -> List[Dict[str, str]]:
+        return self.task.extracted
+
+    @property
+    def file_name(self) -> str:
+        return self.task.file_name
+
+    @property
+    def file_type(self) -> str:
+        return self.task.fileinfo.type
+
+    @property
+    def file_size(self) -> int:
+        return self.task.fileinfo.size
+
+    @property
+    def max_extracted(self) -> int:
+        return self.task.max_extracted
+
+    @property
+    def md5(self) -> str:
+        return self.task.fileinfo.md5
+
+    @property
+    def sha1(self) -> str:
+        return self.task.fileinfo.sha1
+
+    @property
+    def sha256(self) -> str:
+        return self.task.fileinfo.sha256
+
+    @property
+    def sid(self) -> str:
+        return self.task.sid

assemblyline_v4_service/common/result.py

@@ -43,6 +43,11 @@
     ],
 )
 
+PROMOTE_TO = StringTable('PROMOTE_TO', [
+    ('SCREENSHOT', 0),
+    ('ENTROPY', 1)
+])
+
 
 class InvalidHeuristicException(Exception):
     pass
@@ -227,7 +232,7 @@ def body(self) -> str | None:
         if not self._data:
             return None
         elif not isinstance(self._data, str):
-            return json.dumps(self._data)
+            return json.dumps(self._data, allow_nan=False)
         else:
             return self._data
 
@@ -417,7 +422,6 @@ def add_row(self, row: TableRow) -> None:
     def set_column_order(self, order: List[str]):
         self._config = {"column_order": order}
 
-
 class ImageSectionBody(SectionBody):
     def __init__(self, request: ServiceRequest) -> None:
         self._request = request
@@ -516,6 +520,7 @@ def __init__(
         self.zeroize_on_tag_safe = zeroize_on_tag_safe
         self.auto_collapse = auto_collapse
         self.zeroize_on_sig_safe = zeroize_on_sig_safe
+        self._promote_to = None
 
         if isinstance(title_text, list):
             title_text = "".join(title_text)
@@ -549,6 +554,10 @@ def body_config(self):
     def heuristic(self):
         return self._heuristic
 
+    @property
+    def promote_to(self):
+        return self._promote_to
+
     @property
     def subsections(self):
         return self._subsections
@@ -719,6 +728,9 @@ def __init__(self, title_text: Union[str, List], **kwargs):
     def set_colormap(self, cmap_min: int, cmap_max: int, values: List[int]) -> None:
         self.section_body.set_colormap(cmap_min, cmap_max, values)
 
+    def promote_as_entropy(self):
+        self._promote_to = PROMOTE_TO.ENTROPY
+
 
 class ResultURLSection(TypeSpecificResultSection):
     def __init__(self, title_text: Union[str, List], **kwargs):
@@ -806,6 +818,9 @@ def add_image(
 
         return ocr_section
 
+    def promote_as_screenshot(self):
+        self._promote_to = PROMOTE_TO.SCREENSHOT
+
 
 class ResultTimelineSection(TypeSpecificResultSection):
     def __init__(self, title_text: Union[str, List], **kwargs):

assemblyline_v4_service/common/task.py

@@ -49,14 +49,11 @@ def __init__(self, task: ServiceTask):
         self.error_type: str = 'EXCEPTION'
         self.extracted: List[Dict[str, str]] = []
         self.file_name = task.filename
-        self.file_type = task.fileinfo.type
-        self.file_size = task.fileinfo.size
+        self.fileinfo = task.fileinfo
         self.ignore_filtering = task.ignore_filtering
         self.min_classification = task.min_classification.value
         self.max_extracted = task.max_files
         self.metadata = task.metadata
-        self.md5: str = task.fileinfo.md5
-        self.mime: str = task.fileinfo.mime or None
         self.result: Optional[Result] = None
         self.safelist_config: Dict[str, Any] = task.safelist_config
         self.service_config: Dict[str, Any] = dict(task.service_config)
@@ -66,15 +63,12 @@ def __init__(self, task: ServiceTask):
         self.service_name: str = task.service_name
         self.service_tool_version: Optional[str] = None
         self.service_version: Optional[str] = None
-        self.sha1: str = task.fileinfo.sha1
-        self.sha256: str = task.fileinfo.sha256
         self.sid: str = task.sid
         self.supplementary: List[Dict[str, str]] = []
         self.tags = tags
         self.temp_submission_data: Dict[str, Any] = {
             row.name: row.value for row in task.temporary_submission_data
         }
-        self.type: str = task.fileinfo.type
 
     def _add_file(self, path: str, name: str, description: str,
                   classification: Optional[Classification] = None,
@@ -303,3 +297,32 @@ def working_directory(self) -> str:
         if self._working_directory is None:
             self._working_directory = tempfile.mkdtemp(dir=temp_dir)
         return self._working_directory
+
+    @property
+    def file_type(self) -> str:
+        return self.fileinfo.type
+
+    @property
+    def file_size(self) -> int:
+        return self.fileinfo.size
+
+    @property
+    def md5(self) -> str:
+        return self.fileinfo.md5
+
+    @property
+    def mime(self) -> str:
+        return self.fileinfo.mime or None
+
+    @property
+    def sha1(self) -> str:
+        return self.fileinfo.sha1
+
+    @property
+    def sha256(self) -> str:
+        return self.fileinfo.sha256
+
+    # Duplicate of file_type for backward compatibility
+    @property
+    def type(self) -> str:
+        return self.fileinfo.type

assemblyline_v4_service/dev/run_service_once.py

@@ -83,25 +83,21 @@ def try_run(self):
                 k, v = kv.split("=", 1)
                 self.submission_params[k] = v
 
+        file_info.pop("ascii", None)
+        file_info.pop("hex", None)
+        file_info.pop("entropy", None)
         # Create service processing task
         service_task = ServiceTask(dict(
             sid=get_random_id(),
             metadata={},
             service_name=SERVICE_NAME,
             service_config=self.submission_params,
-            fileinfo=dict(
-                magic=file_info['magic'],
-                md5=file_info['md5'],
-                mime=file_info['mime'],
-                sha1=file_info['sha1'],
-                sha256=file_info['sha256'],
-                size=file_info['size'],
-                type=file_info['type'],
-            ),
+            fileinfo=file_info,
             filename=file_name,
             min_classification=forge.get_classification().UNRESTRICTED,
             max_files=501,  # TODO: get the actual value
             ttl=3600
+
         ))
 
         LOG.info(f"Starting task with SID: {service_task.sid}")

assemblyline_v4_service/healthz.py

@@ -32,4 +32,4 @@ def perform_check():
 
 
 if __name__ == '__main__':
-    perform_check()
+    perform_check()

docker/Dockerfile

@@ -16,9 +16,8 @@ COPY setup.py dist* dist/
 RUN pip install --no-cache-dir -f dist --user \
   assemblyline-core==$version \
   assemblyline-service-client==$version \
-  assemblyline-v4-service==$version \
+  assemblyline-v4-service==$version[updater] \
   assemblyline-client \
-  gunicorn[gevent] flask gitpython git-remote-codecommit psutil \
   && rm -rf ~/.cache/pip
 
 # If this is a latest rather than stable build, try to push for unstable packages of assemblyline dependencies