Merge pull request #14 from CybercentreCanada/assemblyline/issues/193

Assemblyline/issues/193
CybercentreCanada · Feb 26, 2024 · c0634c6 · c0634c6
2 parents 89461c1 + b9bbcc5
commit c0634c6
Show file tree

Hide file tree

Showing 2 changed files with 30 additions and 24 deletions.
diff --git a/badlist/badlist.py b/badlist/badlist.py
@@ -116,43 +116,55 @@ def execute(self, request):
         for badlisted in badlisted_tags:
             if badlisted and badlisted["enabled"] and badlisted["type"] == "tag":
                 # Create the bad section
-                bad_ioc_section = ResultOrderedKeyValueSection(
-                    title_text=f"'{badlisted['tag']['value']}' tag was found in the list of bad IOCs",
-                    body={
-                        "IOC": badlisted["tag"]["value"],
-                        "IOC Type": badlisted["tag"]["type"],
-                        "First added": badlisted["added"],
-                        "Last updated": badlisted["updated"],
-                    },
-                    classification=badlisted.get("classification", classification.UNRESTRICTED),
-                    tags={badlisted["tag"]["type"]: [badlisted["tag"]["value"]]},
-                )
+                bad_ioc_section = ResultSection(badlisted["tag"]["value"])
+
+                # Create a metadata body
+                metadata_body = {
+                    "IOC Type": badlisted["tag"]["type"],
+                    "IOC Value": badlisted["tag"]["value"],
+                    "First added": badlisted["added"],
+                    "Last updated": badlisted["updated"],
+                }
 
                 # Add attribution tags
                 attributions = badlisted.get("attribution", {}) or {}
                 for tag_type, values in attributions.items():
                     if values:
                         for v in values:
                             bad_ioc_section.add_tag(f"attribution.{tag_type}", v)
+                        # Add any values that will be tagged in the metadata body
+                        metadata_body[tag_type] = values
+
+                # Add metadata section to parent
+                bad_ioc_section.add_subsection(
+                    ResultOrderedKeyValueSection(
+                        title_text="Metadata",
+                        body=metadata_body,
+                        classification=badlisted.get("classification", classification.UNRESTRICTED),
+                        tags={badlisted["tag"]["type"]: [badlisted["tag"]["value"]]},
+                    )
+                )
 
                 # Create a sub-section per source
-                signatures = {}
                 for source in badlisted["sources"]:
+                    signatures = {}
                     if source["type"] == "user":
-                        msg = f"User '{source['name']}' deemed the tag as bad for the following reason(s):"
+                        msg = f"User '{source['name']}' deemed the tag as bad"
                     else:
                         signatures[source["name"]] = 1
-                        msg = f"External source '{source['name']}' deems the tag as bad for the following reason(s):"
+                        msg = f"External source '{source['name']}' deems the tag as bad"
 
+                    source_classfication = source.pop("classification", classification.UNRESTRICTED)
                     bad_ioc_section.add_subsection(
-                        ResultSection(
+                        ResultOrderedKeyValueSection(
                             msg,
-                            body="\n".join(source["reason"]),
-                            classification=source.get("classification", classification.UNRESTRICTED),
+                            body=source,
+                            classification=source_classfication,
+                            heuristic=Heuristic(2, score_map=self.source_score_override, signatures=signatures),
+                            tags={badlisted["tag"]["type"]: [badlisted["tag"]["value"]]},
                         )
                     )
 
-                bad_ioc_section.set_heuristic(Heuristic(2, score_map=self.source_score_override, signatures=signatures))
                 # Add the bad IOC section to the results
                 result.add_section(bad_ioc_section)
 

diff --git a/badlist/update_server.py b/badlist/update_server.py
@@ -1,6 +1,5 @@
 import csv
 import json
-import os
 import re
 from copy import deepcopy
 from queue import Queue
@@ -18,8 +17,6 @@
 )
 from assemblyline_v4_service.updater.updater import ServiceUpdater
 
-BLOCKLIST_UPDATE_BATCH = int(os.environ.get("BLOCKLIST_UPDATE_BATCH", "1000"))
-
 IOC_CHECK = {
     "ip": re.compile(IP_ONLY_REGEX).match,
     "domain": re.compile(DOMAIN_ONLY_REGEX).match,
@@ -182,9 +179,6 @@ def prepare_item(bl_item):
 
             [prepare_item(bl_item) for bl_item in badlist_items]
             blocklist_batch.extend(badlist_items)
-            if len(blocklist_batch) > BLOCKLIST_UPDATE_BATCH:
-                self.client.badlist.add_update_many(blocklist_batch)
-                blocklist_batch.clear()
 
         source_cfg = self._service.config["updater"][source_name]