Skip to content

Commit

Permalink
Merge pull request #6 from CybercentreCanada/assemblyline/issues/101
Browse files Browse the repository at this point in the history 
Update to use UpdaterClient
  • Loading branch information
@cccs-rs
cccs-rs authored Feb 19, 2024
2 parents 27684b3 + 0d1d226 commit 0ff26ee
Show file tree
Hide file tree
Showing 5 changed files with 105 additions and 64 deletions.
1 change: 0 additions & 1 deletion .vscode/launch.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,6 @@
"UPDATER_DIR": "${workspaceFolder}/updates",
"SERVICE_PATH": "badlist.badlist.Badlist",
"AL_SERVICE_NAME": "Badlist",
"UI_SERVER": "https://nginx/"
},
"justMyCode": false,
},
Expand Down
7 changes: 5 additions & 2 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,18 +9,19 @@ When adding sources to the service, there are two types of expected data formats
- csv
- json

There are also two types of sources for this service:
There are also multiple types of sources for this service:

- blocklist
- malware_family_list
- attribution_list

### Blocklist Data Formats

In order for the service to pull the right IOCs and categorize them per source, you'll have to instruct it on how to using the `config.updater.<source>` key.

Within each `source` map, you'll specify the type of source this is (`blocklist`) as well as set the format (`json` | `csv`).

You'll also have to specify the different IOC types (`domain`, `ip`, `uri`) you expect to find in the data and where.
You'll also have to specify the different IOC types (`domain`, `ip`, `uri`, `md5`, `sha1`, `sha256`, `ssdeep`, `tlsh`) you expect to find in the data and where.

For example if dealing with a CSV file and you expect to find `uri`s in the 3rd column per row:

Expand Down Expand Up @@ -49,3 +50,5 @@ config:
format: json
uri: "bad_uri"
```

You can also override Assemblyline's default scoring of badlist matches (1000 points) by providing a `score` per source.
148 changes: 93 additions & 55 deletions badlist/badlist.py
View file
Original file line number Diff line number Diff line change
@@ -1,9 +1,10 @@
from collections import defaultdict

from assemblyline.common import forge
from assemblyline.common.isotime import epoch_to_iso, now
from assemblyline.common.net import is_valid_ip
from assemblyline_v4_service.common.base import ServiceBase
from assemblyline_v4_service.common.result import Heuristic, Result, ResultOrderedKeyValueSection, ResultSection
from assemblyline_v4_service.common.result import Heuristic, Result, ResultOrderedKeyValueSection, ResultSection

classification = forge.get_classification()

Expand All @@ -15,11 +16,18 @@ def __init__(self, config=None):
self.timeout = 1800
self.similar_api_map = {
"ssdeep": self.api_interface.lookup_badlist_ssdeep,
"tlsh": self.api_interface.lookup_badlist_tlsh
"tlsh": self.api_interface.lookup_badlist_tlsh,
}

# This is a map that allows us to override the default badlist scoring by source
self.source_score_override = {
source_name: int(config["score"])
for source_name, config in self.config.get("updater").items()
if config.get("type") == "blocklist" and config.get("score")
}

def start(self):
self.timeout = self.config.get('cache_timeout_seconds', self.timeout)
self.timeout = self.config.get("cache_timeout_seconds", self.timeout)

def get_tool_version(self):
epoch = now()
Expand All @@ -34,139 +42,169 @@ def execute(self, request):

similar_hash_types = []
hashes = []
if self.config.get('lookup_sha256', False):
if self.config.get("lookup_sha256", False):
hashes.append(request.sha256)
if self.config.get('lookup_sha1', False):
if self.config.get("lookup_sha1", False):
hashes.append(request.sha1)
if self.config.get('lookup_md5', False):
if self.config.get("lookup_md5", False):
hashes.append(request.md5)

if self.config.get('lookup_ssdeep', False):
if self.config.get("lookup_ssdeep", False):
similar_hash_types.append("ssdeep")
if self.config.get('lookup_tlsh', False):
if self.config.get("lookup_tlsh", False):
similar_hash_types.append("tlsh")

# For the list of hashes I'm supposed to check, check them individually
for qhash in hashes:
data = self.api_interface.lookup_badlist(qhash)
if data and data['enabled'] and data['type'] == "file":
if data and data["enabled"] and data["type"] == "file":
# Create the bad section
bad_file_section = ResultSection(
f"{qhash} hash was found in the list of bad files",
heuristic=Heuristic(1),
classification=data.get('classification', classification.UNRESTRICTED))
classification=data.get("classification", classification.UNRESTRICTED),
)

# Add attribution tags
attributions = data.get('attribution', {}) or {}
attributions = data.get("attribution", {}) or {}
for tag_type, values in attributions.items():
if values:
for v in values:
bad_file_section.add_tag(f"attribution.{tag_type}", v)

# Create a sub-section per source
for source in data['sources']:
if source['type'] == 'user':
for source in data["sources"]:
if source["type"] == "user":
msg = f"User {source['name']} deemed this file as bad for the following reason(s):"
else:
msg = f"External badlist source {source['name']} deems this file as bad " \
msg = (
f"External badlist source {source['name']} deems this file as bad "
"for the following reason(s):"
)

bad_file_section.add_subsection(
ResultSection(msg, body="\n".join(source['reason']),
classification=source.get('classification', classification.UNRESTRICTED)))
ResultSection(
msg,
body="\n".join(source["reason"]),
classification=source.get("classification", classification.UNRESTRICTED),
)
)

# Add the bad file section to the results
result.add_section(bad_file_section)

# Add the uri file type data as potential tags to check
tags = request.task.tags
if request.file_type.startswith("uri/") and request.task.fileinfo.uri_info:
tags.setdefault('network.static.uri', [])
tags.setdefault('network.dynamic.uri', [])
tags['network.static.uri'].append(request.task.fileinfo.uri_info.uri)
tags['network.dynamic.uri'].append(request.task.fileinfo.uri_info.uri)
tags.setdefault("network.static.uri", [])
tags.setdefault("network.dynamic.uri", [])
tags["network.static.uri"].append(request.task.fileinfo.uri_info.uri)
tags["network.dynamic.uri"].append(request.task.fileinfo.uri_info.uri)

if is_valid_ip(request.task.fileinfo.uri_info.hostname):
net_type = "ip"
else:
net_type = "domain"

tags.setdefault(f'network.static.{net_type}', [])
tags.setdefault(f'network.dynamic.{net_type}', [])
tags[f'network.static.{net_type}'].append(request.task.fileinfo.uri_info.hostname)
tags[f'network.dynamic.{net_type}'].append(request.task.fileinfo.uri_info.hostname)
tags.setdefault(f"network.static.{net_type}", [])
tags.setdefault(f"network.dynamic.{net_type}", [])
tags[f"network.static.{net_type}"].append(request.task.fileinfo.uri_info.hostname)
tags[f"network.dynamic.{net_type}"].append(request.task.fileinfo.uri_info.hostname)

# Check the list of tags as a batch
badlisted_tags = self.api_interface.lookup_badlist_tags(request.task.tags)
for badlisted in badlisted_tags:
if badlisted and badlisted['enabled'] and badlisted['type'] == "tag":
if badlisted and badlisted["enabled"] and badlisted["type"] == "tag":
# Create the bad section
bad_ioc_section = ResultSection(
f"'{badlisted['tag']['value']}' tag was found in the list of bad IOCs",
heuristic=Heuristic(2),
classification=badlisted.get('classification', classification.UNRESTRICTED),
tags={badlisted['tag']['type']: [badlisted['tag']['value']]})
bad_ioc_section = ResultOrderedKeyValueSection(
title_text=f"'{badlisted['tag']['value']}' tag was found in the list of bad IOCs",
body={
"IOC": badlisted["tag"]["type"],
"IOC Type": badlisted["tag"]["value"],
"First added": badlisted["added"],
"Last updated": badlisted["updated"],
},
classification=badlisted.get("classification", classification.UNRESTRICTED),
tags={badlisted["tag"]["type"]: [badlisted["tag"]["value"]]},
)

# Add attribution tags
attributions = badlisted.get('attribution', {}) or {}
attributions = badlisted.get("attribution", {}) or {}
for tag_type, values in attributions.items():
if values:
for v in values:
bad_ioc_section.add_tag(f"attribution.{tag_type}", v)

# Create a sub-section per source
for source in badlisted['sources']:
if source['type'] == 'user':
msg = f"User {source['name']} deemed the tag as bad for the following reason(s):"
signatures = {}
for source in badlisted["sources"]:
if source["type"] == "user":
msg = f"User '{source['name']}' deemed the tag as bad for the following reason(s):"
else:
msg = f"External badlist source {source['name']} deems the tag as bad for the " \
"following reason(s):"
signatures[source['name']] = 1
msg = f"External source '{source['name']}' deems the tag as bad for the following reason(s):"

bad_ioc_section.add_subsection(
ResultSection(msg, body="\n".join(source['reason']),
classification=source.get('classification', classification.UNRESTRICTED)))

ResultSection(
msg,
body="\n".join(source["reason"]),
classification=source.get("classification", classification.UNRESTRICTED),
)
)

bad_ioc_section.set_heuristic(Heuristic(2, score_map=self.source_score_override, signatures=signatures))
# Add the bad IOC section to the results
result.add_section(bad_ioc_section)

# Check for similarity hashes ssdeep
for hash_type in similar_hash_types:
similar_hashes = self.similar_api_map[hash_type](request.task.fileinfo[hash_type])
for similar in similar_hashes:
if similar and similar['enabled'] and similar['type'] == "file" and \
similar['hashes']['sha256'] != request.sha256:
if (
similar
and similar["enabled"]
and similar["type"] == "file"
and similar["hashes"]["sha256"] != request.sha256
):

# Create the similar section
similar_section = ResultOrderedKeyValueSection(
f"{hash_type.upper()} similarity match: A similar file in the system matches this file",
heuristic=Heuristic(3),
classification=similar.get('classification', classification.UNRESTRICTED))
similar_section.add_item("md5", similar['hashes'].get('md5', None))
similar_section.add_item("sha1", similar['hashes'].get('sha1', None))
similar_section.add_item("sha256", similar['hashes'].get('sha256', None))
similar_section.add_item("ssdeep", similar['hashes'].get('ssdeep', None))
similar_section.add_item("tlsh", similar['hashes'].get('tlsh', None))
similar_section.add_item("size", similar['file'].get('size', None))
similar_section.add_item("type", similar['file'].get('type', None))
classification=similar.get("classification", classification.UNRESTRICTED),
)
similar_section.add_item("md5", similar["hashes"].get("md5", None))
similar_section.add_item("sha1", similar["hashes"].get("sha1", None))
similar_section.add_item("sha256", similar["hashes"].get("sha256", None))
similar_section.add_item("ssdeep", similar["hashes"].get("ssdeep", None))
similar_section.add_item("tlsh", similar["hashes"].get("tlsh", None))
similar_section.add_item("size", similar["file"].get("size", None))
similar_section.add_item("type", similar["file"].get("type", None))

# Add attribution tags
attributions = similar.get('attribution', {}) or {}
attributions = similar.get("attribution", {}) or {}
for tag_type, values in attributions.items():
if values:
for v in values:
similar_section.add_tag(f"attribution.{tag_type}", v)

# Create a sub-section per source
for source in similar['sources']:
if source['type'] == 'user':
for source in similar["sources"]:
if source["type"] == "user":
msg = f"User {source['name']} deemed a similar file as bad for the following reason(s):"
else:
msg = f"External badlist source {source['name']} deems a similar file as bad " \
msg = (
f"External badlist source {source['name']} deems a similar file as bad "
"for the following reason(s):"
)

similar_section.add_subsection(
ResultSection(msg, body="\n".join(source['reason']),
classification=similar.get('classification', classification.UNRESTRICTED)))
ResultSection(
msg,
body="\n".join(source["reason"]),
classification=similar.get("classification", classification.UNRESTRICTED),
)
)

# Add similar section to the result
result.add_section(similar_section)
Expand Down
9 changes: 4 additions & 5 deletions badlist/update_server.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,8 +51,7 @@ def __init__(self, *args, **kwargs):
self.attributions: Set[str] = set()
self.update_queue = Queue()

def do_local_update(self):
...
def do_local_update(self): ...

# A sanity check to make sure we do in fact have things to send to services
def _inventory_check(self) -> bool:
Expand Down Expand Up @@ -96,7 +95,7 @@ def _inventory_check(self) -> bool:

return success

def import_update(self, files_sha256, al_client, source_name, default_classification):
def import_update(self, files_sha256, source_name, default_classification):
blocklist_batch = []

def sanitize_data(data: str, type: str, validate=True) -> List[str]:
Expand Down Expand Up @@ -180,7 +179,7 @@ def prepare_item(bl_item):
[prepare_item(bl_item) for bl_item in badlist_items]
blocklist_batch.extend(badlist_items)
if len(blocklist_batch) > BLOCKLIST_UPDATE_BATCH:
al_client.badlist.add_update_many(blocklist_batch)
self.client.badlist.add_update_many(blocklist_batch)
blocklist_batch.clear()

source_cfg = self._service.config["updater"][source_name]
Expand Down Expand Up @@ -271,7 +270,7 @@ def prepare_item(bl_item):
bl_type="tag" if ioc_type in NETWORK_IOC_TYPES else "file",
)
if blocklist_batch:
al_client.badlist.add_update_many(blocklist_batch)
self.client.badlist.add_update_many(blocklist_batch)

elif source_cfg["type"] == "malware_family_list":
# This source is meant to contributes to the list of valid malware families
Expand Down
4 changes: 3 additions & 1 deletion service_manifest.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,11 +26,13 @@ heuristics:
score: 1000
filetype: "*"
description: This file is found in the list of know bad files
max_score: 1000
- heur_id: 2
name: Badlisted IOC
score: 1000
filetype: "*"
description: This Indicator Of Compromise is found in the list of know bad IOCs
max_score: 1000
- heur_id: 3
name: Badlisted Similar File
score: 500
Expand Down Expand Up @@ -116,7 +118,6 @@ config:
format: csv
sha1: 0


docker_config:
image: ${REGISTRY}cccs/assemblyline-service-badlist:$SERVICE_TAG
cpu_cores: 0.4
Expand All @@ -129,6 +130,7 @@ dependencies:
command: ["python", "-m", "badlist.update_server"]
image: ${REGISTRY}cccs/assemblyline-service-badlist:$SERVICE_TAG
ports: ["5003"]
cpu_cores: 2
ram_mb: 4096
run_as_core: True

Expand Down

0 comments on commit 0ff26ee

Please sign in to comment.