diff --git a/CHANGELOG.md b/CHANGELOG.md index 16cdfaaf..24f9b089 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,14 @@ Every entry has a category for which we use the following visual abbreviations: ## Unreleased +- 🐞 Threatbus now only attempts to load plugins that are explicitly + listed in the config file. + [#150](https://github.com/tenzir/threatbus/pull/140) + +- 🎁 Many configuration options for `threatbus` and `pyvast-threatbus` now have + default values. See the example configs for a detailed list. + [#150](https://github.com/tenzir/threatbus/pull/140) + - 🐞 The content and format of the `threatbus-zmq-app` plugin's subscription success response has changed. Prior to this change, the plugin used to respond with an endpoint in the `host:port` format, which contained a wrong hostname diff --git a/README.md b/README.md index 1519cbfd..ef96a6cb 100644 --- a/README.md +++ b/README.md @@ -62,15 +62,21 @@ The following example shows how to connect [Zeek][zeek] via Threat Bus. There are more integrations available, so make sure to check out all [Threat Bus projects on PyPI](https://pypi.org/search/?q=threatbus). +The example assumes that `threatbus` is available in your PATH. See the +section on [Installation](#installation) below for more information on how to +get there. + *Start Threat Bus* ```sh -mv config.yaml.example config.yaml # rename example config file threatbus ``` *Start with a specially named config file* +The `config.yaml.example` file in this directory gives an overview of +the available config keys and their default values. + ```sh threatbus -c /path/to/your/special-config.yaml ``` @@ -147,7 +153,7 @@ The integration tests require a local [Zeek][zeek] and [Docker](https://www.docker.com/) installation. -## Plugin Development +## Development Setup a virtual environment and install `threatbus` and some plugins with the in development mode: diff --git a/apps/stix-shifter/Makefile b/apps/stix-shifter/Makefile index 304558c1..bb923789 100644 --- a/apps/stix-shifter/Makefile +++ b/apps/stix-shifter/Makefile @@ -33,4 +33,5 @@ install: .PHONY: dev-mode dev-mode: + pip install ../.. pip install --editable . diff --git a/apps/suricata/Makefile b/apps/suricata/Makefile index 304558c1..f87a73cd 100644 --- a/apps/suricata/Makefile +++ b/apps/suricata/Makefile @@ -33,4 +33,5 @@ install: .PHONY: dev-mode dev-mode: + pip install ../../ pip install --editable . diff --git a/apps/suricata/config.yaml.example b/apps/suricata/config.yaml.example index abc071f5..5f844a52 100644 --- a/apps/suricata/config.yaml.example +++ b/apps/suricata/config.yaml.example @@ -8,7 +8,7 @@ logging: threatbus: localhost:13370 snapshot: 30 # The socket to use for connecting with Suricata. -socket: /var/run/suricata/suricata-command.socket -rules_file: /var/lib/suricata/rules/threatbus.rules +socket: /var/run/suricata/suricata-command.socket # Required. +rules_file: /var/lib/suricata/rules/threatbus.rules # Required. # Interval in seconds to trigger `suricatasc -c ruleset-reload-nonblocking` reload_interval: 60 diff --git a/apps/suricata/suricata_threatbus/suricata.py b/apps/suricata/suricata_threatbus/suricata.py index 803234d3..21df0805 100755 --- a/apps/suricata/suricata_threatbus/suricata.py +++ b/apps/suricata/suricata_threatbus/suricata.py @@ -50,25 +50,23 @@ def validate_config(config: Settings): Validates the given Dynaconf object. Throws if the config is invalid. """ validators = [ - Validator("logging.console", is_type_of=bool, required=True, eq=True) - | Validator("logging.file", is_type_of=bool, required=True, eq=True), + Validator("logging.console", is_type_of=bool, default=True), + Validator("logging.file", is_type_of=bool, default=False), Validator( "logging.console_verbosity", is_in=["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"], - required=True, - when=Validator("logging.console", eq=True), + default="INFO", ), Validator( "logging.file_verbosity", is_in=["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"], - required=True, - when=Validator("logging.file", eq=True), + default="INFO", ), - Validator( - "logging.filename", required=True, when=Validator("logging.file", eq=True) - ), - Validator("threatbus", "socket", "rules_file", required=True), - Validator("snapshot", "reload_interval", is_type_of=int, required=True), + Validator("logging.filename", default="suricata-threatbus.log"), + Validator("threatbus", default="localhost:13370"), + Validator("socket", "rules_file", required=True), + Validator("snapshot", is_type_of=int, default=30), + Validator("reload_interval", is_type_of=int, default=60), ] config.validators.register(*validators) diff --git a/apps/vast/Makefile b/apps/vast/Makefile index 304558c1..f87a73cd 100644 --- a/apps/vast/Makefile +++ b/apps/vast/Makefile @@ -33,4 +33,5 @@ install: .PHONY: dev-mode dev-mode: + pip install ../../ pip install --editable . diff --git a/apps/vast/config.yaml.example b/apps/vast/config.yaml.example index 31011b8a..45e4a5a1 100644 --- a/apps/vast/config.yaml.example +++ b/apps/vast/config.yaml.example @@ -1,8 +1,11 @@ +# All config keys are shown with their default values below, +# except where explicitly marked otherwise. + logging: console: true - console_verbosity: DEBUG - file: true - file_verbosity: DEBUG + console_verbosity: INFO + file: false + file_verbosity: INFO filename: pyvast-threatbus.log metrics: @@ -13,14 +16,14 @@ vast: "localhost:42000" vast_binary: vast threatbus: "localhost:13370" snapshot: 30 -# live-matching requires you to install the VAST matcher plugin +# Live matching requires you to install the VAST matcher plugin. live_match: false retro_match: true retro_match_max_events: 0 # set to 0 for unlimited results retro_match_timeout: 5 # set to 0 for no timeout -# optional. remove the field if you don't want to transform sighting context +# Optional. The default is to not apply any transform context. transform_context: fever alertify --alert-prefix 'MY PREFIX' --extra-key my-ioc --ioc %ioc -# optional. remove the field if you simply want to report back sightings to Threat Bus +# Optional. The default is to report back sightings only to Threat Bus. sink: STDOUT -# limits the amount of concurrent background tasks for querying vast +# Limits the amount of concurrent background tasks for querying vast. max_background_tasks: 100 diff --git a/apps/vast/pyvast_threatbus/pyvast_threatbus.py b/apps/vast/pyvast_threatbus/pyvast_threatbus.py index 2990e862..cf3d4f4f 100755 --- a/apps/vast/pyvast_threatbus/pyvast_threatbus.py +++ b/apps/vast/pyvast_threatbus/pyvast_threatbus.py @@ -68,38 +68,33 @@ def validate_config(config: Settings): Validates the given Dynaconf object. Throws if the config is invalid. """ validators = [ - Validator("logging.console", is_type_of=bool, required=True, eq=True) - | Validator("logging.file", is_type_of=bool, required=True, eq=True), + Validator("logging.console", is_type_of=bool, default=True), + Validator("logging.file", is_type_of=bool, default=False), Validator( "logging.console_verbosity", is_in=["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"], - required=True, when=Validator("logging.console", eq=True), + default="INFO", ), Validator( "logging.file_verbosity", is_in=["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"], - required=True, when=Validator("logging.file", eq=True), + default="INFO", ), - Validator( - "logging.filename", required=True, when=Validator("logging.file", eq=True) - ), - Validator( - "vast", "vast_binary", "threatbus", "metrics.filename", required=True - ), - Validator("live_match", "retro_match", is_type_of=bool, required=True), - Validator( - "snapshot", - "retro_match_max_events", - "max_background_tasks", - "metrics.interval", - is_type_of=int, - required=True, - ), - Validator("retro_match_timeout", is_type_of=float, required=True), + Validator("logging.filename", default="pyvast-threatbus.log"), + Validator("vast", default="localhost:42000"), + Validator("vast_binary", default="vast"), + Validator("threatbus", default="localhost:13370"), + Validator("metrics.filename", default="metrics.log"), + Validator("metrics.interval", is_type_of=int, default=10), + Validator("live_match", is_type_of=bool, default=False), + Validator("retro_match", is_type_of=bool, default=True), + Validator("snapshot", is_type_of=int, default=30), + Validator("retro_match_max_events", is_type_of=int, default=0), + Validator("max_background_tasks", is_type_of=int, default=100), + Validator("retro_match_timeout", is_type_of=float, default=5.0), Validator("transform_context", "sink", default=None), - Validator("metrics.interval"), ] config.validators.register(*validators) diff --git a/apps/zmq-app-template/Makefile b/apps/zmq-app-template/Makefile index 304558c1..bb923789 100644 --- a/apps/zmq-app-template/Makefile +++ b/apps/zmq-app-template/Makefile @@ -33,4 +33,5 @@ install: .PHONY: dev-mode dev-mode: + pip install ../.. pip install --editable . diff --git a/config.yaml.example b/config.yaml.example index 9a6e657a..fb8cea59 100644 --- a/config.yaml.example +++ b/config.yaml.example @@ -1,39 +1,52 @@ +# All values below are shown with their default values, except +# for required values which do not have a default value and +# optional settings which are unset by default. +# Note that required settings do not have to be provided via the +# configuration file but can also be passed as environment +# variables; see the README for details. + logging: console: true - console_verbosity: DEBUG + console_verbosity: INFO # One of "DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL". file: false - file_verbosity: DEBUG + file_verbosity: INFO filename: threatbus.log plugins: backbones: + # Requires the 'threatbus-inmem' package to be installed + inmem: {} + + # Requires the 'threatbus-rabbitmq' package to be installed rabbitmq: - host: localhost - port: 5672 - username: guest - password: guest + host: localhost # Required. + port: 5672 # Required. + username: guest # Required. + password: guest # Required. vhost: / exchange_name: threatbus queue: - name_suffix: "my_suffix" # optional. remove property / set empty to use 'hostname' - name_join_symbol: . # queue will be named "threatbus" + join_symbol + name_suffix + name_suffix: "my_suffix" # Optional. Default is the result of `gethostname()`. + name_join_symbol: . # Queue will be named "threatbus" + join_symbol + name_suffix durable: true auto_delete: false lazy: true exclusive: false - max_items: 100000 # optional. remove property / set to 0 to allow infinite length + max_items: 0 # The value 0 to allow infinite length apps: + # Requires the 'threatbus-zeek' package to be installed zeek: host: "127.0.0.1" port: 47761 module_namespace: Tenzir + # Requires the 'threatbus-misp' package to be installed misp: - api: + api: # Optional host: https://localhost ssl: false key: MISP_API_KEY - filter: # filter are optional. you can omit the entire section. + filter: # Optional. - orgs: # org IDs must be strings: https://github.com/MISP/PyMISP/blob/main/pymisp/data/schema.json - "1" - "25" @@ -46,32 +59,36 @@ plugins: - hostname - domain - url + # Requires threatbus-misp[zmp] to be installed. zmq: - host: localhost - port: 50000 - #kafka: - # topics: - # - misp_attribute - # poll_interval: 1.0 - # # All config entries are passed as-is to librdkafka - # # https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md - # config: - # bootstrap.servers: "localhost:9092" - # group.id: "threatbus" - # auto.offset.reset: "earliest" + host: localhost # Required. + port: 50000 # Required. + # Requires threatbus-misp[kafka] to be installed. + kafka: + topics: # Required. + - misp_attribute + poll_interval: 1.0 + # All config entries are passed as-is to librdkafka + # https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md + config: # Required. + bootstrap.servers: "localhost:9092" + group.id: "threatbus" + auto.offset.reset: "earliest" + # Requires the 'threatbus-zmq-app' package to be installed. zmq-app: - host: "127.0.0.1" - manage: 13370 # the port used for management messages - pub: 13371 # the port used to publish messages to connected apps - sub: 13372 # the port used to receive messages from connected apps + host: "127.0.0.1" # Required. + manage: 13370 # Required. The port used for management messages. + pub: 13371 # Required. The port used to publish messages to connected apps. + sub: 13372 # Required. The port used to receive messages from connected apps. + # Requires the 'threatbus-cif3' package to be installed. cif3: api: - host: http://localhost:5000 - ssl: false - token: CIF_TOKEN + host: http://localhost:5000 # Required. + ssl: false # Required. + token: CIF_TOKEN # Required. group: everyone confidence: 7.5 tlp: amber - tags: + tags: # Required. - test - malicious diff --git a/plugins/apps/threatbus_cif3/threatbus_cif3/plugin.py b/plugins/apps/threatbus_cif3/threatbus_cif3/plugin.py index 0142e5f9..175a7b29 100644 --- a/plugins/apps/threatbus_cif3/threatbus_cif3/plugin.py +++ b/plugins/apps/threatbus_cif3/threatbus_cif3/plugin.py @@ -70,13 +70,16 @@ def config_validators() -> List[Validator]: return [ Validator( f"plugins.apps.{plugin_name}.group", + default="everyone", + ), + Validator( f"plugins.apps.{plugin_name}.tlp", - required=True, + default="amber", ), Validator( f"plugins.apps.{plugin_name}.confidence", is_type_of=float, - required=True, + default=7.5, ), Validator( f"plugins.apps.{plugin_name}.tags", diff --git a/plugins/apps/threatbus_misp/threatbus_misp/plugin.py b/plugins/apps/threatbus_misp/threatbus_misp/plugin.py index b3a3e9f6..4e9a02c3 100644 --- a/plugins/apps/threatbus_misp/threatbus_misp/plugin.py +++ b/plugins/apps/threatbus_misp/threatbus_misp/plugin.py @@ -257,7 +257,6 @@ def config_validators() -> List[Validator]: f"plugins.apps.{plugin_name}.kafka.poll_interval", is_type_of=float, default=1.0, - must_exist=True, when=Validator(f"plugins.apps.{plugin_name}.zmq", eq=None), ), Validator( diff --git a/plugins/apps/threatbus_zeek/threatbus_zeek/plugin.py b/plugins/apps/threatbus_zeek/threatbus_zeek/plugin.py index ba8972ff..585810a4 100644 --- a/plugins/apps/threatbus_zeek/threatbus_zeek/plugin.py +++ b/plugins/apps/threatbus_zeek/threatbus_zeek/plugin.py @@ -196,13 +196,18 @@ def config_validators() -> List[Validator]: return [ Validator( f"plugins.apps.{plugin_name}.host", + is_type_of=str, + default="localhost", + ), + Validator( f"plugins.apps.{plugin_name}.module_namespace", - required=True, + is_type_of=str, + default="Tenzir", ), Validator( f"plugins.apps.{plugin_name}.port", is_type_of=int, - required=True, + default=47761, ), ] diff --git a/plugins/backbones/threatbus_rabbitmq/threatbus_rabbitmq/plugin.py b/plugins/backbones/threatbus_rabbitmq/threatbus_rabbitmq/plugin.py index e862843d..49532eae 100644 --- a/plugins/backbones/threatbus_rabbitmq/threatbus_rabbitmq/plugin.py +++ b/plugins/backbones/threatbus_rabbitmq/threatbus_rabbitmq/plugin.py @@ -45,9 +45,17 @@ def config_validators() -> List[Validator]: f"plugins.backbones.{plugin_name}.host", f"plugins.backbones.{plugin_name}.username", f"plugins.backbones.{plugin_name}.password", + is_type_of=str, + required=True, + ), + Validator( f"plugins.backbones.{plugin_name}.vhost", + is_type_of=str, + default="/", + ), + Validator( f"plugins.backbones.{plugin_name}.exchange_name", - required=True, + default="threatbus", ), Validator( f"plugins.backbones.{plugin_name}.port", @@ -56,14 +64,20 @@ def config_validators() -> List[Validator]: ), Validator( f"plugins.backbones.{plugin_name}.queue.durable", - f"plugins.backbones.{plugin_name}.queue.auto_delete", f"plugins.backbones.{plugin_name}.queue.lazy", + is_type_of=bool, + default=True, + ), + Validator( + f"plugins.backbones.{plugin_name}.queue.auto_delete", f"plugins.backbones.{plugin_name}.queue.exclusive", is_type_of=bool, - required=True, + default=False, ), Validator( - f"plugins.backbones.{plugin_name}.queue.name_join_symbol", required=True + f"plugins.backbones.{plugin_name}.queue.name_join_symbol", + required=True, + default=".", ), Validator( f"plugins.backbones.{plugin_name}.queue.name_suffix", diff --git a/threatbus/threatbus.py b/threatbus/threatbus.py index d4998fd8..ad22a18b 100644 --- a/threatbus/threatbus.py +++ b/threatbus/threatbus.py @@ -13,6 +13,11 @@ from threading import Lock from uuid import uuid4 +if sys.version_info >= (3, 8): + from importlib import metadata as importlib_metadata +else: + import importlib_metadata + class ThreatBus(stoppable_worker.StoppableWorker): def __init__( @@ -151,25 +156,26 @@ def run(self): def validate_threatbus_config(config: Settings): """ - Validates the given Dynaconf object. Throws if the config is invalid. + Validates the given Dynaconf object, potentially adding new entries for the default values. + Throws if the config is invalid. """ validators = [ - Validator("logging.console", is_type_of=bool, required=True, eq=True) - | Validator("logging.file", is_type_of=bool, required=True, eq=True), + Validator("logging.console", is_type_of=bool, required=True, default=True), + Validator("logging.file", is_type_of=bool, required=True, default=False), Validator( "logging.console_verbosity", is_in=["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"], - required=True, - when=Validator("logging.console", eq=True), + default="INFO", ), Validator( "logging.file_verbosity", is_in=["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"], - required=True, - when=Validator("logging.file", eq=True), + default="INFO", ), Validator( - "logging.filename", required=True, when=Validator("logging.file", eq=True) + "logging.filename", + required=True, + when=Validator("logging.file", eq=True, default="threatbus.log"), ), Validator("plugins.apps", "plugins.backbones", required=True), ] @@ -177,32 +183,43 @@ def validate_threatbus_config(config: Settings): config.validators.validate() +# Logic for this function taken from pluggy.PluginManager.load_setuptools_entrypoints() +def list_installed(group): + result = [] + for dist in list(importlib_metadata.distributions()): + result += [ep.name for ep in dist.entry_points if ep.group == group] + return result + + def start(config: Settings): + backbones = pluggy.PluginManager("threatbus.backbone") backbones.add_hookspecs(backbonespecs) - backbones.load_setuptools_entrypoints("threatbus.backbone") apps = pluggy.PluginManager("threatbus.app") apps.add_hookspecs(appspecs) - apps.load_setuptools_entrypoints("threatbus.app") tb_logger = logger.setup(config.logging, "threatbus") + installed_apps = set(list_installed("threatbus.app")) configured_apps = set(config.plugins.apps.keys()) - installed_apps = set(dict(apps.list_name_plugin()).keys()) + for app in configured_apps: + apps.load_setuptools_entrypoints("threatbus.app", app) + + installed_backbones = set(list_installed("threatbus.backbone")) + configured_backbones = set(config.plugins.backbones.keys()) + for backbone in configured_backbones: + backbones.load_setuptools_entrypoints("threatbus.backbone", backbone) ## Notify user about configuration mismatches between installed and ## configured plugins. for unwanted_app in installed_apps - configured_apps: - tb_logger.info(f"Disabling installed, but unconfigured app '{unwanted_app}'") - apps.unregister(name=unwanted_app) - configured_backbones = set(config.plugins.backbones.keys()) - installed_backbones = set(dict(backbones.list_name_plugin()).keys()) + tb_logger.info(f"Ignoring installed, but unconfigured app '{unwanted_app}'") + for unwanted_backbones in installed_backbones - configured_backbones: tb_logger.info( - f"Disabling installed, but unconfigured backbones '{unwanted_backbones}'" + f"Ignoring installed, but unconfigured backbones '{unwanted_backbones}'" ) - backbones.unregister(name=unwanted_backbones) for unconfigured_app in (configured_apps - installed_apps).union( configured_backbones - installed_backbones ):