Merge pull request #150

Allow omitting more config values

CHANGELOG.md

@@ -12,6 +12,14 @@ Every entry has a category for which we use the following visual abbreviations:
 
 ## Unreleased
 
+- 🐞 Threatbus now only attempts to load plugins that are explicitly
+  listed in the config file.
+  [#150](https://github.com/tenzir/threatbus/pull/140)
+
+- 🎁 Many configuration options for `threatbus` and `pyvast-threatbus` now have
+  default values. See the example configs for a detailed list.
+  [#150](https://github.com/tenzir/threatbus/pull/140)
+
 - 🐞 The content and format of the `threatbus-zmq-app` plugin's subscription
   success response has changed. Prior to this change, the plugin used to respond
   with an endpoint in the `host:port` format, which contained a wrong hostname

README.md

@@ -62,15 +62,21 @@ The following example shows how to connect [Zeek][zeek] via Threat Bus. There
 are more integrations available, so make sure to check out all
 [Threat Bus projects on PyPI](https://pypi.org/search/?q=threatbus).
 
+The example assumes that `threatbus` is available in your PATH. See the
+section on [Installation](#installation) below for more information on how to
+get there.
+
 *Start Threat Bus*
 
 ```sh
-mv config.yaml.example config.yaml   # rename example config file
 threatbus
 ```
 
 *Start with a specially named config file*
 
+The `config.yaml.example` file in this directory gives an overview of
+the available config keys and their default values.
+
 ```sh
 threatbus -c /path/to/your/special-config.yaml
 ```
@@ -147,7 +153,7 @@ The integration tests require a local [Zeek][zeek] and
 [Docker](https://www.docker.com/) installation.
 
 
-## Plugin Development
+## Development
 
 Setup a virtual environment and install `threatbus` and some plugins with the
 in development mode:

apps/stix-shifter/Makefile

@@ -33,4 +33,5 @@ install:
 
 .PHONY: dev-mode
 dev-mode:
+	pip install ../..
 	pip install --editable .

apps/suricata/Makefile

@@ -33,4 +33,5 @@ install:
 
 .PHONY: dev-mode
 dev-mode:
+	pip install ../../
 	pip install --editable .

apps/suricata/config.yaml.example

@@ -8,7 +8,7 @@ logging:
 threatbus: localhost:13370
 snapshot: 30
 # The socket to use for connecting with Suricata.
-socket: /var/run/suricata/suricata-command.socket
-rules_file: /var/lib/suricata/rules/threatbus.rules
+socket: /var/run/suricata/suricata-command.socket    # Required.
+rules_file: /var/lib/suricata/rules/threatbus.rules  # Required.
 # Interval in seconds to trigger `suricatasc -c ruleset-reload-nonblocking`
 reload_interval: 60

apps/suricata/suricata_threatbus/suricata.py

@@ -50,25 +50,23 @@ def validate_config(config: Settings):
     Validates the given Dynaconf object. Throws if the config is invalid.
     """
     validators = [
-        Validator("logging.console", is_type_of=bool, required=True, eq=True)
-        | Validator("logging.file", is_type_of=bool, required=True, eq=True),
+        Validator("logging.console", is_type_of=bool, default=True),
+        Validator("logging.file", is_type_of=bool, default=False),
         Validator(
             "logging.console_verbosity",
             is_in=["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"],
-            required=True,
-            when=Validator("logging.console", eq=True),
+            default="INFO",
         ),
         Validator(
             "logging.file_verbosity",
             is_in=["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"],
-            required=True,
-            when=Validator("logging.file", eq=True),
+            default="INFO",
         ),
-        Validator(
-            "logging.filename", required=True, when=Validator("logging.file", eq=True)
-        ),
-        Validator("threatbus", "socket", "rules_file", required=True),
-        Validator("snapshot", "reload_interval", is_type_of=int, required=True),
+        Validator("logging.filename", default="suricata-threatbus.log"),
+        Validator("threatbus", default="localhost:13370"),
+        Validator("socket", "rules_file", required=True),
+        Validator("snapshot", is_type_of=int, default=30),
+        Validator("reload_interval", is_type_of=int, default=60),
     ]
 
     config.validators.register(*validators)

apps/vast/Makefile

@@ -33,4 +33,5 @@ install:
 
 .PHONY: dev-mode
 dev-mode:
+	pip install ../../
 	pip install --editable .

apps/vast/config.yaml.example

@@ -1,8 +1,11 @@
+# All config keys are shown with their default values below,
+# except where explicitly marked otherwise.
+
 logging:
   console: true
-  console_verbosity: DEBUG
-  file: true
-  file_verbosity: DEBUG
+  console_verbosity: INFO
+  file: false
+  file_verbosity: INFO
   filename: pyvast-threatbus.log
 
 metrics:
@@ -13,14 +16,14 @@ vast: "localhost:42000"
 vast_binary: vast
 threatbus: "localhost:13370"
 snapshot: 30
-# live-matching requires you to install the VAST matcher plugin
+# Live matching requires you to install the VAST matcher plugin.
 live_match: false
 retro_match: true
 retro_match_max_events: 0 # set to 0 for unlimited results
 retro_match_timeout: 5 # set to 0 for no timeout
-# optional. remove the field if you don't want to transform sighting context
+# Optional. The default is to not apply any transform context.
 transform_context: fever alertify --alert-prefix 'MY PREFIX' --extra-key my-ioc --ioc %ioc
-# optional. remove the field if you simply want to report back sightings to Threat Bus
+# Optional. The default is to report back sightings only to Threat Bus.
 sink: STDOUT
-# limits the amount of concurrent background tasks for querying vast
+# Limits the amount of concurrent background tasks for querying vast.
 max_background_tasks: 100

apps/vast/pyvast_threatbus/pyvast_threatbus.py

@@ -68,38 +68,33 @@ def validate_config(config: Settings):
     Validates the given Dynaconf object. Throws if the config is invalid.
     """
     validators = [
-        Validator("logging.console", is_type_of=bool, required=True, eq=True)
-        | Validator("logging.file", is_type_of=bool, required=True, eq=True),
+        Validator("logging.console", is_type_of=bool, default=True),
+        Validator("logging.file", is_type_of=bool, default=False),
         Validator(
             "logging.console_verbosity",
             is_in=["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"],
-            required=True,
             when=Validator("logging.console", eq=True),
+            default="INFO",
         ),
         Validator(
             "logging.file_verbosity",
             is_in=["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"],
-            required=True,
             when=Validator("logging.file", eq=True),
+            default="INFO",
         ),
-        Validator(
-            "logging.filename", required=True, when=Validator("logging.file", eq=True)
-        ),
-        Validator(
-            "vast", "vast_binary", "threatbus", "metrics.filename", required=True
-        ),
-        Validator("live_match", "retro_match", is_type_of=bool, required=True),
-        Validator(
-            "snapshot",
-            "retro_match_max_events",
-            "max_background_tasks",
-            "metrics.interval",
-            is_type_of=int,
-            required=True,
-        ),
-        Validator("retro_match_timeout", is_type_of=float, required=True),
+        Validator("logging.filename", default="pyvast-threatbus.log"),
+        Validator("vast", default="localhost:42000"),
+        Validator("vast_binary", default="vast"),
+        Validator("threatbus", default="localhost:13370"),
+        Validator("metrics.filename", default="metrics.log"),
+        Validator("metrics.interval", is_type_of=int, default=10),
+        Validator("live_match", is_type_of=bool, default=False),
+        Validator("retro_match", is_type_of=bool, default=True),
+        Validator("snapshot", is_type_of=int, default=30),
+        Validator("retro_match_max_events", is_type_of=int, default=0),
+        Validator("max_background_tasks", is_type_of=int, default=100),
+        Validator("retro_match_timeout", is_type_of=float, default=5.0),
         Validator("transform_context", "sink", default=None),
-        Validator("metrics.interval"),
     ]
 
     config.validators.register(*validators)

apps/zmq-app-template/Makefile

@@ -33,4 +33,5 @@ install:
 
 .PHONY: dev-mode
 dev-mode:
+	pip install ../..
 	pip install --editable .

config.yaml.example

@@ -1,39 +1,52 @@
+# All values below are shown with their default values, except
+# for required values which do not have a default value and
+# optional settings which are unset by default.
+# Note that required settings do not have to be provided via the
+# configuration file but can also be passed as environment
+# variables; see the README for details.
+
 logging:
   console: true
-  console_verbosity: DEBUG
+  console_verbosity: INFO        # One of "DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL".
   file: false
-  file_verbosity: DEBUG
+  file_verbosity: INFO
   filename: threatbus.log
 
 plugins:
   backbones:
+    # Requires the 'threatbus-inmem' package to be installed
+    inmem: {}
+
+    # Requires the 'threatbus-rabbitmq' package to be installed
     rabbitmq:
-      host: localhost
-      port: 5672
-      username: guest
-      password: guest
+      host: localhost            # Required.
+      port: 5672                 # Required.
+      username: guest            # Required.
+      password: guest            # Required.
       vhost: /
       exchange_name: threatbus
       queue:
-        name_suffix: "my_suffix" # optional. remove property / set empty to use 'hostname'
-        name_join_symbol: . # queue will be named "threatbus" + join_symbol + name_suffix
+        name_suffix: "my_suffix" # Optional. Default is the result of `gethostname()`.
+        name_join_symbol: .      # Queue will be named "threatbus" + join_symbol + name_suffix
         durable: true
         auto_delete: false
         lazy: true
         exclusive: false
-        max_items: 100000 # optional. remove property / set to 0 to allow infinite length
+        max_items: 0             # The value 0 to allow infinite length
 
   apps:
+    # Requires the 'threatbus-zeek' package to be installed
     zeek:
       host: "127.0.0.1"
       port: 47761
       module_namespace: Tenzir
+    # Requires the 'threatbus-misp' package to be installed
     misp:
-      api:
+      api:                       # Optional
         host: https://localhost
         ssl: false
         key: MISP_API_KEY
-      filter: # filter are optional. you can omit the entire section.
+      filter:                    # Optional.
         - orgs: # org IDs must be strings: https://github.com/MISP/PyMISP/blob/main/pymisp/data/schema.json
             - "1"
             - "25"
@@ -46,32 +59,36 @@ plugins:
             - hostname
             - domain
             - url
+      # Requires threatbus-misp[zmp] to be installed.
       zmq:
-        host: localhost
-        port: 50000
-      #kafka:
-      #  topics:
-      #  - misp_attribute
-      #  poll_interval: 1.0
-      #  # All config entries are passed as-is to librdkafka
-      #  # https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md
-      #  config:
-      #    bootstrap.servers: "localhost:9092"
-      #    group.id: "threatbus"
-      #    auto.offset.reset: "earliest"
+        host: localhost          # Required.
+        port: 50000              # Required.
+      # Requires threatbus-misp[kafka] to be installed.
+      kafka:
+        topics:                  # Required.
+        - misp_attribute
+        poll_interval: 1.0
+        # All config entries are passed as-is to librdkafka
+        # https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md
+        config:                  # Required.
+          bootstrap.servers: "localhost:9092"
+          group.id: "threatbus"
+          auto.offset.reset: "earliest"
+    # Requires the 'threatbus-zmq-app' package to be installed.
     zmq-app:
-      host: "127.0.0.1"
-      manage: 13370 # the port used for management messages
-      pub: 13371 # the port used to publish messages to connected apps
-      sub: 13372 # the port used to receive messages from connected apps
+      host: "127.0.0.1"          # Required.
+      manage: 13370              # Required. The port used for management messages.
+      pub: 13371                 # Required. The port used to publish messages to connected apps.
+      sub: 13372                 # Required. The port used to receive messages from connected apps.
+    # Requires the 'threatbus-cif3' package to be installed.
     cif3:
       api:
-        host: http://localhost:5000
-        ssl: false
-        token: CIF_TOKEN
+        host: http://localhost:5000  # Required.
+        ssl: false               # Required.
+        token: CIF_TOKEN         # Required.
       group: everyone
       confidence: 7.5
       tlp: amber
-      tags:
+      tags:                      # Required.
         - test
         - malicious

plugins/apps/threatbus_cif3/threatbus_cif3/plugin.py

@@ -70,13 +70,16 @@ def config_validators() -> List[Validator]:
     return [
         Validator(
             f"plugins.apps.{plugin_name}.group",
+            default="everyone",
+        ),
+        Validator(
             f"plugins.apps.{plugin_name}.tlp",
-            required=True,
+            default="amber",
         ),
         Validator(
             f"plugins.apps.{plugin_name}.confidence",
             is_type_of=float,
-            required=True,
+            default=7.5,
         ),
         Validator(
             f"plugins.apps.{plugin_name}.tags",

plugins/apps/threatbus_misp/threatbus_misp/plugin.py

@@ -257,7 +257,6 @@ def config_validators() -> List[Validator]:
             f"plugins.apps.{plugin_name}.kafka.poll_interval",
             is_type_of=float,
             default=1.0,
-            must_exist=True,
             when=Validator(f"plugins.apps.{plugin_name}.zmq", eq=None),
         ),
         Validator(

plugins/apps/threatbus_zeek/threatbus_zeek/plugin.py

@@ -196,13 +196,18 @@ def config_validators() -> List[Validator]:
     return [
         Validator(
             f"plugins.apps.{plugin_name}.host",
+            is_type_of=str,
+            default="localhost",
+        ),
+        Validator(
             f"plugins.apps.{plugin_name}.module_namespace",
-            required=True,
+            is_type_of=str,
+            default="Tenzir",
         ),
         Validator(
             f"plugins.apps.{plugin_name}.port",
             is_type_of=int,
-            required=True,
+            default=47761,
         ),
     ]