diff --git a/.github/workflows/python-egg.yml b/.github/workflows/python-egg.yml index 60219af8..9bb4baf8 100644 --- a/.github/workflows/python-egg.yml +++ b/.github/workflows/python-egg.yml @@ -5,18 +5,74 @@ env: jobs: egg-install: name: Egg Installation - runs-on: ubuntu-latest + runs-on: ${{ matrix.os }} + strategy: + fail-fast: true + matrix: + python-version: [3.7, 3.8] + os: [ubuntu-latest, macOS-latest] steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v2 with: ref: ${{ github.ref }} - - uses: actions/setup-python@v1 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v1 with: - python-version: '3.8' + python-version: ${{ matrix.python-version }} + - name: Set up Dist Environment + run: | + python -m pip install --upgrade pip + pip install wheel setuptools - name: Install Application and Plugins run: | - pip install --upgrade pip + python setup.py build pip install . pip install plugins/apps/threatbus-zeek pip install plugins/apps/threatbus-misp - pip install plugins/backbones/threatbus-inmem \ No newline at end of file + pip install plugins/backbones/threatbus-inmem + - name: Package Application and Plugins + run: | + python setup.py sdist bdist_wheel + python plugins/apps/threatbus-zeek/setup.py sdist bdist_wheel + python plugins/apps/threatbus-misp/setup.py sdist bdist_wheel + python plugins/backbones/threatbus-inmem/setup.py sdist bdist_wheel + + egg-release: + name: Egg Release + needs: egg-install + runs-on: ubuntu-latest + env: + TWINE_USERNAME: __token__ + TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }} + if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags') + steps: + - uses: actions/checkout@v2 + with: + ref: ${{ github.ref }} + - name: Set up Python 3.8 + uses: actions/setup-python@v1 + with: + python-version: 3.8 + - name: Set up Dist Environment + run: | + python -m pip install --upgrade pip + pip install wheel setuptools twine + - name: Package Application and Plugins + run: | + python setup.py sdist bdist_wheel + python plugins/apps/threatbus-zeek/setup.py sdist bdist_wheel + python plugins/apps/threatbus-misp/setup.py sdist bdist_wheel + python plugins/backbones/threatbus-inmem/setup.py sdist bdist_wheel + - name: GitHub Release + id: create_release + uses: actions/create-release@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + tag_name: ${{ github.ref }} + release_name: Threat Bus ${{ github.ref }} + draft: false + prerelease: false + - name: Publish to PyPI + run: | + python -m twine upload --verbose --non-interactive dist dist/* diff --git a/.gitignore b/.gitignore index 2b65c6c5..c7a6cc5f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,8 @@ *.sw? -env -venv -vast -**/__pycache__ -**/*.egg-info \ No newline at end of file +env/ +build/ +dist/ +venv/ +**/__pycache__/ +**/*.egg-info/ +*tar.gz \ No newline at end of file diff --git a/README.md b/README.md index f5a89705..0ba0f253 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,7 @@ The missing tool to interconnect open-source security applications. +[![PyPI Status][pypi-badge]][pypi-url] [![Build Status][ci-badge]][ci-url] [![Total alerts][lgtm-alerts-badge]][lgtm-alerts-url] [![Language grade: Python][lgtm-quality-badge]][lgtm-quality-url] @@ -63,9 +64,11 @@ environment. ``` virtualenv venv # optional source venv/bin/activate # optional -pip install . -pip install plugins/backbones/threatbus-inmem -pip install plugins/ +pip install threatbus +pip install threatbus-inmem +pip install threatbus-misp +pip install threatbus-zeek +pip install threatbus- ``` ### Testing @@ -114,9 +117,9 @@ Example: ```sh plugins ├── apps - │   └── threatbus-zeek - │   ├── setup.py - │   └── threatbus_zeek.py + | └── threatbus-zeek + │ ├── setup.py + | └── threatbus_zeek.py └── backbones └── threatbus-inmem ├── setup.py @@ -167,6 +170,8 @@ Threat Bus comes with a [3-clause BSD license][license-url]. [zeek]: https://www.zeek.org [misp-zmq-config]: https://github.com/MISP/misp-book/tree/master/misp-zmq#misp-zeromq-configuration +[pypi-badge]: https://img.shields.io/pypi/v/threatbus.svg +[pypi-url]: https://pypi.org/project/threatbus [contributing-url]: https://github.com/tenzir/.github/blob/master/contributing.md [latest-release-badge]: https://img.shields.io/github/commits-since/tenzir/threatbus/latest.svg?color=green [latest-release-url]: https://github.com/tenzir/threatbus/releases diff --git a/plugins/apps/threatbus-misp/README.md b/plugins/apps/threatbus-misp/README.md index 588841a4..11ef8bf1 100644 --- a/plugins/apps/threatbus-misp/README.md +++ b/plugins/apps/threatbus-misp/README.md @@ -1,10 +1,22 @@ Threat Bus MISP Plugin ====================== +

+ +[![PyPI Status][pypi-badge]][pypi-url] +[![Build Status][ci-badge]][ci-url] +[![License][license-badge]][license-url] + +

+ A Threat Bus plugin that enables communication to [MISP](https://www.misp-project.org/). -## Installation +The plugin goes against the pub/sub architecture of Threat Bus (for now), +because the plugin subscribes a listener to ZeroMQ / Kafka, rather than having +MISP subscribe itself to Threat Bus. That will be addressed with a MISP module +in the near future. +## Installation ```sh pip install threatbus-misp @@ -19,6 +31,38 @@ you have to install `librdkafka` for the host system that is running `threatbus`. See also the [prerequisites](https://github.com/confluentinc/confluent-kafka-python#prerequisites) section of the `confluent-kafka` python client. +## Configuration + +The plugin can either use ZeroMQ or Kafka to retrieve intelligence items from +MISP. It uses the MISP REST api to report back sightings of indicators. + +ZeroMQ and Kafka are mutually exclusive, such that Threat Bus does not receive +all attribute updates twice. See below for an example configuration. + + +```yaml +... +plugins: + misp: + api: + host: https://localhost + ssl: false + key: MISP_API_KEY + zmq: + host: localhost + port: 50000 + #kafka: + # topics: + # - misp_attribute + # poll_interval: 1.0 + # # All config entries are passed as-is to librdkafka + # # https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md + # config: + # bootstrap.servers: "localhost:9092" + # group.id: "threatbus" + # auto.offset.reset: "earliest" +... +``` ## Development Setup @@ -135,3 +179,15 @@ exit # leave the Docker container shell ```sh make restart-all ``` + + +## License + +Threat Bus comes with a [3-clause BSD license][license-url]. + +[pypi-badge]: https://img.shields.io/pypi/v/threatbus-misp.svg +[pypi-url]: https://pypi.org/project/threatbus-misp +[ci-url]: https://github.com/tenzir/threatbus/actions?query=branch%3Amaster +[ci-badge]: https://github.com/tenzir/threatbus/workflows/Python%20Egg/badge.svg?branch=master +[license-badge]: https://img.shields.io/badge/license-BSD-blue.svg +[license-url]: https://github.com/tenzir/threatbus/blob/master/COPYING \ No newline at end of file diff --git a/plugins/apps/threatbus-misp/setup.py b/plugins/apps/threatbus-misp/setup.py index dfc17a04..8a44643c 100644 --- a/plugins/apps/threatbus-misp/setup.py +++ b/plugins/apps/threatbus-misp/setup.py @@ -1,14 +1,51 @@ from setuptools import setup +import pathlib + +plugin_dir = pathlib.Path(__file__).parent.absolute() + +with open(f"{plugin_dir}/README.md", "r") as fh: + long_description = fh.read() setup( - name="threatbus-misp", + author="Tenzir", + author_email="engineering@tenzir.com", + classifiers=[ + # https://pypi.org/classifiers/ + "Development Status :: 3 - Alpha", + "Environment :: Plugins", + "License :: OSI Approved :: BSD License", + "Operating System :: MacOS :: MacOS X", + "Operating System :: POSIX :: Linux", + "Programming Language :: Python :: 3", + "Topic :: Scientific/Engineering :: Information Analysis", + "Topic :: Scientific/Engineering :: Interface Engine/Protocol Translator", + "Topic :: Security", + "Topic :: Software Development :: Object Brokering", + "Topic :: System :: Distributed Computing", + ], + description="A plugin to enable threatbus communication with MISP.", + entry_points={"threatbus.app": ["misp = threatbus_misp"]}, install_requires=[ - "threatbus>=0.2.0", + "threatbus>=0.3.1", "pymisp>=2.4.120", "pyzmq>=18.1.1", "confluent-kafka>=1.3.0", ], - entry_points={"threatbus.app": ["misp = threatbus_misp"]}, + keywords=[ + "threatbus", + "MISP", + "threat intelligence", + "IDS", + "zeromq", + "zmq", + "kafka", + ], + license="BSD 3-clause", + long_description=long_description, + long_description_content_type="text/markdown", + name="threatbus-misp", py_modules=["threatbus_misp", "misp_message_mapping"], - version="0.3.0", + python_requires=">=3.7", + url="https://github.com/tenzir/threatbus", + version="0.3.1", ) diff --git a/plugins/apps/threatbus-zeek/README.md b/plugins/apps/threatbus-zeek/README.md new file mode 100644 index 00000000..9bcd5138 --- /dev/null +++ b/plugins/apps/threatbus-zeek/README.md @@ -0,0 +1,66 @@ +Threat Bus Zeek Plugin +====================== + +

+ +[![PyPI Status][pypi-badge]][pypi-url] +[![Build Status][ci-badge]][ci-url] +[![License][license-badge]][license-url] + +

+ +A Threat Bus plugin that enables communication to [Zeek](https://zeek.org/). + +## Installation + +```sh +pip install threatbus-zeek +``` + +#### Prerequisites + +*Install [Broker](https://github.com/zeek/broker) on the Threat Bus host* + +The plugin uses the [Broker python bindings](https://docs.zeek.org/projects/broker/en/stable/python.html) +to enable communication with Zeek. You have to install Broker and bindings to +use this plugin. + +## Configuration + +The plugin starts a listening Broker endpoint. The endpoint characteristics for +listening can be cofigure as follows: + +```yaml +... +plugins: + apps: + zeek: + host: "127.0.0.1" + port: 47761 + module_namespace: Tenzir +... +``` + +## Threat Bus Zeek Script + +Threat Bus is a pub/sub broker for threat intelligence data. Applications, like +Zeek, have to register themselves at the bus. Hence, load this [Zeek script](https://github.com/tenzir/threatbus/blob/master/apps/zeek/threatbus.zeek) +into your Zeek installation to make it aware of Threat Bus. + +The script can be configured via certain `option`s for setting topic names or +requesting an intel snapshot: + +```sh +zeek -i -C ./apps/zeek/threatbus.zeek -- "Tenzir::snapshot_intel=-30 days" +``` + +## License + +Threat Bus comes with a [3-clause BSD license][license-url]. + +[pypi-badge]: https://img.shields.io/pypi/v/threatbus-zeek.svg +[pypi-url]: https://pypi.org/project/threatbus-zeek +[ci-url]: https://github.com/tenzir/threatbus/actions?query=branch%3Amaster +[ci-badge]: https://github.com/tenzir/threatbus/workflows/Python%20Egg/badge.svg?branch=master +[license-badge]: https://img.shields.io/badge/license-BSD-blue.svg +[license-url]: https://github.com/tenzir/threatbus/blob/master/COPYING \ No newline at end of file diff --git a/plugins/apps/threatbus-zeek/setup.py b/plugins/apps/threatbus-zeek/setup.py index c2fd1ef3..6c43c815 100644 --- a/plugins/apps/threatbus-zeek/setup.py +++ b/plugins/apps/threatbus-zeek/setup.py @@ -1,9 +1,38 @@ from setuptools import setup +import pathlib + +plugin_dir = pathlib.Path(__file__).parent.absolute() + +with open(f"{plugin_dir}/README.md", "r") as fh: + long_description = fh.read() setup( - name="threatbus-zeek", - install_requires="threatbus", + author="Tenzir", + author_email="engineering@tenzir.com", + classifiers=[ + # https://pypi.org/classifiers/ + "Development Status :: 3 - Alpha", + "Environment :: Plugins", + "License :: OSI Approved :: BSD License", + "Operating System :: MacOS :: MacOS X", + "Operating System :: POSIX :: Linux", + "Programming Language :: Python :: 3", + "Topic :: Scientific/Engineering :: Information Analysis", + "Topic :: Scientific/Engineering :: Interface Engine/Protocol Translator", + "Topic :: Security", + "Topic :: Software Development :: Object Brokering", + "Topic :: System :: Distributed Computing", + ], + description="A plugin to enable threatbus communication with Zeek network monitor.", entry_points={"threatbus.app": ["zeek = threatbus_zeek"]}, + install_requires=["threatbus>=0.3.1",], + keywords=["threatbus", "Zeek", "intrusion detection", "IDS", "broker", "plugin"], + license="BSD 3-clause", + long_description=long_description, + long_description_content_type="text/markdown", + name="threatbus-zeek", py_modules=["threatbus_zeek", "zeek_message_mapping"], - version="0.3.0", + python_requires=">=3.7", + url="https://github.com/tenzir/threatbus", + version="0.3.1", ) diff --git a/plugins/backbones/threatbus-inmem/README.md b/plugins/backbones/threatbus-inmem/README.md new file mode 100644 index 00000000..c41b648a --- /dev/null +++ b/plugins/backbones/threatbus-inmem/README.md @@ -0,0 +1,45 @@ +Threat Bus In-Memory Backbone Plugin +==================================== + +

+ +[![PyPI Status][pypi-badge]][pypi-url] +[![Build Status][ci-badge]][ci-url] +[![License][license-badge]][license-url] + +

+ +A very simplistic Threat Bus plugin that provides an in-memory backbone for data +provisioning. + +## Installation + +```sh +pip install threatbus-inmem +``` + +## Configuration + +Add a placeholder for this plugin's config: + +```yaml +... +plugins: + backbones: + inmem: +... +``` + +There is no further configuration needed. + + +## License + +Threat Bus comes with a [3-clause BSD license][license-url]. + +[pypi-badge]: https://img.shields.io/pypi/v/threatbus-inmem.svg +[pypi-url]: https://pypi.org/project/threatbus-inmem +[ci-url]: https://github.com/tenzir/threatbus/actions?query=branch%3Amaster +[ci-badge]: https://github.com/tenzir/threatbus/workflows/Python%20Egg/badge.svg?branch=master +[license-badge]: https://img.shields.io/badge/license-BSD-blue.svg +[license-url]: https://github.com/tenzir/threatbus/blob/master/COPYING \ No newline at end of file diff --git a/plugins/backbones/threatbus-inmem/setup.py b/plugins/backbones/threatbus-inmem/setup.py index e750d2b0..656f3baa 100644 --- a/plugins/backbones/threatbus-inmem/setup.py +++ b/plugins/backbones/threatbus-inmem/setup.py @@ -1,9 +1,35 @@ from setuptools import setup +import pathlib + +plugin_dir = pathlib.Path(__file__).parent.absolute() + +with open(f"{plugin_dir}/README.md", "r") as fh: + long_description = fh.read() setup( - name="threatbus-inmem", - install_requires="threatbus", + author="Tenzir", + author_email="engineering@tenzir.com", + classifiers=[ + # https://pypi.org/classifiers/ + "Development Status :: 3 - Alpha", + "Environment :: Plugins", + "License :: OSI Approved :: BSD License", + "Operating System :: MacOS :: MacOS X", + "Operating System :: POSIX :: Linux", + "Programming Language :: Python :: 3", + "Topic :: Security", + "Topic :: Software Development :: Object Brokering", + ], + description="A simplistic in-memory backbone for threatbus.", entry_points={"threatbus.backbone": ["inmem = threatbus_inmem"]}, + install_requires=["threatbus>=0.3.1",], + keywords=["threatbus", "plugin"], + license="BSD 3-clause", + long_description=long_description, + long_description_content_type="text/markdown", + name="threatbus-inmem", py_modules=["threatbus_inmem"], - version="0.3.0", + python_requires=">=3.7", + url="https://github.com/tenzir/threatbus", + version="0.3.1", ) diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 00000000..224a7795 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,2 @@ +[metadata] +description-file = README.md \ No newline at end of file diff --git a/setup.py b/setup.py index 353e6074..e1a0ae11 100644 --- a/setup.py +++ b/setup.py @@ -3,25 +3,47 @@ from setuptools import setup, find_packages +with open("README.md", "r") as fh: + long_description = fh.read() + setup( author="Tenzir", - name="threatbus", author_email="engineering@tenzir.com", classifiers=[ - "License :: OSI Approved :: BSD 3-Clause", - "Programming Language :: Python :: 3.8", + # https://pypi.org/classifiers/ + "Development Status :: 3 - Alpha", + "Environment :: Plugins", + "License :: OSI Approved :: BSD License", + "Operating System :: MacOS :: MacOS X", + "Operating System :: POSIX :: Linux", + "Topic :: Scientific/Engineering :: Information Analysis", + "Topic :: Scientific/Engineering :: Interface Engine/Protocol Translator", + "Topic :: Security", + "Topic :: Software Development :: Object Brokering", + "Topic :: System :: Distributed Computing", ], + description="The missing link to connect open-source threat intelligence tools.", + entry_points={"console_scripts": ["threatbus=threatbus.threatbus:main"]}, + include_package_data=True, install_requires=[ "coloredlogs>=10.0", "confuse>=1.0", "pluggy>=0.13", "black>=19.10b", ], - description="Connect open source threat intelligence tools", - license="BSD 3-Clause license", - include_package_data=True, + keywords=[ + "threatbus", + "threat intelligence", + "intel", + "sightings", + "open source threat intelligence", + ], + license="BSD 3-clause", + long_description=long_description, + long_description_content_type="text/markdown", + name="threatbus", packages=find_packages(), - version="0.3.0", - zip_safe=False, - entry_points={"console_scripts": ["threatbus=threatbus.threatbus:main"]}, + python_requires=">=3.7", + url="https://github.com/tenzir/threatbus", + version="0.3.1", )