Merge pull request #11

Automate PyPI distribution
tenzir · Jan 27, 2020 · 8037077 · 8037077
2 parents 09968d0 + 3225d2d
commit 8037077
Show file tree

Hide file tree

Showing 11 changed files with 383 additions and 37 deletions.
diff --git a/.github/workflows/python-egg.yml b/.github/workflows/python-egg.yml
@@ -5,18 +5,74 @@ env:
 jobs:
   egg-install:
     name: Egg Installation
-    runs-on: ubuntu-latest
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: true
+      matrix:
+        python-version: [3.7, 3.8]
+        os: [ubuntu-latest, macOS-latest]
     steps:
-    - uses: actions/checkout@v1
+    - uses: actions/checkout@v2
       with:
         ref: ${{ github.ref }}
-    - uses: actions/setup-python@v1
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v1
       with:
-        python-version: '3.8'
+        python-version: ${{ matrix.python-version }}
+    - name: Set up Dist Environment
+      run: |
+        python -m pip install --upgrade pip
+        pip install wheel setuptools
     - name: Install Application and Plugins
       run: |
-        pip install --upgrade pip
+        python setup.py build
         pip install .
         pip install plugins/apps/threatbus-zeek
         pip install plugins/apps/threatbus-misp
-        pip install plugins/backbones/threatbus-inmem
+        pip install plugins/backbones/threatbus-inmem
+    - name: Package Application and Plugins
+      run: |
+        python setup.py sdist bdist_wheel
+        python plugins/apps/threatbus-zeek/setup.py sdist bdist_wheel
+        python plugins/apps/threatbus-misp/setup.py sdist bdist_wheel
+        python plugins/backbones/threatbus-inmem/setup.py sdist bdist_wheel
+
+  egg-release:
+    name: Egg Release
+    needs: egg-install
+    runs-on: ubuntu-latest
+    env:
+      TWINE_USERNAME: __token__
+      TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }}
+    if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags')
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        ref: ${{ github.ref }}
+    - name: Set up Python 3.8
+      uses: actions/setup-python@v1
+      with:
+        python-version: 3.8
+    - name: Set up Dist Environment
+      run: |
+        python -m pip install --upgrade pip
+        pip install wheel setuptools twine
+    - name: Package Application and Plugins
+      run: |
+        python setup.py sdist bdist_wheel
+        python plugins/apps/threatbus-zeek/setup.py sdist bdist_wheel
+        python plugins/apps/threatbus-misp/setup.py sdist bdist_wheel
+        python plugins/backbones/threatbus-inmem/setup.py sdist bdist_wheel
+    - name: GitHub Release
+      id: create_release
+      uses: actions/create-release@v1
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      with:
+        tag_name: ${{ github.ref }}
+        release_name: Threat Bus ${{ github.ref }}
+        draft: false
+        prerelease: false
+    - name: Publish to PyPI
+      run: |
+        python -m twine upload --verbose --non-interactive dist dist/*
diff --git a/.gitignore b/.gitignore
@@ -1,6 +1,8 @@
 *.sw?
-env
-venv
-vast
-**/__pycache__
-**/*.egg-info
+env/
+build/
+dist/
+venv/
+**/__pycache__/
+**/*.egg-info/
+*tar.gz
diff --git a/README.md b/README.md
@@ -6,6 +6,7 @@
 
 The missing tool to interconnect open-source security applications.
 
+[![PyPI Status][pypi-badge]][pypi-url]
 [![Build Status][ci-badge]][ci-url]
 [![Total alerts][lgtm-alerts-badge]][lgtm-alerts-url]
 [![Language grade: Python][lgtm-quality-badge]][lgtm-quality-url]
@@ -63,9 +64,11 @@ environment.
 ```
 virtualenv venv           # optional
 source venv/bin/activate  # optional
-pip install .
-pip install plugins/backbones/threatbus-inmem
-pip install plugins/<desired-plugins>
+pip install threatbus
+pip install threatbus-inmem
+pip install threatbus-misp
+pip install threatbus-zeek
+pip install threatbus-<plugin_name>
 ```
 
 ### Testing
@@ -114,9 +117,9 @@ Example:
   ```sh
   plugins
   ├── apps
-  │   └── threatbus-zeek
-  │       ├── setup.py
-  │       └── threatbus_zeek.py
+  |   └── threatbus-zeek
+  │       ├── setup.py
+  |       └── threatbus_zeek.py
   └── backbones
       └── threatbus-inmem
           ├── setup.py
@@ -167,6 +170,8 @@ Threat Bus comes with a [3-clause BSD license][license-url].
 [zeek]: https://www.zeek.org
 [misp-zmq-config]: https://github.com/MISP/misp-book/tree/master/misp-zmq#misp-zeromq-configuration
 
+[pypi-badge]: https://img.shields.io/pypi/v/threatbus.svg
+[pypi-url]: https://pypi.org/project/threatbus
 [contributing-url]: https://github.com/tenzir/.github/blob/master/contributing.md
 [latest-release-badge]: https://img.shields.io/github/commits-since/tenzir/threatbus/latest.svg?color=green
 [latest-release-url]: https://github.com/tenzir/threatbus/releases

diff --git a/plugins/apps/threatbus-misp/README.md b/plugins/apps/threatbus-misp/README.md
@@ -1,10 +1,22 @@
 Threat Bus MISP Plugin
 ======================
 
+<h4 align="center">
+
+[![PyPI Status][pypi-badge]][pypi-url]
+[![Build Status][ci-badge]][ci-url]
+[![License][license-badge]][license-url]
+
+</h4>
+
 A Threat Bus plugin that enables communication to [MISP](https://www.misp-project.org/).
 
-## Installation
+The plugin goes against the pub/sub architecture of Threat Bus (for now),
+because the plugin subscribes a listener to ZeroMQ / Kafka, rather than having
+MISP subscribe itself to Threat Bus. That will be addressed with a MISP module
+in the near future.
 
+## Installation
 
 ```sh
 pip install threatbus-misp
@@ -19,6 +31,38 @@ you have to install `librdkafka` for the host system that is running
 `threatbus`. See also the [prerequisites](https://github.com/confluentinc/confluent-kafka-python#prerequisites)
 section of the `confluent-kafka` python client.
 
+## Configuration
+
+The plugin can either use ZeroMQ or Kafka to retrieve intelligence items from
+MISP. It uses the MISP REST api to report back sightings of indicators.
+
+ZeroMQ and Kafka are mutually exclusive, such that Threat Bus does not receive
+all attribute updates twice. See below for an example configuration.
+
+
+```yaml
+...
+plugins:
+  misp:
+    api:
+      host: https://localhost
+      ssl: false
+      key: MISP_API_KEY
+    zmq:
+      host: localhost
+      port: 50000
+    #kafka:
+    #  topics:
+    #  - misp_attribute
+    #  poll_interval: 1.0
+    #  # All config entries are passed as-is to librdkafka
+    #  # https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md
+    #  config:
+    #    bootstrap.servers: "localhost:9092"
+    #    group.id: "threatbus"
+    #    auto.offset.reset: "earliest"
+...
+```
 
 ## Development Setup
 
@@ -135,3 +179,15 @@ exit # leave the Docker container shell
 ```sh
 make restart-all
 ```
+
+
+## License
+
+Threat Bus comes with a [3-clause BSD license][license-url].
+
+[pypi-badge]: https://img.shields.io/pypi/v/threatbus-misp.svg
+[pypi-url]: https://pypi.org/project/threatbus-misp
+[ci-url]: https://github.com/tenzir/threatbus/actions?query=branch%3Amaster
+[ci-badge]: https://github.com/tenzir/threatbus/workflows/Python%20Egg/badge.svg?branch=master
+[license-badge]: https://img.shields.io/badge/license-BSD-blue.svg
+[license-url]: https://github.com/tenzir/threatbus/blob/master/COPYING
diff --git a/plugins/apps/threatbus-misp/setup.py b/plugins/apps/threatbus-misp/setup.py
@@ -1,14 +1,51 @@
 from setuptools import setup
+import pathlib
+
+plugin_dir = pathlib.Path(__file__).parent.absolute()
+
+with open(f"{plugin_dir}/README.md", "r") as fh:
+    long_description = fh.read()
 
 setup(
-    name="threatbus-misp",
+    author="Tenzir",
+    author_email="[email protected]",
+    classifiers=[
+        # https://pypi.org/classifiers/
+        "Development Status :: 3 - Alpha",
+        "Environment :: Plugins",
+        "License :: OSI Approved :: BSD License",
+        "Operating System :: MacOS :: MacOS X",
+        "Operating System :: POSIX :: Linux",
+        "Programming Language :: Python :: 3",
+        "Topic :: Scientific/Engineering :: Information Analysis",
+        "Topic :: Scientific/Engineering :: Interface Engine/Protocol Translator",
+        "Topic :: Security",
+        "Topic :: Software Development :: Object Brokering",
+        "Topic :: System :: Distributed Computing",
+    ],
+    description="A plugin to enable threatbus communication with MISP.",
+    entry_points={"threatbus.app": ["misp = threatbus_misp"]},
     install_requires=[
-        "threatbus>=0.2.0",
+        "threatbus>=0.3.1",
         "pymisp>=2.4.120",
         "pyzmq>=18.1.1",
         "confluent-kafka>=1.3.0",
     ],
-    entry_points={"threatbus.app": ["misp = threatbus_misp"]},
+    keywords=[
+        "threatbus",
+        "MISP",
+        "threat intelligence",
+        "IDS",
+        "zeromq",
+        "zmq",
+        "kafka",
+    ],
+    license="BSD 3-clause",
+    long_description=long_description,
+    long_description_content_type="text/markdown",
+    name="threatbus-misp",
     py_modules=["threatbus_misp", "misp_message_mapping"],
-    version="0.3.0",
+    python_requires=">=3.7",
+    url="https://github.com/tenzir/threatbus",
+    version="0.3.1",
 )
diff --git a/plugins/apps/threatbus-zeek/README.md b/plugins/apps/threatbus-zeek/README.md
@@ -0,0 +1,66 @@
+Threat Bus Zeek Plugin
+======================
+
+<h4 align="center">
+
+[![PyPI Status][pypi-badge]][pypi-url]
+[![Build Status][ci-badge]][ci-url]
+[![License][license-badge]][license-url]
+
+</h4>
+
+A Threat Bus plugin that enables communication to [Zeek](https://zeek.org/).
+
+## Installation
+
+```sh
+pip install threatbus-zeek
+```
+
+#### Prerequisites
+
+*Install [Broker](https://github.com/zeek/broker) on the Threat Bus host*
+
+The plugin uses the [Broker python bindings](https://docs.zeek.org/projects/broker/en/stable/python.html)
+to enable communication with Zeek. You have to install Broker and bindings to
+use this plugin.
+
+## Configuration
+
+The plugin starts a listening Broker endpoint. The endpoint characteristics for
+listening can be cofigure as follows:
+
+```yaml
+...
+plugins:
+  apps:
+    zeek:
+      host: "127.0.0.1"
+      port: 47761
+      module_namespace: Tenzir
+...
+```
+
+## Threat Bus Zeek Script
+
+Threat Bus is a pub/sub broker for threat intelligence data. Applications, like
+Zeek, have to register themselves at the bus. Hence, load this [Zeek script](https://github.com/tenzir/threatbus/blob/master/apps/zeek/threatbus.zeek)
+into your Zeek installation to make it aware of Threat Bus.
+
+The script can be configured via certain `option`s for setting topic names or
+requesting an intel snapshot:
+
+```sh
+zeek -i <INTERFACE> -C ./apps/zeek/threatbus.zeek -- "Tenzir::snapshot_intel=-30 days"
+```
+
+## License
+
+Threat Bus comes with a [3-clause BSD license][license-url].
+
+[pypi-badge]: https://img.shields.io/pypi/v/threatbus-zeek.svg
+[pypi-url]: https://pypi.org/project/threatbus-zeek
+[ci-url]: https://github.com/tenzir/threatbus/actions?query=branch%3Amaster
+[ci-badge]: https://github.com/tenzir/threatbus/workflows/Python%20Egg/badge.svg?branch=master
+[license-badge]: https://img.shields.io/badge/license-BSD-blue.svg
+[license-url]: https://github.com/tenzir/threatbus/blob/master/COPYING
diff --git a/plugins/apps/threatbus-zeek/setup.py b/plugins/apps/threatbus-zeek/setup.py
@@ -1,9 +1,38 @@
 from setuptools import setup
+import pathlib
+
+plugin_dir = pathlib.Path(__file__).parent.absolute()
+
+with open(f"{plugin_dir}/README.md", "r") as fh:
+    long_description = fh.read()
 
 setup(
-    name="threatbus-zeek",
-    install_requires="threatbus",
+    author="Tenzir",
+    author_email="[email protected]",
+    classifiers=[
+        # https://pypi.org/classifiers/
+        "Development Status :: 3 - Alpha",
+        "Environment :: Plugins",
+        "License :: OSI Approved :: BSD License",
+        "Operating System :: MacOS :: MacOS X",
+        "Operating System :: POSIX :: Linux",
+        "Programming Language :: Python :: 3",
+        "Topic :: Scientific/Engineering :: Information Analysis",
+        "Topic :: Scientific/Engineering :: Interface Engine/Protocol Translator",
+        "Topic :: Security",
+        "Topic :: Software Development :: Object Brokering",
+        "Topic :: System :: Distributed Computing",
+    ],
+    description="A plugin to enable threatbus communication with Zeek network monitor.",
     entry_points={"threatbus.app": ["zeek = threatbus_zeek"]},
+    install_requires=["threatbus>=0.3.1",],
+    keywords=["threatbus", "Zeek", "intrusion detection", "IDS", "broker", "plugin"],
+    license="BSD 3-clause",
+    long_description=long_description,
+    long_description_content_type="text/markdown",
+    name="threatbus-zeek",
     py_modules=["threatbus_zeek", "zeek_message_mapping"],
-    version="0.3.0",
+    python_requires=">=3.7",
+    url="https://github.com/tenzir/threatbus",
+    version="0.3.1",
 )