diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 0000000..bc8ea4a --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1,24 @@ +# .readthedocs.yaml +# Read the Docs configuration file +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details + +# Required +version: 2 + +# Set the OS, Python version and other tools you might need +build: + os: ubuntu-22.04 + tools: + python: "3.10" + +sphinx: + configuration: docs/source/conf.py + + +python: + install: + # install itself with pip install . + - method: pip + path: . + extra_requirements: + - docs diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 3fe03a6..73bc6b2 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -17,6 +17,8 @@ Pre-commit will automatically format the code before each commit, It can also be pre-commit run --all-files ``` +Comment style is [Google Python Style Guide](https://google.github.io/styleguide/pyguide.html#38-comments-and-docstrings). + ## Install Locally with Test Dependencies ```bash @@ -34,6 +36,35 @@ pytest --cov=duetector # Generate coverage reports Run unit-test before PR, **ensure that new features are covered by unit tests** +## Generating config + +Use script to generate config after add tracer/filter... + +```bash +python duetector/tools/config_generator.py +``` + +## Build Docs + +Install docs dependencies + +```bash +pip install -e .[docs] +``` + +Build docs + +```bash +make clean && make html +``` + +Use [start-docs-host.sh](dev-tools/start-docs-host.sh) to deploy a local http server to view the docs + +```bash +cd ./dev-tools && ./start-docs-host.sh +``` +Access `http://localhost:8080` for docs. + ## Typing (Optional, python<=3.10) Use [pytype](https://github.com/google/pytype) to check typed diff --git a/README.md b/README.md index 216a910..d97d75c 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,7 @@

duetector🔍: 支持eBPF的可扩展数据使用探测器

Actions Status +Documentation Status pre-commit.ci status LICENSE Releases @@ -157,7 +158,7 @@ Commands: ## API文档与配置文档 -WIP 这一部分内容是PIP相关的，目前还没有完成，完成后将包括可配置的类的内容，以及如何使用duetector作为PIP的内容。 +我们在readthedocs上为开发者和用户提供了API与配置文档，你可以在[这里](https://duetector.readthedocs.io/)查看 ## 维护者 diff --git a/README_en.md b/README_en.md index c63b510..bb598c1 100644 --- a/README_en.md +++ b/README_en.md @@ -1,8 +1,11 @@

duetector🔍: Data Usage Extensible detector

Actions Status +Documentation Status +pre-commit.ci status LICENSE Releases +Pre Releases Last Commit Python version

@@ -152,7 +155,7 @@ More documentation and examples can be found [here](. /docs/). ## API documentation -WIP +See [docs of duetector](https://duetector.readthedocs.io/) ## Maintainers diff --git a/dev-tools/start-docs-host.sh b/dev-tools/start-docs-host.sh new file mode 100755 index 0000000..6e1a451 --- /dev/null +++ b/dev-tools/start-docs-host.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +# Start a nginx server for host ../docs/build/html +# This is useful for testing the docs locally + +set -e +docker run --rm \ +-it \ +-p 8080:80 \ +-v $(pwd)/../docs/build/:/usr/share/nginx/:ro \ +nginx diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 0000000..d0c3cbf --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line, and also +# from the environment for the first two. +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build +SOURCEDIR = source +BUILDDIR = build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 0000000..e69de29 diff --git a/docs/source/cli/daemon.rst b/docs/source/cli/daemon.rst new file mode 100644 index 0000000..c3ed1a0 --- /dev/null +++ b/docs/source/cli/daemon.rst @@ -0,0 +1,3 @@ +.. click:: duetector.cli.daemon:cli + :prog: duectl-daemon + :nested: full diff --git a/docs/source/cli/index.rst b/docs/source/cli/index.rst new file mode 100644 index 0000000..3ab58a3 --- /dev/null +++ b/docs/source/cli/index.rst @@ -0,0 +1,14 @@ +duetector.cli +========================================= + +``duectl``: CLI for Start Monitor, generate config. + +``duectl-daemon``: Allow to run as daemon, and run as a service. + + +.. toctree:: + :maxdepth: 2 + :caption: Reference API docs: + + duectl
+ duectl-daemon diff --git a/docs/source/cli/main.rst b/docs/source/cli/main.rst new file mode 100644 index 0000000..030e941 --- /dev/null +++ b/docs/source/cli/main.rst @@ -0,0 +1,3 @@ +.. click:: duetector.cli.main:cli + :prog: duectl + :nested: full diff --git a/docs/source/collectors/base.rst b/docs/source/collectors/base.rst new file mode 100644 index 0000000..35b9e5e --- /dev/null +++ b/docs/source/collectors/base.rst @@ -0,0 +1,16 @@ +Collector +================== + +Base class for all collectors. + +.. autoclass:: duetector.collectors.base.Collector + :members: + :undoc-members: + :private-members: + + +.. autoclass:: duetector.collectors.base.DequeCollector + :members: + :undoc-members: + :private-members: + :show-inheritance: diff --git a/docs/source/collectors/db.rst b/docs/source/collectors/db.rst new file mode 100644 index 0000000..5bc5df6 --- /dev/null +++ b/docs/source/collectors/db.rst @@ -0,0 +1,10 @@ +DBCollector +================== + +DBCollector use :doc:`SessionManager ` for config the db engine and create sessions. + +.. autoclass:: duetector.collectors.db.DBCollector + :members: + :undoc-members: + :private-members: + :show-inheritance: diff --git a/docs/source/collectors/index.rst b/docs/source/collectors/index.rst new file mode 100644 index 0000000..cca04fd --- /dev/null +++ b/docs/source/collectors/index.rst @@ -0,0 +1,13 @@ +Collector +===================================== + +Collector will convert ``data_t`` to :doc:`Tracking ` +and store them in a somewhere. + +.. toctree:: + :maxdepth: 2 + :caption: Avaliable Collector + + Base Collectors + DB Collectors + Data Models diff --git a/docs/source/collectors/models.rst b/docs/source/collectors/models.rst new file mode 100644 index 0000000..251938e --- /dev/null +++ b/docs/source/collectors/models.rst @@ -0,0 +1,7 @@ +Models for collectors +==================================== + +.. autoclass:: duetector.collectors.models.Tracking + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/conf.py b/docs/source/conf.py new file mode 100644 index 0000000..587145c --- /dev/null +++ b/docs/source/conf.py @@ -0,0 +1,67 @@ +# Configuration file for the Sphinx documentation builder. +# +# This file only contains a selection of the most common options. For a full +# list see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +# -- Path setup -------------------------------------------------------------- + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +import os +import sys + +sys.path.insert(0, os.path.abspath("..")) + +# -- Project information ----------------------------------------------------- + +project = "duetector" +copyright = "2023, hitsz-ids" +author = "hitsz-ids" + +# The full version, including alpha/beta/rc tags +from duetector import __version__ + +release = __version__ + +# -- General configuration --------------------------------------------------- + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + "sphinx.ext.autodoc", + "sphinx.ext.napoleon", + "sphinx.ext.viewcode", + "sphinx.ext.autosectionlabel", + "sphinx_click", +] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ["_templates"] + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = "en" + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This pattern also affects html_static_path and html_extra_path. +exclude_patterns = [] + +# -- Options for HTML output ------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +html_theme = "sphinx_rtd_theme" + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ["_static"] diff --git a/docs/source/config.rst b/docs/source/config.rst new file mode 100644 index 0000000..720be47 --- /dev/null +++ b/docs/source/config.rst @@ -0,0 +1,6 @@ +Config utilities +========================================= +.. automodule:: duetector.config + :members: + :undoc-members: + :inherited-members: diff --git a/docs/source/db.rst b/docs/source/db.rst new file mode 100644 index 0000000..453e654 --- /dev/null +++ b/docs/source/db.rst @@ -0,0 +1,6 @@ +Database utilities +========================================= +.. automodule:: duetector.db + :members: + :undoc-members: + :inherited-members: diff --git a/docs/source/exceptions.rst b/docs/source/exceptions.rst new file mode 100644 index 0000000..7d8af16 --- /dev/null +++ b/docs/source/exceptions.rst @@ -0,0 +1,6 @@ +Exceptions +========================================= +.. automodule:: duetector.exceptions + :members: + :undoc-members: + :inherited-members: diff --git a/docs/source/filters/base.rst b/docs/source/filters/base.rst new file mode 100644 index 0000000..77a9c34 --- /dev/null +++ b/docs/source/filters/base.rst @@ -0,0 +1,10 @@ +BaseFilter +================== + +Base class for all filters. + + +.. autoclass:: duetector.filters.base.Filter + :members: + :undoc-members: + :private-members: diff --git a/docs/source/filters/index.rst b/docs/source/filters/index.rst new file mode 100644 index 0000000..5361bac --- /dev/null +++ b/docs/source/filters/index.rst @@ -0,0 +1,11 @@ +Filter +===================================== + +``Filter`` will filter the data based on the given criteria. + +.. toctree:: + :maxdepth: 2 + :caption: Avaliable Filter + + Base Filter + Pattern Filter diff --git a/docs/source/filters/pattern.rst b/docs/source/filters/pattern.rst new file mode 100644 index 0000000..0f74072 --- /dev/null +++ b/docs/source/filters/pattern.rst @@ -0,0 +1,10 @@ +PatternFilter +================== + +``PatternFilter`` support regex pattern to filter data. + +.. autoclass:: duetector.filters.pattern.PatternFilter + :members: + :undoc-members: + :private-members: + :show-inheritance: diff --git a/docs/source/index.rst b/docs/source/index.rst new file mode 100644 index 0000000..6b1106e --- /dev/null +++ b/docs/source/index.rst @@ -0,0 +1,36 @@ + +Welcome to duetector's documentation! +========================================= + +duetector🔍 is an extensible data usage control detector that provides support for data usage control by probing for data usage behavior in the Linux kernel(based on eBPF). + +For more information, please visit the `project homepage `_. + +Reference +======================================== + +.. toctree:: + :maxdepth: 2 + :caption: Reference Documentation: + + CLI + + Monitors + Managers + + Tracers + Filters + Collectors + + Exceptions + + Database utilities + Config utilities + Tools utilities + + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`search` diff --git a/docs/source/managers/collector.rst b/docs/source/managers/collector.rst new file mode 100644 index 0000000..8c5fc6b --- /dev/null +++ b/docs/source/managers/collector.rst @@ -0,0 +1,13 @@ +CollectorManager +=========================================== + + +.. autodata:: duetector.managers.collector.PROJECT_NAME +.. autofunction:: duetector.managers.collector.init_collector + + +.. autoclass:: duetector.managers.CollectorManager + :members: + :undoc-members: + :inherited-members: + :show-inheritance: diff --git a/docs/source/managers/filter.rst b/docs/source/managers/filter.rst new file mode 100644 index 0000000..548ebe4 --- /dev/null +++ b/docs/source/managers/filter.rst @@ -0,0 +1,13 @@ +FilterManager +=========================================== + + +.. autodata:: duetector.managers.filter.PROJECT_NAME +.. autofunction:: duetector.managers.filter.init_filter + + +.. autoclass:: duetector.managers.FilterManager + :members: + :undoc-members: + :inherited-members: + :show-inheritance: diff --git a/docs/source/managers/index.rst b/docs/source/managers/index.rst new file mode 100644 index 0000000..7ae567d --- /dev/null +++ b/docs/source/managers/index.rst @@ -0,0 +1,19 @@ +Managers +========================================= + +Managers provides a way to get instances of both built-in implementations and extensions. + + +.. autoclass:: duetector.managers.Manager + :members: + :undoc-members: + :inherited-members: + :show-inheritance: + +.. toctree:: + :maxdepth: 1 + :caption: Avaliable Manager + + Collector Manager + Filter Manager + Tracer Manager diff --git a/docs/source/managers/tracer.rst b/docs/source/managers/tracer.rst new file mode 100644 index 0000000..89e09fc --- /dev/null +++ b/docs/source/managers/tracer.rst @@ -0,0 +1,13 @@ +TracerManager +=========================================== + + +.. autodata:: duetector.managers.tracer.PROJECT_NAME +.. autofunction:: duetector.managers.tracer.init_tracer + + +.. autoclass:: duetector.managers.TracerManager + :members: + :undoc-members: + :inherited-members: + :show-inheritance: diff --git a/docs/source/monitors/bcc.rst b/docs/source/monitors/bcc.rst new file mode 100644 index 0000000..f2d6cbd --- /dev/null +++ b/docs/source/monitors/bcc.rst @@ -0,0 +1,11 @@ +BccMonitor +================== + +``BccMonitor`` use `bcc `_. + as backend to monitor the kernel tracepoints. + +.. autoclass:: duetector.monitors.bcc_monitor.BccMonitor + :members: + :undoc-members: + :private-members: + :show-inheritance: diff --git a/docs/source/monitors/index.rst b/docs/source/monitors/index.rst new file mode 100644 index 0000000..f9e51bb --- /dev/null +++ b/docs/source/monitors/index.rst @@ -0,0 +1,22 @@ +Monitor +========================================= + +Monitor combines :doc:`Tracer `, :doc:`Collector ` +and :doc:`Filter ` by :doc:`Manager ` to provide an +entry point for the polling, filtering and collecting of the data. + +:doc:`Poller ` is used to poll the data periodically. + + +.. autoclass:: duetector.monitors.base.Monitor + :members: + :undoc-members: + :show-inheritance: + + +.. toctree:: + :maxdepth: 1 + :caption: Avaliable Monitor + + Bcc Monitor + Shell Monitor diff --git a/docs/source/monitors/sh.rst b/docs/source/monitors/sh.rst new file mode 100644 index 0000000..dbd595b --- /dev/null +++ b/docs/source/monitors/sh.rst @@ -0,0 +1,16 @@ +ShMonitor +================== + +``ShMonitor`` use ``ShTracerHost`` as backend for polling output of a command. + +.. autoclass:: duetector.monitors.sh_monitor.ShTracerHost + :members: + :undoc-members: + :private-members: + :show-inheritance: + +.. autoclass:: duetector.monitors.sh_monitor.ShMonitor + :members: + :undoc-members: + :private-members: + :show-inheritance: diff --git a/docs/source/tools/config_generator.rst b/docs/source/tools/config_generator.rst new file mode 100644 index 0000000..d3af9f9 --- /dev/null +++ b/docs/source/tools/config_generator.rst @@ -0,0 +1,11 @@ +ConfigGenerator +======================================== + +.. autofunction:: duetector.tools.config_generator._recursive_load + + +.. autoclass:: duetector.tools.config_generator.ConfigGenerator + :members: + :undoc-members: + :private-members: + :show-inheritance: diff --git a/docs/source/tools/daemon.rst b/docs/source/tools/daemon.rst new file mode 100644 index 0000000..8bbf67b --- /dev/null +++ b/docs/source/tools/daemon.rst @@ -0,0 +1,10 @@ +Daemon +======================================== + +Allow run command as daemon. Used in :doc:`daemon cli `. + +.. autoclass:: duetector.tools.daemon.Daemon + :members: + :undoc-members: + :private-members: + :show-inheritance: diff --git a/docs/source/tools/index.rst b/docs/source/tools/index.rst new file mode 100644 index 0000000..7df6bac --- /dev/null +++ b/docs/source/tools/index.rst @@ -0,0 +1,13 @@ +Tools +========================================= + +A collection of tools for building and testing the project. + + +.. toctree:: + :maxdepth: 1 + :caption: Avaliable Tools + + poller + daemon + config_generator diff --git a/docs/source/tools/poller.rst b/docs/source/tools/poller.rst new file mode 100644 index 0000000..f4c2055 --- /dev/null +++ b/docs/source/tools/poller.rst @@ -0,0 +1,10 @@ +Poller +======================================== + +A wrapper of ``threading.Thread`` to poll a function periodically. + +.. autoclass:: duetector.tools.poller.Poller + :members: + :undoc-members: + :private-members: + :show-inheritance: diff --git a/docs/source/tracers/clone.rst b/docs/source/tracers/clone.rst new file mode 100644 index 0000000..317456d --- /dev/null +++ b/docs/source/tracers/clone.rst @@ -0,0 +1,8 @@ +CloneTracer +================================== + +.. autoclass:: duetector.tracers.clone.CloneTracer + :members: + :undoc-members: + :private-members: + :show-inheritance: diff --git a/docs/source/tracers/index.rst b/docs/source/tracers/index.rst new file mode 100644 index 0000000..36e9245 --- /dev/null +++ b/docs/source/tracers/index.rst @@ -0,0 +1,17 @@ +Tracer +========================================= + +.. automodule:: duetector.tracers + :members: + :undoc-members: + :inherited-members: + + +.. toctree:: + :maxdepth: 1 + :caption: Avaliable Tracer + + CloneTracer + OpenTracer + TcpconnectTracer + UnameTracer diff --git a/docs/source/tracers/openat2.rst b/docs/source/tracers/openat2.rst new file mode 100644 index 0000000..8e5cd7f --- /dev/null +++ b/docs/source/tracers/openat2.rst @@ -0,0 +1,8 @@ +OpenTracer +================================== + +.. autoclass:: duetector.tracers.openat2.OpenTracer + :members: + :undoc-members: + :private-members: + :show-inheritance: diff --git a/docs/source/tracers/tcpconnect.rst b/docs/source/tracers/tcpconnect.rst new file mode 100644 index 0000000..a922e47 --- /dev/null +++ b/docs/source/tracers/tcpconnect.rst @@ -0,0 +1,8 @@ +TcpconnectTracer +================================== + +.. autoclass:: duetector.tracers.tcpconnect.TcpconnectTracer + :members: + :undoc-members: + :private-members: + :show-inheritance: diff --git a/docs/source/tracers/uname.rst b/docs/source/tracers/uname.rst new file mode 100644 index 0000000..d3c2d9a --- /dev/null +++ b/docs/source/tracers/uname.rst @@ -0,0 +1,8 @@ +UnameTracer +================================== + +.. autoclass:: duetector.tracers.uname.UnameTracer + :members: + :undoc-members: + :private-members: + :show-inheritance: diff --git a/duetector/cli/daemon.py b/duetector/cli/daemon.py index db658bb..00ed638 100644 --- a/duetector/cli/daemon.py +++ b/duetector/cli/daemon.py @@ -29,10 +29,12 @@ @click.pass_context def start(ctx, workdir, loglevel, rotate_log): """ - Start a background process of command `duectl start`. + Start a background process of command ``duectl start``. - All arguments after `--` will be passed to `duectl start` - e.g. `duectl-daemon start -- --config /path/to/config` + All arguments after ``--`` will be passed to ``duectl start``. + + Example: + ``duectl-daemon start -- --config /path/to/config`` """ cmd = ["duectl", "start"] cmd_args = ctx.args @@ -62,7 +64,7 @@ def status(workdir): """ Show status of process. - Determined by the existence of pid file in `workdir`. + Determined by the existence of pid file in ``workdir``. """ if Daemon( workdir=workdir, @@ -82,7 +84,7 @@ def stop(workdir): """ Stop the process. - Determined by the existence of pid file in `workdir`. + Determined by the existence of pid file in ``workdir``. """ Daemon( workdir=workdir, diff --git a/duetector/cli/main.py b/duetector/cli/main.py index 7f68dd7..09892ba 100644 --- a/duetector/cli/main.py +++ b/duetector/cli/main.py @@ -23,25 +23,25 @@ def check_privileges(): @click.option( "--load_current_config", default=True, - help=f"Wheather load current config file, if True, will use --path as origin config file path, default True.", + help=f"Wheather load current config file, if ``True``, will use ``--path`` as origin config file path, default: ``True``.", ) @click.option( "--path", default=CONFIG_PATH, - help=f"Origin config file path, default: {CONFIG_PATH}", + help=f"Origin config file path, default: ``{CONFIG_PATH}``", ) @click.option( "--load_env", default=True, - help=f"Weather load env variables," - f"Prefix: {ConfigLoader.ENV_PREFIX}, Separator:{ConfigLoader.ENV_SEP}, " - f"e.g. {ConfigLoader.ENV_PREFIX}config{ConfigLoader.ENV_SEP}a means config.a, " - f"default: True", + help=f"Weather load env variables, " + f"Prefix: ``{ConfigLoader.ENV_PREFIX}``, Separator:``{ConfigLoader.ENV_SEP}``, " + f"e.g. ``{ConfigLoader.ENV_PREFIX}config{ConfigLoader.ENV_SEP}a`` means ``config.a``, " + f"default: ``True``.", ) @click.option( "--dump_path", default=CONFIG_PATH, - help=f"File path to dump, default: {CONFIG_PATH}", + help=f"File path to dump, default: ``{CONFIG_PATH}``.", ) def generate_dynamic_config(load_current_config, path, load_env, dump_path): """ @@ -62,12 +62,12 @@ def generate_dynamic_config(load_current_config, path, load_env, dump_path): @click.option( "--path", default=CONFIG_PATH, - help=f"Origin config file path, default: {CONFIG_PATH}", + help=f"Origin config file path, default: ``{CONFIG_PATH}``.", ) @click.option( "--dump_path", default=CONFIG_PATH, - help=f"File path to dump, default: {CONFIG_PATH}", + help=f"File path to dump, default: ``{CONFIG_PATH}``.", ) def make_config(path, dump_path): """ @@ -83,7 +83,7 @@ def make_config(path, dump_path): @click.option( "--path", default=CONFIG_PATH, - help=f"Generated config file path, default: {CONFIG_PATH}", + help=f"Generated config file path, default: ``{CONFIG_PATH}``.", ) def generate_config(path): """ @@ -93,19 +93,21 @@ def generate_config(path): @click.command() -@click.option("--config", default=CONFIG_PATH, help=f"Config file path, default: {CONFIG_PATH}") +@click.option( + "--config", default=CONFIG_PATH, help=f"Config file path, default: ``{CONFIG_PATH}``." +) @click.option( "--load_env", default=True, - help=f"Weather load env variables," - f"Prefix: {ConfigLoader.ENV_PREFIX}, Separator:{ConfigLoader.ENV_SEP}, " - f"e.g. {ConfigLoader.ENV_PREFIX}config{ConfigLoader.ENV_SEP}a means config.a, " + help=f"Weather load env variables, " + f"Prefix: ``{ConfigLoader.ENV_PREFIX}``, Separator:``{ConfigLoader.ENV_SEP}``, " + f"e.g. ``{ConfigLoader.ENV_PREFIX}config{ConfigLoader.ENV_SEP}a`` means ``config.a``, " f"default: True", ) @click.option( "--dump_when_load", default=True, - help=f"Weather dump config when load, default: True", + help=f"Weather dump config when load, default: ``True``.", ) @click.option( "--config_dump_dir", @@ -115,12 +117,12 @@ def generate_config(path): @click.option( "--enable_bcc_monitor", default=True, - help=f"Set false or False to disable bcc monitor, default: true", + help=f"Set false or False to disable bcc monitor, default: ``True``.", ) @click.option( "--enable_sh_monitor", default=True, - help=f"Set false or False to disable shell monitor, default: true", + help=f"Set false or False to disable shell monitor, default: ``True``.", ) def start( config, diff --git a/duetector/collectors/base.py b/duetector/collectors/base.py index de18b0f..3242179 100644 --- a/duetector/collectors/base.py +++ b/duetector/collectors/base.py @@ -11,7 +11,11 @@ class Collector(Configuable): """ - Base class for all collectors + Base class for all collectors, provide a ThreadPoolExecutor each instance for async emit. + + By default, the config scope of ``Collector`` is ``collector.{class_name}``. + + Implementations should override ``_emit`` and ``summary`` method, see ``DequeCollector`` as an example. """ default_config = { @@ -21,7 +25,13 @@ class Collector(Configuable): "max_workers": 10, }, } + """ + Default config for ``Collector`` + """ _backend_imp = ThreadPoolExecutor + """ + Default backend implementation for ``Collector`` + """ def __init__(self, config: Optional[Dict[str, Any]] = None, *args, **kwargs): super().__init__(config, *args, **kwargs) @@ -29,34 +39,60 @@ def __init__(self, config: Optional[Dict[str, Any]] = None, *args, **kwargs): @property def config_scope(self): + """ + Config scope for current collector + """ return self.__class__.__name__ @property def disabled(self): + """ + If current collector is disabled + """ return self.config.disabled @property def id(self) -> str: - # ID for current collector - # If not set, use hostname + """ + ID for current collector, used to identify current collector in database + + If not set, use hostname + """ return self.config.statis_id or platform.node() @property def backend_args(self): + """ + Arguments for backend ``self._backend_imp`` + """ + return self.config.backend_args def emit(self, tracer, data: NamedTuple): + """ + Wrapper for ``self._emit``, submit to backend executor + """ + if self.disabled: return self._backend.submit(self._emit, Tracking.from_namedtuple(tracer, data)) def _emit(self, t: Tracking): + """ + Emit a tracking to collector, should be implemented by subclasses + """ raise NotImplementedError def summary(self) -> Dict: + """ + Get summary of current collector, should be implemented by subclasses + """ raise NotImplementedError def shutdown(self): + """ + Shutdown backend executor + """ self._backend.shutdown() @@ -65,7 +101,7 @@ class DequeCollector(Collector): A simple collector using deque, disabled by default Config: - - maxlen: Max length of deque + - ``maxlen``: Max length of deque """ default_config = { @@ -73,9 +109,15 @@ class DequeCollector(Collector): "disabled": True, "maxlen": 1024, } + """ + Default config for ``DequeCollector`` + """ @property def maxlen(self): + """ + Max length of deque + """ return self.config.maxlen def __init__(self, config: Optional[Dict[str, Any]] = None, *args, **kwargs): diff --git a/duetector/collectors/db.py b/duetector/collectors/db.py index b0c966d..846a403 100644 --- a/duetector/collectors/db.py +++ b/duetector/collectors/db.py @@ -13,10 +13,10 @@ class DBCollector(Collector): """ A collector using database, sqlite by default. - Every tracker will create a table in database, see SessionManager.get_tracking_model + Every tracker will create a table in database, see ``SessionManager``.get_tracking_model Config: - - db: A SessionManager config + - ``db``: A ``SessionManager`` config """ default_config = { diff --git a/duetector/collectors/models.py b/duetector/collectors/models.py index 3c20111..169edb9 100644 --- a/duetector/collectors/models.py +++ b/duetector/collectors/models.py @@ -9,24 +9,55 @@ class Tracking(pydantic.BaseModel): """ Tracking model for all tracers, bring tracer's data into a common model - Extended fields will be stored in `extended` field as a dict - Use Tracking.from_namedtuple to create a Tracking instance from tracer's data + Extended fields will be stored in ``_extended`` field as a dict + Use ``Tracking.from_namedtuple`` to create a Tracking instance from tracer's data """ tracer: str + """ + Tracer's name + """ pid: Optional[int] = None + """ + Process ID + """ uid: Optional[int] = None + """ + User ID + """ gid: Optional[int] = None + """ + Group ID of user + """ comm: Optional[str] = "Unknown" + """ + Command name + """ cwd: Optional[str] = None + """ + Current working directory of process + """ fname: Optional[str] = None + """ + File name which is being accessed + """ + timestamp: Optional[int] = None + """ + Timestamp of event, ns since boot + """ extended: Dict[str, Any] = {} + """ + Extended fields, will be stored in ``extended`` field as a dict + """ @staticmethod def from_namedtuple(tracer, data: NamedTuple) -> Tracking: # type: ignore + """ + Create a Tracking instance from tracer's data + """ if isinstance(tracer, type): tracer_name = getattr(tracer, "__name__") elif isinstance(tracer, str): diff --git a/duetector/config.py b/duetector/config.py index 67fc020..1a5aae7 100644 --- a/duetector/config.py +++ b/duetector/config.py @@ -18,6 +18,10 @@ class Config: """ A wrapper for config dict + + All config keys are lower case. + + Access config by ``config.key`` and get all config by ``config._config_dict``. """ def __init__(self, config_dict: Optional[Dict[str, Any]] = None): @@ -42,9 +46,14 @@ def __bool__(self): class ConfigLoader: """ - A loader for config file and environment variables - - User should use CLI for this + A loader for config file and environment variables. + + Attributes: + config_path (Path): Path to config file. + load_env (bool): Load environment variables or not. + dump_when_load (bool): Dump current config to a tmp file when load config. + config_dump_dir (str): Directory to dump config. + generate_config (bool): Generate config file if not exists. """ ENV_PREFIX = "DUETECTOR_" @@ -86,7 +95,9 @@ def generate_config(self): shutil.copy(DEFAULT_CONFIG, self.config_path) def normalize_config(self, config_dict: Dict[str, Any]) -> Dict[str, Any]: - # Make sure all config keys are lower case + """ + Make sure all config keys are lower case. + """ for k in list(config_dict.keys()): v = config_dict[k] if isinstance(v, dict): @@ -97,6 +108,9 @@ def normalize_config(self, config_dict: Dict[str, Any]) -> Dict[str, Any]: return config_dict def load_config(self) -> Dict[str, Any]: + """ + Load config from config file and environment variables. + """ logger.info(f"Loading config from {self.config_path}") if not self.config_path.exists(): raise ConfigFileNotFoundError(f"Config file:{self.config_path} not found.") @@ -123,6 +137,11 @@ def load_config(self) -> Dict[str, Any]: raise e def load_env_config(self, config_dict: Dict[str, Any]) -> Dict[str, Any]: + """ + Load config from environment variables. + + Called by ``load_config``. + """ logger.info( f"Loading config from environment variables, prefix: `{self.ENV_PREFIX}`, sep: `{self.ENV_SEP}`" ) @@ -140,6 +159,10 @@ def load_env_config(self, config_dict: Dict[str, Any]) -> Dict[str, Any]: return config_dict def dump_config(self, config_dict: Dict[str, Any], path: Union[str, Path]): + """ + Dump config to a file. + """ + dump_path = Path(path).expanduser().resolve() dump_path.parent.mkdir(parents=True, exist_ok=True) with dump_path.open("wb") as f: @@ -151,9 +174,10 @@ class Configuable: """ A base class for all configuable classes - default_config: Dict[str, Any] default config for this class - config_scope: str - config scope for this class, e.g. tracer, collector + + Attributes: + default_config (Dict[str, Any]): default config for this class + config_scope (str): config scope for this class, e.g. ``tracer``, ``collector`` """ default_config = {} diff --git a/duetector/db.py b/duetector/db.py index 7afd9bd..457a698 100644 --- a/duetector/db.py +++ b/duetector/db.py @@ -17,6 +17,10 @@ class TrackingMixin: + """ + A mixin for sqlalchemy model to track process + """ + id: Mapped[int] = mapped_column( primary_key=True, autoincrement=True, @@ -41,9 +45,19 @@ class SessionManager(Configuable): """ A wrapper for sqlalchemy session - Config: - - table_prefix: str prefix for all table names - - engine: Dict[str, Any] config for sqlalchemy.create_engine + Special config: + table_prefix (str): prefix for all table names + engine (Dict[str, Any]): config for sqlalchemy.create_engine + + Example: + + .. code-block:: python + + from duetector.db import SessionManager + sessionmanager = SessionManager() + with sessionmanager.begin() as session: + session.add(...) + """ @@ -79,10 +93,17 @@ def debug(self): @property def table_prefix(self): + """ + Prefix for all table names + """ + return self.config.table_prefix @property def engine_config(self) -> Dict[str, Any]: + """ + Config for sqlalchemy.create_engine + """ config = self.config.engine._config_dict if self.debug: config["echo"] = True @@ -97,23 +118,49 @@ def engine_config(self) -> Dict[str, Any]: @property def engine(self): + """ + A sqlalchemy engine + """ if not self._engine: self._engine = sqlalchemy.create_engine(**self.engine_config) return self._engine @property def sessionmaker(self): + """ + A sessionmaker for sqlalchemy session + """ if not self._sessionmaker: self._sessionmaker = sessionmaker(bind=self.engine) return self._sessionmaker @contextmanager def begin(self) -> Generator[Session, None, None]: + """ + Get a sqlalchemy session. + + Example: + + .. code-block:: python + + with session_manager.begin() as session: + session.add(...) + """ with self.sessionmaker.begin() as session: yield session def get_tracking_model(self, tracer: str = "unknown", collector_id: str = "") -> type: - # For thread safety + """ + Get a sqlalchemy model for tracking, each tracer will create a table in database. + + Args: + tracer (str): name of tracer + collector_id (str): id of collector + + Returns: + type: a sqlalchemy model for tracking + """ + with self.mutex: if tracer in self._tracking_models: return self._tracking_models[tracer] diff --git a/duetector/filters/base.py b/duetector/filters/base.py index 3390bb4..7915a7e 100644 --- a/duetector/filters/base.py +++ b/duetector/filters/base.py @@ -7,67 +7,59 @@ class Filter(Configuable): """ - A base class for all filters + A base class for all filters. - Implement `filter` method - User should call Filter() directly to filter data + Default config scope is ``filter.{class_name}``. + + subclass should override ``filter`` method. + + User should call Filter() directly to filter data, + Example: + + .. code-block:: python + + from duetector.filters import Filter + from duetector.collectors.models import Tracking + + class MyFilter(Filter): + def filter(self, data: Tracking) -> Optional[Tracking]: + if data.fname == "/etc/passwd": + return None + return data + + f = MyFilter() + f(Tracking(fname="/etc/passwd")) # None + f(Tracking(fname="/etc/shadow")) # Tracking(fname="/etc/shadow") """ default_config = { "disabled": False, } + """ + Default config for ``Filter``. + """ @property def config_scope(self): + """ + Config scope for current filter. + """ return self.__class__.__name__ @property def disabled(self): + """ + If current filter is disabled. + """ return self.config.disabled def filter(self, data: NamedTuple) -> Optional[NamedTuple]: + """ + Filter data, return ``None`` to drop data, return data to keep data. + """ raise NotImplementedError def __call__(self, data: NamedTuple) -> Optional[NamedTuple]: if self.disabled: return data return self.filter(data) - - -class DefaultFilter(Filter): - """ - A default filter to filter some useless data - - TODO: Split to multiple filters if needed - """ - - def filter(self, data: NamedTuple) -> Optional[NamedTuple]: - fname = getattr(data, "fname", None) - if ( - ( - fname - and any( - [ - fname.startswith(p) - for p in [ - "/proc", - "/sys", - "/lib", - "/dev", - "/run", - "/usr/lib", - "/etc/ld.so.cache", - ] - ] - ) - ) - or getattr(data, "pid", None) == os.getpid() - or getattr(data, "uid", None) == 0 - ): - return None - return data - - -@hookimpl -def init_filter(config=None): - return DefaultFilter(config=config) diff --git a/duetector/filters/pattern.py b/duetector/filters/pattern.py index 74bfe8a..fb76a22 100644 --- a/duetector/filters/pattern.py +++ b/duetector/filters/pattern.py @@ -8,20 +8,25 @@ class PatternFilter(Filter): """ - A Filter support regex pattern to filter data - - Usage: - There are following config build-in: - - re_exclude_fname: Regex pattern to filter out `fname` field - - re_exclude_comm: Regex pattern to filter out `comm` field - - exclude_pid: Filter out `pid` field - - exclude_uid: Filter out `uid` field - - exclude_gid: Filter out `gid` field - - Customize exclude is also supported - e.g.: - - re_exclude_custom: Regex pattern to filter out `custom` - - exclude_custom: Filter out `custom` field + A Filter support regex pattern to filter data. + + There are following config build-in: + - ``re_exclude_fname``: Regex pattern to filter out ``fname`` field + - ``re_exclude_comm``: Regex pattern to filter out ``comm`` field + - ``exclude_pid``: Filter out ``pid`` field + - ``exclude_uid``: Filter out ``uid`` field + - ``exclude_gid``: Filter out ``gid`` field + + Customize exclude is also supported: + - ``re_exclude_custom``: Regex pattern to filter out ``custom`` field + - ``exclude_custom``: Filter out ``custom`` field + + You can change ``custom`` to any field you want to filter out. + + Config ``enable_customize_exclude`` to enable customize exclude, default is ``True``. + + Use ``(?!…)`` for include pattern: + - ``re_exclude_custom``: ``["(?!/proc/)"]`` will include ``/proc`` but exclude others. """ default_config = { @@ -45,13 +50,25 @@ class PatternFilter(Filter): 0, ], } + """ + Default config for ``PatternFilter`` + """ _re_cache = {} + """ + Cache for re pattern + """ @property def enable_customize_exclude(self) -> bool: + """ + If enable customize exclude + """ return bool(self.config.enable_customize_exclude) def customize_exclude(self, data: NamedTuple) -> bool: + """ + Customize exclude function, return ``True`` to drop data, return ``False`` to keep data. + """ for k in self.config._config_dict: if k.startswith("exclude_"): field = k.replace("exclude_", "") @@ -64,6 +81,9 @@ def customize_exclude(self, data: NamedTuple) -> bool: return False def re_exclude(self, field: Optional[str], re_list: Union[str, List[str]]) -> bool: + """ + Check if field match any pattern in re_list + """ if not field: return False if isinstance(re_list, str): @@ -77,6 +97,10 @@ def _cached_search(pattern, field): return any(_cached_search(pattern, field) for pattern in re_list) def filter(self, data: NamedTuple) -> Optional[NamedTuple]: + """ + Filter data, return ``None`` to drop data, return data to keep data. + """ + if getattr(data, "pid", None) == os.getpid(): return if self.re_exclude(getattr(data, "fname", None), self.config.re_exclude_fname): diff --git a/duetector/managers/base.py b/duetector/managers/base.py index cbf8cbc..1255bd9 100644 --- a/duetector/managers/base.py +++ b/duetector/managers/base.py @@ -1,3 +1,5 @@ +from typing import Any, Dict, Optional + import pluggy from duetector.config import Configuable @@ -7,22 +9,58 @@ class Manager(Configuable): """ Manager based on pulggy - FIXME: Need better abstraction, lots of duplicated code in subclasses + Default config scope is ``{class_name}`` + + FIXME: + Need better abstraction, lots of duplicated code in subclasses """ pm: pluggy.PluginManager + """ + PluginManager instance + """ + + default_config = { + "disabled": False, + "include_extension": True, + } + """ + Default config for ``Manager`` + """ + + def __init__(self, config: Optional[Dict[str, Any]] = None, *args, **kwargs): + super().__init__(config, *args, **kwargs) - default_config = {"disabled": False} + # Allow disable extensions when instantiate + self._include_extension = kwargs.get("disable_extensions") + + @property + def include_extension(self): + """ + If include extensions + """ + if self._include_extension is not None: + return self._include_extension + return self.config.include_extension def register(self, subpackage): + """ + Register subpackage to plugin manager + """ registers = getattr(subpackage, "registers", [subpackage]) for register in registers: self.pm.register(register) @property def config_scope(self): + """ + Config scope for current manager. + """ return self.__class__.__name__ @property def disabled(self): + """ + If current manager is disabled. + """ return self.config.disabled diff --git a/duetector/managers/collector.py b/duetector/managers/collector.py index 0d9036f..0c40c4d 100644 --- a/duetector/managers/collector.py +++ b/duetector/managers/collector.py @@ -9,30 +9,48 @@ from duetector.log import logger from duetector.managers import Manager -hookspec = pluggy.HookspecMarker(project_name) +PROJECT_NAME = project_name #: Default project name for pluggy +hookspec = pluggy.HookspecMarker(PROJECT_NAME) @hookspec def init_collector(config) -> Optional[Collector]: """ - Initialize collector from config - None means the collector is not available - Also the collector can be disabled by config, Manager will discard disabled collectors + Initialize collector from config. + + None means the collector is not available. + + Also the collector can be ``disabled`` by config, Manager will discard disabled collectors. """ class CollectorManager(Manager): + """ + Manager for all collectors. + + Collectors are initialized from config, and can be ``disabled`` by config. + """ + config_scope = "collector" + """ + Config scope for ``CollectorManager``. + """ def __init__(self, config: Optional[Dict[str, Any]] = None, *args, **kwargs): super().__init__(config, *args, **kwargs) - self.pm = pluggy.PluginManager(project_name) + self.pm = pluggy.PluginManager(PROJECT_NAME) self.pm.add_hookspecs(sys.modules[__name__]) - self.pm.load_setuptools_entrypoints(project_name) + self.pm.load_setuptools_entrypoints(PROJECT_NAME) self.register(duetector.collectors) def init(self, ignore_disabled=True) -> List[Collector]: + """ + Initialize all collectors from config. + + Args: + ignore_disabled: Ignore disabled collectors + """ if self.disabled: logger.info("CollectorManager disabled.") return [] diff --git a/duetector/managers/filter.py b/duetector/managers/filter.py index 8bbfb5c..16f541b 100644 --- a/duetector/managers/filter.py +++ b/duetector/managers/filter.py @@ -9,7 +9,8 @@ from duetector.log import logger from duetector.managers import Manager -hookspec = pluggy.HookspecMarker(project_name) +PROJECT_NAME = project_name #: Default project name for pluggy +hookspec = pluggy.HookspecMarker(PROJECT_NAME) @hookspec @@ -22,17 +23,32 @@ def init_filter(config) -> Optional[Filter]: class FilterManager(Manager): + """ + Manager for all filters. + + Filters are initialized from config, and can be ``disabled`` by config. + """ + config_scope = "filter" + """ + Config scope for ``FilterManager``. + """ def __init__(self, config: Optional[Dict[str, Any]] = None, *args, **kwargs): super().__init__(config, *args, **kwargs) - self.pm = pluggy.PluginManager(project_name) + self.pm = pluggy.PluginManager(PROJECT_NAME) self.pm.add_hookspecs(sys.modules[__name__]) - self.pm.load_setuptools_entrypoints(project_name) + self.pm.load_setuptools_entrypoints(PROJECT_NAME) self.register(duetector.filters) def init(self, ignore_disabled=True) -> List[Filter]: + """ + Initialize all filters from config. + + Args: + ignore_disabled: Ignore disabled filters + """ if self.disabled: logger.info("FilterManager disabled.") return [] diff --git a/duetector/managers/tracer.py b/duetector/managers/tracer.py index bb3f31a..e7ce10b 100644 --- a/duetector/managers/tracer.py +++ b/duetector/managers/tracer.py @@ -9,7 +9,8 @@ from duetector.managers import Manager from duetector.tracers.base import Tracer -hookspec = pluggy.HookspecMarker(project_name) +PROJECT_NAME = project_name #: Default project name for pluggy +hookspec = pluggy.HookspecMarker(PROJECT_NAME) @hookspec @@ -22,17 +23,33 @@ def init_tracer(config) -> Optional[Tracer]: class TracerManager(Manager): + """ + Manager for all tracers. + + Tracers are initialized from config, and can be ``disabled`` by config. + """ + config_scope = "tracer" + """ + Config scope for ``TracerManager``. + """ def __init__(self, config: Optional[Dict[str, Any]] = None, *args, **kwargs): super().__init__(config, *args, **kwargs) - self.pm = pluggy.PluginManager(project_name) + self.pm = pluggy.PluginManager(PROJECT_NAME) self.pm.add_hookspecs(sys.modules[__name__]) - self.pm.load_setuptools_entrypoints(project_name) + self.pm.load_setuptools_entrypoints(PROJECT_NAME) self.register(duetector.tracers) def init(self, tracer_type=Tracer, ignore_disabled=True) -> List[Tracer]: + """ + Initialize all tracers from config. + + Args: + tracer_type: Only return tracers of this type + ignore_disabled: Ignore disabled tracers + """ if self.disabled: logger.info("TracerManager disabled.") return [] diff --git a/duetector/monitors/base.py b/duetector/monitors/base.py index c80f20d..d0060f5 100644 --- a/duetector/monitors/base.py +++ b/duetector/monitors/base.py @@ -18,10 +18,23 @@ class Monitor(Configuable): """ tracers: List[Tracer] + """ + A list of tracers, should be initialized by ``TracerManager`` + """ + filters: List[Filter] + """ + A list of filters, should be initialized by ``FilterManager`` + """ collectors: List[Collector] + """ + A list of collectors, should be initialized by ``CollectorManager`` + """ config_scope = "monitor" + """ + Config scope for monitor. + """ default_config = { "disabled": False, @@ -32,8 +45,18 @@ class Monitor(Configuable): **Poller.default_config, }, } + """ + Default config for monitor. + + Two sub-configs are available: + - backend_args: config for ``self._backend_imp`` + - poller: config for ``Poller`` + """ _backend_imp = ThreadPoolExecutor + """ + A backend implementation. + """ def __init__(self, config: Optional[Dict[str, Any]] = None, *args, **kwargs): super().__init__(config=config) @@ -42,19 +65,34 @@ def __init__(self, config: Optional[Dict[str, Any]] = None, *args, **kwargs): @property def disabled(self): + """ + If disabled, no tracers, filters, collectors will be initialized. + """ return self.config.disabled @property def backend_args(self): + """ + Config for ``self._backend_imp``. + """ return self.config.backend_args def poll_all(self): + """ + Poll all tracers. Depends on ``self.poll``. + """ return [self._backend.submit(self.poll, tracer) for tracer in self.tracers] def poll(self, tracer: Tracer): + """ + Poll a tracer. Should be implemented by subclass. + """ raise NotImplementedError def summary(self) -> Dict: + """ + Get a summary of all collectors. + """ return { self.__class__.__name__: { collector.__class__.__name__: collector.summary() for collector in self.collectors @@ -62,10 +100,16 @@ def summary(self) -> Dict: } def start_polling(self): + """ + Start polling. Poller will call ``self.poll_all`` periodically. + """ logger.info(f"Start polling {self.__class__.__name__}") self.poller.start(self.poll_all) def shutdown(self): + """ + Shutdown the monitor. + """ self.poller.shutdown() self.poller.wait() self._backend.shutdown() diff --git a/duetector/monitors/bcc_monitor.py b/duetector/monitors/bcc_monitor.py index 54fd315..8494cda 100644 --- a/duetector/monitors/bcc_monitor.py +++ b/duetector/monitors/bcc_monitor.py @@ -12,8 +12,9 @@ class BccMonitor(Monitor): """ A monitor use bcc.BPF host - Config: - - auto_init: Init tracers on init + Special config: + - auto_init: Auto init tracers when init monitor. + - continue_on_exception: Continue on exception when init tracers. """ config_scope = "monitor.bcc" @@ -25,10 +26,16 @@ class BccMonitor(Monitor): @property def continue_on_exception(self): + """ + Continue on exception when init tracers. + """ return self.config.continue_on_exception @property def auto_init(self): + """ + Auto init tracers when init monitor. + """ return self.config.auto_init def __init__(self, config: Optional[Dict[str, Any]] = None, *args, **kwargs): @@ -49,6 +56,10 @@ def __init__(self, config: Optional[Dict[str, Any]] = None, *args, **kwargs): self.init() def init(self): + """ + Init all tracers + """ + # Prevrent ImportError for CI testing without bcc from bcc import BPF # noqa @@ -78,6 +89,10 @@ def init(self): self.tracers.remove(tracer) def _set_callback(self, host, tracer): + """ + Wrap tracer callback with filters and collectors. + """ + def _(data): for filter in self.filters: data = filter(data) @@ -89,6 +104,9 @@ def _(data): tracer.set_callback(host, _) def poll(self, tracer: BccTracer): # type: ignore + """ + Implement poll method for bcc tracers. + """ tracer.get_poller(self.bpf_tracers[tracer])(**tracer.poll_args) diff --git a/duetector/monitors/sh_monitor.py b/duetector/monitors/sh_monitor.py index 328c406..b999992 100644 --- a/duetector/monitors/sh_monitor.py +++ b/duetector/monitors/sh_monitor.py @@ -10,7 +10,9 @@ class ShTracerHost: """ - Host for ShellTracer + Host for Shell, provide a way to poll shell command. + + Use ``subprocess.Popen`` to run shell command. """ def __init__(self, backend, timeout=5): @@ -28,16 +30,16 @@ def delete(self, tracer): def get_poller(self, tracer) -> Callable[[None], None]: """ - Combine tracers' comm and callback - Once poller is called, - it will call tracers' comm and pass its results to all tracers' callback + Provide a callback function for ``Poller``. + + Use ``subprocess.Popen`` to run shell command, pipe stdout to callback. """ comm = self.tracers[tracer] enable_cache = tracer.enable_cache def _(): p = subprocess.Popen(comm, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - p.wait() + p.wait(self.timeout) output = p.stdout.read().decode("utf-8") if enable_cache: if output == tracer.get_cache(): @@ -51,33 +53,59 @@ def _(): return _ def poll(self, tracer): + """ + Poll a tracer. + """ self.get_poller(tracer)() def poll_all(self): + """ + Poll all tracers. + """ return [self.backend.submit(self.poll, tracer) for tracer in self.tracers] def set_callback(self, tracer, callback): + """ + Set callback for tracer. + """ self.callbacks[tracer] = callback class ShMonitor(Monitor): - # A monitor for shell command - # Start shell tracers, daemon them - # Bring tracers, filters, collections together + """ + A monitor for shell command. + """ config_scope = "monitor.sh" + """ + Config scope for this monitor. + """ + default_config = { **Monitor.default_config, "auto_init": True, - "timeout": 5, + "timeout": 30, } + """ + Default config for this monitor. + + Two sub-configs are available: + - auto_init: Auto init tracers when init monitor. + - timeout: Timeout for shell command. + """ @property def auto_init(self): + """ + Auto init tracers when init monitor. + """ return self.config.auto_init @property def timeout(self): + """ + Timeout for shell command. + """ return int(self.config.timeout) def __init__(self, config: Optional[Dict[str, Any]] = None, *args, **kwargs): diff --git a/duetector/static/config.toml b/duetector/static/config.toml index 83e450f..540bd46 100644 --- a/duetector/static/config.toml +++ b/duetector/static/config.toml @@ -7,6 +7,7 @@ [filter] disabled = false +include_extension = true [filter.patternfilter] disabled = false @@ -31,6 +32,7 @@ exclude_gid = [ [tracer] disabled = false +include_extension = true [tracer.clonetracer] disabled = false @@ -52,6 +54,7 @@ poll_timeout = 10 [collector] disabled = false +include_extension = true [collector.dbcollector] disabled = false @@ -88,7 +91,7 @@ interval_ms = 500 [monitor.sh] disabled = false auto_init = true -timeout = 5 +timeout = 30 [monitor.sh.backend_args] max_workers = 10 diff --git a/duetector/tools/config_generator.py b/duetector/tools/config_generator.py index 7abcf7a..92e8416 100644 --- a/duetector/tools/config_generator.py +++ b/duetector/tools/config_generator.py @@ -14,6 +14,10 @@ def _recursive_load(config_scope: str, config_dict: dict, default_config: dict): """ Support .(dot) separated config_scope + Example: + >>> _recursive_load("monitor.bcc", {}, {"auto_init": True}) + {'monitor': {'bcc': {'auto_init': True}}} + """ *prefix, config_scope = config_scope.lower().split(".") last = config_dict @@ -24,7 +28,13 @@ def _recursive_load(config_scope: str, config_dict: dict, default_config: dict): class ConfigGenerator: """ - Tools for generate config file by inspecting all modules + Tools for generate config file by inspecting all modules. + + Args: + load (bool): Load config file or not. + path (str): Path to config file. + load_env (bool): Load environment variables or not. + include_extension (bool): Include extensions or not. """ HEADLINES = """# This is a auto generated config file for duetector🔍 @@ -37,14 +47,27 @@ class ConfigGenerator: """ managers = [FilterManager, TracerManager, CollectorManager] + """ + All managers to inspect. + """ + monitors = [BccMonitor, ShMonitor] + """ + All monitors to inspect. + """ - def __init__(self, load=True, path=None, load_env=True): + def __init__( + self, + load: bool = True, + path: bool = None, + load_env: bool = True, + include_extension: bool = True, + ): # dynamic_config containers all default config for all modules, including extensions self.dynamic_config = {} for manager in self.managers: - m = manager() + m = manager(include_extension=include_extension) _recursive_load(m.config_scope, self.dynamic_config, m.default_config) for c in m.init(ignore_disabled=False): _recursive_load( @@ -72,6 +95,9 @@ def _recursive_update(c, config): _recursive_update(self.dynamic_config, self.loaded_config) def generate(self, dump_path): + """ + Generate config file to dump_path. + """ dump_path = Path(dump_path).expanduser().absolute() logger.info(f"Dumping dynamic config to {dump_path}") @@ -84,6 +110,6 @@ def generate(self, dump_path): if __name__ == "__main__": _HERE = Path(__file__).parent - c = ConfigGenerator(load=False, load_env=False) + c = ConfigGenerator(load=False, load_env=False, include_extension=False) config_path = _HERE / ".." / "static/config.toml" c.generate(config_path) diff --git a/duetector/tools/daemon.py b/duetector/tools/daemon.py index 45e748c..f30c3fb 100644 --- a/duetector/tools/daemon.py +++ b/duetector/tools/daemon.py @@ -1,3 +1,9 @@ +# Modified from https://github.com/Wh1isper/sparglim/blob/0.1.4/sparglim/server/daemon.py +# Original license: +# Copyright (c) 2023 Wh1isper +# Licensed under the BSD 3-Clause License + + import os import subprocess from datetime import datetime @@ -10,6 +16,28 @@ class Daemon: + """ + Start a daemon process and record pid. + + Args: + workdir (str, Path): Working directory for daemon process. + cmd (List[str]): Command to start daemon process. + env_dict (Dict[str, str]): Environment variables for daemon process. + rotate_log (bool): Rotate log file or not. + + Example: + >>> d = Daemon( + ... cmd=["sleep", "100"], + ... workdir="/tmp/duetector", + ... env_dict={"DUETECTOR_LOG_LEVEL": "DEBUG"}, + ... auto_restart=True, + ... rotate_log=True, + ... ) + >>> d.start() + >>> d.poll() + >>> d.stop() + """ + def __init__( self, workdir: Union[str, Path], @@ -29,14 +57,23 @@ def __init__( @property def pid_file(self): + """ + Path to pid file. + """ return self.workdir / "pid" @property def log_file(self): + """ + Path to log file. + """ return self.workdir / "log" @property def pid(self): + """ + Pid of daemon process. + """ if not self.pid_file.exists(): return None @@ -44,12 +81,18 @@ def pid(self): return int(f.read()) def _rotate_log(self): + """ + Rotate log file. + """ now = datetime.now() new_log_file = self.log_file.with_suffix(f".{now:%Y%m%d%H%M%S}") logger.info(f"Rotate log file to {new_log_file}") self.log_file.rename(new_log_file) def start(self): + """ + Start daemon process. + """ if not self.cmd: raise RuntimeError("cmd is empty, nothing to start") @@ -73,6 +116,9 @@ def start(self): assert self.pid def stop(self): + """ + Stop daemon process. + """ if not self.pid: return pid = self.pid @@ -93,6 +139,9 @@ def stop(self): self.pid_file.unlink(missing_ok=True) def poll(self) -> bool: + """ + Poll daemon process. + """ if not self.pid: logger.info("Daemon is not running") return False diff --git a/duetector/tools/poller.py b/duetector/tools/poller.py index 5e9478c..43b213a 100644 --- a/duetector/tools/poller.py +++ b/duetector/tools/poller.py @@ -6,10 +6,24 @@ class Poller(Configuable): + """ + A wrapper for ``threading.Thread`` + + Special config: + - interval_ms: Polling interval in milliseconds + """ + config_scope = "poller" + """ + Config scope for this poller. + """ + default_config = { "interval_ms": 500, } + """ + Default config for this poller. + """ def __init__( self, @@ -23,9 +37,18 @@ def __init__( @property def interval_ms(self): + """ + Polling interval in milliseconds + """ return self.config.interval_ms def start(self, func, *args, **kwargs): + """ + Start a poller thread, until ``shutdown`` is called. + + Exceptions: + - RuntimeError: If poller thread is already started. + """ if self._thread: raise RuntimeError("Poller thread is already started, try shutdown and wait first.") @@ -39,9 +62,19 @@ def _poll(): self._thread.start() def shutdown(self): + """ + Shutdown poller thread. It's safe to call this method multiple times. + + After shutdown, ``wait`` should be called to wait for the thread to exit. + """ self.shutdown_event.set() def wait(self, timeout_ms=None): + """ + Wait for poller thread to exit. + + Call this method after ``shutdown``. + """ if not self._thread: return diff --git a/duetector/tracers/__init__.py b/duetector/tracers/__init__.py index c7d2655..6a3ee06 100644 --- a/duetector/tracers/__init__.py +++ b/duetector/tracers/__init__.py @@ -1,6 +1,6 @@ -from .base import BccTracer +from .base import BccTracer, ShellTracer, Tracer -__all__ = ["BccTracer"] +__all__ = ["Tracer", "BccTracer", "ShellTracer"] # Expose for plugin system from . import clone, openat2, tcpconnect, uname diff --git a/duetector/tracers/base.py b/duetector/tracers/base.py index ba7f703..c462448 100644 --- a/duetector/tracers/base.py +++ b/duetector/tracers/base.py @@ -8,73 +8,126 @@ class Tracer(Configuable): """ - A base class for all tracers + A base class for all tracers. - Subclass should implement attach, detach, get_poller and set_callback - `data_t` is a NamedTuple, which is used to convert raw data to a NamedTuple + As a reverse dependency for host, + subclass should implement ``attach``, ``detach``, ``get_poller`` and ``set_callback``. + This allow tracer to decide how to attach to host, how to detach from host. + + ``data_t`` is a NamedTuple, which is used to convert raw data to a ``NamedTuple``. + + Default scope for config is ``Tracer.__class__.__name__``. """ data_t: NamedTuple + """ + Data type for this tracer. + """ + default_config = { "disabled": False, } + """ + Default config for this tracer. + """ @property def config_scope(self): + """ + Config scope for this tracer. + """ return self.__class__.__name__ @property def disabled(self): + """ + If this tracer is disabled. + """ return self.config.disabled def attach(self, host): + """ + Attach this tracer to host. + """ raise NotImplementedError("attach not implemented") def detach(self, host): + """ + Detach this tracer from host. + """ raise NotImplementedError("detach not implemented") def get_poller(self, host) -> Callable: + """ + Get a poller function from host. + """ raise NotImplementedError("get_poller not implemented") def set_callback(self, host, callback: Callable[[NamedTuple], None]): - # attatch callback to host + """ + Set a callback function to host. + """ raise NotImplementedError("set_callback not implemented") class BccTracer(Tracer): """ - A Tracer use bcc.BPF host + A Tracer use ``bcc.BPF`` as a host - attatch_type: str attatch type for bcc.BPF, e.g. kprobe, kretprobe, etc. - attatch_args: Dict[str, str] args for attatch function - many_attatchs: List[Tuple[str, Dict[str, str]]] list of attatch function name and args - poll_fn: str poll function name in bcc.BPF - poll_args: Dict[str, str] args for poll function - prog: str bpf program - data_t: NamedTuple data type for this tracer + For simple tracers, you can just set ``attach_type``, ``attatch_args`` to attatch to ``bcc.BPF``. + Equal to `bcc.BPF(prog).attatch_{attatch_type}(**attatch_args)` - For simple tracers, you can just set `attach_type`, `attatch_args` to attatch to bcc.BPF - equal to `bcc.BPF(prog).attatch_{attatch_type}(**attatch_args)` + For those tracers need to attatch multiple times, set ``many_attatchs`` to attatch multiple times. - For those tracers need to attatch multiple times, you can set `many_attatchs` to attatch multiple times + ``set_callback`` should attatch ``callback`` to ``bpf``, translate raw data to ``data_t`` then call the ``callback`` - set_callback should attatch callback to bpf, translate raw data to data_t then call the callback - # FIXME: Maybe it's hard for using? Maybe we should use a more simple way to implement this? + FIXME: + - Maybe it's hard for using? Maybe we should use a more simple way to implement this? """ default_config = { **Tracer.default_config, } + """ + Default config for this tracer. + """ attach_type: Optional[str] = None + """ + Attatch type for ``bcc.BPF``, called as ``BPF.attatch_{attach_type}``, + """ + attatch_args: Dict[str, str] = {} + """ + Args for attatch function. + """ + many_attatchs: List[Tuple[str, Dict[str, str]]] = [] + """ + List of attatch function name and args. + ``attatch_type``, ``attatch_args`` will merge to this list. + """ + poll_fn: str + """ + Poll function name in ``bcc.BPF`` + """ + poll_args: Dict[str, str] = {} + """ + Args for poll function. + Remenber to set ``timeout`` for poll function in ``poll_args`` if needed, + """ + prog: str - data_t: NamedTuple + """ + bpf program + """ def _convert_data(self, data) -> NamedTuple: + """ + Convert raw data to ``data_t``. + """ args = {} for k in self.data_t._fields: # type: ignore v = getattr(data, k) @@ -86,6 +139,9 @@ def _convert_data(self, data) -> NamedTuple: return self.data_t(**args) # type: ignore def _attatch(self, host, attatch_type, attatch_args): + """ + Wrapper for ``bcc.BPF.attatch_{attatch_type}``. + """ if not attatch_type: return attatcher = getattr(host, f"attach_{attatch_type}") @@ -94,6 +150,11 @@ def _attatch(self, host, attatch_type, attatch_args): return attatcher(**attatch_args) def attach(self, host): + """ + Attatch to host. + + Merge ``attatch_type``, ``attatch_args`` to ``many_attatchs`` then attatch. + """ if self.disabled: raise TreacerDisabledError("Tracer is disabled") @@ -102,28 +163,47 @@ def attach(self, host): for attatch_type, attatch_args in attatch_list: self._attatch(host, attatch_type, attatch_args) + def _detach(self, host, attatch_type, attatch_args): + """ + Wrapper for ``bcc.BPF.detach_{attatch_type}`` + """ + if not attatch_type: + return + attatcher = getattr(host, f"detach_{attatch_type}") + # Prevent AttributeError + attatch_args = attatch_args or {} + return attatcher(**attatch_args) + def detach(self, host): + """ + Detach from host. + + Merge ``attatch_type``, ``attatch_args`` to ``many_attatchs`` then detach. + + FIXME: + - Maybe we should specify ``detach*`` for detaching? + """ if self.disabled: raise TreacerDisabledError("Tracer is disabled") - if not self.attach_type: - # Means prog is not attached by python's BPF.attatch_() - # So user should detach it manually - raise TracerError("Unable to detach, no attach type specified") - attatcher = getattr(host, f"detach_{self.attach_type}") - return attatcher(**self.attatch_args) + attatch_list = [*self.many_attatchs, (self.attach_type, self.attatch_args)] + + for attatch_type, attatch_args in attatch_list: + self._detach(host, attatch_type, attatch_args) def get_poller(self, host) -> Callable: + """ + Get poller function from host. + """ if self.disabled: raise TreacerDisabledError("Tracer is disabled") if not self.poll_fn: - # Not support poll + # Not support poll, prevent AttributeError, fake one def _(*args, **kwargs): pass - # Prevent AttributeError return _ poller = getattr(host, self.poll_fn) @@ -132,26 +212,58 @@ def _(*args, **kwargs): return poller def set_callback(self, host, callback: Callable[[NamedTuple], None]): + """ + Set callback function to host. + + Should implemented by subclass. + """ raise NotImplementedError("set_callback not implemented") class ShellTracer(Tracer): + """ + A tracer use ``ShTracerHost`` as host. + More detail on :doc:`ShellMonitor and ShTracerHost `. + + Output of shell command will be converted to ``data_t`` and cached by default. + + Attributes: + comm (List[str]): shell command + data_t (NamedTuple): data type for this tracer + + Special config: + - enable_cache: If enable cache for this tracer. + Cache means the same output will not be converted and emited again. + """ + comm = List[str] + """ + shell command + """ data_t = namedtuple("ShellOutput", ["output"]) + """ + data type for this tracer + """ _cache: Optional[Any] = None - default_config = {"disabled": False, "enable_cache": True} + """ + cache for this tracer + """ + default_config = {**Tracer.default_config, "enable_cache": True} + """ + Default config for this tracer. + """ def __init__(self, config: Optional[Union[Config, Dict[str, Any]]] = None, *args, **kwargs): super().__init__(config, *args, **kwargs) self.mutex = Lock() - @property - def config_scope(self): - return self.__class__.__name__ - @property def enable_cache(self): + """ + If enable cache for this tracer. + """ + return self.config.enable_cache @property @@ -159,20 +271,38 @@ def disabled(self): return self.config.disabled def set_cache(self, cache): + """ + Set cache for this tracer. + """ with self.mutex: self._cache = cache def get_cache(self): + """ + Get cache for this tracer. + """ return self._cache def attach(self, host): + """ + Attach to host. + """ host.attach(self) def detach(self, host): + """ + Detach from host. + """ host.detach(self) def get_poller(self, host) -> Callable: + """ + Get poller function from host. + """ return host.get_poller(self) def set_callback(self, host, callback: Callable[[NamedTuple], None]): + """ + Set callback function to host. + """ host.set_callback(self, callback) diff --git a/duetector/tracers/clone.py b/duetector/tracers/clone.py index 39f6d66..2008eee 100644 --- a/duetector/tracers/clone.py +++ b/duetector/tracers/clone.py @@ -7,7 +7,7 @@ class CloneTracer(BccTracer): """ - A tracer for clone syscall + A tracer for clone syscall. """ default_config = { diff --git a/duetector/tracers/dummy.py b/duetector/tracers/dummy.py index 3b2f872..176bffd 100644 --- a/duetector/tracers/dummy.py +++ b/duetector/tracers/dummy.py @@ -19,7 +19,10 @@ def set_callback(self, func): class DummyTracer(BccTracer): - # Fake a tracer that does nothing for testing + """ + Fake a tracer that does nothing for testing. + """ + attach_type = "dummy" poll_fn = "poll_dummy" prog = "This is not a runnable program" diff --git a/duetector/tracers/openat2.py b/duetector/tracers/openat2.py index b367314..0701f8d 100644 --- a/duetector/tracers/openat2.py +++ b/duetector/tracers/openat2.py @@ -7,7 +7,7 @@ class OpenTracer(BccTracer): """ - A tracer for openat2 syscall + A tracer for openat2 syscall. """ default_config = { diff --git a/duetector/tracers/tcpconnect.py b/duetector/tracers/tcpconnect.py index 377fda5..c199e16 100644 --- a/duetector/tracers/tcpconnect.py +++ b/duetector/tracers/tcpconnect.py @@ -29,7 +29,6 @@ def poll_args(self): data_t = namedtuple("TcpTracking", ["pid", "uid", "gid", "comm", "saddr", "daddr", "dport"]) - # define BPF program prog = """ #include #include diff --git a/duetector/tracers/uname.py b/duetector/tracers/uname.py index 8854e24..facd717 100644 --- a/duetector/tracers/uname.py +++ b/duetector/tracers/uname.py @@ -3,6 +3,10 @@ class UnameTracer(ShellTracer): + """ + A tracer for uname command. + """ + comm = ["uname", "-a"] diff --git a/pyproject.toml b/pyproject.toml index c2413c3..4790911 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,5 @@ [build-system] -requires = ["hatchling", ] +requires = ["hatchling"] build-backend = "hatchling.build" [project] @@ -15,9 +15,9 @@ dependencies = [ "tomli-w", "SQLAlchemy>=2", "click", - "psutil" + "psutil", ] -dynamic = ["version", ] +dynamic = ["version"] classifiers = [ "Programming Language :: Python :: 3", 'Programming Language :: Python :: 3.8', @@ -26,11 +26,8 @@ classifiers = [ 'Programming Language :: Python :: 3.11', ] [project.optional-dependencies] -test = [ - "pytest", - "pytest-cov", - "pytype", -] +test = ["pytest", "pytest-cov", "pytype"] +docs = ["Sphinx", "sphinx-rtd-theme", "sphinx-click"] [project.scripts] duectl = "duetector.cli.main:cli" @@ -51,7 +48,7 @@ text = "Apache Software License 2.0" Source = "https://github.com/hitsz-ids/duetector" [tool.check-manifest] -ignore = [".*", ] +ignore = [".*"] [tool.hatch.version] path = "duetector/__init__.py"