diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 73bc6b2..4ff7504 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -63,6 +63,7 @@ Use [start-docs-host.sh](dev-tools/start-docs-host.sh) to deploy a local http se ```bash cd ./dev-tools && ./start-docs-host.sh ``` + Access `http://localhost:8080` for docs. ## Typing @@ -72,3 +73,11 @@ Access `http://localhost:8080` for docs. ``` pytype ./duetector ``` + + +## Contributing a new tracer/filter/collector + +1. Create a new file in `duetector/tracer`, `duetector/filter` or `duetector/collector` directory, with the name `{name}.py` +2. Implement the new tracer/filter/collector +3. Add the new tracer/filter/collector to `registers` list in `duetector/tracer/register.py`, `duetector/filter/register.py` or `duetector/collector/register.py` +4. Test the new tracer/filter/collector diff --git a/docs/design/README.md b/docs/design/README.md index 12ecf22..927a357 100644 --- a/docs/design/README.md +++ b/docs/design/README.md @@ -6,6 +6,7 @@ Key Components and Features: - [ ] **HTTP / RPC Server**: PIP Server, providing API for PDP to get data usage information. - [ ] **Analyzer**: Analyze data usage information and generate data usage behavior. + - [ ] **DBAnalyzer**: Analyze data usage information from database. - [X] **CLI**: CLI for administrator to manage duetector. - [X] **BccMonitor**: Monitor data usage behavior in kernel space. Use BCC to implement. - [X] **ShMonitor**: A general monitor for custom command. Polling the output of command. @@ -32,5 +33,5 @@ Current data flow implementation: The following are not yet realized and may be subject to change. -- [ ] **Collector** will get data from **Collector**. -- [ ] **HTTP / RPC Server** will get data from **Collector**. +- [ ] **Analyzer**'s data stracture and API. +- [ ] **Query Service** will get data from **Analyzer**. diff --git a/docs/design/image/architecture.png b/docs/design/image/architecture.png index 1882824..40510aa 100644 Binary files a/docs/design/image/architecture.png and b/docs/design/image/architecture.png differ diff --git a/docs/design/image/dataflow.png b/docs/design/image/dataflow.png index 96a8091..326a25b 100644 Binary files a/docs/design/image/dataflow.png and b/docs/design/image/dataflow.png differ diff --git a/docs/design/src/architecture.drawio b/docs/design/src/architecture.drawio index 4f24ed2..09ccc2a 100644 --- a/docs/design/src/architecture.drawio +++ b/docs/design/src/architecture.drawio @@ -1,83 +1,169 @@ - + + + + + + + + + + + + + - + - - + + - + - + - + - + - + - + - + - + - + - + - + - - + + - + - + - + - + - + - - + + - - + + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - + + diff --git a/docs/design/src/dataflow.drawio b/docs/design/src/dataflow.drawio index 5ac4266..410780c 100644 --- a/docs/design/src/dataflow.drawio +++ b/docs/design/src/dataflow.drawio @@ -1,37 +1,37 @@ - + - + - + - + - + - + - + - + - + @@ -42,19 +42,19 @@ - + - + - + - + @@ -64,43 +64,43 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -110,7 +110,7 @@ - + @@ -120,7 +120,7 @@ - + @@ -130,51 +130,63 @@ - + - + - + - + - + - + - + - + - + - + - + + + + - + + + + + + + + + + diff --git a/docs/source/analyzer/db.rst b/docs/source/analyzer/db.rst new file mode 100644 index 0000000..8d2cffd --- /dev/null +++ b/docs/source/analyzer/db.rst @@ -0,0 +1,10 @@ +DBAnalyzer +=============================== + +``DBAnalyzer`` + +.. automodule:: duetector.analyzer.db + :members: + :undoc-members: + :private-members: + :show-inheritance: diff --git a/docs/source/analyzer/index.rst b/docs/source/analyzer/index.rst new file mode 100644 index 0000000..ae2bcff --- /dev/null +++ b/docs/source/analyzer/index.rst @@ -0,0 +1,25 @@ +Analyzer +========================================= + + +.. autoclass:: duetector.analyzer.base.Analyzer + :members: + :undoc-members: + :show-inheritance: + + +Avaliable Analyzer +----------------------------------------------- + +.. toctree:: + :maxdepth: 2 + + DB Analyzer + +Data Models +----------------------------------------------- + +.. toctree:: + :maxdepth: 2 + + Data Models diff --git a/docs/source/analyzer/models.rst b/docs/source/analyzer/models.rst new file mode 100644 index 0000000..59d394d --- /dev/null +++ b/docs/source/analyzer/models.rst @@ -0,0 +1,7 @@ +Data Models +=============================== + +.. automodule:: duetector.analyzer.models + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/collectors/index.rst b/docs/source/collectors/index.rst index 718d445..b7f4ba6 100644 --- a/docs/source/collectors/index.rst +++ b/docs/source/collectors/index.rst @@ -18,9 +18,19 @@ and store them in a somewhere. :show-inheritance: +Avaliable Collector +--------------------------------------------------------- + .. toctree:: :maxdepth: 2 - :caption: Avaliable Collector and Data Models DB Collectors + + +Data Models +--------------------------------------------------------- + +.. toctree:: + :maxdepth: 2 + Data Models diff --git a/docs/source/collectors/models.rst b/docs/source/collectors/models.rst index 251938e..dbb757c 100644 --- a/docs/source/collectors/models.rst +++ b/docs/source/collectors/models.rst @@ -1,7 +1,7 @@ Models for collectors ==================================== -.. autoclass:: duetector.collectors.models.Tracking +.. automodule:: duetector.collectors.models :members: :undoc-members: :show-inheritance: diff --git a/docs/source/conf.py b/docs/source/conf.py index 587145c..0846cba 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -37,6 +37,7 @@ "sphinx.ext.viewcode", "sphinx.ext.autosectionlabel", "sphinx_click", + "sphinxcontrib.autodoc_pydantic", ] # Add any paths that contain templates here, relative to this directory. diff --git a/docs/source/filters/base.rst b/docs/source/filters/base.rst deleted file mode 100644 index 77a9c34..0000000 --- a/docs/source/filters/base.rst +++ /dev/null @@ -1,10 +0,0 @@ -BaseFilter -================== - -Base class for all filters. - - -.. autoclass:: duetector.filters.base.Filter - :members: - :undoc-members: - :private-members: diff --git a/docs/source/filters/index.rst b/docs/source/filters/index.rst index 5361bac..249463f 100644 --- a/docs/source/filters/index.rst +++ b/docs/source/filters/index.rst @@ -3,9 +3,16 @@ Filter ``Filter`` will filter the data based on the given criteria. +.. autoclass:: duetector.filters.base.Filter + :members: + :undoc-members: + :private-members: + + +Avaliable Filter +------------------------------------------------------ + .. toctree:: :maxdepth: 2 - :caption: Avaliable Filter - Base Filter Pattern Filter diff --git a/docs/source/index.rst b/docs/source/index.rst index 6b1106e..4381a54 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -15,6 +15,7 @@ Reference CLI + Analyzer Monitors Managers @@ -22,11 +23,7 @@ Reference Filters Collectors - Exceptions - - Database utilities - Config utilities - Tools utilities + Utilities Indices and tables diff --git a/docs/source/managers/index.rst b/docs/source/managers/index.rst index 7ae567d..7defda4 100644 --- a/docs/source/managers/index.rst +++ b/docs/source/managers/index.rst @@ -10,9 +10,12 @@ Managers provides a way to get instances of both built-in implementations and ex :inherited-members: :show-inheritance: + +Avaliable Manager +------------------------------------------- + .. toctree:: - :maxdepth: 1 - :caption: Avaliable Manager + :maxdepth: 2 Collector Manager Filter Manager diff --git a/docs/source/monitors/index.rst b/docs/source/monitors/index.rst index f9e51bb..b46da89 100644 --- a/docs/source/monitors/index.rst +++ b/docs/source/monitors/index.rst @@ -14,9 +14,11 @@ entry point for the polling, filtering and collecting of the data. :show-inheritance: +Avaliable Monitor +------------------------------------------- + .. toctree:: - :maxdepth: 1 - :caption: Avaliable Monitor + :maxdepth: 2 Bcc Monitor Shell Monitor diff --git a/docs/source/tracers/index.rst b/docs/source/tracers/index.rst index 36e9245..15e5ad5 100644 --- a/docs/source/tracers/index.rst +++ b/docs/source/tracers/index.rst @@ -1,5 +1,13 @@ Tracer -========================================= +===================================== + +``Tracer`` will be capturing information in some way. +:doc:`Collector ` will convert ``Tracer``'s ``data_t`` to :doc:`Tracking `. + +.. note:: + Some filed of ``data_t`` will be converted to other more readable filed, + if you want to fit this feature, you should refer to :doc:`Tracking.normalize_field `. + .. automodule:: duetector.tracers :members: @@ -7,9 +15,11 @@ Tracer :inherited-members: +Avaliable Tracer +-------------------------------------- + .. toctree:: - :maxdepth: 1 - :caption: Avaliable Tracer + :maxdepth: 2 CloneTracer OpenTracer diff --git a/docs/source/utilities.rst b/docs/source/utilities.rst new file mode 100644 index 0000000..fad7411 --- /dev/null +++ b/docs/source/utilities.rst @@ -0,0 +1,12 @@ +Utilities +=============================== + +.. toctree:: + :maxdepth: 2 + :caption: Utilities Documentation: + + Exceptions + + Database utilities + Config utilities + Tools utilities diff --git a/duetector/analyzer/__init__.py b/duetector/analyzer/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/duetector/analyzer/base.py b/duetector/analyzer/base.py new file mode 100644 index 0000000..2e8416d --- /dev/null +++ b/duetector/analyzer/base.py @@ -0,0 +1,69 @@ +from datetime import datetime +from typing import List, Optional + +from duetector.analyzer.models import AnalyzerBrief, Tracking +from duetector.config import Configuable + + +class Analyzer(Configuable): + """ + A base class for all analyzers. + """ + + default_config = {} + """ + Default config for ``Analyzer``. + """ + + config_scope = "analyzer" + """ + Config scope for this analyzer. + + Subclasses cloud override this. + """ + + def get_all_tracers(self) -> List[str]: + """ + Get all tracers from storage. + + Returns: + List[str]: List of tracer's name. + """ + raise NotImplementedError + + def get_all_collector_ids(self) -> List[str]: + """ + Get all collector id from storage. + + Returns: + List[str]: List of collector id. + """ + raise NotImplementedError + + def query( + self, + tracer: Optional[str] = None, + collector_id: Optional[str] = None, + start_datetime: Optional[datetime] = None, + end_datetime: Optional[datetime] = None, + start: int = 0, + limit: int = 20, + ) -> List[Tracking]: + """ + Query tracking data from storage. + """ + raise NotImplementedError + + def brief( + self, + start_datetime: Optional[datetime] = None, + end_datetime: Optional[datetime] = None, + ) -> AnalyzerBrief: + """ + Get brief of analyzer. + """ + raise NotImplementedError + + def analyze(self): + # TODO: Not design yet. + pass diff --git a/duetector/analyzer/db.py b/duetector/analyzer/db.py new file mode 100644 index 0000000..ebc750c --- /dev/null +++ b/duetector/analyzer/db.py @@ -0,0 +1,283 @@ +from datetime import datetime +from typing import Any, Dict, List, Optional + +from sqlalchemy import func, select + +from duetector.analyzer.base import Analyzer +from duetector.analyzer.models import AnalyzerBrief, Brief, Tracking +from duetector.db import SessionManager + + +class DBAnalyzer(Analyzer): + """ + A analyzer using database. + + As a top model, it will init a ``SessionManager`` and pass it to submodels. + + Config scope is ``db_analyzer``. + + Example: + + .. code-block:: python + + from duetector.analyzer.db import DBAnalyzer + from duetector.analyzer.models import Tracking as AT + from duetector.collectors.models import Tracking as CT + + + collector_id = "db_analyzer_tests_collector" + c_tracking = CT( + tracer="t", + ) + db_analyzer = DBAnalyzer() + m = db_analyzer.sm.get_tracking_model(c_tracking.tracer, collector_id) + with db_analyzer.sm.begin() as session: + session.add(m(**c_tracking.model_dump(exclude=["tracer"]))) + session.commit() + + a_tracking = AT( + tracer=c_tracking.tracer, + ) + assert a_tracking in db_analyzer.query() + assert a_tracking in db_analyzer.query(tracer=a_tracking.tracer) + assert a_tracking in db_analyzer.query(collector_id=collector_id) + assert a_tracking in db_analyzer.query( + tracer=a_tracking.tracer, collector_id=collector_id + ) + assert not db_analyzer.query(tracer="not-exist") + assert not db_analyzer.query(collector_id="not-exist") + + Note: + Currently, it will **NOT** be configured by ``DBcollector``'s config, + as we design it to be a standalone model. + """ + + default_config = { + **Analyzer.default_config, + "db": { + **SessionManager.default_config, + "engine": { + "url": "sqlite:///duetector-dbcollector.sqlite3", + }, + }, + } + """ + Default config for ``DBAnalyzer``. + """ + + config_scope = "db_analyzer" + """ + Config scope for this analyzer. + """ + + def __init__(self, config: Optional[Dict[str, Any]] = None, *args, **kwargs): + super().__init__(config, *args, **kwargs) + # Init as a submodel + self.sm: SessionManager = SessionManager(self.config._config_dict) + + def query( + self, + tracers: Optional[List[str]] = None, + collector_ids: Optional[List[str]] = None, + start_datetime: Optional[datetime] = None, + end_datetime: Optional[datetime] = None, + start: int = 0, + limit: int = 0, + columns: Optional[List[str]] = None, + where: Optional[Dict[str, Any]] = None, + distinct: bool = False, + order_by_asc: Optional[List[str]] = None, + order_by_desc: Optional[List[str]] = None, + ) -> List[Tracking]: + """ + Query all tracking records from database. + + Args: + tracers (Optional[List[str]], optional): Tracer's name. Defaults to None, all tracers will be queried. + collector_ids (Optional[List[str]], optional): Collector id. Defaults to None, all collector id will be queried. + start_datetime (Optional[datetime], optional): Start time. Defaults to None. + end_datetime (Optional[datetime], optional): End time. Defaults to None. + start (int, optional): Start index. Defaults to 0. + limit (int, optional): Limit of records. Defaults to 20. ``0`` means no limit. + columns (Optional[List[str]], optional): Columns to query. Defaults to None, all columns will be queried. + where (Optional[Dict[str, Any]], optional): Where clause. Defaults to None. + distinct (bool, optional): Distinct. Defaults to False. + order_by_asc (Optional[List[str]], optional): Order by asc. Defaults to None. + order_by_desc (Optional[List[str]], optional): Order by desc. Defaults to None. + Returns: + List[duetector.analyzer.models.Tracking]: List of tracking records. + + """ + + tables = self.sm.inspect_all_tables() + if tracers: + tables = [t for t in tables if self.sm.table_name_to_tracer(t) in tracers] + if collector_ids: + tables = [t for t in tables if self.sm.table_name_to_collector_id(t) in collector_ids] + + r = [] + for t in tables: + tracer = self.sm.table_name_to_tracer(t) + collector_id = self.sm.table_name_to_collector_id(t) + m = self.sm.get_tracking_model(tracer, collector_id) + + columns = columns or m.inspect_fields().keys() + statm = select(*[getattr(m, k) for k in columns]).offset(start) + if start_datetime: + statm = statm.where(m.dt >= start_datetime) + if end_datetime: + statm = statm.where(m.dt <= end_datetime) + if limit: + statm = statm.limit(limit) + if where: + statm = statm.where(*[getattr(m, k) == v for k, v in where.items()]) + if distinct: + statm = statm.distinct() + if order_by_asc: + statm = statm.order_by(*[getattr(m, k).asc() for k in order_by_asc]) + if order_by_desc: + statm = statm.order_by(*[getattr(m, k).desc() for k in order_by_desc]) + + with self.sm.begin() as session: + r.extend( + [ + self._convert_row_to_tracking(columns, r, tracer) + for r in session.execute(statm).fetchall() + ] + ) + + return r + + def get_all_tracers(self) -> List[str]: + """ + Get all tracers from database. + + Returns: + List[str]: List of tracer's name. + """ + return self.sm.inspect_all_tracers() + + def get_all_collector_ids(self) -> List[str]: + """ + Get all collector id from database. + + Returns: + List[str]: List of collector id. + """ + return self.sm.inspect_all_collector_ids() + + def _table_brief( + self, + table_name: str, + start_datetime: Optional[datetime] = None, + end_datetime: Optional[datetime] = None, + inspect: bool = True, + distinct: bool = False, + ) -> Brief: + """ + Get a brief of a table. + + Args: + table_name (str): Table's name. + + Returns: + Brief: A brief of this table. + """ + tracer = self.sm.table_name_to_tracer(table_name) + collector_id = self.sm.table_name_to_collector_id(table_name) + + m = self.sm.get_tracking_model(tracer, collector_id) + + if not inspect: + return Brief(tracer=tracer, collector_id=collector_id, fields=m.inspect_fields()) + columns = m.inspect_fields().keys() + statm = select(*[getattr(m, k) for k in columns]) + if distinct: + statm = statm.distinct() + if start_datetime: + statm = statm.where(m.dt >= start_datetime) + if end_datetime: + statm = statm.where(m.dt <= end_datetime) + + start_statm = statm.order_by(m.dt.asc()) + end_statm = statm.order_by(m.dt.desc()) + count_statm = select(func.count()).select_from(statm.subquery()) + with self.sm.begin() as session: + start_tracking = self._convert_row_to_tracking( + columns, session.execute(start_statm).first(), tracer + ) + end_tracking = self._convert_row_to_tracking( + columns, session.execute(end_statm).first(), tracer + ) + + return Brief( + tracer=tracer, + collector_id=collector_id, + start=start_tracking.dt, + end=end_tracking.dt, + count=session.execute(count_statm).scalar(), + fields=m.inspect_fields(), + ) + + def _convert_row_to_tracking(self, columns: List[str], row: Any, tracer: str) -> Tracking: + """ + Convert a row to a tracking record. + + Args: + columns (List[str]): Columns. + row (Any): Row. + tracer (str): Tracer's name. + + Returns: + duetector.analyzer.models.Tracking: A tracking record. + """ + if not row: + return Tracking(tracer=tracer) + + return Tracking(tracer=tracer, **{k: v for k, v in zip(columns, row)}) + + def brief( + self, + tracers: Optional[List[str]] = None, + collector_ids: Optional[List[str]] = None, + start_datetime: Optional[datetime] = None, + end_datetime: Optional[datetime] = None, + with_details: bool = True, + distinct: bool = False, + ) -> AnalyzerBrief: + """ + Get a brief of this analyzer. + + Args: + tracers (Optional[List[str]], optional): + Tracers. Defaults to None, all tracers will be queried. + If a specific tracer is not found, it will be ignored. + collector_ids (Optional[List[str]], optional): + Collector ids. Defaults to None, all collector ids will be queried. + If a specific collector id is not found, it will be ignored. + start_datetime (Optional[datetime], optional): Start time. Defaults to None. + end_datetime (Optional[datetime], optional): End time. Defaults to None. + with_details (bool, optional): With details. Defaults to True. + distinct (bool, optional): Distinct. Defaults to False. + + Returns: + AnalyzerBrief: A brief of this analyzer. + """ + tables = self.sm.inspect_all_tables() + if tracers: + tables = [t for t in tables if self.sm.table_name_to_tracer(t) in tracers] + if collector_ids: + tables = [t for t in tables if self.sm.table_name_to_collector_id(t) in collector_ids] + + briefs = [ + self._table_brief( + t, start_datetime, end_datetime, inspect=with_details, distinct=distinct + ) + for t in tables + ] + + return AnalyzerBrief( + tracers=[brief.tracer for brief in briefs], + collector_ids=[brief.collector_id for brief in briefs], + briefs=briefs, + ) diff --git a/duetector/analyzer/models.py b/duetector/analyzer/models.py new file mode 100644 index 0000000..d55da4a --- /dev/null +++ b/duetector/analyzer/models.py @@ -0,0 +1,86 @@ +from __future__ import annotations + +from datetime import datetime +from typing import Any, Dict, List, Optional + +import pydantic + + +class Tracking(pydantic.BaseModel): + """ + Tracking model for analyzer. + + Currently, this is a copy of ``duetector.collectors.models.Tracking``. + And as an ACL(anti-corruption layer), we will not use ``duetector.collectors.models.Tracking`` directly. + """ + + tracer: str + """ + Tracer's name + """ + + pid: Optional[int] = None + """ + Process ID + """ + uid: Optional[int] = None + """ + User ID + """ + gid: Optional[int] = None + """ + Group ID of user + """ + comm: Optional[str] = "Unknown" + """ + Command name + """ + cwd: Optional[str] = None + """ + Current working directory of process + """ + fname: Optional[str] = None + """ + File name which is being accessed + """ + + dt: Optional[datetime] = None + """ + datetime of event + """ + + extended: Dict[str, Any] = {} + """ + Extended fields, will be stored in ``extended`` field as a dict + """ + + +class Brief(pydantic.BaseModel): + """ + Brief of a tracking set, mostly a table. + """ + + tracer: str + collector_id: str + start: Optional[datetime] = None + end: Optional[datetime] = None + count: Optional[int] = None + fields: Dict[str, Any] = {} + + +class AnalyzerBrief(pydantic.BaseModel): + """ + Brief of analyzer. + """ + + tracers: List[str] + """ + List of tracers + """ + + collector_ids: List[str] + """ + List of collector ids + """ + + briefs: List[Brief] diff --git a/duetector/collectors/__init__.py b/duetector/collectors/__init__.py index 14b1a86..1fb7d9d 100644 --- a/duetector/collectors/__init__.py +++ b/duetector/collectors/__init__.py @@ -1,9 +1,3 @@ from .base import Collector __all__ = ["Collector"] - - -# Expose for plugin system -from . import base, db - -registers = [base, db] diff --git a/duetector/collectors/base.py b/duetector/collectors/base.py index 3242179..7d8ff5f 100644 --- a/duetector/collectors/base.py +++ b/duetector/collectors/base.py @@ -132,8 +132,8 @@ def summary(self) -> Dict: return { tracer: { "count": len(trackings), - "first": trackings[0].timestamp, - "last": trackings[-1].timestamp, + "first": trackings[0].dt, + "last": trackings[-1].dt, "most_recent": trackings[-1].model_dump(), } for tracer, trackings in self._trackings.items() diff --git a/duetector/collectors/db.py b/duetector/collectors/db.py index 846a403..ccc1469 100644 --- a/duetector/collectors/db.py +++ b/duetector/collectors/db.py @@ -1,7 +1,7 @@ import copy from typing import Any, Dict, Optional -from sqlalchemy import select # type: ignore +from sqlalchemy import func, select # type: ignore from duetector.collectors.base import Collector from duetector.collectors.models import Tracking @@ -50,13 +50,13 @@ def summary(self) -> Dict: with self.sm.begin() as session: return { tracer: { - "count": len(session.execute(select(m)).fetchall()), - "first at": session.execute(select(m)).first()[0].timestamp, + "count": session.execute(select(func.count()).select_from(m)).scalar(), + "first at": session.execute(select(m)).first()[0].dt, "last": session.execute(select(m).order_by(m.id.desc())) # type: ignore .first()[0] - .to_tracking(), + .to_collector_tracking(), } - for tracer, m in self.sm.get_all_model().items() + for tracer, m in self.sm.get_all_models().items() } diff --git a/duetector/collectors/models.py b/duetector/collectors/models.py index 169edb9..b4084fe 100644 --- a/duetector/collectors/models.py +++ b/duetector/collectors/models.py @@ -1,9 +1,12 @@ from __future__ import annotations +from datetime import datetime from typing import Any, Dict, NamedTuple, Optional import pydantic +from duetector.utils import get_boot_time_duration_ns + class Tracking(pydantic.BaseModel): """ @@ -43,9 +46,9 @@ class Tracking(pydantic.BaseModel): File name which is being accessed """ - timestamp: Optional[int] = None + dt: Optional[datetime] = None """ - Timestamp of event, ns since boot + datetime of event """ extended: Dict[str, Any] = {} @@ -53,6 +56,16 @@ class Tracking(pydantic.BaseModel): Extended fields, will be stored in ``extended`` field as a dict """ + @classmethod + def normalize_field(cls, field, data): + """ + Normalize field name and data + """ + if field == "timestamp": + field = "dt" + data = get_boot_time_duration_ns(data) + return field, data + @staticmethod def from_namedtuple(tracer, data: NamedTuple) -> Tracking: # type: ignore """ @@ -71,10 +84,11 @@ def from_namedtuple(tracer, data: NamedTuple) -> Tracking: # type: ignore "extended": {}, } for field in data._fields: # type: ignore - if field in Tracking.model_fields: - args[field] = getattr(data, field) + k, v = Tracking.normalize_field(field, getattr(data, field)) + if k in Tracking.model_fields: + args[k] = v else: - args["extended"][field] = getattr(data, field) + args["extended"][k] = v if not args.get("cwd"): # Try get cwd from /proc//cwd @@ -86,3 +100,7 @@ def from_namedtuple(tracer, data: NamedTuple) -> Tracking: # type: ignore pass return Tracking(**args) + + +if __name__ == "__main__": + Tracking(tracer="test", dt=datetime.now()) diff --git a/duetector/collectors/register.py b/duetector/collectors/register.py new file mode 100644 index 0000000..ac07b16 --- /dev/null +++ b/duetector/collectors/register.py @@ -0,0 +1,4 @@ +# Expose for plugin system +from . import base, db + +registers = [base, db] diff --git a/duetector/db.py b/duetector/db.py index 457a698..ebcf18e 100644 --- a/duetector/db.py +++ b/duetector/db.py @@ -1,6 +1,7 @@ from contextlib import contextmanager +from datetime import datetime from threading import Lock -from typing import Any, Dict, Generator, Optional +from typing import Any, Dict, Generator, List, Optional import sqlalchemy # type: ignore from sqlalchemy.orm import ( # type: ignore @@ -12,7 +13,8 @@ ) from sqlalchemy.types import JSON # type: ignore -from duetector.collectors.models import Tracking +from duetector.analyzer.models import Tracking as AT +from duetector.collectors.models import Tracking as CT from duetector.config import Configuable @@ -29,7 +31,7 @@ class TrackingMixin: pid: Mapped[Optional[int]] uid: Mapped[Optional[int]] gid: Mapped[Optional[int]] - timestamp: Mapped[Optional[int]] + dt: Mapped[Optional[datetime]] comm: Mapped[Optional[str]] cwd: Mapped[Optional[str]] @@ -38,7 +40,32 @@ class TrackingMixin: extended: Mapped[Dict[str, Any]] = mapped_column(type_=JSON, default={}) def __repr__(self): - return f"" + return f"" + + +class TrackingInterface: + """ + A interface for tracking. + """ + + def to_collector_tracking(self) -> CT: + """ + Convert to collector's tracking model. + """ + raise NotImplementedError + + def to_analyzer_tracking(self) -> AT: + """ + Convert to analyzer's tracking model. + """ + raise NotImplementedError + + @classmethod + def inspect_fields(cls) -> Dict[str, Any]: + """ + Inspect fields of this model. + """ + raise NotImplementedError class SessionManager(Configuable): @@ -54,9 +81,21 @@ class SessionManager(Configuable): .. code-block:: python from duetector.db import SessionManager + from duetector.collectors.models import Tracking sessionmanager = SessionManager() + t = Tracking( + tracer="t", + ) + m = sessionmanager.get_tracking_model(t.tracer, "id") + with sessionmanager.begin() as session: - session.add(...) + session.add(m(**t.model_dump(exclude=["tracer"]))) + session.commit() + + assert sessionmanager.inspect_all_tables() == [ + sessionmanager.get_table_names("t", "id") + ] + assert sessionmanager.inspect_all_tables("not-exist") == [] """ @@ -149,7 +188,18 @@ def begin(self) -> Generator[Session, None, None]: with self.sessionmaker.begin() as session: yield session - def get_tracking_model(self, tracer: str = "unknown", collector_id: str = "") -> type: + def get_table_names(self, tracer: str = "unknown", collector_id: str = "") -> str: + return f"{self.table_prefix}:{tracer}@{collector_id}" + + def table_name_to_tracer(self, table_name: str) -> str: + return table_name.split(":")[1].split("@")[0] + + def table_name_to_collector_id(self, table_name: str) -> str: + return table_name.split(":")[1].split("@")[1] + + def get_tracking_model( + self, tracer: str = "unknown", collector_id: str = "" + ) -> TrackingInterface: """ Get a sqlalchemy model for tracking, each tracer will create a table in database. @@ -168,22 +218,41 @@ def get_tracking_model(self, tracer: str = "unknown", collector_id: str = "") -> class Base(DeclarativeBase): pass - class TrackingModel(Base, TrackingMixin): - __tablename__ = f"{self.table_prefix}:{tracer}@{collector_id}" + class TrackingModel(Base, TrackingMixin, TrackingInterface): + __tablename__ = self.get_table_names(tracer, collector_id) - def to_tracking(self) -> Tracking: - return Tracking( + def to_collector_tracking(self) -> CT: + return CT( tracer=tracer, pid=self.pid, uid=self.uid, gid=self.gid, - timestamp=self.timestamp, + dt=self.dt, comm=self.comm, cwd=self.cwd, fname=self.fname, extended=self.extended, ) + def to_analyzer_tracking(self) -> AT: + return AT( + tracer=tracer, + pid=self.pid, + uid=self.uid, + gid=self.gid, + dt=self.dt, + comm=self.comm, + cwd=self.cwd, + fname=self.fname, + extended=self.extended, + ) + + @classmethod + def inspect_fields(cls) -> Dict[str, Any]: + return { + c.name: c.type.python_type for c in cls.__table__.columns if c.name != "id" + } + try: self._tracking_models[tracer] = self._init_tracking_model(TrackingModel) except Exception as e: @@ -191,9 +260,31 @@ def to_tracking(self) -> Tracking: raise return self._tracking_models[tracer] - def get_all_model(self) -> Dict[str, type]: + def get_all_models(self) -> Dict[str, type]: return self._tracking_models + def inspect_all_tables( + self, tracer: Optional[str] = None, collector_id: Optional[str] = None + ) -> str: + def _filter(t): + if tracer and self.table_name_to_tracer(t) != tracer: + return False + if collector_id and self.table_name_to_collector_id(t) != collector_id: + return False + return True + + return [ + t + for t in sqlalchemy.inspect(self.engine).get_table_names() + if t.startswith(self.table_prefix) and _filter(t) + ] + + def inspect_all_tracers(self) -> List[str]: + return list(set(self.table_name_to_tracer(t) for t in self.inspect_all_tables())) + + def inspect_all_collector_ids(self) -> List[str]: + return list(set(self.table_name_to_collector_id(t) for t in self.inspect_all_tables())) + def _init_tracking_model(self, tracking_model: type) -> type: if not sqlalchemy.inspect(self.engine).has_table(tracking_model.__tablename__): tracking_model.__table__.create(self.engine) @@ -201,4 +292,17 @@ def _init_tracking_model(self, tracking_model: type) -> type: if __name__ == "__main__": - print(SessionManager()) + from duetector.collectors.models import Tracking + + sessionmanager = SessionManager() + t = Tracking( + tracer="t", + ) + m = sessionmanager.get_tracking_model(t.tracer, "id") + + with sessionmanager.begin() as session: + session.add(m(**t.model_dump(exclude=["tracer"]))) + session.commit() + + assert sessionmanager.inspect_all_tables() == [sessionmanager.get_table_names("t", "id")] + assert sessionmanager.inspect_all_tables("not-exist") == [] diff --git a/duetector/filters/__init__.py b/duetector/filters/__init__.py index 1cc0953..5fb10ae 100644 --- a/duetector/filters/__init__.py +++ b/duetector/filters/__init__.py @@ -1,9 +1,3 @@ from .base import Filter __all__ = ["Filter"] - - -# Expose for plugin system -from . import pattern - -registers = [pattern] diff --git a/duetector/filters/base.py b/duetector/filters/base.py index 7915a7e..fa8d78c 100644 --- a/duetector/filters/base.py +++ b/duetector/filters/base.py @@ -14,22 +14,23 @@ class Filter(Configuable): subclass should override ``filter`` method. User should call Filter() directly to filter data, + Example: - .. code-block:: python + .. code-block:: python - from duetector.filters import Filter - from duetector.collectors.models import Tracking + from duetector.filters import Filter + from duetector.collectors.models import Tracking - class MyFilter(Filter): - def filter(self, data: Tracking) -> Optional[Tracking]: - if data.fname == "/etc/passwd": - return None - return data + class MyFilter(Filter): + def filter(self, data: Tracking) -> Optional[Tracking]: + if data.fname == "/etc/passwd": + return None + return data - f = MyFilter() - f(Tracking(fname="/etc/passwd")) # None - f(Tracking(fname="/etc/shadow")) # Tracking(fname="/etc/shadow") + f = MyFilter() + f(Tracking(fname="/etc/passwd")) # None + f(Tracking(fname="/etc/shadow")) # Tracking(fname="/etc/shadow") """ default_config = { diff --git a/duetector/filters/register.py b/duetector/filters/register.py new file mode 100644 index 0000000..41f34ed --- /dev/null +++ b/duetector/filters/register.py @@ -0,0 +1,4 @@ +# Expose for plugin system +from . import pattern + +registers = [pattern] diff --git a/duetector/managers/collector.py b/duetector/managers/collector.py index 0c40c4d..5ad7bf3 100644 --- a/duetector/managers/collector.py +++ b/duetector/managers/collector.py @@ -3,7 +3,7 @@ import pluggy -import duetector.collectors +import duetector.collectors.register from duetector.collectors.base import Collector from duetector.extension.collector import project_name from duetector.log import logger @@ -42,7 +42,7 @@ def __init__(self, config: Optional[Dict[str, Any]] = None, *args, **kwargs): self.pm = pluggy.PluginManager(PROJECT_NAME) self.pm.add_hookspecs(sys.modules[__name__]) self.pm.load_setuptools_entrypoints(PROJECT_NAME) - self.register(duetector.collectors) + self.register(duetector.collectors.register) def init(self, ignore_disabled=True) -> List[Collector]: """ diff --git a/duetector/managers/filter.py b/duetector/managers/filter.py index 16f541b..d9a8bd3 100644 --- a/duetector/managers/filter.py +++ b/duetector/managers/filter.py @@ -3,7 +3,7 @@ import pluggy -import duetector.filters +import duetector.filters.register from duetector.extension.filter import project_name from duetector.filters.base import Filter from duetector.log import logger @@ -40,7 +40,7 @@ def __init__(self, config: Optional[Dict[str, Any]] = None, *args, **kwargs): self.pm = pluggy.PluginManager(PROJECT_NAME) self.pm.add_hookspecs(sys.modules[__name__]) self.pm.load_setuptools_entrypoints(PROJECT_NAME) - self.register(duetector.filters) + self.register(duetector.filters.register) def init(self, ignore_disabled=True) -> List[Filter]: """ diff --git a/duetector/managers/tracer.py b/duetector/managers/tracer.py index e7ce10b..0afedf6 100644 --- a/duetector/managers/tracer.py +++ b/duetector/managers/tracer.py @@ -3,7 +3,7 @@ import pluggy -import duetector.tracers +import duetector.tracers.register from duetector.extension.tracer import project_name from duetector.log import logger from duetector.managers import Manager @@ -40,7 +40,7 @@ def __init__(self, config: Optional[Dict[str, Any]] = None, *args, **kwargs): self.pm = pluggy.PluginManager(PROJECT_NAME) self.pm.add_hookspecs(sys.modules[__name__]) self.pm.load_setuptools_entrypoints(PROJECT_NAME) - self.register(duetector.tracers) + self.register(duetector.tracers.register) def init(self, tracer_type=Tracer, ignore_disabled=True) -> List[Tracer]: """ diff --git a/duetector/monitors/sh_monitor.py b/duetector/monitors/sh_monitor.py index b999992..b9044c1 100644 --- a/duetector/monitors/sh_monitor.py +++ b/duetector/monitors/sh_monitor.py @@ -84,7 +84,7 @@ class ShMonitor(Monitor): default_config = { **Monitor.default_config, "auto_init": True, - "timeout": 30, + "timeout": 5, } """ Default config for this monitor. diff --git a/duetector/static/config.toml b/duetector/static/config.toml index 540bd46..c5ea18d 100644 --- a/duetector/static/config.toml +++ b/duetector/static/config.toml @@ -91,10 +91,16 @@ interval_ms = 500 [monitor.sh] disabled = false auto_init = true -timeout = 30 +timeout = 5 [monitor.sh.backend_args] max_workers = 10 [monitor.sh.poller] interval_ms = 500 + +[db_analyzer.db] +table_prefix = "duetector_tracking" + +[db_analyzer.db.engine] +url = "sqlite:///duetector-dbcollector.sqlite3" diff --git a/duetector/tools/config_generator.py b/duetector/tools/config_generator.py index 92e8416..7b6703d 100644 --- a/duetector/tools/config_generator.py +++ b/duetector/tools/config_generator.py @@ -4,6 +4,7 @@ import tomli_w +from duetector.analyzer.db import DBAnalyzer from duetector.config import ConfigLoader from duetector.log import logger from duetector.managers import CollectorManager, FilterManager, TracerManager @@ -56,6 +57,11 @@ class ConfigGenerator: All monitors to inspect. """ + analyzer = [DBAnalyzer] + """ + All analyzers to inspect. + """ + def __init__( self, load: bool = True, @@ -79,6 +85,8 @@ def __init__( for m in self.monitors: _recursive_load(m.config_scope, self.dynamic_config, m.default_config) + for a in self.analyzer: + _recursive_load(a.config_scope, self.dynamic_config, a.default_config) # This will generate default config file if not exists if load: self.loaded_config = ConfigLoader(path, load_env, dump_when_load=False).load_config() diff --git a/duetector/tracers/__init__.py b/duetector/tracers/__init__.py index 6a3ee06..5d8875f 100644 --- a/duetector/tracers/__init__.py +++ b/duetector/tracers/__init__.py @@ -1,8 +1,3 @@ from .base import BccTracer, ShellTracer, Tracer __all__ = ["Tracer", "BccTracer", "ShellTracer"] - -# Expose for plugin system -from . import clone, openat2, tcpconnect, uname - -registers = [openat2, uname, tcpconnect, clone] diff --git a/duetector/tracers/register.py b/duetector/tracers/register.py new file mode 100644 index 0000000..df08838 --- /dev/null +++ b/duetector/tracers/register.py @@ -0,0 +1,4 @@ +# Expose for plugin system +from . import clone, openat2, tcpconnect, uname + +registers = [openat2, uname, tcpconnect, clone] diff --git a/duetector/tracers/tcpconnect.py b/duetector/tracers/tcpconnect.py index c199e16..69e9915 100644 --- a/duetector/tracers/tcpconnect.py +++ b/duetector/tracers/tcpconnect.py @@ -27,7 +27,10 @@ class TcpconnectTracer(BccTracer): def poll_args(self): return {"timeout": int(self.config.poll_timeout)} - data_t = namedtuple("TcpTracking", ["pid", "uid", "gid", "comm", "saddr", "daddr", "dport"]) + data_t = namedtuple( + "TcpTracking", + ["pid", "uid", "gid", "comm", "saddr", "daddr", "dport", "timestamp"], + ) prog = """ #include @@ -45,6 +48,8 @@ def poll_args(self): u32 pid; u32 uid; u32 gid; + + u64 timestamp; char comm[TASK_COMM_LEN]; }; int do_trace(struct pt_regs *ctx, struct sock *sk) @@ -88,6 +93,7 @@ def poll_args(self): event.pid = pid; event.uid = bpf_get_current_uid_gid(); event.gid = bpf_get_current_uid_gid() >> 32; + event.timestamp = bpf_ktime_get_ns(); bpf_get_current_comm(&event.comm, sizeof(event.comm)); // output buffer.ringbuf_output(&event, sizeof(event), 0); @@ -102,7 +108,8 @@ def poll_args(self): def _convert_data(self, data) -> NamedTuple: data = super()._convert_data(data) return data._replace( - saddr=inet_ntoa(data.saddr).decode("utf-8"), daddr=inet_ntoa(data.daddr).decode("utf-8") + saddr=inet_ntoa(data.saddr).decode("utf-8"), + daddr=inet_ntoa(data.daddr).decode("utf-8"), ) # type: ignore def set_callback(self, host, callback: Callable[[NamedTuple], None]): diff --git a/duetector/utils.py b/duetector/utils.py index 8c18017..c67bfa2 100644 --- a/duetector/utils.py +++ b/duetector/utils.py @@ -1,3 +1,11 @@ +from datetime import datetime, timedelta + +try: + from functools import cache +except ImportError: + from functools import lru_cache as cache + + class Singleton(type): _instances = {} @@ -15,3 +23,22 @@ def inet_ntoa(addr) -> bytes: dq = dq + b"." addr = addr >> 8 return dq + + +@cache +def get_boot_time() -> datetime: + with open("/proc/stat", "r") as f: + for line in f: + if line.startswith("btime"): + return datetime.fromtimestamp(int(line.split()[1])) + raise RuntimeError("Could not find btime in /proc/stat") + + +def get_boot_time_duration_ns(ns) -> datetime: + ns = int(ns) + return get_boot_time() + timedelta(microseconds=ns / 1000) + + +if __name__ == "__main__": + print(get_boot_time()) + print(get_boot_time_duration_ns("13205215231927")) diff --git a/pyproject.toml b/pyproject.toml index 54edec7..135da34 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,7 +27,7 @@ classifiers = [ ] [project.optional-dependencies] test = ["pytest", "pytest-cov", "pytype"] -docs = ["Sphinx<=7.2.4", "sphinx-rtd-theme", "sphinx-click"] +docs = ["Sphinx<=7.2.4", "sphinx-rtd-theme", "sphinx-click", "autodoc_pydantic"] [project.scripts] duectl = "duetector.cli.main:cli" diff --git a/tests/config.toml b/tests/config.toml index 5a42e6a..7b3dc63 100644 --- a/tests/config.toml +++ b/tests/config.toml @@ -20,7 +20,7 @@ re_exclude_fname = [ "/run", "/usr/lib", "/etc/ld.so.cache", - "/re/*" + "/re/*", ] re_exclude_comm = [] exclude_pid = [] @@ -61,8 +61,13 @@ maxlen = 1024 disabled = false auto_init = true - [monitor.sh] disabled = false auto_init = true timeout = 5 + +[db_analyzer.db] +table_prefix = "duetector_tracking" + +[db_analyzer.db.engine] +url = "sqlite:///:memory:" diff --git a/tests/test_bcc_monitor.py b/tests/test_bcc_monitor.py index e7c2922..81d779f 100644 --- a/tests/test_bcc_monitor.py +++ b/tests/test_bcc_monitor.py @@ -4,11 +4,13 @@ import pytest from duetector.collectors.models import Tracking -from duetector.config import Configuable from duetector.managers import CollectorManager, FilterManager, TracerManager from duetector.monitors.bcc_monitor import BccMonitor, Monitor from duetector.tracers.base import BccTracer, Tracer -from duetector.tracers.dummy import DummyBPF, DummyTracer +from duetector.utils import get_boot_time_duration_ns + +timestamp = 13205215231927 +datetime = get_boot_time_duration_ns(timestamp) class MockTracer(Tracer): @@ -24,7 +26,7 @@ def get_dummy_data(cls): gid=9999, comm="dummy", fname="dummy.file", - timestamp=13205215231927, + timestamp=timestamp, custom="dummy-xargs", ) @@ -118,7 +120,7 @@ def test_bcc_monitor(bcc_monitor: MockMonitor): comm="dummy", cwd=None, fname="dummy.file", - timestamp=13205215231927, + dt=datetime, extended={"custom": "dummy-xargs"}, ) diff --git a/tests/test_collector.py b/tests/test_collector.py index 9b0424f..0006a36 100644 --- a/tests/test_collector.py +++ b/tests/test_collector.py @@ -5,6 +5,10 @@ from duetector.collectors.db import DBCollector from duetector.collectors.models import Tracking from duetector.managers import CollectorManager +from duetector.utils import get_boot_time_duration_ns + +timestamp = 13205215231927 +datetime = get_boot_time_duration_ns(timestamp) @pytest.fixture @@ -27,7 +31,7 @@ def data_t(): gid=9999, comm="dummy", fname="dummy.file", - timestamp=13205215231927, + timestamp=timestamp, custom="dummy-xargs", ) @@ -38,7 +42,7 @@ def test_dbcollector(dbcollector: DBCollector, data_t): assert dbcollector.summary() == { "dummy": { "count": 1, - "first at": 13205215231927, + "first at": datetime, "last": Tracking( tracer="dummy", pid=9999, @@ -47,7 +51,7 @@ def test_dbcollector(dbcollector: DBCollector, data_t): comm="dummy", cwd=None, fname="dummy.file", - timestamp=13205215231927, + dt=datetime, extended={"custom": "dummy-xargs"}, ), } diff --git a/tests/test_db_analyzer.py b/tests/test_db_analyzer.py new file mode 100644 index 0000000..1595687 --- /dev/null +++ b/tests/test_db_analyzer.py @@ -0,0 +1,109 @@ +from datetime import datetime, timedelta + +import pytest + +from duetector.analyzer.db import DBAnalyzer +from duetector.analyzer.models import Tracking as AT +from duetector.collectors.models import Tracking as CT + +now = datetime.now() + +tracking_kwargs = dict( + tracer="db_analyzer_tests", + pid=9999, + uid=9999, + gid=9999, + comm="dummy", + cwd=None, + fname="dummy.file", + dt=datetime.now(), + extended={"custom": "dummy-xargs"}, +) + + +@pytest.fixture +def tracer_name(): + return "db_analyzer_tests" + + +@pytest.fixture +def collector_id(): + return "db_analyzer_tests_collector" + + +@pytest.fixture +def c_tracking(): + yield CT(**tracking_kwargs) + + +@pytest.fixture +def a_tracking(): + yield AT(**tracking_kwargs) + + +@pytest.fixture +def db_analyzer(full_config, c_tracking, collector_id): + db_analyzer = DBAnalyzer(full_config) + sessionmanager = db_analyzer.sm + + m = sessionmanager.get_tracking_model(c_tracking.tracer, collector_id) + + with sessionmanager.begin() as session: + session.add(m(**c_tracking.model_dump(exclude=["tracer"]))) + session.add(m(**c_tracking.model_dump(exclude=["tracer"]))) + session.commit() + + assert sessionmanager.inspect_all_tables() == [ + sessionmanager.get_table_names(c_tracking.tracer, collector_id) + ] + assert sessionmanager.inspect_all_tables("not-exist") == [] + yield db_analyzer + + +def test_query(db_analyzer: DBAnalyzer, a_tracking, collector_id): + assert a_tracking in db_analyzer.query() + assert a_tracking in db_analyzer.query(tracers=[a_tracking.tracer]) + assert a_tracking in db_analyzer.query(collector_ids=[collector_id]) + assert a_tracking in db_analyzer.query( + tracers=[a_tracking.tracer], collector_ids=[collector_id] + ) + assert a_tracking in db_analyzer.query(start_datetime=now - timedelta(days=1)) + assert a_tracking in db_analyzer.query(end_datetime=now + timedelta(days=1)) + assert a_tracking in db_analyzer.query(order_by_asc=["pid"]) + assert a_tracking in db_analyzer.query(order_by_desc=["pid"]) + + assert len(db_analyzer.query()) == 2 + assert len(db_analyzer.query(distinct=True)) == 1 + + assert AT( + tracer=a_tracking.tracer, + pid=a_tracking.pid, + fname=a_tracking.fname, + ) in db_analyzer.query(columns=["pid", "fname"]) + + assert not db_analyzer.query(tracers=["not-exist"]) + assert not db_analyzer.query(collector_ids=["not-exist"]) + assert not db_analyzer.query(start_datetime=now + timedelta(days=1)) + assert not db_analyzer.query(end_datetime=now - timedelta(days=1)) + assert not db_analyzer.query(start=100) + assert not db_analyzer.query(where={"pid": 1}) + + +def test_brief(db_analyzer: DBAnalyzer, a_tracking, collector_id): + assert db_analyzer.brief() + assert db_analyzer.brief(tracers=[a_tracking.tracer]) + assert db_analyzer.brief(collector_ids=[collector_id]) + assert db_analyzer.brief(tracers=[a_tracking.tracer], collector_ids=[collector_id]) + assert db_analyzer.brief(start_datetime=now - timedelta(days=1)) + assert db_analyzer.brief(end_datetime=now + timedelta(days=1)) + assert db_analyzer.brief(with_details=False) + assert db_analyzer.brief(distinct=True) + + assert not db_analyzer.brief(tracers=["not-exist"]).tracers + assert not db_analyzer.brief(collector_ids=["not-exist"]).collector_ids + assert not db_analyzer.brief(start_datetime=now + timedelta(days=1)).briefs[0].count + assert not db_analyzer.brief(end_datetime=now - timedelta(days=1)).briefs[0].count + + +if __name__ == "__main__": + pytest.main(["-vv", "-s", __file__])