Support jaeger analyzer (#99)

* Add some log for tracking data

* Add jaeger proto bindings

* Add pyi files for type hint

* Demo for jaeger client

* Init otel collector only once

* Switch to otel build proto

* Switch proto

* Fix import issue

* Fixing typo

* Drop auth of image build and only build amd64

* Building test for jaeger

* Trying 127.0.0.1 in github action

* Replacing all localhost to 127.0.0.1

* Add timeout and use http response for checking

* Support grpc secure channel

* Support self-signed creds

* Extract OTelInspector for all Otel code

* Extract OTelInspector

* Improve service api

* Switch to async function

* Add jaeger analyzer docs

* Add docker build test for arm64

* Support query trace

* Fix import issue on 3.11

* Support brief api

* Add testing for all

* Add some comments.

* Fix not_support_params

.github/workflows/docker-image.yml

@@ -21,12 +21,6 @@ jobs:
     -
       name: Set up Docker Buildx
       uses: docker/setup-buildx-action@v2
-    -
-      name: Login to DockerHub
-      uses: docker/login-action@v2
-      with:
-        username: ${{ secrets.DOCKERHUB_USERNAME }}
-        password: ${{ secrets.DOCKERHUB_TOKEN }}
     -
       name: Build image
       id: docker_build_test

.github/workflows/python-package.yml

@@ -27,7 +27,10 @@ jobs:
         python -m pip install -e .[test]
     - name: Test with pytest
       run: |
-        pytest -vv --cov=duetector
+        pytest -vv --cov-append --cov=duetector --cov-report=term-missing tests/standalone
+    - name: Intergration Test for jaeger
+      run: |
+        pytest -vv --cov-append --cov=duetector --cov-report=term-missing tests/integration/jaeger/
     - name: Install dependencies for building
       run: |
         pip install build twine hatch

CONTRIBUTING.md

@@ -25,17 +25,27 @@ Comment style is [Google Python Style Guide](https://google.github.io/styleguide
 pip install -e .[test]
 ```
 
-## Unittest
+## Unit-test
 
 We use pytest to write unit tests, and use pytest-cov to generate coverage reports
 
 ```bash
-pytest -v # Run unit-test
+pytest -v tests/standalone/ # Run unit-test
 pytest --cov=duetector # Generate coverage reports
 ```
 
 Run unit-test before PR, **ensure that new features are covered by unit tests**
 
+## Intergration-test
+
+In order not to be affected by the global variables of unit tests, integration tests need to be performed separately.
+
+Requirement: docker
+
+```
+pytest -vv tests/integration/jaeger
+```
+
 ## Generating config
 
 Use script to generate config after add tracer/filter...

dev-tools/start-jaeger.sh

@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+
+docker run --rm --name jaeger \
+  -e COLLECTOR_ZIPKIN_HOST_PORT=:9411 \
+  -p 6831:6831/udp \
+  -p 6832:6832/udp \
+  -p 5778:5778 \
+  -p 16685:16685 \
+  -p 16686:16686 \
+  -p 4317:4317 \
+  -p 4318:4318 \
+  -p 14250:14250 \
+  -p 14268:14268 \
+  -p 14269:14269 \
+  -p 9411:9411 \
+  jaegertracing/all-in-one:1.50

docs/source/analyzer/index.rst

@@ -15,6 +15,7 @@ Avaliable Analyzer
    :maxdepth: 2
 
    DB Analyzer <db>
+   Jaeger Analyzer <jaeger>
 
 Data Models
 -----------------------------------------------

docs/source/analyzer/jaeger.rst

@@ -0,0 +1,10 @@
+JaegerAnalyzer
+===============================
+
+``JaegerAnalyzer``
+
+.. automodule:: duetector.analyzer.jaeger.analyzer
+   :members:
+   :undoc-members:
+   :private-members:
+   :show-inheritance:

duetector/analyzer/base.py

@@ -51,30 +51,34 @@ def get_all_collector_ids(self) -> List[str]:
         """
         raise NotImplementedError
 
-    def query(
+    async def query(
         self,
         tracers: Optional[List[str]] = None,
         collector_ids: Optional[List[str]] = None,
         start_datetime: Optional[datetime] = None,
         end_datetime: Optional[datetime] = None,
         start: int = 0,
-        limit: int = 0,
+        limit: int = 20,
         columns: Optional[List[str]] = None,
         where: Optional[Dict[str, Any]] = None,
         distinct: bool = False,
         order_by_asc: Optional[List[str]] = None,
         order_by_desc: Optional[List[str]] = None,
     ) -> List[Tracking]:
         """
-        Query all tracking records from database.
+        Query all tracking records from backend.
+
+        Note:
+            Some storage implementations do not guarantee the correct implementation of all parameters.
+            Some parameters may be ignored.
 
         Args:
             tracers (Optional[List[str]], optional): Tracer's name. Defaults to None, all tracers will be queried.
             collector_ids (Optional[List[str]], optional): Collector id. Defaults to None, all collector id will be queried.
             start_datetime (Optional[datetime], optional): Start time. Defaults to None.
             end_datetime (Optional[datetime], optional): End time. Defaults to None.
             start (int, optional): Start index. Defaults to 0.
-            limit (int, optional): Limit of records. Defaults to 20. ``0`` means no limit.
+            limit (int, optional): Limit of records, depends on backend implementations. Defaults to 20. ``0`` means no limit.
             columns (Optional[List[str]], optional): Columns to query. Defaults to None, all columns will be queried.
             where (Optional[Dict[str, Any]], optional): Where clause. Defaults to None.
             distinct (bool, optional): Distinct. Defaults to False.
@@ -85,7 +89,7 @@ def query(
         """
         raise NotImplementedError
 
-    def brief(
+    async def brief(
         self,
         tracers: Optional[List[str]] = None,
         collector_ids: Optional[List[str]] = None,
@@ -116,6 +120,6 @@ def brief(
         """
         raise NotImplementedError
 
-    def analyze(self):
+    async def analyze(self):
         # TODO: Not design yet.
         pass

duetector/analyzer/db.py

@@ -74,14 +74,14 @@ def __init__(self, config: Optional[Dict[str, Any]] = None, *args, **kwargs):
         # Init as a submodel
         self.sm: SessionManager = SessionManager(self.config._config_dict)
 
-    def query(
+    async def query(
         self,
         tracers: Optional[List[str]] = None,
         collector_ids: Optional[List[str]] = None,
         start_datetime: Optional[datetime] = None,
         end_datetime: Optional[datetime] = None,
         start: int = 0,
-        limit: int = 0,
+        limit: int = 20,
         columns: Optional[List[str]] = None,
         where: Optional[Dict[str, Any]] = None,
         distinct: bool = False,
@@ -243,7 +243,7 @@ def _convert_row_to_tracking(self, columns: List[str], row: Any, tracer: str) ->
 
         return Tracking(tracer=tracer, **{k: v for k, v in zip(columns, row)})
 
-    def brief(
+    async def brief(
         self,
         tracers: Optional[List[str]] = None,
         collector_ids: Optional[List[str]] = None,