diff --git a/.github/workflows/security.yml b/.github/workflows/security.yml new file mode 100644 index 00000000..01e48e58 --- /dev/null +++ b/.github/workflows/security.yml @@ -0,0 +1,50 @@ +name: Wasp (Semgrep) - SAST Check + +on: + pull_request_target: + branches: + - main + + schedule: + - cron: '0 */24 * * *' + workflow_dispatch: + +jobs: + wasp-scan: + name: Wasp scan + runs-on: + group: security-lrg + steps: + - name: Setting permission + run: sudo chown runner:runner -R .* + + - name: Repository checkout + uses: actions/checkout@v4 + + - name: Running Wasp scan + uses: freshactions/wasp@latest + env: + WASP_LOG_LEVEL: DEBUG + WASP_SAVE_JSON: true + WASP_SAVE_HTML: true + WASP_SAVE_CSV: true + WASP_FRESHRELEASE_PR_PROJECT_KEY: ${{ vars.SECURITY_APPSEC_FRESHRELEASE_PROJECT_KEY }} + WASP_DRY_RUN: ${{ vars.SECURITY_APPSEC_WASP_DRY_RUN }} + + WASP_FRESHRELEASE_URL: ${{ vars.SECURITY_APPSEC_FRESHRELEASE_URL }} + WASP_FRESHRELEASE_PR_ISSUE_TYPE: ${{ vars.SECURITY_APPSEC_FRESHRELEASE_PR_ISSUE_TYPE }} + + WASP_TOKEN: ${{ secrets.SECURITY_APPSEC_WASP_TOKEN }} + WASP_FRESHRELEASE_TOKEN: ${{ secrets.SECURITY_APPSEC_FRESHRELEASE_TOKEN }} + WASP_SLACK_TOKEN: ${{ secrets.SECURITY_APPSEC_SLACK_TOKEN }} + GITHUB_TOKEN: ${{ secrets.SECURITY_APPSEC_GH_TOKEN }} + + - uses: actions/upload-artifact@v4 + if: always() + with: + name: Wasp scan report archive + retention-days: ${{ vars.SECURITY_APPSEC_WASP_RESULT_RETENTION_DAYS }} + path: | + wasp-report.csv + wasp-report.json + wasp-report.html diff --git a/.gitignore b/.gitignore index e02cc354..05f5ba7f 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,10 @@ __pycache__/ # C extensions *.so +# Pycharn +.idea +models/ + # Distribution / packaging .Python build/ diff --git a/libs/infinity_emb/Dockerfile b/libs/infinity_emb/Dockerfile index a324e022..2c3c6763 100644 --- a/libs/infinity_emb/Dockerfile +++ b/libs/infinity_emb/Dockerfile @@ -1,5 +1,9 @@ +ARG BASE_IMAGE=nvidia/cuda:12.1.0-base-ubuntu22.04 # Use the Python base image -FROM nvidia/cuda:12.1.1-base-ubuntu22.04 AS base +FROM $BASE_IMAGE AS base + +# Define a build-time argument with a default value +ARG DTYPE=auto ENV PYTHONUNBUFFERED=1 \ \ @@ -14,21 +18,29 @@ ENV PYTHONUNBUFFERED=1 \ # do not ask any interactive question POETRY_NO_INTERACTION=1 \ EXTRAS="all" \ - PYTHON="python3.11" -RUN apt-get update && apt-get install build-essential python3-dev $PYTHON-venv $PYTHON curl -y + PYTHON="python3.11" \ + DTYPE=$DTYPE + +RUN apt-get update && apt-get install build-essential python3-dev $PYTHON-venv $PYTHON curl -y + +# Set the working directory for the app WORKDIR /app FROM base as builder -# Set the working directory for the app + # Define the version of Poetry to install (default is 1.7.1) # Define the directory to install Poetry to (default is /opt/poetry) ARG POETRY_VERSION=1.7.1 ARG POETRY_HOME=/opt/poetry + # Create a Python virtual environment for Poetry and install it RUN curl -sSL https://install.python-poetry.org | POETRY_HOME=$POETRY_HOME POETRY_VERSION=$POETRY_VERSION $PYTHON - + ENV PATH=$POETRY_HOME/bin:$PATH + # Test if Poetry is installed in the expected path RUN echo "Poetry version:" && poetry --version + # Copy the rest of the app source code (this layer will be invalidated and rebuilt whenever the source code changes) COPY poetry.lock poetry.toml pyproject.toml README.md /app/ # Install dependencies only @@ -39,75 +51,14 @@ RUN poetry install --no-interaction --no-ansi --extras "${EXTRAS}" --without li # remove cache RUN poetry cache clear pypi --all -FROM builder as testing -# install lint and test dependencies -RUN poetry install --no-interaction --no-ansi --extras "${EXTRAS}" -# lint -RUN poetry run ruff . -RUN poetry run black --check . -RUN poetry run mypy . -# pytest -COPY tests tests -# run end to end tests because of duration of build in github ci. -# Run tests/end_to_end on TARGETPLATFORM x86_64 otherwise run tests/end_to_end_gpu -# poetry run python -m pytest tests/end_to_end -x -RUN if [ "$TARGETPLATFORM" = "linux/amd64" ] ; then \ -poetry run python -m pytest tests/end_to_end -x ; \ -else \ -poetry run python -m pytest tests/end_to_end/test_api_with_dummymodel.py -x ; \ -fi -RUN echo "all tests passed" > "test_results.txt" - +# Use a multi-stage build -> production version +FROM base AS production -# Use a multi-stage build -> production version, with download -FROM base AS tested-builder COPY --from=builder /app /app -# force testing stage to run -COPY --from=testing /app/test_results.txt /app/test_results.txt -ENV HF_HOME=/app/.cache/huggingface -ENV PATH=/app/.venv/bin:$PATH -# do nothing -RUN echo "copied all files" - +COPY /models /models +COPY environment_config.sh ./environment_config.sh -# Export with tensorrt, not recommended. -# docker buildx build --target=production-tensorrt -f Dockerfile . -FROM nvidia/cuda:11.8.0-cudnn8-devel-ubuntu22.04 AS production-tensorrt -ENV PYTHONUNBUFFERED=1 \ - PIP_NO_CACHE_DIR=off \ - PYTHON="python3.11" -RUN apt-get update && apt-get install python3-dev python3-pip $PYTHON build-essential curl -y -COPY --from=builder /app /app -# force testing stage to run -COPY --from=testing /app/test_results.txt /app/test_results.txt -ENV HF_HOME=/app/.cache/torch +ENV SENTENCE_TRANSFORMERS_HOME=/app/.cache/torch ENV PATH=/app/.venv/bin:$PATH -RUN pip install --no-cache-dir "onnxruntime-gpu==1.17.0" "tensorrt==8.6.*" -ENV LD_LIBRARY_PATH /app/.venv/lib/$(PYTHON)/site-packages/tensorrt:/usr/lib/x86_64-linux-gnu:/app/.venv/lib/$(PYTHON)/site-packages/tensorrt_libs:${LD_LIBRARY_PATH} -ENV PATH /app/.venv/lib/$(PYTHON)/site-packages/tensorrt/bin:${PATH} -ENTRYPOINT ["infinity_emb"] - - -# Use a multi-stage build -> production version, with download -# docker buildx build --target=production-with-download \ -# --build-arg MODEL_NAME=BAAI/bge-small-en-v1.5 --build-arg ENGINE=torch -f Dockerfile -t infinity-BAAI-small . -FROM tested-builder AS production-with-download -# collect model name and engine from build args -ARG MODEL_NAME -RUN if [ -z "${MODEL_NAME}" ]; then echo "Error: Build argument MODEL_NAME not set." && exit 1; fi -ARG ENGINE -RUN if [ -z "${ENGINE}" ]; then echo "Error: Build argument ENGINE not set." && exit 1; fi -ARG EXTRA_PACKAGES -RUN if [ -n "${EXTRA_PACKAGES}" ]; then python -m pip install --no-cache-dir ${EXTRA_PACKAGES} ; fi -# will exit with 3 if model is downloaded # TODO: better exit code -RUN infinity_emb v2 --model-id $MODEL_NAME --engine $ENGINE --preload-only || [ $? -eq 3 ] -ENTRYPOINT ["infinity_emb"] -# flash attention fa2 -FROM tested-builder AS production-with-fa2 -RUN python -m pip install https://github.com/Dao-AILab/flash-attention/releases/download/v2.6.1/flash_attn-2.6.1+cu123torch2.3cxx11abiFalse-cp310-cp310-linux_x86_64.whl -ENTRYPOINT ["infinity_emb"] - -# Use a multi-stage build -> production version -FROM tested-builder AS production -ENTRYPOINT ["infinity_emb"] +ENTRYPOINT ["/bin/bash" , "-c", "source ./environment_config.sh $DTYPE"] diff --git a/libs/infinity_emb/environment_config.sh b/libs/infinity_emb/environment_config.sh new file mode 100644 index 00000000..b1061025 --- /dev/null +++ b/libs/infinity_emb/environment_config.sh @@ -0,0 +1,5 @@ +#!/bin/bash +#export OTEL_TRACES_SAMPLER=parentbased_always_off +export OTEL_RESOURCE_ATTRIBUTES=service.name=${SHERLOCK_SERVICE_NAME},host.name=${POD_NAME},host.ip=${POD_IP} +export OTEL_EXPORTER_OTLP_ENDPOINT=http://${HOST_IP}:5680 +infinity_emb v2 --model-id /models --dtype ${DTYPE} diff --git a/libs/infinity_emb/infinity_emb/infinity_server.py b/libs/infinity_emb/infinity_emb/infinity_server.py index 0bcc55c6..2107a574 100644 --- a/libs/infinity_emb/infinity_emb/infinity_server.py +++ b/libs/infinity_emb/infinity_emb/infinity_server.py @@ -25,7 +25,11 @@ RerankInput, ReRankResult, ) -from infinity_emb.log_handler import UVICORN_LOG_LEVELS, logger +from infinity_emb.log_handler import ( + UVICORN_LOG_LEVELS, + logger, + StructuredLoggingMiddleware, +) from infinity_emb.primitives import ( Device, Dtype, @@ -129,6 +133,7 @@ async def validate_token( instrumentator = Instrumentator().instrument(app) app.add_exception_handler(errors.OpenAIException, errors.openai_exception_handler) + app.add_middleware(StructuredLoggingMiddleware) @app.get("/health", operation_id="health", response_class=responses.ORJSONResponse) async def _health() -> dict[str, float]: @@ -220,13 +225,13 @@ async def _embeddings(data: OpenAIEmbeddingInput): if isinstance(data.input, str): data.input = [data.input] - logger.debug("[📝] Received request with %s inputs ", len(data.input)) + logger.info("[📝] Received request with %s inputs ", len(data.input)) start = time.perf_counter() embedding, usage = await engine.embed(sentences=data.input) duration = (time.perf_counter() - start) * 1000 - logger.debug("[✅] Done in %s ms", duration) + logger.info("[✅] Done in %s ms", duration) return OpenAIEmbeddingResult.to_embeddings_response( embeddings=embedding, diff --git a/libs/infinity_emb/infinity_emb/log_handler.py b/libs/infinity_emb/infinity_emb/log_handler.py index 2eca5d34..1c2fbe10 100644 --- a/libs/infinity_emb/infinity_emb/log_handler.py +++ b/libs/infinity_emb/infinity_emb/log_handler.py @@ -1,21 +1,282 @@ # SPDX-License-Identifier: MIT # Copyright (c) 2023-now michaelfeil +import json import logging -import sys +import re +from contextvars import ContextVar from enum import Enum -from typing import Any +from functools import wraps +from time import perf_counter, time +from traceback import format_exception +from uuid import uuid4 -logging.getLogger().handlers.clear() +from fastapi import Request, HTTPException +from opentelemetry.instrumentation.logging import LoggingInstrumentor +from starlette.middleware.base import BaseHTTPMiddleware +from starlette.responses import StreamingResponse -handlers: list[Any] = [] -try: - from rich.console import Console - from rich.logging import RichHandler - handlers.append(RichHandler(console=Console(stderr=True), show_time=False)) -except ImportError: - handlers.append(logging.StreamHandler(sys.stderr)) +def sla_breached(duration): + return duration > 3000 + + +class StructuredLogging: + request_time = ContextVar("request_time", default=perf_counter()) + account_id = ContextVar("account_id", default="null") + request_id = ContextVar("request_id", default="null") + trace_parent = ContextVar("trace_parent", default="null") + controller = ContextVar("controller", default=["null"]) # needed for implementation + + _config_dict = { + "ts": "%(timestamp_ms)d", + "type": "app", + "svc": "freddy-infinity", + "lvl": "%(levelname)s", + "act": "%(controller)s", + "a_id": "%(account_id)s", + "r_id": "%(request_id)s", + "p": "freddy-fs", + "tp": "%(trace_parent)s", + "td": "%(thread)s", + "trace_id": "%(otelTraceID)s", + "span_id": "%(otelSpanID)s", + "msg": "%(message)s", + } + + _dumped_dict = json.dumps(_config_dict) + + _number_in_quotes, _group_without_quotes = r"(\")(%\([^\)]+\)[df])(\")", r"\g<2>" + _config_string_static = re.sub( + _number_in_quotes, _group_without_quotes, _dumped_dict + ) + + config_string = _config_string_static[:-1] + "%(extra_data)s}" + + @classmethod + def modify_logging(cls): + cls._change_factory() + cls._format_extra_data() + LoggingInstrumentor().instrument(logging_format=StructuredLogging.config_string) + + @classmethod + def _change_factory(cls): + def sanitize_string(data) -> str: + return json.dumps(str(data), ensure_ascii=False)[1:-1] + + original_factory = logging.getLogRecordFactory() + + @wraps(original_factory) + def record_factory(*args, **kwargs): + record = original_factory(*args, **kwargs) + record.timestamp_ms = int(time() * 1000) + record.account_id = sanitize_string(cls.account_id.get()) + record.request_id = sanitize_string(cls.request_id.get()) + record.controller = sanitize_string( + cls.controller.get()[0] + ) # first element of list + record.trace_parent = sanitize_string(cls.trace_parent.get()) + record.time_elapsed = perf_counter() - cls.request_time.get() + record.msg = sanitize_string(record.msg) + if record.args: + record.args = tuple( + sanitize_string(arg) if isinstance(arg, str) else arg + for arg in record.args + ) + return record + + logging.setLogRecordFactory(record_factory) + + @staticmethod + def _format_extra_data(): + """changes the logging.Logger.makeRecord method""" + + def nice_format(data): + if type(data) in (int, str, float): + return data + try: + return json.dumps(data, ensure_ascii=False) + except Exception: + return str(data) + + original_make_record = logging.Logger.makeRecord + + @wraps(original_make_record) + def new_make_record(*args, **kwargs): + record = original_make_record(*args, **kwargs) + # original_make_record.__code__.co_varnames[:original_make_record.__code__.co_argcount].index('extra') + extra_index = 9 # obtained from function signature, see above comment. + extra = args[extra_index] + + # correctly format the extra fields + if not extra: + extra = dict() + elif not isinstance(extra, dict): + extra = {"extra_data": nice_format(extra)} + else: + extra = {str(k): nice_format(v) for k, v in extra.items()} + + # add the ex field if needed + if record.exc_info: + extra["ex"] = ( + "".join(format_exception(*record.exc_info)) + if isinstance(record.exc_info, tuple) + else str(record.exc_info) + ) + + # add it to extra_data if not empty + record.extra_data = ( + (", " + json.dumps(extra, ensure_ascii=False)[1:-1]) if extra else "" + ) + return record + + logging.Logger.makeRecord = new_make_record + + +class ControllerFieldHelper: + """Helper class to log act (controller) field""" + + # Note - ContextVar contains list with single element which is mutated + # This helps persist the object beyond its 'official' context, allowing simpler implementation + _request_log_fn = ContextVar( + "_request_log_fn", default=[lambda: logging.critical("REQUEST LOG FAILURE")] + ) + + @classmethod + def modify_fastapi_run_endpoint_function(cls): + """to log act, we override fastapi's routing.run_endpoint_function""" + from fastapi import routing + + old_run_endpoint_function = routing.run_endpoint_function + + async def new_run_endpoint_function(**kwargs): + StructuredLogging.controller.get()[0] = kwargs[ + "dependant" # codespell:ignore + ].call.__name__ + cls._request_log_fn.get()[0]() + cls._request_log_fn.get()[0] = lambda: None + return await old_run_endpoint_function(**kwargs) + + routing.run_endpoint_function = new_run_endpoint_function + + @classmethod + def log_when_controller_found(cls, fn): + StructuredLogging.controller.set(["-"]) + cls._request_log_fn.set([fn]) + + @classmethod + def log_if_controller_not_found(cls): + cls._request_log_fn.get()[0]() + + +# https://stackoverflow.com/questions/71525132/how-to-write-a-custom-fastapi-middleware-class Pure ASGI middleware +# https://www.starlette.io/middleware/#limitations is not used because we don't need the features that it provides +class StructuredLoggingMiddleware(BaseHTTPMiddleware): + def __init__(self, app): + super().__init__(app) + + @staticmethod + async def list_to_async_iterator(body_list): + for part in body_list: + yield part + + @staticmethod + def log_response_info(request, response, content, passed, duration): + if sla_breached(duration): + passed = 0 + response_info = { + "st": response.status_code, + "hdr": json.dumps( + {k.decode(): v.decode() for k, v in response.raw_headers} + ), + "path": request.url.path, + "lg": "http", + "pass": passed, + "body": (b"".join(content)).decode(), + "d": duration, + "m": request.method, + } + if response.status_code >= 400: + logging.error("RESPONSE", extra=response_info) + else: + logging.info("RESPONSE", extra=response_info) + + # for every successful non-health request + if not any(map(request.url.path.__contains__, ("health", "metrics"))): + logging.info( + None, + extra={ + "st": response.status_code, + "path": request.url.path, + "lg": "delight", + "pass": passed, + "d": duration, + "m": request.method, + }, + ) + + async def dispatch(self, request: Request, call_next): + StructuredLogging.request_time.set(perf_counter()) + request_id = request.headers.get( + "request-id", request.headers.get("x-request-id", str(uuid4())) + ) + account_id = request.headers.get("account-id", "null") + trace_parent = request.headers.get("traceparent", "-") + + StructuredLogging.request_id.set(request_id) + StructuredLogging.account_id.set(account_id) + StructuredLogging.trace_parent.set(trace_parent) + + processed_headers = json.dumps(dict(request.headers.items())) + body = await request.body() + + ControllerFieldHelper.log_when_controller_found( + lambda: logging.info( + "REQUEST", + extra={ + "m": request.method, + "path": request.url.path, + "hdr": processed_headers, + "body": body, + "lg": "http", + }, + ), + ) + passed = 1 + content = [] + try: + response = await call_next(request) + ControllerFieldHelper.log_if_controller_not_found() + # GETTING THE ASYNC RESPONSE BODY NEEDS WORKAROUNDS + content = [gen async for gen in response.body_iterator] + response.body_iterator = self.list_to_async_iterator(content) + except HTTPException as http_exc: + ControllerFieldHelper.log_if_controller_not_found() + logging.error("HTTPException occurred", exc_info=http_exc) + error_msg_422 = f"422 Unprocessable Entity: {http_exc.detail}".encode() + response = StreamingResponse(iter((error_msg_422,)), status_code=422) + passed = 0 + except RuntimeError as runtime_exc: + ControllerFieldHelper.log_if_controller_not_found() + logging.critical("RuntimeError occurred", exc_info=runtime_exc) + error_msg_500 = f"500 Internal Server Error: {runtime_exc}".encode() + response = StreamingResponse(iter((error_msg_500,)), status_code=500) + passed = 0 + except Exception as e: + ControllerFieldHelper.log_if_controller_not_found() + logging.critical("Failure occurred in endpoint", exc_info=e) + error_msg_5xx = b"500 internal server error due to: " + str(e).encode() + response = StreamingResponse(iter((error_msg_5xx,)), status_code=500) + passed = 0 + + duration = int(1000 * (perf_counter() - StructuredLogging.request_time.get())) + self.log_response_info(request, response, content, passed, duration) + + return response + + +ControllerFieldHelper.modify_fastapi_run_endpoint_function() +StructuredLogging.modify_logging() LOG_LEVELS: dict[str, int] = { "critical": logging.CRITICAL, @@ -26,14 +287,12 @@ "trace": 5, } -FORMAT = "%(asctime)s %(name)s %(levelname)s: %(message)s" logging.basicConfig( level="INFO", - format=FORMAT, - handlers=handlers, + format=StructuredLogging.config_string, ) -logger = logging.getLogger("infinity_emb") +logger = logging.getLogger() class UVICORN_LOG_LEVELS(Enum): diff --git a/libs/infinity_emb/poetry.lock b/libs/infinity_emb/poetry.lock index 91b81463..ec025fb6 100644 --- a/libs/infinity_emb/poetry.lock +++ b/libs/infinity_emb/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.8.2 and should not be changed by hand. [[package]] name = "aiohttp" @@ -698,6 +698,23 @@ tests = ["Pillow (>=6.2.1)", "absl-py", "apache-beam (>=2.26.0,<2.44.0)", "elast torch = ["torch"] vision = ["Pillow (>=6.2.1)"] +[[package]] +name = "deprecated" +version = "1.2.14" +description = "Python @deprecated decorator to deprecate old python classes, functions or methods." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "Deprecated-1.2.14-py2.py3-none-any.whl", hash = "sha256:6fac8b097794a90302bdbb17b9b815e732d3c4720583ff1b198499d78470466c"}, + {file = "Deprecated-1.2.14.tar.gz", hash = "sha256:e5323eb936458dccc2582dc6f9c322c852a775a27065ff2b0c4970b9d53d01b3"}, +] + +[package.dependencies] +wrapt = ">=1.10,<2" + +[package.extras] +dev = ["PyTest", "PyTest-Cov", "bump2version (<1)", "sphinx (<2)", "tox"] + [[package]] name = "dill" version = "0.3.7" @@ -2111,6 +2128,83 @@ packaging = "*" protobuf = "*" sympy = "*" +[[package]] +name = "opentelemetry-api" +version = "1.27.0" +description = "OpenTelemetry Python API" +optional = false +python-versions = ">=3.8" +files = [ + {file = "opentelemetry_api-1.27.0-py3-none-any.whl", hash = "sha256:953d5871815e7c30c81b56d910c707588000fff7a3ca1c73e6531911d53065e7"}, + {file = "opentelemetry_api-1.27.0.tar.gz", hash = "sha256:ed673583eaa5f81b5ce5e86ef7cdaf622f88ef65f0b9aab40b843dcae5bef342"}, +] + +[package.dependencies] +deprecated = ">=1.2.6" +importlib-metadata = ">=6.0,<=8.4.0" + +[[package]] +name = "opentelemetry-instrumentation" +version = "0.48b0" +description = "Instrumentation Tools & Auto Instrumentation for OpenTelemetry Python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "opentelemetry_instrumentation-0.48b0-py3-none-any.whl", hash = "sha256:a69750dc4ba6a5c3eb67986a337185a25b739966d80479befe37b546fc870b44"}, + {file = "opentelemetry_instrumentation-0.48b0.tar.gz", hash = "sha256:94929685d906380743a71c3970f76b5f07476eea1834abd5dd9d17abfe23cc35"}, +] + +[package.dependencies] +opentelemetry-api = ">=1.4,<2.0" +setuptools = ">=16.0" +wrapt = ">=1.0.0,<2.0.0" + +[[package]] +name = "opentelemetry-instrumentation-logging" +version = "0.48b0" +description = "OpenTelemetry Logging instrumentation" +optional = false +python-versions = ">=3.8" +files = [ + {file = "opentelemetry_instrumentation_logging-0.48b0-py3-none-any.whl", hash = "sha256:75e5357d9b8c12071a19e1fef664dc1f430ef45874445c324ba4439a00972dc0"}, + {file = "opentelemetry_instrumentation_logging-0.48b0.tar.gz", hash = "sha256:529eb13eedf57d6b2f94e20e996271db2957b817b9457fe4796365d6d4238dec"}, +] + +[package.dependencies] +opentelemetry-api = ">=1.12,<2.0" +opentelemetry-instrumentation = "0.48b0" + +[[package]] +name = "opentelemetry-sdk" +version = "1.27.0" +description = "OpenTelemetry Python SDK" +optional = false +python-versions = ">=3.8" +files = [ + {file = "opentelemetry_sdk-1.27.0-py3-none-any.whl", hash = "sha256:365f5e32f920faf0fd9e14fdfd92c086e317eaa5f860edba9cdc17a380d9197d"}, + {file = "opentelemetry_sdk-1.27.0.tar.gz", hash = "sha256:d525017dea0ccce9ba4e0245100ec46ecdc043f2d7b8315d56b19aff0904fa6f"}, +] + +[package.dependencies] +opentelemetry-api = "1.27.0" +opentelemetry-semantic-conventions = "0.48b0" +typing-extensions = ">=3.7.4" + +[[package]] +name = "opentelemetry-semantic-conventions" +version = "0.48b0" +description = "OpenTelemetry Semantic Conventions" +optional = false +python-versions = ">=3.8" +files = [ + {file = "opentelemetry_semantic_conventions-0.48b0-py3-none-any.whl", hash = "sha256:a0de9f45c413a8669788a38569c7e0a11ce6ce97861a628cca785deecdc32a1f"}, + {file = "opentelemetry_semantic_conventions-0.48b0.tar.gz", hash = "sha256:12d74983783b6878162208be57c9effcb89dc88691c64992d70bb89dc00daa1a"}, +] + +[package.dependencies] +deprecated = ">=1.2.6" +opentelemetry-api = "1.27.0" + [[package]] name = "optimum" version = "1.21.2" @@ -3366,7 +3460,7 @@ files = [ name = "setuptools" version = "71.1.0" description = "Easily download, build, install, upgrade, and uninstall Python packages" -optional = true +optional = false python-versions = ">=3.8" files = [ {file = "setuptools-71.1.0-py3-none-any.whl", hash = "sha256:33874fdc59b3188304b2e7c80d9029097ea31627180896fb549c578ceb8a0855"}, @@ -4333,6 +4427,85 @@ files = [ {file = "websockets-12.0.tar.gz", hash = "sha256:81df9cbcbb6c260de1e007e58c011bfebe2dafc8435107b0537f393dd38c8b1b"}, ] +[[package]] +name = "wrapt" +version = "1.16.0" +description = "Module for decorators, wrappers and monkey patching." +optional = false +python-versions = ">=3.6" +files = [ + {file = "wrapt-1.16.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ffa565331890b90056c01db69c0fe634a776f8019c143a5ae265f9c6bc4bd6d4"}, + {file = "wrapt-1.16.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e4fdb9275308292e880dcbeb12546df7f3e0f96c6b41197e0cf37d2826359020"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bb2dee3874a500de01c93d5c71415fcaef1d858370d405824783e7a8ef5db440"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2a88e6010048489cda82b1326889ec075a8c856c2e6a256072b28eaee3ccf487"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ac83a914ebaf589b69f7d0a1277602ff494e21f4c2f743313414378f8f50a4cf"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:73aa7d98215d39b8455f103de64391cb79dfcad601701a3aa0dddacf74911d72"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:807cc8543a477ab7422f1120a217054f958a66ef7314f76dd9e77d3f02cdccd0"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:bf5703fdeb350e36885f2875d853ce13172ae281c56e509f4e6eca049bdfb136"}, + {file = "wrapt-1.16.0-cp310-cp310-win32.whl", hash = "sha256:f6b2d0c6703c988d334f297aa5df18c45e97b0af3679bb75059e0e0bd8b1069d"}, + {file = "wrapt-1.16.0-cp310-cp310-win_amd64.whl", hash = "sha256:decbfa2f618fa8ed81c95ee18a387ff973143c656ef800c9f24fb7e9c16054e2"}, + {file = "wrapt-1.16.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1a5db485fe2de4403f13fafdc231b0dbae5eca4359232d2efc79025527375b09"}, + {file = "wrapt-1.16.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:75ea7d0ee2a15733684badb16de6794894ed9c55aa5e9903260922f0482e687d"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a452f9ca3e3267cd4d0fcf2edd0d035b1934ac2bd7e0e57ac91ad6b95c0c6389"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:43aa59eadec7890d9958748db829df269f0368521ba6dc68cc172d5d03ed8060"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:72554a23c78a8e7aa02abbd699d129eead8b147a23c56e08d08dfc29cfdddca1"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:d2efee35b4b0a347e0d99d28e884dfd82797852d62fcd7ebdeee26f3ceb72cf3"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:6dcfcffe73710be01d90cae08c3e548d90932d37b39ef83969ae135d36ef3956"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:eb6e651000a19c96f452c85132811d25e9264d836951022d6e81df2fff38337d"}, + {file = "wrapt-1.16.0-cp311-cp311-win32.whl", hash = "sha256:66027d667efe95cc4fa945af59f92c5a02c6f5bb6012bff9e60542c74c75c362"}, + {file = "wrapt-1.16.0-cp311-cp311-win_amd64.whl", hash = "sha256:aefbc4cb0a54f91af643660a0a150ce2c090d3652cf4052a5397fb2de549cd89"}, + {file = "wrapt-1.16.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:5eb404d89131ec9b4f748fa5cfb5346802e5ee8836f57d516576e61f304f3b7b"}, + {file = "wrapt-1.16.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9090c9e676d5236a6948330e83cb89969f433b1943a558968f659ead07cb3b36"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:94265b00870aa407bd0cbcfd536f17ecde43b94fb8d228560a1e9d3041462d73"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f2058f813d4f2b5e3a9eb2eb3faf8f1d99b81c3e51aeda4b168406443e8ba809"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:98b5e1f498a8ca1858a1cdbffb023bfd954da4e3fa2c0cb5853d40014557248b"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:14d7dc606219cdd7405133c713f2c218d4252f2a469003f8c46bb92d5d095d81"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:49aac49dc4782cb04f58986e81ea0b4768e4ff197b57324dcbd7699c5dfb40b9"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:418abb18146475c310d7a6dc71143d6f7adec5b004ac9ce08dc7a34e2babdc5c"}, + {file = "wrapt-1.16.0-cp312-cp312-win32.whl", hash = "sha256:685f568fa5e627e93f3b52fda002c7ed2fa1800b50ce51f6ed1d572d8ab3e7fc"}, + {file = "wrapt-1.16.0-cp312-cp312-win_amd64.whl", hash = "sha256:dcdba5c86e368442528f7060039eda390cc4091bfd1dca41e8046af7c910dda8"}, + {file = "wrapt-1.16.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:d462f28826f4657968ae51d2181a074dfe03c200d6131690b7d65d55b0f360f8"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a33a747400b94b6d6b8a165e4480264a64a78c8a4c734b62136062e9a248dd39"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b3646eefa23daeba62643a58aac816945cadc0afaf21800a1421eeba5f6cfb9c"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ebf019be5c09d400cf7b024aa52b1f3aeebeff51550d007e92c3c1c4afc2a40"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:0d2691979e93d06a95a26257adb7bfd0c93818e89b1406f5a28f36e0d8c1e1fc"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:1acd723ee2a8826f3d53910255643e33673e1d11db84ce5880675954183ec47e"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:bc57efac2da352a51cc4658878a68d2b1b67dbe9d33c36cb826ca449d80a8465"}, + {file = "wrapt-1.16.0-cp36-cp36m-win32.whl", hash = "sha256:da4813f751142436b075ed7aa012a8778aa43a99f7b36afe9b742d3ed8bdc95e"}, + {file = "wrapt-1.16.0-cp36-cp36m-win_amd64.whl", hash = "sha256:6f6eac2360f2d543cc875a0e5efd413b6cbd483cb3ad7ebf888884a6e0d2e966"}, + {file = "wrapt-1.16.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:a0ea261ce52b5952bf669684a251a66df239ec6d441ccb59ec7afa882265d593"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7bd2d7ff69a2cac767fbf7a2b206add2e9a210e57947dd7ce03e25d03d2de292"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9159485323798c8dc530a224bd3ffcf76659319ccc7bbd52e01e73bd0241a0c5"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a86373cf37cd7764f2201b76496aba58a52e76dedfaa698ef9e9688bfd9e41cf"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:73870c364c11f03ed072dda68ff7aea6d2a3a5c3fe250d917a429c7432e15228"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:b935ae30c6e7400022b50f8d359c03ed233d45b725cfdd299462f41ee5ffba6f"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:db98ad84a55eb09b3c32a96c576476777e87c520a34e2519d3e59c44710c002c"}, + {file = "wrapt-1.16.0-cp37-cp37m-win32.whl", hash = "sha256:9153ed35fc5e4fa3b2fe97bddaa7cbec0ed22412b85bcdaf54aeba92ea37428c"}, + {file = "wrapt-1.16.0-cp37-cp37m-win_amd64.whl", hash = "sha256:66dfbaa7cfa3eb707bbfcd46dab2bc6207b005cbc9caa2199bcbc81d95071a00"}, + {file = "wrapt-1.16.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1dd50a2696ff89f57bd8847647a1c363b687d3d796dc30d4dd4a9d1689a706f0"}, + {file = "wrapt-1.16.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:44a2754372e32ab315734c6c73b24351d06e77ffff6ae27d2ecf14cf3d229202"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8e9723528b9f787dc59168369e42ae1c3b0d3fadb2f1a71de14531d321ee05b0"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dbed418ba5c3dce92619656802cc5355cb679e58d0d89b50f116e4a9d5a9603e"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:941988b89b4fd6b41c3f0bfb20e92bd23746579736b7343283297c4c8cbae68f"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:6a42cd0cfa8ffc1915aef79cb4284f6383d8a3e9dcca70c445dcfdd639d51267"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:1ca9b6085e4f866bd584fb135a041bfc32cab916e69f714a7d1d397f8c4891ca"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:d5e49454f19ef621089e204f862388d29e6e8d8b162efce05208913dde5b9ad6"}, + {file = "wrapt-1.16.0-cp38-cp38-win32.whl", hash = "sha256:c31f72b1b6624c9d863fc095da460802f43a7c6868c5dda140f51da24fd47d7b"}, + {file = "wrapt-1.16.0-cp38-cp38-win_amd64.whl", hash = "sha256:490b0ee15c1a55be9c1bd8609b8cecd60e325f0575fc98f50058eae366e01f41"}, + {file = "wrapt-1.16.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9b201ae332c3637a42f02d1045e1d0cccfdc41f1f2f801dafbaa7e9b4797bfc2"}, + {file = "wrapt-1.16.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:2076fad65c6736184e77d7d4729b63a6d1ae0b70da4868adeec40989858eb3fb"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c5cd603b575ebceca7da5a3a251e69561bec509e0b46e4993e1cac402b7247b8"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b47cfad9e9bbbed2339081f4e346c93ecd7ab504299403320bf85f7f85c7d46c"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f8212564d49c50eb4565e502814f694e240c55551a5f1bc841d4fcaabb0a9b8a"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:5f15814a33e42b04e3de432e573aa557f9f0f56458745c2074952f564c50e664"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:db2e408d983b0e61e238cf579c09ef7020560441906ca990fe8412153e3b291f"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:edfad1d29c73f9b863ebe7082ae9321374ccb10879eeabc84ba3b69f2579d537"}, + {file = "wrapt-1.16.0-cp39-cp39-win32.whl", hash = "sha256:ed867c42c268f876097248e05b6117a65bcd1e63b779e916fe2e33cd6fd0d3c3"}, + {file = "wrapt-1.16.0-cp39-cp39-win_amd64.whl", hash = "sha256:eb1b046be06b0fce7249f1d025cd359b4b80fc1c3e24ad9eca33e0dcdb2e4a35"}, + {file = "wrapt-1.16.0-py3-none-any.whl", hash = "sha256:6906c4100a8fcbf2fa735f6059214bb13b97f75b1a61777fcf6432121ef12ef1"}, + {file = "wrapt-1.16.0.tar.gz", hash = "sha256:5f370f952971e7d17c7d1ead40e49f32345a7f7a5373571ef44d800d06b1899d"}, +] + [[package]] name = "xxhash" version = "3.4.1" @@ -4584,4 +4757,4 @@ vision = ["pillow", "timm"] [metadata] lock-version = "2.0" python-versions = ">=3.9,<4" -content-hash = "e4148a6463ba6adc8b66d7ad522b0d638f58dc3282d09e17d76bc56a2dc17595" +content-hash = "89bd1b8ad521d3ff58b6d951cf680657fedca56eac9fa64c599ca494235ebd35" diff --git a/libs/infinity_emb/pyproject.toml b/libs/infinity_emb/pyproject.toml index aaa5876a..41560362 100644 --- a/libs/infinity_emb/pyproject.toml +++ b/libs/infinity_emb/pyproject.toml @@ -40,6 +40,8 @@ diskcache = {version = "*", optional=true} # gpu onnxruntime-gpu = {version = "*", optional=true} tensorrt = {version = "^8.6.1", optional=true} +opentelemetry-instrumentation-logging = "^0.48b0" +opentelemetry-sdk = "^1.27.0" [tool.poetry.scripts] infinity_emb = "infinity_emb.infinity_server:cli"