From fd2829f3b6bad27b19d54268deea048fc26edf3a Mon Sep 17 00:00:00 2001 From: Andrew Weiland Date: Wed, 18 Dec 2024 14:13:15 -0500 Subject: [PATCH 1/2] Datadog tweaks (#1681) * Fixed metadata endpoint * More tweaks * Linter is happy --- .github/workflows/pr-open.yml | 5 +++-- .github/workflows/run_build_deploy.yaml | 4 +++- compose/fastapi/Dockerfile | 6 ++++++ compose/fastapi/start-datadog | 6 ++++-- copilot/mindlogger-backend/manifest.yml | 10 +++++----- docker-compose.yaml | 2 +- src/infrastructure/app.py | 17 ----------------- src/infrastructure/datadog.py | 19 ++++++++----------- src/infrastructure/http/execeptions.py | 2 -- src/infrastructure/logger.py | 2 +- src/main.py | 22 ++++++++++++++++++---- 11 files changed, 49 insertions(+), 46 deletions(-) diff --git a/.github/workflows/pr-open.yml b/.github/workflows/pr-open.yml index 26ae6c6dde6..7943fa6b570 100644 --- a/.github/workflows/pr-open.yml +++ b/.github/workflows/pr-open.yml @@ -49,13 +49,13 @@ jobs: permissions: issues: write pull-requests: write - discussions: write + statuses: write needs: [ run-unit-tests ] steps: - name: Post the link to the report uses: guibranco/github-status-action-v2@v1 with: - authToken: ${{secrets.PAT_TOKEN}} + authToken: ${{secrets.GITHUB_TOKEN}} context: 'Unit Test report' state: ${{ needs.run-unit-tests.result }} sha: ${{ github.event.pull_request.head.sha }} @@ -141,6 +141,7 @@ jobs: permissions: issues: write pull-requests: write + statuses: write needs: [run-e2e-tests] steps: - name: "Send Slack message on failure" diff --git a/.github/workflows/run_build_deploy.yaml b/.github/workflows/run_build_deploy.yaml index bc48a18cfec..b2922195a7f 100644 --- a/.github/workflows/run_build_deploy.yaml +++ b/.github/workflows/run_build_deploy.yaml @@ -98,13 +98,15 @@ jobs: echo "IMAGE_NAME=${ECR_REPO}:${SHORT_SHA}" >> "$GITHUB_OUTPUT" - name: Build and push - uses: docker/build-push-action@v5 + uses: docker/build-push-action@v6 with: context: . file: ./compose/fastapi/Dockerfile push: true build-args: | PIPENV_EXTRA_ARGS=--dev + DD_GIT_REPOSITORY_URL=${{ github.repositoryUrl }} + DD_GIT_COMMIT_SHA=${{ github.sha }} cache-to: mode=max,image-manifest=true,oci-mediatypes=true,type=registry,ref=${{ env.ECR_REPO }}:cache cache-from: type=registry,ref=${{ env.ECR_REPO }}:cache tags: | diff --git a/compose/fastapi/Dockerfile b/compose/fastapi/Dockerfile index b87032c7669..b6aaa539c0a 100644 --- a/compose/fastapi/Dockerfile +++ b/compose/fastapi/Dockerfile @@ -14,6 +14,12 @@ RUN apt-get -y update && apt-get -y upgrade \ ARG USERNAME=code ARG USER_UID=1000 ARG USER_GID=$USER_UID + +ARG DD_GIT_REPOSITORY_URL +ARG DD_GIT_COMMIT_SHA +ENV DD_GIT_REPOSITORY_URL=${DD_GIT_REPOSITORY_URL} +ENV DD_GIT_COMMIT_SHA=${DD_GIT_COMMIT_SHA} + RUN groupadd --gid $USER_GID $USERNAME \ && useradd --uid $USER_UID --gid $USER_GID -m $USERNAME -s /bin/bash diff --git a/compose/fastapi/start-datadog b/compose/fastapi/start-datadog index 9123960cf59..34044687be2 100644 --- a/compose/fastapi/start-datadog +++ b/compose/fastapi/start-datadog @@ -1,5 +1,9 @@ #!/bin/bash +# Fargate does not have the metadata service, this will fail there but that's ok +TOKEN=$(curl -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 21600") && \ + export DD_AGENT_HOST=$(curl http://169.254.169.254/latest/meta-data/local-ipv4 -H "X-aws-ec2-metadata-token: $TOKEN") + set -o errexit set -o pipefail set -o nounset @@ -8,8 +12,6 @@ set -o nounset export UVICORN_HOST="0.0.0.0" export UVICORN_PORT=80 -TOKEN=$(curl -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 21600") -export DD_AGENT_HOST=$(curl http://169.254.169.254/latest/meta-data/local-ipv4 -H "X-aws-ec2-metadata-token: $TOKEN") export LOG_JSON_FORMAT=true export DD_TRACE_ENABLED=true diff --git a/copilot/mindlogger-backend/manifest.yml b/copilot/mindlogger-backend/manifest.yml index ca4e006aeb8..a9e6feb8ef0 100644 --- a/copilot/mindlogger-backend/manifest.yml +++ b/copilot/mindlogger-backend/manifest.yml @@ -73,7 +73,6 @@ storage: variables: # Python configurations # ENV: testing - # PostgreSQL configurations DATABASE__HOST: cmiml-feature-db.cjg844g2kgxp.us-east-1.rds.amazonaws.com DATABASE__PORT: 5432 @@ -181,9 +180,9 @@ logging: Name: "datadog" Host: "http-intake.logs.datadoghq.com" dd_service: "backend-api" - dd_source: "backend" + dd_source: "feature" dd_message_key: "log" - dd_tags: "project:fluentbit" + dd_tags: "version:${COPILOT_ENVIRONMENT_NAME},env:feature" TLS: "on" provider: "ecs" configFilePath: "/fluent-bit/configs/parse-json.conf" @@ -198,8 +197,9 @@ sidecars: image: public.ecr.aws/datadog/agent:7 variables: ECS_FARGATE: true - DD_SITE: us5.datadoghq.com - DD_ENV: ${COPILOT_ENVIRONMENT_NAME} + DD_SITE: datadoghq.com + DD_ENV: feature + DD_VERSION: ${COPILOT_ENVIRONMENT_NAME} DD_LOGS_ENABLED: true DD_LOGS_CONFIG_CONTAINER_COLLECT_ALL: true DD_CONTAINER_EXCLUDE: name:datadog-agent diff --git a/docker-compose.yaml b/docker-compose.yaml index d0567b558c9..41b80843da8 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -44,7 +44,7 @@ services: args: - PIPENV_EXTRA_ARGS=--dev entrypoint: /fastapi-entrypoint - command: /fastapi-start-datadog + command: /fastapi-start env_file: .env depends_on: - postgres diff --git a/src/infrastructure/app.py b/src/infrastructure/app.py index 786b11e3e08..500123262cd 100644 --- a/src/infrastructure/app.py +++ b/src/infrastructure/app.py @@ -112,21 +112,4 @@ def create_app(): # https://github.com/Tufin/oasdiff/issues/52 app.openapi_version = "3.0.3" - # UGLY HACK - # Datadog's `TraceMiddleware` is applied as the very first middleware - # in the list, by patching `FastAPI` constructor. - # Unfortunately that means that it is the innermost middleware, so the trace/span are - # created last in the middleware - # chain. Because we want to add the trace_id/span_id in the access log, - # we need to extract it from the middleware list, - # put it back as the outermost middleware, and rebuild the middleware stack. - # tracing_middleware = next( - # (m for m in app.user_middleware if m.cls == TraceMiddleware), None - # ) - # if tracing_middleware is not None: - # app.user_middleware = [m for m in app.user_middleware if m.cls != TraceMiddleware] - # - # app.user_middleware.insert(0, tracing_middleware) - # app.middleware_stack = app.build_middleware_stack() - return app diff --git a/src/infrastructure/datadog.py b/src/infrastructure/datadog.py index bc0bd93dab9..4c7abb35b8b 100644 --- a/src/infrastructure/datadog.py +++ b/src/infrastructure/datadog.py @@ -1,5 +1,4 @@ import logging -import sys import time import structlog @@ -8,8 +7,6 @@ from fastapi import Request, Response from starlette.middleware.base import BaseHTTPMiddleware from structlog.types import EventDict, Processor -from uvicorn.protocols.utils import get_path_with_query_string - # Much of this is borrowed from: https://gist.github.com/Brymes/cd8f9f138e12845417a246822f64ca26 @@ -63,7 +60,7 @@ def setup_structured_logging(json_logs: bool = False, log_level: str = "INFO"): structlog.stdlib.PositionalArgumentsFormatter(), structlog.stdlib.ExtraAdder(), drop_color_message_key, - tracer_injection, + # tracer_injection, # Not needed, patched in timestamper, structlog.processors.dict_tracebacks, structlog.processors.StackInfoRenderer(), @@ -79,10 +76,10 @@ def setup_structured_logging(json_logs: bool = False, log_level: str = "INFO"): structlog.configure( processors=shared_processors - + [ - # Prepare event dict for `ProcessorFormatter`. - structlog.stdlib.ProcessorFormatter.wrap_for_formatter, - ], + + [ + # Prepare event dict for `ProcessorFormatter`. + structlog.stdlib.ProcessorFormatter.wrap_for_formatter, + ], logger_factory=structlog.stdlib.LoggerFactory(), cache_logger_on_first_use=True, ) @@ -164,9 +161,9 @@ async def dispatch(self, request: Request, call_next) -> Response: access_logger = structlog.stdlib.get_logger("api.access") process_time = time.perf_counter_ns() - start_time status_code = response.status_code - url = get_path_with_query_string(request.scope) - client_host = request.client.host - client_port = request.client.port + url = request.url + client_host = request.client.host if request.client else None + client_port = request.client.port if request.client else None real_host = request.headers.get("X-Forwarded-For", client_host) http_method = request.method http_version = request.scope["http_version"] diff --git a/src/infrastructure/http/execeptions.py b/src/infrastructure/http/execeptions.py index ef84f9bcd1d..822bb58c737 100644 --- a/src/infrastructure/http/execeptions.py +++ b/src/infrastructure/http/execeptions.py @@ -27,8 +27,6 @@ def custom_base_errors_handler(_: Request, error: BaseError) -> JSONResponse: ] ) - - return JSONResponse( response.dict(by_alias=True), status_code=error.status_code, diff --git a/src/infrastructure/logger.py b/src/infrastructure/logger.py index 1d64bf60987..237466c8117 100644 --- a/src/infrastructure/logger.py +++ b/src/infrastructure/logger.py @@ -21,4 +21,4 @@ LOG_JSON_FORMAT = parse_obj_as(bool, os.getenv("LOG_JSON_FORMAT", False)) LOG_LEVEL = os.getenv("LOG_LEVEL", "INFO") setup_structured_logging(json_logs=LOG_JSON_FORMAT, log_level=LOG_LEVEL) - logger = structlog.stdlib.get_logger("api") + logger = structlog.stdlib.get_logger("api") # type: ignore diff --git a/src/main.py b/src/main.py index 77b765dad02..0cb3652a9f7 100644 --- a/src/main.py +++ b/src/main.py @@ -1,5 +1,5 @@ -import os import logging +import os # Import DataDog tracer ASAP if os.getenv("DD_TRACE_ENABLED", "false").lower() == "true": @@ -7,10 +7,24 @@ logging.getLogger("main").addHandler(logging.StreamHandler()) logging.getLogger("main").info("Enabling Datadog") # import ddtrace.auto # noqa - from ddtrace import patch + from ddtrace import config, patch + # Manually patch. The auto patcher throws some errors in AMQP (which it doesn't support so why patch it??) - patch(sqlalchemy=True, fastapi=True, botocore=True, asyncpg=True, httpx=True, jinja2=True, - requests=True, starlette=True, structlog=True) + patch( + sqlalchemy=True, + fastapi=True, + botocore=True, + asyncpg=True, + httpx=True, + jinja2=True, + requests=True, + starlette=True, + structlog=True, + futures=True, + gevent=True, + redis=True, + ) + config.botocore["distributed_tracing"] = True from infrastructure.app import create_app From b6fffcfd602dbb4a3027c8c2d0b6dfd1ec81294a Mon Sep 17 00:00:00 2001 From: Ramir Mesquita <790844+ramirlm@users.noreply.github.com> Date: Wed, 18 Dec 2024 21:59:02 -0300 Subject: [PATCH 2/2] fix: changes the invitation footer and signature to greek (M2-8100) (#1692) * fix: changes the invitation footer and signature to greek (M2-8100) * fix: cqf auto fixes --------- Co-authored-by: Ramir Mesquita --- src/apps/mailing/static/templates/invitation_new_user_el.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/apps/mailing/static/templates/invitation_new_user_el.html b/src/apps/mailing/static/templates/invitation_new_user_el.html index 9e226ea8b72..8c659aa5d95 100644 --- a/src/apps/mailing/static/templates/invitation_new_user_el.html +++ b/src/apps/mailing/static/templates/invitation_new_user_el.html @@ -20,9 +20,9 @@ - {% include 'blocks/team_info_en.html' %} + {% include 'blocks/team_info_el.html' %} -{% include 'footers/footer_info_en.html' %} +{% include 'footers/footer_info_el.html' %}