From 05610492af8a01e893656820ddcc1aebfea7ba60 Mon Sep 17 00:00:00 2001 From: Rohan Juneja Date: Mon, 18 Dec 2023 18:37:57 -0800 Subject: [PATCH 1/7] create otel spans for actions in thread --- src/controllers/opentelemetry.js | 4 +-- src/controllers/threading/taskHandler.js | 31 +++++++++++++++--------- 2 files changed, 21 insertions(+), 14 deletions(-) diff --git a/src/controllers/opentelemetry.js b/src/controllers/opentelemetry.js index b711fc5..119aa04 100644 --- a/src/controllers/opentelemetry.js +++ b/src/controllers/opentelemetry.js @@ -13,9 +13,9 @@ const sdk = new opentelemetry.NodeSDK({ traceExporter: new OTLPTraceExporter({ url: process.env.JAEGER_URL ?? "http://localhost:4318/v1/traces", }), - instrumentations: [getNodeAutoInstrumentations()], + instrumentations: isMainThread ? [getNodeAutoInstrumentations()] : [], resource: new Resource({ - "service.name": "biothings-explorer", + "service.name": isMainThread ? "biothings-explorer" : "biothings-explorer-thread", }), }); sdk.start(); diff --git a/src/controllers/threading/taskHandler.js b/src/controllers/threading/taskHandler.js index d27348a..f3a38ce 100644 --- a/src/controllers/threading/taskHandler.js +++ b/src/controllers/threading/taskHandler.js @@ -1,3 +1,5 @@ +require('../opentelemetry') + const { isMainThread, threadId } = require("worker_threads"); const workerData = require("piscina").workerData; const debug = require("debug")(`bte:biothings-explorer-trapi:worker${threadId}`); @@ -58,25 +60,29 @@ const runTask = async ({ req, route, port, job: { jobId, queueName } = {} }) => global.job = await queue.getJob(jobId); } - let transaction; + const routeNames = { + query_v1: "EXEC /v1/query", + query_v1_by_api: "EXEC /v1/smartapi/:/query", + query_v1_by_team: "EXEC /v1/team/:/query", + asyncquery_status: "EXEC /v1/asyncquery_status", + asyncquery_v1: "EXEC /v1/asyncquery", + asyncquery_v1_by_api: "EXEC /v1/smartapi/:/asyncquery", + asyncquery_v1_by_team: "EXEC /v1/team/:/asyncquery", + }; + + let transaction, span; try { - const routeNames = { - query_v1: "EXEC /v1/query", - query_v1_by_api: "EXEC /v1/smartapi/:/query", - query_v1_by_team: "EXEC /v1/team/:/query", - asyncquery_status: "EXEC /v1/asyncquery_status", - asyncquery_v1: "EXEC /v1/asyncquery", - asyncquery_v1_by_api: "EXEC /v1/smartapi/:/asyncquery", - asyncquery_v1_by_team: "EXEC /v1/team/:/asyncquery", - }; transaction = Sentry.startTransaction({ name: routeNames[route] }); transaction.setData("request", req.data.queryGraph); Sentry.getCurrentHub().configureScope(scope => { scope.clearBreadcrumbs(); scope.setSpan(transaction); }); + + span = opentelemetry.trace.getTracer('biothings-explorer-thread').startSpan(routeNames[route]) + span.setAttribute("request", req.data.queryGraph); } catch (error) { - debug("Sentry transaction start error. This does not affect execution."); + debug("Sentry/OpenTelemetry transaction start error. This does not affect execution."); debug(error); } @@ -85,8 +91,9 @@ const runTask = async ({ req, route, port, job: { jobId, queueName } = {} }) => try { transaction.finish(); + span.end(); } catch (error) { - debug("Sentry transaction finish error. This does not affect execution."); + debug("Sentry/OpenTelemetry transaction finish error. This does not affect execution."); debug(error); } From 561581965eef0508b1e1a55ed597ed153521b7b7 Mon Sep 17 00:00:00 2001 From: Rohan Juneja Date: Thu, 21 Dec 2023 17:49:25 -0800 Subject: [PATCH 2/7] fix import --- src/controllers/threading/taskHandler.js | 1 + 1 file changed, 1 insertion(+) diff --git a/src/controllers/threading/taskHandler.js b/src/controllers/threading/taskHandler.js index f3a38ce..f3a8d78 100644 --- a/src/controllers/threading/taskHandler.js +++ b/src/controllers/threading/taskHandler.js @@ -13,6 +13,7 @@ const { tasks } = require("../../routes/index"); const { getQueryQueue } = require("../async/asyncquery_queue"); const Sentry = require("@sentry/node"); const { ProfilingIntegration } = require("@sentry/profiling-node"); +const opentelemetry = require('@opentelemetry/api'); // use SENTRY_DSN environment variable try { From a6369fec2c2ae85016ec3538614148c830fc59bb Mon Sep 17 00:00:00 2001 From: Rohan Juneja Date: Fri, 22 Dec 2023 18:21:52 -0800 Subject: [PATCH 3/7] import fix --- src/controllers/opentelemetry.js | 1 + 1 file changed, 1 insertion(+) diff --git a/src/controllers/opentelemetry.js b/src/controllers/opentelemetry.js index 119aa04..a786599 100644 --- a/src/controllers/opentelemetry.js +++ b/src/controllers/opentelemetry.js @@ -5,6 +5,7 @@ const { OTLPMetricExporter } = require("@opentelemetry/exporter-metrics-otlp-pro const { PeriodicExportingMetricReader, ConsoleMetricExporter } = require("@opentelemetry/sdk-metrics"); const { Resource } = require("@opentelemetry/resources"); const { ConsoleSpanExporter } = require("@opentelemetry/sdk-trace-node"); +const { isMainThread } = require('worker_threads'); const Debug = require("debug"); const debug = Debug("bte:biothings-explorer:otel-init"); From 635226e5319c8fca6b7a1570471c70895c3461e6 Mon Sep 17 00:00:00 2001 From: Rohan Juneja Date: Fri, 22 Dec 2023 18:22:03 -0800 Subject: [PATCH 4/7] manual instrumentation --- src/controllers/threading/taskHandler.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/controllers/threading/taskHandler.js b/src/controllers/threading/taskHandler.js index f3a8d78..7d062d8 100644 --- a/src/controllers/threading/taskHandler.js +++ b/src/controllers/threading/taskHandler.js @@ -14,6 +14,7 @@ const { getQueryQueue } = require("../async/asyncquery_queue"); const Sentry = require("@sentry/node"); const { ProfilingIntegration } = require("@sentry/profiling-node"); const opentelemetry = require('@opentelemetry/api'); +const { Telemetry } = require('@biothings-explorer/utils'); // use SENTRY_DSN environment variable try { @@ -82,6 +83,7 @@ const runTask = async ({ req, route, port, job: { jobId, queueName } = {} }) => span = opentelemetry.trace.getTracer('biothings-explorer-thread').startSpan(routeNames[route]) span.setAttribute("request", req.data.queryGraph); + Telemetry.setOtelContext(opentelemetry.trace.setSpan(opentelemetry.context.active(), span)); } catch (error) { debug("Sentry/OpenTelemetry transaction start error. This does not affect execution."); debug(error); @@ -93,6 +95,7 @@ const runTask = async ({ req, route, port, job: { jobId, queueName } = {} }) => try { transaction.finish(); span.end(); + Telemetry.removeOtelContext(); } catch (error) { debug("Sentry/OpenTelemetry transaction finish error. This does not affect execution."); debug(error); From 829352ae5b42ff3124cc942851659d3a8f504fdf Mon Sep 17 00:00:00 2001 From: Rohan Juneja Date: Wed, 27 Dec 2023 14:46:02 -0800 Subject: [PATCH 5/7] fix otel inner spans --- src/controllers/threading/taskHandler.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/controllers/threading/taskHandler.js b/src/controllers/threading/taskHandler.js index 7d062d8..22ae424 100644 --- a/src/controllers/threading/taskHandler.js +++ b/src/controllers/threading/taskHandler.js @@ -83,7 +83,7 @@ const runTask = async ({ req, route, port, job: { jobId, queueName } = {} }) => span = opentelemetry.trace.getTracer('biothings-explorer-thread').startSpan(routeNames[route]) span.setAttribute("request", req.data.queryGraph); - Telemetry.setOtelContext(opentelemetry.trace.setSpan(opentelemetry.context.active(), span)); + Telemetry.setOtelSpan(span); } catch (error) { debug("Sentry/OpenTelemetry transaction start error. This does not affect execution."); debug(error); @@ -95,7 +95,7 @@ const runTask = async ({ req, route, port, job: { jobId, queueName } = {} }) => try { transaction.finish(); span.end(); - Telemetry.removeOtelContext(); + Telemetry.removeOtelSpan(); } catch (error) { debug("Sentry/OpenTelemetry transaction finish error. This does not affect execution."); debug(error); From 110c407b4d13625e46cbe9cfeb841ac3562225ee Mon Sep 17 00:00:00 2001 From: Rohan Juneja Date: Fri, 29 Dec 2023 18:13:42 -0800 Subject: [PATCH 6/7] record attributes --- src/controllers/threading/taskHandler.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/controllers/threading/taskHandler.js b/src/controllers/threading/taskHandler.js index 22ae424..f056ad5 100644 --- a/src/controllers/threading/taskHandler.js +++ b/src/controllers/threading/taskHandler.js @@ -82,7 +82,7 @@ const runTask = async ({ req, route, port, job: { jobId, queueName } = {} }) => }); span = opentelemetry.trace.getTracer('biothings-explorer-thread').startSpan(routeNames[route]) - span.setAttribute("request", req.data.queryGraph); + span.setAttribute("bte.requestData", JSON.stringify(req.data.queryGraph)); Telemetry.setOtelSpan(span); } catch (error) { debug("Sentry/OpenTelemetry transaction start error. This does not affect execution."); From f0873e9bcda8e82414b0b1509b3ee989f0b926c3 Mon Sep 17 00:00:00 2001 From: tokebe <43009413+tokebe@users.noreply.github.com> Date: Thu, 18 Jan 2024 12:05:17 -0500 Subject: [PATCH 7/7] mild changes --- src/controllers/opentelemetry.js | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/controllers/opentelemetry.js b/src/controllers/opentelemetry.js index 5fa9d61..c892637 100644 --- a/src/controllers/opentelemetry.js +++ b/src/controllers/opentelemetry.js @@ -1,20 +1,20 @@ const opentelemetry = require("@opentelemetry/sdk-node"); const { getNodeAutoInstrumentations } = require("@opentelemetry/auto-instrumentations-node"); const { Resource } = require("@opentelemetry/resources"); -const { isMainThread } = require('worker_threads'); +const { isMainThread } = require("worker_threads"); const Debug = require("debug"); const debug = Debug("bte:biothings-explorer:otel-init"); -const { JaegerExporter } = require("@opentelemetry/exporter-jaeger"); +const { OTLPTraceExporter } = require("@opentelemetry/exporter-trace-otlp-proto"); debug("Initializing Opentelemetry instrumentation..."); const sdk = new opentelemetry.NodeSDK({ - traceExporter: new JaegerExporter({ + traceExporter: new OTLPTraceExporter({ host: process.env.JAEGER_HOST ?? "jaeger-otel-agent.sri", port: parseInt(process.env.JAEGER_PORT ?? "6832"), }), - instrumentations: isMainThread ? [getNodeAutoInstrumentations()] : [], + instrumentations: [getNodeAutoInstrumentations()], resource: new Resource({ - "service.name": isMainThread ? "biothings-explorer" : "biothings-explorer-thread", + "service.name": "biothings-explorer", }), }); sdk.start();