refactor(backend): telemetry service interface

localenv/telemetry/grafana/provisioning/dashboards/example.json

@@ -18,6 +18,7 @@
   "editable": true,
   "fiscalYearStartMonth": 0,
   "graphTooltip": 0,
+  "id": 1,
   "links": [],
   "panels": [
     {
@@ -215,6 +216,101 @@
       ],
       "title": "Transaction Count",
       "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "description": "Maximum time it takes to complete the fastest 25%/50%/75% of ILP payments.",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "ms"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 5,
+        "w": 12,
+        "x": 0,
+        "y": 16
+      },
+      "id": 3,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "none",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "percentChangeColorMode": "standard",
+        "reduceOptions": {
+          "calcs": ["lastNotNull"],
+          "fields": "",
+          "values": false
+        },
+        "showPercentChange": false,
+        "textMode": "auto",
+        "wideLayout": true
+      },
+      "pluginVersion": "11.1.0",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "editorMode": "code",
+          "expr": "histogram_quantile(0.25, ilp_pay_time_ms_bucket)",
+          "instant": false,
+          "legendFormat": "First Quartile",
+          "range": true,
+          "refId": "A"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "editorMode": "code",
+          "expr": "histogram_quantile(0.50, ilp_pay_time_ms_bucket)",
+          "hide": false,
+          "instant": false,
+          "legendFormat": "Second Quartile",
+          "range": true,
+          "refId": "B"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "editorMode": "code",
+          "expr": "histogram_quantile(0.75, ilp_pay_time_ms_bucket)",
+          "hide": false,
+          "instant": false,
+          "legendFormat": "Third Quartile",
+          "range": true,
+          "refId": "C"
+        }
+      ],
+      "title": "ILP Pay Time Quartile Approximations",
+      "type": "stat"
     }
   ],
   "refresh": "15s",

packages/backend/src/open_payments/payment/outgoing/lifecycle.ts

@@ -85,21 +85,14 @@ export async function handleSending(
   const payEndTime = Date.now()
 
   if (deps.telemetry) {
-    deps.telemetry
-      .getOrCreateMetric('transactions_total', {
-        description: 'Count of funded transactions'
-      })
-      .add(1, {
-        source: deps.telemetry.getInstanceName()
-      })
+    deps.telemetry.incrementCounter('transactions_total', 1, {
+      description: 'Count of funded transactions'
+    })
+
     const payDuration = payEndTime - payStartTime
-    deps.telemetry
-      .getOrCreateHistogramMetric('ilp_pay_time_ms', {
-        description: 'Time to complete an ILP payment'
-      })
-      .record(payDuration, {
-        source: deps.telemetry.getInstanceName()
-      })
+    deps.telemetry.recordHistogram('ilp_pay_time_ms', payDuration, {
+      description: 'Time to complete an ILP payment'
+    })
   }
 
   await handleCompleted(deps, payment)

packages/backend/src/payment-method/ilp/connector/core/middleware/telemetry.ts

@@ -1,4 +1,4 @@
-import { collectTelemetryAmount } from '../../../../../telemetry/transaction-amount'
+import { ValueType } from '@opentelemetry/api'
 import { ILPContext, ILPMiddleware } from '../rafiki'
 
 export function createTelemetryMiddleware(): ILPMiddleware {
@@ -7,16 +7,24 @@ export function createTelemetryMiddleware(): ILPMiddleware {
     next: () => Promise<void>
   ): Promise<void> => {
     await next()
-    if (
-      services.telemetry &&
-      Number(request.prepare.amount) &&
-      response.fulfill
-    ) {
-      const { code, scale } = accounts.outgoing.asset
-      collectTelemetryAmount(services.telemetry, services.logger, {
-        amount: BigInt(request.prepare.amount),
-        asset: { code: code, scale: scale }
-      })
+
+    const value = BigInt(request.prepare.amount)
+
+    if (services.telemetry && value && response.fulfill) {
+      const { code: assetCode, scale: assetScale } = accounts.outgoing.asset
+
+      await services.telemetry.incrementCounterWithTransactionAmount(
+        'transactions_amount',
+        {
+          value,
+          assetCode,
+          assetScale
+        },
+        {
+          description: 'Amount sent through the network',
+          valueType: ValueType.DOUBLE
+        }
+      )
     }
   }
 }

packages/backend/src/payment-method/ilp/connector/core/test/middleware/telemetry.test.ts

@@ -3,9 +3,7 @@ import { IlpResponse, OutgoingAccount, ZeroCopyIlpPrepare } from '../..'
 import { IncomingAccountFactory, RafikiServicesFactory } from '../../factories'
 import { createTelemetryMiddleware } from '../../middleware/telemetry'
 import { createILPContext } from '../../utils'
-
 import { IlpFulfill } from 'ilp-packet'
-import * as telemetry from '../../../../../../telemetry/transaction-amount'
 
 const incomingAccount = IncomingAccountFactory.build({ id: 'alice' })
 
@@ -35,7 +33,6 @@ const ctx = createILPContext({
   } as IlpResponse
 })
 
-jest.mock('../../../../../../telemetry/transaction-amount')
 const middleware = createTelemetryMiddleware()
 const next = jest.fn().mockImplementation(() => Promise.resolve())
 
@@ -46,33 +43,38 @@ beforeEach(async () => {
 })
 
 describe('Telemetry Middleware', function () {
-  it('does not gather telemetry if telemetry is not enabled (service is undefined)', async () => {
-    const collectAmountSpy = jest
-      .spyOn(telemetry, 'collectTelemetryAmount')
+  it('should not gather telemetry if telemetry is not enabled (service is undefined)', async () => {
+    const incrementCounterWithTransactionAmountSpy = jest
+      .spyOn(services.telemetry, 'incrementCounterWithTransactionAmount')
       .mockImplementation(() => Promise.resolve())
 
     await middleware(
       { ...ctx, services: { ...ctx.services, telemetry: undefined } },
       next
     )
-    expect(collectAmountSpy).not.toHaveBeenCalled()
+
+    expect(incrementCounterWithTransactionAmountSpy).not.toHaveBeenCalled()
     expect(next).toHaveBeenCalled()
   })
 
-  it('does not gather telemetry if response.fulfill undefined', async () => {
-    const collectAmountSpy = jest.spyOn(telemetry, 'collectTelemetryAmount')
+  it('should not gather telemetry if response.fulfill undefined', async () => {
+    const incrementCounterWithTransactionAmountSpy = jest
+      .spyOn(services.telemetry, 'incrementCounterWithTransactionAmount')
+      .mockImplementation(() => Promise.resolve())
 
     await middleware(
       { ...ctx, response: { fulfill: undefined } as IlpResponse },
       next
     )
 
-    expect(collectAmountSpy).not.toHaveBeenCalled()
+    expect(incrementCounterWithTransactionAmountSpy).not.toHaveBeenCalled()
     expect(next).toHaveBeenCalled()
   })
 
-  it('does not gather telemetry if amount is invalid', async () => {
-    const collectAmountSpy = jest.spyOn(telemetry, 'collectTelemetryAmount')
+  it('should not gather telemetry if request.prepare.amount is 0', async () => {
+    const incrementCounterWithTransactionAmountSpy = jest
+      .spyOn(services.telemetry, 'incrementCounterWithTransactionAmount')
+      .mockImplementation(() => Promise.resolve())
 
     await middleware(
       {
@@ -85,27 +87,27 @@ describe('Telemetry Middleware', function () {
       next
     )
 
-    expect(collectAmountSpy).not.toHaveBeenCalled()
+    expect(incrementCounterWithTransactionAmountSpy).not.toHaveBeenCalled()
     expect(next).toHaveBeenCalled()
   })
 
-  it('gathers telemetry without blocking middleware chain', async () => {
+  it('should gather telemetry without blocking middleware chain', async () => {
     let nextCalled = false
     const next = jest.fn().mockImplementation(() => {
       nextCalled = true
       return Promise.resolve()
     })
 
-    const collectAmountSpy = jest
-      .spyOn(telemetry, 'collectTelemetryAmount')
+    const incrementCounterWithTransactionAmountSpy = jest
+      .spyOn(services.telemetry, 'incrementCounterWithTransactionAmount')
       .mockImplementation(() => {
         expect(nextCalled).toBe(true)
         return Promise.resolve()
       })
 
     await middleware(ctx, next)
 
-    expect(collectAmountSpy).toHaveBeenCalled()
+    expect(incrementCounterWithTransactionAmountSpy).toHaveBeenCalled()
     expect(next).toHaveBeenCalled()
   })
 })