feat(telemetry): instrument rafiki (#2299)

* feat(telemetry): add opentelemetry and instrument rafiki.
localenv now uses otel collector in order to send metrics to AMP

Co-authored-by: JoblersTune <[email protected]>

* feat(telemetry): move transaction_amount metric into ilp connector.

extract metric collection logic from spread codebase into "collector" functions in the telemetry folder.
(this approach intends using callbacks that provide the necessary data after bussiness logic is executed, in order to separate telemetry logic out of the bussiness.
The data is then fed into the "collector" functions)

Increased seed liquidity for localenv

* chore(telemetry): format

* feat(telemetry): go back to usual DI implementation

the collector functions are now part of TelemetryService

* feat(telemetry): remove collector methods from telemetry service & reintroduce logic in respective services

* feat(telemetry): use already existing env variable (instance_name)

* feat(telemetry): move transactions_total metric to outgoing payment instead of accounting.
-this is so that both transaction_amount and transaction_total to metrics to be collected on the sender side of the transaction.

remove Metrics enum
add getOrCreate, so that Counters can be created in respective services instead of telemetryService constructor

* feat(telemetry): telemetry now has it's own middleware instead of injecting logic into handleIlpData

* feat(telemetry): add unit tests for ILP Connector telemetry middleware

* feat(telemetry):remove clog

* feat(telemetry): add telemetry middleware and service tests

* feat(telemetry): mock TelemetryService in createTestApp

* feat(telemetry): fix DI of middleware. Add condition for early return in the case of receiving side

* feat(telemetry): move telemetry middleware after createStreamController in the connector middleware chain.

* feat(telemetry): telemetry service now supports sending metrics to multiple OTEL collectors defined in env variable.

* feat:(telemetry): remove asset_code information from transactions_amount and transactions_count metrics

* feat(telemetry): add rates service that uses external rates

* feat(telemetry): add privacy

* feat(telemetry): rephrase md

* feat(telemetry): extract privacy as a separate module

* feat(telemetry): add tests

* feat(telemetry): remove hostPort

* feat(telemetry): cleanup & add tesets

* feat(telemetry): change privacy doc wording

* feat(telemetry): Remove local otel collector setup from cloud-nine and happy-life.
Keeping it just as an example for ASE integrators

* feat(telemetry): prioritize ase provided exchange rates before external rates

* feat(telemetry): openTelemetryCollectors env var now has a default

* feat(telemetry): remove source from amount metrics for privacy concerns

* feat(telemetry): add exchange rates retrieval lambda to codebase

* feat(telemetry): docs

* Resolve documentation issues

* feat(telemetry): move back from ILP connector to accounting service

* feat(telemetry): base asset code no longer externally configurable
base asset scale added

* feat(telemetry): leftover of last commit

* feat(telemetry): some docs changes

* feat(telemetry):docs changes

* feat(telemetry):docs

* feat(telemetry):more docs changes

* feat(telemetry): remove leftover mock

* feat(telemetry): Move conversion in telemetry & update DI accordingly.
Moved & updated tests

* feat(telemetry): docs - create new integration section

* feat(telemetry): logger DI - add BaseAccountingServiceDependencies interface
test updates
doc updates

* feat(telemetry): telemetry optional in the AppServices interface

* docs(telemetry): remove collection implementation details

* feat(telemetry): cleanup

* feat(telemetry):testing- telemetry service tests now properly mock otel dependencies and tests the actual service.
This meant moving the fallbackRatesService as an AppService, no longer proprietary to telemetry, and just having it injected in telemetry.

* feat(telemetry): opt out of otel sdk periodic logging on periodic exports

* feat(telemetry): meter provider shutdown exposed in telemetryService interface, and called in the gracefull shutdown as to avoid leaks in tests

* feat(telemetry): tests now provide empty list of collector url's, so that no metric exporters are actually instantiated

* feat(telemetry): tests now no longer need to mock exporters, as they are not created when collectorUrl list is empty

* feat(telemetry): code review cleanup

* feat(telemetry): docs updates

* feat(telemetry): docs improvements

* fix(telemetry):docs link

* feat(telemetry): docs change

* feat(telemetry): fix docs link

* feat(telemetry): fix seed not having peeringAsset

* feat(telemetry): remove unused aseRatesService

* feat(telemetry): rename fallbackRates to internalRates

* feat(telemetry): added comments for privacy implementation + added a test

* feat(telemetry): remove testing clog

* feaet(telemetry): remove telemetry from localenv

* feat(telemetry): enableTelemetry defaults to false

* feat(telemetry): turn default telemetry back on, but off on localenv

* feat(telemetry): add distinction between livenet and testnet

* feat(telemetry): update docs to new LIVENET env variable

* feat(telemetry): plural on OPEN_TELEMETRY_COLLECTOR_URLS
& docs changes

---------

Co-authored-by: JoblersTune <[email protected]>

aws/lambdas/exchange-rates/go.mod

@@ -0,0 +1,9 @@
+module exchange-rates-lambda
+
+go 1.21.0
+
+require (
+	github.com/aws/aws-lambda-go v1.42.0 // indirect
+	github.com/aws/aws-sdk-go v1.49.1 // indirect
+	github.com/jmespath/go-jmespath v0.4.0 // indirect
+)

aws/lambdas/exchange-rates/go.sum

@@ -0,0 +1,12 @@
+github.com/aws/aws-lambda-go v1.42.0 h1:U4QKkxLp/il15RJGAANxiT9VumQzimsUER7gokqA0+c=
+github.com/aws/aws-lambda-go v1.42.0/go.mod h1:dpMpZgvWx5vuQJfBt0zqBha60q7Dd7RfgJv23DymV8A=
+github.com/aws/aws-sdk-go v1.49.1 h1:Dsamcd8d/nNb3A+bZ0ucfGl0vGZsW5wlRW0vhoYGoeQ=
+github.com/aws/aws-sdk-go v1.49.1/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
+github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
+github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

aws/lambdas/exchange-rates/main.go

@@ -0,0 +1,119 @@
+package main
+
+import (
+	"bytes"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net/http"
+	"os"
+	"strconv"
+	"time"
+
+	"github.com/aws/aws-lambda-go/lambda"
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/s3/s3manager"
+)
+
+type RatesResponse struct {
+	Base  string             `json:"base"`
+	Rates map[string]float64 `json:"rates"`
+}
+
+type MerchantRates struct {
+	Merchant map[string]map[string]string `json:"merchant"`
+}
+
+func getExchangeRates(apiURL string) (map[string]map[string]string, error) {
+	client := &http.Client{Timeout: 15 * time.Second}
+	response, err := client.Get(apiURL)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get exchange rates from %s: %w", apiURL, err)
+	}
+	defer response.Body.Close()
+
+	if response.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("received non-200 response code: %d", response.StatusCode)
+	}
+
+	var ratesResponse MerchantRates
+	err = json.NewDecoder(response.Body).Decode(&ratesResponse)
+	if err != nil {
+		return nil, fmt.Errorf("failed to decode response body: %w", err)
+	}
+
+	return ratesResponse.Merchant, nil
+}
+
+func transformToBaseCurrency(baseCurrency string, merchantRates map[string]map[string]string) (*RatesResponse, error) {
+
+	usdRates := make(map[string]float64)
+
+	for currency, rateMap := range merchantRates {
+		rateStr, ok := rateMap[baseCurrency]
+		if !ok {
+			return nil, fmt.Errorf("failed to find %s rate for currency %s in merchant rates", baseCurrency, currency)
+		}
+
+		rate, err := strconv.ParseFloat(rateStr, 64)
+		if err != nil {
+			return nil, fmt.Errorf("failed to convert rate to float64 for currency %s: %v", currency, err)
+		}
+
+		usdRates[currency] = rate
+	}
+
+	return &RatesResponse{
+		Base:  baseCurrency,
+		Rates: usdRates,
+	}, nil
+}
+
+func Handler() (string, error) {
+	apiUrl := os.Getenv("API_URL")
+	bucketName := os.Getenv("BUCKET_NAME")
+	keyName := os.Getenv("KEY_NAME")
+	region := os.Getenv("REGION")
+	baseCurrency := os.Getenv("BASE_CURRENCY")
+
+	if apiUrl == "" || bucketName == "" || keyName == "" || region == "" {
+		return "", errors.New("API_URL, BUCKET_NAME, or KEY_NAME environment variable is not set")
+	}
+
+	merchantRates, err := getExchangeRates(apiUrl)
+	if err != nil {
+		return "", err
+	}
+
+	data, err := transformToBaseCurrency(baseCurrency, merchantRates)
+	if err != nil {
+		return "", err
+	}
+
+	dataBytes, err := json.Marshal(data)
+	if err != nil {
+		return "", err
+	}
+
+	sess := session.Must(session.NewSession(&aws.Config{
+		Region: aws.String(region)},
+	))
+
+	uploader := s3manager.NewUploader(sess)
+
+	_, err = uploader.Upload(&s3manager.UploadInput{
+		Bucket: aws.String(bucketName),
+		Key:    aws.String(keyName),
+		Body:   bytes.NewReader(dataBytes),
+	})
+	if err != nil {
+		return "", err
+	}
+
+	return "Successfully uploaded data to " + bucketName + "/" + keyName, nil
+}
+
+func main() {
+	lambda.Start(Handler)
+}

localenv/.gitignore

@@ -0,0 +1 @@
+.env

localenv/cloud-nine-wallet/docker-compose.yml

@@ -40,6 +40,7 @@ services:
       - rafiki
     environment:
       NODE_ENV: ${NODE_ENV:-development}
+      INSTANCE_NAME: CLOUD-NINE
       TRUST_PROXY: ${TRUST_PROXY}
       LOG_LEVEL: debug
       ADMIN_PORT: 3001
@@ -59,6 +60,7 @@ services:
       REDIS_URL: redis://shared-redis:6379/0
       WALLET_ADDRESS_URL: ${CLOUD_NINE_WALLET_ADDRESS_URL:-https://cloud-nine-wallet-backend/.well-known/pay}
       ILP_CONNECTOR_ADDRESS: ${CLOUD_NINE_CONNECTOR_URL}
+      ENABLE_TELEMETRY: false
     depends_on:
       - shared-database
       - shared-redis
@@ -118,6 +120,7 @@ services:
     depends_on:
       - cloud-nine-backend
 
+
 volumes:
   database-data: # named volumes can be managed easier using docker-compose
 

localenv/cloud-nine-wallet/seed.yml

@@ -5,42 +5,42 @@ self:
 assets:
   - code: USD
     scale: 2
-    liquidity: 1000000
-    liquidityThreshold: 100000
+    liquidity: 100000000
+    liquidityThreshold: 10000000
   - code: EUR
     scale: 2
-    liquidity: 1000000
-    liquidityThreshold: 100000
+    liquidity: 100000000
+    liquidityThreshold: 10000000
   - code: MXN
     scale: 2
-    liquidity: 1000000
-    liquidityThreshold: 100000
+    liquidity: 100000000
+    liquidityThreshold: 10000000
   - code: JPY
     scale: 0
-    liquidity: 10000
-    liquidityThreshold: 1000
+    liquidity: 1000000
+    liquidityThreshold: 100000
 peeringAsset: 'USD'
 peers:
-  - initialLiquidity: '100000'
+  - initialLiquidity: '10000000'
     peerUrl: http://happy-life-bank-backend:3002
     peerIlpAddress: test.happy-life-bank
-    liquidityThreshold: 10000
+    liquidityThreshold: 1000000
 accounts:
   - name: 'Grace Franklin'
     path: accounts/gfranklin
     id: 742ab7cd-1624-4d2e-af6e-e15a71638669
-    initialBalance: 50000
+    initialBalance: 40000000
     postmanEnvVar: gfranklinWalletAddress
     assetCode: USD
   - name: 'Bert Hamchest'
     id: a9adbe1a-df31-4766-87c9-d2cb2e636a9b
-    initialBalance: 50000
+    initialBalance: 40000000
     path: accounts/bhamchest
     postmanEnvVar: bhamchestWalletAddress
     assetCode: USD
   - name: "World's Best Donut Co"
     id: 5726eefe-8737-459d-a36b-0acce152cb90
-    initialBalance: 2000
+    initialBalance: 20000000
     path: accounts/wbdc
     postmanEnvVar: wbdcWalletAddress
     assetCode: USD

localenv/happy-life-bank/docker-compose.yml

@@ -34,6 +34,7 @@ services:
       - rafiki
     environment:
       NODE_ENV: development
+      INSTANCE_NAME: HAPPY-LIFE
       LOG_LEVEL: debug
       ADMIN_PORT: 3001
       CONNECTOR_PORT: 3002
@@ -51,6 +52,7 @@ services:
       EXCHANGE_RATES_URL: http://happy-life-bank/rates
       REDIS_URL: redis://shared-redis:6379/1
       WALLET_ADDRESS_URL: ${HAPPY_LIFE_BANK_WALLET_ADDRESS_URL:-https://happy-life-bank-backend/.well-known/pay}
+      ENABLE_TELEMETRY: false
     depends_on:
       - cloud-nine-backend
   happy-life-auth:

localenv/happy-life-bank/seed.yml

@@ -5,55 +5,55 @@ self:
 assets:
   - code: USD
     scale: 2
-    liquidity: 1000000
-    liquidityThreshold: 100000
+    liquidity: 10000000000
+    liquidityThreshold: 100000000
   - code: EUR
     scale: 2
-    liquidity: 1000000
-    liquidityThreshold: 100000
+    liquidity: 10000000000
+    liquidityThreshold: 1000000
   - code: MXN
     scale: 2
-    liquidity: 1000000
-    liquidityThreshold: 100000
+    liquidity: 10000000000
+    liquidityThreshold: 10000000
   - code: JPY
     scale: 0
-    liquidity: 10000
-    liquidityThreshold: 1000
+    liquidity: 1000000000
+    liquidityThreshold: 1000000
 peeringAsset: 'USD'
 peers:
   - initialLiquidity: '1000000000000'
     peerUrl: http://cloud-nine-wallet-backend:3002
     peerIlpAddress: test.cloud-nine-wallet
-    liquidityThreshold: 100000
+    liquidityThreshold: 1000000
 accounts:
   - name: 'Philip Fry'
     path: accounts/pfry
     id: 97a3a431-8ee1-48fc-ac85-70e2f5eba8e5
-    initialBalance: 50000
+    initialBalance: 1
     postmanEnvVar: pfryWalletAddress
     assetCode: USD
   - name: 'PlanEx Corp'
     id: a455cc54-b583-455b-836a-e5275c5c05b7
-    initialBalance: 2000
+    initialBalance: 2000000
     path: accounts/planex
     postmanEnvVar: planexWalletAddress
     assetCode: USD
   - name: 'Alice Smith'
     path: accounts/asmith
     id: f47ac10b-58cc-4372-a567-0e02b2c3d479
-    initialBalance: 500
+    initialBalance: 5000000
     postmanEnvVar: asmithWalletAddress
     skipWalletAddressCreation: true
     assetCode: USD
   - name: 'Lars'
     path: accounts/lars
     id: fd4ecbc9-205d-4ecd-a030-507d6ce2bde6
-    initialBalance: 50000
+    initialBalance: 50000000
     assetCode: EUR
   - name: 'David'
     path: accounts/david
     id: 60257507-3191-4507-9d77-9071fd6b3c30
-    initialBalance: 150000
+    initialBalance: 1500000000
     assetCode: MXN
 rates:
   EUR:

packages/backend/package.json

@@ -56,6 +56,10 @@
     "@interledger/pay": "0.4.0-alpha.9",
     "@interledger/stream-receiver": "^0.3.3-alpha.3",
     "@koa/router": "^12.0.0",
+    "@opentelemetry/api": "^1.6.0",
+    "@opentelemetry/exporter-metrics-otlp-grpc": "^0.43.0",
+    "@opentelemetry/resources": "^1.17.0",
+    "@opentelemetry/sdk-metrics": "^1.17.0",
     "ajv": "^8.12.0",
     "axios": "1.6.7",
     "base64url": "^3.0.1",

packages/backend/src/accounting/psql/service.ts

@@ -3,17 +3,18 @@ import { TransactionOrKnex } from 'objection'
 import { v4 as uuid } from 'uuid'
 import { Asset } from '../../asset/model'
 import { BaseService } from '../../shared/baseService'
+import { TelemetryService } from '../../telemetry/service'
 import { isTransferError, TransferError } from '../errors'
 import {
   AccountingService,
+  createAccountToAccountTransfer,
   Deposit,
   LiquidityAccount,
   LiquidityAccountType,
   Transaction,
   TransferOptions,
   TransferToCreate,
-  Withdrawal,
-  createAccountToAccountTransfer
+  Withdrawal
 } from '../service'
 import { getAccountBalances } from './balance'
 import {
@@ -35,6 +36,7 @@ import {
 import { LedgerTransfer, LedgerTransferType } from './ledger-transfer/model'
 
 export interface ServiceDependencies extends BaseService {
+  telemetry?: TelemetryService
   knex: TransactionOrKnex
   withdrawalThrottleDelay?: number
 }
@@ -200,9 +202,8 @@ export async function createTransfer(
   deps: ServiceDependencies,
   args: TransferOptions
 ): Promise<Transaction | TransferError> {
-  return createAccountToAccountTransfer({
+  return createAccountToAccountTransfer(deps, {
     transferArgs: args,
-    withdrawalThrottleDelay: deps.withdrawalThrottleDelay,
     voidTransfers: async (transferRefs) => voidTransfers(deps, transferRefs),
     postTransfers: async (transferRefs) => postTransfers(deps, transferRefs),
     getAccountReceived: async (accountRef) =>