diff --git a/build.sbt b/build.sbt index 41fa57a7f..9c7800ebb 100644 --- a/build.sbt +++ b/build.sbt @@ -824,7 +824,8 @@ lazy val docs = project `sdk-exporter-prometheus`.jvm, `sdk-contrib-aws-resource`.jvm, `sdk-contrib-aws-xray`.jvm, - `sdk-contrib-aws-xray-propagator`.jvm + `sdk-contrib-aws-xray-propagator`.jvm, + `sdk-testkit`.jvm ) .settings( libraryDependencies ++= Seq( @@ -837,6 +838,7 @@ lazy val docs = project "io.opentelemetry.instrumentation" % "opentelemetry-runtime-telemetry-java17" % OpenTelemetryInstrumentationAlphaVersion ), mdocVariables ++= Map( + "CATS_EFFECT_VERSION" -> CatsEffectVersion, "OPEN_TELEMETRY_VERSION" -> OpenTelemetryVersion, "OPEN_TELEMETRY_INSTRUMENTATION_ALPHA_VERSION" -> OpenTelemetryInstrumentationAlphaVersion ), diff --git a/docs/sdk/directory.conf b/docs/sdk/directory.conf index 52f524011..0a26d6ba0 100644 --- a/docs/sdk/directory.conf +++ b/docs/sdk/directory.conf @@ -4,6 +4,7 @@ laika.navigationOrder = [ overview.md configuration.md prometheus-exporter.md + testkit.md aws-resource-detectors.md aws-xray.md aws-xray-propagator.md diff --git a/docs/sdk/testkit.md b/docs/sdk/testkit.md new file mode 100644 index 000000000..8ea1f7a69 --- /dev/null +++ b/docs/sdk/testkit.md @@ -0,0 +1,269 @@ +# Testkit + +The `otel4s-sdk-testkit` provides in-memory implementations of metric and trace exporters. +In-memory data can be used to test the structure of the spans, the names of instruments, and many more. + +The testkit is framework-agnostic, so it can be used with any test framework, such as weaver, munit, scalatest. + +## Getting started + +@:select(build-tool) + +@:choice(sbt) + +Add settings to the `build.sbt`: + +```scala +libraryDependencies ++= Seq( + "org.typelevel" %%% "otel4s-sdk-testkit" % "@VERSION@" % Test, // <1> +) +``` + +@:choice(scala-cli) + +Add directives to the `*.scala` file: + +```scala +//> using test.dep "org.typelevel::otel4s-sdk-testkit::@VERSION@" // <1> +``` + +@:@ + +1. Add the `otel4s-sdk-testkit` library + +## Testing metrics + +Let's assume we have a program that increments a counter by one and sets the gauge's value to 42. +Here is how we can test this program: + +```scala mdoc:silent:reset +import cats.effect.IO +import org.typelevel.otel4s.metrics.MeterProvider +import org.typelevel.otel4s.sdk.metrics.data.{MetricData, MetricPoints, PointData} +import org.typelevel.otel4s.sdk.testkit.OpenTelemetrySdkTestkit + +// the program that we want to test +def program(meterProvider: MeterProvider[IO]): IO[Unit] = + for { + meter <- meterProvider.get("service") + + counter <- meter.counter[Long]("service.counter").create + _ <- counter.inc() + + gauge <- meter.gauge[Long]("service.gauge").create + _ <- gauge.record(42L) + } yield () + +// the test +def test: IO[Unit] = + OpenTelemetrySdkTestkit.inMemory[IO]().use { testkit => + // the list of expected metrics + val expected = List( + TelemetryMetric.CounterLong("service.counter", Vector(1L)), + TelemetryMetric.GaugeLong("service.gauge", Vector(42L)) + ) + + for { + // invoke the program + _ <- program(testkit.meterProvider) + // collect the metrics + metrics <- testkit.collectMetrics + // verify the collected metrics + _ <- assertMetrics(metrics, expected) + } yield () + } + +// here you can use an assertion mechanism from your favorite testing framework +def assertMetrics(metrics: List[MetricData], expected: List[TelemetryMetric]): IO[Unit] = + IO { + assert(metrics.map(TelemetryMetric.fromMetricData) == expected) + } + +// a minimized representation of the MetricData to simplify testing +sealed trait TelemetryMetric +object TelemetryMetric { + case class CounterLong(name: String, values: Vector[Long]) extends TelemetryMetric + case class CounterDouble(name: String, values: Vector[Double]) extends TelemetryMetric + + case class GaugeLong(name: String, values: Vector[Long]) extends TelemetryMetric + case class GaugeDouble(name: String, values: Vector[Double]) extends TelemetryMetric + + case class Histogram(name: String, values: Vector[Double]) extends TelemetryMetric + + def fromMetricData(metric: MetricData): TelemetryMetric = + metric.data match { + // Counter, UpDownCounter, ObservableCounter, ObservableUpDownCounter + case sum: MetricPoints.Sum => + val (doubles, longs) = split(sum.points.toVector) + if (doubles.nonEmpty) TelemetryMetric.CounterDouble(metric.name, doubles) + else TelemetryMetric.CounterLong(metric.name, longs) + + // Gauge, ObservableGauge + case gauge: MetricPoints.Gauge => + val (doubles, longs) = split(gauge.points.toVector) + if (doubles.nonEmpty) TelemetryMetric.GaugeDouble(metric.name, doubles) + else TelemetryMetric.GaugeLong(metric.name, longs) + + // Histogram + case histogram: MetricPoints.Histogram => + TelemetryMetric.Histogram( + metric.name, + histogram.points.toVector.flatMap(_.stats.map(_.sum)) + ) + } + + private def split(points: Vector[PointData.NumberPoint]): (Vector[Double], Vector[Long]) = + points.partitionMap { + case point: PointData.LongNumber => Right(point.value) + case double: PointData.DoubleNumber => Left(double.value) + } +} +``` + +`MetricData` provides **all** information about the metric: +name, instrumentation scope, telemetry resource, data points, +associated attributes, collection time window, and so on. + +It's hard to implement an assertion that verifies **all** aspects of the metric +because many things must be considered, such as time window, attributes, exemplars, etc. +To simplify the testing process, we can define a minimized projection of `MetricData` such as `TelemetryMetric`. + +```scala mdoc:invisible +// we silently run the test to ensure it's actually correct +import cats.effect.unsafe.implicits.global +test.unsafeRunSync() +``` + +## Testing spans + +Let's assume we want to test the structure of created spans: + +```scala mdoc:reset +import cats.effect.IO +import org.typelevel.otel4s.sdk.testkit.OpenTelemetrySdkTestkit +import org.typelevel.otel4s.sdk.trace.data.SpanData +import org.typelevel.otel4s.trace.TracerProvider +import scala.concurrent.duration._ + +// the program that we want to test +def program(tracerProvider: TracerProvider[IO]): IO[Unit] = + for { + tracer <- tracerProvider.get("service") + _ <- tracer.span("app.span").surround { + tracer.span("app.nested.1").surround(IO.sleep(200.millis)) >> + tracer.span("app.nested.2").surround(IO.sleep(300.millis)) + } + } yield () + +// the test +def test: IO[Unit] = + OpenTelemetrySdkTestkit.inMemory[IO]().use { testkit => + // the list of expected spans + val expected = List( + SpanTree( + TelemetrySpan("app.span"), + List( + SpanTree(TelemetrySpan("app.nested.1"), Nil), + SpanTree(TelemetrySpan("app.nested.2"), Nil) + ) + ) + ) + + for { + // invoke the program + _ <- program(testkit.tracerProvider) + // collect the finished spans + spans <- testkit.finishedSpans + // verify the collected spans + _ <- assertSpans(spans, expected) + } yield () + } + +// here you can use an assertion mechanism from your favorite testing framework +def assertSpans(spans: List[SpanData], expected: List[SpanTree[TelemetrySpan]]): IO[Unit] = + IO { + val trees = SpanTree.fromSpans(spans) + assert(trees.map(_.map(data => TelemetrySpan(data.name))) == expected) + } + +// a minimized representation of the SpanData to simplify testing +case class TelemetrySpan(name: String) + +// a tree-like representation of the spans +case class SpanTree[A](current: A, children: List[SpanTree[A]]) { + def map[B](f: A => B): SpanTree[B] = SpanTree(f(current), children.map(_.map(f))) +} +object SpanTree { + def fromSpans(spans: List[SpanData]): List[SpanTree[SpanData]] = { + val byParent = spans.groupBy(s => s.parentSpanContext.map(_.spanIdHex)) + val topNodes = byParent.getOrElse(None, Nil) + val bottomToTop = sortNodesByDepth(0, topNodes, byParent, Nil) + val maxDepth = bottomToTop.headOption.map(_.depth).getOrElse(0) + buildFromBottom(maxDepth, bottomToTop, byParent, Map.empty) + } + + private case class EntryWithDepth(data: SpanData, depth: Int) + + @annotation.tailrec + private def sortNodesByDepth( + depth: Int, + nodesInDepth: List[SpanData], + nodesByParent: Map[Option[String], List[SpanData]], + acc: List[EntryWithDepth] + ): List[EntryWithDepth] = { + val withDepth = nodesInDepth.map(n => EntryWithDepth(n, depth)) + val calculated = withDepth ++ acc + val children = nodesInDepth.flatMap { n => + nodesByParent.getOrElse(Some(n.spanContext.spanIdHex), Nil) + } + children match { + case Nil => + calculated + case _ => + sortNodesByDepth(depth + 1, children, nodesByParent, calculated) + } + } + + @annotation.tailrec + private def buildFromBottom( + depth: Int, + remaining: List[EntryWithDepth], + nodesByParent: Map[Option[String], List[SpanData]], + processedNodesById: Map[String, SpanTree[SpanData]] + ): List[SpanTree[SpanData]] = { + val (nodesOnCurrentDepth, rest) = remaining.span(_.depth == depth) + val newProcessedNodes = nodesOnCurrentDepth.map { n => + val nodeId = n.data.spanContext.spanIdHex + val children = nodesByParent + .getOrElse(Some(nodeId), Nil) + .flatMap(c => processedNodesById.get(c.spanContext.spanIdHex)) + val leaf = SpanTree(n.data, children) + nodeId -> leaf + }.toMap + if (depth > 0) { + buildFromBottom( + depth - 1, + rest, + nodesByParent, + processedNodesById ++ newProcessedNodes + ) + } else { + // top nodes + newProcessedNodes.values.toList + } + } +} +``` + +`SpanData` provides **all** information about the span: +name, instrumentation scope, telemetry resource, associated attributes, time window, and so on. + +It's difficult to implement an assertion that verifies **all** aspects of the span +because many things must be considered, such as time windows, attributes, etc. +To simplify the testing process, we can define a minimized projection of `SpanData`, such as `TelemetrySpan`. + +```scala mdoc:invisible +// we silently run the test to ensure it's actually correct +import cats.effect.unsafe.implicits.global +test.unsafeRunSync() +```