From 5db972eacbad20afe91a6a328833f2d7deaadb15 Mon Sep 17 00:00:00 2001
From: Fletcher Nichol <fletcher@systeminit.com>
Date: Wed, 7 Feb 2024 13:54:20 -0700
Subject: [PATCH] chore(telemetry): add a span around tracing level change
 events

This change will ensure that anytime we change the telemetry tracing log
levels, we get an `INFO` level span transmitted to the OpenTelemetry
endpoint with its associated events.

Signed-off-by: Fletcher Nichol <fletcher@systeminit.com>
---
 lib/telemetry-application-rs/src/lib.rs | 32 ++++++++++++++++---------
 1 file changed, 21 insertions(+), 11 deletions(-)

diff --git a/lib/telemetry-application-rs/src/lib.rs b/lib/telemetry-application-rs/src/lib.rs
index 9350fc67d9..41ec7d60d9 100644
--- a/lib/telemetry-application-rs/src/lib.rs
+++ b/lib/telemetry-application-rs/src/lib.rs
@@ -580,21 +580,31 @@ impl TelemetryUpdateTask {
         while let Some(command) = self.update_command_rx.recv().await {
             match command {
                 TelemetryCommand::TracingLevel { level, wait } => {
-                    if let Err(err) = self.update_tracing_level(level) {
-                        warn!(
-                            task = Self::NAME,
-                            error = ?err,
-                            "failed to update tracing level, using prior value",
-                        );
-                    }
-                    if let Some(tx) = wait {
-                        if let Err(err) = tx.send(()) {
+                    // We want a span around the update logging so this is transmitted to our
+                    // OpenTelemetry endpoint. We may use this span (and associated events) as a
+                    // deployment mutation event, for example adding a mark in Honeycomb.
+                    //
+                    // Also note that we're using the `in_scope` method as none of the containing
+                    // code is asynchronous--if there were async code then we'd use the
+                    // `.instrument()` combinator on the future.
+                    let span = info_span!("telemetry_update_task.update_tracing_level");
+                    span.in_scope(|| {
+                        if let Err(err) = self.update_tracing_level(level) {
                             warn!(
+                                task = Self::NAME,
                                 error = ?err,
-                                "receiver already closed when waiting on changing tracing level",
+                                "failed to update tracing level, using prior value",
                             );
                         }
-                    }
+                        if let Some(tx) = wait {
+                            if let Err(err) = tx.send(()) {
+                                warn!(
+                                    error = ?err,
+                                    "receiver already closed when waiting on changing tracing level",
+                                );
+                            }
+                        }
+                    })
                 }
                 TelemetryCommand::Shutdown(token) => {
                     if !self.is_shutdown {