merge: #3272

3272: feat: support telemetry logging option as JSON lines r=fnichol a=fnichol

This change adds a new `--log-json` flag to all services which will configure the telemetry to output logging as JSON objects, one per line. An associated environment variable of `SI_LOG_JSON` is also supported to enable this output mode.

<img src="https://media2.giphy.com/media/xT9KVn8tApfgnNLpL2/giphy.gif"/>

Co-authored-by: Fletcher Nichol <[email protected]>

Cargo.toml

@@ -149,7 +149,7 @@ tower = "0.4.13"
 tower-http = { version = "0.4", features = ["compression-br", "compression-deflate", "compression-gzip", "cors", "trace"] } # todo: pinning back to 0.4.4, upgrade this alongside hyper/http/axum/tokio-tungstenite
 tracing = { version = "0.1" }
 tracing-opentelemetry = "0.22.0"
-tracing-subscriber = { version = "0.3.18", features = ["env-filter", "std"] }
+tracing-subscriber = { version = "0.3.18", features = ["env-filter", "json", "std"] }
 ulid = { version = "1.0.0", features = ["serde"] }
 url = { version = "2.3.1", features = ["serde"] }
 uuid = { version = "1.3.2", features = ["serde", "v4"] }

bin/council/src/args.rs

@@ -43,6 +43,17 @@ pub(crate) struct Args {
     )]
     pub(crate) force_color: bool,
 
+    /// Prints telemetry logging as JSON lines.
+    ///
+    /// For more details, visit: <https://jsonlines.org/>.
+    #[arg(
+        long = "log-json",
+        default_value = "false",
+        env = "SI_LOG_JSON",
+        hide_env_values = true
+    )]
+    pub(crate) log_json: bool,
+
     /// NATS connection URL [example: demo.nats.io]
     #[arg(long)]
     pub(crate) nats_url: Option<String>,

bin/council/src/main.rs

@@ -30,6 +30,11 @@ async fn async_main() -> Result<()> {
         let config = TelemetryConfig::builder()
             .force_color(args.force_color.then_some(true))
             .no_color(args.no_color.then_some(true))
+            .console_log_format(
+                args.log_json
+                    .then_some(ConsoleLogFormat::Json)
+                    .unwrap_or_default(),
+            )
             .service_name("council")
             .service_namespace("si")
             .log_env_var_prefix("SI")

bin/cyclone/src/args.rs

@@ -48,6 +48,17 @@ pub(crate) struct Args {
     )]
     pub(crate) force_color: bool,
 
+    /// Prints telemetry logging as JSON lines.
+    ///
+    /// For more details, visit: <https://jsonlines.org/>.
+    #[arg(
+        long = "log-json",
+        default_value = "false",
+        env = "SI_LOG_JSON",
+        hide_env_values = true
+    )]
+    pub(crate) log_json: bool,
+
     /// Binds service to a socket address [example: 0.0.0.0:5157]
     #[arg(long, group = "bind")]
     pub(crate) bind_addr: Option<SocketAddr>,

bin/cyclone/src/main.rs

@@ -24,6 +24,11 @@ async fn main() -> Result<()> {
         let config = TelemetryConfig::builder()
             .force_color(args.force_color.then_some(true))
             .no_color(args.no_color.then_some(true))
+            .console_log_format(
+                args.log_json
+                    .then_some(ConsoleLogFormat::Json)
+                    .unwrap_or_default(),
+            )
             .service_name("cyclone")
             .service_namespace("si")
             .log_env_var_prefix("SI")

bin/module-index/src/args.rs

@@ -45,6 +45,17 @@ pub(crate) struct Args {
     )]
     pub(crate) force_color: bool,
 
+    /// Prints telemetry logging as JSON lines.
+    ///
+    /// For more details, visit: <https://jsonlines.org/>.
+    #[arg(
+        long = "log-json",
+        default_value = "false",
+        env = "SI_LOG_JSON",
+        hide_env_values = true
+    )]
+    pub(crate) log_json: bool,
+
     /// PostgreSQL connection pool dbname [example: myapp]
     #[arg(long, env)]
     pub(crate) pg_dbname: Option<String>,

bin/module-index/src/main.rs

@@ -30,6 +30,11 @@ async fn async_main() -> Result<()> {
         let config = TelemetryConfig::builder()
             .force_color(args.force_color.then_some(true))
             .no_color(args.no_color.then_some(true))
+            .console_log_format(
+                args.log_json
+                    .then_some(ConsoleLogFormat::Json)
+                    .unwrap_or_default(),
+            )
             .service_name("module-index")
             .service_namespace("si")
             .log_env_var_prefix("SI")

bin/pinga/src/args.rs

@@ -45,6 +45,17 @@ pub(crate) struct Args {
     )]
     pub(crate) force_color: bool,
 
+    /// Prints telemetry logging as JSON lines.
+    ///
+    /// For more details, visit: <https://jsonlines.org/>.
+    #[arg(
+        long = "log-json",
+        default_value = "false",
+        env = "SI_LOG_JSON",
+        hide_env_values = true
+    )]
+    pub(crate) log_json: bool,
+
     /// PostgreSQL connection pool dbname [example: myapp]
     #[arg(long)]
     pub(crate) pg_dbname: Option<String>,

bin/pinga/src/main.rs

@@ -30,6 +30,11 @@ async fn async_main() -> Result<()> {
         let config = TelemetryConfig::builder()
             .force_color(args.force_color.then_some(true))
             .no_color(args.no_color.then_some(true))
+            .console_log_format(
+                args.log_json
+                    .then_some(ConsoleLogFormat::Json)
+                    .unwrap_or_default(),
+            )
             .service_name("pinga")
             .service_namespace("si")
             .log_env_var_prefix("SI")

bin/sdf/src/args.rs

@@ -47,6 +47,17 @@ pub(crate) struct Args {
     )]
     pub(crate) force_color: bool,
 
+    /// Prints telemetry logging as JSON lines.
+    ///
+    /// For more details, visit: <https://jsonlines.org/>.
+    #[arg(
+        long = "log-json",
+        default_value = "false",
+        env = "SI_LOG_JSON",
+        hide_env_values = true
+    )]
+    pub(crate) log_json: bool,
+
     /// PostgreSQL connection pool dbname [example: myapp]
     #[arg(long)]
     pub(crate) pg_dbname: Option<String>,

bin/sdf/src/main.rs

@@ -41,6 +41,11 @@ async fn async_main() -> Result<()> {
         let config = TelemetryConfig::builder()
             .force_color(args.force_color.then_some(true))
             .no_color(args.no_color.then_some(true))
+            .console_log_format(
+                args.log_json
+                    .then_some(ConsoleLogFormat::Json)
+                    .unwrap_or_default(),
+            )
             .service_name("sdf")
             .service_namespace("si")
             .log_env_var_prefix("SI")

bin/veritech/src/args.rs

@@ -41,6 +41,17 @@ pub(crate) struct Args {
     )]
     pub(crate) force_color: bool,
 
+    /// Prints telemetry logging as JSON lines.
+    ///
+    /// For more details, visit: <https://jsonlines.org/>.
+    #[arg(
+        long = "log-json",
+        default_value = "false",
+        env = "SI_LOG_JSON",
+        hide_env_values = true
+    )]
+    pub(crate) log_json: bool,
+
     /// NATS connection URL [example: 0.0.0.0:4222]
     #[arg(long, short = 'u')]
     pub(crate) nats_url: Option<String>,

bin/veritech/src/main.rs

@@ -16,6 +16,11 @@ async fn main() -> Result<()> {
         let config = TelemetryConfig::builder()
             .force_color(args.force_color.then_some(true))
             .no_color(args.no_color.then_some(true))
+            .console_log_format(
+                args.log_json
+                    .then_some(ConsoleLogFormat::Json)
+                    .unwrap_or_default(),
+            )
             .service_name("veritech")
             .service_namespace("si")
             .log_env_var_prefix("SI")

lib/telemetry-application-rs/src/lib.rs

@@ -52,7 +52,7 @@ pub use telemetry::tracing;
 pub use telemetry::{ApplicationTelemetryClient, TelemetryClient};
 
 pub mod prelude {
-    pub use super::TelemetryConfig;
+    pub use super::{ConsoleLogFormat, TelemetryConfig};
     pub use telemetry::prelude::*;
     pub use telemetry::{ApplicationTelemetryClient, TelemetryClient};
 }
@@ -128,6 +128,9 @@ pub struct TelemetryConfig {
     #[builder(setter(into), default = "None")]
     force_color: Option<bool>,
 
+    #[builder(setter(into), default)]
+    console_log_format: ConsoleLogFormat,
+
     #[builder(default = "true")]
     signal_handlers: bool,
 }
@@ -224,6 +227,8 @@ pub fn init(
     let (subscriber, handles) = tracing_subscriber(&config, &tracing_level, span_events_fmt)?;
     subscriber.try_init()?;
 
+    debug!(?config, "telemetry configuration");
+
     let (client, guard) = create_client(config, tracing_level, handles, tracker, shutdown_token)?;
 
     Ok((client, guard))
@@ -306,10 +311,20 @@ fn tracing_subscriber(
     let directives = TracingDirectives::from(tracing_level);
 
     let (console_log_layer, console_log_filter_reload) = {
-        let layer = tracing_subscriber::fmt::layer()
-            .with_thread_ids(true)
-            .with_ansi(should_add_ansi(config))
-            .with_span_events(span_events_fmt);
+        let layer: Box<dyn Layer<Registry> + Send + Sync> = match config.console_log_format {
+            ConsoleLogFormat::Json => Box::new(
+                tracing_subscriber::fmt::layer()
+                    .json()
+                    .with_thread_ids(true)
+                    .with_span_events(span_events_fmt),
+            ),
+            ConsoleLogFormat::Text => Box::new(
+                tracing_subscriber::fmt::layer()
+                    .with_thread_ids(true)
+                    .with_ansi(should_add_ansi(config))
+                    .with_span_events(span_events_fmt),
+            ),
+        };
 
         let env_filter = EnvFilter::try_new(directives.as_str())?;
         let (filter, handle) = reload::Layer::new(env_filter);
@@ -416,6 +431,19 @@ fn should_add_ansi(config: &TelemetryConfig) -> bool {
     }
 }
 
+#[remain::sorted]
+#[derive(Copy, Clone, Debug)]
+pub enum ConsoleLogFormat {
+    Json,
+    Text,
+}
+
+impl Default for ConsoleLogFormat {
+    fn default() -> Self {
+        Self::Text
+    }
+}
+
 #[must_use]
 pub struct TelemetryShutdownGuard {
     update_telemetry_tx: mpsc::UnboundedSender<TelemetryCommand>,