Merge pull request #503 from cbgbt/log-format-option

Enable configuration of log formatting

README.md

@@ -146,6 +146,17 @@ scheduler_cron_expression: "* * * * * * *"
 apiserver_internal_port: "8443"
 # API server internal address where the CRD version conversion webhook is served
 apiserver_service_port: "443"
+
+# Formatter for the logs emitted by brupop.
+# Options are:
+# * full - Human-readable, single-line logs
+# * compact - A variant of full optimized for shorter line lengths
+# * pretty - "Excessively pretty" logs optimized for human-readable terminal output.
+# * json - Newline-delimited JSON-formatted logs.
+logging_formatter: "pretty"
+# Whether or not to enable ANSI colors on log messages.
+# Makes the output "pretty" in terminals, but may add noise to web-based log utilities.
+logging_ansi_enabled: "true"
 ```
 
 #### Configure API server ports

agent/Cargo.toml

@@ -11,10 +11,7 @@ apiserver = { path = "../apiserver", version = "0.1.0", default-features = false
 
 dotenv = "0.15"
 futures = "0.3"
-opentelemetry = { version = "0.18", features = ["rt-tokio-current-thread"] }
 tracing = "0.1"
-tracing-subscriber = { version = "0.3", features = ["registry", "env-filter"] }
-tracing-opentelemetry = "0.18"
 
 # k8s-openapi must match the version required by kube and enable a k8s version feature
 k8s-openapi = { version = "0.18", default-features = false, features = ["v1_24"] }

agent/src/main.rs

@@ -12,11 +12,10 @@ use kube::{
 };
 use models::constants::{AGENT_TOKEN_PATH, AGENT_TOKEN_PROJECTION_MOUNT_PATH};
 use models::node::{brs_name_from_node_name, BottlerocketShadow};
+use models::telemetry;
 
-use opentelemetry::sdk::propagation::TraceContextPropagator;
 use snafu::{OptionExt, ResultExt};
 use tracing::{event, Level};
-use tracing_subscriber::{fmt, layer::SubscriberExt, EnvFilter, Registry};
 
 use std::convert::TryFrom;
 use std::env;
@@ -40,7 +39,7 @@ async fn main() {
 }
 
 async fn run_agent() -> Result<()> {
-    init_telemetry()?;
+    telemetry::init_telemetry_from_env().context(agent_error::TelemetryInitSnafu)?;
 
     let incluster_config = kube::Config::incluster_dns().context(agent_error::ConfigCreateSnafu)?;
     let namespace = incluster_config.default_namespace.to_string();
@@ -119,23 +118,9 @@ async fn run_agent() -> Result<()> {
     Ok(())
 }
 
-/// Initializes global tracing and telemetry state for the agent.
-pub fn init_telemetry() -> Result<()> {
-    opentelemetry::global::set_text_map_propagator(TraceContextPropagator::new());
-
-    let env_filter = EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info"));
-    let stdio_formatting_layer = fmt::layer().pretty();
-    let subscriber = Registry::default()
-        .with(env_filter)
-        .with(stdio_formatting_layer);
-    tracing::subscriber::set_global_default(subscriber)
-        .context(agent_error::TracingConfigurationSnafu)?;
-
-    Ok(())
-}
-
 pub mod agent_error {
     use agent::agentclient::agentclient_error;
+    use models::telemetry;
     use snafu::Snafu;
 
     #[derive(Debug, Snafu)]
@@ -168,6 +153,11 @@ pub mod agent_error {
         #[snafu(display("The Kubernetes WATCH on {} objects has failed.", object))]
         KubernetesWatcherFailed { object: String },
 
+        #[snafu(display("Error configuring telemetry: '{}'", source))]
+        TelemetryInit {
+            source: telemetry::TelemetryConfigError,
+        },
+
         #[snafu(display("Error configuring tracing: '{}'", source))]
         TracingConfiguration {
             source: tracing::subscriber::SetGlobalDefaultError,

apiserver/Cargo.toml

@@ -25,8 +25,6 @@ opentelemetry = { version = "0.18", features = ["rt-tokio-current-thread"]}
 opentelemetry-prometheus = "0.11"
 tracing = "0.1"
 tracing-actix-web = "0.7"
-tracing-subscriber = { version = "0.3", features = ["registry", "env-filter"] }
-tracing-opentelemetry = "0.18"
 
 # k8s-openapi must match the version required by kube and enable a k8s version feature
 k8s-openapi = { version = "0.18", default-features = false, features = ["v1_24"] }

apiserver/src/main.rs

@@ -1,7 +1,7 @@
 use apiserver::api::{self, APIServerSettings};
-use apiserver::telemetry::init_telemetry;
 use apiserver_error::{StartServerSnafu, StartTelemetrySnafu};
 use models::node::K8SBottlerocketShadowClient;
+use models::telemetry;
 use tracing::{event, Level};
 
 use opentelemetry::sdk::export::metrics::aggregation;
@@ -33,7 +33,7 @@ async fn main() {
 }
 
 async fn run_server() -> Result<(), apiserver_error::Error> {
-    init_telemetry().context(StartTelemetrySnafu)?;
+    telemetry::init_telemetry_from_env().context(StartTelemetrySnafu)?;
     let controller = controllers::basic(
         processors::factory(
             selectors::simple::histogram([1.0, 2.0, 5.0, 10.0, 20.0, 50.0]),
@@ -102,7 +102,7 @@ pub mod apiserver_error {
 
         #[snafu(display("Unable to start API server telemetry: '{}'", source))]
         StartTelemetry {
-            source: apiserver::telemetry::telemetry_error::Error,
+            source: models::telemetry::TelemetryConfigError,
         },
 
         #[snafu(display("Unable to start API server: '{}'", source))]

apiserver/src/telemetry.rs

@@ -4,11 +4,8 @@ use crate::constants::HEADER_BRUPOP_NODE_NAME;
 use actix_web::body::MessageBody;
 use actix_web::dev::{ServiceRequest, ServiceResponse};
 use lazy_static::lazy_static;
-use opentelemetry::sdk::propagation::TraceContextPropagator;
-use snafu::ResultExt;
 use tracing::Span;
 use tracing_actix_web::{DefaultRootSpanBuilder, RootSpanBuilder};
-use tracing_subscriber::{fmt, layer::SubscriberExt, EnvFilter, Registry};
 
 use std::collections::HashSet;
 
@@ -51,31 +48,3 @@ impl RootSpanBuilder for BrupopApiserverRootSpanBuilder {
         DefaultRootSpanBuilder::on_request_end(span, response);
     }
 }
-
-/// Initializes global tracing and telemetry state for the apiserver.
-pub fn init_telemetry() -> Result<(), telemetry_error::Error> {
-    opentelemetry::global::set_text_map_propagator(TraceContextPropagator::new());
-
-    let env_filter = EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info"));
-    let stdio_formatting_layer = fmt::layer().pretty();
-    let subscriber = Registry::default()
-        .with(env_filter)
-        .with(stdio_formatting_layer);
-    tracing::subscriber::set_global_default(subscriber)
-        .context(telemetry_error::TracingConfigurationSnafu)?;
-
-    Ok(())
-}
-
-pub mod telemetry_error {
-    use snafu::Snafu;
-
-    #[derive(Debug, Snafu)]
-    #[snafu(visibility(pub))]
-    pub enum Error {
-        #[snafu(display("Error configuring tracing: '{}'", source))]
-        TracingConfiguration {
-            source: tracing::subscriber::SetGlobalDefaultError,
-        },
-    }
-}

controller/Cargo.toml

@@ -15,18 +15,17 @@ lazy_static = "1.2"
 maplit = "1.0"
 regex = "1.1"
 semver = "1.0"
+serde = "1"
+serde_plain = "1"
 # k8s-openapi must match the version required by kube and enable a k8s version feature
 k8s-openapi = { version = "0.18", default-features = false, features = ["v1_24"] }
 kube = { version = "0.84", default-features = false, features = [ "derive", "runtime", "rustls-tls" ] }
 models = { path = "../models", version = "0.1.0" }
 opentelemetry = { version = "0.18", features = ["rt-tokio-current-thread"] }
 opentelemetry-prometheus = "0.11"
 prometheus = "0.13.0"
-serde_plain = "1.0.1"
 
 snafu = "0.7"
 tokio = { version = "1", features = ["macros", "rt-multi-thread", "time"] }
 tracing = "0.1"
-tracing-opentelemetry = "0.18"
-tracing-subscriber = { version = "0.3", features = ["registry", "env-filter"] }
 validator = { version = "0.16", features = ["derive"] }

controller/src/main.rs

@@ -4,6 +4,7 @@ use controller::{telemetry::vending_metrics, BrupopController};
 use models::{
     constants::CONTROLLER_INTERNAL_PORT,
     node::{BottlerocketShadow, K8SBottlerocketShadowClient},
+    telemetry,
 };
 
 use actix_web::{web::Data, App, HttpServer};
@@ -22,19 +23,15 @@ use kube::{
 
 use opentelemetry::sdk::export::metrics::aggregation;
 use opentelemetry::sdk::metrics::{controllers, processors, selectors};
-use opentelemetry::sdk::propagation::TraceContextPropagator;
 use snafu::ResultExt;
 use tracing::{event, Level};
-use tracing_subscriber::{fmt, layer::SubscriberExt, EnvFilter, Registry};
-
-const DEFAULT_TRACE_LEVEL: &str = "info";
 
 /// The module-wide result type.
 type Result<T> = std::result::Result<T, controller_error::Error>;
 
 #[actix_web::main]
 async fn main() -> Result<()> {
-    init_telemetry()?;
+    telemetry::init_telemetry_from_env().context(controller_error::TelemetryInitSnafu)?;
 
     let incluster_config =
         kube::Config::incluster_dns().context(controller_error::ConfigCreateSnafu)?;
@@ -135,23 +132,9 @@ async fn main() -> Result<()> {
     Ok(())
 }
 
-fn init_telemetry() -> Result<()> {
-    opentelemetry::global::set_text_map_propagator(TraceContextPropagator::new());
-
-    let env_filter =
-        EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new(DEFAULT_TRACE_LEVEL));
-    let stdio_formatting_layer = fmt::layer().pretty();
-    let subscriber = Registry::default()
-        .with(env_filter)
-        .with(stdio_formatting_layer);
-    tracing::subscriber::set_global_default(subscriber)
-        .context(controller_error::TracingConfigurationSnafu)?;
-
-    Ok(())
-}
-
 pub mod controller_error {
     use controller::controllerclient_error;
+    use models::telemetry;
     use snafu::Snafu;
 
     #[derive(Debug, Snafu)]
@@ -176,11 +159,6 @@ pub mod controller_error {
         #[snafu(display("The Kubernetes WATCH on {} objects has failed.", object))]
         KubernetesWatcherFailed { object: String },
 
-        #[snafu(display("Error configuring tracing: '{}'", source))]
-        TracingConfiguration {
-            source: tracing::subscriber::SetGlobalDefaultError,
-        },
-
         #[snafu(display("Error determining the cluster server address: '{}'", source))]
         MissingClusterIPFamily { source: std::env::VarError },
 
@@ -189,5 +167,10 @@ pub mod controller_error {
 
         #[snafu(display("Failed to run prometheus on controller"))]
         PrometheusError,
+
+        #[snafu(display("Error configuring telemetry: '{}'", source))]
+        TelemetryInit {
+            source: telemetry::TelemetryConfigError,
+        },
     }
 }

deploy/charts/bottlerocket-update-operator/README.md

@@ -135,4 +135,15 @@ scheduler_cron_expression: "* * * * * * *"
 apiserver_internal_port: "8443"
 # API server internal address where the CRD version conversion webhook is served
 apiserver_service_port: "443"
+
+# Formatter for the logs emitted by brupop.
+# Options are:
+# * full - Human-readable, single-line logs
+# * compact - A variant of full optimized for shorter line lengths
+# * pretty - "Excessively pretty" logs optimized for human-readable terminal output.
+# * json - Newline-delimited JSON-formatted logs.
+logging_formatter: "pretty"
+# Whether or not to enable ANSI colors on log messages.
+# Makes the output "pretty" in terminals, but may add noise to web-based log utilities.
+logging_ansi_enabled: "true"
 ```

deploy/charts/bottlerocket-update-operator/templates/agent-daemonset.yaml

@@ -49,6 +49,10 @@ spec:
               value: "{{ .Values.exclude_from_lb_wait_time_in_sec }}"
             - name: APISERVER_SERVICE_PORT
               value: "{{ .Values.apiserver_service_port }}"
+            - name: LOGGING_FORMATTER
+              value: "{{ .Values.logging_formatter }}"
+            - name: LOGGING_ANSI_ENABLED
+              value: "{{ .Values.logging_ansi_enabled }}"
           image: {{ .Values.image }}
           name: brupop
           resources: