organic/src/init_tracing.rs

#[cfg(feature = "tracing")]
use opentelemetry_otlp::WithExportConfig;
#[cfg(feature = "tracing")]
use tracing_subscriber::prelude::__tracing_subscriber_SubscriberExt;
#[cfg(feature = "tracing")]
use tracing_subscriber::util::SubscriberInitExt;

const SERVICE_NAME: &'static str = "organic";

// Despite the obvious verbosity that fully-qualifying everything causes, in these functions I am fully-qualifying everything relating to tracing. This is because the tracing feature involves multiple libraries working together and so I think it is beneficial to see which libraries contribute which bits.

#[cfg(feature = "tracing")]
pub fn init_telemetry() -> Result<(), Box<dyn std::error::Error>> {
    // by default it will hit http://localhost:4317 with a gRPC payload
    // TODO: I think the endpoint can be controlled by the OTEL_EXPORTER_OTLP_TRACES_ENDPOINT env variable instead of hard-coded into this code base. Regardless, I am the only developer right now so I am not too concerned.
    let exporter = opentelemetry_otlp::new_exporter()
        .tonic()
        // Using "localhost" is broken inside the docker container when tracing
        .with_endpoint("http://127.0.0.1:4317/v1/traces");

    let tracer = opentelemetry_otlp::new_pipeline()
        .tracing()
        .with_exporter(exporter)
        .with_trace_config(opentelemetry::sdk::trace::config().with_resource(
            opentelemetry::sdk::Resource::new(vec![opentelemetry::KeyValue::new(
                opentelemetry_semantic_conventions::resource::SERVICE_NAME,
                SERVICE_NAME.to_string(),
            )]),
        ))
        // If I do install_batch then 1K+ spans will get orphaned off into their own trace and I get the error message "OpenTelemetry trace error occurred. cannot send message to batch processor as the channel is closed"
        //
        // If I do install_simple then it only creates 1 trace (which is good!) but my console gets spammed with this concerning log message that makes me think it might be dropping the extra spans on the floor: "OpenTelemetry trace error occurred. Exporter otlp encountered the following error(s): the grpc server returns error (Unknown error): , detailed error message: Service was not ready: transport error"
        //
        // I suspect it is related to this bug: https://github.com/open-telemetry/opentelemetry-rust/issues/888
        //
        // .install_simple()
        .install_batch(opentelemetry::runtime::Tokio)
        .expect("Error: Failed to initialize the tracer.");

    let subscriber = tracing_subscriber::Registry::default();
    let level_filter_layer = tracing_subscriber::EnvFilter::try_from_default_env()
        .unwrap_or(tracing_subscriber::EnvFilter::new("WARN"));
    let tracing_layer = tracing_opentelemetry::layer().with_tracer(tracer);

    opentelemetry::global::set_text_map_propagator(
        opentelemetry::sdk::propagation::TraceContextPropagator::new(),
    );

    subscriber
        .with(level_filter_layer)
        .with(tracing_layer)
        .try_init()?;

    Ok(())
}

#[cfg(feature = "tracing")]
pub fn shutdown_telemetry() -> Result<(), Box<dyn std::error::Error>> {
    opentelemetry::global::shutdown_tracer_provider();
    Ok(())
}

#[cfg(not(feature = "tracing"))]
pub fn init_telemetry() -> Result<(), Box<dyn std::error::Error>> {
    Ok(())
}

#[cfg(not(feature = "tracing"))]
pub fn shutdown_telemetry() -> Result<(), Box<dyn std::error::Error>> {
    Ok(())
}