Replace manual Sentry trace IDs with OpenTelemetry span propagation
Add opentelemetry, opentelemetry_sdk, sentry-opentelemetry, and tracing-opentelemetry to the dependency stack. The tracing subscriber now includes an OTEL layer wired to SentrySpanProcessor, so tracing spans become Sentry transactions automatically. Cross-process trace continuity: quire-server creates an info_span for each CI run and extracts the W3C traceparent header via current_traceparent(), which travels to quire-ci through the bootstrap API (new `traceparent` column, migration 0006). quire-ci calls attach_traceparent() before entering its own root span, making it a child of the orchestrator's span in Sentry's trace view. Removes the manual sentry::protocol::TraceId/SpanId generation and the hand-rolled TraceContext scope setup in both binaries. https://claude.ai/code/session_01Tbgz29e8A9KS4Bh94skkFX
diff --git a/Cargo.lock b/Cargo.lock
index affd8d3..89a8257 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1152,12 +1152,34 @@ version = "0.3.32"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d"
+[[package]]
+name = "futures-executor"
+version = "0.3.32"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "baf29c38818342a3b26b5b923639e7b1f4a61fc5e76102d4b1981c6dc7a7579d"
+dependencies = [
+ "futures-core",
+ "futures-task",
+ "futures-util",
+]
+
[[package]]
name = "futures-io"
version = "0.3.32"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718"
+[[package]]
+name = "futures-macro"
+version = "0.3.32"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
[[package]]
name = "futures-sink"
version = "0.3.32"
@@ -1178,6 +1200,7 @@ checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6"
dependencies = [
"futures-core",
"futures-io",
+ "futures-macro",
"futures-sink",
"futures-task",
"memchr",
@@ -2251,6 +2274,38 @@ version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe"
+[[package]]
+name = "opentelemetry"
+version = "0.29.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9e87237e2775f74896f9ad219d26a2081751187eb7c9f5c58dde20a23b95d16c"
+dependencies = [
+ "futures-core",
+ "futures-sink",
+ "js-sys",
+ "pin-project-lite",
+ "thiserror",
+ "tracing",
+]
+
+[[package]]
+name = "opentelemetry_sdk"
+version = "0.29.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "afdefb21d1d47394abc1ba6c57363ab141be19e27cc70d0e422b7f303e4d290b"
+dependencies = [
+ "futures-channel",
+ "futures-executor",
+ "futures-util",
+ "glob",
+ "opentelemetry",
+ "percent-encoding",
+ "rand",
+ "serde_json",
+ "thiserror",
+ "tracing",
+]
+
[[package]]
name = "ordered-float"
version = "2.10.1"
@@ -2519,16 +2574,20 @@ dependencies = [
"jiff",
"miette",
"mlua",
+ "opentelemetry",
+ "opentelemetry_sdk",
"petgraph",
"rand",
"regex",
"sentry",
+ "sentry-opentelemetry",
"sentry-tracing",
"serde",
"serde_json",
"tempfile",
"thiserror",
"tracing",
+ "tracing-opentelemetry",
"tracing-subscriber",
]
@@ -2990,9 +3049,9 @@ dependencies = [
[[package]]
name = "sentry-core"
-version = "0.48.0"
+version = "0.48.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "56de6f8c10ed1b74543b9654b99d3d9a2d876bd5996f3ecd60afdc30ef40ad0e"
+checksum = "545dc562b6758d646ac19e1407f4ebc26d452111386743e03323464bc48bb2e0"
dependencies = [
"rand",
"sentry-types",
@@ -3011,6 +3070,17 @@ dependencies = [
"sentry-core",
]
+[[package]]
+name = "sentry-opentelemetry"
+version = "0.48.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "df2d807edab875bc9d133de3106a295796b6c9692a149799fb5157108f5636d5"
+dependencies = [
+ "opentelemetry",
+ "opentelemetry_sdk",
+ "sentry-core",
+]
+
[[package]]
name = "sentry-panic"
version = "0.48.0"
@@ -3036,9 +3106,9 @@ dependencies = [
[[package]]
name = "sentry-types"
-version = "0.48.0"
+version = "0.48.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "00c69667ff14f47aad798e6be2ad8409e350b3ab0b8d72b338b462ed35f7bcc4"
+checksum = "041359745a44dd2e14fe21b7510fe7ca8b5beffce6636a0b52e5bc7d5f736887"
dependencies = [
"debugid",
"hex",
@@ -3612,6 +3682,24 @@ dependencies = [
"tracing-core",
]
+[[package]]
+name = "tracing-opentelemetry"
+version = "0.30.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fd8e764bd6f5813fd8bebc3117875190c5b0415be8f7f8059bffb6ecd979c444"
+dependencies = [
+ "js-sys",
+ "once_cell",
+ "opentelemetry",
+ "opentelemetry_sdk",
+ "smallvec 1.15.1",
+ "tracing",
+ "tracing-core",
+ "tracing-log",
+ "tracing-subscriber",
+ "web-time",
+]
+
[[package]]
name = "tracing-serde"
version = "0.2.0"
diff --git a/Cargo.toml b/Cargo.toml
index 0713734..bf45c81 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -13,6 +13,10 @@ petgraph = "*"
regex = "*"
sentry = { version = "*", features = ["backtrace", "contexts", "debug-images", "panic", "release-health", "reqwest", "rustls", "tokio"], default-features = false }
sentry-tracing = "*"
+opentelemetry = "0.29.1"
+opentelemetry_sdk = "0.29.0"
+sentry-opentelemetry = "0.48.2"
+tracing-opentelemetry = "0.30.0"
serde = { version = "*", features = ["derive"] }
serde_json = "*"
tempfile = "*"
diff --git a/quire-ci/src/main.rs b/quire-ci/src/main.rs
index 89faa0a..21b4c51 100644
--- a/quire-ci/src/main.rs
+++ b/quire-ci/src/main.rs
@@ -246,7 +246,7 @@ impl RunClient {
bootstrap.git_dir,
bootstrap.meta,
SentryContext {
- trace_id: bootstrap.sentry_trace_id,
+ traceparent: bootstrap.traceparent,
repo: Some(bootstrap.repo),
run_id: Some(bootstrap.run_id),
},
@@ -359,7 +359,16 @@ fn main() -> Result<()> {
// for them. The layer stays installed in case future ops
// errors want to register types.
let miette_layer = MietteLayer::new();
- telemetry::init_tracing(miette_layer, FmtMode::Plain)?;
+ // _tracing_guard must be declared AFTER _sentry so it drops
+ // BEFORE _sentry — OTEL provider flushes spans to Sentry SDK
+ // before the Sentry client flushes to the server.
+ let _tracing_guard = telemetry::init_tracing(miette_layer, FmtMode::Plain)?;
+
+ // Attach the remote trace context so quire-ci spans appear
+ // as children of the orchestrator's span in Sentry.
+ let _cx_guard = sentry_ctx.traceparent.as_deref()
+ .map(|tp| telemetry::attach_traceparent(tp));
+ let _run_span = tracing::info_span!("quire.ci.run", sha = %meta.sha, r#ref = %meta.r#ref).entered();
let registry = SecretRegistry::new(move |name| client.fetch_secret(name));
@@ -375,7 +384,7 @@ fn main() -> Result<()> {
/// these tag observability only.
#[derive(Default)]
struct SentryContext {
- trace_id: Option<String>,
+ traceparent: Option<String>,
repo: Option<String>,
run_id: Option<String>,
}
@@ -398,30 +407,6 @@ fn init_sentry(dsn: &str, meta: &RunMeta, ctx: &SentryContext) -> sentry::Client
if let Some(run_id) = &ctx.run_id {
scope.set_tag("run_id", run_id);
}
- if let Some(tid) = ctx.trace_id.as_deref() {
- match tid.parse::<sentry::protocol::TraceId>() {
- Ok(trace_id) => {
- scope.set_context(
- "trace",
- sentry::protocol::Context::Trace(Box::new(
- sentry::protocol::TraceContext {
- trace_id,
- span_id: sentry::protocol::SpanId::default(),
- op: Some("quire.ci.run".into()),
- ..Default::default()
- },
- )),
- );
- }
- Err(e) => {
- tracing::warn!(
- trace_id = %tid,
- error = %e,
- "malformed trace_id in bootstrap; quire-ci events won't link to orchestrator",
- );
- }
- }
- }
});
guard
}
diff --git a/quire-core/Cargo.toml b/quire-core/Cargo.toml
index f48944a..c1774c7 100644
--- a/quire-core/Cargo.toml
+++ b/quire-core/Cargo.toml
@@ -12,8 +12,12 @@ mlua = { workspace = true }
petgraph = { workspace = true }
rand = "*"
regex = { workspace = true }
+opentelemetry = { workspace = true }
+opentelemetry_sdk = { workspace = true }
sentry = { workspace = true }
+sentry-opentelemetry = { workspace = true }
sentry-tracing = { workspace = true }
+tracing-opentelemetry = { workspace = true }
serde = { workspace = true }
serde_json = { workspace = true }
thiserror = { workspace = true }
diff --git a/quire-core/src/ci/bootstrap.rs b/quire-core/src/ci/bootstrap.rs
index 0d51ccb..56bbc81 100644
--- a/quire-core/src/ci/bootstrap.rs
+++ b/quire-core/src/ci/bootstrap.rs
@@ -28,10 +28,10 @@ pub struct Bootstrap {
/// The server-assigned run id (UUIDv7, the `runs.id` PK).
/// quire-ci tags Sentry events with it.
pub run_id: String,
- /// Sentry trace id for the orchestrator's span, present only when
+ /// W3C traceparent header value for the orchestrator's span, present only when
/// the global config sets a DSN. Allows quire-ci to attach its
- /// events to the same trace. The DSN itself travels via the
+ /// events to the same trace via OTEL context propagation. The DSN itself travels via the
/// `QUIRE__SENTRY_DSN` environment variable.
#[serde(default, skip_serializing_if = "Option::is_none")]
- pub sentry_trace_id: Option<String>,
+ pub traceparent: Option<String>,
}
diff --git a/quire-core/src/telemetry.rs b/quire-core/src/telemetry.rs
index 0068f66..7113cf4 100644
--- a/quire-core/src/telemetry.rs
+++ b/quire-core/src/telemetry.rs
@@ -187,14 +187,51 @@ pub fn sentry_client_options(release: &'static str) -> sentry::ClientOptions {
}
}
+/// Returned by `init_tracing`; shuts down the OTEL TracerProvider on drop.
+pub struct TracingGuard {
+ provider: opentelemetry_sdk::trace::SdkTracerProvider,
+}
+
+impl Drop for TracingGuard {
+ fn drop(&mut self) {
+ let _ = self.provider.shutdown();
+ }
+}
+
+/// Opaque guard that restores the previous OTEL context on drop.
+#[allow(dead_code)]
+pub struct TraceparentGuard(opentelemetry::ContextGuard);
+
+/// Extract the W3C traceparent for the currently active tracing span.
+/// Returns None when no OTEL span is active (e.g. no DSN, not yet entered a span).
+pub fn current_traceparent() -> Option<String> {
+ use opentelemetry::propagation::TextMapPropagator;
+ let propagator = opentelemetry_sdk::propagation::TraceContextPropagator::new();
+ let cx = opentelemetry::Context::current();
+ let mut carrier = std::collections::HashMap::new();
+ propagator.inject_context(&cx, &mut carrier);
+ carrier.remove("traceparent")
+}
+
+/// Inject a W3C traceparent into the current thread's OTEL context.
+/// The returned guard restores the previous context on drop.
+pub fn attach_traceparent(traceparent: &str) -> TraceparentGuard {
+ use opentelemetry::propagation::TextMapPropagator;
+ let propagator = opentelemetry_sdk::propagation::TraceContextPropagator::new();
+ let mut carrier = std::collections::HashMap::new();
+ carrier.insert("traceparent".to_string(), traceparent.to_string());
+ let cx = propagator.extract(&carrier);
+ TraceparentGuard(cx.attach())
+}
+
/// Initialize the global tracing subscriber with `QUIRE_LOG`-driven filtering,
-/// a stderr fmt layer per `fmt_mode`, the `sentry-tracing` bridge, and the
-/// supplied [`MietteLayer`].
+/// a stderr fmt layer per `fmt_mode`, the `sentry-tracing` bridge, the
+/// supplied [`MietteLayer`], and an OTEL tracing layer wired to Sentry.
///
/// Layer ordering is baked in: `miette_layer` registers before
/// `sentry_tracing::layer()` so its thread-local is populated when
/// sentry-tracing's `on_event` calls `capture_event`.
-pub fn init_tracing(miette_layer: MietteLayer, fmt_mode: FmtMode) -> miette::Result<()> {
+pub fn init_tracing(miette_layer: MietteLayer, fmt_mode: FmtMode) -> miette::Result<TracingGuard> {
let filter = EnvFilter::builder()
.with_env_var("QUIRE_LOG")
.from_env()
@@ -212,14 +249,22 @@ pub fn init_tracing(miette_layer: MietteLayer, fmt_mode: FmtMode) -> miette::Res
}
};
+ use opentelemetry::trace::TracerProvider as _;
+ let provider = opentelemetry_sdk::trace::SdkTracerProvider::builder()
+ .with_span_processor(sentry_opentelemetry::SentrySpanProcessor::new())
+ .build();
+ opentelemetry::global::set_tracer_provider(provider.clone());
+ let tracer = provider.tracer("quire");
+
tracing_subscriber::registry()
.with(miette_layer)
.with(sentry_tracing::layer())
+ .with(tracing_opentelemetry::layer().with_tracer(tracer))
.with(fmt_layer)
.with(filter)
.init();
- Ok(())
+ Ok(TracingGuard { provider })
}
#[cfg(test)]
diff --git a/quire-server/migrations/0006_traceparent.sql b/quire-server/migrations/0006_traceparent.sql
new file mode 100644
index 0000000..2989404
--- /dev/null
+++ b/quire-server/migrations/0006_traceparent.sql
@@ -0,0 +1 @@
+ALTER TABLE runs ADD COLUMN traceparent TEXT;
diff --git a/quire-server/src/ci/mod.rs b/quire-server/src/ci/mod.rs
index 6d70d57..cde1875 100644
--- a/quire-server/src/ci/mod.rs
+++ b/quire-server/src/ci/mod.rs
@@ -11,6 +11,7 @@ pub use quire_core::ci::pipeline::{
pub use quire_core::ci::run::ApiSession;
pub use quire_core::ci::run::RunMeta;
pub use quire_core::ci::{pipeline, registration, runtime};
+use quire_core::telemetry;
pub use run::{Executor, Run, RunState, Runs, materialize_workspace, reconcile_orphans};
/// A resolved commit reference.
@@ -158,16 +159,7 @@ pub fn trigger(quire: &crate::Quire, event: &PushEvent) {
};
for push_ref in event.updated_refs() {
- // One trace per push_ref. The trace context is set on the
- // orchestrator's scope for the duration of this iteration and
- // propagated to quire-ci through the dispatch file, so a
- // quire-ci panic and the orchestrator-side "CI trigger
- // failed" event end up on the same trace in Sentry. DSN and
- // trace_id travel together — no DSN, no handoff, no trace
- // tagging is observable.
- let trace_id = sentry::protocol::TraceId::default();
- let span_id = sentry::protocol::SpanId::default();
- run_ref(&ctx, event.pushed_at, push_ref, trace_id, span_id);
+ run_ref(&ctx, event.pushed_at, push_ref);
}
}
@@ -176,23 +168,19 @@ fn run_ref(
ctx: &TriggerContext<'_>,
pushed_at: jiff::Timestamp,
push_ref: &PushRef,
- trace_id: sentry::protocol::TraceId,
- span_id: sentry::protocol::SpanId,
) {
let session = ApiSession::new(ctx.port);
- let sentry_trace_id = ctx.sentry_dsn.as_ref().map(|_| trace_id.to_string());
+
+ let span = tracing::info_span!("quire.ci.run", repo = %ctx.event_repo);
+ let _guard = span.enter();
+
+ let traceparent = ctx.sentry_dsn.as_ref()
+ .and_then(|_| telemetry::current_traceparent());
+
sentry::with_scope(
|scope| {
scope.set_tag("repo", ctx.event_repo);
- scope.set_context(
- "trace",
- sentry::protocol::Context::Trace(Box::new(sentry::protocol::TraceContext {
- trace_id,
- span_id,
- op: Some("quire.ci.run".into()),
- ..Default::default()
- })),
- );
+ // TraceContext is now managed by sentry-opentelemetry
},
|| {
if let Err(e) = run_ref_inner(
@@ -200,7 +188,7 @@ fn run_ref(
pushed_at,
push_ref,
&session,
- sentry_trace_id.as_deref(),
+ traceparent.as_deref(),
ctx.sentry_dsn.as_deref(),
) {
tracing::error!(
@@ -220,7 +208,7 @@ fn run_ref_inner(
pushed_at: jiff::Timestamp,
push_ref: &PushRef,
session: &ApiSession,
- sentry_trace_id: Option<&str>,
+ traceparent: Option<&str>,
sentry_dsn: Option<&str>,
) -> error::Result<()> {
let ci = ctx.repo.ci();
@@ -258,7 +246,7 @@ fn run_ref_inner(
run.execute(
&ctx.repo.path(),
&workspace,
- sentry_trace_id,
+ traceparent,
sentry_dsn,
Some(session),
)?;
diff --git a/quire-server/src/ci/run.rs b/quire-server/src/ci/run.rs
index 5adf3c8..ad281c6 100644
--- a/quire-server/src/ci/run.rs
+++ b/quire-server/src/ci/run.rs
@@ -284,7 +284,7 @@ impl Run {
mut self,
git_dir: &Path,
workspace: &Path,
- sentry_trace_id: Option<&str>,
+ traceparent: Option<&str>,
sentry_dsn: Option<&str>,
session: Option<&ApiSession>,
) -> Result<()> {
@@ -329,7 +329,7 @@ impl Run {
cmd.arg("--local").arg("--git-dir").arg(git_dir);
}
Some(s) => {
- self.store_bootstrap_data(git_dir, sentry_trace_id)?;
+ self.store_bootstrap_data(git_dir, traceparent)?;
cmd.env("QUIRE__SERVER_URL", &s.server_url);
cmd.env("QUIRE__RUN_TOKEN", &s.run_token);
}
@@ -476,7 +476,7 @@ impl Run {
/// Called by `execute` when the API transport is active, before spawning
/// quire-ci. quire-ci fetches this via `GET /api/runs/:id/bootstrap`
/// instead of reading a file.
- fn store_bootstrap_data(&self, git_dir: &Path, sentry_trace_id: Option<&str>) -> Result<()> {
+ fn store_bootstrap_data(&self, git_dir: &Path, traceparent: Option<&str>) -> Result<()> {
let git_dir_str = git_dir.to_str().ok_or_else(|| {
std::io::Error::new(
std::io::ErrorKind::InvalidData,
@@ -485,8 +485,8 @@ impl Run {
})?;
let db = crate::db::open(&self.db_path)?;
db.execute(
- "UPDATE runs SET git_dir = ?1, sentry_trace_id = ?2 WHERE id = ?3",
- rusqlite::params![git_dir_str, sentry_trace_id, &self.id],
+ "UPDATE runs SET git_dir = ?1, traceparent = ?2 WHERE id = ?3",
+ rusqlite::params![git_dir_str, traceparent, &self.id],
)?;
Ok(())
}
diff --git a/quire-server/src/db.rs b/quire-server/src/db.rs
index e699abc..c43d1f8 100644
--- a/quire-server/src/db.rs
+++ b/quire-server/src/db.rs
@@ -17,6 +17,7 @@ static MIGRATIONS: std::sync::LazyLock<Migrations<'static>> = std::sync::LazyLoc
M::up(include_str!("../migrations/0003_ci_api.sql")),
M::up(include_str!("../migrations/0004_bootstrap_api.sql")),
M::up(include_str!("../migrations/0005_rename_run_token.sql")),
+ M::up(include_str!("../migrations/0006_traceparent.sql")),
])
});
diff --git a/quire-server/src/quire/web/api.rs b/quire-server/src/quire/web/api.rs
index 9ef61b7..a18e6cf 100644
--- a/quire-server/src/quire/web/api.rs
+++ b/quire-server/src/quire/web/api.rs
@@ -154,14 +154,14 @@ async fn get_bootstrap(
ref_name: String,
pushed_at_ms: i64,
git_dir: Option<String>,
- sentry_trace_id: Option<String>,
+ traceparent: Option<String>,
state: String,
repo: String,
}
let row: RunRow = db
.prepare(
- "SELECT sha, ref_name, pushed_at_ms, git_dir, sentry_trace_id, state, repo
+ "SELECT sha, ref_name, pushed_at_ms, git_dir, traceparent, state, repo
FROM runs WHERE id = ?1",
)?
.query_and_then(rusqlite::params![run_id], serde_rusqlite::from_row)?
@@ -192,7 +192,7 @@ async fn get_bootstrap(
git_dir,
repo: row.repo,
run_id,
- sentry_trace_id: row.sentry_trace_id,
+ traceparent: row.traceparent,
})
})
.await
@@ -284,7 +284,7 @@ mod tests {
env: &TestEnv,
session: &ApiSession,
git_dir: &str,
- sentry_trace_id: Option<&str>,
+ traceparent: Option<&str>,
) -> String {
let run = env
.runs()
@@ -294,8 +294,8 @@ mod tests {
let db = crate::db::open(&env.quire.db_path()).expect("db open");
db.execute(
- "UPDATE runs SET git_dir = ?1, sentry_trace_id = ?2 WHERE id = ?3",
- rusqlite::params![git_dir, sentry_trace_id, &run_id],
+ "UPDATE runs SET git_dir = ?1, traceparent = ?2 WHERE id = ?3",
+ rusqlite::params![git_dir, traceparent, &run_id],
)
.expect("update bootstrap data");
run_id