Add API-backed secret fetching for quire-ci
When `:ci {:transport :api}` is set in config.fnl, quire-ci ignores the
secrets map in the bootstrap file and fetches each value on demand from
GET /api/runs/{run_id}/secrets/{name} using the per-run bearer token when
a run-fn calls `(secret :name)`. The bootstrap file is otherwise unchanged.
The filesystem path remains the default. The API path lets you validate
that secret fetching works over HTTP before a follow-up change stops
writing secrets into the bootstrap file entirely.
Changes:
- quire-core/secret: add SecretRegistry::with_fallback() — a Fn called
when a name is absent from the declared map; revealed values are still
registered for redaction regardless of which path produced them.
- quire-core/runtime: Runtime::new now accepts a SecretRegistry directly
instead of HashMap<String, SecretString>, so callers can attach a
fallback before handing it to the runtime.
- quire-ci: when --transport api, build an empty registry with an
API-fetch fallback instead of populating from the bootstrap secrets.
Uses Handle::block_on to drive the async reqwest call from synchronous
Lua callback context.
- quire-server/ci/run: pass --bootstrap and --transport api for API-mode
dispatches (previously --bootstrap was omitted, causing a CLI parse
error on the quire-ci side).
- docs/config.md: document :ci :transport and the two transport modes.
https://claude.ai/code/session_01VCCZYDCHDbr4LgadiJ6QAi
diff --git a/docs/config.md b/docs/config.md
index a9ae633..24f923f 100644
--- a/docs/config.md
+++ b/docs/config.md
@@ -9,11 +9,12 @@ loaded via the embedding described in [`fennel.md`](fennel.md).
Lives at `/var/quire/config.fnl` on the bind-mounted volume.
Operator-created. Re-read on every call (no caching today).
-| Key | Type | Required | Purpose |
-|----------------|----------------|----------|----------------------------------------------------------|
-| `:port` | integer | no | TCP port the HTTP server binds to (on `0.0.0.0`). Default: `3000`. |
-| `:sentry :dsn` | `SecretString` | no | Sentry DSN for error reporting from both `quire` and `quire-ci`. Omit to disable. |
-| `:secrets` | table | no | Named secrets exposed to `ci.fnl` jobs as `(secret :name)`. |
+| Key | Type | Required | Purpose |
+|----------------------|----------------|----------|----------------------------------------------------------|
+| `:port` | integer | no | TCP port the HTTP server binds to (on `0.0.0.0`). Default: `3000`. |
+| `:sentry :dsn` | `SecretString` | no | Sentry DSN for error reporting from both `quire` and `quire-ci`. Omit to disable. |
+| `:secrets` | table | no | Named secrets exposed to `ci.fnl` jobs as `(secret :name)`. |
+| `:ci :transport` | string | no | How `quire-ci` receives secrets. `"filesystem"` (default) bakes revealed values into the bootstrap file; `"api"` serves them on demand via the HTTP API instead. See [Secret transport modes](#secret-transport-modes) below. |
Minimal (no Sentry, no secrets):
@@ -83,6 +84,31 @@ Limits worth knowing:
their *recorded* output redacted at record time.
- Tracing output is not yet redacted (tracked separately).
+## Secret transport modes
+
+The `:ci :transport` key controls how `quire-ci` obtains secret values at
+run time. The two modes are:
+
+**`"filesystem"` (default):** Secrets are revealed into the bootstrap JSON
+file (mode `0600`) before `quire-ci` is spawned. The subprocess reads and
+immediately unlinks the file.
+
+**`"api"`:** `quire-ci` ignores the secrets map in the bootstrap file and
+fetches each value on demand from `GET /api/runs/{run_id}/secrets/{name}`
+using the per-run bearer token when a run-fn calls `(secret :name)`. The
+bootstrap file is still written and used for run metadata; only the secret
+values travel over the loopback HTTP channel instead of disk.
+
+To opt in:
+
+```fennel
+{:ci {:transport :api}}
+```
+
+The filesystem path remains the default. Use `:api` to validate the HTTP
+path before a future change stops writing secrets to the bootstrap file
+entirely.
+
## See also
- [`fennel.md`](fennel.md) — how Fennel files are loaded into Rust structs.
diff --git a/quire-ci/src/main.rs b/quire-ci/src/main.rs
index 30b2782..a94707d 100644
--- a/quire-ci/src/main.rs
+++ b/quire-ci/src/main.rs
@@ -14,6 +14,7 @@ use quire_core::ci::run::RunMeta;
use quire_core::ci::runtime::{Runtime, RuntimeError, RuntimeEvent, RuntimeHandle};
use quire_core::ci::transport::{ApiSession, Transport, TransportMode};
use quire_core::fennel::FennelError;
+use quire_core::secret::SecretRegistry;
use quire_core::telemetry::{self, FmtMode, MietteLayer};
/// Errors from running a job's `run_fn`. Lua errors are re-wrapped
@@ -235,13 +236,25 @@ fn main() -> miette::Result<()> {
let miette_layer = MietteLayer::new();
telemetry::init_tracing(miette_layer, FmtMode::Plain)?;
+ // API transport fetches secrets from the server on demand
+ // instead of reading them from the bootstrap file. The
+ // bootstrap secrets map is ignored in this path; a later
+ // change will stop writing secrets to bootstrap entirely.
+ let registry = if transport.mode == TransportMode::Api {
+ let session = transport.session.clone();
+ SecretRegistry::from(HashMap::new())
+ .with_fallback(move |name| fetch_secret_from_api(&session, name))
+ } else {
+ SecretRegistry::from(secrets)
+ };
+
run_pipeline(
cli.workspace,
sink,
log_dir,
git_dir,
meta,
- secrets,
+ registry,
transport,
)
}
@@ -356,13 +369,56 @@ fn load_bootstrap(
Ok((bootstrap.git_dir, bootstrap.meta, secrets, bootstrap.sentry))
}
+/// Fetch a single secret value from quire-server's API.
+///
+/// Called by the [`SecretRegistry`] fallback when API transport is
+/// selected. Uses [`tokio::runtime::Handle::block_on`] to drive the
+/// async HTTP call from synchronous Lua callback context. The caller
+/// must be on a thread that has entered a Tokio runtime (guaranteed
+/// by `rt.enter()` in `main`).
+///
+/// Maps the server's 404 to [`quire_core::secret::Error::UnknownSecret`]
+/// so the error message mirrors the filesystem-backed path.
+fn fetch_secret_from_api(session: &ApiSession, name: &str) -> quire_core::secret::Result<String> {
+ use quire_core::secret::Error as SecretError;
+
+ let url = format!(
+ "{}/api/runs/{}/secrets/{}",
+ session.server_url, session.run_id, name
+ );
+ let token = session.auth_token.clone();
+ let name_owned = name.to_string();
+
+ tokio::runtime::Handle::current().block_on(async move {
+ let resp = reqwest::Client::new()
+ .get(&url)
+ .bearer_auth(&token)
+ .send()
+ .await
+ .map_err(|e| SecretError::Resolve(e.to_string()))?;
+
+ let status = resp.status();
+ if status.is_success() {
+ resp.text()
+ .await
+ .map_err(|e| SecretError::Resolve(e.to_string()))
+ } else if status == reqwest::StatusCode::NOT_FOUND {
+ Err(SecretError::UnknownSecret(name_owned))
+ } else {
+ Err(SecretError::Resolve(format!(
+ "secret API returned {status} for {name_owned:?}"
+ )))
+ }
+ })
+}
+
fn run_pipeline(
workspace: PathBuf,
mut sink: Box<dyn EventSink>,
log_dir: PathBuf,
git_dir: PathBuf,
meta: RunMeta,
- secrets: HashMap<String, quire_core::secret::SecretString>,
+ registry: SecretRegistry,
_transport: Transport,
) -> miette::Result<()> {
let pipeline = match compile_at(&workspace) {
@@ -404,7 +460,7 @@ fn run_pipeline(
let sink: Rc<RefCell<Box<dyn EventSink>>> = Rc::new(RefCell::new(sink));
let runtime = Rc::new(Runtime::new(
- pipeline, secrets, &meta, &git_dir, workspace, log_dir,
+ pipeline, registry, &meta, &git_dir, workspace, log_dir,
));
// Active job pointer, shared between the main loop and the
diff --git a/quire-core/src/ci/runtime.rs b/quire-core/src/ci/runtime.rs
index 1fcbaf3..b75089f 100644
--- a/quire-core/src/ci/runtime.rs
+++ b/quire-core/src/ci/runtime.rs
@@ -15,7 +15,9 @@ use mlua::{IntoLua, Lua, LuaSerdeExt};
use super::pipeline::{Job, Pipeline};
use super::run::RunMeta;
-use crate::secret::{self, SecretRegistry, SecretString, redact};
+#[cfg(test)]
+use crate::secret::SecretString;
+use crate::secret::{self, SecretRegistry, redact};
/// Errors produced by [`Runtime`] methods and the `RunFn::Rust`
/// callbacks that hold them. A small sum carved out of the
@@ -128,6 +130,9 @@ impl Runtime {
///
/// Takes ownership of the pipeline (including its Lua VM). `meta`
/// provides the push data for `:quire/push` source outputs.
+ /// `registry` supplies secrets for `(secret :name)` calls; build
+ /// it with [`SecretRegistry::with_fallback`] to enable API-backed
+ /// fetching.
///
/// Panics if any of the Lua table operations below fail. They run
/// against a freshly initialized VM with `String`/`&str` keys and
@@ -135,7 +140,7 @@ impl Runtime {
/// failure — abort is the right answer there.
pub fn new(
pipeline: Pipeline,
- secrets: HashMap<String, SecretString>,
+ registry: SecretRegistry,
meta: &RunMeta,
git_dir: &std::path::Path,
workspace: std::path::PathBuf,
@@ -175,7 +180,7 @@ impl Runtime {
Self {
pipeline,
inputs,
- registry: RefCell::new(SecretRegistry::from(secrets)),
+ registry: RefCell::new(registry),
current_job: RefCell::new(None),
outputs: RefCell::new(HashMap::new()),
sh_timings: RefCell::new(HashMap::new()),
diff --git a/quire-core/src/secret.rs b/quire-core/src/secret.rs
index 512618a..37d5e68 100644
--- a/quire-core/src/secret.rs
+++ b/quire-core/src/secret.rs
@@ -157,6 +157,10 @@ impl Revealed {
// Explicitly no Debug impl — revealed values must never be printed.
+/// Signature for a secret fallback fetcher: given a name, return the
+/// revealed value or an error.
+type SecretFetcher = Box<dyn Fn(&str) -> Result<String>>;
+
/// Per-run secret store: holds declared secrets and their revealed
/// values for both lookup and redaction.
///
@@ -173,6 +177,11 @@ pub struct SecretRegistry {
/// name → revealed value (opaque, zeroed on drop).
/// Populated on first `(secret :name)` call.
revealed: HashMap<String, Revealed>,
+ /// Optional fallback invoked when a name is not in `declared`.
+ /// Intended for API transport: quire-ci installs a closure that
+ /// fetches the value from the server. Names resolved via the
+ /// fallback are registered for redaction just like declared ones.
+ fallback: Option<SecretFetcher>,
}
impl std::fmt::Debug for SecretRegistry {
@@ -180,6 +189,7 @@ impl std::fmt::Debug for SecretRegistry {
f.debug_struct("SecretRegistry")
.field("declared", &self.declared.keys().collect::<Vec<_>>())
.field("revealed", &self.revealed.keys().collect::<Vec<_>>())
+ .field("fallback", &self.fallback.as_ref().map(|_| "<fn>"))
.finish()
}
}
@@ -189,6 +199,7 @@ impl From<HashMap<String, SecretString>> for SecretRegistry {
Self {
declared,
revealed: HashMap::new(),
+ fallback: None,
}
}
}
@@ -210,9 +221,26 @@ impl From<Vec<(&str, &str)>> for SecretRegistry {
}
impl SecretRegistry {
- /// Resolve a declared secret by name, caching the revealed value
- /// for redaction. Returns `Err` if the name isn't declared or
- /// the source can't be read.
+ /// Install a fallback resolver called when a secret name is not
+ /// found in the declared map. Intended for API transport: quire-ci
+ /// installs a closure that fetches the value from quire-server.
+ ///
+ /// The fallback is tried exactly once per `resolve` call — results
+ /// are not cached between calls, so the closure is responsible for
+ /// any fetch-level caching it needs. Revealed values are still
+ /// registered for redaction regardless of which path produced them.
+ pub fn with_fallback<F>(mut self, fallback: F) -> Self
+ where
+ F: Fn(&str) -> Result<String> + 'static,
+ {
+ self.fallback = Some(Box::new(fallback));
+ self
+ }
+
+ /// Resolve a secret by name, caching the revealed value for
+ /// redaction. Checks `declared` first; if the name is absent and a
+ /// fallback is installed, calls it. Returns `Err` if the name is
+ /// unknown through both paths or the source can't be read.
///
/// Values shorter than 8 characters are returned to the caller
/// but not registered for redaction — the false-positive rate on
@@ -227,11 +255,13 @@ impl SecretRegistry {
/// through [`redact`] (e.g. `sh` command args, ShOutput) or wrap
/// it in a type whose `Debug`/`Display` impl redacts.
pub fn resolve(&mut self, name: &str) -> Result<String> {
- let secret = self
- .declared
- .get(name)
- .ok_or_else(|| Error::UnknownSecret(name.to_string()))?;
- let value = secret.reveal()?.to_string();
+ let value = if let Some(secret) = self.declared.get(name) {
+ secret.reveal()?.to_string()
+ } else if let Some(ref fallback) = self.fallback {
+ fallback(name)?
+ } else {
+ return Err(Error::UnknownSecret(name.to_string()));
+ };
if value.len() >= 8 {
self.revealed
.insert(name.to_string(), Revealed::new(value.clone()));
diff --git a/quire-server/src/ci/run.rs b/quire-server/src/ci/run.rs
index 37edde8..121d53c 100644
--- a/quire-server/src/ci/run.rs
+++ b/quire-server/src/ci/run.rs
@@ -349,18 +349,14 @@ impl Run {
.arg("--server-url")
.arg(&t.session.server_url);
cmd.env("QUIRE_CI_TOKEN", &t.session.auth_token);
- match t.mode {
- TransportMode::Filesystem => {
- cmd.arg("--out-dir")
- .arg(&run_dir)
- .arg("--events")
- .arg(&events_path)
- .arg("--bootstrap")
- .arg(&bootstrap_path);
- }
- TransportMode::Api => {
- cmd.arg("--transport").arg("api");
- }
+ cmd.arg("--out-dir")
+ .arg(&run_dir)
+ .arg("--events")
+ .arg(&events_path)
+ .arg("--bootstrap")
+ .arg(&bootstrap_path);
+ if t.mode == TransportMode::Api {
+ cmd.arg("--transport").arg("api");
}
}
}