From 355a430c7ba68324da6ea9ef24e5473f9618bacd Mon Sep 17 00:00:00 2001 From: Petteri Raatikainen Date: Thu, 19 Nov 2026 12:30:00 +0200 Subject: [PATCH 1/7] =?UTF-8?q?=F0=9F=93=A1=20Say=20at=20startup=20whether?= =?UTF-8?q?=20usage=20is=20forwarded=20to=20Valohai=20LLM?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit At the default log level, Conduit now prints one line at startup that tells whether usage leaves the machine. Before, a run with forwarding on looked the same as one without an API key. INFO conduit_proxy: forwarding usage to Valohai LLM at https://llm.valohai.com INFO conduit_proxy: Valohai LLM forwarding off; set VALOHAI_LLM_API_KEY or [valohai-llm] api_key to turn it on The line names the URL and never the API key. Refs #41 Co-Authored-By: Claude Opus 5.5 --- crates/proxy/src/lib.rs | 2 ++ crates/proxy/src/sync.rs | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 37 insertions(+) diff --git a/crates/proxy/src/lib.rs b/crates/proxy/src/lib.rs index b7051fb..9ca8a0a 100644 --- a/crates/proxy/src/lib.rs +++ b/crates/proxy/src/lib.rs @@ -53,6 +53,8 @@ pub async fn start(config: Config, storages: Storages) -> anyhow::Result<()> { let (report_tx, report_rx) = mpsc::unbounded_channel(); tokio::spawn(report_processor(report_rx, storages.clone())); + // the default log level should tell whether usage leaves this machine + tracing::info!("{}", sync::forwarding_status(&config.valohai_llm)); if config.valohai_llm.enabled() { let config_for_sync = config.clone(); let http_client_for_sync = reqwest::Client::builder() diff --git a/crates/proxy/src/sync.rs b/crates/proxy/src/sync.rs index 3718bbd..562023f 100644 --- a/crates/proxy/src/sync.rs +++ b/crates/proxy/src/sync.rs @@ -1,5 +1,6 @@ use std::time::Duration; +use conduit_core::config::{LLM_API_KEY_ENV_VAR, ValohaiLlmConfig}; use conduit_core::{Config, Storages, TransitRecord}; use serde::Serialize; @@ -10,6 +11,17 @@ const SYNC_POST_BATCH_SIZE: u32 = 64; // The cadence to flush transits to Valohai LLM. const SYNC_FLUSH_INTERVAL: Duration = Duration::from_secs(5); +// never include the api key, not even a part of it +pub fn forwarding_status(config: &ValohaiLlmConfig) -> String { + if config.enabled() { + format!("forwarding usage to Valohai LLM at {}", config.url) + } else { + format!( + "Valohai LLM forwarding off; set {LLM_API_KEY_ENV_VAR} or [valohai-llm] api_key to turn it on" + ) + } +} + pub async fn valohai_llm_poster(storages: Storages, config: Config, http_client: reqwest::Client) { let api_key = &config.valohai_llm.api_key; let endpoint = format!("{}/api/ingest/transits/", config.valohai_llm.url); @@ -103,3 +115,26 @@ impl From<&TransitRecord> for TransitPost { } } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn forwarding_status_names_the_url_but_not_the_key() { + let config = ValohaiLlmConfig { + api_key: "dummy-key-123".into(), + url: "http://127.0.0.1:1234".into(), + }; + let status = forwarding_status(&config); + assert!(status.contains("http://127.0.0.1:1234")); + assert!(!status.contains("dummy-key-123")); + } + + #[test] + fn forwarding_status_says_how_to_turn_forwarding_on() { + let status = forwarding_status(&ValohaiLlmConfig::default()); + assert!(status.contains(LLM_API_KEY_ENV_VAR)); + assert!(status.contains("[valohai-llm] api_key")); + } +} From ed3f0253d2d5d4224ca831af3823b62a33b9d0b1 Mon Sep 17 00:00:00 2001 From: Petteri Raatikainen Date: Thu, 19 Nov 2026 12:31:18 +0200 Subject: [PATCH 2/7] =?UTF-8?q?=F0=9F=A7=B9=20Pull=20one=20Valohai=20LLM?= =?UTF-8?q?=20flush=20out=20of=20the=20poster=20loop?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The body of each 5-second tick is now its own function that returns how many records the endpoint accepted. Nothing changes in what is sent or logged; this lets the next commit log a per-flush count, and lets tests drive a flush against a local endpoint and an in-memory storage. Refs #41 Co-Authored-By: Claude Opus 5.5 --- crates/proxy/src/lib.rs | 94 ++++++++++++++++++++ crates/proxy/src/sync.rs | 179 +++++++++++++++++++++++++++++---------- 2 files changed, 227 insertions(+), 46 deletions(-) diff --git a/crates/proxy/src/lib.rs b/crates/proxy/src/lib.rs index 9ca8a0a..a28fd03 100644 --- a/crates/proxy/src/lib.rs +++ b/crates/proxy/src/lib.rs @@ -281,6 +281,14 @@ async fn health() -> (StatusCode, &'static str) { #[cfg(test)] pub(crate) mod testutil { use super::*; + use conduit_core::{ + IdentityDeclaration, TransitPage, TransitQuery, TransitRecord, TransitStorage, + UsageDeclaration, + }; + use std::collections::HashMap; + use std::future::Future; + use std::pin::Pin; + use std::sync::Mutex; pub fn test_state() -> Arc { test_state_with_client(reqwest::Client::new()) @@ -293,6 +301,92 @@ pub(crate) mod testutil { report_tx, }) } + + // keeps only transit ids, which is all the flush and report paths look at + #[derive(Default)] + pub struct FakeTransitStorage { + pub unsent: Mutex>, + pub sent: Mutex>, + } + + impl FakeTransitStorage { + pub fn with_unsent(count: usize) -> Arc { + Arc::new(Self { + unsent: Mutex::new((0..count).map(|_| Uuid::now_v7()).collect()), + ..Self::default() + }) + } + } + + impl TransitStorage for FakeTransitStorage { + fn store_identities( + &self, + _declarations: Vec, + ) -> Pin> + Send + '_>> { + Box::pin(async { Ok(()) }) + } + + fn store_usages( + &self, + _declarations: Vec, + ) -> Pin> + Send + '_>> { + Box::pin(async { Ok(()) }) + } + + fn list_transits( + &self, + _query: TransitQuery, + ) -> Pin> + Send + '_>> { + Box::pin(async { anyhow::bail!("not faked") }) + } + + fn get_transits( + &self, + _transit_ids: Vec, + ) -> Pin>> + Send + '_>> { + Box::pin(async { anyhow::bail!("not faked") }) + } + + fn list_unsent( + &self, + limit: u32, + ) -> Pin>> + Send + '_>> { + let records = self + .unsent + .lock() + .unwrap() + .iter() + .take(limit as usize) + .map(|&transit_id| TransitRecord { + transit_id, + stored_at: chrono::Utc::now(), + provider: Provider::default(), + header_id: None, + body_id: None, + vh_headers: Some(HashMap::from([( + "x-vh-tag-context".to_string(), + "test".to_string(), + )])), + model: None, + usage: None, + estimated_cost_usd: None, + }) + .collect(); + Box::pin(async { Ok(records) }) + } + + fn mark_sent( + &self, + transit_ids: Vec, + ) -> Pin> + Send + '_>> { + self.unsent + .lock() + .unwrap() + .retain(|id| !transit_ids.contains(id)); + self.sent.lock().unwrap().extend(transit_ids); + Box::pin(async { Ok(()) }) + } + } } #[cfg(test)] diff --git a/crates/proxy/src/sync.rs b/crates/proxy/src/sync.rs index 562023f..873785a 100644 --- a/crates/proxy/src/sync.rs +++ b/crates/proxy/src/sync.rs @@ -30,57 +30,72 @@ pub async fn valohai_llm_poster(storages: Storages, config: Config, http_client: loop { interval.tick().await; + flush_unsent(&storages, &http_client, &endpoint, api_key).await; + } +} - loop { - let records = match storages.transit.list_unsent(SYNC_POST_BATCH_SIZE).await { - Ok(r) if r.is_empty() => break, - Ok(r) => r, - Err(e) => { - tracing::error!(error = %e, "failed to list unsent transit records"); - break; - } - }; - - let ids: Vec<_> = records.iter().map(|r| r.transit_id).collect(); - let payload: Vec = records.iter().map(TransitPost::from).collect(); - - tracing::trace!( - count = ids.len(), - payload = %serde_json::to_string_pretty(&payload).unwrap_or_default(), - "posting transit records", - ); - - let result = http_client - .post(&endpoint) - .bearer_auth(api_key) - .json(&payload) - .send() - .await; - - match result { - Ok(resp) if resp.status().is_success() => { - tracing::debug!(count = ids.len(), "sent transit records"); - if let Err(e) = storages.transit.mark_sent(ids).await { - tracing::error!(error = %e, "failed to mark transit records as sent"); - } - } - Ok(resp) => { - let status = resp.status(); - let body = resp.text().await.unwrap_or_default(); - tracing::warn!( - %status, - body = body.chars().take(200).collect::(), - "endpoint rejected transit records, will retry later", - ); - break; - } - Err(e) => { - tracing::warn!(error = %e, "failed to send transit records, will retry later"); - break; +// one flush: posts unsent records in batches until none are left or a post fails, +// and returns how many records the endpoint accepted +async fn flush_unsent( + storages: &Storages, + http_client: &reqwest::Client, + endpoint: &str, + api_key: &str, +) -> usize { + let mut sent = 0; + + loop { + let records = match storages.transit.list_unsent(SYNC_POST_BATCH_SIZE).await { + Ok(r) if r.is_empty() => break, + Ok(r) => r, + Err(e) => { + tracing::error!(error = %e, "failed to list unsent transit records"); + break; + } + }; + + let ids: Vec<_> = records.iter().map(|r| r.transit_id).collect(); + let payload: Vec = records.iter().map(TransitPost::from).collect(); + + tracing::trace!( + count = ids.len(), + payload = %serde_json::to_string_pretty(&payload).unwrap_or_default(), + "posting transit records", + ); + + let result = http_client + .post(endpoint) + .bearer_auth(api_key) + .json(&payload) + .send() + .await; + + match result { + Ok(resp) if resp.status().is_success() => { + tracing::debug!(count = ids.len(), "sent transit records"); + sent += ids.len(); + if let Err(e) = storages.transit.mark_sent(ids).await { + tracing::error!(error = %e, "failed to mark transit records as sent"); } } + Ok(resp) => { + let status = resp.status(); + let body = resp.text().await.unwrap_or_default(); + tracing::warn!( + %status, + body = body.chars().take(200).collect::(), + "endpoint rejected transit records, will retry later", + ); + break; + } + Err(e) => { + tracing::warn!(error = %e, "failed to send transit records, will retry later"); + break; + } } } + + sent } #[derive(Serialize)] @@ -119,6 +134,78 @@ impl From<&TransitRecord> for TransitPost { #[cfg(test)] mod tests { use super::*; + use crate::testutil::FakeTransitStorage; + use axum::Json; + use axum::Router; + use axum::http::StatusCode; + use axum::routing::post; + use std::sync::{Arc, Mutex}; + use tokio::net::TcpListener; + + // a stand-in for the Valohai LLM ingest endpoint that answers every post with + // `status` and keeps the size of each posted batch + async fn ingest_endpoint(status: StatusCode) -> (String, Arc>>) { + let batches = Arc::new(Mutex::new(Vec::new())); + let app = Router::new().route( + "/api/ingest/transits/", + post({ + let batches = batches.clone(); + move |Json(body): Json>| { + batches.lock().unwrap().push(body.len()); + async move { status } + } + }), + ); + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + tokio::spawn(async { axum::serve(listener, app).await.unwrap() }); + (format!("http://{addr}/api/ingest/transits/"), batches) + } + + #[tokio::test] + async fn flush_sends_every_unsent_record_across_batches() { + let storage = FakeTransitStorage::with_unsent(150); + let storages = Storages { + transit: storage.clone(), + }; + let (endpoint, batches) = ingest_endpoint(StatusCode::OK).await; + + let sent = flush_unsent(&storages, &reqwest::Client::new(), &endpoint, "dummy").await; + + assert_eq!(sent, 150); + assert_eq!(*batches.lock().unwrap(), vec![64, 64, 22]); + assert!(storage.unsent.lock().unwrap().is_empty()); + assert_eq!(storage.sent.lock().unwrap().len(), 150); + } + + #[tokio::test] + async fn flush_sends_nothing_when_the_endpoint_rejects() { + let storage = FakeTransitStorage::with_unsent(150); + let storages = Storages { + transit: storage.clone(), + }; + let (endpoint, batches) = ingest_endpoint(StatusCode::UNAUTHORIZED).await; + + let sent = flush_unsent(&storages, &reqwest::Client::new(), &endpoint, "dummy").await; + + assert_eq!(sent, 0); + // the first rejection ends the flush; the rest wait for the next one + assert_eq!(*batches.lock().unwrap(), vec![64]); + assert_eq!(storage.unsent.lock().unwrap().len(), 150); + assert!(storage.sent.lock().unwrap().is_empty()); + } + + #[tokio::test] + async fn flush_with_nothing_unsent_posts_nothing() { + let storage = FakeTransitStorage::with_unsent(0); + let storages = Storages { transit: storage }; + let (endpoint, batches) = ingest_endpoint(StatusCode::OK).await; + + let sent = flush_unsent(&storages, &reqwest::Client::new(), &endpoint, "dummy").await; + + assert_eq!(sent, 0); + assert!(batches.lock().unwrap().is_empty()); + } #[test] fn forwarding_status_names_the_url_but_not_the_key() { From 36580e48a08162eb934cf53a4f6f2c5f2427a392 Mon Sep 17 00:00:00 2001 From: Petteri Raatikainen Date: Thu, 19 Nov 2026 12:31:51 +0200 Subject: [PATCH 3/7] =?UTF-8?q?=F0=9F=93=A4=20Log=20how=20many=20usage=20r?= =?UTF-8?q?ecords=20each=20flush=20sent=20to=20Valohai=20LLM?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Each 5-second flush that sends something now logs one info line with the total across all of its batches: INFO conduit_proxy::sync: sent 6 usage records to Valohai LLM Before, sends were logged at debug only, so at the default level there was no way to tell from the log that forwarding worked. Flushes that send nothing stay quiet, and the per-POST debug line is kept. Refs #41 Co-Authored-By: Claude Opus 5.5 --- crates/proxy/src/sync.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/crates/proxy/src/sync.rs b/crates/proxy/src/sync.rs index 873785a..a5f5878 100644 --- a/crates/proxy/src/sync.rs +++ b/crates/proxy/src/sync.rs @@ -30,7 +30,11 @@ pub async fn valohai_llm_poster(storages: Storages, config: Config, http_client: loop { interval.tick().await; - flush_unsent(&storages, &http_client, &endpoint, api_key).await; + let sent = flush_unsent(&storages, &http_client, &endpoint, api_key).await; + if sent > 0 { + let records = if sent == 1 { "record" } else { "records" }; + tracing::info!("sent {sent} usage {records} to Valohai LLM"); + } } } From b9c751bfdc22c18b0dfce3e1b50c6ff57e2333cd Mon Sep 17 00:00:00 2001 From: Petteri Raatikainen Date: Thu, 19 Nov 2026 12:36:12 +0200 Subject: [PATCH 4/7] =?UTF-8?q?=F0=9F=8F=B7=EF=B8=8F=20Say=20when=20record?= =?UTF-8?q?ed=20calls=20stay=20local=20for=20lack=20of=20X-VH-=20headers?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Conduit only forwards calls that carry an X-VH- header, and calls without one were not mentioned at any level. With forwarding on, the first flush that finds such calls from this run now logs one info line with a hint on how to tag them: INFO conduit_proxy::sync: 3 recorded calls had no X-VH- headers and stay local; add an X-VH-Tag- header (e.g. X-VH-Tag-Context) to forward them Later counts go to debug, so a steady stream of untagged calls does not add an info line every five seconds. The count is kept in memory for this run only, so records from earlier runs never trigger it. With forwarding off, the startup line already says nothing is sent. Fixes #41 Co-Authored-By: Claude Opus 5.5 --- crates/proxy/src/inspect/mod.rs | 62 +++++++++++++++++++++++++++- crates/proxy/src/lib.rs | 8 +++- crates/proxy/src/sync.rs | 72 ++++++++++++++++++++++++++++++++- 3 files changed, 139 insertions(+), 3 deletions(-) diff --git a/crates/proxy/src/inspect/mod.rs b/crates/proxy/src/inspect/mod.rs index 2297b20..f7a4815 100644 --- a/crates/proxy/src/inspect/mod.rs +++ b/crates/proxy/src/inspect/mod.rs @@ -19,6 +19,7 @@ use uuid::Uuid; use axum::http::HeaderMap; use crate::frame::Frame; +use crate::sync::LocalOnlyCalls; pub trait Inspector: Send { fn on_request(&mut self, _headers: &HeaderMap, _body_json: Option<&Value>) {} @@ -84,7 +85,11 @@ impl fmt::Display for ReportPayload { const FLUSH_THRESHOLD: usize = 64; const FLUSH_INTERVAL: Duration = Duration::from_secs(1); -pub async fn report_processor(mut rx: mpsc::UnboundedReceiver, storages: Storages) { +pub async fn report_processor( + mut rx: mpsc::UnboundedReceiver, + storages: Storages, + local_only_calls: LocalOnlyCalls, +) { let mut identity_deadline = pin!(sleep(FLUSH_INTERVAL)); let mut pending_identities: Vec = Vec::with_capacity(FLUSH_THRESHOLD); @@ -103,6 +108,11 @@ pub async fn report_processor(mut rx: mpsc::UnboundedReceiver, storages: match report.payload { ReportPayload::Identity { provider, header_id, body_id, vh_headers } => { + // `list_unsent` never sends these; the inspector gives None + // for no headers, but an empty map would mean the same + if vh_headers.as_ref().is_none_or(HashMap::is_empty) { + local_only_calls.add_one(); + } pending_identities.push(IdentityDeclaration { transit_id: report.transit_id, provider, @@ -169,3 +179,53 @@ async fn take_and_store_usages(pending: &mut Vec, storages: &S tracing::error!(error = %err, "failed to store usages"); } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::testutil::FakeTransitStorage; + + fn identity(vh_headers: Option>) -> Report { + Report { + transit_id: Uuid::now_v7(), + payload: ReportPayload::Identity { + provider: Provider::default(), + header_id: None, + body_id: Some("chatcmpl-test".into()), + vh_headers, + }, + } + } + + #[tokio::test] + async fn identities_without_vh_headers_are_counted_as_local_only() { + let (tx, rx) = mpsc::unbounded_channel(); + tx.send(identity(None)).unwrap(); + tx.send(identity(Some(HashMap::new()))).unwrap(); + tx.send(identity(Some(HashMap::from([( + "x-vh-tag-context".to_string(), + "support-bot".to_string(), + )])))) + .unwrap(); + tx.send(Report { + transit_id: Uuid::now_v7(), + payload: ReportPayload::Usage { + provider: Provider::default(), + model: None, + usage: serde_json::json!({"total_tokens": 29}), + }, + }) + .unwrap(); + drop(tx); + + let local_only_calls = LocalOnlyCalls::default(); + let storages = Storages { + transit: FakeTransitStorage::with_unsent(0), + }; + // returns once the channel is closed and drained + report_processor(rx, storages, local_only_calls.clone()).await; + + assert_eq!(local_only_calls.take(), 2); + assert_eq!(local_only_calls.take(), 0); + } +} diff --git a/crates/proxy/src/lib.rs b/crates/proxy/src/lib.rs index a28fd03..8fdc7eb 100644 --- a/crates/proxy/src/lib.rs +++ b/crates/proxy/src/lib.rs @@ -51,7 +51,12 @@ pub async fn start(config: Config, storages: Storages) -> anyhow::Result<()> { anyhow::bail!("No providers specified, gateway will not be functional."); } let (report_tx, report_rx) = mpsc::unbounded_channel(); - tokio::spawn(report_processor(report_rx, storages.clone())); + let local_only_calls = sync::LocalOnlyCalls::default(); + tokio::spawn(report_processor( + report_rx, + storages.clone(), + local_only_calls.clone(), + )); // the default log level should tell whether usage leaves this machine tracing::info!("{}", sync::forwarding_status(&config.valohai_llm)); @@ -64,6 +69,7 @@ pub async fn start(config: Config, storages: Storages) -> anyhow::Result<()> { storages, config_for_sync, http_client_for_sync, + local_only_calls, )); } diff --git a/crates/proxy/src/sync.rs b/crates/proxy/src/sync.rs index a5f5878..2504059 100644 --- a/crates/proxy/src/sync.rs +++ b/crates/proxy/src/sync.rs @@ -1,9 +1,12 @@ +use std::sync::Arc; +use std::sync::atomic::{AtomicU64, Ordering}; use std::time::Duration; use conduit_core::config::{LLM_API_KEY_ENV_VAR, ValohaiLlmConfig}; use conduit_core::{Config, Storages, TransitRecord}; use serde::Serialize; +use tracing::Level; // Max records sent per outgoing POST; further records are sent in subsequent // requests within the same flush. @@ -22,11 +25,51 @@ pub fn forwarding_status(config: &ValohaiLlmConfig) -> String { } } -pub async fn valohai_llm_poster(storages: Storages, config: Config, http_client: reqwest::Client) { +// Recorded calls without X-VH- headers in this run; `list_unsent` never picks +// them, so they are counted here to tell the user why nothing gets sent. +#[derive(Clone, Default)] +pub struct LocalOnlyCalls(Arc); + +impl LocalOnlyCalls { + pub fn add_one(&self) { + self.0.fetch_add(1, Ordering::Relaxed); + } + + pub fn take(&self) -> u64 { + self.0.swap(0, Ordering::Relaxed) + } +} + +// the hint on tagging is worth one info line; repeating it every flush would +// bury the rest of the log under a steady stream of untagged calls +#[derive(Default)] +struct LocalOnlyNotes { + told: bool, +} + +impl LocalOnlyNotes { + fn level(&mut self, count: u64) -> Option { + if count == 0 { + None + } else if std::mem::replace(&mut self.told, true) { + Some(Level::DEBUG) + } else { + Some(Level::INFO) + } + } +} + +pub async fn valohai_llm_poster( + storages: Storages, + config: Config, + http_client: reqwest::Client, + local_only_calls: LocalOnlyCalls, +) { let api_key = &config.valohai_llm.api_key; let endpoint = format!("{}/api/ingest/transits/", config.valohai_llm.url); let mut interval = tokio::time::interval(SYNC_FLUSH_INTERVAL); + let mut local_only_notes = LocalOnlyNotes::default(); loop { interval.tick().await; @@ -35,6 +78,23 @@ pub async fn valohai_llm_poster(storages: Storages, config: Config, http_client: let records = if sent == 1 { "record" } else { "records" }; tracing::info!("sent {sent} usage {records} to Valohai LLM"); } + + let local_only = local_only_calls.take(); + let (calls, stay) = if local_only == 1 { + ("call", "stays") + } else { + ("calls", "stay") + }; + match local_only_notes.level(local_only) { + Some(Level::INFO) => tracing::info!( + "{local_only} recorded {calls} had no X-VH- headers and {stay} local; \ + add an X-VH-Tag- header (e.g. X-VH-Tag-Context) to forward them" + ), + Some(_) => tracing::debug!( + "{local_only} more recorded {calls} had no X-VH- headers and {stay} local" + ), + None => {} + } } } @@ -199,6 +259,16 @@ mod tests { assert!(storage.sent.lock().unwrap().is_empty()); } + #[test] + fn local_only_calls_are_noted_at_info_once_then_at_debug() { + let mut notes = LocalOnlyNotes::default(); + assert_eq!(notes.level(0), None); + assert_eq!(notes.level(3), Some(Level::INFO)); + assert_eq!(notes.level(0), None); + assert_eq!(notes.level(2), Some(Level::DEBUG)); + assert_eq!(notes.level(5), Some(Level::DEBUG)); + } + #[tokio::test] async fn flush_with_nothing_unsent_posts_nothing() { let storage = FakeTransitStorage::with_unsent(0); From ef05322153b3320d1943b93216029cb16f4b7e31 Mon Sep 17 00:00:00 2001 From: Petteri Raatikainen Date: Thu, 19 Nov 2026 14:59:40 +0200 Subject: [PATCH 5/7] =?UTF-8?q?=F0=9F=99=88=20Keep=20URL=20credentials=20o?= =?UTF-8?q?ut=20of=20the=20forwarding=20startup=20line?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Valohai LLM url is free-form config, so it can carry credentials as userinfo (`https://user:secret@host`) or in the query string, and the startup line printed it as written at the default log level. The line now shows only scheme, host, port and path. A url that doesn't parse to a host is shown as `` instead of being echoed. Refs #41 Co-Authored-By: Claude Fable 5.2 --- crates/proxy/src/sync.rs | 54 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 53 insertions(+), 1 deletion(-) diff --git a/crates/proxy/src/sync.rs b/crates/proxy/src/sync.rs index 2504059..89f3b00 100644 --- a/crates/proxy/src/sync.rs +++ b/crates/proxy/src/sync.rs @@ -17,7 +17,10 @@ const SYNC_FLUSH_INTERVAL: Duration = Duration::from_secs(5); // never include the api key, not even a part of it pub fn forwarding_status(config: &ValohaiLlmConfig) -> String { if config.enabled() { - format!("forwarding usage to Valohai LLM at {}", config.url) + format!( + "forwarding usage to Valohai LLM at {}", + loggable_url(&config.url) + ) } else { format!( "Valohai LLM forwarding off; set {LLM_API_KEY_ENV_VAR} or [valohai-llm] api_key to turn it on" @@ -25,6 +28,20 @@ pub fn forwarding_status(config: &ValohaiLlmConfig) -> String { } } +// the url is free-form config, so it may carry credentials as userinfo or in the +// query; this goes to the log at the default level, so keep only where it points +fn loggable_url(url: &str) -> String { + // anything that doesn't parse to a host can't be trusted to have no secrets either + let Some(mut url) = reqwest::Url::parse(url).ok().filter(|u| u.has_host()) else { + return "".to_string(); + }; + let _ = url.set_username(""); + let _ = url.set_password(None); + url.set_query(None); + url.set_fragment(None); + url.as_str().trim_end_matches('/').to_string() +} + // Recorded calls without X-VH- headers in this run; `list_unsent` never picks // them, so they are counted here to tell the user why nothing gets sent. #[derive(Clone, Default)] @@ -292,6 +309,41 @@ mod tests { assert!(!status.contains("dummy-key-123")); } + #[test] + fn forwarding_status_leaves_out_credentials_in_the_url() { + let config = ValohaiLlmConfig { + api_key: "dummy-key-123".into(), + url: "https://someone:hunter2@llm.example.com/base?token=abc123#frag".into(), + }; + assert_eq!( + forwarding_status(&config), + "forwarding usage to Valohai LLM at https://llm.example.com/base" + ); + } + + #[test] + fn forwarding_status_shows_the_default_url_as_configured() { + let config = ValohaiLlmConfig { + api_key: "dummy-key-123".into(), + ..ValohaiLlmConfig::default() + }; + assert_eq!( + forwarding_status(&config), + "forwarding usage to Valohai LLM at https://llm.valohai.com" + ); + } + + #[test] + fn forwarding_status_does_not_echo_a_url_it_cannot_parse() { + let config = ValohaiLlmConfig { + api_key: "dummy-key-123".into(), + url: "someone:hunter2@not a url".into(), + }; + let status = forwarding_status(&config); + assert!(!status.contains("hunter2"), "{status}"); + assert!(status.contains(""), "{status}"); + } + #[test] fn forwarding_status_says_how_to_turn_forwarding_on() { let status = forwarding_status(&ValohaiLlmConfig::default()); From fb8a221aae30266235392cdeab14ae7892fd1c70 Mon Sep 17 00:00:00 2001 From: Petteri Raatikainen Date: Thu, 19 Nov 2026 15:00:37 +0200 Subject: [PATCH 6/7] =?UTF-8?q?=F0=9F=A7=AE=20Count=20calls=20as=20staying?= =?UTF-8?q?=20local=20only=20once=20they=20are=20stored?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The stay-local note counted an identity as soon as its report arrived. If storing the batch then failed, say on a full disk, the log still said the calls were recorded, and could spend the one info-level note on calls that were lost. The count is now taken per batch and added only after the batch is stored. Refs #41 Co-Authored-By: Claude Fable 5.2 --- crates/proxy/src/inspect/mod.rs | 46 +++++++++++++++++++++++++-------- crates/proxy/src/lib.rs | 15 ++++++++++- crates/proxy/src/sync.rs | 4 +-- 3 files changed, 51 insertions(+), 14 deletions(-) diff --git a/crates/proxy/src/inspect/mod.rs b/crates/proxy/src/inspect/mod.rs index f7a4815..3bfea2d 100644 --- a/crates/proxy/src/inspect/mod.rs +++ b/crates/proxy/src/inspect/mod.rs @@ -101,18 +101,13 @@ pub async fn report_processor( maybe_report = rx.recv() => { let Some(report) = maybe_report else { // channel closed, flush everything pending - take_and_store_identities(&mut pending_identities, &storages).await; + take_and_store_identities(&mut pending_identities, &storages, &local_only_calls).await; take_and_store_usages(&mut pending_usages, &storages).await; break; }; match report.payload { ReportPayload::Identity { provider, header_id, body_id, vh_headers } => { - // `list_unsent` never sends these; the inspector gives None - // for no headers, but an empty map would mean the same - if vh_headers.as_ref().is_none_or(HashMap::is_empty) { - local_only_calls.add_one(); - } pending_identities.push(IdentityDeclaration { transit_id: report.transit_id, provider, @@ -132,7 +127,7 @@ pub async fn report_processor( } if pending_identities.len() >= FLUSH_THRESHOLD { - take_and_store_identities(&mut pending_identities, &storages).await; + take_and_store_identities(&mut pending_identities, &storages, &local_only_calls).await; identity_deadline.as_mut().reset(Instant::now() + FLUSH_INTERVAL); } if pending_usages.len() >= FLUSH_THRESHOLD { @@ -144,7 +139,7 @@ pub async fn report_processor( // on intervals, flush the related declarations even if the batch size hasn't been reached _ = &mut identity_deadline => { - take_and_store_identities(&mut pending_identities, &storages).await; + take_and_store_identities(&mut pending_identities, &storages, &local_only_calls).await; identity_deadline.as_mut().reset(Instant::now() + FLUSH_INTERVAL); } _ = &mut usage_deadline => { @@ -155,17 +150,30 @@ pub async fn report_processor( } } -async fn take_and_store_identities(pending: &mut Vec, storages: &Storages) { +async fn take_and_store_identities( + pending: &mut Vec, + storages: &Storages, + local_only_calls: &LocalOnlyCalls, +) { if pending.is_empty() { return; } + // `list_unsent` never sends these; the inspector gives None for no headers, + // but an empty map would mean the same + let local_only = pending + .iter() + .filter(|identity| identity.vh_headers.as_ref().is_none_or(HashMap::is_empty)) + .count(); + let result = storages .transit .store_identities(std::mem::take(pending)) .await; - if let Err(err) = result { - tracing::error!(error = %err, "failed to store identities"); + match result { + // counted only once stored, as the note calls them recorded + Ok(()) => local_only_calls.add(local_only as u64), + Err(err) => tracing::error!(error = %err, "failed to store identities"), } } @@ -228,4 +236,20 @@ mod tests { assert_eq!(local_only_calls.take(), 2); assert_eq!(local_only_calls.take(), 0); } + + #[tokio::test] + async fn identities_that_fail_to_store_are_not_counted_as_local_only() { + let (tx, rx) = mpsc::unbounded_channel(); + tx.send(identity(None)).unwrap(); + tx.send(identity(None)).unwrap(); + drop(tx); + + let local_only_calls = LocalOnlyCalls::default(); + let storages = Storages { + transit: FakeTransitStorage::failing_to_store(), + }; + report_processor(rx, storages, local_only_calls.clone()).await; + + assert_eq!(local_only_calls.take(), 0); + } } diff --git a/crates/proxy/src/lib.rs b/crates/proxy/src/lib.rs index 8fdc7eb..48913a6 100644 --- a/crates/proxy/src/lib.rs +++ b/crates/proxy/src/lib.rs @@ -313,6 +313,7 @@ pub(crate) mod testutil { pub struct FakeTransitStorage { pub unsent: Mutex>, pub sent: Mutex>, + fail_stores: bool, } impl FakeTransitStorage { @@ -322,6 +323,14 @@ pub(crate) mod testutil { ..Self::default() }) } + + // a storage that can't be written to, e.g. a full disk + pub fn failing_to_store() -> Arc { + Arc::new(Self { + fail_stores: true, + ..Self::default() + }) + } } impl TransitStorage for FakeTransitStorage { @@ -329,7 +338,11 @@ pub(crate) mod testutil { &self, _declarations: Vec, ) -> Pin> + Send + '_>> { - Box::pin(async { Ok(()) }) + let fail = self.fail_stores; + Box::pin(async move { + anyhow::ensure!(!fail, "storage is failing"); + Ok(()) + }) } fn store_usages( diff --git a/crates/proxy/src/sync.rs b/crates/proxy/src/sync.rs index 89f3b00..6d47a54 100644 --- a/crates/proxy/src/sync.rs +++ b/crates/proxy/src/sync.rs @@ -48,8 +48,8 @@ fn loggable_url(url: &str) -> String { pub struct LocalOnlyCalls(Arc); impl LocalOnlyCalls { - pub fn add_one(&self) { - self.0.fetch_add(1, Ordering::Relaxed); + pub fn add(&self, count: u64) { + self.0.fetch_add(count, Ordering::Relaxed); } pub fn take(&self) -> u64 { From 0784b4b63d1d08658bb38ab7fb3ae89b9b0f1424 Mon Sep 17 00:00:00 2001 From: Petteri Raatikainen Date: Thu, 19 Nov 2026 15:02:48 +0200 Subject: [PATCH 7/7] =?UTF-8?q?=E2=8F=B1=EF=B8=8F=20Bound=20each=20Valohai?= =?UTF-8?q?=20LLM=20flush=20so=20its=20log=20lines=20keep=20coming?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A flush kept posting until no unsent records were left. With records arriving as fast as they are sent it would not return, so neither the sent count nor the stay-local note was logged while batches went out fine. A flush now also ends once it has run for the flush interval. Its tick is already due by then, so the next flush starts right away and the backlog keeps draining at the same rate. Refs #41 Co-Authored-By: Claude Fable 5.2 --- crates/proxy/src/sync.rs | 73 ++++++++++++++++++++++++++++++++++++---- 1 file changed, 67 insertions(+), 6 deletions(-) diff --git a/crates/proxy/src/sync.rs b/crates/proxy/src/sync.rs index 6d47a54..2ddab94 100644 --- a/crates/proxy/src/sync.rs +++ b/crates/proxy/src/sync.rs @@ -90,7 +90,16 @@ pub async fn valohai_llm_poster( loop { interval.tick().await; - let sent = flush_unsent(&storages, &http_client, &endpoint, api_key).await; + // a flush that used up its time is followed by the next one right away, + // as its tick is already due + let sent = flush_unsent( + &storages, + &http_client, + &endpoint, + api_key, + SYNC_FLUSH_INTERVAL, + ) + .await; if sent > 0 { let records = if sent == 1 { "record" } else { "records" }; tracing::info!("sent {sent} usage {records} to Valohai LLM"); @@ -115,14 +124,17 @@ pub async fn valohai_llm_poster( } } -// one flush: posts unsent records in batches until none are left or a post fails, -// and returns how many records the endpoint accepted +// one flush: posts unsent records in batches until none are left, a post fails or +// `time_limit` has passed, and returns how many records the endpoint accepted; +// the limit keeps the caller reporting while records come in as fast as they go out async fn flush_unsent( storages: &Storages, http_client: &reqwest::Client, endpoint: &str, api_key: &str, + time_limit: Duration, ) -> usize { + let started = std::time::Instant::now(); let mut sent = 0; loop { @@ -158,6 +170,9 @@ async fn flush_unsent( if let Err(e) = storages.transit.mark_sent(ids).await { tracing::error!(error = %e, "failed to mark transit records as sent"); } + if started.elapsed() >= time_limit { + break; + } } Ok(resp) => { let status = resp.status(); @@ -223,6 +238,8 @@ mod tests { use std::sync::{Arc, Mutex}; use tokio::net::TcpListener; + const NO_TIME_LIMIT: Duration = Duration::MAX; + // a stand-in for the Valohai LLM ingest endpoint that answers every post with // `status` and keeps the size of each posted batch async fn ingest_endpoint(status: StatusCode) -> (String, Arc>>) { @@ -251,7 +268,14 @@ mod tests { }; let (endpoint, batches) = ingest_endpoint(StatusCode::OK).await; - let sent = flush_unsent(&storages, &reqwest::Client::new(), &endpoint, "dummy").await; + let sent = flush_unsent( + &storages, + &reqwest::Client::new(), + &endpoint, + "dummy", + NO_TIME_LIMIT, + ) + .await; assert_eq!(sent, 150); assert_eq!(*batches.lock().unwrap(), vec![64, 64, 22]); @@ -259,6 +283,29 @@ mod tests { assert_eq!(storage.sent.lock().unwrap().len(), 150); } + #[tokio::test] + async fn flush_stops_at_its_time_limit_and_leaves_the_rest_for_the_next_one() { + let storage = FakeTransitStorage::with_unsent(150); + let storages = Storages { + transit: storage.clone(), + }; + let (endpoint, batches) = ingest_endpoint(StatusCode::OK).await; + + // no time at all still lets one batch through, so a flush always makes progress + let sent = flush_unsent( + &storages, + &reqwest::Client::new(), + &endpoint, + "dummy", + Duration::ZERO, + ) + .await; + + assert_eq!(sent, 64); + assert_eq!(*batches.lock().unwrap(), vec![64]); + assert_eq!(storage.unsent.lock().unwrap().len(), 86); + } + #[tokio::test] async fn flush_sends_nothing_when_the_endpoint_rejects() { let storage = FakeTransitStorage::with_unsent(150); @@ -267,7 +314,14 @@ mod tests { }; let (endpoint, batches) = ingest_endpoint(StatusCode::UNAUTHORIZED).await; - let sent = flush_unsent(&storages, &reqwest::Client::new(), &endpoint, "dummy").await; + let sent = flush_unsent( + &storages, + &reqwest::Client::new(), + &endpoint, + "dummy", + NO_TIME_LIMIT, + ) + .await; assert_eq!(sent, 0); // the first rejection ends the flush; the rest wait for the next one @@ -292,7 +346,14 @@ mod tests { let storages = Storages { transit: storage }; let (endpoint, batches) = ingest_endpoint(StatusCode::OK).await; - let sent = flush_unsent(&storages, &reqwest::Client::new(), &endpoint, "dummy").await; + let sent = flush_unsent( + &storages, + &reqwest::Client::new(), + &endpoint, + "dummy", + NO_TIME_LIMIT, + ) + .await; assert_eq!(sent, 0); assert!(batches.lock().unwrap().is_empty());