Skip to content
Open
Show file tree
Hide file tree
Changes from 4 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
62 changes: 61 additions & 1 deletion crates/proxy/src/inspect/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ use uuid::Uuid;
use axum::http::HeaderMap;

use crate::frame::Frame;
use crate::sync::LocalOnlyCalls;

pub trait Inspector: Send {
fn on_request(&mut self, _headers: &HeaderMap, _body_json: Option<&Value>) {}
Expand Down Expand Up @@ -84,7 +85,11 @@ impl fmt::Display for ReportPayload {
const FLUSH_THRESHOLD: usize = 64;
const FLUSH_INTERVAL: Duration = Duration::from_secs(1);

pub async fn report_processor(mut rx: mpsc::UnboundedReceiver<Report>, storages: Storages) {
pub async fn report_processor(
mut rx: mpsc::UnboundedReceiver<Report>,
storages: Storages,
local_only_calls: LocalOnlyCalls,
) {
let mut identity_deadline = pin!(sleep(FLUSH_INTERVAL));
let mut pending_identities: Vec<IdentityDeclaration> = Vec::with_capacity(FLUSH_THRESHOLD);

Expand All @@ -103,6 +108,11 @@ pub async fn report_processor(mut rx: mpsc::UnboundedReceiver<Report>, storages:

match report.payload {
ReportPayload::Identity { provider, header_id, body_id, vh_headers } => {
// `list_unsent` never sends these; the inspector gives None
// for no headers, but an empty map would mean the same
if vh_headers.as_ref().is_none_or(HashMap::is_empty) {
local_only_calls.add_one();
Comment thread
petterir marked this conversation as resolved.
Resolved
Comment thread
petterir marked this conversation as resolved.
Resolved
}
pending_identities.push(IdentityDeclaration {
transit_id: report.transit_id,
provider,
Expand Down Expand Up @@ -169,3 +179,53 @@ async fn take_and_store_usages(pending: &mut Vec<UsageDeclaration>, storages: &S
tracing::error!(error = %err, "failed to store usages");
}
}

#[cfg(test)]
mod tests {
use super::*;
use crate::testutil::FakeTransitStorage;

fn identity(vh_headers: Option<HashMap<String, String>>) -> Report {
Report {
transit_id: Uuid::now_v7(),
payload: ReportPayload::Identity {
provider: Provider::default(),
header_id: None,
body_id: Some("chatcmpl-test".into()),
vh_headers,
},
}
}

#[tokio::test]
async fn identities_without_vh_headers_are_counted_as_local_only() {
let (tx, rx) = mpsc::unbounded_channel();
tx.send(identity(None)).unwrap();
tx.send(identity(Some(HashMap::new()))).unwrap();
tx.send(identity(Some(HashMap::from([(
"x-vh-tag-context".to_string(),
"support-bot".to_string(),
)]))))
.unwrap();
tx.send(Report {
transit_id: Uuid::now_v7(),
payload: ReportPayload::Usage {
provider: Provider::default(),
model: None,
usage: serde_json::json!({"total_tokens": 29}),
},
})
.unwrap();
drop(tx);

let local_only_calls = LocalOnlyCalls::default();
let storages = Storages {
transit: FakeTransitStorage::with_unsent(0),
};
// returns once the channel is closed and drained
report_processor(rx, storages, local_only_calls.clone()).await;

assert_eq!(local_only_calls.take(), 2);
assert_eq!(local_only_calls.take(), 0);
}
}
106 changes: 104 additions & 2 deletions crates/proxy/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -51,8 +51,15 @@ pub async fn start(config: Config, storages: Storages) -> anyhow::Result<()> {
anyhow::bail!("No providers specified, gateway will not be functional.");
}
let (report_tx, report_rx) = mpsc::unbounded_channel();
tokio::spawn(report_processor(report_rx, storages.clone()));

let local_only_calls = sync::LocalOnlyCalls::default();
tokio::spawn(report_processor(
report_rx,
storages.clone(),
local_only_calls.clone(),
));

// the default log level should tell whether usage leaves this machine
tracing::info!("{}", sync::forwarding_status(&config.valohai_llm));
if config.valohai_llm.enabled() {
let config_for_sync = config.clone();
let http_client_for_sync = reqwest::Client::builder()
Expand All @@ -62,6 +69,7 @@ pub async fn start(config: Config, storages: Storages) -> anyhow::Result<()> {
storages,
config_for_sync,
http_client_for_sync,
local_only_calls,
));
}

Expand Down Expand Up @@ -279,6 +287,14 @@ async fn health() -> (StatusCode, &'static str) {
#[cfg(test)]
pub(crate) mod testutil {
use super::*;
use conduit_core::{
IdentityDeclaration, TransitPage, TransitQuery, TransitRecord, TransitStorage,
UsageDeclaration,
};
use std::collections::HashMap;
use std::future::Future;
use std::pin::Pin;
use std::sync::Mutex;

pub fn test_state() -> Arc<AppState> {
test_state_with_client(reqwest::Client::new())
Expand All @@ -291,6 +307,92 @@ pub(crate) mod testutil {
report_tx,
})
}

// keeps only transit ids, which is all the flush and report paths look at
#[derive(Default)]
pub struct FakeTransitStorage {
pub unsent: Mutex<Vec<Uuid>>,
pub sent: Mutex<Vec<Uuid>>,
}

impl FakeTransitStorage {
pub fn with_unsent(count: usize) -> Arc<Self> {
Arc::new(Self {
unsent: Mutex::new((0..count).map(|_| Uuid::now_v7()).collect()),
..Self::default()
})
}
}

impl TransitStorage for FakeTransitStorage {
fn store_identities(
&self,
_declarations: Vec<IdentityDeclaration>,
) -> Pin<Box<dyn Future<Output = anyhow::Result<()>> + Send + '_>> {
Box::pin(async { Ok(()) })
}

fn store_usages(
&self,
_declarations: Vec<UsageDeclaration>,
) -> Pin<Box<dyn Future<Output = anyhow::Result<()>> + Send + '_>> {
Box::pin(async { Ok(()) })
}

fn list_transits(
&self,
_query: TransitQuery,
) -> Pin<Box<dyn Future<Output = anyhow::Result<TransitPage>> + Send + '_>> {
Box::pin(async { anyhow::bail!("not faked") })
}

fn get_transits(
&self,
_transit_ids: Vec<Uuid>,
) -> Pin<Box<dyn Future<Output = anyhow::Result<Vec<TransitRecord>>> + Send + '_>> {
Box::pin(async { anyhow::bail!("not faked") })
}

fn list_unsent(
&self,
limit: u32,
) -> Pin<Box<dyn Future<Output = anyhow::Result<Vec<TransitRecord>>> + Send + '_>> {
let records = self
.unsent
.lock()
.unwrap()
.iter()
.take(limit as usize)
.map(|&transit_id| TransitRecord {
transit_id,
stored_at: chrono::Utc::now(),
provider: Provider::default(),
header_id: None,
body_id: None,
vh_headers: Some(HashMap::from([(
"x-vh-tag-context".to_string(),
"test".to_string(),
)])),
model: None,
usage: None,
estimated_cost_usd: None,
})
.collect();
Box::pin(async { Ok(records) })
}

fn mark_sent(
&self,
transit_ids: Vec<Uuid>,
) -> Pin<Box<dyn Future<Output = anyhow::Result<()>> + Send + '_>> {
self.unsent
.lock()
.unwrap()
.retain(|id| !transit_ids.contains(id));
self.sent.lock().unwrap().extend(transit_ids);
Box::pin(async { Ok(()) })
}
}
}

#[cfg(test)]
Expand Down
Loading