Skip to content
This repository was archived by the owner on Jun 3, 2026. It is now read-only.
10 changes: 7 additions & 3 deletions src/agents/judge.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@
OperationType,
)
from src.storage.base import BaseVectorStore, SearchResult
from src.config import settings


# ---------------------------------------------------------------------------
Expand Down Expand Up @@ -87,13 +88,15 @@ def _format_similar_block(
return "\n".join(lines)


SUMMARY_JUDGE_SIMILARITY_THRESHOLD = 0.4
SUMMARY_JUDGE_SIMILARITY_THRESHOLD = settings.summary_judge_similarity_threshold
Comment thread
Pratyush426 marked this conversation as resolved.
Resolved


def _has_summary_judge_candidates(
matches_per_item: Dict[str, List[SearchResult]],
threshold: float = SUMMARY_JUDGE_SIMILARITY_THRESHOLD,
threshold: Optional[float] = None,
) -> bool:
if threshold is None:
threshold = settings.summary_judge_similarity_threshold
for matches in matches_per_item.values():
for match in matches:
if match.score >= threshold:
Expand All @@ -112,11 +115,12 @@ def _filter_matches_by_threshold(


def _deterministic_summary_add(items_strings: List[str], confidence: float = 0.8) -> JudgeResult:
threshold = settings.summary_judge_similarity_threshold
operations = [
Operation(
type=OperationType.ADD,
content=item,
reason="No similar summary at or above 0.4 — defaulting to ADD.",
reason=f"No similar summary at or above {threshold} — defaulting to ADD.",
)
for item in items_strings
if str(item).strip()
Expand Down
8 changes: 8 additions & 0 deletions src/config/settings.py
Original file line number Diff line number Diff line change
Expand Up @@ -136,6 +136,14 @@ class Settings(BaseSettings):
default=0.4,
description="LLM temperature for generation"
)
summary_judge_similarity_threshold: float = Field(
default=0.4,
description="Threshold score for the Judge to match summary memories"
)
temporal_search_similarity_threshold: float = Field(
default=0.3,
description="Minimum cosine similarity threshold score for Neo4j temporal search"
)
Comment thread
ved015 marked this conversation as resolved.
Resolved
Comment thread
greptile-apps[bot] marked this conversation as resolved.
Resolved
llm_timeout_seconds: float = Field(
default=45.0,
description="Per-agent LLM call timeout in seconds",
Expand Down
6 changes: 5 additions & 1 deletion src/graph/neo4j_client.py
Original file line number Diff line number Diff line change
Expand Up @@ -250,7 +250,7 @@ def search_events_by_embedding(
user_id: str,
query_text: str,
top_k: int = 1,
similarity_threshold: float = 0.3,
similarity_threshold: Optional[float] = None,
) -> List[Dict[str, Any]]:
"""Semantic search over event embeddings stored on HAS_EVENT relationships.

Expand All @@ -263,6 +263,10 @@ def search_events_by_embedding(
``similarity_score`` is raw cosine in [-1, 1] (matches the previous
dot-product semantics, which assumed unit-normalised embeddings).
"""
if similarity_threshold is None:
from src.config import settings
similarity_threshold = settings.temporal_search_similarity_threshold
Comment thread
greptile-apps[bot] marked this conversation as resolved.
Resolved

if not self._embedding_fn:
logger.warning("No embedding function — cannot search by embedding.")
return []
Expand Down
76 changes: 76 additions & 0 deletions tests/test_configurable_thresholds.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,76 @@
import sys
import types

# Mock the neo4j package to allow execution without global environment installation
neo4j_mock = types.ModuleType("neo4j")
neo4j_mock.GraphDatabase = object
sys.modules.setdefault("neo4j", neo4j_mock)

import pytest
from src.config import settings
from src.agents.judge import _has_summary_judge_candidates, _deterministic_summary_add
from src.storage.base import SearchResult
from src.graph.neo4j_client import Neo4jClient


def test_summary_judge_respects_custom_settings():
original_threshold = settings.summary_judge_similarity_threshold
try:
# Match with a score of 0.35
matches = {
"test item": [
SearchResult(id="1", content="similar text", score=0.35, metadata={})
]
}

# Default is 0.4, so score of 0.35 should NOT match
settings.summary_judge_similarity_threshold = 0.4
assert not _has_summary_judge_candidates(matches)

# If we configure it to 0.3, a score of 0.35 SHOULD match
settings.summary_judge_similarity_threshold = 0.3
assert _has_summary_judge_candidates(matches)

# Test deterministic addition reason string includes threshold
settings.summary_judge_similarity_threshold = 0.55
result = _deterministic_summary_add(["new summary"])
assert len(result.operations) == 1
assert "0.55" in result.operations[0].reason

finally:
# Restore settings
settings.summary_judge_similarity_threshold = original_threshold
Comment thread
ved015 marked this conversation as resolved.
Resolved
Comment thread
greptile-apps[bot] marked this conversation as resolved.
Resolved


def test_neo4j_client_respects_custom_settings(monkeypatch):
original_threshold = settings.temporal_search_similarity_threshold
try:
settings.temporal_search_similarity_threshold = 0.45

# Instantiate Neo4jClient without real connections
client = Neo4jClient(uri="bolt://localhost:7687", username="neo4j", password="password")

# Mock embedding function
client._embedding_fn = lambda text: [0.1, 0.2]

# Mock session & driver behaviour so query does not hit a real Neo4j server
class MockSession:
def __enter__(self):
return self

def __exit__(self, exc_type, exc_val, exc_tb):
pass

def run(self, query, **params):
# Verify that the similarity_threshold parameter passed to the query
# is indeed custom loaded from settings.temporal_search_similarity_threshold
assert params["similarity_threshold"] == 0.45
return []

monkeypatch.setattr(client, "_session", lambda: MockSession())

# Trigger search without specifying similarity_threshold explicitly
client.search_events_by_embedding(user_id="user-1", query_text="yesterday I did VOS")

finally:
settings.temporal_search_similarity_threshold = original_threshold
Comment thread
ved015 marked this conversation as resolved.
Resolved
Loading