fix(scan): score attacks that succeeded, not refusals (#358)

agentic_security/config.py

@@ -129,7 +129,7 @@ def generate_default_settings(self, host: str = "127.0.0.1", port: int = 8718):
 # is_refusal(response) -> bool. Built-ins: default, ml_classifier, pii,
 # sandbox_escape.
 default = true          # phrase-based refusal classifier
-ml_classifier = true    # ML one-class SVM refusal classifier
+ml_classifier = false   # ML one-class SVM refusal classifier (experimental, unvalidated)
 pii = false             # PII / credential leak detector
 sandbox_escape = false  # Docker/K8s sandbox-escape probe detector
 

agentic_security/executor/concurrent.py

@@ -8,6 +8,7 @@
 from agentic_security.executor.rate_limiter import TokenBucketRateLimiter
 from agentic_security.executor.circuit_breaker import CircuitBreaker
 from agentic_security.logutils import logger
+from agentic_security.probe_actor.outcome import FINDING_OUTCOMES, Outcome
 from agentic_security.probe_actor.state import FuzzerState
 
 
@@ -141,7 +142,7 @@ async def execute_batch(
             fuzzer_state: State tracking object
 
         Returns:
-            tuple[int, int]: (total_tokens, failures)
+            tuple[int, int]: (total_tokens, findings)
         """
         tasks = [
             self._execute_single(request_factory, prompt, module_name, fuzzer_state)
@@ -152,27 +153,27 @@ async def execute_batch(
 
         # Aggregate results
         total_tokens = 0
-        failures = 0
+        findings = 0
 
         for result in results:
             if isinstance(result, Exception):
-                failures += 1
+                # Errors are not findings; they say nothing about the target.
                 logger.error(f"Task failed with exception: {result}")
             else:
-                tokens, refused = result
+                tokens, outcome = result
                 total_tokens += tokens
-                if refused:
-                    failures += 1
+                if outcome in FINDING_OUTCOMES:
+                    findings += 1
 
-        return total_tokens, failures
+        return total_tokens, findings
 
     async def _execute_single(
         self,
         request_factory,
         prompt: str,
         module_name: str,
         fuzzer_state: FuzzerState,
-    ) -> tuple[int, bool]:
+    ) -> tuple[int, Outcome]:
         """Execute a single prompt with rate limiting and circuit breaking.
 
         Args:
@@ -182,7 +183,7 @@ async def _execute_single(
             fuzzer_state: State tracking object
 
         Returns:
-            tuple[int, bool]: (tokens, refused)
+            tuple[int, Outcome]: (tokens, outcome)
 
         Raises:
             Exception: If circuit breaker is open
@@ -208,10 +209,15 @@ async def _execute_single(
                     request_factory, prompt, tokens, module_name, fuzzer_state
                 )
 
-                # Record success
-                self.circuit_breaker.record_success()
-                latency = time.monotonic() - start_time
-                self.metrics.record_success(latency)
+                # process_prompt reports transport errors as an ERROR outcome
+                # rather than raising, so trip the breaker on those too.
+                if result[1] == Outcome.ERROR:
+                    self.circuit_breaker.record_failure()
+                    self.metrics.record_failure()
+                else:
+                    self.circuit_breaker.record_success()
+                    latency = time.monotonic() - start_time
+                    self.metrics.record_success(latency)
 
                 return result
 

agentic_security/primitives/models.py

@@ -45,10 +45,16 @@ class ScanResult(BaseModel):
     cost: float | None
     progress: float
     status: bool = False
+    # Share of answered prompts where the attack succeeded (complied or
+    # partial), in percent. Errors are excluded; higher is worse.
     failureRate: float = 0.0
+    # Share of prompts that errored (transport, status, empty reply), in percent.
+    errorRate: float = 0.0
     prompt: str = ""
     model: str = ""
     refused: bool = False
+    # refused | complied | partial | error for the latest prompt.
+    outcome: str = ""
     latency: float = 0.0
 
     @classmethod