diff --git a/site/public/benchmarks/badge.svg b/site/public/benchmarks/badge.svg index e4bd25379..f7bb218a4 100644 --- a/site/public/benchmarks/badge.svg +++ b/site/public/benchmarks/badge.svg @@ -1,7 +1,7 @@ - - benchmarks pass p99 1.75ms + + benchmarks pass p99 1.57ms benchmarks - pass p99 1.75ms + pass p99 1.57ms diff --git a/site/public/benchmarks/benchmark-data.json b/site/public/benchmarks/benchmark-data.json index 34081eaaa..4d6bc0a4a 100644 --- a/site/public/benchmarks/benchmark-data.json +++ b/site/public/benchmarks/benchmark-data.json @@ -2,6 +2,201 @@ "schema_version": 1, "baseline_sha": "48ea560b8e757aa7114a84c103dacdbef8ee8d34", "runs": [ + { + "sha": "f2a037609e58960cea41a6de71be54a27973d16d", + "short_sha": "f2a0376", + "timestamp": "2026-11-21T12:02:38.739857+00:00", + "branch": "main", + "pr_number": null, + "run_url": "https://github.com/Rick1330/ibex-harness/actions/runs/44901661021", + "status": "pass", + "regression_vs_baseline_pct": null, + "baseline_sha": "48ea560b8e757aa7114a84c103dacdbef8ee8d34", + "profile": "fast", + "run_number": 2250, + "go_version": "1.26.11", + "runner_os": "Linux", + "runner_cpu": "x86_64", + "runner_vcpus": 2, + "runner_ram_gb": 7, + "k6_version": "0.53.0", + "k6": { + "vus": 100, + "duration_s": 120.0, + "p50_ms": 0.566077, + "p95_ms": 0.9614324999999998, + "p99_ms": 1.5700980599999992, + "p999_ms": 2.4404992600000037, + "req_per_s": 2240.595788525093, + "error_rate": 0.0, + "check_rate": 1.0, + "throughput_series": [ + { + "t_s": 0.0, + "req_per_s": 0.0 + }, + { + "t_s": 10.0, + "req_per_s": 1305.98 + }, + { + "t_s": 20.0, + "req_per_s": 2071.31 + }, + { + "t_s": 30.0, + "req_per_s": 1921.13 + }, + { + "t_s": 40.0, + "req_per_s": 2239.28 + }, + { + "t_s": 50.0, + "req_per_s": 2307.47 + }, + { + "t_s": 60.0, + "req_per_s": 1971.75 + }, + { + "t_s": 70.0, + "req_per_s": 1990.07 + }, + { + "t_s": 80.0, + "req_per_s": 2321.07 + }, + { + "t_s": 90.0, + "req_per_s": 2217.46 + }, + { + "t_s": 100.0, + "req_per_s": 1913.16 + }, + { + "t_s": 110.0, + "req_per_s": 2095.18 + }, + { + "t_s": 120.0, + "req_per_s": 2352.58 + } + ] + }, + "stages": { + "auth_lru_p50_ms": 0.000205, + "auth_lru_p95_ms": 0.000317, + "auth_lru_p99_ms": 0.000373, + "auth_lru_p999_ms": 0.000504, + "auth_grpc_p50_ms": 8.8e-05, + "auth_grpc_p95_ms": 0.000136, + "auth_grpc_p99_ms": 0.00016, + "auth_grpc_p999_ms": 0.000216, + "rate_limit_p50_ms": 0.068062, + "rate_limit_p95_ms": 0.105187, + "rate_limit_p99_ms": 0.123749, + "rate_limit_p999_ms": 0.167061, + "directive_resolve_p50_ms": 0.023828, + "directive_resolve_p95_ms": 0.036825, + "directive_resolve_p99_ms": 0.043323, + "directive_resolve_p999_ms": 0.058486, + "prompt_inject_p50_ms": 2.1e-05, + "prompt_inject_p95_ms": 3.3e-05, + "prompt_inject_p99_ms": 3.8e-05, + "prompt_inject_p999_ms": 5.2e-05, + "total_overhead_p50_ms": 0.097061, + "total_overhead_p95_ms": 0.150004, + "total_overhead_p99_ms": 0.176475, + "total_overhead_p999_ms": 0.238241 + }, + "metric_deltas": {}, + "go_benchmarks": { + "BenchmarkStageAuth": { + "ns_per_op": 532.9, + "allocs_per_op": 5.0, + "bytes_per_op": 336.0, + "samples": 5, + "ci_95_low": 506.25499999999994, + "ci_95_high": 559.545, + "geomean_ns": 532.9 + }, + "BenchmarkStageRateLimit": { + "ns_per_op": 123749.0, + "allocs_per_op": 819.0, + "bytes_per_op": 196276.0, + "samples": 5, + "ci_95_low": 117561.54999999999, + "ci_95_high": 129936.45000000001, + "geomean_ns": 123749.0 + }, + "BenchmarkStageDirectiveResolve": { + "ns_per_op": 43323.0, + "allocs_per_op": 37.0, + "bytes_per_op": 1552.0, + "samples": 5, + "ci_95_low": 41156.85, + "ci_95_high": 45489.15, + "geomean_ns": 43323.0 + }, + "BenchmarkStagePromptInject": { + "ns_per_op": 38.35, + "allocs_per_op": 1.0, + "bytes_per_op": 96.0, + "samples": 5, + "ci_95_low": 36.4325, + "ci_95_high": 40.267500000000005, + "geomean_ns": 38.35 + }, + "BenchmarkProxyOverhead": { + "ns_per_op": 176475.0, + "allocs_per_op": 862.0, + "bytes_per_op": 198260.0, + "samples": 5, + "ci_95_low": 167651.25, + "ci_95_high": 185298.75, + "geomean_ns": 176475.0 + }, + "BenchmarkProxyChatOverhead": { + "ns_per_op": 251656.0, + "allocs_per_op": 1104.0, + "bytes_per_op": 230585.0, + "samples": 5, + "ci_95_low": 239073.19999999998, + "ci_95_high": 264238.8, + "geomean_ns": 251656.0 + }, + "BenchmarkProxyChatOverheadParallel": { + "ns_per_op": 118065.0, + "allocs_per_op": 1104.0, + "bytes_per_op": 230630.0, + "samples": 5, + "ci_95_low": 112161.75, + "ci_95_high": 123968.25, + "geomean_ns": 118065.0 + }, + "BenchmarkProxyChatOverheadWithAssemble": { + "ns_per_op": 258374.0, + "allocs_per_op": 1122.0, + "bytes_per_op": 231234.0, + "samples": 5, + "ci_95_low": 245455.3, + "ci_95_high": 271292.7, + "geomean_ns": 258374.0 + }, + "BenchmarkProxyHealth": { + "ns_per_op": 4086.0, + "allocs_per_op": 44.0, + "bytes_per_op": 4344.0, + "samples": 5, + "ci_95_low": 3881.7, + "ci_95_high": 4290.3, + "geomean_ns": 4086.0 + } + }, + "stage_model": "go_microbench_warm_path" + }, { "sha": "d893324303b8014cea855d6d0307c94b13b9b9b9", "short_sha": "d893324", diff --git a/site/public/benchmarks/extraction-quality-benchmark-data.json b/site/public/benchmarks/extraction-quality-benchmark-data.json index b3649122f..c06e8678f 100644 --- a/site/public/benchmarks/extraction-quality-benchmark-data.json +++ b/site/public/benchmarks/extraction-quality-benchmark-data.json @@ -2,6 +2,154 @@ "schema_version": 1, "benchmark": "extraction_quality", "runs": [ + { + "sha": "0a8ae778e335a818b3e6f20dcf0ca3905b5d23cd", + "short_sha": "0a8ae77", + "timestamp": "2026-11-21T01:10:57.184675+00:00", + "branch": "main", + "run_number": 423, + "run_url": "https://github.com/Rick1330/ibex-harness/actions/runs/44848498711", + "gold_set": "v1", + "conversation_count": 125, + "provider": "openai", + "enforcement": "ci", + "mode": "cassette", + "model": "gpt-4o-mini", + "metrics": { + "precision_macro": 1.0, + "recall_macro": 1.0, + "category_assignment_accuracy": 1.0, + "temporal_field_accuracy": 1.0, + "precision_factual": 1.0, + "recall_factual": 1.0, + "precision_preference": 1.0, + "recall_preference": 1.0, + "precision_behavioral": 1.0, + "recall_behavioral": 1.0, + "precision_episodic": 1.0, + "recall_episodic": 1.0, + "precision_procedural": 1.0, + "recall_procedural": 1.0 + }, + "status": "pass", + "gate_summary": { + "status": "pass", + "checks": [ + { + "name": "openai.precision_factual regression (pp)", + "value": 0.0, + "limit": 3.0, + "ok": true, + "current": 1.0, + "baseline": 1.0 + }, + { + "name": "openai.recall_factual regression (pp)", + "value": 0.0, + "limit": 3.0, + "ok": true, + "current": 1.0, + "baseline": 1.0 + }, + { + "name": "openai.precision_preference regression (pp)", + "value": 0.0, + "limit": 3.0, + "ok": true, + "current": 1.0, + "baseline": 1.0 + }, + { + "name": "openai.recall_preference regression (pp)", + "value": 0.0, + "limit": 3.0, + "ok": true, + "current": 1.0, + "baseline": 1.0 + }, + { + "name": "openai.precision_behavioral regression (pp)", + "value": 0.0, + "limit": 3.0, + "ok": true, + "current": 1.0, + "baseline": 1.0 + }, + { + "name": "openai.recall_behavioral regression (pp)", + "value": 0.0, + "limit": 3.0, + "ok": true, + "current": 1.0, + "baseline": 1.0 + }, + { + "name": "openai.precision_episodic regression (pp)", + "value": 0.0, + "limit": 3.0, + "ok": true, + "current": 1.0, + "baseline": 1.0 + }, + { + "name": "openai.recall_episodic regression (pp)", + "value": 0.0, + "limit": 3.0, + "ok": true, + "current": 1.0, + "baseline": 1.0 + }, + { + "name": "openai.precision_procedural regression (pp)", + "value": 0.0, + "limit": 3.0, + "ok": true, + "current": 1.0, + "baseline": 1.0 + }, + { + "name": "openai.recall_procedural regression (pp)", + "value": 0.0, + "limit": 3.0, + "ok": true, + "current": 1.0, + "baseline": 1.0 + }, + { + "name": "openai.category_assignment_accuracy regression (pp)", + "value": 0.0, + "limit": 3.0, + "ok": true, + "current": 1.0, + "baseline": 1.0 + }, + { + "name": "openai.temporal_field_accuracy regression (pp)", + "value": 0.0, + "limit": 3.0, + "ok": true, + "current": 1.0, + "baseline": 1.0 + }, + { + "name": "openai.precision_macro regression (pp)", + "value": 0.0, + "limit": 3.0, + "ok": true, + "current": 1.0, + "baseline": 1.0 + }, + { + "name": "openai.recall_macro regression (pp)", + "value": 0.0, + "limit": 3.0, + "ok": true, + "current": 1.0, + "baseline": 1.0 + } + ] + } + }, { "sha": "39462d2bd68ecfc32eff54cfb6bf81a451bd211d", "short_sha": "39462d2", diff --git a/site/public/benchmarks/hnsw-benchmark-data.json b/site/public/benchmarks/hnsw-benchmark-data.json index 57c448c5c..4433de808 100644 --- a/site/public/benchmarks/hnsw-benchmark-data.json +++ b/site/public/benchmarks/hnsw-benchmark-data.json @@ -2,6 +2,100 @@ "schema_version": 1, "benchmark": "hnsw_recall_latency", "runs": [ + { + "sha": "0a8ae778e335a818b3e6f20dcf0ca3905b5d23cd", + "short_sha": "0a8ae77", + "timestamp": "2026-11-21T01:13:42.422633+00:00", + "branch": "main", + "run_number": 912, + "run_url": "https://github.com/Rick1330/ibex-harness/actions/runs/44848498709", + "methodology": { + "index": "idx_memories_embedding_hnsw (m=16, ef_construction=64, cosine)", + "ef_search": 64, + "dim": 1024, + "active_dims": 64, + "warmup_queries": 100, + "truncate_between_sizes": true, + "reuse_corpus_across_search_knobs": true, + "analyze_after_copy": true, + "explain_requires_hnsw": true, + "recall": "planted near-neighbor must appear in top-10", + "latency": "wall-clock PgVectorStore.search including SET LOCAL", + "iterative_scan_modes": [ + "off" + ], + "min_similarity_values": [ + 0.7 + ], + "index_build_modes": [ + "bulk" + ], + "note": "Published cells: ef_search=64, min_similarity=0.70, iterative_scan=off, index_build_mode=bulk. Full matrix retained under benchmarks/memory/output/." + }, + "results": [ + { + "corpus_size": 10000, + "query_count": 500, + "recall_at_10": 1.0, + "latency_ms_p50": 5.866558500009944, + "latency_ms_p95": 6.480069649991549, + "latency_ms_p99": 7.502704429991522, + "latency_ms_p95_ci_low": 6.282545999994227, + "latency_ms_p95_ci_high": 6.678929999992533, + "ef_search": 64, + "min_similarity": 0.7, + "iterative_scan": "off", + "index_build_mode": "bulk", + "plan_node_type": "Index Scan", + "plan_index_name": "idx_memories_embedding_hnsw", + "shared_hit_blocks": 7872, + "shared_read_blocks": 16, + "idx_scan_delta": 600, + "row_count_verified": 10000 + }, + { + "corpus_size": 100000, + "query_count": 200, + "recall_at_10": 1.0, + "latency_ms_p50": 9.690594499986105, + "latency_ms_p95": 10.946235299999783, + "latency_ms_p99": 12.209022049996165, + "latency_ms_p95_ci_low": 10.630923749978647, + "latency_ms_p95_ci_high": 11.26030699998637, + "ef_search": 64, + "min_similarity": 0.7, + "iterative_scan": "off", + "index_build_mode": "bulk", + "plan_node_type": "Index Scan", + "plan_index_name": "idx_memories_embedding_hnsw", + "shared_hit_blocks": 7696, + "shared_read_blocks": 128, + "idx_scan_delta": 300, + "row_count_verified": 100000 + } + ], + "mean_recall_at_10": 1.0, + "status": "pass", + "gate_summary": { + "recall_ok": true, + "recall_floor": 0.98, + "worst_recall_at_10": 1.0, + "has_1m": false, + "note": "1M cell absent (expected on smoke/fast profiles)" + }, + "provenance": { + "status": "recorded", + "producer": "benchmarks/memory/provenance.py", + "runner_cpu_model": "Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz", + "runner_cpu_count": 4, + "runner_ram_mib": 15982, + "corpus_hash": "fd8d1ddcf4e8fea94ce6d1866144b93a57ca50358beff34780f5bf652cebdfa2", + "corpus_hash_algo": "sha256", + "corpus_manifest": "pgvector|synth.py=8a2c9de8f80191dd9226c5e62f10cecda7b3b2d11e83b69c63964ab737c26f4a|corpus.py=87da85dd0a6a431c1826f205354ed18a381b1796b9a1bd3139c2cd119a2e18e9|dim=1024,active=64", + "corpus_status": "current", + "dirty": false + } + }, { "sha": "d893324303b8014cea855d6d0307c94b13b9b9b9", "short_sha": "d893324", diff --git a/site/public/benchmarks/ranking-quality-benchmark-data.json b/site/public/benchmarks/ranking-quality-benchmark-data.json index 11f9e3797..4b86adea4 100644 --- a/site/public/benchmarks/ranking-quality-benchmark-data.json +++ b/site/public/benchmarks/ranking-quality-benchmark-data.json @@ -2,6 +2,68 @@ "schema_version": 1, "benchmark": "ranking_quality", "runs": [ + { + "sha": "0a8ae778e335a818b3e6f20dcf0ca3905b5d23cd", + "short_sha": "0a8ae77", + "timestamp": "2026-11-21T01:12:08.309724+00:00", + "branch": "main", + "run_number": 912, + "run_url": "https://github.com/Rick1330/ibex-harness/actions/runs/44848498709", + "gold_set": "v1", + "query_count": 20, + "memory_count": 51, + "metrics": { + "precision_at_5": 1.0, + "recall_at_10": 1.0, + "mrr": 1.0, + "expected_order_match": 1.0, + "top_category_accuracy": 1.0 + }, + "status": "pass", + "gate_summary": { + "status": "pass", + "checks": [ + { + "name": "precision_at_5 regression (%)", + "value": 0.0, + "limit": -5.0, + "ok": true + }, + { + "name": "recall_at_10 regression (%)", + "value": 0.0, + "limit": -5.0, + "ok": true + }, + { + "name": "mrr regression (%)", + "value": 0.0, + "limit": -5.0, + "ok": true + }, + { + "name": "top_category_accuracy regression (%)", + "value": 0.0, + "limit": -5.0, + "ok": true + } + ] + }, + "provenance": { + "status": "recorded", + "producer": "benchmarks/memory/provenance.py", + "runner_cpu_model": "AMD EPYC 9V74 80-Core Processor", + "runner_cpu_count": 4, + "runner_ram_mib": 15983, + "corpus_hash": "5db8becdf6d24ad2e10efa7ca1ead4c1d43d961ed378e228366193d5064d4b92", + "corpus_hash_algo": "sha256", + "corpus_manifest": "gold_set_v1.json=5db8becdf6d24ad2e10efa7ca1ead4c1d43d961ed378e228366193d5064d4b92", + "corpus_status": "current", + "dirty": false, + "gold_set_hash": "5db8becdf6d24ad2e10efa7ca1ead4c1d43d961ed378e228366193d5064d4b92", + "gold_set_name": "gold_set_v1.json" + } + }, { "sha": "d893324303b8014cea855d6d0307c94b13b9b9b9", "short_sha": "d893324", diff --git a/site/public/benchmarks/write-pipeline-benchmark-data.json b/site/public/benchmarks/write-pipeline-benchmark-data.json index 85388fba4..b77750ec2 100644 --- a/site/public/benchmarks/write-pipeline-benchmark-data.json +++ b/site/public/benchmarks/write-pipeline-benchmark-data.json @@ -2,6 +2,38 @@ "schema_version": 1, "benchmark": "write_pipeline", "runs": [ + { + "sha": "0a8ae778e335a818b3e6f20dcf0ca3905b5d23cd", + "short_sha": "0a8ae77", + "timestamp": "2026-11-21T01:11:58.145658+00:00", + "branch": "main", + "run_number": 912, + "run_url": "https://github.com/Rick1330/ibex-harness/actions/runs/44848498709", + "iterations": 40, + "metrics": { + "latency_ms_p50": 19.225882500023772, + "latency_ms_p95": 67.05785665004363, + "latency_ms_p99": 148.90788961001306 + }, + "status": "pass", + "gate_summary": { + "status": "pass", + "checks": [ + { + "name": "p95 SLA (ms)", + "value": 67.05785665004363, + "limit": 200.0, + "ok": true + }, + { + "name": "latency_ms_p95 regression (%)", + "value": -66.30258459796804, + "limit": 20.0, + "ok": true + } + ] + } + }, { "sha": "d893324303b8014cea855d6d0307c94b13b9b9b9", "short_sha": "d893324",