Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
142 commits
Select commit Hold shift + click to select a range
78f7979
chore(bench): extraction-quality benchmark data update (run #261)
Nov 3, 2026
b2b4c47
chore(bench): hnsw benchmark data update (run #490)
Nov 3, 2026
215152f
chore(bench): ranking-quality benchmark data update (run #490)
Nov 3, 2026
4596514
chore(bench): write-pipeline benchmark data update (run #490)
Nov 3, 2026
e92861a
chore(bench): proxy benchmark data update (run #1876)
Nov 3, 2026
6f111a1
chore(bench): proxy benchmark data update (run #1888)
Nov 4, 2026
729466c
chore(bench): hnsw benchmark data update (run #505)
Nov 4, 2026
7de95cb
chore(bench): ranking-quality benchmark data update (run #505)
Nov 4, 2026
7da5709
chore(bench): write-pipeline benchmark data update (run #505)
Nov 4, 2026
479dedd
chore(bench): proxy benchmark data update (run #1908)
Nov 4, 2026
fb9807f
chore(bench): hnsw benchmark data update (run #520)
Nov 4, 2026
ca68fb7
chore(bench): ranking-quality benchmark data update (run #520)
Nov 4, 2026
13c48fe
chore(bench): write-pipeline benchmark data update (run #520)
Nov 4, 2026
7d69dba
chore(bench): hnsw benchmark data update (run #524)
Nov 4, 2026
eb9fe04
chore(bench): ranking-quality benchmark data update (run #524)
Nov 4, 2026
99772b0
chore(bench): proxy benchmark data update (run #1912)
Nov 4, 2026
53f87ab
chore(bench): write-pipeline benchmark data update (run #524)
Nov 4, 2026
c203c7e
chore(bench): hnsw benchmark data update (run #532)
Nov 5, 2026
f1b5186
chore(bench): ranking-quality benchmark data update (run #532)
Nov 5, 2026
e797883
chore(bench): write-pipeline benchmark data update (run #532)
Nov 5, 2026
3710822
chore(bench): proxy benchmark data update (run #1926)
Nov 5, 2026
8da4967
chore(bench): hnsw benchmark data update (run #538)
Nov 5, 2026
54f9e80
chore(bench): ranking-quality benchmark data update (run #538)
Nov 5, 2026
a63c4b9
chore(bench): write-pipeline benchmark data update (run #538)
Nov 5, 2026
2ebae3c
chore(bench): proxy benchmark data update (run #1931)
Nov 6, 2026
5470568
chore(bench): proxy benchmark data update (run #1938)
Nov 6, 2026
660f889
chore(bench): proxy benchmark data update (run #1943)
Nov 6, 2026
85851ef
chore(bench): extraction-quality benchmark data update (run #290)
Nov 6, 2026
421c239
chore(bench): proxy benchmark data update (run #1948)
Nov 6, 2026
5386697
chore(bench): hnsw benchmark data update (run #542)
Nov 6, 2026
65e392f
chore(bench): ranking-quality benchmark data update (run #542)
Nov 6, 2026
8f74332
chore(bench): write-pipeline benchmark data update (run #542)
Nov 6, 2026
2de06ad
chore(bench): hnsw benchmark data update (run #554)
Nov 7, 2026
91531f6
chore(bench): ranking-quality benchmark data update (run #554)
Nov 7, 2026
8e56635
chore(bench): write-pipeline benchmark data update (run #554)
Nov 7, 2026
4fe463d
chore(bench): proxy benchmark data update (run #1966)
Nov 7, 2026
3299235
chore(bench): extraction-quality benchmark data update (run #299)
Nov 8, 2026
710fa11
chore(bench): proxy benchmark data update (run #1983)
Nov 8, 2026
b303816
chore(bench): proxy benchmark data update (run #1987)
Nov 8, 2026
afdc950
chore(bench): hnsw benchmark data update (run #559)
Nov 8, 2026
3bb0fd0
chore(bench): ranking-quality benchmark data update (run #559)
Nov 8, 2026
daff0e6
chore(bench): write-pipeline benchmark data update (run #559)
Nov 8, 2026
e340b0e
chore(bench): hnsw benchmark data update (run #565)
Nov 9, 2026
4779c45
chore(bench): ranking-quality benchmark data update (run #565)
Nov 9, 2026
d6a4c9c
chore(bench): write-pipeline benchmark data update (run #565)
Nov 9, 2026
b9531fa
chore(bench): proxy benchmark data update (run #2000)
Nov 10, 2026
1789b02
chore(bench): proxy benchmark data update (run #2008)
Nov 10, 2026
d9124d2
chore(bench): proxy benchmark data update (run #2017)
Nov 11, 2026
80f0454
chore(bench): hnsw benchmark data update (run #573)
Nov 11, 2026
1e64763
chore(bench): ranking-quality benchmark data update (run #573)
Nov 11, 2026
1767a88
chore(bench): write-pipeline benchmark data update (run #573)
Nov 11, 2026
103d3ad
chore(bench): proxy benchmark data update (run #2020)
Nov 11, 2026
860379a
chore(bench): proxy benchmark data update (run #2038)
Nov 11, 2026
b1f89bf
chore(bench): hnsw benchmark data update (run #582)
Nov 11, 2026
5105bcf
chore(bench): ranking-quality benchmark data update (run #582)
Nov 11, 2026
8a1b9e1
chore(bench): write-pipeline benchmark data update (run #582)
Nov 11, 2026
88616d3
chore(bench): proxy benchmark data update (run #2042)
Nov 11, 2026
5c38f0f
chore(bench): extraction-quality benchmark data update (run #330)
Nov 11, 2026
2e8d4b2
chore(bench): proxy benchmark data update (run #2052)
Nov 11, 2026
316c2ad
chore(bench): proxy benchmark data update (run #2057)
Nov 12, 2026
a0bca8e
chore(bench): proxy benchmark data update (run #2061)
Nov 12, 2026
c52bdb6
chore(bench): hnsw benchmark data update (run #585)
Nov 12, 2026
cc2b5f0
chore(bench): ranking-quality benchmark data update (run #585)
Nov 12, 2026
3227f07
chore(bench): write-pipeline benchmark data update (run #585)
Nov 12, 2026
35ac604
chore(bench): proxy benchmark data update (run #2065)
Nov 12, 2026
7f5d87c
chore(bench): proxy benchmark data update (run #2071)
Nov 12, 2026
b618015
chore(bench): proxy benchmark data update (run #2078)
Nov 13, 2026
650d627
chore(bench): proxy benchmark data update (run #2082)
Nov 13, 2026
b2b049c
chore(bench): proxy benchmark data update (run #2084)
Nov 13, 2026
cd50c5c
chore(bench): hnsw benchmark data update (run #598)
Nov 13, 2026
b30ff7f
chore(bench): ranking-quality benchmark data update (run #598)
Nov 13, 2026
42b191b
chore(bench): write-pipeline benchmark data update (run #598)
Nov 13, 2026
cccf58f
chore(bench): proxy benchmark data update (run #2089)
Nov 13, 2026
213f017
chore(bench): hnsw benchmark data update (run #618)
Nov 14, 2026
a4e5448
chore(bench): ranking-quality benchmark data update (run #618)
Nov 14, 2026
0772a9d
chore(bench): write-pipeline benchmark data update (run #618)
Nov 14, 2026
19ea364
chore(bench): proxy benchmark data update (run #2104)
Nov 14, 2026
98bf3e1
chore(bench): hnsw benchmark data update (run #628)
Nov 14, 2026
f1ceac2
chore(bench): ranking-quality benchmark data update (run #628)
Nov 14, 2026
7c8ce8a
chore(bench): write-pipeline benchmark data update (run #628)
Nov 14, 2026
7180b05
chore(bench): proxy benchmark data update (run #2111)
Nov 14, 2026
312dfab
chore(bench): proxy benchmark data update (run #2119)
Nov 14, 2026
9f5c419
chore(bench): hnsw benchmark data update (run #639)
Nov 14, 2026
27cc268
chore(bench): ranking-quality benchmark data update (run #639)
Nov 14, 2026
c2546f5
chore(bench): write-pipeline benchmark data update (run #639)
Nov 14, 2026
2628fbc
chore(bench): extraction-quality benchmark data update (run #359)
Nov 15, 2026
0d9ec54
chore(bench): hnsw benchmark data update (run #655)
Nov 15, 2026
3c9acb5
chore(bench): ranking-quality benchmark data update (run #655)
Nov 15, 2026
580797a
chore(bench): write-pipeline benchmark data update (run #655)
Nov 15, 2026
28397fd
chore(bench): proxy benchmark data update (run #2134)
Nov 15, 2026
1b83b61
chore(bench): hnsw benchmark data update (run #665)
Nov 15, 2026
43fa22a
chore(bench): ranking-quality benchmark data update (run #665)
Nov 15, 2026
2e7a488
chore(bench): write-pipeline benchmark data update (run #665)
Nov 15, 2026
a63e4ab
chore(bench): hnsw benchmark data update (run #675)
Nov 15, 2026
e34aef0
chore(bench): ranking-quality benchmark data update (run #675)
Nov 15, 2026
51613dc
chore(bench): write-pipeline benchmark data update (run #675)
Nov 15, 2026
19aa462
chore(bench): proxy benchmark data update (run #2143)
Nov 16, 2026
d616ae7
chore(bench): hnsw benchmark data update (run #680)
Nov 16, 2026
222e4df
chore(bench): ranking-quality benchmark data update (run #680)
Nov 16, 2026
2a433ae
chore(bench): write-pipeline benchmark data update (run #680)
Nov 16, 2026
08a6795
chore(bench): hnsw benchmark data update (run #710)
Nov 17, 2026
d76f399
chore(bench): ranking-quality benchmark data update (run #710)
Nov 17, 2026
8bb492e
chore(bench): write-pipeline benchmark data update (run #710)
Nov 17, 2026
7dc854b
chore(bench): proxy benchmark data update (run #2154)
Nov 17, 2026
d35008e
chore(bench): hnsw benchmark data update (run #714)
Nov 17, 2026
dda6529
chore(bench): ranking-quality benchmark data update (run #714)
Nov 17, 2026
25df568
chore(bench): write-pipeline benchmark data update (run #714)
Nov 17, 2026
c607cf5
chore(bench): proxy benchmark data update (run #2162)
Nov 17, 2026
2144794
chore(bench): hnsw benchmark data update (run #723)
Nov 17, 2026
3ce909d
chore(bench): ranking-quality benchmark data update (run #723)
Nov 17, 2026
cbc9b4d
chore(bench): write-pipeline benchmark data update (run #723)
Nov 17, 2026
995c6d5
chore(bench): proxy benchmark data update (run #2171)
Nov 17, 2026
833aa8e
chore(bench): hnsw benchmark data update (run #748)
Nov 17, 2026
21d2ce2
chore(bench): ranking-quality benchmark data update (run #748)
Nov 17, 2026
7ed1e3c
chore(bench): write-pipeline benchmark data update (run #748)
Nov 17, 2026
450621a
chore(bench): proxy benchmark data update (run #2182)
Nov 17, 2026
403d872
chore(bench): hnsw benchmark data update (run #762)
Nov 17, 2026
f47879b
chore(bench): ranking-quality benchmark data update (run #762)
Nov 17, 2026
257a59e
chore(bench): write-pipeline benchmark data update (run #762)
Nov 17, 2026
b26fd22
chore(bench): proxy benchmark data update (run #2197)
Nov 19, 2026
e4087c3
chore(bench): hnsw benchmark data update (run #796)
Nov 19, 2026
e9a3f30
chore(bench): ranking-quality benchmark data update (run #796)
Nov 19, 2026
2a00398
chore(bench): write-pipeline benchmark data update (run #796)
Nov 19, 2026
af2defb
chore(bench): proxy benchmark data update (run #2203)
Nov 19, 2026
69da6bb
chore(bench): proxy benchmark data update (run #2210)
Nov 19, 2026
d4f3423
chore(bench): hnsw benchmark data update (run #868)
Nov 20, 2026
72ab0a0
chore(bench): ranking-quality benchmark data update (run #868)
Nov 20, 2026
8b63024
chore(bench): write-pipeline benchmark data update (run #868)
Nov 20, 2026
091d10e
chore(bench): proxy benchmark data update (run #2220)
Nov 20, 2026
116f277
chore(bench): extraction-quality benchmark data update (run #415)
Nov 20, 2026
f897645
chore(bench): proxy benchmark data update (run #2226)
Nov 20, 2026
b27ec0a
chore(bench): hnsw benchmark data update (run #884)
Nov 20, 2026
74e94fe
chore(bench): ranking-quality benchmark data update (run #884)
Nov 20, 2026
b936ba5
chore(bench): write-pipeline benchmark data update (run #884)
Nov 20, 2026
53f4e83
chore(bench): proxy benchmark data update (run #2236)
Nov 20, 2026
b008def
chore(bench): extraction-quality benchmark data update (run #423)
Nov 21, 2026
4008eed
chore(bench): proxy benchmark data update (run #2241)
Nov 21, 2026
b3d9f85
chore(bench): hnsw benchmark data update (run #912)
Nov 21, 2026
00b90cf
chore(bench): ranking-quality benchmark data update (run #912)
Nov 21, 2026
54c969f
chore(bench): write-pipeline benchmark data update (run #912)
Nov 21, 2026
966ae06
chore(bench): proxy benchmark data update (run #2248)
Nov 21, 2026
e488dc7
chore(bench): proxy benchmark data update (run #2250)
Nov 21, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions site/public/benchmarks/badge.svg
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
195 changes: 195 additions & 0 deletions site/public/benchmarks/benchmark-data.json
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,201 @@
"schema_version": 1,
"baseline_sha": "48ea560b8e757aa7114a84c103dacdbef8ee8d34",
"runs": [
{
"sha": "f2a037609e58960cea41a6de71be54a27973d16d",
"short_sha": "f2a0376",
"timestamp": "2026-11-21T12:02:38.739857+00:00",
"branch": "main",
"pr_number": null,
"run_url": "https://github.com/Rick1330/ibex-harness/actions/runs/44901661021",
"status": "pass",
"regression_vs_baseline_pct": null,
"baseline_sha": "48ea560b8e757aa7114a84c103dacdbef8ee8d34",
"profile": "fast",
"run_number": 2250,
"go_version": "1.26.11",
"runner_os": "Linux",
"runner_cpu": "x86_64",
"runner_vcpus": 2,
"runner_ram_gb": 7,
"k6_version": "0.53.0",
"k6": {
"vus": 100,
"duration_s": 120.0,
"p50_ms": 0.566077,
"p95_ms": 0.9614324999999998,
"p99_ms": 1.5700980599999992,
"p999_ms": 2.4404992600000037,
"req_per_s": 2240.595788525093,
"error_rate": 0.0,
"check_rate": 1.0,
"throughput_series": [
{
"t_s": 0.0,
"req_per_s": 0.0
},
{
"t_s": 10.0,
"req_per_s": 1305.98
},
{
"t_s": 20.0,
"req_per_s": 2071.31
},
{
"t_s": 30.0,
"req_per_s": 1921.13
},
{
"t_s": 40.0,
"req_per_s": 2239.28
},
{
"t_s": 50.0,
"req_per_s": 2307.47
},
{
"t_s": 60.0,
"req_per_s": 1971.75
},
{
"t_s": 70.0,
"req_per_s": 1990.07
},
{
"t_s": 80.0,
"req_per_s": 2321.07
},
{
"t_s": 90.0,
"req_per_s": 2217.46
},
{
"t_s": 100.0,
"req_per_s": 1913.16
},
{
"t_s": 110.0,
"req_per_s": 2095.18
},
{
"t_s": 120.0,
"req_per_s": 2352.58
}
]
},
"stages": {
"auth_lru_p50_ms": 0.000205,
"auth_lru_p95_ms": 0.000317,
"auth_lru_p99_ms": 0.000373,
"auth_lru_p999_ms": 0.000504,
"auth_grpc_p50_ms": 8.8e-05,
"auth_grpc_p95_ms": 0.000136,
"auth_grpc_p99_ms": 0.00016,
"auth_grpc_p999_ms": 0.000216,
"rate_limit_p50_ms": 0.068062,
"rate_limit_p95_ms": 0.105187,
"rate_limit_p99_ms": 0.123749,
"rate_limit_p999_ms": 0.167061,
"directive_resolve_p50_ms": 0.023828,
"directive_resolve_p95_ms": 0.036825,
"directive_resolve_p99_ms": 0.043323,
"directive_resolve_p999_ms": 0.058486,
"prompt_inject_p50_ms": 2.1e-05,
"prompt_inject_p95_ms": 3.3e-05,
"prompt_inject_p99_ms": 3.8e-05,
"prompt_inject_p999_ms": 5.2e-05,
"total_overhead_p50_ms": 0.097061,
"total_overhead_p95_ms": 0.150004,
"total_overhead_p99_ms": 0.176475,
"total_overhead_p999_ms": 0.238241
},
"metric_deltas": {},
"go_benchmarks": {
"BenchmarkStageAuth": {
"ns_per_op": 532.9,
"allocs_per_op": 5.0,
"bytes_per_op": 336.0,
"samples": 5,
"ci_95_low": 506.25499999999994,
"ci_95_high": 559.545,
"geomean_ns": 532.9
},
"BenchmarkStageRateLimit": {
"ns_per_op": 123749.0,
"allocs_per_op": 819.0,
"bytes_per_op": 196276.0,
"samples": 5,
"ci_95_low": 117561.54999999999,
"ci_95_high": 129936.45000000001,
"geomean_ns": 123749.0
},
"BenchmarkStageDirectiveResolve": {
"ns_per_op": 43323.0,
"allocs_per_op": 37.0,
"bytes_per_op": 1552.0,
"samples": 5,
"ci_95_low": 41156.85,
"ci_95_high": 45489.15,
"geomean_ns": 43323.0
},
"BenchmarkStagePromptInject": {
"ns_per_op": 38.35,
"allocs_per_op": 1.0,
"bytes_per_op": 96.0,
"samples": 5,
"ci_95_low": 36.4325,
"ci_95_high": 40.267500000000005,
"geomean_ns": 38.35
},
"BenchmarkProxyOverhead": {
"ns_per_op": 176475.0,
"allocs_per_op": 862.0,
"bytes_per_op": 198260.0,
"samples": 5,
"ci_95_low": 167651.25,
"ci_95_high": 185298.75,
"geomean_ns": 176475.0
},
"BenchmarkProxyChatOverhead": {
"ns_per_op": 251656.0,
"allocs_per_op": 1104.0,
"bytes_per_op": 230585.0,
"samples": 5,
"ci_95_low": 239073.19999999998,
"ci_95_high": 264238.8,
"geomean_ns": 251656.0
},
"BenchmarkProxyChatOverheadParallel": {
"ns_per_op": 118065.0,
"allocs_per_op": 1104.0,
"bytes_per_op": 230630.0,
"samples": 5,
"ci_95_low": 112161.75,
"ci_95_high": 123968.25,
"geomean_ns": 118065.0
},
"BenchmarkProxyChatOverheadWithAssemble": {
"ns_per_op": 258374.0,
"allocs_per_op": 1122.0,
"bytes_per_op": 231234.0,
"samples": 5,
"ci_95_low": 245455.3,
"ci_95_high": 271292.7,
"geomean_ns": 258374.0
},
"BenchmarkProxyHealth": {
"ns_per_op": 4086.0,
"allocs_per_op": 44.0,
"bytes_per_op": 4344.0,
"samples": 5,
"ci_95_low": 3881.7,
"ci_95_high": 4290.3,
"geomean_ns": 4086.0
}
},
"stage_model": "go_microbench_warm_path"
},
{
"sha": "d893324303b8014cea855d6d0307c94b13b9b9b9",
"short_sha": "d893324",
Expand Down
148 changes: 148 additions & 0 deletions site/public/benchmarks/extraction-quality-benchmark-data.json
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,154 @@
"schema_version": 1,
"benchmark": "extraction_quality",
"runs": [
{
"sha": "0a8ae778e335a818b3e6f20dcf0ca3905b5d23cd",
"short_sha": "0a8ae77",
"timestamp": "2026-11-21T01:10:57.184675+00:00",
"branch": "main",
"run_number": 423,
"run_url": "https://github.com/Rick1330/ibex-harness/actions/runs/44848498711",
"gold_set": "v1",
"conversation_count": 125,
"provider": "openai",
"enforcement": "ci",
"mode": "cassette",
"model": "gpt-4o-mini",
"metrics": {
"precision_macro": 1.0,
"recall_macro": 1.0,
"category_assignment_accuracy": 1.0,
"temporal_field_accuracy": 1.0,
"precision_factual": 1.0,
"recall_factual": 1.0,
"precision_preference": 1.0,
"recall_preference": 1.0,
"precision_behavioral": 1.0,
"recall_behavioral": 1.0,
"precision_episodic": 1.0,
"recall_episodic": 1.0,
"precision_procedural": 1.0,
"recall_procedural": 1.0
},
"status": "pass",
"gate_summary": {
"status": "pass",
"checks": [
{
"name": "openai.precision_factual regression (pp)",
"value": 0.0,
"limit": 3.0,
"ok": true,
"current": 1.0,
"baseline": 1.0
},
{
"name": "openai.recall_factual regression (pp)",
"value": 0.0,
"limit": 3.0,
"ok": true,
"current": 1.0,
"baseline": 1.0
},
{
"name": "openai.precision_preference regression (pp)",
"value": 0.0,
"limit": 3.0,
"ok": true,
"current": 1.0,
"baseline": 1.0
},
{
"name": "openai.recall_preference regression (pp)",
"value": 0.0,
"limit": 3.0,
"ok": true,
"current": 1.0,
"baseline": 1.0
},
{
"name": "openai.precision_behavioral regression (pp)",
"value": 0.0,
"limit": 3.0,
"ok": true,
"current": 1.0,
"baseline": 1.0
},
{
"name": "openai.recall_behavioral regression (pp)",
"value": 0.0,
"limit": 3.0,
"ok": true,
"current": 1.0,
"baseline": 1.0
},
{
"name": "openai.precision_episodic regression (pp)",
"value": 0.0,
"limit": 3.0,
"ok": true,
"current": 1.0,
"baseline": 1.0
},
{
"name": "openai.recall_episodic regression (pp)",
"value": 0.0,
"limit": 3.0,
"ok": true,
"current": 1.0,
"baseline": 1.0
},
{
"name": "openai.precision_procedural regression (pp)",
"value": 0.0,
"limit": 3.0,
"ok": true,
"current": 1.0,
"baseline": 1.0
},
{
"name": "openai.recall_procedural regression (pp)",
"value": 0.0,
"limit": 3.0,
"ok": true,
"current": 1.0,
"baseline": 1.0
},
{
"name": "openai.category_assignment_accuracy regression (pp)",
"value": 0.0,
"limit": 3.0,
"ok": true,
"current": 1.0,
"baseline": 1.0
},
{
"name": "openai.temporal_field_accuracy regression (pp)",
"value": 0.0,
"limit": 3.0,
"ok": true,
"current": 1.0,
"baseline": 1.0
},
{
"name": "openai.precision_macro regression (pp)",
"value": 0.0,
"limit": 3.0,
"ok": true,
"current": 1.0,
"baseline": 1.0
},
{
"name": "openai.recall_macro regression (pp)",
"value": 0.0,
"limit": 3.0,
"ok": true,
"current": 1.0,
"baseline": 1.0
}
]
}
},
{
"sha": "39462d2bd68ecfc32eff54cfb6bf81a451bd211d",
"short_sha": "39462d2",
Expand Down
Loading
Loading