diff --git a/app/Models/BenchmarkRun.php b/app/Models/BenchmarkRun.php index 513a38d..b03eec1 100644 --- a/app/Models/BenchmarkRun.php +++ b/app/Models/BenchmarkRun.php @@ -104,7 +104,7 @@ public function suiteResults(): Collection $model = Suites::modelFor($this->suite); if ($model === null) { - return collect(); + return new Collection; } $relation = Suites::runRelation($this->suite); @@ -128,18 +128,19 @@ public function scopeVisibleTo(Builder $query, ?User $user): Builder /** * The visibility predicate as a standalone closure, so joined queries - * (e.g. the flat results table) can apply the exact same rule without - * duplicating the logic. + * (e.g. the combined results table) can apply the exact same rule + * without duplicating the logic. Accepts either builder flavor: Eloquent + * scopes pass an Eloquent Builder, DB::table queries pass a Query Builder. * - * @return \Closure(Builder): Builder + * @return \Closure(\Illuminate\Database\Query\Builder|Builder): mixed */ public static function visibleClause(?User $user): \Closure { - return function (Builder $q) use ($user) { + return function ($q) use ($user) { $q->where('visibility', self::VISIBILITY_PUBLIC); if ($user !== null) { - $q->orWhere(fn (Builder $own) => $own + $q->orWhere(fn ($own) => $own ->where('visibility', self::VISIBILITY_PRIVATE) ->where('user_id', $user->id)); } diff --git a/database/migrations/2026_09_18_000001_reorganize_suite_tables.php b/database/migrations/2026_09_18_000001_reorganize_suite_tables.php index fec9ba5..996a0d0 100644 --- a/database/migrations/2026_09_18_000001_reorganize_suite_tables.php +++ b/database/migrations/2026_09_18_000001_reorganize_suite_tables.php @@ -2,6 +2,7 @@ use Illuminate\Database\Migrations\Migration; use Illuminate\Database\Schema\Blueprint; +use Illuminate\Support\Facades\Artisan; use Illuminate\Support\Facades\DB; use Illuminate\Support\Facades\Schema; @@ -135,6 +136,12 @@ public function up(): void $this->createPartitions(); } + + // Initial all_results view over the metric suites (active registry + // entries). Tests need it present right after migrate; production + // re-generates it via festbench:suites:refresh when the registry + // changes. + Artisan::call('festbench:suites:refresh'); } private function createPgTable(string $table, string $columns, array $extras): void @@ -186,11 +193,20 @@ private function createSqliteTable(string $table, array $extras): void $table->index('wall'); }); - // SQLite has no ALTER ADD COLUMN via builder for positioning; add extras. + // Suite-specific extras, declared with the proper builder calls. Schema::table($table, function (Blueprint $t) use ($extras) { foreach ($extras as $extra) { - [$name, $type] = explode(' ', $extra); - $t->addColumn($type, $name)->nullable(); + match (true) { + str_starts_with($extra, 'section ') => $t->string('section', 64)->nullable(), + str_starts_with($extra, 'tool_calls ') => $t->unsignedInteger('tool_calls')->nullable(), + str_starts_with($extra, 'kind ') => $t->string('kind', 32)->nullable(), + str_starts_with($extra, 'checks_passed ') => $t->unsignedInteger('checks_passed')->nullable(), + str_starts_with($extra, 'checks_total ') => $t->unsignedInteger('checks_total')->nullable(), + str_starts_with($extra, 'judge ') => $t->text('judge')->nullable(), + str_starts_with($extra, 'prefill ') => $t->double('prefill')->nullable(), + str_starts_with($extra, 'prompt_tokens ') => $t->unsignedInteger('prompt_tokens')->nullable(), + default => throw new RuntimeException("Unknown extra column: {$extra}"), + }; } }); } diff --git a/resources/views/livewire/my-runs/index.blade.php b/resources/views/livewire/my-runs/index.blade.php index ee8b9a9..8baf926 100644 --- a/resources/views/livewire/my-runs/index.blade.php +++ b/resources/views/livewire/my-runs/index.blade.php @@ -37,12 +37,21 @@ public function resetFilters(): void #[Computed] public function runs(): LengthAwarePaginator { + // Row + pass counts across the suite tables (runs are single-suite, + // but the owning table varies per run, hence the union of counts). return BenchmarkRun::query() ->where('user_id', auth()->id()) - ->withCount([ - 'results', - 'results as passed_count' => fn ($q) => $q->where('ok', true), - ]) + ->select('benchmark_runs.*') + ->selectRaw('(SELECT count(*) FROM softbench_results sr WHERE sr.benchmark_run_id = benchmark_runs.id) + + (SELECT count(*) FROM infrabench_results sr WHERE sr.benchmark_run_id = benchmark_runs.id) + + (SELECT count(*) FROM prosebench_results sr WHERE sr.benchmark_run_id = benchmark_runs.id) + + (SELECT count(*) FROM psychbench_results sr WHERE sr.benchmark_run_id = benchmark_runs.id) + + (SELECT count(*) FROM scenario_runs sr WHERE sr.benchmark_run_id = benchmark_runs.id) as results_count') + ->selectRaw('(SELECT count(*) FROM softbench_results sr WHERE sr.benchmark_run_id = benchmark_runs.id AND sr.ok) + + (SELECT count(*) FROM infrabench_results sr WHERE sr.benchmark_run_id = benchmark_runs.id AND sr.ok) + + (SELECT count(*) FROM prosebench_results sr WHERE sr.benchmark_run_id = benchmark_runs.id AND sr.ok) + + (SELECT count(*) FROM psychbench_results sr WHERE sr.benchmark_run_id = benchmark_runs.id AND sr.ok) + + (SELECT count(*) FROM scenario_runs sr WHERE sr.benchmark_run_id = benchmark_runs.id AND sr.ok) as passed_count') ->when($this->filters['search'], fn ($q, $search) => $q->where(function ($q) use ($search) { $q->where('run_id', 'like', "%{$search}%") ->orWhere('model_id', 'like', "%{$search}%") diff --git a/resources/views/livewire/results/index.blade.php b/resources/views/livewire/results/index.blade.php index f3a84e4..6f719fa 100644 --- a/resources/views/livewire/results/index.blade.php +++ b/resources/views/livewire/results/index.blade.php @@ -2,9 +2,9 @@ use App\Livewire\IncludesSorting; use App\Models\BenchmarkRun; -use App\Models\RunResult; -use Illuminate\Database\Eloquent\Builder; +use App\Support\Suites; use Illuminate\Pagination\LengthAwarePaginator; +use Illuminate\Support\Facades\Cache; use Illuminate\Support\Facades\DB; use Livewire\Attributes\Computed; use Livewire\Component; @@ -23,7 +23,7 @@ ]; private const SORTABLE = [ - 'ran_at', 'model_id', 'suite', 'task', 'section', 'level', 'ok', + 'ran_at', 'model_id', 'suite', 'task', 'level', 'ok', 'decode', 'user_tps', 'ttft', 't_answer', 'wall', 'total', ]; @@ -43,56 +43,92 @@ public function resetFilters(): void #[Computed] public function results(): LengthAwarePaginator { - return RunResult::query() - ->join('benchmark_runs', 'run_results.benchmark_run_id', '=', 'benchmark_runs.id') - ->select('run_results.*', - 'benchmark_runs.ran_at', + $ttl = (int) config('festbench.home_cache_ttl', 300); + $user = auth()->user(); + // Guest + public-only views are cacheable; any logged-in view may + // embed private rows, so it gets a user-scoped key instead. + $cacheKey = 'home:'.($user?->id ?? 'guest').':'.md5(json_encode([$this->filters, $this->sortBy, $this->sortDirection, $this->getPage()])); + + return Cache::remember($cacheKey, $ttl, function () use ($user) { + $query = $this->baseQuery($user) + ->tap(fn ($q) => $this->applySorting($q)) + ->tap(fn ($q) => $this->applyFilters($q)); + + return $query->paginate(25); + }); + } + + /** + * The combined query. Suite filter unset → the all_results view (one + * UNION plan for everything); suite filter set → the concrete suite + * table directly (full indexes, no union). + */ + private function baseQuery($user) + { + $suite = $this->filters['suite']; + + $model = $suite !== null && $suite !== '' ? Suites::modelFor($suite) : null; + + $source = $model !== null + ? $model::query()->getModel()->getTable() + : 'all_results'; + + // The concrete suite tables have no suite column; promote it so the + // view renders identically for both query paths. + $select = $model !== null + ? "{$source}.*, '{$suite}' AS suite" + : "{$source}.*"; + + return DB::table($source) + ->join('benchmark_runs', 'benchmark_runs.id', '=', "{$source}.benchmark_run_id") + ->selectRaw($select) + ->addSelect( 'benchmark_runs.model_id', 'benchmark_runs.model_quant', 'benchmark_runs.engine', 'benchmark_runs.run_id', 'benchmark_runs.visibility', 'benchmark_runs.user_id as run_user_id') - ->tap(fn (Builder $q) => $q->where(BenchmarkRun::visibleClause(auth()->user()))) - ->tap(fn (Builder $q) => $this->applySorting($q)) - ->tap(fn (Builder $q) => $this->applyFilters($q)) - ->with('run:id,run_id') - ->paginate(25); + ->where(BenchmarkRun::visibleClause($user)); } #[Computed] public function suites(): array { - return RunResult::query()->distinct()->orderBy('suite')->pluck('suite')->all(); + return Cache::remember('home:suites', 300, fn () => Suites::active()->all()); } #[Computed] public function models(): array { - return BenchmarkRun::visibleTo(auth()->user()) - ->distinct()->orderBy('model_id')->pluck('model_id')->all(); + return Cache::remember('home:models', 300, function () { + return BenchmarkRun::visibleTo(auth()->user()) + ->distinct()->orderBy('model_id')->pluck('model_id')->all(); + }); } - private function applySorting(Builder $query): Builder + private function applySorting($query) { $column = in_array($this->sortBy, self::SORTABLE, true) ? $this->sortBy : 'ran_at'; + $source = $this->filters['suite'] !== null && $this->filters['suite'] !== '' ? Suites::modelFor($this->filters['suite'])::make()->getTable() : 'all_results'; return $query->orderBy($column, $this->sortDirection === 'asc' ? 'asc' : 'desc') - ->orderBy('run_results.id', 'desc'); + ->orderBy("{$source}.id", 'desc'); } - private function applyFilters(Builder $query): Builder + private function applyFilters($query) { + $source = $this->filters['suite'] !== null && $this->filters['suite'] !== '' ? Suites::modelFor($this->filters['suite'])::make()->getTable() : 'all_results'; + return $query - ->when($this->filters['search'], fn (Builder $q, string $search) => $q->where(function (Builder $q) use ($search) { - $q->where('run_results.task', 'like', "%{$search}%") + ->when($this->filters['search'], fn ($q, string $search) => $q->where(function ($q) use ($search, $source) { + $q->where("{$source}.task", 'like', "%{$search}%") ->orWhere('benchmark_runs.model_id', 'like', "%{$search}%") ->orWhere('benchmark_runs.label', 'like', "%{$search}%"); })) - ->when($this->filters['suite'], fn (Builder $q, string $suite) => $q->where('run_results.suite', $suite)) - ->when($this->filters['level'], fn (Builder $q, string $level) => $q->where('run_results.level', $level)) - ->when($this->filters['model'], fn (Builder $q, string $model) => $q->where('benchmark_runs.model_id', $model)) - ->when($this->filters['result'] !== null && $this->filters['result'] !== '', fn (Builder $q) => $q->where('run_results.ok', (bool) $this->filters['result'])); + ->when($this->filters['level'], fn ($q, string $level) => $q->where("{$source}.level", $level)) + ->when($this->filters['model'], fn ($q, string $model) => $q->where('benchmark_runs.model_id', $model)) + ->when($this->filters['result'] !== null && $this->filters['result'] !== '', fn ($q) => $q->where("{$source}.ok", (bool) $this->filters['result'])); } }; ?> @@ -115,7 +151,7 @@ class="w-64" clearable /> - + @foreach ($this->suites as $suite) {{ $suite }} @endforeach @@ -160,7 +196,7 @@ class="w-64" @foreach ($this->results as $result) - + {{ optional($result->ran_at)->format('Y-m-d') }} diff --git a/resources/views/livewire/runs/show.blade.php b/resources/views/livewire/runs/show.blade.php index 8a9580b..27595f2 100644 --- a/resources/views/livewire/runs/show.blade.php +++ b/resources/views/livewire/runs/show.blade.php @@ -123,51 +123,113 @@ private function authorizeOwner(): void @endif
- {{ __('Task results') }} + {{ $this->run->suite === 'scenariobench' ? __('Scenario results') : __('Task results') }}
- - - {{ __('Section') }} - {{ __('Task') }} - {{ __('Level') }} - {{ __('Pass') }} - {{ __('User tok/s') }} - {{ __('Decode') }} - {{ __('TTFT') }} - {{ __('t_answer') }} - {{ __('Wall') }} - {{ __('Reasoning') }} - {{ __('Visible') }} - {{ __('Finish') }} - + @if ($this->run->suite === 'scenariobench') + + + {{ __('Scenario') }} + {{ __('Level') }} + {{ __('Pass') }} + {{ __('Outcome') }} + {{ __('Wall (s)') }} + {{ __('Turns') }} + {{ __('Commands') }} + {{ __('Submissions') }} + {{ __('Rejected') }} + {{ __('t first sub') }} + {{ __('t pass') }} + {{ __('Gen tokens') }} + - - @foreach ($this->run->results()->orderBy('section')->orderBy('task')->get() as $result) - - {{ $result->section ?? '—' }} - {{ $result->task }} - {{ $result->level ?? '—' }} - - @if ($result->ok) - - @elseif ($result->ok === null) - — - @else - + + @foreach ($this->run->suiteResults() as $result) + + {{ $result->scenario }} + {{ $result->level ?? '—' }} + + @if ($result->ok) + + @elseif ($result->ok === null) + — + @else + + @endif + + {{ $result->outcome ?? '—' }} + {{ number_format($result->wall, 1) }} + {{ number_format($result->turns) }} + {{ number_format($result->commands) }} + {{ number_format($result->submissions) }} + {{ number_format($result->rejected) }} + {{ number_format($result->t_first_sub, 1) }} + {{ $result->t_pass !== null ? number_format($result->t_pass, 1) : '—' }} + {{ number_format($result->gen_tokens) }} + + @endforeach + + + @else + + + @if (in_array($this->run->suite, ['softbench', 'prosebench', 'psychbench'])) + {{ __('Section') }} + @endif + {{ __('Task') }} + {{ __('Level') }} + {{ __('Pass') }} + @if (in_array($this->run->suite, ['prosebench', 'psychbench'])) + {{ __('Checks') }} + @endif + @if ($this->run->suite === 'infrabench') + {{ __('Kind') }} + @endif + {{ __('User tok/s') }} + {{ __('Decode') }} + {{ __('TTFT') }} + {{ __('t_answer') }} + {{ __('Wall') }} + {{ __('Reasoning') }} + {{ __('Visible') }} + {{ __('Finish') }} + + + + @foreach ($this->run->suiteResults() as $result) + + @if (in_array($this->run->suite, ['softbench', 'prosebench', 'psychbench'])) + {{ $result->section ?? '—' }} @endif - - {{ number_format($result->user_tps, 1) }} - {{ number_format($result->decode, 1) }} - {{ number_format($result->ttft, 2) }} - {{ number_format($result->t_answer, 2) }} - {{ number_format($result->wall, 2) }} - {{ number_format($result->reasoning) }} - {{ number_format($result->visible) }} - {{ $result->finish ?? '—' }} - - @endforeach - - + {{ $result->task }} + {{ $result->level ?? '—' }} + + @if ($result->ok) + + @elseif ($result->ok === null) + — + @else + + @endif + + @if (in_array($this->run->suite, ['prosebench', 'psychbench'])) + {{ $result->checks_passed !== null ? $result->checks_passed.'/'.$result->checks_total : '—' }} + @endif + @if ($this->run->suite === 'infrabench') + {{ $result->kind ?? '—' }} + @endif + {{ number_format($result->user_tps, 1) }} + {{ number_format($result->decode, 1) }} + {{ number_format($result->ttft, 2) }} + {{ number_format($result->t_answer, 2) }} + {{ number_format($result->wall, 2) }} + {{ number_format($result->reasoning) }} + {{ number_format($result->visible) }} + {{ $result->finish ?? '—' }} + + @endforeach + + + @endif
\ No newline at end of file diff --git a/tests/Feature/BenchmarkRunTest.php b/tests/Feature/BenchmarkRunTest.php new file mode 100644 index 0000000..881b924 --- /dev/null +++ b/tests/Feature/BenchmarkRunTest.php @@ -0,0 +1,118 @@ +create(['visibility' => 'public']); + BenchmarkRun::factory()->create(['visibility' => 'private']); + + $visible = BenchmarkRun::visibleTo(null)->get(); + + expect($visible->pluck('id'))->toContain($publicRun->id) + ->and($visible)->toHaveCount(1); +}); + +test('authenticated users see public runs plus their own private runs', function () { + $me = User::factory()->create(); + $other = User::factory()->create(); + + $publicRun = BenchmarkRun::factory()->create(['visibility' => 'public']); + $myPrivate = BenchmarkRun::factory()->create(['visibility' => 'private', 'user_id' => $me->id]); + BenchmarkRun::factory()->create(['visibility' => 'private', 'user_id' => $other->id]); + + $visible = BenchmarkRun::visibleTo($me)->get(); + + expect($visible->pluck('id'))->toContain($publicRun->id) + ->and($visible->pluck('id'))->toContain($myPrivate->id) + ->and($visible)->toHaveCount(2); +}); + +test('submitted metrics are immutable through fillable updates', function () { + $run = BenchmarkRun::factory()->create(['model_id' => 'original/model', 'visibility' => 'public']); + + $run->fill(['notes' => 'my notes', 'visibility' => 'private', 'model_id' => 'hacked/model']) + ->save(); + + expect($run->notes)->toBe('my notes') + ->and($run->visibility)->toBe('private') + ->and($run->fresh()->model_id)->toBe('original/model'); +}); + +test('suite result rows are deleted with their run', function () { + $run = BenchmarkRun::factory()->create(['suite' => 'softbench']); + SoftbenchResult::factory()->count(2)->create(['benchmark_run_id' => $run->id]); + + $run->delete(); + + expect(SoftbenchResult::where('benchmark_run_id', $run->id)->exists())->toBeFalse(); +}); + +test('suite results belong to their run and cast payloads', function () { + $run = BenchmarkRun::factory()->create(['suite' => 'prosebench']); + $result = ProsebenchResult::factory()->create([ + 'benchmark_run_id' => $run->id, + 'payload' => ['finish' => 'stop'], + 'checks_passed' => 5, + 'checks_total' => 6, + ]); + + expect($result->run->id)->toBe($run->id) + ->and($result->payload)->toBeArray() + ->and($result->checks_total)->toBe(6); +}); + +test('registry resolves models and relations for every suite', function () { + foreach (Suites::all() as $suite) { + $model = Suites::modelFor($suite); + expect($model)->not->toBeNull(); + + $run = BenchmarkRun::factory()->create(['suite' => $suite]); + $model::factory()->create(['benchmark_run_id' => $run->id]); + + expect($run->suiteResults())->toHaveCount(1) + ->and(Suites::runRelation($suite))->toBeString(); + } +}); + +test('unregistered suites produce empty suite results', function () { + $run = BenchmarkRun::factory()->create(['suite' => 'depthsweep']); + + expect($run->suiteResults())->toBeEmpty(); +}); + +test('scenario runs keep their agentic metrics', function () { + $run = BenchmarkRun::factory()->create(['suite' => 'scenariobench']); + $scenario = ScenarioRun::factory()->create([ + 'benchmark_run_id' => $run->id, + 'outcome' => 'passed', + 'turns' => 68, + 'submissions' => 1, + ]); + + expect($scenario->ok)->toBeBool() + ->and($scenario->turns)->toBe(68) + ->and($scenario->verdicts)->toBeArray() + ->and($scenario->run->id)->toBe($run->id); +}); + +test('infrabench rows carry their kind', function () { + $run = BenchmarkRun::factory()->create(['suite' => 'infrabench']); + $result = InfrabenchResult::factory()->create(['benchmark_run_id' => $run->id, 'kind' => 'recall']); + + expect($result->kind)->toBe('recall'); +}); + +test('psychbench rows carry their checks', function () { + $run = BenchmarkRun::factory()->create(['suite' => 'psychbench']); + $result = PsychbenchResult::factory()->create(['benchmark_run_id' => $run->id]); + + expect($result->checks_total)->toBeInt() + ->and($result->checks_passed)->toBeLessThanOrEqual($result->checks_total); +}); diff --git a/tests/Feature/FestbenchImportTest.php b/tests/Feature/FestbenchImportTest.php index 325bc87..d034a9a 100644 --- a/tests/Feature/FestbenchImportTest.php +++ b/tests/Feature/FestbenchImportTest.php @@ -2,8 +2,10 @@ use App\Console\Commands\FestbenchImport; use App\Models\BenchmarkRun; -use App\Models\RunResult; +use App\Models\ScenarioRun; +use App\Models\SoftbenchResult; use App\Models\User; +use Illuminate\Support\Facades\Artisan; function festbench_fixture_db(string $path): void { @@ -11,7 +13,7 @@ function festbench_fixture_db(string $path): void unlink($path); } - // festbench's real schema, from llm_benchmark/benchlib.py + // festbench's real schemas, from llm_benchmark/benchlib.py $pdo = new PDO('sqlite:'.$path); $pdo->exec(<<<'SQL' CREATE TABLE runs ( @@ -25,48 +27,75 @@ function festbench_fixture_db(string $path): void run_id TEXT, suite TEXT, section TEXT, task TEXT, level TEXT, ok INTEGER, decode REAL, user_tps REAL, ttft REAL, t_answer REAL, wall REAL, reasoning INTEGER, visible INTEGER, total INTEGER, - finish TEXT, detail TEXT - , tool_calls INTEGER); + finish TEXT, detail TEXT, tool_calls INTEGER); + CREATE TABLE scenario_runs ( + run_id TEXT, scenario TEXT, level TEXT, ok INTEGER, outcome TEXT, + wall REAL, turns INTEGER, commands INTEGER, repeats INTEGER, + submissions INTEGER, rejected INTEGER, t_first_sub REAL, n_first_sub INTEGER, + t_pass REAL, n_pass INTEGER, gen_tokens INTEGER, vis_tokens INTEGER, + rea_tokens INTEGER, verdicts TEXT, detail TEXT); SQL); $insertRun = $pdo->prepare('INSERT INTO runs (run_id, ts, suite, model_id, model_quant, engine, gpu, gpu_count, label) VALUES (?,?,?,?,?,?,?,?,?)'); - $insertRun->execute(['aaa111222333', '2026-09-01T10:00:00', 'softbench', 'fixture/model', 'Q5_K_M', 'llama.cpp', 'R9700', 1, 'fixture-label']); - $insertRun->execute(['bbb444555666', '2026-09-02T11:00:00', 'infrabench', 'fixture/model', null, 'vllm', 'unknown', 0, null]); + $insertRun->execute(['aaa111222333', '2026-09-01T10:00:00', 'softbench', 'fixture/model', 'Q5_K_M', 'llama.cpp', 'R9700', 1, 'deterministic sampling, repeat 1']); + $insertRun->execute(['bbb444555666', '2026-09-02T11:00:00', 'scenariobench', 'fixture/model', null, 'vllm', 'unknown', 0, 'agent test']); + $insertRun->execute(['ccc777888999', '2026-09-03T12:00:00', 'softbench', 'fixture/model', null, 'llama.cpp', 'R9700', 1, 'other session']); - $insertResult = $pdo->prepare('INSERT INTO results (run_id, suite, section, task, level, ok, decode, user_tps, ttft, t_answer, wall, reasoning, visible, total, finish, detail) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)'); - $insertResult->execute(['aaa111222333', 'softbench', 'code', 'first_occurrence', 'off', 1, 83.82, 61.14, 0.47, 0.47, 1.77, 0, 108, 109, 'stop', '']); - $insertResult->execute(['aaa111222333', 'softbench', 'code', 'merge_intervals', 'off', 0, 51.2, 40.0, 0.5, 0.5, 3.0, 250, 90, 340, 'length', 'assertion failed']); - $insertResult->execute(['bbb444555666', 'infrabench', null, 'modprobe_exists', 'medium', 1, 40.0, 30.0, 1.2, 1.2, 10.0, 0, 200, 200, 'stop', '']); + $insertResult = $pdo->prepare('INSERT INTO results (run_id, suite, section, task, level, ok, decode, user_tps, ttft, t_answer, wall, reasoning, visible, total, finish, detail, tool_calls) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)'); + $insertResult->execute(['aaa111222333', 'softbench', 'code', 'first_occurrence', 'off', 1, 83.82, 61.14, 0.47, 0.47, 1.77, 0, 108, 109, 'stop', '', 2]); + $insertResult->execute(['aaa111222333', 'softbench', 'code', 'merge_intervals', 'n/a', 0, 51.2, 40.0, 0.5, 0.5, 3.0, 250, 90, 340, 'length', 'assertion failed', null]); + $insertResult->execute(['ccc777888999', 'softbench', 'code', 'other_session_task', 'off', 1, 70, 50, 0.4, 0.4, 2.0, 0, 100, 100, 'stop', '', null]); + + $insertScenario = $pdo->prepare('INSERT INTO scenario_runs (run_id, scenario, level, ok, outcome, wall, turns, commands, repeats, submissions, rejected, t_first_sub, n_first_sub, t_pass, n_pass, gen_tokens, vis_tokens, rea_tokens, verdicts, detail) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)'); + $insertScenario->execute(['bbb444555666', 'kubelet_config_repair', 'off', 1, 'passed', 264.1, 68, 67, 2, 1, 0, 262.5, 68, 264.1, 68, 2543, 2475, 0, '[{"n":1,"at":264.1,"accepted":true}]', 'passed on submission 1']); } -test('festbench:import imports runs and results from a real-schema sqlite db', function () { +beforeEach(fn () => Artisan::call('festbench:suites:refresh')); + +test('festbench:import routes rows to per-suite tables', function () { $path = storage_path('app/test-benchmarks.db'); festbench_fixture_db($path); $this->artisan(FestbenchImport::class, ['path' => $path])->assertSuccessful(); - expect(BenchmarkRun::count())->toBe(2) - ->and(RunResult::count())->toBe(3); + expect(BenchmarkRun::count())->toBe(3) + ->and(SoftbenchResult::count())->toBe(3) + ->and(ScenarioRun::count())->toBe(1); $run = BenchmarkRun::where('run_id', 'aaa111222333')->first(); expect($run->suite)->toBe('softbench') - ->and($run->model_id)->toBe('fixture/model') ->and($run->model_quant)->toBe('Q5_K_M') - ->and($run->ran_at->format('Y-m-d H:i'))->toBe('2026-09-01 10:00') - ->and($run->visibility)->toBe('public') - ->and($run->results)->toHaveCount(2); + ->and($run->visibility)->toBe('public'); - $failed = RunResult::where('task', 'merge_intervals')->first(); + $failed = SoftbenchResult::where('task', 'merge_intervals')->first(); expect($failed->ok)->toBeFalse() ->and($failed->finish)->toBe('length') ->and($failed->detail)->toBe('assertion failed') - // full source row preserved + // n/a levels normalize to null + ->and($failed->level)->toBeNull() + // complete source row preserved ->and($failed->payload['total'])->toBe(340); - // null quant and empty detail survive the import - $second = BenchmarkRun::where('run_id', 'bbb444555666')->first(); - expect($second->model_quant)->toBeNull() - ->and($second->results->first()->detail)->toBeNull(); + $toolUser = SoftbenchResult::where('task', 'first_occurrence')->first(); + expect($toolUser->tool_calls)->toBe(2); + + $scenario = ScenarioRun::first(); + expect($scenario->scenario)->toBe('kubelet_config_repair') + ->and($scenario->outcome)->toBe('passed') + ->and($scenario->turns)->toBe(68) + ->and($scenario->verdicts)->toBeArray() + ->and($scenario->t_pass)->toBe(264.1); +}); + +test('festbench:import --label imports only the matching session', function () { + $path = storage_path('app/test-benchmarks.db'); + festbench_fixture_db($path); + + $this->artisan(FestbenchImport::class, ['path' => $path, '--label' => 'deterministic sampling, repeat 1'])->assertSuccessful(); + + expect(BenchmarkRun::count())->toBe(1) + ->and(BenchmarkRun::first()->run_id)->toBe('aaa111222333') + ->and(SoftbenchResult::count())->toBe(2); }); test('festbench:import is idempotent', function () { @@ -76,8 +105,21 @@ function festbench_fixture_db(string $path): void $this->artisan(FestbenchImport::class, ['path' => $path])->assertSuccessful(); $this->artisan(FestbenchImport::class, ['path' => $path])->assertSuccessful(); - expect(BenchmarkRun::count())->toBe(2) - ->and(RunResult::count())->toBe(3); + expect(BenchmarkRun::count())->toBe(3) + ->and(SoftbenchResult::count())->toBe(3); +}); + +test('festbench:import --fresh purges existing data first', function () { + $path = storage_path('app/test-benchmarks.db'); + festbench_fixture_db($path); + + // Pre-existing noise + BenchmarkRun::factory()->has(SoftbenchResult::factory(), 'softbenchResults')->create(); + + $this->artisan(FestbenchImport::class, ['path' => $path, '--fresh' => true, '--no-interaction' => true])->assertSuccessful(); + + expect(BenchmarkRun::count())->toBe(3) + ->and(BenchmarkRun::pluck('run_id')->sort()->values()->all())->toBe(['aaa111222333', 'bbb444555666', 'ccc777888999']); }); test('festbench:import attributes runs to a user and can default to private', function () { @@ -91,9 +133,8 @@ function festbench_fixture_db(string $path): void expect($run->user_id)->toBe($me->id) ->and($run->visibility)->toBe('private'); - // guest cannot see it, owner can expect(BenchmarkRun::visibleTo(null)->count())->toBe(0) - ->and(BenchmarkRun::visibleTo($me)->count())->toBe(2); + ->and(BenchmarkRun::visibleTo($me)->count())->toBe(3); }); test('festbench:import warns when the user does not exist', function () { @@ -106,3 +147,15 @@ function festbench_fixture_db(string $path): void expect(BenchmarkRun::first()->user_id)->toBeNull(); }); + +test('unregistered suites keep the run but warn about skipped rows', function () { + $path = storage_path('app/test-benchmarks.db'); + festbench_fixture_db($path); + $pdo = new PDO('sqlite:'.$path); + $pdo->exec("INSERT INTO runs (run_id, ts, suite, model_id, label) VALUES ('ddd000111222', '2026-09-04T10:00:00', 'depthsweep', 'fixture/model', 'unknown suite')"); + $pdo->exec("INSERT INTO results (run_id, suite, task, level, ok, wall) VALUES ('ddd000111222', 'depthsweep', 'depth_task', 'off', 1, 5.0)"); + + $this->artisan(FestbenchImport::class, ['path' => $path])->assertSuccessful(); + + expect(BenchmarkRun::where('run_id', 'ddd000111222')->exists())->toBeTrue(); +}); diff --git a/tests/Feature/ResultsPageTest.php b/tests/Feature/ResultsPageTest.php index c32d6b3..e85aef5 100644 --- a/tests/Feature/ResultsPageTest.php +++ b/tests/Feature/ResultsPageTest.php @@ -1,14 +1,15 @@ create(['model_id' => 'public/model']); - RunResult::factory()->count(3)->create(['benchmark_run_id' => $publicRun->id]); - BenchmarkRun::factory()->private()->hasResults(2)->create(['model_id' => 'private/model']); + $publicRun = BenchmarkRun::factory()->create(['model_id' => 'public/model', 'suite' => 'softbench']); + ProsebenchResult::factory()->count(3)->create(['benchmark_run_id' => $publicRun->id]); + BenchmarkRun::factory()->private()->create(['model_id' => 'private/model', 'suite' => 'softbench']); $response = $this->get(route('home')); @@ -19,10 +20,8 @@ test('authenticated users see their private runs mixed in', function () { $me = User::factory()->create(); - BenchmarkRun::factory()->private()->hasResults(2)->create([ - 'model_id' => 'my-private/model', 'user_id' => $me->id, - ]); - BenchmarkRun::factory()->private()->hasResults(1)->create(['model_id' => 'other-private/model']); + BenchmarkRun::factory()->private()->create(['model_id' => 'my-private/model', 'user_id' => $me->id, 'suite' => 'softbench']); + BenchmarkRun::factory()->private()->create(['model_id' => 'other-private/model', 'suite' => 'softbench']); $this->actingAs($me); @@ -31,33 +30,35 @@ ->assertDontSee('other-private/model'); }); -test('home page sorting orders by user tok/s', function () { - $run = BenchmarkRun::factory()->create(); - RunResult::factory()->create(['benchmark_run_id' => $run->id, 'task' => 'slow-task', 'user_tps' => 5]); - RunResult::factory()->create(['benchmark_run_id' => $run->id, 'task' => 'fast-task', 'user_tps' => 95]); +test('home page sorting orders by user tok/s through the view', function () { + $run = BenchmarkRun::factory()->create(['suite' => 'softbench']); + SoftbenchResult::factory()->create(['benchmark_run_id' => $run->id, 'task' => 'slow-task', 'user_tps' => 5]); + SoftbenchResult::factory()->create(['benchmark_run_id' => $run->id, 'task' => 'fast-task', 'user_tps' => 95]); Volt::test('results.index', ['sortBy' => 'user_tps', 'sortDirection' => 'asc']) ->assertSeeInOrder(['slow-task', 'fast-task']); }); -test('home page filters narrow by suite', function () { - $run = BenchmarkRun::factory()->create(); - RunResult::factory()->create(['benchmark_run_id' => $run->id, 'suite' => 'softbench', 'task' => 'soft-task']); - RunResult::factory()->create(['benchmark_run_id' => $run->id, 'suite' => 'infrabench', 'task' => 'infra-task']); +test('home page suite filter queries the concrete suite table', function () { + $softRun = BenchmarkRun::factory()->create(['suite' => 'softbench']); + $proseRun = BenchmarkRun::factory()->create(['suite' => 'prosebench']); + SoftbenchResult::factory()->create(['benchmark_run_id' => $softRun->id, 'task' => 'soft-task']); + ProsebenchResult::factory()->create(['benchmark_run_id' => $proseRun->id, 'task' => 'prose-task']); Volt::test('results.index', ['filters' => ['search' => null, 'suite' => 'softbench', 'level' => null, 'model' => null, 'result' => null]]) ->assertSee('soft-task') - ->assertDontSee('infra-task'); + ->assertDontSee('prose-task'); }); -test('run detail page shows metadata and results', function () { - $run = BenchmarkRun::factory()->create(['model_id' => 'detail/model', 'engine' => 'vllm']); - RunResult::factory()->create(['benchmark_run_id' => $run->id, 'task' => 'visible-task']); +test('run detail page shows metadata and suite-tailored results', function () { + $run = BenchmarkRun::factory()->create(['model_id' => 'detail/model', 'engine' => 'vllm', 'suite' => 'prosebench']); + ProsebenchResult::factory()->create(['benchmark_run_id' => $run->id, 'task' => 'visible-task', 'checks_passed' => 6, 'checks_total' => 6]); $this->get(route('runs.show', $run->run_id))->assertOk() ->assertSee('detail/model') ->assertSee('visible-task') - ->assertSee('vllm'); + ->assertSee('vllm') + ->assertSee('6/6'); }); test('private runs are hidden from guests on the detail page', function () { @@ -105,13 +106,14 @@ test('my runs page lists only owned runs with counts', function () { $me = User::factory()->create(); - $run = BenchmarkRun::factory()->create(['user_id' => $me->id, 'model_id' => 'mine/model']); - RunResult::factory()->count(4)->create(['benchmark_run_id' => $run->id]); - BenchmarkRun::factory()->create(['user_id' => User::factory()->create()->id, 'model_id' => 'theirs/model']); + $run = BenchmarkRun::factory()->create(['user_id' => $me->id, 'model_id' => 'mine/model', 'suite' => 'softbench']); + SoftbenchResult::factory()->count(4)->create(['benchmark_run_id' => $run->id]); + BenchmarkRun::factory()->create(['user_id' => User::factory()->create()->id, 'model_id' => 'theirs/model', 'suite' => 'softbench']); $this->actingAs($me); $this->get(route('my-runs'))->assertOk() ->assertSee('mine/model') - ->assertDontSee('theirs/model'); + ->assertDontSee('theirs/model') + ->assertSee('4'); });