pch/festbench-web
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

v2 Phase C: suite-aware home table via all_results view, tailored run detail, per-suite tests

Browse source
This commit is contained in:
PeterChrz 2026-09-18 14:49:09 -04:00
parent 8c75db59f0
commit 72cde51978
Signed by untrusted user who does not match committer: pch
GPG key ID: 8F0826ECF7302C63
8 changed files with 430 additions and 133 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

13
app/Models/BenchmarkRun.php
View file

@ -104,7 +104,7 @@ public function suiteResults(): Collection
$model = Suites::modelFor($this->suite); $model = Suites::modelFor($this->suite);
if ($model === null) { if ($model === null) {
return collect(); return new Collection;
} }
$relation = Suites::runRelation($this->suite); $relation = Suites::runRelation($this->suite);
@ -128,18 +128,19 @@ public function scopeVisibleTo(Builder $query, ?User $user): Builder
/** /**
* The visibility predicate as a standalone closure, so joined queries * The visibility predicate as a standalone closure, so joined queries
* (e.g. the flat results table) can apply the exact same rule without * (e.g. the combined results table) can apply the exact same rule
* duplicating the logic. * without duplicating the logic. Accepts either builder flavor: Eloquent
* scopes pass an Eloquent Builder, DB::table queries pass a Query Builder.
* *
* @return \Closure(Builder): Builder * @return \Closure(\Illuminate\Database\Query\Builder|Builder): mixed
*/ */
public static function visibleClause(?User $user): \Closure public static function visibleClause(?User $user): \Closure
{ {
return function (Builder $q) use ($user) { return function ($q) use ($user) {
$q->where('visibility', self::VISIBILITY_PUBLIC); $q->where('visibility', self::VISIBILITY_PUBLIC);
if ($user !== null) { if ($user !== null) {
$q->orWhere(fn (Builder $own) => $own $q->orWhere(fn ($own) => $own
->where('visibility', self::VISIBILITY_PRIVATE) ->where('visibility', self::VISIBILITY_PRIVATE)
->where('user_id', $user->id)); ->where('user_id', $user->id));
} }

22
database/migrations/2026_09_18_000001_reorganize_suite_tables.php
View file

@ -2,6 +2,7 @@
use Illuminate\Database\Migrations\Migration; use Illuminate\Database\Migrations\Migration;
use Illuminate\Database\Schema\Blueprint; use Illuminate\Database\Schema\Blueprint;
use Illuminate\Support\Facades\Artisan;
use Illuminate\Support\Facades\DB; use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Schema; use Illuminate\Support\Facades\Schema;
@ -135,6 +136,12 @@ public function up(): void
$this->createPartitions(); $this->createPartitions();
} }
// Initial all_results view over the metric suites (active registry
// entries). Tests need it present right after migrate; production
// re-generates it via festbench:suites:refresh when the registry
// changes.
Artisan::call('festbench:suites:refresh');
} }
private function createPgTable(string $table, string $columns, array $extras): void private function createPgTable(string $table, string $columns, array $extras): void
@ -186,11 +193,20 @@ private function createSqliteTable(string $table, array $extras): void
$table->index('wall'); $table->index('wall');
}); });
// SQLite has no ALTER ADD COLUMN via builder for positioning; add extras. // Suite-specific extras, declared with the proper builder calls.
Schema::table($table, function (Blueprint $t) use ($extras) { Schema::table($table, function (Blueprint $t) use ($extras) {
foreach ($extras as $extra) { foreach ($extras as $extra) {
[$name, $type] = explode(' ', $extra); match (true) {
$t->addColumn($type, $name)->nullable(); str_starts_with($extra, 'section ') => $t->string('section', 64)->nullable(),
str_starts_with($extra, 'tool_calls ') => $t->unsignedInteger('tool_calls')->nullable(),
str_starts_with($extra, 'kind ') => $t->string('kind', 32)->nullable(),
str_starts_with($extra, 'checks_passed ') => $t->unsignedInteger('checks_passed')->nullable(),
str_starts_with($extra, 'checks_total ') => $t->unsignedInteger('checks_total')->nullable(),
str_starts_with($extra, 'judge ') => $t->text('judge')->nullable(),
str_starts_with($extra, 'prefill ') => $t->double('prefill')->nullable(),
str_starts_with($extra, 'prompt_tokens ') => $t->unsignedInteger('prompt_tokens')->nullable(),
default => throw new RuntimeException("Unknown extra column: {$extra}"),
};
} }
}); });
} }

17
resources/views/livewire/my-runs/index.blade.php
View file

@ -37,12 +37,21 @@ public function resetFilters(): void
#[Computed] #[Computed]
public function runs(): LengthAwarePaginator public function runs(): LengthAwarePaginator
{ {
// Row + pass counts across the suite tables (runs are single-suite,
// but the owning table varies per run, hence the union of counts).
return BenchmarkRun::query() return BenchmarkRun::query()
->where('user_id', auth()->id()) ->where('user_id', auth()->id())
->withCount([ ->select('benchmark_runs.*')
'results', ->selectRaw('(SELECT count(*) FROM softbench_results sr WHERE sr.benchmark_run_id = benchmark_runs.id)
'results as passed_count' => fn ($q) => $q->where('ok', true), + (SELECT count(*) FROM infrabench_results sr WHERE sr.benchmark_run_id = benchmark_runs.id)
]) + (SELECT count(*) FROM prosebench_results sr WHERE sr.benchmark_run_id = benchmark_runs.id)
+ (SELECT count(*) FROM psychbench_results sr WHERE sr.benchmark_run_id = benchmark_runs.id)
+ (SELECT count(*) FROM scenario_runs sr WHERE sr.benchmark_run_id = benchmark_runs.id) as results_count')
->selectRaw('(SELECT count(*) FROM softbench_results sr WHERE sr.benchmark_run_id = benchmark_runs.id AND sr.ok)
+ (SELECT count(*) FROM infrabench_results sr WHERE sr.benchmark_run_id = benchmark_runs.id AND sr.ok)
+ (SELECT count(*) FROM prosebench_results sr WHERE sr.benchmark_run_id = benchmark_runs.id AND sr.ok)
+ (SELECT count(*) FROM psychbench_results sr WHERE sr.benchmark_run_id = benchmark_runs.id AND sr.ok)
+ (SELECT count(*) FROM scenario_runs sr WHERE sr.benchmark_run_id = benchmark_runs.id AND sr.ok) as passed_count')
->when($this->filters['search'], fn ($q, $search) => $q->where(function ($q) use ($search) { ->when($this->filters['search'], fn ($q, $search) => $q->where(function ($q) use ($search) {
$q->where('run_id', 'like', "%{$search}%") $q->where('run_id', 'like', "%{$search}%")
->orWhere('model_id', 'like', "%{$search}%") ->orWhere('model_id', 'like', "%{$search}%")

88
resources/views/livewire/results/index.blade.php
View file

@ -2,9 +2,9 @@
use App\Livewire\IncludesSorting; use App\Livewire\IncludesSorting;
use App\Models\BenchmarkRun; use App\Models\BenchmarkRun;
use App\Models\RunResult; use App\Support\Suites;
use Illuminate\Database\Eloquent\Builder;
use Illuminate\Pagination\LengthAwarePaginator; use Illuminate\Pagination\LengthAwarePaginator;
use Illuminate\Support\Facades\Cache;
use Illuminate\Support\Facades\DB; use Illuminate\Support\Facades\DB;
use Livewire\Attributes\Computed; use Livewire\Attributes\Computed;
use Livewire\Component; use Livewire\Component;
@ -23,7 +23,7 @@
]; ];
private const SORTABLE = [ private const SORTABLE = [
'ran_at', 'model_id', 'suite', 'task', 'section', 'level', 'ok', 'ran_at', 'model_id', 'suite', 'task', 'level', 'ok',
'decode', 'user_tps', 'ttft', 't_answer', 'wall', 'total', 'decode', 'user_tps', 'ttft', 't_answer', 'wall', 'total',
]; ];
@ -43,56 +43,92 @@ public function resetFilters(): void
#[Computed] #[Computed]
public function results(): LengthAwarePaginator public function results(): LengthAwarePaginator
{ {
return RunResult::query() $ttl = (int) config('festbench.home_cache_ttl', 300);
->join('benchmark_runs', 'run_results.benchmark_run_id', '=', 'benchmark_runs.id') $user = auth()->user();
->select('run_results.*', // Guest + public-only views are cacheable; any logged-in view may
'benchmark_runs.ran_at', // embed private rows, so it gets a user-scoped key instead.
$cacheKey = 'home:'.($user?->id ?? 'guest').':'.md5(json_encode([$this->filters, $this->sortBy, $this->sortDirection, $this->getPage()]));
return Cache::remember($cacheKey, $ttl, function () use ($user) {
$query = $this->baseQuery($user)
->tap(fn ($q) => $this->applySorting($q))
->tap(fn ($q) => $this->applyFilters($q));
return $query->paginate(25);
});
}
/**
* The combined query. Suite filter unset → the all_results view (one
* UNION plan for everything); suite filter set → the concrete suite
* table directly (full indexes, no union).
*/
private function baseQuery($user)
{
$suite = $this->filters['suite'];
$model = $suite !== null && $suite !== '' ? Suites::modelFor($suite) : null;
$source = $model !== null
? $model::query()->getModel()->getTable()
: 'all_results';
// The concrete suite tables have no suite column; promote it so the
// view renders identically for both query paths.
$select = $model !== null
? "{$source}.*, '{$suite}' AS suite"
: "{$source}.*";
return DB::table($source)
->join('benchmark_runs', 'benchmark_runs.id', '=', "{$source}.benchmark_run_id")
->selectRaw($select)
->addSelect(
'benchmark_runs.model_id', 'benchmark_runs.model_id',
'benchmark_runs.model_quant', 'benchmark_runs.model_quant',
'benchmark_runs.engine', 'benchmark_runs.engine',
'benchmark_runs.run_id', 'benchmark_runs.run_id',
'benchmark_runs.visibility', 'benchmark_runs.visibility',
'benchmark_runs.user_id as run_user_id') 'benchmark_runs.user_id as run_user_id')
->tap(fn (Builder $q) => $q->where(BenchmarkRun::visibleClause(auth()->user()))) ->where(BenchmarkRun::visibleClause($user));
->tap(fn (Builder $q) => $this->applySorting($q))
->tap(fn (Builder $q) => $this->applyFilters($q))
->with('run:id,run_id')
->paginate(25);
} }
#[Computed] #[Computed]
public function suites(): array public function suites(): array
{ {
return RunResult::query()->distinct()->orderBy('suite')->pluck('suite')->all(); return Cache::remember('home:suites', 300, fn () => Suites::active()->all());
} }
#[Computed] #[Computed]
public function models(): array public function models(): array
{ {
return BenchmarkRun::visibleTo(auth()->user()) return Cache::remember('home:models', 300, function () {
->distinct()->orderBy('model_id')->pluck('model_id')->all(); return BenchmarkRun::visibleTo(auth()->user())
->distinct()->orderBy('model_id')->pluck('model_id')->all();
});
} }
private function applySorting(Builder $query): Builder private function applySorting($query)
{ {
$column = in_array($this->sortBy, self::SORTABLE, true) ? $this->sortBy : 'ran_at'; $column = in_array($this->sortBy, self::SORTABLE, true) ? $this->sortBy : 'ran_at';
$source = $this->filters['suite'] !== null && $this->filters['suite'] !== '' ? Suites::modelFor($this->filters['suite'])::make()->getTable() : 'all_results';
return $query->orderBy($column, $this->sortDirection === 'asc' ? 'asc' : 'desc') return $query->orderBy($column, $this->sortDirection === 'asc' ? 'asc' : 'desc')
->orderBy('run_results.id', 'desc'); ->orderBy("{$source}.id", 'desc');
} }
private function applyFilters(Builder $query): Builder private function applyFilters($query)
{ {
$source = $this->filters['suite'] !== null && $this->filters['suite'] !== '' ? Suites::modelFor($this->filters['suite'])::make()->getTable() : 'all_results';
return $query return $query
->when($this->filters['search'], fn (Builder $q, string $search) => $q->where(function (Builder $q) use ($search) { ->when($this->filters['search'], fn ($q, string $search) => $q->where(function ($q) use ($search, $source) {
$q->where('run_results.task', 'like', "%{$search}%") $q->where("{$source}.task", 'like', "%{$search}%")
->orWhere('benchmark_runs.model_id', 'like', "%{$search}%") ->orWhere('benchmark_runs.model_id', 'like', "%{$search}%")
->orWhere('benchmark_runs.label', 'like', "%{$search}%"); ->orWhere('benchmark_runs.label', 'like', "%{$search}%");
})) }))
->when($this->filters['suite'], fn (Builder $q, string $suite) => $q->where('run_results.suite', $suite)) ->when($this->filters['level'], fn ($q, string $level) => $q->where("{$source}.level", $level))
->when($this->filters['level'], fn (Builder $q, string $level) => $q->where('run_results.level', $level)) ->when($this->filters['model'], fn ($q, string $model) => $q->where('benchmark_runs.model_id', $model))
->when($this->filters['model'], fn (Builder $q, string $model) => $q->where('benchmark_runs.model_id', $model)) ->when($this->filters['result'] !== null && $this->filters['result'] !== '', fn ($q) => $q->where("{$source}.ok", (bool) $this->filters['result']));
->when($this->filters['result'] !== null && $this->filters['result'] !== '', fn (Builder $q) => $q->where('run_results.ok', (bool) $this->filters['result']));
} }
}; };
?> ?>
@ -115,7 +151,7 @@ class="w-64"
clearable clearable
/> />
<flux:select wire:model.live="filters.suite" :label="__('Suite')" placeholder="{{ __('All suites') }}" size="sm" class="w-40"> <flux:select wire:model.live="filters.suite" :label="__('Suite')" placeholder="{{ __('All suites') }}" size="sm" class="w-44">
@foreach ($this->suites as $suite) @foreach ($this->suites as $suite)
<flux:select.option value="{{ $suite }}">{{ $suite }}</flux:select.option> <flux:select.option value="{{ $suite }}">{{ $suite }}</flux:select.option>
@endforeach @endforeach
@ -160,7 +196,7 @@ class="w-64"
<flux:table.rows> <flux:table.rows>
@foreach ($this->results as $result) @foreach ($this->results as $result)
<flux:table.row :key="$result->id"> <flux:table.row :key="$result->suite.'-'.$result->id">
<flux:table.cell>{{ optional($result->ran_at)->format('Y-m-d') }}</flux:table.cell> <flux:table.cell>{{ optional($result->ran_at)->format('Y-m-d') }}</flux:table.cell>
<flux:table.cell> <flux:table.cell>
<a href="{{ route('runs.show', $result->run_id) }}" class="flex items-center hover:underline" wire:navigate> <a href="{{ route('runs.show', $result->run_id) }}" class="flex items-center hover:underline" wire:navigate>

146
resources/views/livewire/runs/show.blade.php
View file

@ -123,51 +123,113 @@ private function authorizeOwner(): void
@endif @endif
<div> <div>
<flux:heading size="md">{{ __('Task results') }}</flux:heading> <flux:heading size="md">{{ $this->run->suite === 'scenariobench' ? __('Scenario results') : __('Task results') }}</flux:heading>
<div class="mt-3"> <div class="mt-3">
<flux:table> @if ($this->run->suite === 'scenariobench')
<flux:table.columns> <flux:table>
<flux:table.column>{{ __('Section') }}</flux:table.column> <flux:table.columns>
<flux:table.column>{{ __('Task') }}</flux:table.column> <flux:table.column>{{ __('Scenario') }}</flux:table.column>
<flux:table.column>{{ __('Level') }}</flux:table.column> <flux:table.column>{{ __('Level') }}</flux:table.column>
<flux:table.column>{{ __('Pass') }}</flux:table.column> <flux:table.column>{{ __('Pass') }}</flux:table.column>
<flux:table.column class="text-right">{{ __('User tok/s') }}</flux:table.column> <flux:table.column>{{ __('Outcome') }}</flux:table.column>
<flux:table.column class="text-right">{{ __('Decode') }}</flux:table.column> <flux:table.column class="text-right">{{ __('Wall (s)') }}</flux:table.column>
<flux:table.column class="text-right">{{ __('TTFT') }}</flux:table.column> <flux:table.column class="text-right">{{ __('Turns') }}</flux:table.column>
<flux:table.column class="text-right">{{ __('t_answer') }}</flux:table.column> <flux:table.column class="text-right">{{ __('Commands') }}</flux:table.column>
<flux:table.column class="text-right">{{ __('Wall') }}</flux:table.column> <flux:table.column class="text-right">{{ __('Submissions') }}</flux:table.column>
<flux:table.column class="text-right">{{ __('Reasoning') }}</flux:table.column> <flux:table.column class="text-right">{{ __('Rejected') }}</flux:table.column>
<flux:table.column class="text-right">{{ __('Visible') }}</flux:table.column> <flux:table.column class="text-right">{{ __('t first sub') }}</flux:table.column>
<flux:table.column>{{ __('Finish') }}</flux:table.column> <flux:table.column class="text-right">{{ __('t pass') }}</flux:table.column>
</flux:table.columns> <flux:table.column class="text-right">{{ __('Gen tokens') }}</flux:table.column>
</flux:table.columns>
<flux:table.rows> <flux:table.rows>
@foreach ($this->run->results()->orderBy('section')->orderBy('task')->get() as $result) @foreach ($this->run->suiteResults() as $result)
<flux:table.row :key="$result->id"> <flux:table.row :key="$result->id">
<flux:table.cell>{{ $result->section ?? '—' }}</flux:table.cell> <flux:table.cell>{{ $result->scenario }}</flux:table.cell>
<flux:table.cell>{{ $result->task }}</flux:table.cell> <flux:table.cell>{{ $result->level ?? '—' }}</flux:table.cell>
<flux:table.cell>{{ $result->level ?? '—' }}</flux:table.cell> <flux:table.cell>
<flux:table.cell> @if ($result->ok)
@if ($result->ok) <flux:icon name="check-circle" variant="solid" class="size-4 text-emerald-500" />
<flux:icon name="check-circle" variant="solid" class="size-4 text-emerald-500" /> @elseif ($result->ok === null)
@elseif ($result->ok === null) <span class="text-zinc-400">—</span>
<span class="text-zinc-400">—</span> @else
@else <flux:icon name="x-circle" variant="solid" class="size-4 text-red-500" />
<flux:icon name="x-circle" variant="solid" class="size-4 text-red-500" /> @endif
</flux:table.cell>
<flux:table.cell><flux:badge size="xs" color="{{ $result->outcome === 'passed' ? 'emerald' : 'rose' }}">{{ $result->outcome ?? '—' }}</flux:badge></flux:table.cell>
<flux:table.cell class="text-right font-mono text-sm">{{ number_format($result->wall, 1) }}</flux:table.cell>
<flux:table.cell class="text-right font-mono text-sm">{{ number_format($result->turns) }}</flux:table.cell>
<flux:table.cell class="text-right font-mono text-sm">{{ number_format($result->commands) }}</flux:table.cell>
<flux:table.cell class="text-right font-mono text-sm">{{ number_format($result->submissions) }}</flux:table.cell>
<flux:table.cell class="text-right font-mono text-sm">{{ number_format($result->rejected) }}</flux:table.cell>
<flux:table.cell class="text-right font-mono text-sm">{{ number_format($result->t_first_sub, 1) }}</flux:table.cell>
<flux:table.cell class="text-right font-mono text-sm">{{ $result->t_pass !== null ? number_format($result->t_pass, 1) : '—' }}</flux:table.cell>
<flux:table.cell class="text-right font-mono text-sm">{{ number_format($result->gen_tokens) }}</flux:table.cell>
</flux:table.row>
@endforeach
</flux:table.rows>
</flux:table>
@else
<flux:table>
<flux:table.columns>
@if (in_array($this->run->suite, ['softbench', 'prosebench', 'psychbench']))
<flux:table.column>{{ __('Section') }}</flux:table.column>
@endif
<flux:table.column>{{ __('Task') }}</flux:table.column>
<flux:table.column>{{ __('Level') }}</flux:table.column>
<flux:table.column>{{ __('Pass') }}</flux:table.column>
@if (in_array($this->run->suite, ['prosebench', 'psychbench']))
<flux:table.column class="text-right">{{ __('Checks') }}</flux:table.column>
@endif
@if ($this->run->suite === 'infrabench')
<flux:table.column>{{ __('Kind') }}</flux:table.column>
@endif
<flux:table.column class="text-right">{{ __('User tok/s') }}</flux:table.column>
<flux:table.column class="text-right">{{ __('Decode') }}</flux:table.column>
<flux:table.column class="text-right">{{ __('TTFT') }}</flux:table.column>
<flux:table.column class="text-right">{{ __('t_answer') }}</flux:table.column>
<flux:table.column class="text-right">{{ __('Wall') }}</flux:table.column>
<flux:table.column class="text-right">{{ __('Reasoning') }}</flux:table.column>
<flux:table.column class="text-right">{{ __('Visible') }}</flux:table.column>
<flux:table.column>{{ __('Finish') }}</flux:table.column>
</flux:table.columns>
<flux:table.rows>
@foreach ($this->run->suiteResults() as $result)
<flux:table.row :key="$result->id">
@if (in_array($this->run->suite, ['softbench', 'prosebench', 'psychbench']))
<flux:table.cell>{{ $result->section ?? '—' }}</flux:table.cell>
@endif @endif
</flux:table.cell> <flux:table.cell>{{ $result->task }}</flux:table.cell>
<flux:table.cell class="text-right font-mono text-sm">{{ number_format($result->user_tps, 1) }}</flux:table.cell> <flux:table.cell>{{ $result->level ?? '—' }}</flux:table.cell>
<flux:table.cell class="text-right font-mono text-sm">{{ number_format($result->decode, 1) }}</flux:table.cell> <flux:table.cell>
<flux:table.cell class="text-right font-mono text-sm">{{ number_format($result->ttft, 2) }}</flux:table.cell> @if ($result->ok)
<flux:table.cell class="text-right font-mono text-sm">{{ number_format($result->t_answer, 2) }}</flux:table.cell> <flux:icon name="check-circle" variant="solid" class="size-4 text-emerald-500" />
<flux:table.cell class="text-right font-mono text-sm">{{ number_format($result->wall, 2) }}</flux:table.cell> @elseif ($result->ok === null)
<flux:table.cell class="text-right font-mono text-sm">{{ number_format($result->reasoning) }}</flux:table.cell> <span class="text-zinc-400">—</span>
<flux:table.cell class="text-right font-mono text-sm">{{ number_format($result->visible) }}</flux:table.cell> @else
<flux:table.cell><flux:badge size="xs" color="{{ $result->finish === 'stop' ? 'emerald' : 'amber' }}">{{ $result->finish ?? '—' }}</flux:badge></flux:table.cell> <flux:icon name="x-circle" variant="solid" class="size-4 text-red-500" />
</flux:table.row> @endif
@endforeach </flux:table.cell>
</flux:table.rows> @if (in_array($this->run->suite, ['prosebench', 'psychbench']))
</flux:table> <flux:table.cell class="text-right font-mono text-sm">{{ $result->checks_passed !== null ? $result->checks_passed.'/'.$result->checks_total : '—' }}</flux:table.cell>
@endif
@if ($this->run->suite === 'infrabench')
<flux:table.cell>{{ $result->kind ?? '—' }}</flux:table.cell>
@endif
<flux:table.cell class="text-right font-mono text-sm">{{ number_format($result->user_tps, 1) }}</flux:table.cell>
<flux:table.cell class="text-right font-mono text-sm">{{ number_format($result->decode, 1) }}</flux:table.cell>
<flux:table.cell class="text-right font-mono text-sm">{{ number_format($result->ttft, 2) }}</flux:table.cell>
<flux:table.cell class="text-right font-mono text-sm">{{ number_format($result->t_answer, 2) }}</flux:table.cell>
<flux:table.cell class="text-right font-mono text-sm">{{ number_format($result->wall, 2) }}</flux:table.cell>
<flux:table.cell class="text-right font-mono text-sm">{{ number_format($result->reasoning) }}</flux:table.cell>
<flux:table.cell class="text-right font-mono text-sm">{{ number_format($result->visible) }}</flux:table.cell>
<flux:table.cell><flux:badge size="xs" color="{{ $result->finish === 'stop' ? 'emerald' : 'amber' }}">{{ $result->finish ?? '—' }}</flux:badge></flux:table.cell>
</flux:table.row>
@endforeach
</flux:table.rows>
</flux:table>
@endif
</div> </div>
</div> </div>
</div> </div>

118
tests/Feature/BenchmarkRunTest.php Normal file
View file

@ -0,0 +1,118 @@
<?php
use App\Models\BenchmarkRun;
use App\Models\InfrabenchResult;
use App\Models\ProsebenchResult;
use App\Models\PsychbenchResult;
use App\Models\ScenarioRun;
use App\Models\SoftbenchResult;
use App\Models\User;
use App\Support\Suites;
test('guests see only public runs', function () {
$publicRun = BenchmarkRun::factory()->create(['visibility' => 'public']);
BenchmarkRun::factory()->create(['visibility' => 'private']);
$visible = BenchmarkRun::visibleTo(null)->get();
expect($visible->pluck('id'))->toContain($publicRun->id)
->and($visible)->toHaveCount(1);
});
test('authenticated users see public runs plus their own private runs', function () {
$me = User::factory()->create();
$other = User::factory()->create();
$publicRun = BenchmarkRun::factory()->create(['visibility' => 'public']);
$myPrivate = BenchmarkRun::factory()->create(['visibility' => 'private', 'user_id' => $me->id]);
BenchmarkRun::factory()->create(['visibility' => 'private', 'user_id' => $other->id]);
$visible = BenchmarkRun::visibleTo($me)->get();
expect($visible->pluck('id'))->toContain($publicRun->id)
->and($visible->pluck('id'))->toContain($myPrivate->id)
->and($visible)->toHaveCount(2);
});
test('submitted metrics are immutable through fillable updates', function () {
$run = BenchmarkRun::factory()->create(['model_id' => 'original/model', 'visibility' => 'public']);
$run->fill(['notes' => 'my notes', 'visibility' => 'private', 'model_id' => 'hacked/model'])
->save();
expect($run->notes)->toBe('my notes')
->and($run->visibility)->toBe('private')
->and($run->fresh()->model_id)->toBe('original/model');
});
test('suite result rows are deleted with their run', function () {
$run = BenchmarkRun::factory()->create(['suite' => 'softbench']);
SoftbenchResult::factory()->count(2)->create(['benchmark_run_id' => $run->id]);
$run->delete();
expect(SoftbenchResult::where('benchmark_run_id', $run->id)->exists())->toBeFalse();
});
test('suite results belong to their run and cast payloads', function () {
$run = BenchmarkRun::factory()->create(['suite' => 'prosebench']);
$result = ProsebenchResult::factory()->create([
'benchmark_run_id' => $run->id,
'payload' => ['finish' => 'stop'],
'checks_passed' => 5,
'checks_total' => 6,
]);
expect($result->run->id)->toBe($run->id)
->and($result->payload)->toBeArray()
->and($result->checks_total)->toBe(6);
});
test('registry resolves models and relations for every suite', function () {
foreach (Suites::all() as $suite) {
$model = Suites::modelFor($suite);
expect($model)->not->toBeNull();
$run = BenchmarkRun::factory()->create(['suite' => $suite]);
$model::factory()->create(['benchmark_run_id' => $run->id]);
expect($run->suiteResults())->toHaveCount(1)
->and(Suites::runRelation($suite))->toBeString();
}
});
test('unregistered suites produce empty suite results', function () {
$run = BenchmarkRun::factory()->create(['suite' => 'depthsweep']);
expect($run->suiteResults())->toBeEmpty();
});
test('scenario runs keep their agentic metrics', function () {
$run = BenchmarkRun::factory()->create(['suite' => 'scenariobench']);
$scenario = ScenarioRun::factory()->create([
'benchmark_run_id' => $run->id,
'outcome' => 'passed',
'turns' => 68,
'submissions' => 1,
]);
expect($scenario->ok)->toBeBool()
->and($scenario->turns)->toBe(68)
->and($scenario->verdicts)->toBeArray()
->and($scenario->run->id)->toBe($run->id);
});
test('infrabench rows carry their kind', function () {
$run = BenchmarkRun::factory()->create(['suite' => 'infrabench']);
$result = InfrabenchResult::factory()->create(['benchmark_run_id' => $run->id, 'kind' => 'recall']);
expect($result->kind)->toBe('recall');
});
test('psychbench rows carry their checks', function () {
$run = BenchmarkRun::factory()->create(['suite' => 'psychbench']);
$result = PsychbenchResult::factory()->create(['benchmark_run_id' => $run->id]);
expect($result->checks_total)->toBeInt()
->and($result->checks_passed)->toBeLessThanOrEqual($result->checks_total);
});

107
tests/Feature/FestbenchImportTest.php
View file

@ -2,8 +2,10 @@
use App\Console\Commands\FestbenchImport; use App\Console\Commands\FestbenchImport;
use App\Models\BenchmarkRun; use App\Models\BenchmarkRun;
use App\Models\RunResult; use App\Models\ScenarioRun;
use App\Models\SoftbenchResult;
use App\Models\User; use App\Models\User;
use Illuminate\Support\Facades\Artisan;
function festbench_fixture_db(string $path): void function festbench_fixture_db(string $path): void
{ {
@ -11,7 +13,7 @@ function festbench_fixture_db(string $path): void
unlink($path); unlink($path);
} }
// festbench's real schema, from llm_benchmark/benchlib.py // festbench's real schemas, from llm_benchmark/benchlib.py
$pdo = new PDO('sqlite:'.$path); $pdo = new PDO('sqlite:'.$path);
$pdo->exec(<<<'SQL' $pdo->exec(<<<'SQL'
CREATE TABLE runs ( CREATE TABLE runs (
@ -25,48 +27,75 @@ function festbench_fixture_db(string $path): void
run_id TEXT, suite TEXT, section TEXT, task TEXT, level TEXT, run_id TEXT, suite TEXT, section TEXT, task TEXT, level TEXT,
ok INTEGER, decode REAL, user_tps REAL, ttft REAL, t_answer REAL, ok INTEGER, decode REAL, user_tps REAL, ttft REAL, t_answer REAL,
wall REAL, reasoning INTEGER, visible INTEGER, total INTEGER, wall REAL, reasoning INTEGER, visible INTEGER, total INTEGER,
finish TEXT, detail TEXT finish TEXT, detail TEXT, tool_calls INTEGER);
, tool_calls INTEGER); CREATE TABLE scenario_runs (
run_id TEXT, scenario TEXT, level TEXT, ok INTEGER, outcome TEXT,
wall REAL, turns INTEGER, commands INTEGER, repeats INTEGER,
submissions INTEGER, rejected INTEGER, t_first_sub REAL, n_first_sub INTEGER,
t_pass REAL, n_pass INTEGER, gen_tokens INTEGER, vis_tokens INTEGER,
rea_tokens INTEGER, verdicts TEXT, detail TEXT);
SQL); SQL);
$insertRun = $pdo->prepare('INSERT INTO runs (run_id, ts, suite, model_id, model_quant, engine, gpu, gpu_count, label) VALUES (?,?,?,?,?,?,?,?,?)'); $insertRun = $pdo->prepare('INSERT INTO runs (run_id, ts, suite, model_id, model_quant, engine, gpu, gpu_count, label) VALUES (?,?,?,?,?,?,?,?,?)');
$insertRun->execute(['aaa111222333', '2026-09-01T10:00:00', 'softbench', 'fixture/model', 'Q5_K_M', 'llama.cpp', 'R9700', 1, 'fixture-label']); $insertRun->execute(['aaa111222333', '2026-09-01T10:00:00', 'softbench', 'fixture/model', 'Q5_K_M', 'llama.cpp', 'R9700', 1, 'deterministic sampling, repeat 1']);
$insertRun->execute(['bbb444555666', '2026-09-02T11:00:00', 'infrabench', 'fixture/model', null, 'vllm', 'unknown', 0, null]); $insertRun->execute(['bbb444555666', '2026-09-02T11:00:00', 'scenariobench', 'fixture/model', null, 'vllm', 'unknown', 0, 'agent test']);
$insertRun->execute(['ccc777888999', '2026-09-03T12:00:00', 'softbench', 'fixture/model', null, 'llama.cpp', 'R9700', 1, 'other session']);
$insertResult = $pdo->prepare('INSERT INTO results (run_id, suite, section, task, level, ok, decode, user_tps, ttft, t_answer, wall, reasoning, visible, total, finish, detail) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)'); $insertResult = $pdo->prepare('INSERT INTO results (run_id, suite, section, task, level, ok, decode, user_tps, ttft, t_answer, wall, reasoning, visible, total, finish, detail, tool_calls) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)');
$insertResult->execute(['aaa111222333', 'softbench', 'code', 'first_occurrence', 'off', 1, 83.82, 61.14, 0.47, 0.47, 1.77, 0, 108, 109, 'stop', '']); $insertResult->execute(['aaa111222333', 'softbench', 'code', 'first_occurrence', 'off', 1, 83.82, 61.14, 0.47, 0.47, 1.77, 0, 108, 109, 'stop', '', 2]);
$insertResult->execute(['aaa111222333', 'softbench', 'code', 'merge_intervals', 'off', 0, 51.2, 40.0, 0.5, 0.5, 3.0, 250, 90, 340, 'length', 'assertion failed']); $insertResult->execute(['aaa111222333', 'softbench', 'code', 'merge_intervals', 'n/a', 0, 51.2, 40.0, 0.5, 0.5, 3.0, 250, 90, 340, 'length', 'assertion failed', null]);
$insertResult->execute(['bbb444555666', 'infrabench', null, 'modprobe_exists', 'medium', 1, 40.0, 30.0, 1.2, 1.2, 10.0, 0, 200, 200, 'stop', '']); $insertResult->execute(['ccc777888999', 'softbench', 'code', 'other_session_task', 'off', 1, 70, 50, 0.4, 0.4, 2.0, 0, 100, 100, 'stop', '', null]);
$insertScenario = $pdo->prepare('INSERT INTO scenario_runs (run_id, scenario, level, ok, outcome, wall, turns, commands, repeats, submissions, rejected, t_first_sub, n_first_sub, t_pass, n_pass, gen_tokens, vis_tokens, rea_tokens, verdicts, detail) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)');
$insertScenario->execute(['bbb444555666', 'kubelet_config_repair', 'off', 1, 'passed', 264.1, 68, 67, 2, 1, 0, 262.5, 68, 264.1, 68, 2543, 2475, 0, '[{"n":1,"at":264.1,"accepted":true}]', 'passed on submission 1']);
} }
test('festbench:import imports runs and results from a real-schema sqlite db', function () { beforeEach(fn () => Artisan::call('festbench:suites:refresh'));
test('festbench:import routes rows to per-suite tables', function () {
$path = storage_path('app/test-benchmarks.db'); $path = storage_path('app/test-benchmarks.db');
festbench_fixture_db($path); festbench_fixture_db($path);
$this->artisan(FestbenchImport::class, ['path' => $path])->assertSuccessful(); $this->artisan(FestbenchImport::class, ['path' => $path])->assertSuccessful();
expect(BenchmarkRun::count())->toBe(2) expect(BenchmarkRun::count())->toBe(3)
->and(RunResult::count())->toBe(3); ->and(SoftbenchResult::count())->toBe(3)
->and(ScenarioRun::count())->toBe(1);
$run = BenchmarkRun::where('run_id', 'aaa111222333')->first(); $run = BenchmarkRun::where('run_id', 'aaa111222333')->first();
expect($run->suite)->toBe('softbench') expect($run->suite)->toBe('softbench')
->and($run->model_id)->toBe('fixture/model')
->and($run->model_quant)->toBe('Q5_K_M') ->and($run->model_quant)->toBe('Q5_K_M')
->and($run->ran_at->format('Y-m-d H:i'))->toBe('2026-09-01 10:00') ->and($run->visibility)->toBe('public');
->and($run->visibility)->toBe('public')
->and($run->results)->toHaveCount(2);
$failed = RunResult::where('task', 'merge_intervals')->first(); $failed = SoftbenchResult::where('task', 'merge_intervals')->first();
expect($failed->ok)->toBeFalse() expect($failed->ok)->toBeFalse()
->and($failed->finish)->toBe('length') ->and($failed->finish)->toBe('length')
->and($failed->detail)->toBe('assertion failed') ->and($failed->detail)->toBe('assertion failed')
// full source row preserved // n/a levels normalize to null
->and($failed->level)->toBeNull()
// complete source row preserved
->and($failed->payload['total'])->toBe(340); ->and($failed->payload['total'])->toBe(340);
// null quant and empty detail survive the import $toolUser = SoftbenchResult::where('task', 'first_occurrence')->first();
$second = BenchmarkRun::where('run_id', 'bbb444555666')->first(); expect($toolUser->tool_calls)->toBe(2);
expect($second->model_quant)->toBeNull()
->and($second->results->first()->detail)->toBeNull(); $scenario = ScenarioRun::first();
expect($scenario->scenario)->toBe('kubelet_config_repair')
->and($scenario->outcome)->toBe('passed')
->and($scenario->turns)->toBe(68)
->and($scenario->verdicts)->toBeArray()
->and($scenario->t_pass)->toBe(264.1);
});
test('festbench:import --label imports only the matching session', function () {
$path = storage_path('app/test-benchmarks.db');
festbench_fixture_db($path);
$this->artisan(FestbenchImport::class, ['path' => $path, '--label' => 'deterministic sampling, repeat 1'])->assertSuccessful();
expect(BenchmarkRun::count())->toBe(1)
->and(BenchmarkRun::first()->run_id)->toBe('aaa111222333')
->and(SoftbenchResult::count())->toBe(2);
}); });
test('festbench:import is idempotent', function () { test('festbench:import is idempotent', function () {
@ -76,8 +105,21 @@ function festbench_fixture_db(string $path): void
$this->artisan(FestbenchImport::class, ['path' => $path])->assertSuccessful(); $this->artisan(FestbenchImport::class, ['path' => $path])->assertSuccessful();
$this->artisan(FestbenchImport::class, ['path' => $path])->assertSuccessful(); $this->artisan(FestbenchImport::class, ['path' => $path])->assertSuccessful();
expect(BenchmarkRun::count())->toBe(2) expect(BenchmarkRun::count())->toBe(3)
->and(RunResult::count())->toBe(3); ->and(SoftbenchResult::count())->toBe(3);
});
test('festbench:import --fresh purges existing data first', function () {
$path = storage_path('app/test-benchmarks.db');
festbench_fixture_db($path);
// Pre-existing noise
BenchmarkRun::factory()->has(SoftbenchResult::factory(), 'softbenchResults')->create();
$this->artisan(FestbenchImport::class, ['path' => $path, '--fresh' => true, '--no-interaction' => true])->assertSuccessful();
expect(BenchmarkRun::count())->toBe(3)
->and(BenchmarkRun::pluck('run_id')->sort()->values()->all())->toBe(['aaa111222333', 'bbb444555666', 'ccc777888999']);
}); });
test('festbench:import attributes runs to a user and can default to private', function () { test('festbench:import attributes runs to a user and can default to private', function () {
@ -91,9 +133,8 @@ function festbench_fixture_db(string $path): void
expect($run->user_id)->toBe($me->id) expect($run->user_id)->toBe($me->id)
->and($run->visibility)->toBe('private'); ->and($run->visibility)->toBe('private');
// guest cannot see it, owner can
expect(BenchmarkRun::visibleTo(null)->count())->toBe(0) expect(BenchmarkRun::visibleTo(null)->count())->toBe(0)
->and(BenchmarkRun::visibleTo($me)->count())->toBe(2); ->and(BenchmarkRun::visibleTo($me)->count())->toBe(3);
}); });
test('festbench:import warns when the user does not exist', function () { test('festbench:import warns when the user does not exist', function () {
@ -106,3 +147,15 @@ function festbench_fixture_db(string $path): void
expect(BenchmarkRun::first()->user_id)->toBeNull(); expect(BenchmarkRun::first()->user_id)->toBeNull();
}); });
test('unregistered suites keep the run but warn about skipped rows', function () {
$path = storage_path('app/test-benchmarks.db');
festbench_fixture_db($path);
$pdo = new PDO('sqlite:'.$path);
$pdo->exec("INSERT INTO runs (run_id, ts, suite, model_id, label) VALUES ('ddd000111222', '2026-09-04T10:00:00', 'depthsweep', 'fixture/model', 'unknown suite')");
$pdo->exec("INSERT INTO results (run_id, suite, task, level, ok, wall) VALUES ('ddd000111222', 'depthsweep', 'depth_task', 'off', 1, 5.0)");
$this->artisan(FestbenchImport::class, ['path' => $path])->assertSuccessful();
expect(BenchmarkRun::where('run_id', 'ddd000111222')->exists())->toBeTrue();
});

52
tests/Feature/ResultsPageTest.php
View file

@ -1,14 +1,15 @@
<?php <?php
use App\Models\BenchmarkRun; use App\Models\BenchmarkRun;
use App\Models\RunResult; use App\Models\ProsebenchResult;
use App\Models\SoftbenchResult;
use App\Models\User; use App\Models\User;
use Livewire\Volt\Volt; use Livewire\Volt\Volt;
test('guests see public task results on the home page', function () { test('guests see public task results on the home page', function () {
$publicRun = BenchmarkRun::factory()->create(['model_id' => 'public/model']); $publicRun = BenchmarkRun::factory()->create(['model_id' => 'public/model', 'suite' => 'softbench']);
RunResult::factory()->count(3)->create(['benchmark_run_id' => $publicRun->id]); ProsebenchResult::factory()->count(3)->create(['benchmark_run_id' => $publicRun->id]);
BenchmarkRun::factory()->private()->hasResults(2)->create(['model_id' => 'private/model']); BenchmarkRun::factory()->private()->create(['model_id' => 'private/model', 'suite' => 'softbench']);
$response = $this->get(route('home')); $response = $this->get(route('home'));
@ -19,10 +20,8 @@
test('authenticated users see their private runs mixed in', function () { test('authenticated users see their private runs mixed in', function () {
$me = User::factory()->create(); $me = User::factory()->create();
BenchmarkRun::factory()->private()->hasResults(2)->create([ BenchmarkRun::factory()->private()->create(['model_id' => 'my-private/model', 'user_id' => $me->id, 'suite' => 'softbench']);
'model_id' => 'my-private/model', 'user_id' => $me->id, BenchmarkRun::factory()->private()->create(['model_id' => 'other-private/model', 'suite' => 'softbench']);
]);
BenchmarkRun::factory()->private()->hasResults(1)->create(['model_id' => 'other-private/model']);
$this->actingAs($me); $this->actingAs($me);
@ -31,33 +30,35 @@
->assertDontSee('other-private/model'); ->assertDontSee('other-private/model');
}); });
test('home page sorting orders by user tok/s', function () { test('home page sorting orders by user tok/s through the view', function () {
$run = BenchmarkRun::factory()->create(); $run = BenchmarkRun::factory()->create(['suite' => 'softbench']);
RunResult::factory()->create(['benchmark_run_id' => $run->id, 'task' => 'slow-task', 'user_tps' => 5]); SoftbenchResult::factory()->create(['benchmark_run_id' => $run->id, 'task' => 'slow-task', 'user_tps' => 5]);
RunResult::factory()->create(['benchmark_run_id' => $run->id, 'task' => 'fast-task', 'user_tps' => 95]); SoftbenchResult::factory()->create(['benchmark_run_id' => $run->id, 'task' => 'fast-task', 'user_tps' => 95]);
Volt::test('results.index', ['sortBy' => 'user_tps', 'sortDirection' => 'asc']) Volt::test('results.index', ['sortBy' => 'user_tps', 'sortDirection' => 'asc'])
->assertSeeInOrder(['slow-task', 'fast-task']); ->assertSeeInOrder(['slow-task', 'fast-task']);
}); });
test('home page filters narrow by suite', function () { test('home page suite filter queries the concrete suite table', function () {
$run = BenchmarkRun::factory()->create(); $softRun = BenchmarkRun::factory()->create(['suite' => 'softbench']);
RunResult::factory()->create(['benchmark_run_id' => $run->id, 'suite' => 'softbench', 'task' => 'soft-task']); $proseRun = BenchmarkRun::factory()->create(['suite' => 'prosebench']);
RunResult::factory()->create(['benchmark_run_id' => $run->id, 'suite' => 'infrabench', 'task' => 'infra-task']); SoftbenchResult::factory()->create(['benchmark_run_id' => $softRun->id, 'task' => 'soft-task']);
ProsebenchResult::factory()->create(['benchmark_run_id' => $proseRun->id, 'task' => 'prose-task']);
Volt::test('results.index', ['filters' => ['search' => null, 'suite' => 'softbench', 'level' => null, 'model' => null, 'result' => null]]) Volt::test('results.index', ['filters' => ['search' => null, 'suite' => 'softbench', 'level' => null, 'model' => null, 'result' => null]])
->assertSee('soft-task') ->assertSee('soft-task')
->assertDontSee('infra-task'); ->assertDontSee('prose-task');
}); });
test('run detail page shows metadata and results', function () { test('run detail page shows metadata and suite-tailored results', function () {
$run = BenchmarkRun::factory()->create(['model_id' => 'detail/model', 'engine' => 'vllm']); $run = BenchmarkRun::factory()->create(['model_id' => 'detail/model', 'engine' => 'vllm', 'suite' => 'prosebench']);
RunResult::factory()->create(['benchmark_run_id' => $run->id, 'task' => 'visible-task']); ProsebenchResult::factory()->create(['benchmark_run_id' => $run->id, 'task' => 'visible-task', 'checks_passed' => 6, 'checks_total' => 6]);
$this->get(route('runs.show', $run->run_id))->assertOk() $this->get(route('runs.show', $run->run_id))->assertOk()
->assertSee('detail/model') ->assertSee('detail/model')
->assertSee('visible-task') ->assertSee('visible-task')
->assertSee('vllm'); ->assertSee('vllm')
->assertSee('6/6');
}); });
test('private runs are hidden from guests on the detail page', function () { test('private runs are hidden from guests on the detail page', function () {
@ -105,13 +106,14 @@
test('my runs page lists only owned runs with counts', function () { test('my runs page lists only owned runs with counts', function () {
$me = User::factory()->create(); $me = User::factory()->create();
$run = BenchmarkRun::factory()->create(['user_id' => $me->id, 'model_id' => 'mine/model']); $run = BenchmarkRun::factory()->create(['user_id' => $me->id, 'model_id' => 'mine/model', 'suite' => 'softbench']);
RunResult::factory()->count(4)->create(['benchmark_run_id' => $run->id]); SoftbenchResult::factory()->count(4)->create(['benchmark_run_id' => $run->id]);
BenchmarkRun::factory()->create(['user_id' => User::factory()->create()->id, 'model_id' => 'theirs/model']); BenchmarkRun::factory()->create(['user_id' => User::factory()->create()->id, 'model_id' => 'theirs/model', 'suite' => 'softbench']);
$this->actingAs($me); $this->actingAs($me);
$this->get(route('my-runs'))->assertOk() $this->get(route('my-runs'))->assertOk()
->assertSee('mine/model') ->assertSee('mine/model')
->assertDontSee('theirs/model'); ->assertDontSee('theirs/model')
->assertSee('4');
}); });