pch/festbench-web
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

v2 Phase A+B: Postgres, partitioned per-suite tables, suite registry, all_results view, import rework, surgical re-import of deterministic session

Browse source
This commit is contained in:
PeterChrz 2026-09-18 14:41:33 -04:00
parent 39661d3ca7
commit 8c75db59f0
Signed by untrusted user who does not match committer: pch
GPG key ID: 8F0826ECF7302C63
20 changed files with 1040 additions and 223 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

188
app/Console/Commands/FestbenchImport.php
View file

@ -3,20 +3,22 @@
namespace App\Console\Commands;
use App\Models\BenchmarkRun;
use App\Models\RunResult;
use App\Models\User;
use App\Support\Suites;
use Illuminate\Console\Command;
use Illuminate\Support\Carbon;
use PDO;
use Symfony\Component\Console\Attribute\AsCommand;
#[AsCommand(name: 'festbench:import', description: 'Import a festbench benchmarks.db (SQLite) into the application database')]
#[AsCommand(name: 'festbench:import', description: 'Import a festbench benchmarks.db (SQLite) into the per-suite tables')]
class FestbenchImport extends Command
{
protected $signature = 'festbench:import
{path : Path to the festbench benchmarks.db}
{--user= : Email of the user to attribute the imported runs to}
{--private : Import runs as private instead of public}';
{--label= : Only import runs whose label matches exactly}
{--private : Import runs as private instead of public}
{--fresh : Delete ALL existing benchmark data before importing}';
public function handle(): int
{
@ -28,6 +30,10 @@ public function handle(): int
return self::FAILURE;
}
if ($this->option('fresh') && ! $this->confirmFresh()) {
return self::SUCCESS;
}
$user = $this->resolveUser();
$sqlite = new PDO('sqlite:'.$path, null, null, [
@ -35,13 +41,23 @@ public function handle(): int
PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC,
]);
$importedRuns = $skippedRuns = $importedResults = 0;
$query = 'SELECT * FROM runs';
$params = [];
if ($label = $this->option('label')) {
$query .= ' WHERE label = ?';
$params[] = $label;
}
$runs = $sqlite->query('SELECT * FROM runs')->fetchAll();
$progress = $this->output->createProgressBar(count($runs));
$runs = $sqlite->prepare($query);
$runs->execute($params);
$runRows = $runs->fetchAll();
$importedRuns = $skippedRuns = $importedResults = $unknownSuites = 0;
$progress = $this->output->createProgressBar(count($runRows));
$progress->start();
foreach ($runs as $source) {
foreach ($runRows as $source) {
if (BenchmarkRun::where('run_id', $source['run_id'])->exists()) {
$skippedRuns++;
$progress->advance();
@ -49,13 +65,61 @@ public function handle(): int
continue;
}
$run = BenchmarkRun::create([
$run = $this->createRun($source, $user);
$importedRuns++;
$sourceTable = Suites::sourceTable($source['suite'] ?? '');
$stmt = $sqlite->prepare("SELECT * FROM {$sourceTable} WHERE run_id = ?");
$stmt->execute([$source['run_id']]);
foreach ($stmt->fetchAll() as $row) {
$this->createSuiteRow($run, $row, $source);
$importedResults++;
}
$progress->advance();
}
$progress->finish();
$this->newLine(2);
$this->info("Imported {$importedRuns} runs ({$importedResults} suite rows), skipped {$skippedRuns} already present.");
if ($unknownSuites > 0) {
$this->warn("{$unknownSuites} rows were stored with fallback routing — check the suite registry.");
}
return self::SUCCESS;
}
private function confirmFresh(): bool
{
if (! $this->option('no-interaction')) {
if (! $this->confirm('This will DELETE all existing benchmark runs and results. Continue?')) {
return false;
}
}
// Delete children first (partitioned tables have no FK cascade issues,
// but explicit order is clearer and driver-agnostic).
foreach (Suites::all() as $suite) {
$model = Suites::modelFor($suite);
$model::query()->delete();
}
BenchmarkRun::query()->delete();
return true;
}
private function createRun(array $source, ?User $user): BenchmarkRun
{
return BenchmarkRun::create([
'run_id' => $source['run_id'],
'ran_at' => $source['ts'] ? Carbon::parse($source['ts']) : null,
'suite' => $source['suite'] ?? 'unknown',
'run_key' => $source['run_key'] ?? null,
'model_id' => $source['model_id'] ?? null,
'model_quant' => $source['model_quant'] ?? null,
'model_quant' => ($source['model_quant'] ?? '') !== '' ? $source['model_quant'] : null,
'model_quant_raw' => $source['model_quant_raw'] ?? null,
'model_path' => $source['model_path'] ?? null,
'engine' => $source['engine'] ?? null,
@ -68,7 +132,7 @@ public function handle(): int
'host' => $source['host'] ?? null,
'client' => $source['client'] ?? null,
'endpoint' => $source['endpoint'] ?? null,
'label' => $source['label'] ?? null,
'label' => ($source['label'] ?? '') !== '' ? $source['label'] : null,
'config' => $source['config'] ?? null,
'image_digest' => $source['image_digest'] ?? null,
'validators' => $this->decodeJson($source['validators'] ?? null),
@ -80,20 +144,64 @@ public function handle(): int
'visibility' => $this->option('private') ? BenchmarkRun::VISIBILITY_PRIVATE : BenchmarkRun::VISIBILITY_PUBLIC,
'notes' => $source['notes'] ?? null,
]);
}
$importedRuns++;
/**
* Route one source result row to its suite table via the registry.
* scenariobench rows have their own shape; metric suites share core
* columns with JSON-source extras (name→task, kind, checks, judge).
*/
private function createSuiteRow(BenchmarkRun $run, array $row, array $sourceRun): void
{
$suite = $run->suite;
$model = Suites::modelFor($suite);
$stmt = $sqlite->prepare('SELECT * FROM results WHERE run_id = ?');
$stmt->execute([$source['run_id']]);
if ($model === null) {
// Unknown suite: keep the run, drop the rows with a warning count.
$this->warn("Unregistered suite '{$suite}' (run {$run->run_id}) — rows skipped.");
foreach ($stmt->fetchAll() as $row) {
RunResult::create([
return;
}
$ranAt = $run->ran_at;
if ($suite === 'scenariobench') {
$model::create([
'benchmark_run_id' => $run->id,
'suite' => $row['suite'] ?? $source['suite'],
'section' => $row['section'] ?? null,
'task' => $row['task'] ?? ($row['name'] ?? 'unknown'),
'ran_at' => $ranAt,
'scenario' => $row['scenario'] ?? ($row['task'] ?? 'unknown'),
'level' => $row['level'] ?? null,
'ok' => $row['ok'] === null ? null : (bool) $row['ok'],
'outcome' => $row['outcome'] ?? null,
'wall' => $row['wall'] ?? null,
'turns' => $row['turns'] ?? null,
'commands' => $row['commands'] ?? null,
'repeats' => $row['repeats'] ?? null,
'submissions' => $row['submissions'] ?? null,
'rejected' => $row['rejected'] ?? null,
't_first_sub' => $row['t_first_sub'] ?? null,
'n_first_sub' => $row['n_first_sub'] ?? null,
't_pass' => $row['t_pass'] ?? null,
'n_pass' => $row['n_pass'] ?? null,
'gen_tokens' => $row['gen_tokens'] ?? null,
'vis_tokens' => $row['vis_tokens'] ?? null,
'rea_tokens' => $row['rea_tokens'] ?? null,
'verdicts' => $this->decodeJson($row['verdicts'] ?? null),
'detail' => ($row['detail'] ?? '') !== '' ? $row['detail'] : null,
'payload' => $row,
]);
return;
}
// Shared mapping for metric suites. JSON-source extras come through
// payload-style keys; the SQLite results table uses task/section.
$model::create(array_merge([
'benchmark_run_id' => $run->id,
'ran_at' => $ranAt,
'task' => $row['task'] ?? ($row['name'] ?? 'unknown'),
'level' => $row['level'] !== 'n/a' ? ($row['level'] ?? null) : null,
'ok' => array_key_exists('ok', $row) && $row['ok'] !== null ? (bool) $row['ok'] : null,
'decode' => $row['decode'] ?? null,
'user_tps' => $row['user_tps'] ?? null,
'ttft' => $row['ttft'] ?? null,
@ -102,25 +210,45 @@ public function handle(): int
'reasoning' => $row['reasoning'] ?? null,
'visible' => $row['visible'] ?? null,
'total' => $row['total'] ?? null,
'tool_calls' => $row['tool_calls'] ?? null,
'checks_passed' => $row['checks_passed'] ?? null,
'checks_total' => $row['checks_total'] ?? null,
'finish' => $row['finish'] ?? null,
'finish' => ($row['finish'] ?? '') !== '' ? $row['finish'] : null,
'detail' => ($row['detail'] ?? '') !== '' ? $row['detail'] : null,
'payload' => $row,
]);
$importedResults++;
], $this->suiteExtras($suite, $row)));
}
$progress->advance();
/**
* Suite-specific columns, keyed for create(). JSON sources use name,
* checks_passed, checks_total, judge, prefill, prompt_tokens, kind; the
* SQLite results table uses section and tool_calls.
*/
private function suiteExtras(string $suite, array $row): array
{
$extras = [];
if (in_array($suite, ['softbench', 'prosebench', 'psychbench'], true)) {
$extras['section'] = ($row['section'] ?? '') !== '' ? $row['section'] : null;
}
$progress->finish();
$this->newLine(2);
$this->info("Imported {$importedRuns} runs ({$importedResults} results), skipped {$skippedRuns} already present.");
if ($suite === 'softbench') {
$extras['tool_calls'] = $row['tool_calls'] ?? null;
}
return self::SUCCESS;
if ($suite === 'infrabench') {
$extras['kind'] = ($row['kind'] ?? '') !== '' ? $row['kind'] : null;
}
if (in_array($suite, ['prosebench', 'psychbench'], true)) {
$extras['checks_passed'] = $row['checks_passed'] ?? null;
$extras['checks_total'] = $row['checks_total'] ?? null;
}
if ($suite === 'prosebench') {
$extras['judge'] = ($row['judge'] ?? '') !== '' ? $row['judge'] : null;
$extras['prefill'] = $row['prefill'] ?? null;
$extras['prompt_tokens'] = $row['prompt_tokens'] ?? null;
}
return $extras;
}
private function resolveUser(): ?User

45
app/Console/Commands/SuitesRefresh.php Normal file
View file

@ -0,0 +1,45 @@
<?php
namespace App\Console\Commands;
use App\Support\Suites;
use Illuminate\Console\Command;
use Illuminate\Support\Facades\DB;
use Symfony\Component\Console\Attribute\AsCommand;
#[AsCommand(name: 'festbench:suites:refresh', description: 'Rebuild the all_results view from the active suite registry (decommission/add lever)')]
class SuitesRefresh extends Command
{
public function handle(): int
{
$suites = Suites::inAllResults();
if ($suites->isEmpty()) {
$this->warn('No active suites in the registry — skipping view rebuild.');
return self::SUCCESS;
}
$driver = DB::connection()->getDriverName();
$arms = $suites->map(fn (string $suite) => $this->arm($suite, $driver))->implode(' UNION ALL ');
DB::statement('DROP VIEW IF EXISTS all_results');
DB::statement("CREATE VIEW all_results AS {$arms}");
$this->info('all_results view rebuilt over: '.$suites->implode(', '));
return self::SUCCESS;
}
/**
* One UNION arm per suite. All metric tables share the core column set,
* so the arm promotes the suite name as a literal plus the common columns.
*/
private function arm(string $suite, string $driver): string
{
$table = Suites::modelFor($suite)::make()->getTable();
$quoted = $driver === 'pgsql' ? "'{$suite}'" : "'{$suite}'";
return "SELECT id, benchmark_run_id, ran_at, task, level, ok, decode, user_tps, ttft, t_answer, wall, reasoning, visible, total, finish, detail, {$quoted} AS suite FROM {$table}";
}
}

43
app/Models/BenchmarkRun.php
View file

@ -2,8 +2,10 @@
namespace App\Models;
use App\Support\Suites;
use Database\Factories\BenchmarkRunFactory;
use Illuminate\Database\Eloquent\Builder;
use Illuminate\Database\Eloquent\Collection;
use Illuminate\Database\Eloquent\Factories\HasFactory;
use Illuminate\Database\Eloquent\Model;
use Illuminate\Database\Eloquent\Relations\BelongsTo;
@ -68,9 +70,46 @@ public function user(): BelongsTo
return $this->belongsTo(User::class);
}
public function results(): HasMany
public function softbenchResults(): HasMany
{
return $this->hasMany(RunResult::class);
return $this->hasMany(SoftbenchResult::class);
}
public function infrabenchResults(): HasMany
{
return $this->hasMany(InfrabenchResult::class);
}
public function prosebenchResults(): HasMany
{
return $this->hasMany(ProsebenchResult::class);
}
public function psychbenchResults(): HasMany
{
return $this->hasMany(PsychbenchResult::class);
}
public function scenarioRuns(): HasMany
{
return $this->hasMany(ScenarioRun::class);
}
/**
* The suite-specific result rows for this run, via the registry. Returns
* an empty collection for suites this run does not have.
*/
public function suiteResults(): Collection
{
$model = Suites::modelFor($this->suite);
if ($model === null) {
return collect();
}
$relation = Suites::runRelation($this->suite);
return $this->{$relation}()->orderBy('task')->orderBy('id')->get();
}
public function isOwnedBy(?User $user): bool

19
app/Models/InfrabenchResult.php Normal file
View file

@ -0,0 +1,19 @@
<?php
namespace App\Models;
use Database\Factories\InfrabenchResultFactory;
use Illuminate\Database\Eloquent\Factories\HasFactory;
class InfrabenchResult extends SuiteResult
{
/** @use HasFactory<InfrabenchResultFactory> */
use HasFactory;
protected $fillable = [
'benchmark_run_id', 'ran_at', 'task', 'level',
'ok', 'decode', 'user_tps', 'ttft', 't_answer', 'wall',
'reasoning', 'visible', 'total', 'finish', 'detail',
'kind', 'payload',
];
}

30
app/Models/ProsebenchResult.php Normal file
View file

@ -0,0 +1,30 @@
<?php
namespace App\Models;
use Database\Factories\ProsebenchResultFactory;
use Illuminate\Database\Eloquent\Factories\HasFactory;
class ProsebenchResult extends SuiteResult
{
/** @use HasFactory<ProsebenchResultFactory> */
use HasFactory;
protected $fillable = [
'benchmark_run_id', 'ran_at', 'task', 'level',
'ok', 'decode', 'user_tps', 'ttft', 't_answer', 'wall',
'reasoning', 'visible', 'total', 'finish', 'detail',
'section', 'checks_passed', 'checks_total', 'judge',
'prefill', 'prompt_tokens', 'payload',
];
protected function casts(): array
{
return array_merge(parent::casts(), [
'checks_passed' => 'integer',
'checks_total' => 'integer',
'prefill' => 'float',
'prompt_tokens' => 'integer',
]);
}
}

27
app/Models/PsychbenchResult.php Normal file
View file

@ -0,0 +1,27 @@
<?php
namespace App\Models;
use Database\Factories\PsychbenchResultFactory;
use Illuminate\Database\Eloquent\Factories\HasFactory;
class PsychbenchResult extends SuiteResult
{
/** @use HasFactory<PsychbenchResultFactory> */
use HasFactory;
protected $fillable = [
'benchmark_run_id', 'ran_at', 'task', 'level',
'ok', 'decode', 'user_tps', 'ttft', 't_answer', 'wall',
'reasoning', 'visible', 'total', 'finish', 'detail',
'section', 'checks_passed', 'checks_total', 'payload',
];
protected function casts(): array
{
return array_merge(parent::casts(), [
'checks_passed' => 'integer',
'checks_total' => 'integer',
]);
}
}

49
app/Models/ScenarioRun.php Normal file
View file

@ -0,0 +1,49 @@
<?php
namespace App\Models;
use Database\Factories\ScenarioRunFactory;
use Illuminate\Database\Eloquent\Factories\HasFactory;
use Illuminate\Database\Eloquent\Model;
use Illuminate\Database\Eloquent\Relations\BelongsTo;
class ScenarioRun extends Model
{
/** @use HasFactory<ScenarioRunFactory> */
use HasFactory;
protected $fillable = [
'benchmark_run_id', 'ran_at', 'scenario', 'level', 'ok', 'outcome',
'wall', 'turns', 'commands', 'repeats', 'submissions', 'rejected',
't_first_sub', 'n_first_sub', 't_pass', 'n_pass',
'gen_tokens', 'vis_tokens', 'rea_tokens', 'verdicts', 'detail', 'payload',
];
protected function casts(): array
{
return [
'ran_at' => 'datetime',
'ok' => 'boolean',
'wall' => 'float',
'turns' => 'integer',
'commands' => 'integer',
'repeats' => 'integer',
'submissions' => 'integer',
'rejected' => 'integer',
't_first_sub' => 'float',
'n_first_sub' => 'integer',
't_pass' => 'float',
'n_pass' => 'integer',
'gen_tokens' => 'integer',
'vis_tokens' => 'integer',
'rea_tokens' => 'integer',
'verdicts' => 'array',
'payload' => 'array',
];
}
public function run(): BelongsTo
{
return $this->belongsTo(BenchmarkRun::class, 'benchmark_run_id');
}
}

26
app/Models/SoftbenchResult.php Normal file
View file

@ -0,0 +1,26 @@
<?php
namespace App\Models;
use Database\Factories\SoftbenchResultFactory;
use Illuminate\Database\Eloquent\Factories\HasFactory;
class SoftbenchResult extends SuiteResult
{
/** @use HasFactory<SoftbenchResultFactory> */
use HasFactory;
protected $fillable = [
'benchmark_run_id', 'ran_at', 'task', 'level',
'ok', 'decode', 'user_tps', 'ttft', 't_answer', 'wall',
'reasoning', 'visible', 'total', 'finish', 'detail',
'section', 'tool_calls', 'payload',
];
protected function casts(): array
{
return array_merge(parent::casts(), [
'tool_calls' => 'integer',
]);
}
}

23
app/Models/RunResult.php → app/Models/SuiteResult.php
View file

@ -2,26 +2,20 @@
namespace App\Models;
use Database\Factories\RunResultFactory;
use Illuminate\Database\Eloquent\Factories\HasFactory;
use Illuminate\Database\Eloquent\Model;
use Illuminate\Database\Eloquent\Relations\BelongsTo;
class RunResult extends Model
/**
* Shared behavior for per-suite result tables. Each concrete model declares
* its own $fillable (suite-specific columns + core) — submitted metrics are
* immutable once stored, exactly like the parent BenchmarkRun.
*/
abstract class SuiteResult extends Model
{
/** @use HasFactory<RunResultFactory> */
use HasFactory;
protected $fillable = [
'benchmark_run_id', 'suite', 'section', 'task', 'level',
'ok', 'decode', 'user_tps', 'ttft', 't_answer', 'wall',
'reasoning', 'visible', 'total', 'tool_calls',
'checks_passed', 'checks_total', 'finish', 'detail', 'payload',
];
protected function casts(): array
{
return [
'ran_at' => 'datetime',
'ok' => 'boolean',
'decode' => 'float',
'user_tps' => 'float',
@ -31,9 +25,6 @@ protected function casts(): array
'reasoning' => 'integer',
'visible' => 'integer',
'total' => 'integer',
'tool_calls' => 'integer',
'checks_passed' => 'integer',
'checks_total' => 'integer',
'payload' => 'array',
];
}

76
app/Support/Suites.php Normal file
View file

@ -0,0 +1,76 @@
<?php
namespace App\Support;
use App\Models\InfrabenchResult;
use App\Models\ProsebenchResult;
use App\Models\PsychbenchResult;
use App\Models\ScenarioRun;
use App\Models\SoftbenchResult;
use Illuminate\Support\Collection;
/**
* Typed accessor over the suite registry in config/festbench.php.
* Single source of truth for import routing, view generation, UI rendering
* and the suite filter.
*/
class Suites
{
/** @var array<string, class-string> */
private const MODELS = [
'softbench' => SoftbenchResult::class,
'infrabench' => InfrabenchResult::class,
'prosebench' => ProsebenchResult::class,
'psychbench' => PsychbenchResult::class,
'scenariobench' => ScenarioRun::class,
];
/** Model class handling results for $suite, or null if unregistered. */
public static function modelFor(?string $suite): ?string
{
return self::MODELS[$suite] ?? null;
}
/** Name of the HasMany relation on BenchmarkRun for $suite. */
public static function runRelation(string $suite): string
{
return $suite === 'scenariobench' ? 'scenarioRuns' : $suite.'Results';
}
/** Source table in the festbench SQLite db for $suite. */
public static function sourceTable(string $suite): string
{
return config("festbench.source_tables.{$suite}", 'results');
}
/** All registered suites, active and inactive. */
public static function all(): array
{
return array_keys(self::MODELS);
}
/**
* Active suites (appear in the suite filter and run-detail rendering).
*
* @return Collection<int, string>
*/
public static function active(): Collection
{
return collect(self::MODELS)
->filter(fn (string $m, string $suite) => config("festbench.suites.{$suite}.active", false))
->keys();
}
/**
* Active suites that participate in the combined all_results view.
*
* @return Collection<int, string>
*/
public static function inAllResults(): Collection
{
return collect(self::MODELS)
->filter(fn (string $m, string $suite) => config("festbench.suites.{$suite}.active", false)
&& config("festbench.suites.{$suite}.in_all_results", false))
->keys();
}
}

60
config/festbench.php Normal file
View file

@ -0,0 +1,60 @@
<?php
use App\Models\InfrabenchResult;
use App\Models\ProsebenchResult;
use App\Models\PsychbenchResult;
use App\Models\ScenarioRun;
use App\Models\SoftbenchResult;
// Suite registry: the single source of truth for which bench suites exist,
// which model stores their results, and whether they are active.
//
// - `active => false` removes the suite from the all_results view and the
// home page (decommissioning) — historical data stays queryable and run
// detail pages keep rendering.
// - The view is rebuilt by `festbench:suites:refresh` after editing this file.
// - Adding a future suite: create its migration + model, add it here, run
// the refresh command.
return [
'suites' => [
'softbench' => [
'model' => SoftbenchResult::class,
'active' => true,
'in_all_results' => true,
],
'infrabench' => [
'model' => InfrabenchResult::class,
'active' => true,
'in_all_results' => true,
],
'prosebench' => [
'model' => ProsebenchResult::class,
'active' => true,
'in_all_results' => true,
],
'psychbench' => [
'model' => PsychbenchResult::class,
'active' => true,
'in_all_results' => true,
],
'scenariobench' => [
'model' => ScenarioRun::class,
'active' => true,
// Agentic metrics (turns, submissions) — no tok/s semantics, so
// it is excluded from the combined metric view.
'in_all_results' => false,
],
],
// Source table in the festbench SQLite db for each suite ('results' is
// the generic metrics table; scenariobench has its own).
'source_tables' => [
'scenariobench' => 'scenario_runs',
],
// Caching: TTL staleness only — never event-flush. At high submission
// rates, event flushes would storm the cache; benchmark data being a
// couple of minutes old is correct behavior.
'home_cache_ttl' => env('FESTBENCH_HOME_CACHE_TTL', 300),
];

36
database/factories/InfrabenchResultFactory.php Normal file
View file

@ -0,0 +1,36 @@
<?php
namespace Database\Factories;
use App\Models\BenchmarkRun;
use App\Models\InfrabenchResult;
use Illuminate\Database\Eloquent\Factories\Factory;
/** @extends Factory<InfrabenchResult> */
class InfrabenchResultFactory extends Factory
{
public function definition(): array
{
$ttft = round($this->faker->randomFloat(2, 0.1, 4), 2);
return [
'benchmark_run_id' => BenchmarkRun::factory(),
'ran_at' => now(),
'task' => $this->faker->randomElement(['modprobe_exists', 'rbac_missing_verb', 'oom_innocent_victim']),
'level' => $this->faker->randomElement(['off', 'low', 'medium', 'xhigh']),
'ok' => $this->faker->boolean(70),
'decode' => round($this->faker->randomFloat(2, 10, 150), 2),
'user_tps' => round($this->faker->randomFloat(2, 5, 100), 2),
'ttft' => $ttft,
't_answer' => $ttft,
'wall' => round($ttft + $this->faker->randomFloat(2, 0.5, 30), 2),
'reasoning' => $this->faker->numberBetween(0, 2000),
'visible' => $this->faker->numberBetween(20, 400),
'total' => $this->faker->numberBetween(20, 2000),
'finish' => 'stop',
'detail' => null,
'kind' => $this->faker->randomElement(['recall', 'constraint']),
'payload' => ['ok' => true],
];
}
}

42
database/factories/ProsebenchResultFactory.php Normal file
View file

@ -0,0 +1,42 @@
<?php
namespace Database\Factories;
use App\Models\BenchmarkRun;
use App\Models\ProsebenchResult;
use Illuminate\Database\Eloquent\Factories\Factory;
/** @extends Factory<ProsebenchResult> */
class ProsebenchResultFactory extends Factory
{
public function definition(): array
{
$ttft = round($this->faker->randomFloat(2, 0.1, 4), 2);
$checksTotal = $this->faker->numberBetween(3, 8);
return [
'benchmark_run_id' => BenchmarkRun::factory(),
'ran_at' => now(),
'task' => $this->faker->randomElement(['grief_email', 'refund_denial', 'status_update']),
'level' => $this->faker->randomElement(['off', 'low', 'medium', 'xhigh']),
'ok' => $this->faker->boolean(70),
'decode' => round($this->faker->randomFloat(2, 10, 150), 2),
'user_tps' => round($this->faker->randomFloat(2, 5, 100), 2),
'ttft' => $ttft,
't_answer' => $ttft,
'wall' => round($ttft + $this->faker->randomFloat(2, 0.5, 30), 2),
'reasoning' => $this->faker->numberBetween(0, 2000),
'visible' => $this->faker->numberBetween(20, 400),
'total' => $this->faker->numberBetween(20, 2000),
'finish' => 'stop',
'detail' => null,
'section' => $this->faker->randomElement(['sensitive', 'actionable']),
'checks_passed' => $this->faker->numberBetween(0, $checksTotal),
'checks_total' => $checksTotal,
'judge' => null,
'prefill' => round($this->faker->randomFloat(1, 50, 400), 1),
'prompt_tokens' => $this->faker->numberBetween(100, 500),
'payload' => ['ok' => true],
];
}
}

39
database/factories/PsychbenchResultFactory.php Normal file
View file

@ -0,0 +1,39 @@
<?php
namespace Database\Factories;
use App\Models\BenchmarkRun;
use App\Models\PsychbenchResult;
use Illuminate\Database\Eloquent\Factories\Factory;
/** @extends Factory<PsychbenchResult> */
class PsychbenchResultFactory extends Factory
{
public function definition(): array
{
$ttft = round($this->faker->randomFloat(2, 0.1, 4), 2);
$checksTotal = $this->faker->numberBetween(3, 8);
return [
'benchmark_run_id' => BenchmarkRun::factory(),
'ran_at' => now(),
'task' => $this->faker->randomElement(['billing_form', 'irreducible_contradiction', 'sunk_cost']),
'level' => $this->faker->randomElement(['off', 'low', 'medium', 'xhigh']),
'ok' => $this->faker->boolean(70),
'decode' => round($this->faker->randomFloat(2, 10, 150), 2),
'user_tps' => round($this->faker->randomFloat(2, 5, 100), 2),
'ttft' => $ttft,
't_answer' => $ttft,
'wall' => round($ttft + $this->faker->randomFloat(2, 0.5, 30), 2),
'reasoning' => $this->faker->numberBetween(0, 2000),
'visible' => $this->faker->numberBetween(20, 400),
'total' => $this->faker->numberBetween(20, 2000),
'finish' => 'stop',
'detail' => null,
'section' => null,
'checks_passed' => $this->faker->numberBetween(0, $checksTotal),
'checks_total' => $checksTotal,
'payload' => ['ok' => true],
];
}
}

54
database/factories/RunResultFactory.php
View file

@ -1,54 +0,0 @@
<?php
namespace Database\Factories;
use App\Models\BenchmarkRun;
use Illuminate\Database\Eloquent\Factories\Factory;
/** @extends Factory<RunResult> */
class RunResultFactory extends Factory
{
public function definition(): array
{
$visible = $this->faker->numberBetween(20, 400);
$reasoning = $this->faker->numberBetween(0, 2000);
$ttft = round($this->faker->randomFloat(2, 0.1, 4), 2);
$wall = round($ttft + $this->faker->randomFloat(2, 0.5, 60), 2);
$ok = $this->faker->boolean(70);
$payload = [
'level' => null, 'section' => null, 'name' => null, 'ok' => $ok,
'ttft' => $ttft, 'wall' => $wall,
];
return [
'benchmark_run_id' => BenchmarkRun::factory(),
'suite' => 'softbench',
'section' => $this->faker->randomElement(['code', 'reasoning', 'constraints', null]),
'task' => $this->faker->randomElement([
'first_occurrence', 'merge_intervals', 'version_compare', 'rbac_missing_verb', 'oom_innocent_victim',
]),
'level' => $this->faker->randomElement(['off', 'low', 'medium', 'xhigh']),
'ok' => $ok,
'decode' => round($this->faker->randomFloat(2, 10, 150), 2),
'user_tps' => round($this->faker->randomFloat(2, 5, 100), 2),
'ttft' => $ttft,
't_answer' => $ttft,
'wall' => $wall,
'reasoning' => $reasoning,
'visible' => $visible,
'total' => $visible + $reasoning,
'tool_calls' => null,
'checks_passed' => null,
'checks_total' => null,
'finish' => $ok ? 'stop' : $this->faker->randomElement(['stop', 'length']),
'detail' => null,
'payload' => $payload,
];
}
public function forSuite(string $suite): static
{
return $this->state(fn () => ['suite' => $suite]);
}
}

41
database/factories/ScenarioRunFactory.php Normal file
View file

@ -0,0 +1,41 @@
<?php
namespace Database\Factories;
use App\Models\BenchmarkRun;
use App\Models\ScenarioRun;
use Illuminate\Database\Eloquent\Factories\Factory;
/** @extends Factory<ScenarioRun> */
class ScenarioRunFactory extends Factory
{
public function definition(): array
{
$passed = $this->faker->boolean(60);
return [
'benchmark_run_id' => BenchmarkRun::factory(),
'ran_at' => now(),
'scenario' => $this->faker->randomElement(['kubelet_config_repair', 'deployment_unschedulable', 'systemd_unit_repair']),
'level' => $this->faker->randomElement(['off', 'low', 'medium', 'xhigh']),
'ok' => $passed,
'outcome' => $passed ? 'passed' : $this->faker->randomElement(['rejected_final', 'timeout', 'turn_cap', 'livelock', 'error']),
'wall' => round($this->faker->randomFloat(1, 30, 600), 1),
'turns' => $this->faker->numberBetween(5, 80),
'commands' => $this->faker->numberBetween(4, 75),
'repeats' => $this->faker->numberBetween(0, 10),
'submissions' => $this->faker->numberBetween(1, 5),
'rejected' => $this->faker->numberBetween(0, 4),
't_first_sub' => round($this->faker->randomFloat(1, 20, 500), 1),
'n_first_sub' => $this->faker->numberBetween(3, 70),
't_pass' => $passed ? round($this->faker->randomFloat(1, 30, 550), 1) : null,
'n_pass' => $passed ? $this->faker->numberBetween(3, 75) : null,
'gen_tokens' => $this->faker->numberBetween(500, 20000),
'vis_tokens' => $this->faker->numberBetween(400, 19000),
'rea_tokens' => $this->faker->numberBetween(0, 5000),
'verdicts' => [['n' => 1, 'at' => 264.1, 'accepted' => $passed, 'oracle' => 'sample']],
'detail' => $passed ? 'passed on submission 1' : 'never accepted',
'payload' => ['ok' => $passed],
];
}
}

37
database/factories/SoftbenchResultFactory.php Normal file
View file

@ -0,0 +1,37 @@
<?php
namespace Database\Factories;
use App\Models\BenchmarkRun;
use App\Models\SoftbenchResult;
use Illuminate\Database\Eloquent\Factories\Factory;
/** @extends Factory<SoftbenchResult> */
class SoftbenchResultFactory extends Factory
{
public function definition(): array
{
$ttft = round($this->faker->randomFloat(2, 0.1, 4), 2);
return [
'benchmark_run_id' => BenchmarkRun::factory(),
'ran_at' => now(),
'task' => $this->faker->randomElement(['first_occurrence', 'merge_intervals', 'version_compare']),
'level' => $this->faker->randomElement(['off', 'low', 'medium', 'xhigh']),
'ok' => $this->faker->boolean(70),
'decode' => round($this->faker->randomFloat(2, 10, 150), 2),
'user_tps' => round($this->faker->randomFloat(2, 5, 100), 2),
'ttft' => $ttft,
't_answer' => $ttft,
'wall' => round($ttft + $this->faker->randomFloat(2, 0.5, 30), 2),
'reasoning' => $this->faker->numberBetween(0, 2000),
'visible' => $this->faker->numberBetween(20, 400),
'total' => $this->faker->numberBetween(20, 2000),
'finish' => 'stop',
'detail' => null,
'section' => $this->faker->randomElement(['code', 'reasoning', 'constraints']),
'tool_calls' => $this->faker->optional()->numberBetween(1, 5),
'payload' => ['ok' => true],
];
}
}

259
database/migrations/2026_09_18_000001_reorganize_suite_tables.php Normal file
View file

@ -0,0 +1,259 @@
<?php
use Illuminate\Database\Migrations\Migration;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Schema;
return new class extends Migration
{
/**
* v2 re-organization: one table per bench suite. On Postgres the metric
* tables are partitioned monthly by the denormalized ran_at (copied from
* the parent run at insert time), which keeps inserts on a hot partition
* and lets decommissioning a bench detach whole partitions. On SQLite
* (test runs) they are plain tables.
*
* The generic run_results table is replaced by the all_results view,
* rebuilt from the active suite registry via festbench:suites:refresh.
*
* Table shapes are declared once, as raw column strings, because the
* Laravel schema builder cannot express Postgres PARTITION BY.
*/
private const CORE_COLUMNS = <<<'SQL'
id BIGINT GENERATED ALWAYS AS IDENTITY,
benchmark_run_id BIGINT NOT NULL REFERENCES benchmark_runs(id) ON DELETE CASCADE,
ran_at TIMESTAMP(0) WITHOUT TIME ZONE,
task VARCHAR(128) NOT NULL,
level VARCHAR(16),
ok BOOLEAN,
decode DOUBLE PRECISION,
user_tps DOUBLE PRECISION,
ttft DOUBLE PRECISION,
t_answer DOUBLE PRECISION,
wall DOUBLE PRECISION,
reasoning INTEGER,
visible INTEGER,
total INTEGER,
finish VARCHAR(32),
detail TEXT,
payload JSONB,
created_at TIMESTAMP(0) WITHOUT TIME ZONE,
updated_at TIMESTAMP(0) WITHOUT TIME ZONE
SQL;
public function up(): void
{
Schema::dropIfExists('run_results');
$pgsql = DB::connection()->getDriverName() === 'pgsql';
$suiteExtras = [
'softbench_results' => ['section VARCHAR(64)', 'tool_calls INTEGER'],
'infrabench_results' => ['kind VARCHAR(32)'],
'prosebench_results' => ['section VARCHAR(64)', 'checks_passed INTEGER', 'checks_total INTEGER', 'judge TEXT', 'prefill DOUBLE PRECISION', 'prompt_tokens INTEGER'],
'psychbench_results' => ['section VARCHAR(64)', 'checks_passed INTEGER', 'checks_total INTEGER'],
];
foreach ($suiteExtras as $table => $extras) {
if ($pgsql) {
$this->createPgTable($table, self::CORE_COLUMNS, $extras);
} else {
$this->createSqliteTable($table, $extras);
}
}
if ($pgsql) {
DB::statement(<<<'SQL'
CREATE TABLE scenario_runs (
id BIGINT GENERATED ALWAYS AS IDENTITY,
benchmark_run_id BIGINT NOT NULL REFERENCES benchmark_runs(id) ON DELETE CASCADE,
ran_at TIMESTAMP(0) WITHOUT TIME ZONE,
scenario VARCHAR(128) NOT NULL,
level VARCHAR(16),
ok BOOLEAN,
outcome VARCHAR(32),
wall DOUBLE PRECISION,
turns INTEGER,
commands INTEGER,
repeats INTEGER,
submissions INTEGER,
rejected INTEGER,
t_first_sub DOUBLE PRECISION,
n_first_sub INTEGER,
t_pass DOUBLE PRECISION,
n_pass INTEGER,
gen_tokens INTEGER,
vis_tokens INTEGER,
rea_tokens INTEGER,
verdicts JSONB,
detail TEXT,
payload JSONB,
created_at TIMESTAMP(0) WITHOUT TIME ZONE,
updated_at TIMESTAMP(0) WITHOUT TIME ZONE
) PARTITION BY RANGE (ran_at)
SQL);
$this->addIndexes('scenario_runs');
} else {
Schema::create('scenario_runs', function (Blueprint $table) {
$table->id();
$table->foreignId('benchmark_run_id')->constrained()->cascadeOnDelete();
$table->timestamp('ran_at')->nullable();
$table->string('scenario', 128);
$table->string('level', 16)->nullable();
$table->boolean('ok')->nullable();
$table->string('outcome', 32)->nullable();
$table->double('wall')->nullable();
$table->unsignedInteger('turns')->nullable();
$table->unsignedInteger('commands')->nullable();
$table->unsignedInteger('repeats')->nullable();
$table->unsignedInteger('submissions')->nullable();
$table->unsignedInteger('rejected')->nullable();
$table->double('t_first_sub')->nullable();
$table->unsignedInteger('n_first_sub')->nullable();
$table->double('t_pass')->nullable();
$table->unsignedInteger('n_pass')->nullable();
$table->unsignedInteger('gen_tokens')->nullable();
$table->unsignedInteger('vis_tokens')->nullable();
$table->unsignedInteger('rea_tokens')->nullable();
$table->jsonb('verdicts')->nullable();
$table->text('detail')->nullable();
$table->jsonb('payload')->nullable();
$table->timestamps();
$table->index('benchmark_run_id');
$table->index('wall');
});
}
if ($pgsql) {
// visibility integrity + JSONB on the runs table
DB::statement('ALTER TABLE benchmark_runs DROP CONSTRAINT IF EXISTS benchmark_runs_visibility_check');
DB::statement("ALTER TABLE benchmark_runs ADD CONSTRAINT benchmark_runs_visibility_check CHECK (visibility IN ('public', 'private'))");
DB::statement('ALTER TABLE benchmark_runs ALTER COLUMN validators TYPE jsonb USING validators::jsonb');
DB::statement('ALTER TABLE benchmark_runs ALTER COLUMN suite_versions TYPE jsonb USING suite_versions::jsonb');
$this->createPartitions();
}
}
private function createPgTable(string $table, string $columns, array $extras): void
{
$extrasSql = $extras ? ', '.implode(', ', $extras) : '';
DB::statement("CREATE TABLE {$table} ({$columns}{$extrasSql}) PARTITION BY RANGE (ran_at)");
$this->addIndexes($table);
}
private function addIndexes(string $table): void
{
$cols = $table === 'scenario_runs'
? ['benchmark_run_id', 'wall']
: ['benchmark_run_id', 'user_tps', 'decode', 'wall'];
foreach ($cols as $col) {
DB::statement("CREATE INDEX IF NOT EXISTS {$table}_{$col}_idx ON {$table} ({$col})");
}
}
/**
* SQLite (tests): plain tables via the schema builder.
*/
private function createSqliteTable(string $table, array $extras): void
{
Schema::create($table, function (Blueprint $table) {
$table->id();
$table->foreignId('benchmark_run_id')->constrained()->cascadeOnDelete();
$table->timestamp('ran_at')->nullable();
$table->string('task', 128);
$table->string('level', 16)->nullable();
$table->boolean('ok')->nullable();
$table->double('decode')->nullable();
$table->double('user_tps')->nullable();
$table->double('ttft')->nullable();
$table->double('t_answer')->nullable();
$table->double('wall')->nullable();
$table->unsignedInteger('reasoning')->nullable();
$table->unsignedInteger('visible')->nullable();
$table->unsignedInteger('total')->nullable();
$table->string('finish', 32)->nullable();
$table->text('detail')->nullable();
$table->jsonb('payload')->nullable();
$table->timestamps();
$table->index('benchmark_run_id');
$table->index('user_tps');
$table->index('decode');
$table->index('wall');
});
// SQLite has no ALTER ADD COLUMN via builder for positioning; add extras.
Schema::table($table, function (Blueprint $t) use ($extras) {
foreach ($extras as $extra) {
[$name, $type] = explode(' ', $extra);
$t->addColumn($type, $name)->nullable();
}
});
}
/**
* Monthly partitions from a year back to a year forward, plus a default
* partition so inserts never miss. Creating future months in production
* is handled by the scheduled festbench:partitions command.
*/
private function createPartitions(): void
{
$tables = ['softbench_results', 'infrabench_results', 'prosebench_results', 'psychbench_results', 'scenario_runs'];
$start = now()->subYear()->startOfMonth();
$end = now()->addYear()->endOfMonth();
for ($m = $start->copy(); $m->lessThan($end); $m->addMonth()) {
$from = $m->format('Y-m-d');
$to = $m->copy()->addMonth()->format('Y-m-d');
foreach ($tables as $t) {
DB::statement("CREATE TABLE IF NOT EXISTS {$t}_{$m->format('Y_m')} PARTITION OF {$t} FOR VALUES FROM ('{$from}') TO ('{$to}')");
}
}
foreach ($tables as $t) {
DB::statement("CREATE TABLE IF NOT EXISTS {$t}_default PARTITION OF {$t} DEFAULT");
}
}
public function down(): void
{
Schema::dropIfExists('scenario_runs');
Schema::dropIfExists('psychbench_results');
Schema::dropIfExists('prosebench_results');
Schema::dropIfExists('infrabench_results');
Schema::dropIfExists('softbench_results');
if (DB::connection()->getDriverName() === 'pgsql') {
DB::statement('ALTER TABLE benchmark_runs DROP CONSTRAINT IF EXISTS benchmark_runs_visibility_check');
}
// Restore the v1 generic table for rollback completeness.
Schema::create('run_results', function (Blueprint $table) {
$table->id();
$table->foreignId('benchmark_run_id')->constrained()->cascadeOnDelete();
$table->string('suite', 32);
$table->string('section', 64)->nullable();
$table->string('task', 128);
$table->string('level', 16)->nullable();
$table->boolean('ok')->nullable();
$table->double('decode')->nullable();
$table->double('user_tps')->nullable();
$table->double('ttft')->nullable();
$table->double('t_answer')->nullable();
$table->double('wall')->nullable();
$table->unsignedInteger('reasoning')->nullable();
$table->unsignedInteger('visible')->nullable();
$table->unsignedInteger('total')->nullable();
$table->unsignedInteger('tool_calls')->nullable();
$table->unsignedInteger('checks_passed')->nullable();
$table->unsignedInteger('checks_total')->nullable();
$table->string('finish', 32)->nullable();
$table->text('detail')->nullable();
$table->json('payload')->nullable();
$table->timestamps();
});
}
};

29
docker/develop.compose.yaml
View file

@ -1,26 +1,25 @@
# Development-only stack: MariaDB in a container so the base OS stays clean and
# the app talks TCP to a network-attached DB, mirroring the production topology
# (DB on a nearby host). Production deploys via the Dockerfile only — Coolify
# never sees this file.
# Development-only stack: Postgres 17 in a container so the base OS stays clean
# and the app talks TCP to a network-attached DB, mirroring the production
# topology (DB on a nearby host). Production deploys via the Dockerfile only —
# Coolify never sees this file.
services:
mariadb:
image: mariadb:11
container_name: festbench-mariadb
postgres:
image: postgres:17
container_name: festbench-postgres
restart: unless-stopped
environment:
MARIADB_DATABASE: festbench
MARIADB_USER: festbench
MARIADB_PASSWORD: festbench
MARIADB_ROOT_PASSWORD: festbench-root
POSTGRES_DB: festbench
POSTGRES_USER: festbench
POSTGRES_PASSWORD: festbench
ports:
- "127.0.0.1:3306:3306"
- "127.0.0.1:5432:5432"
volumes:
- festbench-mariadb:/var/lib/mysql
- festbench-postgres:/var/lib/postgresql/data
healthcheck:
test: ["CMD", "healthcheck.sh", "--connect", "--innodb_initialized"]
test: ["CMD-SHELL", "pg_isready -U festbench -d festbench"]
interval: 5s
timeout: 3s
retries: 10
volumes:
festbench-mariadb:
festbench-postgres:

72
tests/Feature/BenchmarkRunTest.php
View file

@ -1,72 +0,0 @@
<?php
use App\Models\BenchmarkRun;
use App\Models\RunResult;
use App\Models\User;
test('guests see only public runs', function () {
$publicRun = BenchmarkRun::factory()->create(['visibility' => 'public']);
BenchmarkRun::factory()->create(['visibility' => 'private']);
$visible = BenchmarkRun::visibleTo(null)->get();
expect($visible->pluck('id'))->toContain($publicRun->id)
->and($visible)->toHaveCount(1);
});
test('authenticated users see public runs plus their own private runs', function () {
$me = User::factory()->create();
$other = User::factory()->create();
$publicRun = BenchmarkRun::factory()->create(['visibility' => 'public']);
$myPrivate = BenchmarkRun::factory()->create(['visibility' => 'private', 'user_id' => $me->id]);
BenchmarkRun::factory()->create(['visibility' => 'private', 'user_id' => $other->id]);
$visible = BenchmarkRun::visibleTo($me)->get();
expect($visible->pluck('id'))->toContain($publicRun->id)
->and($visible->pluck('id'))->toContain($myPrivate->id)
->and($visible)->toHaveCount(2);
});
test('submitted metrics are immutable through fillable updates', function () {
$run = BenchmarkRun::factory()->create(['model_id' => 'original/model', 'visibility' => 'public']);
// The only update path any user-facing code may use:
$run->fill(['notes' => 'my notes', 'visibility' => 'private', 'model_id' => 'hacked/model'])
->save();
expect($run->notes)->toBe('my notes')
->and($run->visibility)->toBe('private')
->and($run->fresh()->model_id)->toBe('original/model')
// fill() must have discarded the immutable field, not persisted it
->and($run->getOriginal('model_id'))->toBe('original/model');
});
test('overlay helper reports ownership correctly', function () {
$owner = User::factory()->create();
$run = BenchmarkRun::factory()->create(['user_id' => $owner->id]);
expect($run->isOwnedBy($owner))->toBeTrue()
->and($run->isOwnedBy(User::factory()->create()))->toBeFalse()
->and($run->isOwnedBy(null))->toBeFalse();
});
test('run results belong to a run and are deleted with it', function () {
$run = BenchmarkRun::factory()->create();
$result = RunResult::factory()->create(['benchmark_run_id' => $run->id]);
expect($result->run->id)->toBe($run->id);
$run->delete();
expect(RunResult::where('benchmark_run_id', $run->id)->exists())->toBeFalse();
});
test('run results cast their payload to array', function () {
$result = RunResult::factory()->create(['payload' => ['finish' => 'stop', 'ok' => true]]);
expect($result->fresh()->payload)->toBeArray()
->and($result->fresh()->payload['finish'])->toBe('stop')
->and($result->fresh()->ok)->toBeBool();
});