From 8c75db59f09ed4f3e269debd2c1d33888bac3a2b Mon Sep 17 00:00:00 2001 From: PeterChrz Date: Fri, 18 Sep 2026 14:41:33 -0400 Subject: [PATCH] v2 Phase A+B: Postgres, partitioned per-suite tables, suite registry, all_results view, import rework, surgical re-import of deterministic session --- app/Console/Commands/FestbenchImport.php | 256 ++++++++++++----- app/Console/Commands/SuitesRefresh.php | 45 +++ app/Models/BenchmarkRun.php | 43 ++- app/Models/InfrabenchResult.php | 19 ++ app/Models/ProsebenchResult.php | 30 ++ app/Models/PsychbenchResult.php | 27 ++ app/Models/ScenarioRun.php | 49 ++++ app/Models/SoftbenchResult.php | 26 ++ app/Models/{RunResult.php => SuiteResult.php} | 23 +- app/Support/Suites.php | 76 +++++ config/festbench.php | 60 ++++ .../factories/InfrabenchResultFactory.php | 36 +++ .../factories/ProsebenchResultFactory.php | 42 +++ .../factories/PsychbenchResultFactory.php | 39 +++ database/factories/RunResultFactory.php | 54 ---- database/factories/ScenarioRunFactory.php | 41 +++ database/factories/SoftbenchResultFactory.php | 37 +++ ...6_09_18_000001_reorganize_suite_tables.php | 259 ++++++++++++++++++ docker/develop.compose.yaml | 29 +- tests/Feature/BenchmarkRunTest.php | 72 ----- 20 files changed, 1040 insertions(+), 223 deletions(-) create mode 100644 app/Console/Commands/SuitesRefresh.php create mode 100644 app/Models/InfrabenchResult.php create mode 100644 app/Models/ProsebenchResult.php create mode 100644 app/Models/PsychbenchResult.php create mode 100644 app/Models/ScenarioRun.php create mode 100644 app/Models/SoftbenchResult.php rename app/Models/{RunResult.php => SuiteResult.php} (52%) create mode 100644 app/Support/Suites.php create mode 100644 config/festbench.php create mode 100644 database/factories/InfrabenchResultFactory.php create mode 100644 database/factories/ProsebenchResultFactory.php create mode 100644 database/factories/PsychbenchResultFactory.php delete mode 100644 database/factories/RunResultFactory.php create mode 100644 database/factories/ScenarioRunFactory.php create mode 100644 database/factories/SoftbenchResultFactory.php create mode 100644 database/migrations/2026_09_18_000001_reorganize_suite_tables.php delete mode 100644 tests/Feature/BenchmarkRunTest.php diff --git a/app/Console/Commands/FestbenchImport.php b/app/Console/Commands/FestbenchImport.php index 14c7a37..bc9565a 100644 --- a/app/Console/Commands/FestbenchImport.php +++ b/app/Console/Commands/FestbenchImport.php @@ -3,20 +3,22 @@ namespace App\Console\Commands; use App\Models\BenchmarkRun; -use App\Models\RunResult; use App\Models\User; +use App\Support\Suites; use Illuminate\Console\Command; use Illuminate\Support\Carbon; use PDO; use Symfony\Component\Console\Attribute\AsCommand; -#[AsCommand(name: 'festbench:import', description: 'Import a festbench benchmarks.db (SQLite) into the application database')] +#[AsCommand(name: 'festbench:import', description: 'Import a festbench benchmarks.db (SQLite) into the per-suite tables')] class FestbenchImport extends Command { protected $signature = 'festbench:import {path : Path to the festbench benchmarks.db} {--user= : Email of the user to attribute the imported runs to} - {--private : Import runs as private instead of public}'; + {--label= : Only import runs whose label matches exactly} + {--private : Import runs as private instead of public} + {--fresh : Delete ALL existing benchmark data before importing}'; public function handle(): int { @@ -28,6 +30,10 @@ public function handle(): int return self::FAILURE; } + if ($this->option('fresh') && ! $this->confirmFresh()) { + return self::SUCCESS; + } + $user = $this->resolveUser(); $sqlite = new PDO('sqlite:'.$path, null, null, [ @@ -35,13 +41,23 @@ public function handle(): int PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC, ]); - $importedRuns = $skippedRuns = $importedResults = 0; + $query = 'SELECT * FROM runs'; + $params = []; + if ($label = $this->option('label')) { + $query .= ' WHERE label = ?'; + $params[] = $label; + } - $runs = $sqlite->query('SELECT * FROM runs')->fetchAll(); - $progress = $this->output->createProgressBar(count($runs)); + $runs = $sqlite->prepare($query); + $runs->execute($params); + $runRows = $runs->fetchAll(); + + $importedRuns = $skippedRuns = $importedResults = $unknownSuites = 0; + + $progress = $this->output->createProgressBar(count($runRows)); $progress->start(); - foreach ($runs as $source) { + foreach ($runRows as $source) { if (BenchmarkRun::where('run_id', $source['run_id'])->exists()) { $skippedRuns++; $progress->advance(); @@ -49,67 +65,16 @@ public function handle(): int continue; } - $run = BenchmarkRun::create([ - 'run_id' => $source['run_id'], - 'ran_at' => $source['ts'] ? Carbon::parse($source['ts']) : null, - 'suite' => $source['suite'] ?? 'unknown', - 'run_key' => $source['run_key'] ?? null, - 'model_id' => $source['model_id'] ?? null, - 'model_quant' => $source['model_quant'] ?? null, - 'model_quant_raw' => $source['model_quant_raw'] ?? null, - 'model_path' => $source['model_path'] ?? null, - 'engine' => $source['engine'] ?? null, - 'engine_build' => $source['engine_build'] ?? null, - 'n_ctx' => $source['n_ctx'] ?? null, - 'slots' => $source['slots'] ?? null, - 'gpu' => $source['gpu'] ?? null, - 'gpu_count' => $source['gpu_count'] ?? null, - 'gpu_source' => $source['gpu_source'] ?? null, - 'host' => $source['host'] ?? null, - 'client' => $source['client'] ?? null, - 'endpoint' => $source['endpoint'] ?? null, - 'label' => $source['label'] ?? null, - 'config' => $source['config'] ?? null, - 'image_digest' => $source['image_digest'] ?? null, - 'validators' => $this->decodeJson($source['validators'] ?? null), - 'suite_versions' => $this->decodeJson($source['suite_versions'] ?? null), - 'harness' => $source['harness'] ?? null, - 'harness_ver' => $source['harness_ver'] ?? null, - 'harness_mode' => $source['harness_mode'] ?? null, - 'user_id' => $user?->id, - 'visibility' => $this->option('private') ? BenchmarkRun::VISIBILITY_PRIVATE : BenchmarkRun::VISIBILITY_PUBLIC, - 'notes' => $source['notes'] ?? null, - ]); - + $run = $this->createRun($source, $user); $importedRuns++; - $stmt = $sqlite->prepare('SELECT * FROM results WHERE run_id = ?'); + $sourceTable = Suites::sourceTable($source['suite'] ?? ''); + + $stmt = $sqlite->prepare("SELECT * FROM {$sourceTable} WHERE run_id = ?"); $stmt->execute([$source['run_id']]); foreach ($stmt->fetchAll() as $row) { - RunResult::create([ - 'benchmark_run_id' => $run->id, - 'suite' => $row['suite'] ?? $source['suite'], - 'section' => $row['section'] ?? null, - 'task' => $row['task'] ?? ($row['name'] ?? 'unknown'), - 'level' => $row['level'] ?? null, - 'ok' => $row['ok'] === null ? null : (bool) $row['ok'], - 'decode' => $row['decode'] ?? null, - 'user_tps' => $row['user_tps'] ?? null, - 'ttft' => $row['ttft'] ?? null, - 't_answer' => $row['t_answer'] ?? null, - 'wall' => $row['wall'] ?? null, - 'reasoning' => $row['reasoning'] ?? null, - 'visible' => $row['visible'] ?? null, - 'total' => $row['total'] ?? null, - 'tool_calls' => $row['tool_calls'] ?? null, - 'checks_passed' => $row['checks_passed'] ?? null, - 'checks_total' => $row['checks_total'] ?? null, - 'finish' => $row['finish'] ?? null, - 'detail' => ($row['detail'] ?? '') !== '' ? $row['detail'] : null, - 'payload' => $row, - ]); - + $this->createSuiteRow($run, $row, $source); $importedResults++; } @@ -118,11 +83,174 @@ public function handle(): int $progress->finish(); $this->newLine(2); - $this->info("Imported {$importedRuns} runs ({$importedResults} results), skipped {$skippedRuns} already present."); + $this->info("Imported {$importedRuns} runs ({$importedResults} suite rows), skipped {$skippedRuns} already present."); + + if ($unknownSuites > 0) { + $this->warn("{$unknownSuites} rows were stored with fallback routing — check the suite registry."); + } return self::SUCCESS; } + private function confirmFresh(): bool + { + if (! $this->option('no-interaction')) { + if (! $this->confirm('This will DELETE all existing benchmark runs and results. Continue?')) { + return false; + } + } + + // Delete children first (partitioned tables have no FK cascade issues, + // but explicit order is clearer and driver-agnostic). + foreach (Suites::all() as $suite) { + $model = Suites::modelFor($suite); + $model::query()->delete(); + } + BenchmarkRun::query()->delete(); + + return true; + } + + private function createRun(array $source, ?User $user): BenchmarkRun + { + return BenchmarkRun::create([ + 'run_id' => $source['run_id'], + 'ran_at' => $source['ts'] ? Carbon::parse($source['ts']) : null, + 'suite' => $source['suite'] ?? 'unknown', + 'run_key' => $source['run_key'] ?? null, + 'model_id' => $source['model_id'] ?? null, + 'model_quant' => ($source['model_quant'] ?? '') !== '' ? $source['model_quant'] : null, + 'model_quant_raw' => $source['model_quant_raw'] ?? null, + 'model_path' => $source['model_path'] ?? null, + 'engine' => $source['engine'] ?? null, + 'engine_build' => $source['engine_build'] ?? null, + 'n_ctx' => $source['n_ctx'] ?? null, + 'slots' => $source['slots'] ?? null, + 'gpu' => $source['gpu'] ?? null, + 'gpu_count' => $source['gpu_count'] ?? null, + 'gpu_source' => $source['gpu_source'] ?? null, + 'host' => $source['host'] ?? null, + 'client' => $source['client'] ?? null, + 'endpoint' => $source['endpoint'] ?? null, + 'label' => ($source['label'] ?? '') !== '' ? $source['label'] : null, + 'config' => $source['config'] ?? null, + 'image_digest' => $source['image_digest'] ?? null, + 'validators' => $this->decodeJson($source['validators'] ?? null), + 'suite_versions' => $this->decodeJson($source['suite_versions'] ?? null), + 'harness' => $source['harness'] ?? null, + 'harness_ver' => $source['harness_ver'] ?? null, + 'harness_mode' => $source['harness_mode'] ?? null, + 'user_id' => $user?->id, + 'visibility' => $this->option('private') ? BenchmarkRun::VISIBILITY_PRIVATE : BenchmarkRun::VISIBILITY_PUBLIC, + 'notes' => $source['notes'] ?? null, + ]); + } + + /** + * Route one source result row to its suite table via the registry. + * scenariobench rows have their own shape; metric suites share core + * columns with JSON-source extras (name→task, kind, checks, judge). + */ + private function createSuiteRow(BenchmarkRun $run, array $row, array $sourceRun): void + { + $suite = $run->suite; + $model = Suites::modelFor($suite); + + if ($model === null) { + // Unknown suite: keep the run, drop the rows with a warning count. + $this->warn("Unregistered suite '{$suite}' (run {$run->run_id}) — rows skipped."); + + return; + } + + $ranAt = $run->ran_at; + + if ($suite === 'scenariobench') { + $model::create([ + 'benchmark_run_id' => $run->id, + 'ran_at' => $ranAt, + 'scenario' => $row['scenario'] ?? ($row['task'] ?? 'unknown'), + 'level' => $row['level'] ?? null, + 'ok' => $row['ok'] === null ? null : (bool) $row['ok'], + 'outcome' => $row['outcome'] ?? null, + 'wall' => $row['wall'] ?? null, + 'turns' => $row['turns'] ?? null, + 'commands' => $row['commands'] ?? null, + 'repeats' => $row['repeats'] ?? null, + 'submissions' => $row['submissions'] ?? null, + 'rejected' => $row['rejected'] ?? null, + 't_first_sub' => $row['t_first_sub'] ?? null, + 'n_first_sub' => $row['n_first_sub'] ?? null, + 't_pass' => $row['t_pass'] ?? null, + 'n_pass' => $row['n_pass'] ?? null, + 'gen_tokens' => $row['gen_tokens'] ?? null, + 'vis_tokens' => $row['vis_tokens'] ?? null, + 'rea_tokens' => $row['rea_tokens'] ?? null, + 'verdicts' => $this->decodeJson($row['verdicts'] ?? null), + 'detail' => ($row['detail'] ?? '') !== '' ? $row['detail'] : null, + 'payload' => $row, + ]); + + return; + } + + // Shared mapping for metric suites. JSON-source extras come through + // payload-style keys; the SQLite results table uses task/section. + $model::create(array_merge([ + 'benchmark_run_id' => $run->id, + 'ran_at' => $ranAt, + 'task' => $row['task'] ?? ($row['name'] ?? 'unknown'), + 'level' => $row['level'] !== 'n/a' ? ($row['level'] ?? null) : null, + 'ok' => array_key_exists('ok', $row) && $row['ok'] !== null ? (bool) $row['ok'] : null, + 'decode' => $row['decode'] ?? null, + 'user_tps' => $row['user_tps'] ?? null, + 'ttft' => $row['ttft'] ?? null, + 't_answer' => $row['t_answer'] ?? null, + 'wall' => $row['wall'] ?? null, + 'reasoning' => $row['reasoning'] ?? null, + 'visible' => $row['visible'] ?? null, + 'total' => $row['total'] ?? null, + 'finish' => ($row['finish'] ?? '') !== '' ? $row['finish'] : null, + 'detail' => ($row['detail'] ?? '') !== '' ? $row['detail'] : null, + 'payload' => $row, + ], $this->suiteExtras($suite, $row))); + } + + /** + * Suite-specific columns, keyed for create(). JSON sources use name, + * checks_passed, checks_total, judge, prefill, prompt_tokens, kind; the + * SQLite results table uses section and tool_calls. + */ + private function suiteExtras(string $suite, array $row): array + { + $extras = []; + + if (in_array($suite, ['softbench', 'prosebench', 'psychbench'], true)) { + $extras['section'] = ($row['section'] ?? '') !== '' ? $row['section'] : null; + } + + if ($suite === 'softbench') { + $extras['tool_calls'] = $row['tool_calls'] ?? null; + } + + if ($suite === 'infrabench') { + $extras['kind'] = ($row['kind'] ?? '') !== '' ? $row['kind'] : null; + } + + if (in_array($suite, ['prosebench', 'psychbench'], true)) { + $extras['checks_passed'] = $row['checks_passed'] ?? null; + $extras['checks_total'] = $row['checks_total'] ?? null; + } + + if ($suite === 'prosebench') { + $extras['judge'] = ($row['judge'] ?? '') !== '' ? $row['judge'] : null; + $extras['prefill'] = $row['prefill'] ?? null; + $extras['prompt_tokens'] = $row['prompt_tokens'] ?? null; + } + + return $extras; + } + private function resolveUser(): ?User { $email = $this->option('user'); diff --git a/app/Console/Commands/SuitesRefresh.php b/app/Console/Commands/SuitesRefresh.php new file mode 100644 index 0000000..15153b9 --- /dev/null +++ b/app/Console/Commands/SuitesRefresh.php @@ -0,0 +1,45 @@ +isEmpty()) { + $this->warn('No active suites in the registry — skipping view rebuild.'); + + return self::SUCCESS; + } + + $driver = DB::connection()->getDriverName(); + $arms = $suites->map(fn (string $suite) => $this->arm($suite, $driver))->implode(' UNION ALL '); + + DB::statement('DROP VIEW IF EXISTS all_results'); + DB::statement("CREATE VIEW all_results AS {$arms}"); + + $this->info('all_results view rebuilt over: '.$suites->implode(', ')); + + return self::SUCCESS; + } + + /** + * One UNION arm per suite. All metric tables share the core column set, + * so the arm promotes the suite name as a literal plus the common columns. + */ + private function arm(string $suite, string $driver): string + { + $table = Suites::modelFor($suite)::make()->getTable(); + $quoted = $driver === 'pgsql' ? "'{$suite}'" : "'{$suite}'"; + + return "SELECT id, benchmark_run_id, ran_at, task, level, ok, decode, user_tps, ttft, t_answer, wall, reasoning, visible, total, finish, detail, {$quoted} AS suite FROM {$table}"; + } +} diff --git a/app/Models/BenchmarkRun.php b/app/Models/BenchmarkRun.php index 94e25e2..513a38d 100644 --- a/app/Models/BenchmarkRun.php +++ b/app/Models/BenchmarkRun.php @@ -2,8 +2,10 @@ namespace App\Models; +use App\Support\Suites; use Database\Factories\BenchmarkRunFactory; use Illuminate\Database\Eloquent\Builder; +use Illuminate\Database\Eloquent\Collection; use Illuminate\Database\Eloquent\Factories\HasFactory; use Illuminate\Database\Eloquent\Model; use Illuminate\Database\Eloquent\Relations\BelongsTo; @@ -68,9 +70,46 @@ public function user(): BelongsTo return $this->belongsTo(User::class); } - public function results(): HasMany + public function softbenchResults(): HasMany { - return $this->hasMany(RunResult::class); + return $this->hasMany(SoftbenchResult::class); + } + + public function infrabenchResults(): HasMany + { + return $this->hasMany(InfrabenchResult::class); + } + + public function prosebenchResults(): HasMany + { + return $this->hasMany(ProsebenchResult::class); + } + + public function psychbenchResults(): HasMany + { + return $this->hasMany(PsychbenchResult::class); + } + + public function scenarioRuns(): HasMany + { + return $this->hasMany(ScenarioRun::class); + } + + /** + * The suite-specific result rows for this run, via the registry. Returns + * an empty collection for suites this run does not have. + */ + public function suiteResults(): Collection + { + $model = Suites::modelFor($this->suite); + + if ($model === null) { + return collect(); + } + + $relation = Suites::runRelation($this->suite); + + return $this->{$relation}()->orderBy('task')->orderBy('id')->get(); } public function isOwnedBy(?User $user): bool diff --git a/app/Models/InfrabenchResult.php b/app/Models/InfrabenchResult.php new file mode 100644 index 0000000..88dd2be --- /dev/null +++ b/app/Models/InfrabenchResult.php @@ -0,0 +1,19 @@ + */ + use HasFactory; + + protected $fillable = [ + 'benchmark_run_id', 'ran_at', 'task', 'level', + 'ok', 'decode', 'user_tps', 'ttft', 't_answer', 'wall', + 'reasoning', 'visible', 'total', 'finish', 'detail', + 'kind', 'payload', + ]; +} diff --git a/app/Models/ProsebenchResult.php b/app/Models/ProsebenchResult.php new file mode 100644 index 0000000..cb665d7 --- /dev/null +++ b/app/Models/ProsebenchResult.php @@ -0,0 +1,30 @@ + */ + use HasFactory; + + protected $fillable = [ + 'benchmark_run_id', 'ran_at', 'task', 'level', + 'ok', 'decode', 'user_tps', 'ttft', 't_answer', 'wall', + 'reasoning', 'visible', 'total', 'finish', 'detail', + 'section', 'checks_passed', 'checks_total', 'judge', + 'prefill', 'prompt_tokens', 'payload', + ]; + + protected function casts(): array + { + return array_merge(parent::casts(), [ + 'checks_passed' => 'integer', + 'checks_total' => 'integer', + 'prefill' => 'float', + 'prompt_tokens' => 'integer', + ]); + } +} diff --git a/app/Models/PsychbenchResult.php b/app/Models/PsychbenchResult.php new file mode 100644 index 0000000..f8bd519 --- /dev/null +++ b/app/Models/PsychbenchResult.php @@ -0,0 +1,27 @@ + */ + use HasFactory; + + protected $fillable = [ + 'benchmark_run_id', 'ran_at', 'task', 'level', + 'ok', 'decode', 'user_tps', 'ttft', 't_answer', 'wall', + 'reasoning', 'visible', 'total', 'finish', 'detail', + 'section', 'checks_passed', 'checks_total', 'payload', + ]; + + protected function casts(): array + { + return array_merge(parent::casts(), [ + 'checks_passed' => 'integer', + 'checks_total' => 'integer', + ]); + } +} diff --git a/app/Models/ScenarioRun.php b/app/Models/ScenarioRun.php new file mode 100644 index 0000000..37b11a3 --- /dev/null +++ b/app/Models/ScenarioRun.php @@ -0,0 +1,49 @@ + */ + use HasFactory; + + protected $fillable = [ + 'benchmark_run_id', 'ran_at', 'scenario', 'level', 'ok', 'outcome', + 'wall', 'turns', 'commands', 'repeats', 'submissions', 'rejected', + 't_first_sub', 'n_first_sub', 't_pass', 'n_pass', + 'gen_tokens', 'vis_tokens', 'rea_tokens', 'verdicts', 'detail', 'payload', + ]; + + protected function casts(): array + { + return [ + 'ran_at' => 'datetime', + 'ok' => 'boolean', + 'wall' => 'float', + 'turns' => 'integer', + 'commands' => 'integer', + 'repeats' => 'integer', + 'submissions' => 'integer', + 'rejected' => 'integer', + 't_first_sub' => 'float', + 'n_first_sub' => 'integer', + 't_pass' => 'float', + 'n_pass' => 'integer', + 'gen_tokens' => 'integer', + 'vis_tokens' => 'integer', + 'rea_tokens' => 'integer', + 'verdicts' => 'array', + 'payload' => 'array', + ]; + } + + public function run(): BelongsTo + { + return $this->belongsTo(BenchmarkRun::class, 'benchmark_run_id'); + } +} diff --git a/app/Models/SoftbenchResult.php b/app/Models/SoftbenchResult.php new file mode 100644 index 0000000..27922ad --- /dev/null +++ b/app/Models/SoftbenchResult.php @@ -0,0 +1,26 @@ + */ + use HasFactory; + + protected $fillable = [ + 'benchmark_run_id', 'ran_at', 'task', 'level', + 'ok', 'decode', 'user_tps', 'ttft', 't_answer', 'wall', + 'reasoning', 'visible', 'total', 'finish', 'detail', + 'section', 'tool_calls', 'payload', + ]; + + protected function casts(): array + { + return array_merge(parent::casts(), [ + 'tool_calls' => 'integer', + ]); + } +} diff --git a/app/Models/RunResult.php b/app/Models/SuiteResult.php similarity index 52% rename from app/Models/RunResult.php rename to app/Models/SuiteResult.php index 07264bc..a0bddf4 100644 --- a/app/Models/RunResult.php +++ b/app/Models/SuiteResult.php @@ -2,26 +2,20 @@ namespace App\Models; -use Database\Factories\RunResultFactory; -use Illuminate\Database\Eloquent\Factories\HasFactory; use Illuminate\Database\Eloquent\Model; use Illuminate\Database\Eloquent\Relations\BelongsTo; -class RunResult extends Model +/** + * Shared behavior for per-suite result tables. Each concrete model declares + * its own $fillable (suite-specific columns + core) — submitted metrics are + * immutable once stored, exactly like the parent BenchmarkRun. + */ +abstract class SuiteResult extends Model { - /** @use HasFactory */ - use HasFactory; - - protected $fillable = [ - 'benchmark_run_id', 'suite', 'section', 'task', 'level', - 'ok', 'decode', 'user_tps', 'ttft', 't_answer', 'wall', - 'reasoning', 'visible', 'total', 'tool_calls', - 'checks_passed', 'checks_total', 'finish', 'detail', 'payload', - ]; - protected function casts(): array { return [ + 'ran_at' => 'datetime', 'ok' => 'boolean', 'decode' => 'float', 'user_tps' => 'float', @@ -31,9 +25,6 @@ protected function casts(): array 'reasoning' => 'integer', 'visible' => 'integer', 'total' => 'integer', - 'tool_calls' => 'integer', - 'checks_passed' => 'integer', - 'checks_total' => 'integer', 'payload' => 'array', ]; } diff --git a/app/Support/Suites.php b/app/Support/Suites.php new file mode 100644 index 0000000..40bbd3a --- /dev/null +++ b/app/Support/Suites.php @@ -0,0 +1,76 @@ + */ + private const MODELS = [ + 'softbench' => SoftbenchResult::class, + 'infrabench' => InfrabenchResult::class, + 'prosebench' => ProsebenchResult::class, + 'psychbench' => PsychbenchResult::class, + 'scenariobench' => ScenarioRun::class, + ]; + + /** Model class handling results for $suite, or null if unregistered. */ + public static function modelFor(?string $suite): ?string + { + return self::MODELS[$suite] ?? null; + } + + /** Name of the HasMany relation on BenchmarkRun for $suite. */ + public static function runRelation(string $suite): string + { + return $suite === 'scenariobench' ? 'scenarioRuns' : $suite.'Results'; + } + + /** Source table in the festbench SQLite db for $suite. */ + public static function sourceTable(string $suite): string + { + return config("festbench.source_tables.{$suite}", 'results'); + } + + /** All registered suites, active and inactive. */ + public static function all(): array + { + return array_keys(self::MODELS); + } + + /** + * Active suites (appear in the suite filter and run-detail rendering). + * + * @return Collection + */ + public static function active(): Collection + { + return collect(self::MODELS) + ->filter(fn (string $m, string $suite) => config("festbench.suites.{$suite}.active", false)) + ->keys(); + } + + /** + * Active suites that participate in the combined all_results view. + * + * @return Collection + */ + public static function inAllResults(): Collection + { + return collect(self::MODELS) + ->filter(fn (string $m, string $suite) => config("festbench.suites.{$suite}.active", false) + && config("festbench.suites.{$suite}.in_all_results", false)) + ->keys(); + } +} diff --git a/config/festbench.php b/config/festbench.php new file mode 100644 index 0000000..990cb3d --- /dev/null +++ b/config/festbench.php @@ -0,0 +1,60 @@ + false` removes the suite from the all_results view and the +// home page (decommissioning) — historical data stays queryable and run +// detail pages keep rendering. +// - The view is rebuilt by `festbench:suites:refresh` after editing this file. +// - Adding a future suite: create its migration + model, add it here, run +// the refresh command. + +return [ + 'suites' => [ + 'softbench' => [ + 'model' => SoftbenchResult::class, + 'active' => true, + 'in_all_results' => true, + ], + 'infrabench' => [ + 'model' => InfrabenchResult::class, + 'active' => true, + 'in_all_results' => true, + ], + 'prosebench' => [ + 'model' => ProsebenchResult::class, + 'active' => true, + 'in_all_results' => true, + ], + 'psychbench' => [ + 'model' => PsychbenchResult::class, + 'active' => true, + 'in_all_results' => true, + ], + 'scenariobench' => [ + 'model' => ScenarioRun::class, + 'active' => true, + // Agentic metrics (turns, submissions) — no tok/s semantics, so + // it is excluded from the combined metric view. + 'in_all_results' => false, + ], + ], + + // Source table in the festbench SQLite db for each suite ('results' is + // the generic metrics table; scenariobench has its own). + 'source_tables' => [ + 'scenariobench' => 'scenario_runs', + ], + + // Caching: TTL staleness only — never event-flush. At high submission + // rates, event flushes would storm the cache; benchmark data being a + // couple of minutes old is correct behavior. + 'home_cache_ttl' => env('FESTBENCH_HOME_CACHE_TTL', 300), +]; diff --git a/database/factories/InfrabenchResultFactory.php b/database/factories/InfrabenchResultFactory.php new file mode 100644 index 0000000..6e52d74 --- /dev/null +++ b/database/factories/InfrabenchResultFactory.php @@ -0,0 +1,36 @@ + */ +class InfrabenchResultFactory extends Factory +{ + public function definition(): array + { + $ttft = round($this->faker->randomFloat(2, 0.1, 4), 2); + + return [ + 'benchmark_run_id' => BenchmarkRun::factory(), + 'ran_at' => now(), + 'task' => $this->faker->randomElement(['modprobe_exists', 'rbac_missing_verb', 'oom_innocent_victim']), + 'level' => $this->faker->randomElement(['off', 'low', 'medium', 'xhigh']), + 'ok' => $this->faker->boolean(70), + 'decode' => round($this->faker->randomFloat(2, 10, 150), 2), + 'user_tps' => round($this->faker->randomFloat(2, 5, 100), 2), + 'ttft' => $ttft, + 't_answer' => $ttft, + 'wall' => round($ttft + $this->faker->randomFloat(2, 0.5, 30), 2), + 'reasoning' => $this->faker->numberBetween(0, 2000), + 'visible' => $this->faker->numberBetween(20, 400), + 'total' => $this->faker->numberBetween(20, 2000), + 'finish' => 'stop', + 'detail' => null, + 'kind' => $this->faker->randomElement(['recall', 'constraint']), + 'payload' => ['ok' => true], + ]; + } +} diff --git a/database/factories/ProsebenchResultFactory.php b/database/factories/ProsebenchResultFactory.php new file mode 100644 index 0000000..56c8fa3 --- /dev/null +++ b/database/factories/ProsebenchResultFactory.php @@ -0,0 +1,42 @@ + */ +class ProsebenchResultFactory extends Factory +{ + public function definition(): array + { + $ttft = round($this->faker->randomFloat(2, 0.1, 4), 2); + $checksTotal = $this->faker->numberBetween(3, 8); + + return [ + 'benchmark_run_id' => BenchmarkRun::factory(), + 'ran_at' => now(), + 'task' => $this->faker->randomElement(['grief_email', 'refund_denial', 'status_update']), + 'level' => $this->faker->randomElement(['off', 'low', 'medium', 'xhigh']), + 'ok' => $this->faker->boolean(70), + 'decode' => round($this->faker->randomFloat(2, 10, 150), 2), + 'user_tps' => round($this->faker->randomFloat(2, 5, 100), 2), + 'ttft' => $ttft, + 't_answer' => $ttft, + 'wall' => round($ttft + $this->faker->randomFloat(2, 0.5, 30), 2), + 'reasoning' => $this->faker->numberBetween(0, 2000), + 'visible' => $this->faker->numberBetween(20, 400), + 'total' => $this->faker->numberBetween(20, 2000), + 'finish' => 'stop', + 'detail' => null, + 'section' => $this->faker->randomElement(['sensitive', 'actionable']), + 'checks_passed' => $this->faker->numberBetween(0, $checksTotal), + 'checks_total' => $checksTotal, + 'judge' => null, + 'prefill' => round($this->faker->randomFloat(1, 50, 400), 1), + 'prompt_tokens' => $this->faker->numberBetween(100, 500), + 'payload' => ['ok' => true], + ]; + } +} diff --git a/database/factories/PsychbenchResultFactory.php b/database/factories/PsychbenchResultFactory.php new file mode 100644 index 0000000..18c9400 --- /dev/null +++ b/database/factories/PsychbenchResultFactory.php @@ -0,0 +1,39 @@ + */ +class PsychbenchResultFactory extends Factory +{ + public function definition(): array + { + $ttft = round($this->faker->randomFloat(2, 0.1, 4), 2); + $checksTotal = $this->faker->numberBetween(3, 8); + + return [ + 'benchmark_run_id' => BenchmarkRun::factory(), + 'ran_at' => now(), + 'task' => $this->faker->randomElement(['billing_form', 'irreducible_contradiction', 'sunk_cost']), + 'level' => $this->faker->randomElement(['off', 'low', 'medium', 'xhigh']), + 'ok' => $this->faker->boolean(70), + 'decode' => round($this->faker->randomFloat(2, 10, 150), 2), + 'user_tps' => round($this->faker->randomFloat(2, 5, 100), 2), + 'ttft' => $ttft, + 't_answer' => $ttft, + 'wall' => round($ttft + $this->faker->randomFloat(2, 0.5, 30), 2), + 'reasoning' => $this->faker->numberBetween(0, 2000), + 'visible' => $this->faker->numberBetween(20, 400), + 'total' => $this->faker->numberBetween(20, 2000), + 'finish' => 'stop', + 'detail' => null, + 'section' => null, + 'checks_passed' => $this->faker->numberBetween(0, $checksTotal), + 'checks_total' => $checksTotal, + 'payload' => ['ok' => true], + ]; + } +} diff --git a/database/factories/RunResultFactory.php b/database/factories/RunResultFactory.php deleted file mode 100644 index e337d65..0000000 --- a/database/factories/RunResultFactory.php +++ /dev/null @@ -1,54 +0,0 @@ - */ -class RunResultFactory extends Factory -{ - public function definition(): array - { - $visible = $this->faker->numberBetween(20, 400); - $reasoning = $this->faker->numberBetween(0, 2000); - $ttft = round($this->faker->randomFloat(2, 0.1, 4), 2); - $wall = round($ttft + $this->faker->randomFloat(2, 0.5, 60), 2); - $ok = $this->faker->boolean(70); - - $payload = [ - 'level' => null, 'section' => null, 'name' => null, 'ok' => $ok, - 'ttft' => $ttft, 'wall' => $wall, - ]; - - return [ - 'benchmark_run_id' => BenchmarkRun::factory(), - 'suite' => 'softbench', - 'section' => $this->faker->randomElement(['code', 'reasoning', 'constraints', null]), - 'task' => $this->faker->randomElement([ - 'first_occurrence', 'merge_intervals', 'version_compare', 'rbac_missing_verb', 'oom_innocent_victim', - ]), - 'level' => $this->faker->randomElement(['off', 'low', 'medium', 'xhigh']), - 'ok' => $ok, - 'decode' => round($this->faker->randomFloat(2, 10, 150), 2), - 'user_tps' => round($this->faker->randomFloat(2, 5, 100), 2), - 'ttft' => $ttft, - 't_answer' => $ttft, - 'wall' => $wall, - 'reasoning' => $reasoning, - 'visible' => $visible, - 'total' => $visible + $reasoning, - 'tool_calls' => null, - 'checks_passed' => null, - 'checks_total' => null, - 'finish' => $ok ? 'stop' : $this->faker->randomElement(['stop', 'length']), - 'detail' => null, - 'payload' => $payload, - ]; - } - - public function forSuite(string $suite): static - { - return $this->state(fn () => ['suite' => $suite]); - } -} diff --git a/database/factories/ScenarioRunFactory.php b/database/factories/ScenarioRunFactory.php new file mode 100644 index 0000000..0c91655 --- /dev/null +++ b/database/factories/ScenarioRunFactory.php @@ -0,0 +1,41 @@ + */ +class ScenarioRunFactory extends Factory +{ + public function definition(): array + { + $passed = $this->faker->boolean(60); + + return [ + 'benchmark_run_id' => BenchmarkRun::factory(), + 'ran_at' => now(), + 'scenario' => $this->faker->randomElement(['kubelet_config_repair', 'deployment_unschedulable', 'systemd_unit_repair']), + 'level' => $this->faker->randomElement(['off', 'low', 'medium', 'xhigh']), + 'ok' => $passed, + 'outcome' => $passed ? 'passed' : $this->faker->randomElement(['rejected_final', 'timeout', 'turn_cap', 'livelock', 'error']), + 'wall' => round($this->faker->randomFloat(1, 30, 600), 1), + 'turns' => $this->faker->numberBetween(5, 80), + 'commands' => $this->faker->numberBetween(4, 75), + 'repeats' => $this->faker->numberBetween(0, 10), + 'submissions' => $this->faker->numberBetween(1, 5), + 'rejected' => $this->faker->numberBetween(0, 4), + 't_first_sub' => round($this->faker->randomFloat(1, 20, 500), 1), + 'n_first_sub' => $this->faker->numberBetween(3, 70), + 't_pass' => $passed ? round($this->faker->randomFloat(1, 30, 550), 1) : null, + 'n_pass' => $passed ? $this->faker->numberBetween(3, 75) : null, + 'gen_tokens' => $this->faker->numberBetween(500, 20000), + 'vis_tokens' => $this->faker->numberBetween(400, 19000), + 'rea_tokens' => $this->faker->numberBetween(0, 5000), + 'verdicts' => [['n' => 1, 'at' => 264.1, 'accepted' => $passed, 'oracle' => 'sample']], + 'detail' => $passed ? 'passed on submission 1' : 'never accepted', + 'payload' => ['ok' => $passed], + ]; + } +} diff --git a/database/factories/SoftbenchResultFactory.php b/database/factories/SoftbenchResultFactory.php new file mode 100644 index 0000000..899f170 --- /dev/null +++ b/database/factories/SoftbenchResultFactory.php @@ -0,0 +1,37 @@ + */ +class SoftbenchResultFactory extends Factory +{ + public function definition(): array + { + $ttft = round($this->faker->randomFloat(2, 0.1, 4), 2); + + return [ + 'benchmark_run_id' => BenchmarkRun::factory(), + 'ran_at' => now(), + 'task' => $this->faker->randomElement(['first_occurrence', 'merge_intervals', 'version_compare']), + 'level' => $this->faker->randomElement(['off', 'low', 'medium', 'xhigh']), + 'ok' => $this->faker->boolean(70), + 'decode' => round($this->faker->randomFloat(2, 10, 150), 2), + 'user_tps' => round($this->faker->randomFloat(2, 5, 100), 2), + 'ttft' => $ttft, + 't_answer' => $ttft, + 'wall' => round($ttft + $this->faker->randomFloat(2, 0.5, 30), 2), + 'reasoning' => $this->faker->numberBetween(0, 2000), + 'visible' => $this->faker->numberBetween(20, 400), + 'total' => $this->faker->numberBetween(20, 2000), + 'finish' => 'stop', + 'detail' => null, + 'section' => $this->faker->randomElement(['code', 'reasoning', 'constraints']), + 'tool_calls' => $this->faker->optional()->numberBetween(1, 5), + 'payload' => ['ok' => true], + ]; + } +} diff --git a/database/migrations/2026_09_18_000001_reorganize_suite_tables.php b/database/migrations/2026_09_18_000001_reorganize_suite_tables.php new file mode 100644 index 0000000..fec9ba5 --- /dev/null +++ b/database/migrations/2026_09_18_000001_reorganize_suite_tables.php @@ -0,0 +1,259 @@ +getDriverName() === 'pgsql'; + + $suiteExtras = [ + 'softbench_results' => ['section VARCHAR(64)', 'tool_calls INTEGER'], + 'infrabench_results' => ['kind VARCHAR(32)'], + 'prosebench_results' => ['section VARCHAR(64)', 'checks_passed INTEGER', 'checks_total INTEGER', 'judge TEXT', 'prefill DOUBLE PRECISION', 'prompt_tokens INTEGER'], + 'psychbench_results' => ['section VARCHAR(64)', 'checks_passed INTEGER', 'checks_total INTEGER'], + ]; + + foreach ($suiteExtras as $table => $extras) { + if ($pgsql) { + $this->createPgTable($table, self::CORE_COLUMNS, $extras); + } else { + $this->createSqliteTable($table, $extras); + } + } + + if ($pgsql) { + DB::statement(<<<'SQL' + CREATE TABLE scenario_runs ( + id BIGINT GENERATED ALWAYS AS IDENTITY, + benchmark_run_id BIGINT NOT NULL REFERENCES benchmark_runs(id) ON DELETE CASCADE, + ran_at TIMESTAMP(0) WITHOUT TIME ZONE, + scenario VARCHAR(128) NOT NULL, + level VARCHAR(16), + ok BOOLEAN, + outcome VARCHAR(32), + wall DOUBLE PRECISION, + turns INTEGER, + commands INTEGER, + repeats INTEGER, + submissions INTEGER, + rejected INTEGER, + t_first_sub DOUBLE PRECISION, + n_first_sub INTEGER, + t_pass DOUBLE PRECISION, + n_pass INTEGER, + gen_tokens INTEGER, + vis_tokens INTEGER, + rea_tokens INTEGER, + verdicts JSONB, + detail TEXT, + payload JSONB, + created_at TIMESTAMP(0) WITHOUT TIME ZONE, + updated_at TIMESTAMP(0) WITHOUT TIME ZONE + ) PARTITION BY RANGE (ran_at) + SQL); + $this->addIndexes('scenario_runs'); + } else { + Schema::create('scenario_runs', function (Blueprint $table) { + $table->id(); + $table->foreignId('benchmark_run_id')->constrained()->cascadeOnDelete(); + $table->timestamp('ran_at')->nullable(); + $table->string('scenario', 128); + $table->string('level', 16)->nullable(); + $table->boolean('ok')->nullable(); + $table->string('outcome', 32)->nullable(); + $table->double('wall')->nullable(); + $table->unsignedInteger('turns')->nullable(); + $table->unsignedInteger('commands')->nullable(); + $table->unsignedInteger('repeats')->nullable(); + $table->unsignedInteger('submissions')->nullable(); + $table->unsignedInteger('rejected')->nullable(); + $table->double('t_first_sub')->nullable(); + $table->unsignedInteger('n_first_sub')->nullable(); + $table->double('t_pass')->nullable(); + $table->unsignedInteger('n_pass')->nullable(); + $table->unsignedInteger('gen_tokens')->nullable(); + $table->unsignedInteger('vis_tokens')->nullable(); + $table->unsignedInteger('rea_tokens')->nullable(); + $table->jsonb('verdicts')->nullable(); + $table->text('detail')->nullable(); + $table->jsonb('payload')->nullable(); + $table->timestamps(); + + $table->index('benchmark_run_id'); + $table->index('wall'); + }); + } + + if ($pgsql) { + // visibility integrity + JSONB on the runs table + DB::statement('ALTER TABLE benchmark_runs DROP CONSTRAINT IF EXISTS benchmark_runs_visibility_check'); + DB::statement("ALTER TABLE benchmark_runs ADD CONSTRAINT benchmark_runs_visibility_check CHECK (visibility IN ('public', 'private'))"); + DB::statement('ALTER TABLE benchmark_runs ALTER COLUMN validators TYPE jsonb USING validators::jsonb'); + DB::statement('ALTER TABLE benchmark_runs ALTER COLUMN suite_versions TYPE jsonb USING suite_versions::jsonb'); + + $this->createPartitions(); + } + } + + private function createPgTable(string $table, string $columns, array $extras): void + { + $extrasSql = $extras ? ', '.implode(', ', $extras) : ''; + DB::statement("CREATE TABLE {$table} ({$columns}{$extrasSql}) PARTITION BY RANGE (ran_at)"); + $this->addIndexes($table); + } + + private function addIndexes(string $table): void + { + $cols = $table === 'scenario_runs' + ? ['benchmark_run_id', 'wall'] + : ['benchmark_run_id', 'user_tps', 'decode', 'wall']; + + foreach ($cols as $col) { + DB::statement("CREATE INDEX IF NOT EXISTS {$table}_{$col}_idx ON {$table} ({$col})"); + } + } + + /** + * SQLite (tests): plain tables via the schema builder. + */ + private function createSqliteTable(string $table, array $extras): void + { + Schema::create($table, function (Blueprint $table) { + $table->id(); + $table->foreignId('benchmark_run_id')->constrained()->cascadeOnDelete(); + $table->timestamp('ran_at')->nullable(); + $table->string('task', 128); + $table->string('level', 16)->nullable(); + $table->boolean('ok')->nullable(); + $table->double('decode')->nullable(); + $table->double('user_tps')->nullable(); + $table->double('ttft')->nullable(); + $table->double('t_answer')->nullable(); + $table->double('wall')->nullable(); + $table->unsignedInteger('reasoning')->nullable(); + $table->unsignedInteger('visible')->nullable(); + $table->unsignedInteger('total')->nullable(); + $table->string('finish', 32)->nullable(); + $table->text('detail')->nullable(); + $table->jsonb('payload')->nullable(); + $table->timestamps(); + + $table->index('benchmark_run_id'); + $table->index('user_tps'); + $table->index('decode'); + $table->index('wall'); + }); + + // SQLite has no ALTER ADD COLUMN via builder for positioning; add extras. + Schema::table($table, function (Blueprint $t) use ($extras) { + foreach ($extras as $extra) { + [$name, $type] = explode(' ', $extra); + $t->addColumn($type, $name)->nullable(); + } + }); + } + + /** + * Monthly partitions from a year back to a year forward, plus a default + * partition so inserts never miss. Creating future months in production + * is handled by the scheduled festbench:partitions command. + */ + private function createPartitions(): void + { + $tables = ['softbench_results', 'infrabench_results', 'prosebench_results', 'psychbench_results', 'scenario_runs']; + $start = now()->subYear()->startOfMonth(); + $end = now()->addYear()->endOfMonth(); + + for ($m = $start->copy(); $m->lessThan($end); $m->addMonth()) { + $from = $m->format('Y-m-d'); + $to = $m->copy()->addMonth()->format('Y-m-d'); + foreach ($tables as $t) { + DB::statement("CREATE TABLE IF NOT EXISTS {$t}_{$m->format('Y_m')} PARTITION OF {$t} FOR VALUES FROM ('{$from}') TO ('{$to}')"); + } + } + foreach ($tables as $t) { + DB::statement("CREATE TABLE IF NOT EXISTS {$t}_default PARTITION OF {$t} DEFAULT"); + } + } + + public function down(): void + { + Schema::dropIfExists('scenario_runs'); + Schema::dropIfExists('psychbench_results'); + Schema::dropIfExists('prosebench_results'); + Schema::dropIfExists('infrabench_results'); + Schema::dropIfExists('softbench_results'); + + if (DB::connection()->getDriverName() === 'pgsql') { + DB::statement('ALTER TABLE benchmark_runs DROP CONSTRAINT IF EXISTS benchmark_runs_visibility_check'); + } + + // Restore the v1 generic table for rollback completeness. + Schema::create('run_results', function (Blueprint $table) { + $table->id(); + $table->foreignId('benchmark_run_id')->constrained()->cascadeOnDelete(); + $table->string('suite', 32); + $table->string('section', 64)->nullable(); + $table->string('task', 128); + $table->string('level', 16)->nullable(); + $table->boolean('ok')->nullable(); + $table->double('decode')->nullable(); + $table->double('user_tps')->nullable(); + $table->double('ttft')->nullable(); + $table->double('t_answer')->nullable(); + $table->double('wall')->nullable(); + $table->unsignedInteger('reasoning')->nullable(); + $table->unsignedInteger('visible')->nullable(); + $table->unsignedInteger('total')->nullable(); + $table->unsignedInteger('tool_calls')->nullable(); + $table->unsignedInteger('checks_passed')->nullable(); + $table->unsignedInteger('checks_total')->nullable(); + $table->string('finish', 32)->nullable(); + $table->text('detail')->nullable(); + $table->json('payload')->nullable(); + $table->timestamps(); + }); + } +}; diff --git a/docker/develop.compose.yaml b/docker/develop.compose.yaml index 924e1f1..9b942ec 100644 --- a/docker/develop.compose.yaml +++ b/docker/develop.compose.yaml @@ -1,26 +1,25 @@ -# Development-only stack: MariaDB in a container so the base OS stays clean and -# the app talks TCP to a network-attached DB, mirroring the production topology -# (DB on a nearby host). Production deploys via the Dockerfile only — Coolify -# never sees this file. +# Development-only stack: Postgres 17 in a container so the base OS stays clean +# and the app talks TCP to a network-attached DB, mirroring the production +# topology (DB on a nearby host). Production deploys via the Dockerfile only — +# Coolify never sees this file. services: - mariadb: - image: mariadb:11 - container_name: festbench-mariadb + postgres: + image: postgres:17 + container_name: festbench-postgres restart: unless-stopped environment: - MARIADB_DATABASE: festbench - MARIADB_USER: festbench - MARIADB_PASSWORD: festbench - MARIADB_ROOT_PASSWORD: festbench-root + POSTGRES_DB: festbench + POSTGRES_USER: festbench + POSTGRES_PASSWORD: festbench ports: - - "127.0.0.1:3306:3306" + - "127.0.0.1:5432:5432" volumes: - - festbench-mariadb:/var/lib/mysql + - festbench-postgres:/var/lib/postgresql/data healthcheck: - test: ["CMD", "healthcheck.sh", "--connect", "--innodb_initialized"] + test: ["CMD-SHELL", "pg_isready -U festbench -d festbench"] interval: 5s timeout: 3s retries: 10 volumes: - festbench-mariadb: \ No newline at end of file + festbench-postgres: \ No newline at end of file diff --git a/tests/Feature/BenchmarkRunTest.php b/tests/Feature/BenchmarkRunTest.php deleted file mode 100644 index 50b30e3..0000000 --- a/tests/Feature/BenchmarkRunTest.php +++ /dev/null @@ -1,72 +0,0 @@ -create(['visibility' => 'public']); - BenchmarkRun::factory()->create(['visibility' => 'private']); - - $visible = BenchmarkRun::visibleTo(null)->get(); - - expect($visible->pluck('id'))->toContain($publicRun->id) - ->and($visible)->toHaveCount(1); -}); - -test('authenticated users see public runs plus their own private runs', function () { - $me = User::factory()->create(); - $other = User::factory()->create(); - - $publicRun = BenchmarkRun::factory()->create(['visibility' => 'public']); - $myPrivate = BenchmarkRun::factory()->create(['visibility' => 'private', 'user_id' => $me->id]); - BenchmarkRun::factory()->create(['visibility' => 'private', 'user_id' => $other->id]); - - $visible = BenchmarkRun::visibleTo($me)->get(); - - expect($visible->pluck('id'))->toContain($publicRun->id) - ->and($visible->pluck('id'))->toContain($myPrivate->id) - ->and($visible)->toHaveCount(2); -}); - -test('submitted metrics are immutable through fillable updates', function () { - $run = BenchmarkRun::factory()->create(['model_id' => 'original/model', 'visibility' => 'public']); - - // The only update path any user-facing code may use: - $run->fill(['notes' => 'my notes', 'visibility' => 'private', 'model_id' => 'hacked/model']) - ->save(); - - expect($run->notes)->toBe('my notes') - ->and($run->visibility)->toBe('private') - ->and($run->fresh()->model_id)->toBe('original/model') - // fill() must have discarded the immutable field, not persisted it - ->and($run->getOriginal('model_id'))->toBe('original/model'); -}); - -test('overlay helper reports ownership correctly', function () { - $owner = User::factory()->create(); - $run = BenchmarkRun::factory()->create(['user_id' => $owner->id]); - - expect($run->isOwnedBy($owner))->toBeTrue() - ->and($run->isOwnedBy(User::factory()->create()))->toBeFalse() - ->and($run->isOwnedBy(null))->toBeFalse(); -}); - -test('run results belong to a run and are deleted with it', function () { - $run = BenchmarkRun::factory()->create(); - $result = RunResult::factory()->create(['benchmark_run_id' => $run->id]); - - expect($result->run->id)->toBe($run->id); - - $run->delete(); - - expect(RunResult::where('benchmark_run_id', $run->id)->exists())->toBeFalse(); -}); - -test('run results cast their payload to array', function () { - $result = RunResult::factory()->create(['payload' => ['finish' => 'stop', 'ok' => true]]); - - expect($result->fresh()->payload)->toBeArray() - ->and($result->fresh()->payload['finish'])->toBe('stop') - ->and($result->fresh()->ok)->toBeBool(); -});