Phase 4: festbench:import artisan command, real-schema fixture tests, imported 109 real runs

2026-09-16 22:42:38 -04:00 · 2026-09-16 22:42:38 -04:00 · 8863585fd6
commit 8863585fd6
parent 5ec35c367b
2 changed files with 261 additions and 0 deletions
--- a/app/Console/Commands/FestbenchImport.php
+++ b/app/Console/Commands/FestbenchImport.php
@ -0,0 +1,153 @@
+<?php
+
+namespace App\Console\Commands;
+
+use App\Models\BenchmarkRun;
+use App\Models\RunResult;
+use App\Models\User;
+use Illuminate\Console\Command;
+use Illuminate\Support\Carbon;
+use PDO;
+use Symfony\Component\Console\Attribute\AsCommand;
+
+#[AsCommand(name: 'festbench:import', description: 'Import a festbench benchmarks.db (SQLite) into the application database')]
+class FestbenchImport extends Command
+{
+    protected $signature = 'festbench:import
+        {path : Path to the festbench benchmarks.db}
+        {--user= : Email of the user to attribute the imported runs to}
+        {--private : Import runs as private instead of public}';
+
+    public function handle(): int
+    {
+        $path = $this->argument('path');
+
+        if (! file_exists($path)) {
+            $this->error("Database not found: {$path}");
+
+            return self::FAILURE;
+        }
+
+        $user = $this->resolveUser();
+
+        $sqlite = new PDO('sqlite:'.$path, null, null, [
+            PDO::ATTR_ERRMODE => PDO::ERRMODE_EXCEPTION,
+            PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC,
+        ]);
+
+        $importedRuns = $skippedRuns = $importedResults = 0;
+
+        $runs = $sqlite->query('SELECT * FROM runs')->fetchAll();
+        $progress = $this->output->createProgressBar(count($runs));
+        $progress->start();
+
+        foreach ($runs as $source) {
+            if (BenchmarkRun::where('run_id', $source['run_id'])->exists()) {
+                $skippedRuns++;
+                $progress->advance();
+
+                continue;
+            }
+
+            $run = BenchmarkRun::create([
+                'run_id' => $source['run_id'],
+                'ran_at' => $source['ts'] ? Carbon::parse($source['ts']) : null,
+                'suite' => $source['suite'] ?? 'unknown',
+                'run_key' => $source['run_key'] ?? null,
+                'model_id' => $source['model_id'] ?? null,
+                'model_quant' => $source['model_quant'] ?? null,
+                'model_quant_raw' => $source['model_quant_raw'] ?? null,
+                'model_path' => $source['model_path'] ?? null,
+                'engine' => $source['engine'] ?? null,
+                'engine_build' => $source['engine_build'] ?? null,
+                'n_ctx' => $source['n_ctx'] ?? null,
+                'slots' => $source['slots'] ?? null,
+                'gpu' => $source['gpu'] ?? null,
+                'gpu_count' => $source['gpu_count'] ?? null,
+                'gpu_source' => $source['gpu_source'] ?? null,
+                'host' => $source['host'] ?? null,
+                'client' => $source['client'] ?? null,
+                'endpoint' => $source['endpoint'] ?? null,
+                'label' => $source['label'] ?? null,
+                'config' => $source['config'] ?? null,
+                'image_digest' => $source['image_digest'] ?? null,
+                'validators' => $this->decodeJson($source['validators'] ?? null),
+                'suite_versions' => $this->decodeJson($source['suite_versions'] ?? null),
+                'harness' => $source['harness'] ?? null,
+                'harness_ver' => $source['harness_ver'] ?? null,
+                'harness_mode' => $source['harness_mode'] ?? null,
+                'user_id' => $user?->id,
+                'visibility' => $this->option('private') ? BenchmarkRun::VISIBILITY_PRIVATE : BenchmarkRun::VISIBILITY_PUBLIC,
+                'notes' => $source['notes'] ?? null,
+            ]);
+
+            $importedRuns++;
+
+            $stmt = $sqlite->prepare('SELECT * FROM results WHERE run_id = ?');
+            $stmt->execute([$source['run_id']]);
+
+            foreach ($stmt->fetchAll() as $row) {
+                RunResult::create([
+                    'benchmark_run_id' => $run->id,
+                    'suite' => $row['suite'] ?? $source['suite'],
+                    'section' => $row['section'] ?? null,
+                    'task' => $row['task'] ?? ($row['name'] ?? 'unknown'),
+                    'level' => $row['level'] ?? null,
+                    'ok' => $row['ok'] === null ? null : (bool) $row['ok'],
+                    'decode' => $row['decode'] ?? null,
+                    'user_tps' => $row['user_tps'] ?? null,
+                    'ttft' => $row['ttft'] ?? null,
+                    't_answer' => $row['t_answer'] ?? null,
+                    'wall' => $row['wall'] ?? null,
+                    'reasoning' => $row['reasoning'] ?? null,
+                    'visible' => $row['visible'] ?? null,
+                    'total' => $row['total'] ?? null,
+                    'tool_calls' => $row['tool_calls'] ?? null,
+                    'checks_passed' => $row['checks_passed'] ?? null,
+                    'checks_total' => $row['checks_total'] ?? null,
+                    'finish' => $row['finish'] ?? null,
+                    'detail' => ($row['detail'] ?? '') !== '' ? $row['detail'] : null,
+                    'payload' => $row,
+                ]);
+
+                $importedResults++;
+            }
+
+            $progress->advance();
+        }
+
+        $progress->finish();
+        $this->newLine(2);
+        $this->info("Imported {$importedRuns} runs ({$importedResults} results), skipped {$skippedRuns} already present.");
+
+        return self::SUCCESS;
+    }
+
+    private function resolveUser(): ?User
+    {
+        $email = $this->option('user');
+
+        if ($email === null) {
+            return null;
+        }
+
+        $user = User::where('email', $email)->first();
+
+        if ($user === null) {
+            $this->warn("User {$email} not found — importing without attribution.");
+        }
+
+        return $user;
+    }
+
+    private function decodeJson(?string $value): ?array
+    {
+        if ($value === null || $value === '' || $value === '{}') {
+            return null;
+        }
+
+        $decoded = json_decode($value, true);
+
+        return is_array($decoded) ? $decoded : null;
+    }
+}
--- a/tests/Feature/FestbenchImportTest.php
+++ b/tests/Feature/FestbenchImportTest.php
@ -0,0 +1,108 @@
+<?php
+
+use App\Console\Commands\FestbenchImport;
+use App\Models\BenchmarkRun;
+use App\Models\RunResult;
+use App\Models\User;
+
+function festbench_fixture_db(string $path): void
+{
+    if (file_exists($path)) {
+        unlink($path);
+    }
+
+    // festbench's real schema, from llm_benchmark/benchlib.py
+    $pdo = new PDO('sqlite:'.$path);
+    $pdo->exec(<<<'SQL'
+        CREATE TABLE runs (
+          run_id TEXT PRIMARY KEY, ts TEXT, suite TEXT, run_key TEXT,
+          endpoint TEXT, model_id TEXT, model_quant TEXT, model_path TEXT,
+          engine TEXT, engine_build TEXT, n_ctx INTEGER, slots INTEGER,
+          gpu TEXT, gpu_count INTEGER, gpu_source TEXT, host TEXT, client TEXT, label TEXT,
+          config TEXT, notes TEXT,
+          model_quant_raw TEXT, image_digest TEXT, validators TEXT, suite_versions TEXT, harness TEXT, harness_ver TEXT, harness_mode TEXT);
+        CREATE TABLE results (
+          run_id TEXT, suite TEXT, section TEXT, task TEXT, level TEXT,
+          ok INTEGER, decode REAL, user_tps REAL, ttft REAL, t_answer REAL,
+          wall REAL, reasoning INTEGER, visible INTEGER, total INTEGER,
+          finish TEXT, detail TEXT
+        , tool_calls INTEGER);
+    SQL);
+
+    $insertRun = $pdo->prepare('INSERT INTO runs (run_id, ts, suite, model_id, model_quant, engine, gpu, gpu_count, label) VALUES (?,?,?,?,?,?,?,?,?)');
+    $insertRun->execute(['aaa111222333', '2026-09-01T10:00:00', 'softbench', 'fixture/model', 'Q5_K_M', 'llama.cpp', 'R9700', 1, 'fixture-label']);
+    $insertRun->execute(['bbb444555666', '2026-09-02T11:00:00', 'infrabench', 'fixture/model', null, 'vllm', 'unknown', 0, null]);
+
+    $insertResult = $pdo->prepare('INSERT INTO results (run_id, suite, section, task, level, ok, decode, user_tps, ttft, t_answer, wall, reasoning, visible, total, finish, detail) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)');
+    $insertResult->execute(['aaa111222333', 'softbench', 'code', 'first_occurrence', 'off', 1, 83.82, 61.14, 0.47, 0.47, 1.77, 0, 108, 109, 'stop', '']);
+    $insertResult->execute(['aaa111222333', 'softbench', 'code', 'merge_intervals', 'off', 0, 51.2, 40.0, 0.5, 0.5, 3.0, 250, 90, 340, 'length', 'assertion failed']);
+    $insertResult->execute(['bbb444555666', 'infrabench', null, 'modprobe_exists', 'medium', 1, 40.0, 30.0, 1.2, 1.2, 10.0, 0, 200, 200, 'stop', '']);
+}
+
+test('festbench:import imports runs and results from a real-schema sqlite db', function () {
+    $path = storage_path('app/test-benchmarks.db');
+    festbench_fixture_db($path);
+
+    $this->artisan(FestbenchImport::class, ['path' => $path])->assertSuccessful();
+
+    expect(BenchmarkRun::count())->toBe(2)
+        ->and(RunResult::count())->toBe(3);
+
+    $run = BenchmarkRun::where('run_id', 'aaa111222333')->first();
+    expect($run->suite)->toBe('softbench')
+        ->and($run->model_id)->toBe('fixture/model')
+        ->and($run->model_quant)->toBe('Q5_K_M')
+        ->and($run->ran_at->format('Y-m-d H:i'))->toBe('2026-09-01 10:00')
+        ->and($run->visibility)->toBe('public')
+        ->and($run->results)->toHaveCount(2);
+
+    $failed = RunResult::where('task', 'merge_intervals')->first();
+    expect($failed->ok)->toBeFalse()
+        ->and($failed->finish)->toBe('length')
+        ->and($failed->detail)->toBe('assertion failed')
+        // full source row preserved
+        ->and($failed->payload['total'])->toBe(340);
+
+    // null quant and empty detail survive the import
+    $second = BenchmarkRun::where('run_id', 'bbb444555666')->first();
+    expect($second->model_quant)->toBeNull()
+        ->and($second->results->first()->detail)->toBeNull();
+});
+
+test('festbench:import is idempotent', function () {
+    $path = storage_path('app/test-benchmarks.db');
+    festbench_fixture_db($path);
+
+    $this->artisan(FestbenchImport::class, ['path' => $path])->assertSuccessful();
+    $this->artisan(FestbenchImport::class, ['path' => $path])->assertSuccessful();
+
+    expect(BenchmarkRun::count())->toBe(2)
+        ->and(RunResult::count())->toBe(3);
+});
+
+test('festbench:import attributes runs to a user and can default to private', function () {
+    $path = storage_path('app/test-benchmarks.db');
+    festbench_fixture_db($path);
+    $me = User::factory()->create();
+
+    $this->artisan(FestbenchImport::class, ['path' => $path, '--user' => $me->email, '--private' => true])->assertSuccessful();
+
+    $run = BenchmarkRun::where('run_id', 'aaa111222333')->first();
+    expect($run->user_id)->toBe($me->id)
+        ->and($run->visibility)->toBe('private');
+
+    // guest cannot see it, owner can
+    expect(BenchmarkRun::visibleTo(null)->count())->toBe(0)
+        ->and(BenchmarkRun::visibleTo($me)->count())->toBe(2);
+});
+
+test('festbench:import warns when the user does not exist', function () {
+    $path = storage_path('app/test-benchmarks.db');
+    festbench_fixture_db($path);
+
+    $this->artisan(FestbenchImport::class, ['path' => $path, '--user' => 'ghost@example.com'])
+        ->expectsOutputToContain('not found')
+        ->assertSuccessful();
+
+    expect(BenchmarkRun::first()->user_id)->toBeNull();
+});