diff --git a/app/Console/Commands/FestbenchImport.php b/app/Console/Commands/FestbenchImport.php new file mode 100644 index 0000000..14c7a37 --- /dev/null +++ b/app/Console/Commands/FestbenchImport.php @@ -0,0 +1,153 @@ +argument('path'); + + if (! file_exists($path)) { + $this->error("Database not found: {$path}"); + + return self::FAILURE; + } + + $user = $this->resolveUser(); + + $sqlite = new PDO('sqlite:'.$path, null, null, [ + PDO::ATTR_ERRMODE => PDO::ERRMODE_EXCEPTION, + PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC, + ]); + + $importedRuns = $skippedRuns = $importedResults = 0; + + $runs = $sqlite->query('SELECT * FROM runs')->fetchAll(); + $progress = $this->output->createProgressBar(count($runs)); + $progress->start(); + + foreach ($runs as $source) { + if (BenchmarkRun::where('run_id', $source['run_id'])->exists()) { + $skippedRuns++; + $progress->advance(); + + continue; + } + + $run = BenchmarkRun::create([ + 'run_id' => $source['run_id'], + 'ran_at' => $source['ts'] ? Carbon::parse($source['ts']) : null, + 'suite' => $source['suite'] ?? 'unknown', + 'run_key' => $source['run_key'] ?? null, + 'model_id' => $source['model_id'] ?? null, + 'model_quant' => $source['model_quant'] ?? null, + 'model_quant_raw' => $source['model_quant_raw'] ?? null, + 'model_path' => $source['model_path'] ?? null, + 'engine' => $source['engine'] ?? null, + 'engine_build' => $source['engine_build'] ?? null, + 'n_ctx' => $source['n_ctx'] ?? null, + 'slots' => $source['slots'] ?? null, + 'gpu' => $source['gpu'] ?? null, + 'gpu_count' => $source['gpu_count'] ?? null, + 'gpu_source' => $source['gpu_source'] ?? null, + 'host' => $source['host'] ?? null, + 'client' => $source['client'] ?? null, + 'endpoint' => $source['endpoint'] ?? null, + 'label' => $source['label'] ?? null, + 'config' => $source['config'] ?? null, + 'image_digest' => $source['image_digest'] ?? null, + 'validators' => $this->decodeJson($source['validators'] ?? null), + 'suite_versions' => $this->decodeJson($source['suite_versions'] ?? null), + 'harness' => $source['harness'] ?? null, + 'harness_ver' => $source['harness_ver'] ?? null, + 'harness_mode' => $source['harness_mode'] ?? null, + 'user_id' => $user?->id, + 'visibility' => $this->option('private') ? BenchmarkRun::VISIBILITY_PRIVATE : BenchmarkRun::VISIBILITY_PUBLIC, + 'notes' => $source['notes'] ?? null, + ]); + + $importedRuns++; + + $stmt = $sqlite->prepare('SELECT * FROM results WHERE run_id = ?'); + $stmt->execute([$source['run_id']]); + + foreach ($stmt->fetchAll() as $row) { + RunResult::create([ + 'benchmark_run_id' => $run->id, + 'suite' => $row['suite'] ?? $source['suite'], + 'section' => $row['section'] ?? null, + 'task' => $row['task'] ?? ($row['name'] ?? 'unknown'), + 'level' => $row['level'] ?? null, + 'ok' => $row['ok'] === null ? null : (bool) $row['ok'], + 'decode' => $row['decode'] ?? null, + 'user_tps' => $row['user_tps'] ?? null, + 'ttft' => $row['ttft'] ?? null, + 't_answer' => $row['t_answer'] ?? null, + 'wall' => $row['wall'] ?? null, + 'reasoning' => $row['reasoning'] ?? null, + 'visible' => $row['visible'] ?? null, + 'total' => $row['total'] ?? null, + 'tool_calls' => $row['tool_calls'] ?? null, + 'checks_passed' => $row['checks_passed'] ?? null, + 'checks_total' => $row['checks_total'] ?? null, + 'finish' => $row['finish'] ?? null, + 'detail' => ($row['detail'] ?? '') !== '' ? $row['detail'] : null, + 'payload' => $row, + ]); + + $importedResults++; + } + + $progress->advance(); + } + + $progress->finish(); + $this->newLine(2); + $this->info("Imported {$importedRuns} runs ({$importedResults} results), skipped {$skippedRuns} already present."); + + return self::SUCCESS; + } + + private function resolveUser(): ?User + { + $email = $this->option('user'); + + if ($email === null) { + return null; + } + + $user = User::where('email', $email)->first(); + + if ($user === null) { + $this->warn("User {$email} not found — importing without attribution."); + } + + return $user; + } + + private function decodeJson(?string $value): ?array + { + if ($value === null || $value === '' || $value === '{}') { + return null; + } + + $decoded = json_decode($value, true); + + return is_array($decoded) ? $decoded : null; + } +} diff --git a/tests/Feature/FestbenchImportTest.php b/tests/Feature/FestbenchImportTest.php new file mode 100644 index 0000000..325bc87 --- /dev/null +++ b/tests/Feature/FestbenchImportTest.php @@ -0,0 +1,108 @@ +exec(<<<'SQL' + CREATE TABLE runs ( + run_id TEXT PRIMARY KEY, ts TEXT, suite TEXT, run_key TEXT, + endpoint TEXT, model_id TEXT, model_quant TEXT, model_path TEXT, + engine TEXT, engine_build TEXT, n_ctx INTEGER, slots INTEGER, + gpu TEXT, gpu_count INTEGER, gpu_source TEXT, host TEXT, client TEXT, label TEXT, + config TEXT, notes TEXT, + model_quant_raw TEXT, image_digest TEXT, validators TEXT, suite_versions TEXT, harness TEXT, harness_ver TEXT, harness_mode TEXT); + CREATE TABLE results ( + run_id TEXT, suite TEXT, section TEXT, task TEXT, level TEXT, + ok INTEGER, decode REAL, user_tps REAL, ttft REAL, t_answer REAL, + wall REAL, reasoning INTEGER, visible INTEGER, total INTEGER, + finish TEXT, detail TEXT + , tool_calls INTEGER); + SQL); + + $insertRun = $pdo->prepare('INSERT INTO runs (run_id, ts, suite, model_id, model_quant, engine, gpu, gpu_count, label) VALUES (?,?,?,?,?,?,?,?,?)'); + $insertRun->execute(['aaa111222333', '2026-09-01T10:00:00', 'softbench', 'fixture/model', 'Q5_K_M', 'llama.cpp', 'R9700', 1, 'fixture-label']); + $insertRun->execute(['bbb444555666', '2026-09-02T11:00:00', 'infrabench', 'fixture/model', null, 'vllm', 'unknown', 0, null]); + + $insertResult = $pdo->prepare('INSERT INTO results (run_id, suite, section, task, level, ok, decode, user_tps, ttft, t_answer, wall, reasoning, visible, total, finish, detail) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)'); + $insertResult->execute(['aaa111222333', 'softbench', 'code', 'first_occurrence', 'off', 1, 83.82, 61.14, 0.47, 0.47, 1.77, 0, 108, 109, 'stop', '']); + $insertResult->execute(['aaa111222333', 'softbench', 'code', 'merge_intervals', 'off', 0, 51.2, 40.0, 0.5, 0.5, 3.0, 250, 90, 340, 'length', 'assertion failed']); + $insertResult->execute(['bbb444555666', 'infrabench', null, 'modprobe_exists', 'medium', 1, 40.0, 30.0, 1.2, 1.2, 10.0, 0, 200, 200, 'stop', '']); +} + +test('festbench:import imports runs and results from a real-schema sqlite db', function () { + $path = storage_path('app/test-benchmarks.db'); + festbench_fixture_db($path); + + $this->artisan(FestbenchImport::class, ['path' => $path])->assertSuccessful(); + + expect(BenchmarkRun::count())->toBe(2) + ->and(RunResult::count())->toBe(3); + + $run = BenchmarkRun::where('run_id', 'aaa111222333')->first(); + expect($run->suite)->toBe('softbench') + ->and($run->model_id)->toBe('fixture/model') + ->and($run->model_quant)->toBe('Q5_K_M') + ->and($run->ran_at->format('Y-m-d H:i'))->toBe('2026-09-01 10:00') + ->and($run->visibility)->toBe('public') + ->and($run->results)->toHaveCount(2); + + $failed = RunResult::where('task', 'merge_intervals')->first(); + expect($failed->ok)->toBeFalse() + ->and($failed->finish)->toBe('length') + ->and($failed->detail)->toBe('assertion failed') + // full source row preserved + ->and($failed->payload['total'])->toBe(340); + + // null quant and empty detail survive the import + $second = BenchmarkRun::where('run_id', 'bbb444555666')->first(); + expect($second->model_quant)->toBeNull() + ->and($second->results->first()->detail)->toBeNull(); +}); + +test('festbench:import is idempotent', function () { + $path = storage_path('app/test-benchmarks.db'); + festbench_fixture_db($path); + + $this->artisan(FestbenchImport::class, ['path' => $path])->assertSuccessful(); + $this->artisan(FestbenchImport::class, ['path' => $path])->assertSuccessful(); + + expect(BenchmarkRun::count())->toBe(2) + ->and(RunResult::count())->toBe(3); +}); + +test('festbench:import attributes runs to a user and can default to private', function () { + $path = storage_path('app/test-benchmarks.db'); + festbench_fixture_db($path); + $me = User::factory()->create(); + + $this->artisan(FestbenchImport::class, ['path' => $path, '--user' => $me->email, '--private' => true])->assertSuccessful(); + + $run = BenchmarkRun::where('run_id', 'aaa111222333')->first(); + expect($run->user_id)->toBe($me->id) + ->and($run->visibility)->toBe('private'); + + // guest cannot see it, owner can + expect(BenchmarkRun::visibleTo(null)->count())->toBe(0) + ->and(BenchmarkRun::visibleTo($me)->count())->toBe(2); +}); + +test('festbench:import warns when the user does not exist', function () { + $path = storage_path('app/test-benchmarks.db'); + festbench_fixture_db($path); + + $this->artisan(FestbenchImport::class, ['path' => $path, '--user' => 'ghost@example.com']) + ->expectsOutputToContain('not found') + ->assertSuccessful(); + + expect(BenchmarkRun::first()->user_id)->toBeNull(); +});