285 lines
10 KiB
PHP
285 lines
10 KiB
PHP
<?php
|
|
|
|
namespace App\Console\Commands;
|
|
|
|
use App\Models\BenchmarkRun;
|
|
use App\Models\User;
|
|
use App\Support\Suites;
|
|
use Illuminate\Console\Command;
|
|
use Illuminate\Support\Carbon;
|
|
use PDO;
|
|
use Symfony\Component\Console\Attribute\AsCommand;
|
|
|
|
#[AsCommand(name: 'festbench:import', description: 'Import a festbench benchmarks.db (SQLite) into the per-suite tables')]
|
|
class FestbenchImport extends Command
|
|
{
|
|
protected $signature = 'festbench:import
|
|
{path : Path to the festbench benchmarks.db}
|
|
{--user= : Email of the user to attribute the imported runs to}
|
|
{--label= : Only import runs whose label matches exactly}
|
|
{--private : Import runs as private instead of public}
|
|
{--fresh : Delete ALL existing benchmark data before importing}';
|
|
|
|
public function handle(): int
|
|
{
|
|
$path = $this->argument('path');
|
|
|
|
if (! file_exists($path)) {
|
|
$this->error("Database not found: {$path}");
|
|
|
|
return self::FAILURE;
|
|
}
|
|
|
|
if ($this->option('fresh') && ! $this->confirmFresh()) {
|
|
return self::SUCCESS;
|
|
}
|
|
|
|
$user = $this->resolveUser();
|
|
|
|
$sqlite = new PDO('sqlite:'.$path, null, null, [
|
|
PDO::ATTR_ERRMODE => PDO::ERRMODE_EXCEPTION,
|
|
PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC,
|
|
]);
|
|
|
|
$query = 'SELECT * FROM runs';
|
|
$params = [];
|
|
if ($label = $this->option('label')) {
|
|
$query .= ' WHERE label = ?';
|
|
$params[] = $label;
|
|
}
|
|
|
|
$runs = $sqlite->prepare($query);
|
|
$runs->execute($params);
|
|
$runRows = $runs->fetchAll();
|
|
|
|
$importedRuns = $skippedRuns = $importedResults = $unknownSuites = 0;
|
|
|
|
$progress = $this->output->createProgressBar(count($runRows));
|
|
$progress->start();
|
|
|
|
foreach ($runRows as $source) {
|
|
if (BenchmarkRun::where('run_id', $source['run_id'])->exists()) {
|
|
$skippedRuns++;
|
|
$progress->advance();
|
|
|
|
continue;
|
|
}
|
|
|
|
$run = $this->createRun($source, $user);
|
|
$importedRuns++;
|
|
|
|
$sourceTable = Suites::sourceTable($source['suite'] ?? '');
|
|
|
|
$stmt = $sqlite->prepare("SELECT * FROM {$sourceTable} WHERE run_id = ?");
|
|
$stmt->execute([$source['run_id']]);
|
|
|
|
foreach ($stmt->fetchAll() as $row) {
|
|
$this->createSuiteRow($run, $row, $source);
|
|
$importedResults++;
|
|
}
|
|
|
|
$progress->advance();
|
|
}
|
|
|
|
$progress->finish();
|
|
$this->newLine(2);
|
|
$this->info("Imported {$importedRuns} runs ({$importedResults} suite rows), skipped {$skippedRuns} already present.");
|
|
|
|
if ($unknownSuites > 0) {
|
|
$this->warn("{$unknownSuites} rows were stored with fallback routing — check the suite registry.");
|
|
}
|
|
|
|
// Keep the precomputed summary + snapshot in step with the new data.
|
|
$this->call('festbench:summary');
|
|
$this->call('festbench:snapshot');
|
|
|
|
return self::SUCCESS;
|
|
}
|
|
|
|
private function confirmFresh(): bool
|
|
{
|
|
if (! $this->option('no-interaction')) {
|
|
if (! $this->confirm('This will DELETE all existing benchmark runs and results. Continue?')) {
|
|
return false;
|
|
}
|
|
}
|
|
|
|
// Delete children first (partitioned tables have no FK cascade issues,
|
|
// but explicit order is clearer and driver-agnostic).
|
|
foreach (Suites::all() as $suite) {
|
|
$model = Suites::modelFor($suite);
|
|
$model::query()->delete();
|
|
}
|
|
BenchmarkRun::query()->delete();
|
|
|
|
return true;
|
|
}
|
|
|
|
private function createRun(array $source, ?User $user): BenchmarkRun
|
|
{
|
|
return BenchmarkRun::create([
|
|
'run_id' => $source['run_id'],
|
|
'ran_at' => $source['ts'] ? Carbon::parse($source['ts']) : null,
|
|
'suite' => $source['suite'] ?? 'unknown',
|
|
'run_key' => $source['run_key'] ?? null,
|
|
'model_id' => $source['model_id'] ?? null,
|
|
'model_quant' => ($source['model_quant'] ?? '') !== '' ? $source['model_quant'] : null,
|
|
'model_quant_raw' => $source['model_quant_raw'] ?? null,
|
|
'model_path' => $source['model_path'] ?? null,
|
|
'engine' => $source['engine'] ?? null,
|
|
'engine_build' => $source['engine_build'] ?? null,
|
|
'n_ctx' => $source['n_ctx'] ?? null,
|
|
'slots' => $source['slots'] ?? null,
|
|
'gpu' => $source['gpu'] ?? null,
|
|
'gpu_count' => $source['gpu_count'] ?? null,
|
|
'gpu_source' => $source['gpu_source'] ?? null,
|
|
'host' => $source['host'] ?? null,
|
|
'client' => $source['client'] ?? null,
|
|
'endpoint' => $source['endpoint'] ?? null,
|
|
'label' => ($source['label'] ?? '') !== '' ? $source['label'] : null,
|
|
'config' => $source['config'] ?? null,
|
|
'image_digest' => $source['image_digest'] ?? null,
|
|
'validators' => $this->decodeJson($source['validators'] ?? null),
|
|
'suite_versions' => $this->decodeJson($source['suite_versions'] ?? null),
|
|
'harness' => $source['harness'] ?? null,
|
|
'harness_ver' => $source['harness_ver'] ?? null,
|
|
'harness_mode' => $source['harness_mode'] ?? null,
|
|
'user_id' => $user?->id,
|
|
'visibility' => $this->option('private') ? BenchmarkRun::VISIBILITY_PRIVATE : BenchmarkRun::VISIBILITY_PUBLIC,
|
|
'notes' => $source['notes'] ?? null,
|
|
]);
|
|
}
|
|
|
|
/**
|
|
* Route one source result row to its suite table via the registry.
|
|
* scenariobench rows have their own shape; metric suites share core
|
|
* columns with JSON-source extras (name→task, kind, checks, judge).
|
|
*/
|
|
private function createSuiteRow(BenchmarkRun $run, array $row, array $sourceRun): void
|
|
{
|
|
$suite = $run->suite;
|
|
$model = Suites::modelFor($suite);
|
|
|
|
if ($model === null) {
|
|
// Unknown suite: keep the run, drop the rows with a warning count.
|
|
$this->warn("Unregistered suite '{$suite}' (run {$run->run_id}) — rows skipped.");
|
|
|
|
return;
|
|
}
|
|
|
|
$ranAt = $run->ran_at;
|
|
|
|
if ($suite === 'scenariobench') {
|
|
$model::create([
|
|
'benchmark_run_id' => $run->id,
|
|
'ran_at' => $ranAt,
|
|
'scenario' => $row['scenario'] ?? ($row['task'] ?? 'unknown'),
|
|
'level' => $row['level'] ?? null,
|
|
'ok' => $row['ok'] === null ? null : (bool) $row['ok'],
|
|
'outcome' => $row['outcome'] ?? null,
|
|
'wall' => $row['wall'] ?? null,
|
|
'turns' => $row['turns'] ?? null,
|
|
'commands' => $row['commands'] ?? null,
|
|
'repeats' => $row['repeats'] ?? null,
|
|
'submissions' => $row['submissions'] ?? null,
|
|
'rejected' => $row['rejected'] ?? null,
|
|
't_first_sub' => $row['t_first_sub'] ?? null,
|
|
'n_first_sub' => $row['n_first_sub'] ?? null,
|
|
't_pass' => $row['t_pass'] ?? null,
|
|
'n_pass' => $row['n_pass'] ?? null,
|
|
'gen_tokens' => $row['gen_tokens'] ?? null,
|
|
'vis_tokens' => $row['vis_tokens'] ?? null,
|
|
'rea_tokens' => $row['rea_tokens'] ?? null,
|
|
'verdicts' => $this->decodeJson($row['verdicts'] ?? null),
|
|
'detail' => ($row['detail'] ?? '') !== '' ? $row['detail'] : null,
|
|
'payload' => $row,
|
|
]);
|
|
|
|
return;
|
|
}
|
|
|
|
// Shared mapping for metric suites. JSON-source extras come through
|
|
// payload-style keys; the SQLite results table uses task/section.
|
|
$model::create(array_merge([
|
|
'benchmark_run_id' => $run->id,
|
|
'ran_at' => $ranAt,
|
|
'task' => $row['task'] ?? ($row['name'] ?? 'unknown'),
|
|
'level' => $row['level'] !== 'n/a' ? ($row['level'] ?? null) : null,
|
|
'ok' => array_key_exists('ok', $row) && $row['ok'] !== null ? (bool) $row['ok'] : null,
|
|
'decode' => $row['decode'] ?? null,
|
|
'user_tps' => $row['user_tps'] ?? null,
|
|
'ttft' => $row['ttft'] ?? null,
|
|
't_answer' => $row['t_answer'] ?? null,
|
|
'wall' => $row['wall'] ?? null,
|
|
'reasoning' => $row['reasoning'] ?? null,
|
|
'visible' => $row['visible'] ?? null,
|
|
'total' => $row['total'] ?? null,
|
|
'finish' => ($row['finish'] ?? '') !== '' ? $row['finish'] : null,
|
|
'detail' => ($row['detail'] ?? '') !== '' ? $row['detail'] : null,
|
|
'payload' => $row,
|
|
], $this->suiteExtras($suite, $row)));
|
|
}
|
|
|
|
/**
|
|
* Suite-specific columns, keyed for create(). JSON sources use name,
|
|
* checks_passed, checks_total, judge, prefill, prompt_tokens, kind; the
|
|
* SQLite results table uses section and tool_calls.
|
|
*/
|
|
private function suiteExtras(string $suite, array $row): array
|
|
{
|
|
$extras = [];
|
|
|
|
if (in_array($suite, ['softbench', 'prosebench', 'psychbench'], true)) {
|
|
$extras['section'] = ($row['section'] ?? '') !== '' ? $row['section'] : null;
|
|
}
|
|
|
|
if ($suite === 'softbench') {
|
|
$extras['tool_calls'] = $row['tool_calls'] ?? null;
|
|
}
|
|
|
|
if ($suite === 'infrabench') {
|
|
$extras['kind'] = ($row['kind'] ?? '') !== '' ? $row['kind'] : null;
|
|
}
|
|
|
|
if (in_array($suite, ['prosebench', 'psychbench'], true)) {
|
|
$extras['checks_passed'] = $row['checks_passed'] ?? null;
|
|
$extras['checks_total'] = $row['checks_total'] ?? null;
|
|
}
|
|
|
|
if ($suite === 'prosebench') {
|
|
$extras['judge'] = ($row['judge'] ?? '') !== '' ? $row['judge'] : null;
|
|
$extras['prefill'] = $row['prefill'] ?? null;
|
|
$extras['prompt_tokens'] = $row['prompt_tokens'] ?? null;
|
|
}
|
|
|
|
return $extras;
|
|
}
|
|
|
|
private function resolveUser(): ?User
|
|
{
|
|
$email = $this->option('user');
|
|
|
|
if ($email === null) {
|
|
return null;
|
|
}
|
|
|
|
$user = User::where('email', $email)->first();
|
|
|
|
if ($user === null) {
|
|
$this->warn("User {$email} not found — importing without attribution.");
|
|
}
|
|
|
|
return $user;
|
|
}
|
|
|
|
private function decodeJson(?string $value): ?array
|
|
{
|
|
if ($value === null || $value === '' || $value === '{}') {
|
|
return null;
|
|
}
|
|
|
|
$decoded = json_decode($value, true);
|
|
|
|
return is_array($decoded) ? $decoded : null;
|
|
}
|
|
}
|