dedupe keys for warnOnce() */ private array $warned = []; protected $signature = 'festbench:import {path : Path to the festbench benchmarks.db} {--user= : Email of the user to attribute the imported runs to} {--label= : Only import runs whose label matches exactly} {--run-id= : Comma-separated list of run_ids to import} {--gpu= : Override the GPU name on imported runs (backfill or correction)} {--gpu-count= : Override GPU count when --gpu is supplied (defaults to 1 if source count is 0)} {--relabel= : Replace the label on imported runs (build one session from many)} {--config-id= : Link imported runs to a server config by id} {--private : Import runs as private instead of public} {--fresh : Delete ALL existing benchmark data before importing}'; public function handle(): int { $path = $this->argument('path'); if (! file_exists($path)) { $this->error("Database not found: {$path}"); return self::FAILURE; } if ($this->option('fresh') && ! $this->confirmFresh()) { return self::SUCCESS; } $user = $this->resolveUser(); $sqlite = new PDO('sqlite:'.$path, null, null, [ PDO::ATTR_ERRMODE => PDO::ERRMODE_EXCEPTION, PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC, ]); $query = 'SELECT * FROM runs'; $where = []; $params = []; if ($label = $this->option('label')) { $where[] = 'label = ?'; $params[] = $label; } if ($runIds = $this->option('run-id')) { $ids = array_map('trim', explode(',', (string) $runIds)); $where[] = 'run_id IN ('.implode(',', array_fill(0, count($ids), '?')).')'; $params = [...$params, ...$ids]; } if ($where !== []) { $query .= ' WHERE '.implode(' AND ', $where); } $runs = $sqlite->prepare($query); $runs->execute($params); $runRows = $runs->fetchAll(); $importedRuns = $skippedRuns = $importedResults = $unknownSuites = 0; $progress = $this->output->createProgressBar(count($runRows)); $progress->start(); foreach ($runRows as $source) { if (BenchmarkRun::where('run_id', $source['run_id'])->exists()) { $skippedRuns++; $progress->advance(); continue; } $run = $this->createRun($source, $user); $importedRuns++; $sourceTable = Suites::sourceTable($source['suite'] ?? ''); $stmt = $sqlite->prepare("SELECT * FROM {$sourceTable} WHERE run_id = ?"); $stmt->execute([$source['run_id']]); foreach ($stmt->fetchAll() as $row) { $this->createSuiteRow($run, $row, $source); $importedResults++; } $progress->advance(); } $progress->finish(); $this->newLine(2); $this->info("Imported {$importedRuns} runs ({$importedResults} suite rows), skipped {$skippedRuns} already present."); if ($unknownSuites > 0) { $this->warn("{$unknownSuites} rows were stored with fallback routing — check the suite registry."); } // Keep the precomputed summary + snapshot in step with the new data. $this->call('festbench:summary'); $this->call('festbench:snapshot'); return self::SUCCESS; } private function confirmFresh(): bool { if (! $this->option('no-interaction')) { if (! $this->confirm('This will DELETE all existing benchmark runs and results. Continue?')) { return false; } } // Delete children first (partitioned tables have no FK cascade issues, // but explicit order is clearer and driver-agnostic). foreach (Suites::all() as $suite) { $model = Suites::modelFor($suite); $model::query()->delete(); } BenchmarkRun::query()->delete(); return true; } private function createRun(array $source, ?User $user): BenchmarkRun { // --gpu overrides the recorded hardware (backfill/correction); the // source row keeps its provenance inside payload/run_key untouched. $gpuName = $this->option('gpu') ?? ($source['gpu'] ?? null); $gpu = Gpu::match($gpuName); $gpuCount = $this->option('gpu-count') ?? ($source['gpu_count'] ?? null); if ($this->option('gpu') !== null && ((int) $gpuCount) < 1) { $gpuCount = 1; } // --relabel rewrites the session glue so runs imported together form // one band; --config-id links the named server config for grouping. $label = $this->option('relabel') ?? (($source['label'] ?? '') !== '' ? $source['label'] : null); $configId = $this->option('config-id'); if ($configId !== null && ! ServerConfig::whereKey((int) $configId)->exists()) { $this->warnOnce("Server config {$configId} not found -- runs will import without a config link."); $configId = null; } if ($gpuName !== null && $gpu === null && ! in_array(strtolower((string) $gpuName), ['unknown', 'hosted', 'cloud'], true)) { $this->warnOnce("GPU '{$gpuName}' is not in the catalog (run {$source['run_id']}) — it can be confirmed on the website later."); } return BenchmarkRun::create([ 'run_id' => $source['run_id'], 'ran_at' => $source['ts'] ? Carbon::parse($source['ts']) : null, 'suite' => $source['suite'] ?? 'unknown', 'run_key' => $source['run_key'] ?? null, 'model_id' => $source['model_id'] ?? null, 'model_quant' => ($source['model_quant'] ?? '') !== '' ? $source['model_quant'] : null, 'model_quant_raw' => $source['model_quant_raw'] ?? null, 'model_path' => $source['model_path'] ?? null, 'engine' => $source['engine'] ?? null, 'engine_build' => $source['engine_build'] ?? null, 'n_ctx' => $source['n_ctx'] ?? null, 'slots' => $source['slots'] ?? null, 'gpu' => $gpuName, 'gpu_count' => $gpuCount, 'gpu_id' => $gpu?->id, 'gpu_source' => $this->option('gpu') ? ($source['gpu_source'] ?? 'backfill-manual') : ($source['gpu_source'] ?? null), 'host' => $source['host'] ?? null, 'client' => $source['client'] ?? null, 'endpoint' => $source['endpoint'] ?? null, 'label' => $label, 'config_id' => $configId, 'config' => $source['config'] ?? null, 'image_digest' => $source['image_digest'] ?? null, 'validators' => $this->decodeJson($source['validators'] ?? null), 'suite_versions' => $this->decodeJson($source['suite_versions'] ?? null), 'harness' => $source['harness'] ?? null, 'harness_ver' => $source['harness_ver'] ?? null, 'harness_mode' => $source['harness_mode'] ?? null, 'user_id' => $user?->id, 'visibility' => $this->option('private') ? BenchmarkRun::VISIBILITY_PRIVATE : BenchmarkRun::VISIBILITY_PUBLIC, 'notes' => $source['notes'] ?? null, ]); } /** * Route one source result row to its suite table via the registry. * scenariobench rows have their own shape; metric suites share core * columns with JSON-source extras (name→task, kind, checks, judge). */ private function createSuiteRow(BenchmarkRun $run, array $row, array $sourceRun): void { $suite = $run->suite; $model = Suites::modelFor($suite); if ($model === null) { // Unknown suite: keep the run, drop the rows with a warning count. $this->warn("Unregistered suite '{$suite}' (run {$run->run_id}) — rows skipped."); return; } $ranAt = $run->ran_at; if ($suite === 'scenariobench') { $model::create([ 'benchmark_run_id' => $run->id, 'ran_at' => $ranAt, 'scenario' => $row['scenario'] ?? ($row['task'] ?? 'unknown'), 'level' => $row['level'] ?? null, 'ok' => $row['ok'] === null ? null : (bool) $row['ok'], 'outcome' => $row['outcome'] ?? null, 'wall' => $row['wall'] ?? null, 'turns' => $row['turns'] ?? null, 'commands' => $row['commands'] ?? null, 'repeats' => $row['repeats'] ?? null, 'submissions' => $row['submissions'] ?? null, 'rejected' => $row['rejected'] ?? null, 't_first_sub' => $row['t_first_sub'] ?? null, 'n_first_sub' => $row['n_first_sub'] ?? null, 't_pass' => $row['t_pass'] ?? null, 'n_pass' => $row['n_pass'] ?? null, 'gen_tokens' => $row['gen_tokens'] ?? null, 'vis_tokens' => $row['vis_tokens'] ?? null, 'rea_tokens' => $row['rea_tokens'] ?? null, 'verdicts' => $this->decodeJson($row['verdicts'] ?? null), 'detail' => ($row['detail'] ?? '') !== '' ? $row['detail'] : null, 'payload' => $row, ]); return; } // Shared mapping for metric suites. JSON-source extras come through // payload-style keys; the SQLite results table uses task/section. $model::create(array_merge([ 'benchmark_run_id' => $run->id, 'ran_at' => $ranAt, 'task' => $row['task'] ?? ($row['name'] ?? 'unknown'), 'level' => $row['level'] !== 'n/a' ? ($row['level'] ?? null) : null, 'ok' => array_key_exists('ok', $row) && $row['ok'] !== null ? (bool) $row['ok'] : null, 'decode' => $row['decode'] ?? null, 'user_tps' => $row['user_tps'] ?? null, 'ttft' => $row['ttft'] ?? null, 't_answer' => $row['t_answer'] ?? null, 'wall' => $row['wall'] ?? null, 'reasoning' => $row['reasoning'] ?? null, 'visible' => $row['visible'] ?? null, 'total' => $row['total'] ?? null, 'finish' => ($row['finish'] ?? '') !== '' ? $row['finish'] : null, 'detail' => ($row['detail'] ?? '') !== '' ? $row['detail'] : null, 'payload' => $row, ], $this->suiteExtras($suite, $row))); } /** * Suite-specific columns, keyed for create(). JSON sources use name, * checks_passed, checks_total, judge, prefill, prompt_tokens, kind; the * SQLite results table uses section and tool_calls. */ private function suiteExtras(string $suite, array $row): array { $extras = []; if (in_array($suite, ['softbench', 'prosebench', 'psychbench'], true)) { $extras['section'] = ($row['section'] ?? '') !== '' ? $row['section'] : null; } if ($suite === 'softbench') { $extras['tool_calls'] = $row['tool_calls'] ?? null; } if ($suite === 'infrabench') { $extras['kind'] = ($row['kind'] ?? '') !== '' ? $row['kind'] : null; } if (in_array($suite, ['prosebench', 'psychbench'], true)) { $extras['checks_passed'] = $row['checks_passed'] ?? null; $extras['checks_total'] = $row['checks_total'] ?? null; } if ($suite === 'prosebench') { $extras['judge'] = ($row['judge'] ?? '') !== '' ? $row['judge'] : null; $extras['prefill'] = $row['prefill'] ?? null; $extras['prompt_tokens'] = $row['prompt_tokens'] ?? null; } return $extras; } /** Warn for a repeated condition only once per import run. */ private function warnOnce(string $message): void { $key = md5($message); if (isset($this->warned[$key])) { return; } $this->warned[$key] = true; $this->warn($message); } private function resolveUser(): ?User { $email = $this->option('user'); if ($email === null) { return null; } $user = User::where('email', $email)->first(); if ($user === null) { $this->warn("User {$email} not found — importing without attribution."); } return $user; } private function decodeJson(?string $value): ?array { if ($value === null || $value === '' || $value === '{}') { return null; } $decoded = json_decode($value, true); return is_array($decoded) ? $decoded : null; } }