festbench-web/tests/Feature/FestbenchImportTest.php

<?php

use App\Console\Commands\FestbenchImport;
use App\Models\BenchmarkRun;
use App\Models\ScenarioRun;
use App\Models\SoftbenchResult;
use App\Models\User;
use Illuminate\Support\Facades\Artisan;

function festbench_fixture_db(string $path): void
{
    if (file_exists($path)) {
        unlink($path);
    }

    // festbench's real schemas, from llm_benchmark/benchlib.py
    $pdo = new PDO('sqlite:'.$path);
    $pdo->exec(<<<'SQL'
        CREATE TABLE runs (
          run_id TEXT PRIMARY KEY, ts TEXT, suite TEXT, run_key TEXT,
          endpoint TEXT, model_id TEXT, model_quant TEXT, model_path TEXT,
          engine TEXT, engine_build TEXT, n_ctx INTEGER, slots INTEGER,
          gpu TEXT, gpu_count INTEGER, gpu_source TEXT, host TEXT, client TEXT, label TEXT,
          config TEXT, notes TEXT,
          model_quant_raw TEXT, image_digest TEXT, validators TEXT, suite_versions TEXT, harness TEXT, harness_ver TEXT, harness_mode TEXT);
        CREATE TABLE results (
          run_id TEXT, suite TEXT, section TEXT, task TEXT, level TEXT,
          ok INTEGER, decode REAL, user_tps REAL, ttft REAL, t_answer REAL,
          wall REAL, reasoning INTEGER, visible INTEGER, total INTEGER,
          finish TEXT, detail TEXT, tool_calls INTEGER);
        CREATE TABLE scenario_runs (
          run_id TEXT, scenario TEXT, level TEXT, ok INTEGER, outcome TEXT,
          wall REAL, turns INTEGER, commands INTEGER, repeats INTEGER,
          submissions INTEGER, rejected INTEGER, t_first_sub REAL, n_first_sub INTEGER,
          t_pass REAL, n_pass INTEGER, gen_tokens INTEGER, vis_tokens INTEGER,
          rea_tokens INTEGER, verdicts TEXT, detail TEXT);
    SQL);

    $insertRun = $pdo->prepare('INSERT INTO runs (run_id, ts, suite, model_id, model_quant, engine, gpu, gpu_count, label) VALUES (?,?,?,?,?,?,?,?,?)');
    $insertRun->execute(['aaa111222333', '2026-09-01T10:00:00', 'softbench', 'fixture/model', 'Q5_K_M', 'llama.cpp', 'R9700', 1, 'deterministic sampling, repeat 1']);
    $insertRun->execute(['bbb444555666', '2026-09-02T11:00:00', 'scenariobench', 'fixture/model', null, 'vllm', 'unknown', 0, 'agent test']);
    $insertRun->execute(['ccc777888999', '2026-09-03T12:00:00', 'softbench', 'fixture/model', null, 'llama.cpp', 'R9700', 1, 'other session']);

    $insertResult = $pdo->prepare('INSERT INTO results (run_id, suite, section, task, level, ok, decode, user_tps, ttft, t_answer, wall, reasoning, visible, total, finish, detail, tool_calls) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)');
    $insertResult->execute(['aaa111222333', 'softbench', 'code', 'first_occurrence', 'off', 1, 83.82, 61.14, 0.47, 0.47, 1.77, 0, 108, 109, 'stop', '', 2]);
    $insertResult->execute(['aaa111222333', 'softbench', 'code', 'merge_intervals', 'n/a', 0, 51.2, 40.0, 0.5, 0.5, 3.0, 250, 90, 340, 'length', 'assertion failed', null]);
    $insertResult->execute(['ccc777888999', 'softbench', 'code', 'other_session_task', 'off', 1, 70, 50, 0.4, 0.4, 2.0, 0, 100, 100, 'stop', '', null]);

    $insertScenario = $pdo->prepare('INSERT INTO scenario_runs (run_id, scenario, level, ok, outcome, wall, turns, commands, repeats, submissions, rejected, t_first_sub, n_first_sub, t_pass, n_pass, gen_tokens, vis_tokens, rea_tokens, verdicts, detail) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)');
    $insertScenario->execute(['bbb444555666', 'kubelet_config_repair', 'off', 1, 'passed', 264.1, 68, 67, 2, 1, 0, 262.5, 68, 264.1, 68, 2543, 2475, 0, '[{"n":1,"at":264.1,"accepted":true}]', 'passed on submission 1']);
}

beforeEach(fn () => Artisan::call('festbench:suites:refresh'));

test('festbench:import routes rows to per-suite tables', function () {
    $path = storage_path('app/test-benchmarks.db');
    festbench_fixture_db($path);

    $this->artisan(FestbenchImport::class, ['path' => $path])->assertSuccessful();

    expect(BenchmarkRun::count())->toBe(3)
        ->and(SoftbenchResult::count())->toBe(3)
        ->and(ScenarioRun::count())->toBe(1);

    $run = BenchmarkRun::where('run_id', 'aaa111222333')->first();
    expect($run->suite)->toBe('softbench')
        ->and($run->model_quant)->toBe('Q5_K_M')
        ->and($run->visibility)->toBe('public');

    $failed = SoftbenchResult::where('task', 'merge_intervals')->first();
    expect($failed->ok)->toBeFalse()
        ->and($failed->finish)->toBe('length')
        ->and($failed->detail)->toBe('assertion failed')
        // n/a levels normalize to null
        ->and($failed->level)->toBeNull()
        // complete source row preserved
        ->and($failed->payload['total'])->toBe(340);

    $toolUser = SoftbenchResult::where('task', 'first_occurrence')->first();
    expect($toolUser->tool_calls)->toBe(2);

    $scenario = ScenarioRun::first();
    expect($scenario->scenario)->toBe('kubelet_config_repair')
        ->and($scenario->outcome)->toBe('passed')
        ->and($scenario->turns)->toBe(68)
        ->and($scenario->verdicts)->toBeArray()
        ->and($scenario->t_pass)->toBe(264.1);
});

test('festbench:import --label imports only the matching session', function () {
    $path = storage_path('app/test-benchmarks.db');
    festbench_fixture_db($path);

    $this->artisan(FestbenchImport::class, ['path' => $path, '--label' => 'deterministic sampling, repeat 1'])->assertSuccessful();

    expect(BenchmarkRun::count())->toBe(1)
        ->and(BenchmarkRun::first()->run_id)->toBe('aaa111222333')
        ->and(SoftbenchResult::count())->toBe(2);
});

test('festbench:import is idempotent', function () {
    $path = storage_path('app/test-benchmarks.db');
    festbench_fixture_db($path);

    $this->artisan(FestbenchImport::class, ['path' => $path])->assertSuccessful();
    $this->artisan(FestbenchImport::class, ['path' => $path])->assertSuccessful();

    expect(BenchmarkRun::count())->toBe(3)
        ->and(SoftbenchResult::count())->toBe(3);
});

test('festbench:import --fresh purges existing data first', function () {
    $path = storage_path('app/test-benchmarks.db');
    festbench_fixture_db($path);

    // Pre-existing noise
    BenchmarkRun::factory()->has(SoftbenchResult::factory(), 'softbenchResults')->create();

    $this->artisan(FestbenchImport::class, ['path' => $path, '--fresh' => true, '--no-interaction' => true])->assertSuccessful();

    expect(BenchmarkRun::count())->toBe(3)
        ->and(BenchmarkRun::pluck('run_id')->sort()->values()->all())->toBe(['aaa111222333', 'bbb444555666', 'ccc777888999']);
});

test('festbench:import attributes runs to a user and can default to private', function () {
    $path = storage_path('app/test-benchmarks.db');
    festbench_fixture_db($path);
    $me = User::factory()->create();

    $this->artisan(FestbenchImport::class, ['path' => $path, '--user' => $me->email, '--private' => true])->assertSuccessful();

    $run = BenchmarkRun::where('run_id', 'aaa111222333')->first();
    expect($run->user_id)->toBe($me->id)
        ->and($run->visibility)->toBe('private');

    expect(BenchmarkRun::visibleTo(null)->count())->toBe(0)
        ->and(BenchmarkRun::visibleTo($me)->count())->toBe(3);
});

test('festbench:import warns when the user does not exist', function () {
    $path = storage_path('app/test-benchmarks.db');
    festbench_fixture_db($path);

    $this->artisan(FestbenchImport::class, ['path' => $path, '--user' => 'ghost@example.com'])
        ->expectsOutputToContain('not found')
        ->assertSuccessful();

    expect(BenchmarkRun::first()->user_id)->toBeNull();
});

test('unregistered suites keep the run but warn about skipped rows', function () {
    $path = storage_path('app/test-benchmarks.db');
    festbench_fixture_db($path);
    $pdo = new PDO('sqlite:'.$path);
    $pdo->exec("INSERT INTO runs (run_id, ts, suite, model_id, label) VALUES ('ddd000111222', '2026-09-04T10:00:00', 'depthsweep', 'fixture/model', 'unknown suite')");
    $pdo->exec("INSERT INTO results (run_id, suite, task, level, ok, wall) VALUES ('ddd000111222', 'depthsweep', 'depth_task', 'off', 1, 5.0)");

    $this->artisan(FestbenchImport::class, ['path' => $path])->assertSuccessful();

    expect(BenchmarkRun::where('run_id', 'ddd000111222')->exists())->toBeTrue();
});