festbench-web/tests/Feature/FestbenchImportTest.php

<?php

use App\Console\Commands\FestbenchImport;
use App\Models\BenchmarkRun;
use App\Models\RunResult;
use App\Models\User;

function festbench_fixture_db(string $path): void
{
    if (file_exists($path)) {
        unlink($path);
    }

    // festbench's real schema, from llm_benchmark/benchlib.py
    $pdo = new PDO('sqlite:'.$path);
    $pdo->exec(<<<'SQL'
        CREATE TABLE runs (
          run_id TEXT PRIMARY KEY, ts TEXT, suite TEXT, run_key TEXT,
          endpoint TEXT, model_id TEXT, model_quant TEXT, model_path TEXT,
          engine TEXT, engine_build TEXT, n_ctx INTEGER, slots INTEGER,
          gpu TEXT, gpu_count INTEGER, gpu_source TEXT, host TEXT, client TEXT, label TEXT,
          config TEXT, notes TEXT,
          model_quant_raw TEXT, image_digest TEXT, validators TEXT, suite_versions TEXT, harness TEXT, harness_ver TEXT, harness_mode TEXT);
        CREATE TABLE results (
          run_id TEXT, suite TEXT, section TEXT, task TEXT, level TEXT,
          ok INTEGER, decode REAL, user_tps REAL, ttft REAL, t_answer REAL,
          wall REAL, reasoning INTEGER, visible INTEGER, total INTEGER,
          finish TEXT, detail TEXT
        , tool_calls INTEGER);
    SQL);

    $insertRun = $pdo->prepare('INSERT INTO runs (run_id, ts, suite, model_id, model_quant, engine, gpu, gpu_count, label) VALUES (?,?,?,?,?,?,?,?,?)');
    $insertRun->execute(['aaa111222333', '2026-09-01T10:00:00', 'softbench', 'fixture/model', 'Q5_K_M', 'llama.cpp', 'R9700', 1, 'fixture-label']);
    $insertRun->execute(['bbb444555666', '2026-09-02T11:00:00', 'infrabench', 'fixture/model', null, 'vllm', 'unknown', 0, null]);

    $insertResult = $pdo->prepare('INSERT INTO results (run_id, suite, section, task, level, ok, decode, user_tps, ttft, t_answer, wall, reasoning, visible, total, finish, detail) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)');
    $insertResult->execute(['aaa111222333', 'softbench', 'code', 'first_occurrence', 'off', 1, 83.82, 61.14, 0.47, 0.47, 1.77, 0, 108, 109, 'stop', '']);
    $insertResult->execute(['aaa111222333', 'softbench', 'code', 'merge_intervals', 'off', 0, 51.2, 40.0, 0.5, 0.5, 3.0, 250, 90, 340, 'length', 'assertion failed']);
    $insertResult->execute(['bbb444555666', 'infrabench', null, 'modprobe_exists', 'medium', 1, 40.0, 30.0, 1.2, 1.2, 10.0, 0, 200, 200, 'stop', '']);
}

test('festbench:import imports runs and results from a real-schema sqlite db', function () {
    $path = storage_path('app/test-benchmarks.db');
    festbench_fixture_db($path);

    $this->artisan(FestbenchImport::class, ['path' => $path])->assertSuccessful();

    expect(BenchmarkRun::count())->toBe(2)
        ->and(RunResult::count())->toBe(3);

    $run = BenchmarkRun::where('run_id', 'aaa111222333')->first();
    expect($run->suite)->toBe('softbench')
        ->and($run->model_id)->toBe('fixture/model')
        ->and($run->model_quant)->toBe('Q5_K_M')
        ->and($run->ran_at->format('Y-m-d H:i'))->toBe('2026-09-01 10:00')
        ->and($run->visibility)->toBe('public')
        ->and($run->results)->toHaveCount(2);

    $failed = RunResult::where('task', 'merge_intervals')->first();
    expect($failed->ok)->toBeFalse()
        ->and($failed->finish)->toBe('length')
        ->and($failed->detail)->toBe('assertion failed')
        // full source row preserved
        ->and($failed->payload['total'])->toBe(340);

    // null quant and empty detail survive the import
    $second = BenchmarkRun::where('run_id', 'bbb444555666')->first();
    expect($second->model_quant)->toBeNull()
        ->and($second->results->first()->detail)->toBeNull();
});

test('festbench:import is idempotent', function () {
    $path = storage_path('app/test-benchmarks.db');
    festbench_fixture_db($path);

    $this->artisan(FestbenchImport::class, ['path' => $path])->assertSuccessful();
    $this->artisan(FestbenchImport::class, ['path' => $path])->assertSuccessful();

    expect(BenchmarkRun::count())->toBe(2)
        ->and(RunResult::count())->toBe(3);
});

test('festbench:import attributes runs to a user and can default to private', function () {
    $path = storage_path('app/test-benchmarks.db');
    festbench_fixture_db($path);
    $me = User::factory()->create();

    $this->artisan(FestbenchImport::class, ['path' => $path, '--user' => $me->email, '--private' => true])->assertSuccessful();

    $run = BenchmarkRun::where('run_id', 'aaa111222333')->first();
    expect($run->user_id)->toBe($me->id)
        ->and($run->visibility)->toBe('private');

    // guest cannot see it, owner can
    expect(BenchmarkRun::visibleTo(null)->count())->toBe(0)
        ->and(BenchmarkRun::visibleTo($me)->count())->toBe(2);
});

test('festbench:import warns when the user does not exist', function () {
    $path = storage_path('app/test-benchmarks.db');
    festbench_fixture_db($path);

    $this->artisan(FestbenchImport::class, ['path' => $path, '--user' => 'ghost@example.com'])
        ->expectsOutputToContain('not found')
        ->assertSuccessful();

    expect(BenchmarkRun::first()->user_id)->toBeNull();
});