161 lines
8 KiB
PHP
161 lines
8 KiB
PHP
<?php
|
|
|
|
use App\Console\Commands\FestbenchImport;
|
|
use App\Models\BenchmarkRun;
|
|
use App\Models\ScenarioRun;
|
|
use App\Models\SoftbenchResult;
|
|
use App\Models\User;
|
|
use Illuminate\Support\Facades\Artisan;
|
|
|
|
function festbench_fixture_db(string $path): void
|
|
{
|
|
if (file_exists($path)) {
|
|
unlink($path);
|
|
}
|
|
|
|
// festbench's real schemas, from llm_benchmark/benchlib.py
|
|
$pdo = new PDO('sqlite:'.$path);
|
|
$pdo->exec(<<<'SQL'
|
|
CREATE TABLE runs (
|
|
run_id TEXT PRIMARY KEY, ts TEXT, suite TEXT, run_key TEXT,
|
|
endpoint TEXT, model_id TEXT, model_quant TEXT, model_path TEXT,
|
|
engine TEXT, engine_build TEXT, n_ctx INTEGER, slots INTEGER,
|
|
gpu TEXT, gpu_count INTEGER, gpu_source TEXT, host TEXT, client TEXT, label TEXT,
|
|
config TEXT, notes TEXT,
|
|
model_quant_raw TEXT, image_digest TEXT, validators TEXT, suite_versions TEXT, harness TEXT, harness_ver TEXT, harness_mode TEXT);
|
|
CREATE TABLE results (
|
|
run_id TEXT, suite TEXT, section TEXT, task TEXT, level TEXT,
|
|
ok INTEGER, decode REAL, user_tps REAL, ttft REAL, t_answer REAL,
|
|
wall REAL, reasoning INTEGER, visible INTEGER, total INTEGER,
|
|
finish TEXT, detail TEXT, tool_calls INTEGER);
|
|
CREATE TABLE scenario_runs (
|
|
run_id TEXT, scenario TEXT, level TEXT, ok INTEGER, outcome TEXT,
|
|
wall REAL, turns INTEGER, commands INTEGER, repeats INTEGER,
|
|
submissions INTEGER, rejected INTEGER, t_first_sub REAL, n_first_sub INTEGER,
|
|
t_pass REAL, n_pass INTEGER, gen_tokens INTEGER, vis_tokens INTEGER,
|
|
rea_tokens INTEGER, verdicts TEXT, detail TEXT);
|
|
SQL);
|
|
|
|
$insertRun = $pdo->prepare('INSERT INTO runs (run_id, ts, suite, model_id, model_quant, engine, gpu, gpu_count, label) VALUES (?,?,?,?,?,?,?,?,?)');
|
|
$insertRun->execute(['aaa111222333', '2026-09-01T10:00:00', 'softbench', 'fixture/model', 'Q5_K_M', 'llama.cpp', 'R9700', 1, 'deterministic sampling, repeat 1']);
|
|
$insertRun->execute(['bbb444555666', '2026-09-02T11:00:00', 'scenariobench', 'fixture/model', null, 'vllm', 'unknown', 0, 'agent test']);
|
|
$insertRun->execute(['ccc777888999', '2026-09-03T12:00:00', 'softbench', 'fixture/model', null, 'llama.cpp', 'R9700', 1, 'other session']);
|
|
|
|
$insertResult = $pdo->prepare('INSERT INTO results (run_id, suite, section, task, level, ok, decode, user_tps, ttft, t_answer, wall, reasoning, visible, total, finish, detail, tool_calls) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)');
|
|
$insertResult->execute(['aaa111222333', 'softbench', 'code', 'first_occurrence', 'off', 1, 83.82, 61.14, 0.47, 0.47, 1.77, 0, 108, 109, 'stop', '', 2]);
|
|
$insertResult->execute(['aaa111222333', 'softbench', 'code', 'merge_intervals', 'n/a', 0, 51.2, 40.0, 0.5, 0.5, 3.0, 250, 90, 340, 'length', 'assertion failed', null]);
|
|
$insertResult->execute(['ccc777888999', 'softbench', 'code', 'other_session_task', 'off', 1, 70, 50, 0.4, 0.4, 2.0, 0, 100, 100, 'stop', '', null]);
|
|
|
|
$insertScenario = $pdo->prepare('INSERT INTO scenario_runs (run_id, scenario, level, ok, outcome, wall, turns, commands, repeats, submissions, rejected, t_first_sub, n_first_sub, t_pass, n_pass, gen_tokens, vis_tokens, rea_tokens, verdicts, detail) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)');
|
|
$insertScenario->execute(['bbb444555666', 'kubelet_config_repair', 'off', 1, 'passed', 264.1, 68, 67, 2, 1, 0, 262.5, 68, 264.1, 68, 2543, 2475, 0, '[{"n":1,"at":264.1,"accepted":true}]', 'passed on submission 1']);
|
|
}
|
|
|
|
beforeEach(fn () => Artisan::call('festbench:suites:refresh'));
|
|
|
|
test('festbench:import routes rows to per-suite tables', function () {
|
|
$path = storage_path('app/test-benchmarks.db');
|
|
festbench_fixture_db($path);
|
|
|
|
$this->artisan(FestbenchImport::class, ['path' => $path])->assertSuccessful();
|
|
|
|
expect(BenchmarkRun::count())->toBe(3)
|
|
->and(SoftbenchResult::count())->toBe(3)
|
|
->and(ScenarioRun::count())->toBe(1);
|
|
|
|
$run = BenchmarkRun::where('run_id', 'aaa111222333')->first();
|
|
expect($run->suite)->toBe('softbench')
|
|
->and($run->model_quant)->toBe('Q5_K_M')
|
|
->and($run->visibility)->toBe('public');
|
|
|
|
$failed = SoftbenchResult::where('task', 'merge_intervals')->first();
|
|
expect($failed->ok)->toBeFalse()
|
|
->and($failed->finish)->toBe('length')
|
|
->and($failed->detail)->toBe('assertion failed')
|
|
// n/a levels normalize to null
|
|
->and($failed->level)->toBeNull()
|
|
// complete source row preserved
|
|
->and($failed->payload['total'])->toBe(340);
|
|
|
|
$toolUser = SoftbenchResult::where('task', 'first_occurrence')->first();
|
|
expect($toolUser->tool_calls)->toBe(2);
|
|
|
|
$scenario = ScenarioRun::first();
|
|
expect($scenario->scenario)->toBe('kubelet_config_repair')
|
|
->and($scenario->outcome)->toBe('passed')
|
|
->and($scenario->turns)->toBe(68)
|
|
->and($scenario->verdicts)->toBeArray()
|
|
->and($scenario->t_pass)->toBe(264.1);
|
|
});
|
|
|
|
test('festbench:import --label imports only the matching session', function () {
|
|
$path = storage_path('app/test-benchmarks.db');
|
|
festbench_fixture_db($path);
|
|
|
|
$this->artisan(FestbenchImport::class, ['path' => $path, '--label' => 'deterministic sampling, repeat 1'])->assertSuccessful();
|
|
|
|
expect(BenchmarkRun::count())->toBe(1)
|
|
->and(BenchmarkRun::first()->run_id)->toBe('aaa111222333')
|
|
->and(SoftbenchResult::count())->toBe(2);
|
|
});
|
|
|
|
test('festbench:import is idempotent', function () {
|
|
$path = storage_path('app/test-benchmarks.db');
|
|
festbench_fixture_db($path);
|
|
|
|
$this->artisan(FestbenchImport::class, ['path' => $path])->assertSuccessful();
|
|
$this->artisan(FestbenchImport::class, ['path' => $path])->assertSuccessful();
|
|
|
|
expect(BenchmarkRun::count())->toBe(3)
|
|
->and(SoftbenchResult::count())->toBe(3);
|
|
});
|
|
|
|
test('festbench:import --fresh purges existing data first', function () {
|
|
$path = storage_path('app/test-benchmarks.db');
|
|
festbench_fixture_db($path);
|
|
|
|
// Pre-existing noise
|
|
BenchmarkRun::factory()->has(SoftbenchResult::factory(), 'softbenchResults')->create();
|
|
|
|
$this->artisan(FestbenchImport::class, ['path' => $path, '--fresh' => true, '--no-interaction' => true])->assertSuccessful();
|
|
|
|
expect(BenchmarkRun::count())->toBe(3)
|
|
->and(BenchmarkRun::pluck('run_id')->sort()->values()->all())->toBe(['aaa111222333', 'bbb444555666', 'ccc777888999']);
|
|
});
|
|
|
|
test('festbench:import attributes runs to a user and can default to private', function () {
|
|
$path = storage_path('app/test-benchmarks.db');
|
|
festbench_fixture_db($path);
|
|
$me = User::factory()->create();
|
|
|
|
$this->artisan(FestbenchImport::class, ['path' => $path, '--user' => $me->email, '--private' => true])->assertSuccessful();
|
|
|
|
$run = BenchmarkRun::where('run_id', 'aaa111222333')->first();
|
|
expect($run->user_id)->toBe($me->id)
|
|
->and($run->visibility)->toBe('private');
|
|
|
|
expect(BenchmarkRun::visibleTo(null)->count())->toBe(0)
|
|
->and(BenchmarkRun::visibleTo($me)->count())->toBe(3);
|
|
});
|
|
|
|
test('festbench:import warns when the user does not exist', function () {
|
|
$path = storage_path('app/test-benchmarks.db');
|
|
festbench_fixture_db($path);
|
|
|
|
$this->artisan(FestbenchImport::class, ['path' => $path, '--user' => 'ghost@example.com'])
|
|
->expectsOutputToContain('not found')
|
|
->assertSuccessful();
|
|
|
|
expect(BenchmarkRun::first()->user_id)->toBeNull();
|
|
});
|
|
|
|
test('unregistered suites keep the run but warn about skipped rows', function () {
|
|
$path = storage_path('app/test-benchmarks.db');
|
|
festbench_fixture_db($path);
|
|
$pdo = new PDO('sqlite:'.$path);
|
|
$pdo->exec("INSERT INTO runs (run_id, ts, suite, model_id, label) VALUES ('ddd000111222', '2026-09-04T10:00:00', 'depthsweep', 'fixture/model', 'unknown suite')");
|
|
$pdo->exec("INSERT INTO results (run_id, suite, task, level, ok, wall) VALUES ('ddd000111222', 'depthsweep', 'depth_task', 'off', 1, 5.0)");
|
|
|
|
$this->artisan(FestbenchImport::class, ['path' => $path])->assertSuccessful();
|
|
|
|
expect(BenchmarkRun::where('run_id', 'ddd000111222')->exists())->toBeTrue();
|
|
});
|