pch/festbench-web
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

v2.2: depthsweep suite (table, model, registry, run-detail branch), band column headers, GpuSeeder in DatabaseSeeder, device-flow + catalog + confirm-picker tests

Browse source
This commit is contained in:
PeterChrz 2026-09-19 21:11:20 -04:00
parent 646cecc403
commit 3695632e58
Signed by untrusted user who does not match committer: pch
GPG key ID: 8F0826ECF7302C63
11 changed files with 285 additions and 4 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
app/Models/BenchmarkRun.php
View file

@ -101,6 +101,11 @@ public function scenarioRuns(): HasMany
return $this->hasMany(ScenarioRun::class);
}
public function depthsweepResults(): HasMany
{
return $this->hasMany(DepthsweepResult::class);
}
/**
* The suite-specific result rows for this run, via the registry. Returns
* an empty collection for suites this run does not have.

19
app/Models/DepthsweepResult.php Normal file
View file

@ -0,0 +1,19 @@
<?php
namespace App\Models;
use Database\Factories\DepthsweepResultFactory;
use Illuminate\Database\Eloquent\Factories\HasFactory;
class DepthsweepResult extends SuiteResult
{
/** @use HasFactory<DepthsweepResultFactory> */
use HasFactory;
protected $fillable = [
'benchmark_run_id', 'ran_at', 'task', 'level',
'ok', 'decode', 'user_tps', 'ttft', 't_answer', 'wall',
'reasoning', 'visible', 'total', 'finish', 'detail',
'section', 'payload',
];
}

2
app/Support/Suites.php
View file

@ -2,6 +2,7 @@
namespace App\Support;
use App\Models\DepthsweepResult;
use App\Models\InfrabenchResult;
use App\Models\ProsebenchResult;
use App\Models\PsychbenchResult;
@ -22,6 +23,7 @@ class Suites
'infrabench' => InfrabenchResult::class,
'prosebench' => ProsebenchResult::class,
'psychbench' => PsychbenchResult::class,
'depthsweep' => DepthsweepResult::class,
'scenariobench' => ScenarioRun::class,
];

7
config/festbench.php
View file

@ -1,5 +1,6 @@
<?php
use App\Models\DepthsweepResult;
use App\Models\InfrabenchResult;
use App\Models\ProsebenchResult;
use App\Models\PsychbenchResult;
@ -38,6 +39,12 @@
'active' => true,
'in_all_results' => true,
],
'depthsweep' => [
'model' => DepthsweepResult::class,
'active' => true,
// ttft/wall are meaningful in the combined view; tok/s stays null.
'in_all_results' => true,
],
'scenariobench' => [
'model' => ScenarioRun::class,
'active' => true,

35
database/factories/DepthsweepResultFactory.php Normal file
View file

@ -0,0 +1,35 @@
<?php
namespace Database\Factories;
use App\Models\BenchmarkRun;
use App\Models\DepthsweepResult;
use Illuminate\Database\Eloquent\Factories\Factory;
/** @extends Factory<DepthsweepResult> */
class DepthsweepResultFactory extends Factory
{
public function definition(): array
{
// depthsweep: needle-in-haystack at a depth point — no tok/s columns.
return [
'benchmark_run_id' => BenchmarkRun::factory(),
'ran_at' => now(),
'task' => $this->faker->randomElement(['housing_ref', 'quote_price']),
'level' => $this->faker->randomElement(['off', 'low', 'medium', 'xhigh']),
'ok' => $this->faker->boolean(80),
'decode' => null,
'user_tps' => null,
'ttft' => round($this->faker->randomFloat(2, 60, 90), 2),
't_answer' => null,
'wall' => round($this->faker->randomFloat(2, 60, 95), 2),
'reasoning' => null,
'visible' => $this->faker->numberBetween(2, 10),
'total' => null,
'finish' => null,
'detail' => 'BX-'.$this->faker->numberBetween(1000, 9999),
'section' => 'd'.$this->faker->randomElement(['0.1', '0.3', '0.5', '0.7', '0.9']),
'payload' => ['ok' => true],
];
}
}

6
database/migrations/2026_09_18_000001_reorganize_suite_tables.php
View file

@ -54,6 +54,9 @@ public function up(): void
'infrabench_results' => ['kind VARCHAR(32)'],
'prosebench_results' => ['section VARCHAR(64)', 'checks_passed INTEGER', 'checks_total INTEGER', 'judge TEXT', 'prefill DOUBLE PRECISION', 'prompt_tokens INTEGER'],
'psychbench_results' => ['section VARCHAR(64)', 'checks_passed INTEGER', 'checks_total INTEGER'],
// depthsweep: section holds the depth point (d0.1..d0.9); only
// ttft/wall/visible carry metrics — tok/s columns stay null.
'depthsweep_results' => ['section VARCHAR(64)'],
];
foreach ($suiteExtras as $table => $extras) {
@ -218,7 +221,7 @@ private function createSqliteTable(string $table, array $extras): void
*/
private function createPartitions(): void
{
$tables = ['softbench_results', 'infrabench_results', 'prosebench_results', 'psychbench_results', 'scenario_runs'];
$tables = ['softbench_results', 'infrabench_results', 'prosebench_results', 'psychbench_results', 'depthsweep_results', 'scenario_runs'];
$start = now()->subYear()->startOfMonth();
$end = now()->addYear()->endOfMonth();
@ -237,6 +240,7 @@ private function createPartitions(): void
public function down(): void
{
Schema::dropIfExists('scenario_runs');
Schema::dropIfExists('depthsweep_results');
Schema::dropIfExists('psychbench_results');
Schema::dropIfExists('prosebench_results');
Schema::dropIfExists('infrabench_results');

4
database/seeders/DatabaseSeeder.php
View file

@ -19,5 +19,9 @@ public function run(): void
'name' => 'Test User',
'email' => 'test@example.com',
]);
$this->call([
GpuSeeder::class,
]);
}
}

4
resources/views/livewire/my-runs/index.blade.php
View file

@ -68,8 +68,8 @@ public function confirmGpu(int $gpuId): void
$run->fill(['gpu' => $gpu->name, 'gpu_id' => $gpu->id])->save();
// Historical regroup: the run moves into the correct session band.
$this->call('festbench:summary');
$this->call('festbench:snapshot');
Illuminate\Support\Facades\Artisan::call('festbench:summary');
Illuminate\Support\Facades\Artisan::call('festbench:snapshot');
$this->modal('gpu-confirm-modal')->close();
$this->confirmingRunId = null;

36
resources/views/livewire/runs/show.blade.php
View file

@ -123,7 +123,7 @@ private function authorizeOwner(): void
@endif
<div>
<flux:heading size="md">{{ $this->run->suite === 'scenariobench' ? __('Scenario results') : __('Task results') }}</flux:heading>
<flux:heading size="md">{{ $this->run->suite === 'scenariobench' ? __('Scenario results') : ($this->run->suite === 'depthsweep' ? __('Depth sweep results') : __('Task results')) }}</flux:heading>
<div class="mt-3">
@if ($this->run->suite === 'scenariobench')
<flux:table>
@ -169,6 +169,40 @@ private function authorizeOwner(): void
@endforeach
</flux:table.rows>
</flux:table>
@elseif ($this->run->suite === 'depthsweep')
<flux:table>
<flux:table.columns>
<flux:table.column>{{ __('Depth') }}</flux:table.column>
<flux:table.column>{{ __('Task') }}</flux:table.column>
<flux:table.column>{{ __('Level') }}</flux:table.column>
<flux:table.column>{{ __('Found') }}</flux:table.column>
<flux:table.column>{{ __('Needle') }}</flux:table.column>
<flux:table.column class="text-right">{{ __('TTFT') }}</flux:table.column>
<flux:table.column class="text-right">{{ __('Wall') }}</flux:table.column>
<flux:table.column class="text-right">{{ __('Visible') }}</flux:table.column>
</flux:table.columns>
<flux:table.rows>
@foreach ($this->run->suiteResults() as $result)
<flux:table.row :key="$result->id">
<flux:table.cell><flux:badge size="xs" color="sky">{{ $result->section }}</flux:badge></flux:table.cell>
<flux:table.cell>{{ $result->task }}</flux:table.cell>
<flux:table.cell>{{ $result->level ?? '—' }}</flux:table.cell>
<flux:table.cell>
@if ($result->ok)
<flux:icon name="check-circle" variant="solid" class="size-4 text-emerald-500" />
@else
<flux:icon name="x-circle" variant="solid" class="size-4 text-red-500" />
@endif
</flux:table.cell>
<flux:table.cell class="font-mono text-xs">{{ $result->detail ?? '—' }}</flux:table.cell>
<flux:table.cell class="text-right font-mono text-sm">{{ number_format($result->ttft, 2) }}</flux:table.cell>
<flux:table.cell class="text-right font-mono text-sm">{{ number_format($result->wall, 2) }}</flux:table.cell>
<flux:table.cell class="text-right font-mono text-sm">{{ number_format($result->visible) }}</flux:table.cell>
</flux:table.row>
@endforeach
</flux:table.rows>
</flux:table>
@else
<flux:table>
<flux:table.columns>

8
resources/views/livewire/summary/index.blade.php
View file

@ -158,6 +158,14 @@ class="w-64"
@if ($session->band->label)
<div class="px-4 pt-2 text-xs text-zinc-500">{{ $session->band->label }}</div>
@endif
{{-- Column titles aligned over the suite rows (md+ only) --}}
<div class="hidden px-4 pt-2 md:flex flex-wrap items-center gap-x-4 gap-y-1">
<span class="w-24 text-[10px] font-semibold uppercase tracking-wider text-zinc-400">{{ __('Benchmark') }}</span>
<span class="w-14 text-[10px] font-semibold uppercase tracking-wider text-zinc-400">{{ __('Thinking') }}</span>
<span class="w-24 text-[10px] font-semibold uppercase tracking-wider text-zinc-400">{{ __('Result') }}</span>
<span class="w-40 text-[10px] font-semibold uppercase tracking-wider text-zinc-400">{{ __('Speed (tok/s)') }} <span class="font-normal normal-case">avg · wtd</span></span>
<span class="w-20 text-[10px] font-semibold uppercase tracking-wider text-zinc-400">{{ __('Tokens') }}</span>
</div>
<div class="divide-y divide-zinc-100 dark:divide-zinc-800">
@foreach ($session->suites as $row)
<a href="{{ route('runs.show', $row->run_id) }}" wire:navigate

163
tests/Feature/CliLoginTest.php Normal file
View file

@ -0,0 +1,163 @@
<?php
use App\Models\BenchmarkRun;
use App\Models\DepthsweepResult;
use App\Models\DeviceCode;
use App\Models\Gpu;
use App\Models\SoftbenchResult;
use App\Models\User;
use Illuminate\Support\Facades\Artisan;
use Illuminate\Support\Facades\DB;
use Livewire\Volt\Volt;
function start_device_flow(): array
{
$response = test()->postJson('/api/cli/device');
$response->assertStatus(201)
->assertJsonStructure(['code', 'poll_token', 'verify_url', 'expires_in']);
return $response->json();
}
test('the device flow grants a one-time usable token', function () {
$me = User::factory()->create();
$this->actingAs($me);
$start = start_device_flow();
// Poll before confirming: pending
$this->postJson('/api/cli/poll', ['code' => $start['code'], 'poll_token' => $start['poll_token']])
->assertOk()->assertJson(['status' => 'pending']);
// Confirm via the authorize screen logic (the grant path the page runs)
$device = DeviceCode::where('code', $start['code'])->first();
$plain = $me->createToken('festbench-cli', ['gpus:read', 'results:submit'], now()->addYear());
$device->forceFill([
'user_id' => $me->id,
'personal_access_token_id' => $plain->accessToken->id,
'plain_token' => $plain->plainTextToken,
])->save();
// Poll after: granted with a usable id|secret token
$granted = $this->postJson('/api/cli/poll', ['code' => $start['code'], 'poll_token' => $start['poll_token']])
->assertOk()->assertJson(['status' => 'granted'])
->json();
expect($granted['token'])->toContain('|')
->and($granted['abilities'])->toContain('gpus:read');
// The token authenticates against the API
$this->withToken($granted['token'])->getJson('/api/me')
->assertOk()->assertJson(['id' => $me->id, 'email' => $me->email]);
$this->withToken($granted['token'])->getJson('/api/gpus')->assertOk();
// Replay: bare granted, no second token handout
$replay = $this->postJson('/api/cli/poll', ['code' => $start['code'], 'poll_token' => $start['poll_token']])
->assertOk()->assertJson(['status' => 'granted'])->json();
expect($replay)->not->toHaveKey('token');
// The secret is erased server-side after pickup
expect(DeviceCode::where('code', $start['code'])->first()->plain_token)->toBeNull();
});
test('poll with a wrong token is rejected without an oracle', function () {
$start = start_device_flow();
$this->postJson('/api/cli/poll', ['code' => $start['code'], 'poll_token' => 'wrong-token-entirely'])
->assertStatus(404);
$this->postJson('/api/cli/poll', ['code' => 'NOPE-NOPE', 'poll_token' => $start['poll_token']])
->assertStatus(404);
});
test('expired codes report expired on poll', function () {
$start = start_device_flow();
DeviceCode::where('code', $start['code'])->update(['expires_at' => now()->subMinute()]);
$this->postJson('/api/cli/poll', ['code' => $start['code'], 'poll_token' => $start['poll_token']])
->assertOk()->assertJson(['status' => 'expired']);
});
test('the gpu catalog requires authentication and the gpus:read ability', function () {
// Guest: redirected to login (no token at all)
$this->getJson('/api/gpus')->assertStatus(401);
// Token with the wrong ability
$me = User::factory()->create();
$limited = $me->createToken('no-gpus', ['results:submit'], now()->addYear());
$this->withToken($limited->plainTextToken)->getJson('/api/gpus')->assertStatus(403);
});
test('the catalog returns only active gpus', function () {
$me = User::factory()->create();
$token = $me->createToken('reader', ['gpus:read'], now()->addYear());
// Seed the catalog for this test (RefreshDatabase resets between tests)
Artisan::call('db:seed', ['--class' => 'GpuSeeder', '--force' => true]);
// Deactivate one entry
DB::table('gpus')->where('name', 'AMD Radeon RX 7900 XTX')->update(['active' => false]);
$response = $this->withToken($token->plainTextToken)->getJson('/api/gpus')->assertOk()->json();
$names = array_column($response['gpus'], 'name');
expect($names)->toContain('AMD Radeon AI PRO R9700')
->not->toContain('AMD Radeon RX 7900 XTX');
});
test('gpu confirm on my-runs links the run and regroups it', function () {
Artisan::call('db:seed', ['--class' => 'GpuSeeder', '--force' => true]);
$me = User::factory()->create();
// A run whose raw gpu string missed the catalog (typo case)
$run = BenchmarkRun::factory()->create([
'suite' => 'softbench', 'user_id' => $me->id, 'gpu' => 'amd radeon ai pro r9700', 'gpu_id' => null,
]);
SoftbenchResult::factory()->create(['benchmark_run_id' => $run->id]);
Artisan::call('festbench:summary');
// Owner confirms via the picker flow
$this->actingAs($me);
Volt::test('my-runs.index')
->call('startGpuConfirm', $run->id)
->call('confirmGpu', Gpu::match('AMD Radeon AI PRO R9700')->id);
$run->refresh();
expect($run->gpu_id)->not->toBeNull()
->and($run->gpu)->toBe('AMD Radeon AI PRO R9700');
});
test('depthsweep results import and aggregate without tok/s', function () {
$run = BenchmarkRun::factory()->create(['suite' => 'depthsweep', 'visibility' => 'public']);
DepthsweepResult::factory()->count(5)->create(['benchmark_run_id' => $run->id]);
Artisan::call('festbench:summary');
$row = DB::table('session_suite_rows')->where('suite', 'depthsweep')->first();
expect($row->passed)->toBeInt()
->and($row->avg_user_tps)->toBeNull()
->and($row->total_tokens)->toBe(0);
// Run detail renders the depthsweep column set
$this->get(route('runs.show', $run->run_id))->assertOk()
->assertSee('Depth sweep results')
->assertSee('Needle');
});
test('session bands show column titles', function () {
$run = BenchmarkRun::factory()->create(['suite' => 'softbench', 'visibility' => 'public']);
SoftbenchResult::factory()->create(['benchmark_run_id' => $run->id]);
Artisan::call('festbench:summary');
$this->get(route('home'))->assertOk()
->assertSee('Benchmark', false)
->assertSee('Thinking', false)
->assertSee('Speed (tok/s)', false)
->assertSee('Tokens', false);
});