v2.2: depthsweep suite (table, model, registry, run-detail branch), band column headers, GpuSeeder in DatabaseSeeder, device-flow + catalog + confirm-picker tests

app/Models/BenchmarkRun.php

@@ -101,6 +101,11 @@ public function scenarioRuns(): HasMany
         return $this->hasMany(ScenarioRun::class);
     }
 
+    public function depthsweepResults(): HasMany
+    {
+        return $this->hasMany(DepthsweepResult::class);
+    }
+
     /**
      * The suite-specific result rows for this run, via the registry. Returns
      * an empty collection for suites this run does not have.

app/Models/DepthsweepResult.php

@@ -0,0 +1,19 @@
+<?php
+
+namespace App\Models;
+
+use Database\Factories\DepthsweepResultFactory;
+use Illuminate\Database\Eloquent\Factories\HasFactory;
+
+class DepthsweepResult extends SuiteResult
+{
+    /** @use HasFactory<DepthsweepResultFactory> */
+    use HasFactory;
+
+    protected $fillable = [
+        'benchmark_run_id', 'ran_at', 'task', 'level',
+        'ok', 'decode', 'user_tps', 'ttft', 't_answer', 'wall',
+        'reasoning', 'visible', 'total', 'finish', 'detail',
+        'section', 'payload',
+    ];
+}

app/Support/Suites.php

@@ -2,6 +2,7 @@
 
 namespace App\Support;
 
+use App\Models\DepthsweepResult;
 use App\Models\InfrabenchResult;
 use App\Models\ProsebenchResult;
 use App\Models\PsychbenchResult;
@@ -22,6 +23,7 @@ class Suites
         'infrabench' => InfrabenchResult::class,
         'prosebench' => ProsebenchResult::class,
         'psychbench' => PsychbenchResult::class,
+        'depthsweep' => DepthsweepResult::class,
         'scenariobench' => ScenarioRun::class,
     ];
 

config/festbench.php

@@ -1,5 +1,6 @@
 <?php
 
+use App\Models\DepthsweepResult;
 use App\Models\InfrabenchResult;
 use App\Models\ProsebenchResult;
 use App\Models\PsychbenchResult;
@@ -38,6 +39,12 @@
             'active' => true,
             'in_all_results' => true,
         ],
+        'depthsweep' => [
+            'model' => DepthsweepResult::class,
+            'active' => true,
+            // ttft/wall are meaningful in the combined view; tok/s stays null.
+            'in_all_results' => true,
+        ],
         'scenariobench' => [
             'model' => ScenarioRun::class,
             'active' => true,

database/factories/DepthsweepResultFactory.php

@@ -0,0 +1,35 @@
+<?php
+
+namespace Database\Factories;
+
+use App\Models\BenchmarkRun;
+use App\Models\DepthsweepResult;
+use Illuminate\Database\Eloquent\Factories\Factory;
+
+/** @extends Factory<DepthsweepResult> */
+class DepthsweepResultFactory extends Factory
+{
+    public function definition(): array
+    {
+        // depthsweep: needle-in-haystack at a depth point — no tok/s columns.
+        return [
+            'benchmark_run_id' => BenchmarkRun::factory(),
+            'ran_at' => now(),
+            'task' => $this->faker->randomElement(['housing_ref', 'quote_price']),
+            'level' => $this->faker->randomElement(['off', 'low', 'medium', 'xhigh']),
+            'ok' => $this->faker->boolean(80),
+            'decode' => null,
+            'user_tps' => null,
+            'ttft' => round($this->faker->randomFloat(2, 60, 90), 2),
+            't_answer' => null,
+            'wall' => round($this->faker->randomFloat(2, 60, 95), 2),
+            'reasoning' => null,
+            'visible' => $this->faker->numberBetween(2, 10),
+            'total' => null,
+            'finish' => null,
+            'detail' => 'BX-'.$this->faker->numberBetween(1000, 9999),
+            'section' => 'd'.$this->faker->randomElement(['0.1', '0.3', '0.5', '0.7', '0.9']),
+            'payload' => ['ok' => true],
+        ];
+    }
+}

database/migrations/2026_09_18_000001_reorganize_suite_tables.php

@@ -54,6 +54,9 @@ public function up(): void
             'infrabench_results' => ['kind VARCHAR(32)'],
             'prosebench_results' => ['section VARCHAR(64)', 'checks_passed INTEGER', 'checks_total INTEGER', 'judge TEXT', 'prefill DOUBLE PRECISION', 'prompt_tokens INTEGER'],
             'psychbench_results' => ['section VARCHAR(64)', 'checks_passed INTEGER', 'checks_total INTEGER'],
+            // depthsweep: section holds the depth point (d0.1..d0.9); only
+            // ttft/wall/visible carry metrics — tok/s columns stay null.
+            'depthsweep_results' => ['section VARCHAR(64)'],
         ];
 
         foreach ($suiteExtras as $table => $extras) {
@@ -218,7 +221,7 @@ private function createSqliteTable(string $table, array $extras): void
      */
     private function createPartitions(): void
     {
-        $tables = ['softbench_results', 'infrabench_results', 'prosebench_results', 'psychbench_results', 'scenario_runs'];
+        $tables = ['softbench_results', 'infrabench_results', 'prosebench_results', 'psychbench_results', 'depthsweep_results', 'scenario_runs'];
         $start = now()->subYear()->startOfMonth();
         $end = now()->addYear()->endOfMonth();
 
@@ -237,6 +240,7 @@ private function createPartitions(): void
     public function down(): void
     {
         Schema::dropIfExists('scenario_runs');
+        Schema::dropIfExists('depthsweep_results');
         Schema::dropIfExists('psychbench_results');
         Schema::dropIfExists('prosebench_results');
         Schema::dropIfExists('infrabench_results');

database/seeders/DatabaseSeeder.php

@@ -19,5 +19,9 @@ public function run(): void
             'name' => 'Test User',
             'email' => 'test@example.com',
         ]);
+
+        $this->call([
+            GpuSeeder::class,
+        ]);
     }
 }

resources/views/livewire/my-runs/index.blade.php

@@ -68,8 +68,8 @@ public function confirmGpu(int $gpuId): void
         $run->fill(['gpu' => $gpu->name, 'gpu_id' => $gpu->id])->save();
 
         // Historical regroup: the run moves into the correct session band.
-        $this->call('festbench:summary');
+        Illuminate\Support\Facades\Artisan::call('festbench:summary');
-        $this->call('festbench:snapshot');
+        Illuminate\Support\Facades\Artisan::call('festbench:snapshot');
 
         $this->modal('gpu-confirm-modal')->close();
         $this->confirmingRunId = null;

resources/views/livewire/runs/show.blade.php

@@ -123,7 +123,7 @@ private function authorizeOwner(): void
     @endif
 
     <div>
-        <flux:heading size="md">{{ $this->run->suite === 'scenariobench' ? __('Scenario results') : __('Task results') }}</flux:heading>
+        <flux:heading size="md">{{ $this->run->suite === 'scenariobench' ? __('Scenario results') : ($this->run->suite === 'depthsweep' ? __('Depth sweep results') : __('Task results')) }}</flux:heading>
         <div class="mt-3">
             @if ($this->run->suite === 'scenariobench')
                 <flux:table>
@@ -169,6 +169,40 @@ private function authorizeOwner(): void
                         @endforeach
                     </flux:table.rows>
                 </flux:table>
+            @elseif ($this->run->suite === 'depthsweep')
+                <flux:table>
+                    <flux:table.columns>
+                        <flux:table.column>{{ __('Depth') }}</flux:table.column>
+                        <flux:table.column>{{ __('Task') }}</flux:table.column>
+                        <flux:table.column>{{ __('Level') }}</flux:table.column>
+                        <flux:table.column>{{ __('Found') }}</flux:table.column>
+                        <flux:table.column>{{ __('Needle') }}</flux:table.column>
+                        <flux:table.column class="text-right">{{ __('TTFT') }}</flux:table.column>
+                        <flux:table.column class="text-right">{{ __('Wall') }}</flux:table.column>
+                        <flux:table.column class="text-right">{{ __('Visible') }}</flux:table.column>
+                    </flux:table.columns>
+
+                    <flux:table.rows>
+                        @foreach ($this->run->suiteResults() as $result)
+                            <flux:table.row :key="$result->id">
+                                <flux:table.cell><flux:badge size="xs" color="sky">{{ $result->section }}</flux:badge></flux:table.cell>
+                                <flux:table.cell>{{ $result->task }}</flux:table.cell>
+                                <flux:table.cell>{{ $result->level ?? '—' }}</flux:table.cell>
+                                <flux:table.cell>
+                                    @if ($result->ok)
+                                        <flux:icon name="check-circle" variant="solid" class="size-4 text-emerald-500" />
+                                    @else
+                                        <flux:icon name="x-circle" variant="solid" class="size-4 text-red-500" />
+                                    @endif
+                                </flux:table.cell>
+                                <flux:table.cell class="font-mono text-xs">{{ $result->detail ?? '—' }}</flux:table.cell>
+                                <flux:table.cell class="text-right font-mono text-sm">{{ number_format($result->ttft, 2) }}</flux:table.cell>
+                                <flux:table.cell class="text-right font-mono text-sm">{{ number_format($result->wall, 2) }}</flux:table.cell>
+                                <flux:table.cell class="text-right font-mono text-sm">{{ number_format($result->visible) }}</flux:table.cell>
+                            </flux:table.row>
+                        @endforeach
+                    </flux:table.rows>
+                </flux:table>
             @else
                 <flux:table>
                     <flux:table.columns>

resources/views/livewire/summary/index.blade.php

@@ -158,6 +158,14 @@ class="w-64"
                     @if ($session->band->label)
                         <div class="px-4 pt-2 text-xs text-zinc-500">{{ $session->band->label }}</div>
                     @endif
+                    {{-- Column titles aligned over the suite rows (md+ only) --}}
+                    <div class="hidden px-4 pt-2 md:flex flex-wrap items-center gap-x-4 gap-y-1">
+                        <span class="w-24 text-[10px] font-semibold uppercase tracking-wider text-zinc-400">{{ __('Benchmark') }}</span>
+                        <span class="w-14 text-[10px] font-semibold uppercase tracking-wider text-zinc-400">{{ __('Thinking') }}</span>
+                        <span class="w-24 text-[10px] font-semibold uppercase tracking-wider text-zinc-400">{{ __('Result') }}</span>
+                        <span class="w-40 text-[10px] font-semibold uppercase tracking-wider text-zinc-400">{{ __('Speed (tok/s)') }} <span class="font-normal normal-case">avg · wtd</span></span>
+                        <span class="w-20 text-[10px] font-semibold uppercase tracking-wider text-zinc-400">{{ __('Tokens') }}</span>
+                    </div>
                     <div class="divide-y divide-zinc-100 dark:divide-zinc-800">
                         @foreach ($session->suites as $row)
                             <a href="{{ route('runs.show', $row->run_id) }}" wire:navigate

tests/Feature/CliLoginTest.php

@@ -0,0 +1,163 @@
+<?php
+
+use App\Models\BenchmarkRun;
+use App\Models\DepthsweepResult;
+use App\Models\DeviceCode;
+use App\Models\Gpu;
+use App\Models\SoftbenchResult;
+use App\Models\User;
+use Illuminate\Support\Facades\Artisan;
+use Illuminate\Support\Facades\DB;
+use Livewire\Volt\Volt;
+
+function start_device_flow(): array
+{
+    $response = test()->postJson('/api/cli/device');
+
+    $response->assertStatus(201)
+        ->assertJsonStructure(['code', 'poll_token', 'verify_url', 'expires_in']);
+
+    return $response->json();
+}
+
+test('the device flow grants a one-time usable token', function () {
+    $me = User::factory()->create();
+    $this->actingAs($me);
+
+    $start = start_device_flow();
+
+    // Poll before confirming: pending
+    $this->postJson('/api/cli/poll', ['code' => $start['code'], 'poll_token' => $start['poll_token']])
+        ->assertOk()->assertJson(['status' => 'pending']);
+
+    // Confirm via the authorize screen logic (the grant path the page runs)
+    $device = DeviceCode::where('code', $start['code'])->first();
+    $plain = $me->createToken('festbench-cli', ['gpus:read', 'results:submit'], now()->addYear());
+    $device->forceFill([
+        'user_id' => $me->id,
+        'personal_access_token_id' => $plain->accessToken->id,
+        'plain_token' => $plain->plainTextToken,
+    ])->save();
+
+    // Poll after: granted with a usable id|secret token
+    $granted = $this->postJson('/api/cli/poll', ['code' => $start['code'], 'poll_token' => $start['poll_token']])
+        ->assertOk()->assertJson(['status' => 'granted'])
+        ->json();
+
+    expect($granted['token'])->toContain('|')
+        ->and($granted['abilities'])->toContain('gpus:read');
+
+    // The token authenticates against the API
+    $this->withToken($granted['token'])->getJson('/api/me')
+        ->assertOk()->assertJson(['id' => $me->id, 'email' => $me->email]);
+
+    $this->withToken($granted['token'])->getJson('/api/gpus')->assertOk();
+
+    // Replay: bare granted, no second token handout
+    $replay = $this->postJson('/api/cli/poll', ['code' => $start['code'], 'poll_token' => $start['poll_token']])
+        ->assertOk()->assertJson(['status' => 'granted'])->json();
+
+    expect($replay)->not->toHaveKey('token');
+
+    // The secret is erased server-side after pickup
+    expect(DeviceCode::where('code', $start['code'])->first()->plain_token)->toBeNull();
+});
+
+test('poll with a wrong token is rejected without an oracle', function () {
+    $start = start_device_flow();
+
+    $this->postJson('/api/cli/poll', ['code' => $start['code'], 'poll_token' => 'wrong-token-entirely'])
+        ->assertStatus(404);
+
+    $this->postJson('/api/cli/poll', ['code' => 'NOPE-NOPE', 'poll_token' => $start['poll_token']])
+        ->assertStatus(404);
+});
+
+test('expired codes report expired on poll', function () {
+    $start = start_device_flow();
+
+    DeviceCode::where('code', $start['code'])->update(['expires_at' => now()->subMinute()]);
+
+    $this->postJson('/api/cli/poll', ['code' => $start['code'], 'poll_token' => $start['poll_token']])
+        ->assertOk()->assertJson(['status' => 'expired']);
+});
+
+test('the gpu catalog requires authentication and the gpus:read ability', function () {
+    // Guest: redirected to login (no token at all)
+    $this->getJson('/api/gpus')->assertStatus(401);
+
+    // Token with the wrong ability
+    $me = User::factory()->create();
+    $limited = $me->createToken('no-gpus', ['results:submit'], now()->addYear());
+
+    $this->withToken($limited->plainTextToken)->getJson('/api/gpus')->assertStatus(403);
+});
+
+test('the catalog returns only active gpus', function () {
+    $me = User::factory()->create();
+    $token = $me->createToken('reader', ['gpus:read'], now()->addYear());
+
+    // Seed the catalog for this test (RefreshDatabase resets between tests)
+    Artisan::call('db:seed', ['--class' => 'GpuSeeder', '--force' => true]);
+
+    // Deactivate one entry
+    DB::table('gpus')->where('name', 'AMD Radeon RX 7900 XTX')->update(['active' => false]);
+
+    $response = $this->withToken($token->plainTextToken)->getJson('/api/gpus')->assertOk()->json();
+
+    $names = array_column($response['gpus'], 'name');
+    expect($names)->toContain('AMD Radeon AI PRO R9700')
+        ->not->toContain('AMD Radeon RX 7900 XTX');
+});
+
+test('gpu confirm on my-runs links the run and regroups it', function () {
+    Artisan::call('db:seed', ['--class' => 'GpuSeeder', '--force' => true]);
+
+    $me = User::factory()->create();
+
+    // A run whose raw gpu string missed the catalog (typo case)
+    $run = BenchmarkRun::factory()->create([
+        'suite' => 'softbench', 'user_id' => $me->id, 'gpu' => 'amd radeon ai pro r9700', 'gpu_id' => null,
+    ]);
+    SoftbenchResult::factory()->create(['benchmark_run_id' => $run->id]);
+    Artisan::call('festbench:summary');
+
+    // Owner confirms via the picker flow
+    $this->actingAs($me);
+    Volt::test('my-runs.index')
+        ->call('startGpuConfirm', $run->id)
+        ->call('confirmGpu', Gpu::match('AMD Radeon AI PRO R9700')->id);
+
+    $run->refresh();
+    expect($run->gpu_id)->not->toBeNull()
+        ->and($run->gpu)->toBe('AMD Radeon AI PRO R9700');
+});
+
+test('depthsweep results import and aggregate without tok/s', function () {
+    $run = BenchmarkRun::factory()->create(['suite' => 'depthsweep', 'visibility' => 'public']);
+    DepthsweepResult::factory()->count(5)->create(['benchmark_run_id' => $run->id]);
+
+    Artisan::call('festbench:summary');
+
+    $row = DB::table('session_suite_rows')->where('suite', 'depthsweep')->first();
+    expect($row->passed)->toBeInt()
+        ->and($row->avg_user_tps)->toBeNull()
+        ->and($row->total_tokens)->toBe(0);
+
+    // Run detail renders the depthsweep column set
+    $this->get(route('runs.show', $run->run_id))->assertOk()
+        ->assertSee('Depth sweep results')
+        ->assertSee('Needle');
+});
+
+test('session bands show column titles', function () {
+    $run = BenchmarkRun::factory()->create(['suite' => 'softbench', 'visibility' => 'public']);
+    SoftbenchResult::factory()->create(['benchmark_run_id' => $run->id]);
+    Artisan::call('festbench:summary');
+
+    $this->get(route('home'))->assertOk()
+        ->assertSee('Benchmark', false)
+        ->assertSee('Thinking', false)
+        ->assertSee('Speed (tok/s)', false)
+        ->assertSee('Tokens', false);
+});