diff --git a/app/Models/BenchmarkRun.php b/app/Models/BenchmarkRun.php index 0671069..cf3f9c7 100644 --- a/app/Models/BenchmarkRun.php +++ b/app/Models/BenchmarkRun.php @@ -101,6 +101,11 @@ public function scenarioRuns(): HasMany return $this->hasMany(ScenarioRun::class); } + public function depthsweepResults(): HasMany + { + return $this->hasMany(DepthsweepResult::class); + } + /** * The suite-specific result rows for this run, via the registry. Returns * an empty collection for suites this run does not have. diff --git a/app/Models/DepthsweepResult.php b/app/Models/DepthsweepResult.php new file mode 100644 index 0000000..67cd56f --- /dev/null +++ b/app/Models/DepthsweepResult.php @@ -0,0 +1,19 @@ + */ + use HasFactory; + + protected $fillable = [ + 'benchmark_run_id', 'ran_at', 'task', 'level', + 'ok', 'decode', 'user_tps', 'ttft', 't_answer', 'wall', + 'reasoning', 'visible', 'total', 'finish', 'detail', + 'section', 'payload', + ]; +} diff --git a/app/Support/Suites.php b/app/Support/Suites.php index 40bbd3a..5f76b28 100644 --- a/app/Support/Suites.php +++ b/app/Support/Suites.php @@ -2,6 +2,7 @@ namespace App\Support; +use App\Models\DepthsweepResult; use App\Models\InfrabenchResult; use App\Models\ProsebenchResult; use App\Models\PsychbenchResult; @@ -22,6 +23,7 @@ class Suites 'infrabench' => InfrabenchResult::class, 'prosebench' => ProsebenchResult::class, 'psychbench' => PsychbenchResult::class, + 'depthsweep' => DepthsweepResult::class, 'scenariobench' => ScenarioRun::class, ]; diff --git a/config/festbench.php b/config/festbench.php index 990cb3d..b0a8697 100644 --- a/config/festbench.php +++ b/config/festbench.php @@ -1,5 +1,6 @@ true, 'in_all_results' => true, ], + 'depthsweep' => [ + 'model' => DepthsweepResult::class, + 'active' => true, + // ttft/wall are meaningful in the combined view; tok/s stays null. + 'in_all_results' => true, + ], 'scenariobench' => [ 'model' => ScenarioRun::class, 'active' => true, diff --git a/database/factories/DepthsweepResultFactory.php b/database/factories/DepthsweepResultFactory.php new file mode 100644 index 0000000..a9abaee --- /dev/null +++ b/database/factories/DepthsweepResultFactory.php @@ -0,0 +1,35 @@ + */ +class DepthsweepResultFactory extends Factory +{ + public function definition(): array + { + // depthsweep: needle-in-haystack at a depth point — no tok/s columns. + return [ + 'benchmark_run_id' => BenchmarkRun::factory(), + 'ran_at' => now(), + 'task' => $this->faker->randomElement(['housing_ref', 'quote_price']), + 'level' => $this->faker->randomElement(['off', 'low', 'medium', 'xhigh']), + 'ok' => $this->faker->boolean(80), + 'decode' => null, + 'user_tps' => null, + 'ttft' => round($this->faker->randomFloat(2, 60, 90), 2), + 't_answer' => null, + 'wall' => round($this->faker->randomFloat(2, 60, 95), 2), + 'reasoning' => null, + 'visible' => $this->faker->numberBetween(2, 10), + 'total' => null, + 'finish' => null, + 'detail' => 'BX-'.$this->faker->numberBetween(1000, 9999), + 'section' => 'd'.$this->faker->randomElement(['0.1', '0.3', '0.5', '0.7', '0.9']), + 'payload' => ['ok' => true], + ]; + } +} diff --git a/database/migrations/2026_09_18_000001_reorganize_suite_tables.php b/database/migrations/2026_09_18_000001_reorganize_suite_tables.php index 996a0d0..fcc663e 100644 --- a/database/migrations/2026_09_18_000001_reorganize_suite_tables.php +++ b/database/migrations/2026_09_18_000001_reorganize_suite_tables.php @@ -54,6 +54,9 @@ public function up(): void 'infrabench_results' => ['kind VARCHAR(32)'], 'prosebench_results' => ['section VARCHAR(64)', 'checks_passed INTEGER', 'checks_total INTEGER', 'judge TEXT', 'prefill DOUBLE PRECISION', 'prompt_tokens INTEGER'], 'psychbench_results' => ['section VARCHAR(64)', 'checks_passed INTEGER', 'checks_total INTEGER'], + // depthsweep: section holds the depth point (d0.1..d0.9); only + // ttft/wall/visible carry metrics — tok/s columns stay null. + 'depthsweep_results' => ['section VARCHAR(64)'], ]; foreach ($suiteExtras as $table => $extras) { @@ -218,7 +221,7 @@ private function createSqliteTable(string $table, array $extras): void */ private function createPartitions(): void { - $tables = ['softbench_results', 'infrabench_results', 'prosebench_results', 'psychbench_results', 'scenario_runs']; + $tables = ['softbench_results', 'infrabench_results', 'prosebench_results', 'psychbench_results', 'depthsweep_results', 'scenario_runs']; $start = now()->subYear()->startOfMonth(); $end = now()->addYear()->endOfMonth(); @@ -237,6 +240,7 @@ private function createPartitions(): void public function down(): void { Schema::dropIfExists('scenario_runs'); + Schema::dropIfExists('depthsweep_results'); Schema::dropIfExists('psychbench_results'); Schema::dropIfExists('prosebench_results'); Schema::dropIfExists('infrabench_results'); diff --git a/database/seeders/DatabaseSeeder.php b/database/seeders/DatabaseSeeder.php index d01a0ef..0c25d5a 100644 --- a/database/seeders/DatabaseSeeder.php +++ b/database/seeders/DatabaseSeeder.php @@ -19,5 +19,9 @@ public function run(): void 'name' => 'Test User', 'email' => 'test@example.com', ]); + + $this->call([ + GpuSeeder::class, + ]); } } diff --git a/resources/views/livewire/my-runs/index.blade.php b/resources/views/livewire/my-runs/index.blade.php index 9aa73c5..5a4a108 100644 --- a/resources/views/livewire/my-runs/index.blade.php +++ b/resources/views/livewire/my-runs/index.blade.php @@ -68,8 +68,8 @@ public function confirmGpu(int $gpuId): void $run->fill(['gpu' => $gpu->name, 'gpu_id' => $gpu->id])->save(); // Historical regroup: the run moves into the correct session band. - $this->call('festbench:summary'); - $this->call('festbench:snapshot'); + Illuminate\Support\Facades\Artisan::call('festbench:summary'); + Illuminate\Support\Facades\Artisan::call('festbench:snapshot'); $this->modal('gpu-confirm-modal')->close(); $this->confirmingRunId = null; diff --git a/resources/views/livewire/runs/show.blade.php b/resources/views/livewire/runs/show.blade.php index 393db72..e3e5df0 100644 --- a/resources/views/livewire/runs/show.blade.php +++ b/resources/views/livewire/runs/show.blade.php @@ -123,7 +123,7 @@ private function authorizeOwner(): void @endif
- {{ $this->run->suite === 'scenariobench' ? __('Scenario results') : __('Task results') }} + {{ $this->run->suite === 'scenariobench' ? __('Scenario results') : ($this->run->suite === 'depthsweep' ? __('Depth sweep results') : __('Task results')) }}
@if ($this->run->suite === 'scenariobench') @@ -169,6 +169,40 @@ private function authorizeOwner(): void @endforeach + @elseif ($this->run->suite === 'depthsweep') + + + {{ __('Depth') }} + {{ __('Task') }} + {{ __('Level') }} + {{ __('Found') }} + {{ __('Needle') }} + {{ __('TTFT') }} + {{ __('Wall') }} + {{ __('Visible') }} + + + + @foreach ($this->run->suiteResults() as $result) + + {{ $result->section }} + {{ $result->task }} + {{ $result->level ?? '—' }} + + @if ($result->ok) + + @else + + @endif + + {{ $result->detail ?? '—' }} + {{ number_format($result->ttft, 2) }} + {{ number_format($result->wall, 2) }} + {{ number_format($result->visible) }} + + @endforeach + + @else diff --git a/resources/views/livewire/summary/index.blade.php b/resources/views/livewire/summary/index.blade.php index 402fa56..a1c836d 100644 --- a/resources/views/livewire/summary/index.blade.php +++ b/resources/views/livewire/summary/index.blade.php @@ -158,6 +158,14 @@ class="w-64" @if ($session->band->label)
{{ $session->band->label }}
@endif + {{-- Column titles aligned over the suite rows (md+ only) --}} +
@foreach ($session->suites as $row) postJson('/api/cli/device'); + + $response->assertStatus(201) + ->assertJsonStructure(['code', 'poll_token', 'verify_url', 'expires_in']); + + return $response->json(); +} + +test('the device flow grants a one-time usable token', function () { + $me = User::factory()->create(); + $this->actingAs($me); + + $start = start_device_flow(); + + // Poll before confirming: pending + $this->postJson('/api/cli/poll', ['code' => $start['code'], 'poll_token' => $start['poll_token']]) + ->assertOk()->assertJson(['status' => 'pending']); + + // Confirm via the authorize screen logic (the grant path the page runs) + $device = DeviceCode::where('code', $start['code'])->first(); + $plain = $me->createToken('festbench-cli', ['gpus:read', 'results:submit'], now()->addYear()); + $device->forceFill([ + 'user_id' => $me->id, + 'personal_access_token_id' => $plain->accessToken->id, + 'plain_token' => $plain->plainTextToken, + ])->save(); + + // Poll after: granted with a usable id|secret token + $granted = $this->postJson('/api/cli/poll', ['code' => $start['code'], 'poll_token' => $start['poll_token']]) + ->assertOk()->assertJson(['status' => 'granted']) + ->json(); + + expect($granted['token'])->toContain('|') + ->and($granted['abilities'])->toContain('gpus:read'); + + // The token authenticates against the API + $this->withToken($granted['token'])->getJson('/api/me') + ->assertOk()->assertJson(['id' => $me->id, 'email' => $me->email]); + + $this->withToken($granted['token'])->getJson('/api/gpus')->assertOk(); + + // Replay: bare granted, no second token handout + $replay = $this->postJson('/api/cli/poll', ['code' => $start['code'], 'poll_token' => $start['poll_token']]) + ->assertOk()->assertJson(['status' => 'granted'])->json(); + + expect($replay)->not->toHaveKey('token'); + + // The secret is erased server-side after pickup + expect(DeviceCode::where('code', $start['code'])->first()->plain_token)->toBeNull(); +}); + +test('poll with a wrong token is rejected without an oracle', function () { + $start = start_device_flow(); + + $this->postJson('/api/cli/poll', ['code' => $start['code'], 'poll_token' => 'wrong-token-entirely']) + ->assertStatus(404); + + $this->postJson('/api/cli/poll', ['code' => 'NOPE-NOPE', 'poll_token' => $start['poll_token']]) + ->assertStatus(404); +}); + +test('expired codes report expired on poll', function () { + $start = start_device_flow(); + + DeviceCode::where('code', $start['code'])->update(['expires_at' => now()->subMinute()]); + + $this->postJson('/api/cli/poll', ['code' => $start['code'], 'poll_token' => $start['poll_token']]) + ->assertOk()->assertJson(['status' => 'expired']); +}); + +test('the gpu catalog requires authentication and the gpus:read ability', function () { + // Guest: redirected to login (no token at all) + $this->getJson('/api/gpus')->assertStatus(401); + + // Token with the wrong ability + $me = User::factory()->create(); + $limited = $me->createToken('no-gpus', ['results:submit'], now()->addYear()); + + $this->withToken($limited->plainTextToken)->getJson('/api/gpus')->assertStatus(403); +}); + +test('the catalog returns only active gpus', function () { + $me = User::factory()->create(); + $token = $me->createToken('reader', ['gpus:read'], now()->addYear()); + + // Seed the catalog for this test (RefreshDatabase resets between tests) + Artisan::call('db:seed', ['--class' => 'GpuSeeder', '--force' => true]); + + // Deactivate one entry + DB::table('gpus')->where('name', 'AMD Radeon RX 7900 XTX')->update(['active' => false]); + + $response = $this->withToken($token->plainTextToken)->getJson('/api/gpus')->assertOk()->json(); + + $names = array_column($response['gpus'], 'name'); + expect($names)->toContain('AMD Radeon AI PRO R9700') + ->not->toContain('AMD Radeon RX 7900 XTX'); +}); + +test('gpu confirm on my-runs links the run and regroups it', function () { + Artisan::call('db:seed', ['--class' => 'GpuSeeder', '--force' => true]); + + $me = User::factory()->create(); + + // A run whose raw gpu string missed the catalog (typo case) + $run = BenchmarkRun::factory()->create([ + 'suite' => 'softbench', 'user_id' => $me->id, 'gpu' => 'amd radeon ai pro r9700', 'gpu_id' => null, + ]); + SoftbenchResult::factory()->create(['benchmark_run_id' => $run->id]); + Artisan::call('festbench:summary'); + + // Owner confirms via the picker flow + $this->actingAs($me); + Volt::test('my-runs.index') + ->call('startGpuConfirm', $run->id) + ->call('confirmGpu', Gpu::match('AMD Radeon AI PRO R9700')->id); + + $run->refresh(); + expect($run->gpu_id)->not->toBeNull() + ->and($run->gpu)->toBe('AMD Radeon AI PRO R9700'); +}); + +test('depthsweep results import and aggregate without tok/s', function () { + $run = BenchmarkRun::factory()->create(['suite' => 'depthsweep', 'visibility' => 'public']); + DepthsweepResult::factory()->count(5)->create(['benchmark_run_id' => $run->id]); + + Artisan::call('festbench:summary'); + + $row = DB::table('session_suite_rows')->where('suite', 'depthsweep')->first(); + expect($row->passed)->toBeInt() + ->and($row->avg_user_tps)->toBeNull() + ->and($row->total_tokens)->toBe(0); + + // Run detail renders the depthsweep column set + $this->get(route('runs.show', $run->run_id))->assertOk() + ->assertSee('Depth sweep results') + ->assertSee('Needle'); +}); + +test('session bands show column titles', function () { + $run = BenchmarkRun::factory()->create(['suite' => 'softbench', 'visibility' => 'public']); + SoftbenchResult::factory()->create(['benchmark_run_id' => $run->id]); + Artisan::call('festbench:summary'); + + $this->get(route('home'))->assertOk() + ->assertSee('Benchmark', false) + ->assertSee('Thinking', false) + ->assertSee('Speed (tok/s)', false) + ->assertSee('Tokens', false); +});