v2.4: per-band scenariobench aggregation, band scenario detail page, nickname badges, orderBy fix

2026-09-20 22:49:23 -04:00 · 2026-09-20 22:49:23 -04:00 · 06908e252d
commit 06908e252d
parent 9f9327bb27
6 changed files with 225 additions and 16 deletions
--- a/app/Console/Commands/SessionSummary.php
+++ b/app/Console/Commands/SessionSummary.php
@ -73,6 +73,15 @@ public function handle(): int
                $sessions[$key]['labels']->push($run->label);
            }

+            // Scenariobench is aggregated at band level (one row for all
+            // scenario runs in the band) rather than per-run, so collect the
+            // run rows and defer to a second pass below.
+            if ($run->suite === 'scenariobench') {
+                $sessions[$key]['scenario_runs'][] = $run;
+
+                continue;
+            }
+
            $aggs = $this->aggregateRun($run);

            $sessions[$key]['passed'] += $aggs->sum('passed');
@ -81,6 +90,20 @@ public function handle(): int
            $sessions[$key]['runs'] = $sessions[$key]['runs']->merge($aggs);
        }

+        // Second pass: emit one aggregated scenariobench row per band,
+        // covering all scenario runs in that band.
+        foreach ($sessions as $key => $band) {
+            $scenarioRuns = $band['scenario_runs'] ?? [];
+            if (empty($scenarioRuns)) {
+                continue;
+            }
+            $agg = $this->aggregateScenarioBand($scenarioRuns);
+            $sessions[$key]['passed'] += $agg['passed'];
+            $sessions[$key]['failed'] += $agg['failed'];
+            $sessions[$key]['tokens'] += $agg['total_tokens'];
+            $sessions[$key]['runs'] = $band['runs']->merge([$agg]);
+        }
+
        DB::transaction(function () use ($sessions) {
            DB::table('session_summaries')->truncate();

@ -234,6 +257,40 @@ private function aggregateRun(object $run)
        ]]);
    }

+    /**
+     * Aggregate all scenariobench runs in a band into one summary row.
+     * Aggregates passed/failed/tokens and generated tok/s across every
+     * scenario run that belongs to the band.
+     *
+     * @param  array<int, object>  $runs
+     */
+    private function aggregateScenarioBand(array $runs): array
+    {
+        $runPks = array_map(fn ($r) => $r->id, $runs);
+        $first = $runs[0];
+
+        $agg = DB::table('scenario_runs')
+            ->whereIn('benchmark_run_id', $runPks)
+            ->selectRaw('count(*) filter (where ok) as passed')
+            ->selectRaw('count(*) filter (where not ok) as failed')
+            ->selectRaw('sum(gen_tokens) as total_tokens')
+            ->selectRaw('sum(gen_tokens) / nullif(sum(wall), 0) as gen_tps')
+            ->first();
+
+        return [
+            'run_pk' => $first->id,
+            'run_id' => $first->run_id,
+            'suite' => 'scenariobench',
+            'level' => null,
+            'passed' => (int) ($agg->passed ?? 0),
+            'failed' => (int) ($agg->failed ?? 0),
+            'avg_user_tps' => null,
+            'weighted_user_tps' => null,
+            'gen_tps' => $this->roundOrNull($agg->gen_tps),
+            'total_tokens' => (int) ($agg->total_tokens ?? 0),
+        ];
+    }
+
    private function emptyAgg(object $run): array
    {
        return [
--- a/app/Models/BenchmarkRun.php
+++ b/app/Models/BenchmarkRun.php
@ -126,9 +126,11 @@ public function suiteResults(?string $level = null): Collection

        $relation = Suites::runRelation($this->suite);

+        $orderCol = $this->suite === 'scenariobench' ? 'scenario' : 'task';
+
        return $this->{$relation}()
            ->when($level !== null && $level !== '', fn ($q) => $q->where('level', $level))
-            ->orderBy('task')
+            ->orderBy($orderCol)
            ->orderBy('id')
            ->get();
    }
--- a/resources/views/livewire/runs/show.blade.php
+++ b/resources/views/livewire/runs/show.blade.php
@ -75,9 +75,12 @@ private function authorizeOwner(): void
        <div>
            <flux:heading size="xl">{{ $this->run->suite }} <span class="text-zinc-400">·</span> <span class="font-mono text-lg">{{ $this->run->model_id }}</span></flux:heading>
            <flux:subheading>
+                @if ($this->run->label)
+                    <flux:badge color="amber" size="xs">{{ $this->run->label }}</flux:badge>
+                @else
                    @if ($this->run->model_quant)<flux:badge color="zinc" size="xs">{{ $this->run->model_quant }}</flux:badge> @endif
                    @if ($this->run->engine)<flux:badge color="sky" size="xs">{{ $this->run->engine }}</flux:badge> @endif
-                @if ($this->run->label)<flux:badge color="amber" size="xs">{{ $this->run->label }}</flux:badge> @endif
+                @endif
                @if ($this->run->visibility === 'private')<flux:badge color="rose" size="xs">{{ __('Private') }}</flux:badge> @endif
                {{ optional($this->run->ran_at)->format('Y-m-d H:i') }}
            </flux:subheading>
--- a/resources/views/livewire/sessions/scenarios.blade.php
+++ b/resources/views/livewire/sessions/scenarios.blade.php
@ -0,0 +1,140 @@
+<?php
+
+use App\Models\BenchmarkRun;
+use Illuminate\Support\Facades\DB;
+use Livewire\Attributes\Computed;
+use Livewire\Attributes\Layout;
+use Livewire\Component;
+
+new #[Layout('components.layouts.app')] class extends Component {
+    public ?int $sessionId = null;
+
+    public function mount(int $session): void
+    {
+        $this->sessionId = $session;
+    }
+
+    #[Computed]
+    public function band(): object
+    {
+        return DB::table('session_summaries')
+            ->where('id', $this->sessionId)
+            ->where(function ($q) {
+                $q->where('visibility', 'public');
+                if (auth()->user() !== null) {
+                    $q->orWhere(function ($own) {
+                        $own->where('visibility', 'private')
+                            ->where('user_id', auth()->id());
+                    });
+                }
+            })
+            ->firstOrFail();
+    }
+
+    #[Computed]
+    public function runs()
+    {
+        $band = $this->band;
+
+        return BenchmarkRun::where('suite', 'scenariobench')
+            ->where('model_id', $band->model_id)
+            ->where('model_quant', $band->model_quant)
+            ->where('gpu', $band->gpu)
+            ->where('gpu_count', $band->gpu_count)
+            ->where('user_id', $band->user_id)
+            ->where('visibility', $band->visibility)
+            ->where('engine', $band->engine)
+            ->where(function ($q) use ($band) {
+                $q->whereNull('config_id')
+                  ->orWhere('config_id', $this->configIdFromName($band->config_name));
+            })
+            ->orderBy('ran_at')
+            ->get();
+    }
+
+    private function configIdFromName(?string $name): ?int
+    {
+        if ($name === null) {
+            return null;
+        }
+
+        return DB::table('server_configs')->where('name', $name)->value('id');
+    }
+
+    #[Computed]
+    public function results()
+    {
+        $runIds = $this->runs->pluck('id');
+
+        return DB::table('scenario_runs')
+            ->whereIn('benchmark_run_id', $runIds)
+            ->orderBy('scenario')
+            ->orderBy('benchmark_run_id')
+            ->get();
+    }
+};
+?>
+
+<div class="flex flex-col gap-6">
+    <div class="flex flex-wrap items-center justify-between gap-3">
+        <div>
+            <flux:heading size="xl">{{ __('Scenario results') }}</flux:heading>
+            <flux:subheading>
+                @if ($this->band->label)
+                    <flux:badge size="xs" color="amber">{{ $this->band->label }}</flux:badge>
+                @endif
+                <span class="text-sm">{{ $this->band->model_id ?? 'unknown model' }}</span>
+                @if ($this->band->model_quant)
+                    <flux:badge size="xs" color="zinc">{{ $this->band->model_quant }}</flux:badge>
+                @endif
+                <span class="text-sm text-zinc-500">{{ $this->band->gpu ?? 'unknown GPU' }}</span>
+            </flux:subheading>
+        </div>
+
+        <flux:button variant="ghost" icon="arrow-left" href="{{ route('home') }}" wire:navigate>{{ __('Sessions') }}</flux:button>
+    </div>
+
+    <flux:table>
+        <flux:table.columns>
+            <flux:table.column>{{ __('Scenario') }}</flux:table.column>
+            <flux:table.column>{{ __('Level') }}</flux:table.column>
+            <flux:table.column>{{ __('Pass') }}</flux:table.column>
+            <flux:table.column>{{ __('Outcome') }}</flux:table.column>
+            <flux:table.column class="text-right">{{ __('Wall (s)') }}</flux:table.column>
+            <flux:table.column class="text-right">{{ __('Turns') }}</flux:table.column>
+            <flux:table.column class="text-right">{{ __('Commands') }}</flux:table.column>
+            <flux:table.column class="text-right">{{ __('Submissions') }}</flux:table.column>
+            <flux:table.column class="text-right">{{ __('Rejected') }}</flux:table.column>
+            <flux:table.column class="text-right">{{ __('t first sub') }}</flux:table.column>
+            <flux:table.column class="text-right">{{ __('t pass') }}</flux:table.column>
+            <flux:table.column class="text-right">{{ __('Gen tokens') }}</flux:table.column>
+        </flux:table.columns>
+
+        <flux:table.rows>
+            @foreach ($this->results as $result)
+                <flux:table.row :key="$result->id">
+                    <flux:table.cell>{{ $result->scenario }}</flux:table.cell>
+                    <flux:table.cell>{{ $result->level ?? '—' }}</flux:table.cell>
+                    <flux:table.cell>
+                        @if ($result->ok)
+                            <flux:icon name="check-circle" variant="solid" class="size-4 text-emerald-500" />
+                        @elseif ($result->ok === null)
+                            <span class="text-zinc-400">—</span>
+                        @else
+                            <flux:icon name="x-circle" variant="solid" class="size-4 text-red-500" />
+                        @endif
+                    </flux:table.cell>
+                    <flux:table.cell><flux:badge size="xs" color="{{ $result->outcome === 'passed' ? 'emerald' : 'rose' }}">{{ $result->outcome ?? '—' }}</flux:badge></flux:table.cell>
+                    <flux:table.cell class="text-right font-mono text-sm">{{ number_format($result->wall, 1) }}</flux:table.cell>
+                    <flux:table.cell class="text-right font-mono text-sm">{{ number_format($result->turns) }}</flux:table.cell>
+                    <flux:table.cell class="text-right font-mono text-sm">{{ number_format($result->commands) }}</flux:table.cell>
+                    <flux:table.cell class="text-right font-mono text-sm">{{ number_format($result->submissions) }}</flux:table.cell>
+                    <flux:table.cell class="text-right font-mono text-sm">{{ number_format($result->rejected) }}</flux:table.cell>
+                    <flux:table.cell class="text-right font-mono text-sm">{{ number_format($result->t_first_sub, 1) }}</flux:table.cell>
+                    <flux:table.cell class="text-right font-mono text-sm">{{ $result->t_pass !== null ? number_format($result->t_pass, 1) : '—' }}</flux:table.cell>
+                    <flux:table.cell class="text-right font-mono text-sm">{{ number_format($result->gen_tokens) }}</flux:table.cell>
+                </flux:table.row>
+            @endforeach
+        </flux:table.rows>
+    </flux:table>
+</div>
--- a/resources/views/livewire/summary/index.blade.php
+++ b/resources/views/livewire/summary/index.blade.php
@ -148,6 +148,9 @@ class="w-64"
                            <flux:badge size="xs" color="zinc">×{{ $session->band->gpu_count }}</flux:badge>
                        @endif
                        <span class="text-sm">{{ $session->band->model_id ?? 'unknown model' }}</span>
+                        @if ($session->band->label)
+                            <flux:badge size="xs" color="amber">{{ $session->band->label }}</flux:badge>
+                        @else
                            @if ($session->band->model_quant)
                                <flux:badge size="xs" color="zinc">{{ $session->band->model_quant }}</flux:badge>
                            @endif
@ -157,13 +160,11 @@ class="w-64"
                            @if ($session->band->config_name)
                                <flux:badge size="xs" color="amber">{{ $session->band->config_name }}</flux:badge>
                            @endif
+                        @endif
                        <flux:spacer />
                        <span class="text-sm text-zinc-500">{{ $session->band->runner_name ?? 'anonymous' }}</span>
                        <span class="text-xs text-zinc-400">{{ optional($session->band->ran_at)->format('M j Y') }}</span>
                    </div>
-                    @if ($session->band->label)
-                        <div class="px-4 pt-2 text-xs text-zinc-500">{{ $session->band->label }}</div>
-                    @endif
                    {{-- Column titles aligned over the suite rows (md+ only) --}}
                    <div class="hidden px-4 pt-2 md:flex flex-wrap items-center gap-x-4 gap-y-1">
                        <span class="w-24 text-[10px] font-semibold uppercase tracking-wider text-zinc-400">{{ __('Benchmark') }}</span>
@ -174,7 +175,12 @@ class="w-64"
                    </div>
                    <div class="divide-y divide-zinc-100 dark:divide-zinc-800">
                        @foreach ($session->suites as $row)
-                            <a href="{{ route('runs.show', $row->run_id) }}{{ $row->level && $row->suite === 'depthsweep' ? '?level='.$row->level : '' }}" wire:navigate
+                            @php
+                                $detailUrl = $row->suite === 'scenariobench'
+                                    ? route('sessions.scenarios', $session->band->id)
+                                    : route('runs.show', $row->run_id) . ($row->level && $row->suite === 'depthsweep' ? '?level=' . $row->level : '');
+                            @endphp
+                            <a href="{{ $detailUrl }}" wire:navigate
                               class="flex flex-wrap items-center gap-x-4 gap-y-1 px-4 py-2.5 hover:bg-zinc-50 dark:hover:bg-zinc-800/60">
                                <flux:badge size="xs">{{ $row->suite }}</flux:badge>
                                @if ($row->level)
--- a/routes/web.php
+++ b/routes/web.php
@ -7,6 +7,7 @@
 Route::livewire('results', 'results.index')->name('results');

 Route::livewire('runs/{run}', 'runs.show')->name('runs.show');
+Route::livewire('sessions/{session}/scenarios', 'sessions.scenarios')->name('sessions.scenarios');

 // Device authorization for the festbench CLI (Geekbench-style login flow).
 Route::livewire('cli/authorize/{code}', 'cli.authorize')