v2.1: session-grouped summary home page (GPU/model/quant/runner bands, per-suite pass/fail, avg + weighted tok/s), flat task table moved to /results, summary command + tables on schedule and after import
This commit is contained in:
parent
53a0ee54c1
commit
2e205ef792
13 changed files with 637 additions and 15 deletions
|
|
@ -89,6 +89,10 @@ public function handle(): int
|
|||
$this->warn("{$unknownSuites} rows were stored with fallback routing — check the suite registry.");
|
||||
}
|
||||
|
||||
// Keep the precomputed summary + snapshot in step with the new data.
|
||||
$this->call('festbench:summary');
|
||||
$this->call('festbench:snapshot');
|
||||
|
||||
return self::SUCCESS;
|
||||
}
|
||||
|
||||
|
|
|
|||
186
app/Console/Commands/SessionSummary.php
Normal file
186
app/Console/Commands/SessionSummary.php
Normal file
|
|
@ -0,0 +1,186 @@
|
|||
<?php
|
||||
|
||||
namespace App\Console\Commands;
|
||||
|
||||
use App\Support\Suites;
|
||||
use Illuminate\Console\Command;
|
||||
use Illuminate\Support\Facades\DB;
|
||||
use Symfony\Component\Console\Attribute\AsCommand;
|
||||
|
||||
#[AsCommand(name: 'festbench:summary', description: 'Rebuild the session-grouped home summary tables')]
|
||||
class SessionSummary extends Command
|
||||
{
|
||||
public function handle(): int
|
||||
{
|
||||
$sessions = [];
|
||||
|
||||
$runs = DB::table('benchmark_runs')
|
||||
->leftJoin('users', 'users.id', '=', 'benchmark_runs.user_id')
|
||||
->orderByDesc('ran_at')
|
||||
->orderByDesc('benchmark_runs.id')
|
||||
->get([
|
||||
'benchmark_runs.*',
|
||||
'users.name as runner_name',
|
||||
]);
|
||||
|
||||
foreach ($runs as $run) {
|
||||
// Visibility participates in the grouping key so private runs
|
||||
// form their own band — the guest view can then filter bands by
|
||||
// visibility without any per-request recomputation.
|
||||
$key = implode('|', [
|
||||
$run->label ?? '-',
|
||||
$run->model_id ?? '-',
|
||||
$run->model_quant ?? '-',
|
||||
$run->gpu ?? '-',
|
||||
$run->gpu_count ?? '-',
|
||||
$run->user_id ?? '-',
|
||||
$run->visibility,
|
||||
]);
|
||||
|
||||
$sessions[$key] ??= [
|
||||
'key' => $key,
|
||||
'label' => $run->label,
|
||||
'model_id' => $run->model_id,
|
||||
'model_quant' => $run->model_quant,
|
||||
'gpu' => $run->gpu,
|
||||
'gpu_count' => $run->gpu_count,
|
||||
'runner_name' => $run->runner_name,
|
||||
'user_id' => $run->user_id,
|
||||
'visibility' => $run->visibility,
|
||||
'ran_at' => $run->ran_at,
|
||||
'passed' => 0,
|
||||
'failed' => 0,
|
||||
'tokens' => 0,
|
||||
'runs' => collect(),
|
||||
];
|
||||
|
||||
$agg = $this->aggregateRun($run);
|
||||
|
||||
$sessions[$key]['passed'] += $agg['passed'];
|
||||
$sessions[$key]['failed'] += $agg['failed'];
|
||||
$sessions[$key]['tokens'] += $agg['total_tokens'];
|
||||
$sessions[$key]['runs']->push($agg + ['suite' => $run->suite, 'level' => $agg['level']]);
|
||||
}
|
||||
|
||||
DB::transaction(function () use ($sessions) {
|
||||
DB::table('session_summaries')->truncate();
|
||||
|
||||
foreach (array_values($sessions) as $bandIndex => $band) {
|
||||
$summaryId = DB::table('session_summaries')->insertGetId([
|
||||
'session_key' => $band['key'],
|
||||
'label' => $band['label'],
|
||||
'model_id' => $band['model_id'],
|
||||
'model_quant' => $band['model_quant'],
|
||||
'gpu' => $band['gpu'],
|
||||
'gpu_count' => $band['gpu_count'],
|
||||
'runner_name' => $band['runner_name'],
|
||||
'user_id' => $band['user_id'],
|
||||
'visibility' => $band['visibility'],
|
||||
'ran_at' => $band['ran_at'],
|
||||
'total_passed' => $band['passed'],
|
||||
'total_failed' => $band['failed'],
|
||||
'total_tokens' => $band['tokens'],
|
||||
'sequence' => $bandIndex,
|
||||
'created_at' => now(),
|
||||
'updated_at' => now(),
|
||||
]);
|
||||
|
||||
foreach ($band['runs'] as $suiteIndex => $row) {
|
||||
DB::table('session_suite_rows')->insert([
|
||||
'session_summary_id' => $summaryId,
|
||||
'benchmark_run_id' => $row['run_pk'],
|
||||
'run_id' => $row['run_id'],
|
||||
'suite' => $row['suite'],
|
||||
'level' => $row['level'],
|
||||
'passed' => $row['passed'],
|
||||
'failed' => $row['failed'],
|
||||
'avg_user_tps' => $row['avg_user_tps'],
|
||||
'weighted_user_tps' => $row['weighted_user_tps'],
|
||||
'total_tokens' => $row['total_tokens'],
|
||||
'sequence' => $suiteIndex,
|
||||
'created_at' => now(),
|
||||
'updated_at' => now(),
|
||||
]);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
$this->info('session summaries rebuilt: '.count($sessions).' session(s).');
|
||||
|
||||
return self::SUCCESS;
|
||||
}
|
||||
|
||||
/**
|
||||
* Aggregate one run's suite table: pass/fail counts, simple + weighted
|
||||
* user tok/s, generated tokens, and the run's thinking level.
|
||||
*/
|
||||
private function aggregateRun(object $run): array
|
||||
{
|
||||
$model = Suites::modelFor($run->suite);
|
||||
|
||||
if ($model === null) {
|
||||
return [
|
||||
'run_pk' => $run->id,
|
||||
'run_id' => $run->run_id,
|
||||
'level' => null,
|
||||
'passed' => 0,
|
||||
'failed' => 0,
|
||||
'avg_user_tps' => null,
|
||||
'weighted_user_tps' => null,
|
||||
'total_tokens' => 0,
|
||||
];
|
||||
}
|
||||
|
||||
$table = $model::make()->getTable();
|
||||
|
||||
if ($run->suite === 'scenariobench') {
|
||||
$agg = DB::table($table)->where('benchmark_run_id', $run->id)
|
||||
->selectRaw('count(*) filter (where ok) as passed')
|
||||
->selectRaw('count(*) filter (where not ok) as failed')
|
||||
->selectRaw('sum(gen_tokens) as total_tokens')
|
||||
->first();
|
||||
|
||||
return [
|
||||
'run_pk' => $run->id,
|
||||
'run_id' => $run->run_id,
|
||||
'level' => $this->runLevel($table, $run->id),
|
||||
'passed' => (int) ($agg->passed ?? 0),
|
||||
'failed' => (int) ($agg->failed ?? 0),
|
||||
'avg_user_tps' => null,
|
||||
'weighted_user_tps' => null,
|
||||
'total_tokens' => (int) ($agg->total_tokens ?? 0),
|
||||
];
|
||||
}
|
||||
|
||||
$agg = DB::table($table)->where('benchmark_run_id', $run->id)
|
||||
->selectRaw('count(*) filter (where ok) as passed')
|
||||
->selectRaw('count(*) filter (where not ok) as failed')
|
||||
->selectRaw('avg(user_tps) as avg_user_tps')
|
||||
->selectRaw('sum(user_tps * total) filter (where total > 0) / nullif(sum(total) filter (where total > 0), 0) as weighted_user_tps')
|
||||
->selectRaw('sum(total) as total_tokens')
|
||||
->first();
|
||||
|
||||
return [
|
||||
'run_pk' => $run->id,
|
||||
'run_id' => $run->run_id,
|
||||
'level' => $this->runLevel($table, $run->id),
|
||||
'passed' => (int) ($agg->passed ?? 0),
|
||||
'failed' => (int) ($agg->failed ?? 0),
|
||||
'avg_user_tps' => $agg->avg_user_tps !== null ? round((float) $agg->avg_user_tps, 1) : null,
|
||||
'weighted_user_tps' => $agg->weighted_user_tps !== null ? round((float) $agg->weighted_user_tps, 1) : null,
|
||||
'total_tokens' => (int) ($agg->total_tokens ?? 0),
|
||||
];
|
||||
}
|
||||
|
||||
/**
|
||||
* The run's thinking level: the distinct level across its result rows
|
||||
* (a run is single-level; falls back to null when mixed or empty).
|
||||
*/
|
||||
private function runLevel(string $table, int $runPk): ?string
|
||||
{
|
||||
$level = DB::table($table)->where('benchmark_run_id', $runPk)
|
||||
->distinct()->pluck('level')->filter()->values();
|
||||
|
||||
return $level->count() === 1 ? $level->first() : null;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,61 @@
|
|||
<?php
|
||||
|
||||
use Illuminate\Database\Migrations\Migration;
|
||||
use Illuminate\Database\Schema\Blueprint;
|
||||
use Illuminate\Support\Facades\Schema;
|
||||
|
||||
return new class extends Migration
|
||||
{
|
||||
/**
|
||||
* Precomputed session-grouped summary for the home page.
|
||||
*
|
||||
* A session = runs sharing label + model + quant + gpu + gpu_count +
|
||||
* user. Bands render as session_summaries rows; the per-suite aggregate
|
||||
* lines under each band render as session_suite_rows. Rebuilt wholesale
|
||||
* by festbench:summary (scheduled + after import) — the request path
|
||||
* only reads, never aggregates.
|
||||
*/
|
||||
public function up(): void
|
||||
{
|
||||
Schema::create('session_summaries', function (Blueprint $table) {
|
||||
$table->id();
|
||||
$table->string('session_key', 255)->unique();
|
||||
$table->string('label', 255)->nullable();
|
||||
$table->string('model_id', 255)->nullable();
|
||||
$table->string('model_quant', 64)->nullable();
|
||||
$table->string('gpu', 128)->nullable();
|
||||
$table->unsignedInteger('gpu_count')->nullable();
|
||||
$table->string('runner_name', 255)->nullable();
|
||||
$table->foreignId('user_id')->nullable();
|
||||
$table->string('visibility', 16)->default('public');
|
||||
$table->timestamp('ran_at')->nullable();
|
||||
$table->unsignedInteger('total_passed')->default(0);
|
||||
$table->unsignedInteger('total_failed')->default(0);
|
||||
$table->unsignedInteger('total_tokens')->default(0);
|
||||
$table->unsignedInteger('sequence')->index()->comment('band order, newest first');
|
||||
$table->timestamps();
|
||||
});
|
||||
|
||||
Schema::create('session_suite_rows', function (Blueprint $table) {
|
||||
$table->id();
|
||||
$table->foreignId('session_summary_id')->constrained('session_summaries')->cascadeOnDelete();
|
||||
$table->foreignId('benchmark_run_id')->constrained()->cascadeOnDelete();
|
||||
$table->string('run_id', 32);
|
||||
$table->string('suite', 32);
|
||||
$table->string('level', 16)->nullable();
|
||||
$table->unsignedInteger('passed')->default(0);
|
||||
$table->unsignedInteger('failed')->default(0);
|
||||
$table->double('avg_user_tps')->nullable();
|
||||
$table->double('weighted_user_tps')->nullable();
|
||||
$table->unsignedInteger('total_tokens')->default(0);
|
||||
$table->unsignedInteger('sequence')->comment('suite order within band');
|
||||
$table->timestamps();
|
||||
});
|
||||
}
|
||||
|
||||
public function down(): void
|
||||
{
|
||||
Schema::dropIfExists('session_suite_rows');
|
||||
Schema::dropIfExists('session_summaries');
|
||||
}
|
||||
};
|
||||
|
|
@ -13,7 +13,7 @@
|
|||
|
||||
<flux:navlist variant="outline">
|
||||
<flux:navlist.group heading="Platform" class="grid">
|
||||
<flux:navlist.item icon="home" :href="route('home')" :current="request()->routeIs('home')" wire:navigate>Results</flux:navlist.item>
|
||||
<flux:navlist.item icon="home" :href="route('home')" :current="request()->routeIs('home')" wire:navigate>Sessions</flux:navlist.item>
|
||||
@auth
|
||||
<flux:navlist.item icon="lock-closed" :href="route('my-runs')" :current="request()->routeIs('my-runs')" wire:navigate>My Runs</flux:navlist.item>
|
||||
<flux:navlist.item icon="layout-grid" :href="route('home')" :current="request()->routeIs('home')" wire:navigate>Dashboard</flux:navlist.item>
|
||||
|
|
|
|||
56
resources/views/flux/icon/cpu.blade.php
Normal file
56
resources/views/flux/icon/cpu.blade.php
Normal file
|
|
@ -0,0 +1,56 @@
|
|||
@blaze(fold: true)
|
||||
|
||||
{{-- Credit: Lucide (https://lucide.dev) --}}
|
||||
|
||||
@props([
|
||||
'variant' => 'outline',
|
||||
])
|
||||
|
||||
@php
|
||||
if ($variant === 'solid') {
|
||||
throw new \Exception('The "solid" variant is not supported in Lucide.');
|
||||
}
|
||||
|
||||
$classes = Flux::classes('shrink-0')
|
||||
->add(match($variant) {
|
||||
'outline' => '[:where(&)]:size-6',
|
||||
'solid' => '[:where(&)]:size-6',
|
||||
'mini' => '[:where(&)]:size-5',
|
||||
'micro' => '[:where(&)]:size-4',
|
||||
});
|
||||
|
||||
$strokeWidth = match ($variant) {
|
||||
'outline' => 2,
|
||||
'mini' => 2.25,
|
||||
'micro' => 2.5,
|
||||
};
|
||||
@endphp
|
||||
|
||||
<svg
|
||||
{{ $attributes->class($classes) }}
|
||||
data-flux-icon
|
||||
xmlns="http://www.w3.org/2000/svg"
|
||||
viewBox="0 0 24 24"
|
||||
fill="none"
|
||||
stroke="currentColor"
|
||||
stroke-width="{{ $strokeWidth }}"
|
||||
stroke-linecap="round"
|
||||
stroke-linejoin="round"
|
||||
aria-hidden="true"
|
||||
data-slot="icon"
|
||||
>
|
||||
<path d="M12 20v2" />
|
||||
<path d="M12 2v2" />
|
||||
<path d="M17 20v2" />
|
||||
<path d="M17 2v2" />
|
||||
<path d="M2 12h2" />
|
||||
<path d="M2 17h2" />
|
||||
<path d="M2 7h2" />
|
||||
<path d="M20 12h2" />
|
||||
<path d="M20 17h2" />
|
||||
<path d="M20 7h2" />
|
||||
<path d="M7 20v2" />
|
||||
<path d="M7 2v2" />
|
||||
<rect x="4" y="4" width="16" height="16" rx="2" />
|
||||
<rect x="8" y="8" width="8" height="8" rx="1" />
|
||||
</svg>
|
||||
|
|
@ -150,9 +150,10 @@ private function applyFilters($query)
|
|||
<div>
|
||||
<div class="flex items-center justify-between">
|
||||
<div>
|
||||
<flux:heading size="xl">{{ __('Benchmark Results') }}</flux:heading>
|
||||
<flux:heading size="xl">{{ __('All Task Rows') }}</flux:heading>
|
||||
<flux:subheading>{{ __('Every datapoint festbench captures — sortable.') }}</flux:subheading>
|
||||
</div>
|
||||
<flux:button variant="ghost" icon="arrow-left" href="{{ route('home') }}" wire:navigate>{{ __('Session summary') }}</flux:button>
|
||||
</div>
|
||||
|
||||
<div class="mt-6 flex flex-wrap items-end gap-3">
|
||||
|
|
|
|||
|
|
@ -80,7 +80,7 @@ private function authorizeOwner(): void
|
|||
</div>
|
||||
|
||||
<div class="flex items-center gap-2">
|
||||
<flux:button variant="ghost" icon="arrow-left" href="{{ route('home') }}" wire:navigate>{{ __('Back') }}</flux:button>
|
||||
<flux:button variant="ghost" icon="arrow-left" href="{{ route('home') }}" wire:navigate>{{ __('Sessions') }}</flux:button>
|
||||
@if ($this->run->isOwnedBy(auth()->user()))
|
||||
<flux:button
|
||||
size="sm"
|
||||
|
|
|
|||
197
resources/views/livewire/summary/index.blade.php
Normal file
197
resources/views/livewire/summary/index.blade.php
Normal file
|
|
@ -0,0 +1,197 @@
|
|||
<?php
|
||||
|
||||
use App\Models\BenchmarkRun;
|
||||
use Illuminate\Support\Facades\Cache;
|
||||
use Livewire\Attributes\Computed;
|
||||
use Livewire\Component;
|
||||
use Livewire\WithPagination;
|
||||
|
||||
new class extends Component {
|
||||
use WithPagination;
|
||||
|
||||
public array $filters = [
|
||||
'search' => null,
|
||||
'model' => null,
|
||||
'suite' => null,
|
||||
];
|
||||
|
||||
public function resetFilters(): void
|
||||
{
|
||||
$this->filters = ['search' => null, 'model' => null, 'suite' => null];
|
||||
$this->resetPage();
|
||||
}
|
||||
|
||||
#[Computed]
|
||||
public function sessions()
|
||||
{
|
||||
$ttl = (int) config('festbench.home_cache_ttl', 300);
|
||||
$user = auth()->user();
|
||||
// Guest views are shared-cacheable; logged-in views may contain the
|
||||
// user's private bands, so they get a user-scoped key.
|
||||
$cacheKey = 'summary:'.($user?->id ?? 'guest').':'.md5(json_encode([$this->filters, $this->getPage()]));
|
||||
|
||||
return Cache::remember($cacheKey, $ttl, function () use ($user) {
|
||||
$bands = $this->bandQuery($user)->paginate(10);
|
||||
|
||||
$runIds = $bands->getCollection()->map(fn ($b) => $b->id)->all();
|
||||
$suiteRows = $runIds === []
|
||||
? collect()
|
||||
: \Illuminate\Support\Facades\DB::table('session_suite_rows')
|
||||
->whereIn('session_summary_id', $runIds)
|
||||
->orderBy('sequence')
|
||||
->get()
|
||||
->groupBy('session_summary_id');
|
||||
|
||||
$bands->getCollection()->transform(fn ($band) => (object) [
|
||||
'band' => $band,
|
||||
'suites' => $suiteRows->get($band->id, collect()),
|
||||
]);
|
||||
|
||||
return $bands;
|
||||
});
|
||||
}
|
||||
|
||||
private function bandQuery($user)
|
||||
{
|
||||
return \Illuminate\Support\Facades\DB::table('session_summaries')
|
||||
->where(function ($q) use ($user) {
|
||||
$q->where('visibility', 'public');
|
||||
if ($user !== null) {
|
||||
$q->orWhere(function ($own) use ($user) {
|
||||
$own->where('visibility', 'private')->where('user_id', $user->id);
|
||||
});
|
||||
}
|
||||
})
|
||||
->when($this->filters['search'], fn ($q, $search) => $q->where(function ($q) use ($search) {
|
||||
$q->where('label', 'like', "%{$search}%")
|
||||
->orWhere('model_id', 'like', "%{$search}%")
|
||||
->orWhere('gpu', 'like', "%{$search}%")
|
||||
->orWhere('runner_name', 'like', "%{$search}%");
|
||||
}))
|
||||
->when($this->filters['model'], fn ($q, $model) => $q->where('model_id', $model))
|
||||
// Suite filter: keep bands containing the chosen suite
|
||||
->when($this->filters['suite'], fn ($q, $suite) => $q->whereExists(function ($sub) use ($suite) {
|
||||
$sub->select(\Illuminate\Support\Facades\DB::raw(1))
|
||||
->from('session_suite_rows')
|
||||
->whereColumn('session_suite_rows.session_summary_id', 'session_summaries.id')
|
||||
->where('session_suite_rows.suite', $suite);
|
||||
}))
|
||||
->orderByDesc('ran_at')
|
||||
->orderBy('sequence');
|
||||
}
|
||||
|
||||
#[Computed]
|
||||
public function models(): array
|
||||
{
|
||||
return Cache::remember('summary:models', 300, function () {
|
||||
return \Illuminate\Support\Facades\DB::table('session_summaries')
|
||||
->where('visibility', 'public')
|
||||
->distinct()->orderBy('model_id')->pluck('model_id')->all();
|
||||
});
|
||||
}
|
||||
|
||||
#[Computed]
|
||||
public function suites(): array
|
||||
{
|
||||
return Cache::remember('summary:suites', 300, function () {
|
||||
return \Illuminate\Support\Facades\DB::table('session_suite_rows')
|
||||
->join('session_summaries', 'session_summaries.id', '=', 'session_suite_rows.session_summary_id')
|
||||
->where('session_summaries.visibility', 'public')
|
||||
->distinct()->orderBy('suite')->pluck('suite')->all();
|
||||
});
|
||||
}
|
||||
};
|
||||
?>
|
||||
|
||||
<div>
|
||||
<div class="flex items-center justify-between">
|
||||
<div>
|
||||
<flux:heading size="xl">{{ __('Benchmark Sessions') }}</flux:heading>
|
||||
<flux:subheading>{{ __('GPU, model, suite scores — click a suite line for per-task detail.') }}</flux:subheading>
|
||||
</div>
|
||||
<flux:button variant="ghost" icon="table-cells" href="{{ route('results') }}" wire:navigate>{{ __('All task rows') }}</flux:button>
|
||||
</div>
|
||||
|
||||
<div class="mt-6 flex flex-wrap items-end gap-3">
|
||||
<flux:input
|
||||
wire:model.live.debounce.300ms="filters.search"
|
||||
:label="__('Search')"
|
||||
:placeholder="__('Session, model, GPU or runner…')"
|
||||
size="sm"
|
||||
class="w-64"
|
||||
clearable
|
||||
/>
|
||||
|
||||
<flux:select wire:model.live="filters.model" :label="__('Model')" placeholder="{{ __('All models') }}" size="sm" class="w-56">
|
||||
@foreach ($this->models as $model)
|
||||
<flux:select.option value="{{ $model }}">{{ $model }}</flux:select.option>
|
||||
@endforeach
|
||||
</flux:select>
|
||||
|
||||
<flux:select wire:model.live="filters.suite" :label="__('Suite')" placeholder="{{ __('All suites') }}" size="sm" class="w-44">
|
||||
@foreach ($this->suites as $suite)
|
||||
<flux:select.option value="{{ $suite }}">{{ $suite }}</flux:select.option>
|
||||
@endforeach
|
||||
</flux:select>
|
||||
|
||||
<flux:button size="sm" variant="ghost" wire:click="resetFilters" icon="x-mark">{{ __('Clear') }}</flux:button>
|
||||
</div>
|
||||
|
||||
@if ($this->sessions->count() > 0)
|
||||
<div class="mt-6 flex flex-col gap-4">
|
||||
@foreach ($this->sessions as $session)
|
||||
<flux:card class="!p-0">
|
||||
<div class="flex flex-wrap items-center gap-x-4 gap-y-1 border-b border-zinc-200 px-4 py-3 dark:border-zinc-700">
|
||||
<flux:icon name="cpu" class="size-4 text-zinc-500" />
|
||||
<span class="text-sm font-semibold">{{ $session->band->gpu ?? 'unknown GPU' }}</span>
|
||||
@if ($session->band->gpu_count)
|
||||
<flux:badge size="xs" color="zinc">×{{ $session->band->gpu_count }}</flux:badge>
|
||||
@endif
|
||||
<span class="text-sm">{{ $session->band->model_id ?? 'unknown model' }}</span>
|
||||
@if ($session->band->model_quant)
|
||||
<flux:badge size="xs" color="zinc">{{ $session->band->model_quant }}</flux:badge>
|
||||
@endif
|
||||
<flux:spacer />
|
||||
<span class="text-sm text-zinc-500">{{ $session->band->runner_name ?? 'anonymous' }}</span>
|
||||
<span class="text-xs text-zinc-400">{{ optional($session->band->ran_at)->format('M j Y') }}</span>
|
||||
</div>
|
||||
@if ($session->band->label)
|
||||
<div class="px-4 pt-2 text-xs text-zinc-500">{{ $session->band->label }}</div>
|
||||
@endif
|
||||
<div class="divide-y divide-zinc-100 dark:divide-zinc-800">
|
||||
@foreach ($session->suites as $row)
|
||||
<a href="{{ route('runs.show', $row->run_id) }}" wire:navigate
|
||||
class="flex flex-wrap items-center gap-x-4 gap-y-1 px-4 py-2.5 hover:bg-zinc-50 dark:hover:bg-zinc-800/60">
|
||||
<flux:badge size="xs">{{ $row->suite }}</flux:badge>
|
||||
@if ($row->level)
|
||||
<flux:badge size="xs" color="sky">{{ $row->level }}</flux:badge>
|
||||
@endif
|
||||
<span class="flex items-center gap-1.5">
|
||||
<flux:badge size="xs" color="emerald">✔ {{ $row->passed }}</flux:badge>
|
||||
<flux:badge size="xs" color="{{ $row->failed > 0 ? 'rose' : 'zinc' }}">✘ {{ $row->failed }}</flux:badge>
|
||||
</span>
|
||||
@if ($row->avg_user_tps !== null)
|
||||
<span class="font-mono text-sm">{{ number_format($row->avg_user_tps, 1) }} <span class="text-xs text-zinc-400">avg</span></span>
|
||||
<span class="font-mono text-sm">{{ number_format($row->weighted_user_tps, 1) }} <span class="text-xs text-zinc-400">wtd tok/s</span></span>
|
||||
@endif
|
||||
<span class="font-mono text-sm">{{ number_format($row->total_tokens) }} <span class="text-xs text-zinc-400">tokens</span></span>
|
||||
<span class="ms-auto text-xs text-zinc-400">{{ __('per-task detail') }} →</span>
|
||||
</a>
|
||||
@endforeach
|
||||
</div>
|
||||
</flux:card>
|
||||
@endforeach
|
||||
</div>
|
||||
|
||||
<div class="mt-4">
|
||||
{{ $this->sessions->links() }}
|
||||
</div>
|
||||
@else
|
||||
<div class="mt-6">
|
||||
<flux:callout variant="brand">
|
||||
<flux:callout.heading>{{ __('No sessions') }}</flux:callout.heading>
|
||||
<flux:callout.text>{{ __('No benchmark sessions match the current filters.') }}</flux:callout.text>
|
||||
</flux:callout>
|
||||
</div>
|
||||
@endif
|
||||
</div>
|
||||
|
|
@ -6,6 +6,5 @@
|
|||
// request path never runs the UNION. Cheap on this data volume.
|
||||
Schedule::command('festbench:snapshot')->everyFiveMinutes();
|
||||
|
||||
// Ensure next month's Postgres partitions exist before inserts need them.
|
||||
// (Created by the migration for +/-1 year; this keeps future months coming.)
|
||||
// Placeholder command added when partition maintenance is wired up.
|
||||
// Rebuild the session-grouped home summary (bands + per-suite rows).
|
||||
Schedule::command('festbench:summary')->everyFiveMinutes();
|
||||
|
|
|
|||
|
|
@ -3,7 +3,8 @@
|
|||
use Illuminate\Support\Facades\Route;
|
||||
use Livewire\Volt\Volt;
|
||||
|
||||
Route::livewire('/', 'results.index')->name('home');
|
||||
Route::livewire('/', 'summary.index')->name('home');
|
||||
Route::livewire('results', 'results.index')->name('results');
|
||||
|
||||
Route::livewire('runs/{run}', 'runs.show')->name('runs.show');
|
||||
|
||||
|
|
|
|||
|
|
@ -20,23 +20,23 @@
|
|||
->and($rows[0]->run_id)->toBe($run->run_id);
|
||||
});
|
||||
|
||||
test('guest default home view reads the snapshot, no union needed', function () {
|
||||
test('guest default results view reads the snapshot, no union needed', function () {
|
||||
$run = BenchmarkRun::factory()->create(['suite' => 'softbench', 'visibility' => 'public']);
|
||||
SoftbenchResult::factory()->create(['benchmark_run_id' => $run->id, 'task' => 'snapshotted-task']);
|
||||
Artisan::call('festbench:snapshot');
|
||||
|
||||
// The snapshot is the only path that knows this task name; seeing it
|
||||
// proves the default view served from home_snapshot.
|
||||
$this->get(route('home'))->assertOk()->assertSee('snapshotted-task');
|
||||
$this->get(route('results'))->assertOk()->assertSee('snapshotted-task');
|
||||
});
|
||||
|
||||
test('guest pages get public cache headers, authenticated pages do not', function () {
|
||||
$guest = $this->get(route('home'));
|
||||
$guest = $this->get(route('results'));
|
||||
expect($guest->headers->get('Cache-Control'))->toContain('public')
|
||||
->and($guest->headers->get('Cache-Control'))->toContain('s-maxage');
|
||||
|
||||
$this->actingAs(User::factory()->create());
|
||||
$auth = $this->get(route('home'));
|
||||
$auth = $this->get(route('results'));
|
||||
|
||||
expect($auth->headers->get('Cache-Control'))->toContain('private')
|
||||
->and($auth->headers->get('Cache-Control'))->not->toContain('public');
|
||||
|
|
|
|||
|
|
@ -6,26 +6,26 @@
|
|||
use App\Models\User;
|
||||
use Livewire\Volt\Volt;
|
||||
|
||||
test('guests see public task results on the home page', function () {
|
||||
test('guests see public task results on the results page', function () {
|
||||
$publicRun = BenchmarkRun::factory()->create(['model_id' => 'public/model', 'suite' => 'softbench']);
|
||||
ProsebenchResult::factory()->count(3)->create(['benchmark_run_id' => $publicRun->id]);
|
||||
BenchmarkRun::factory()->private()->create(['model_id' => 'private/model', 'suite' => 'softbench']);
|
||||
|
||||
$response = $this->get(route('home'));
|
||||
$response = $this->get(route('results'));
|
||||
|
||||
$response->assertOk()
|
||||
->assertSee('public/model')
|
||||
->assertDontSee('private/model');
|
||||
});
|
||||
|
||||
test('authenticated users see their private runs mixed in', function () {
|
||||
test('authenticated users see their private runs mixed in on results', function () {
|
||||
$me = User::factory()->create();
|
||||
BenchmarkRun::factory()->private()->create(['model_id' => 'my-private/model', 'user_id' => $me->id, 'suite' => 'softbench']);
|
||||
BenchmarkRun::factory()->private()->create(['model_id' => 'other-private/model', 'suite' => 'softbench']);
|
||||
|
||||
$this->actingAs($me);
|
||||
|
||||
$this->get(route('home'))->assertOk()
|
||||
$this->get(route('results'))->assertOk()
|
||||
->assertSee('my-private/model')
|
||||
->assertDontSee('other-private/model');
|
||||
});
|
||||
|
|
|
|||
117
tests/Feature/SessionSummaryTest.php
Normal file
117
tests/Feature/SessionSummaryTest.php
Normal file
|
|
@ -0,0 +1,117 @@
|
|||
<?php
|
||||
|
||||
use App\Models\BenchmarkRun;
|
||||
use App\Models\InfrabenchResult;
|
||||
use App\Models\SoftbenchResult;
|
||||
use App\Models\User;
|
||||
use Illuminate\Support\Facades\Artisan;
|
||||
|
||||
test('festbench:summary groups runs into session bands with weighted tok/s', function () {
|
||||
$me = User::factory()->create(['name' => 'peterZ']);
|
||||
|
||||
// One session: two softbench runs sharing label/model/gpu/user, plus a
|
||||
// different-session run (different label).
|
||||
$a = BenchmarkRun::factory()->create([
|
||||
'suite' => 'softbench', 'label' => 'session A', 'model_id' => 'm', 'model_quant' => 'Q5_K_M',
|
||||
'gpu' => 'AMD R9700', 'gpu_count' => 1, 'user_id' => $me->id, 'visibility' => 'public',
|
||||
'ran_at' => '2026-09-03 12:00:00',
|
||||
]);
|
||||
SoftbenchResult::factory()->create(['benchmark_run_id' => $a->id, 'ok' => true, 'user_tps' => 60.0, 'total' => 100, 'level' => 'off']);
|
||||
SoftbenchResult::factory()->create(['benchmark_run_id' => $a->id, 'ok' => false, 'user_tps' => 20.0, 'total' => 300, 'level' => 'off']);
|
||||
|
||||
$b = BenchmarkRun::factory()->create([
|
||||
'suite' => 'infrabench', 'label' => 'session A', 'model_id' => 'm', 'model_quant' => 'Q5_K_M',
|
||||
'gpu' => 'AMD R9700', 'gpu_count' => 1, 'user_id' => $me->id, 'visibility' => 'public',
|
||||
'ran_at' => '2026-09-03 12:05:00',
|
||||
]);
|
||||
InfrabenchResult::factory()->create(['benchmark_run_id' => $b->id, 'ok' => true, 'user_tps' => 40.0, 'total' => 200, 'level' => 'low']);
|
||||
|
||||
$c = BenchmarkRun::factory()->create([
|
||||
'suite' => 'softbench', 'label' => 'session B', 'model_id' => 'm', 'model_quant' => 'Q5_K_M',
|
||||
'gpu' => 'AMD R9700', 'gpu_count' => 1, 'user_id' => $me->id, 'visibility' => 'public',
|
||||
'ran_at' => '2026-09-04 12:00:00',
|
||||
]);
|
||||
SoftbenchResult::factory()->create(['benchmark_run_id' => $c->id, 'ok' => true, 'user_tps' => 50.0, 'total' => 50, 'level' => 'medium']);
|
||||
|
||||
Artisan::call('festbench:summary');
|
||||
|
||||
$bands = DB::table('session_summaries')->orderBy('sequence')->get();
|
||||
expect($bands)->toHaveCount(2);
|
||||
|
||||
$bandA = $bands->firstWhere('label', 'session A');
|
||||
expect($bandA->runner_name)->toBe('peterZ')
|
||||
->and($bandA->gpu)->toBe('AMD R9700')
|
||||
->and($bandA->gpu_count)->toBe(1)
|
||||
->and($bandA->total_passed)->toBe(2)
|
||||
->and($bandA->total_failed)->toBe(1)
|
||||
->and($bandA->total_tokens)->toBe(600);
|
||||
|
||||
$rowsA = DB::table('session_suite_rows')->where('session_summary_id', $bandA->id)->orderBy('sequence')->get();
|
||||
expect($rowsA)->toHaveCount(2);
|
||||
|
||||
$soft = $rowsA->firstWhere('suite', 'softbench');
|
||||
// simple avg of (60, 20) = 40; weighted = (60*100 + 20*300)/400 = 30
|
||||
expect($soft->passed)->toBe(1)
|
||||
->and($soft->failed)->toBe(1)
|
||||
->and($soft->avg_user_tps)->toBe(40.0)
|
||||
->and($soft->weighted_user_tps)->toBe(30.0)
|
||||
->and($soft->total_tokens)->toBe(400)
|
||||
->and($soft->level)->toBe('off');
|
||||
|
||||
$infra = $rowsA->firstWhere('suite', 'infrabench');
|
||||
expect($infra->passed)->toBe(1)->and($infra->level)->toBe('low');
|
||||
|
||||
// Newest session first
|
||||
expect($bands->first()->label)->toBe('session B');
|
||||
});
|
||||
|
||||
test('private runs form their own band and are hidden from guests', function () {
|
||||
$me = User::factory()->create();
|
||||
BenchmarkRun::factory()->create([
|
||||
'suite' => 'softbench', 'label' => 'pub', 'model_id' => 'm', 'visibility' => 'public',
|
||||
'gpu' => 'g', 'gpu_count' => 1, 'user_id' => $me->id,
|
||||
]);
|
||||
SoftbenchResult::factory()->create(['benchmark_run_id' => BenchmarkRun::latest('id')->first()->id]);
|
||||
|
||||
BenchmarkRun::factory()->create([
|
||||
'suite' => 'softbench', 'label' => 'pub', 'model_id' => 'm', 'visibility' => 'private',
|
||||
'gpu' => 'g', 'gpu_count' => 1, 'user_id' => $me->id,
|
||||
]);
|
||||
|
||||
Artisan::call('festbench:summary');
|
||||
|
||||
expect(DB::table('session_summaries')->count())->toBe(2)
|
||||
->and(DB::table('session_summaries')->where('visibility', 'private')->count())->toBe(1);
|
||||
|
||||
// Guest home shows the public band only; owner sees both
|
||||
$this->get(route('home'))->assertOk()->assertSee('pub');
|
||||
|
||||
$bands = DB::table('session_summaries')->pluck('visibility');
|
||||
expect($bands)->toContain('public')->toContain('private');
|
||||
});
|
||||
|
||||
test('home renders the session summary and links into per-task detail', function () {
|
||||
$run = BenchmarkRun::factory()->create([
|
||||
'suite' => 'softbench', 'label' => 'rendered session', 'visibility' => 'public',
|
||||
'model_id' => 'render/model', 'gpu' => 'AMD R9700', 'gpu_count' => 2,
|
||||
]);
|
||||
SoftbenchResult::factory()->create(['benchmark_run_id' => $run->id, 'ok' => true, 'level' => 'off']);
|
||||
|
||||
Artisan::call('festbench:summary');
|
||||
|
||||
$this->get(route('home'))->assertOk()
|
||||
->assertSee('rendered session')
|
||||
->assertSee('render/model')
|
||||
->assertSee('AMD R9700')
|
||||
->assertSee(route('runs.show', $run->run_id))
|
||||
->assertSee('All task rows');
|
||||
});
|
||||
|
||||
test('results page still lists flat task rows', function () {
|
||||
$run = BenchmarkRun::factory()->create(['suite' => 'softbench', 'visibility' => 'public']);
|
||||
SoftbenchResult::factory()->create(['benchmark_run_id' => $run->id, 'task' => 'flat-task']);
|
||||
|
||||
$this->get(route('results'))->assertOk()
|
||||
->assertSee('All Task Rows')
|
||||
->assertSee('flat-task');
|
||||
});
|
||||
Loading…
Reference in a new issue