pch/festbench-web
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

v2.3: group runs by actual harness config, per-level depthsweep, scenario gen speed, named server configs

Browse source
This commit is contained in:
PeterChrz 2026-09-19 23:45:04 -04:00
parent 3695632e58
commit 9f9327bb27
Signed by untrusted user who does not match committer: pch
GPG key ID: 8F0826ECF7302C63
13 changed files with 530 additions and 56 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

22
app/Console/Commands/FestbenchImport.php
View file

@ -4,6 +4,7 @@
use App\Models\BenchmarkRun;
use App\Models\Gpu;
use App\Models\ServerConfig;
use App\Models\User;
use App\Support\Suites;
use Illuminate\Console\Command;
@ -23,6 +24,9 @@ class FestbenchImport extends Command
{--label= : Only import runs whose label matches exactly}
{--run-id= : Comma-separated list of run_ids to import}
{--gpu= : Override the GPU name on imported runs (backfill or correction)}
{--gpu-count= : Override GPU count when --gpu is supplied (defaults to 1 if source count is 0)}
{--relabel= : Replace the label on imported runs (build one session from many)}
{--config-id= : Link imported runs to a server config by id}
{--private : Import runs as private instead of public}
{--fresh : Delete ALL existing benchmark data before importing}';
@ -136,6 +140,19 @@ private function createRun(array $source, ?User $user): BenchmarkRun
// source row keeps its provenance inside payload/run_key untouched.
$gpuName = $this->option('gpu') ?? ($source['gpu'] ?? null);
$gpu = Gpu::match($gpuName);
$gpuCount = $this->option('gpu-count') ?? ($source['gpu_count'] ?? null);
if ($this->option('gpu') !== null && ((int) $gpuCount) < 1) {
$gpuCount = 1;
}
// --relabel rewrites the session glue so runs imported together form
// one band; --config-id links the named server config for grouping.
$label = $this->option('relabel') ?? (($source['label'] ?? '') !== '' ? $source['label'] : null);
$configId = $this->option('config-id');
if ($configId !== null && ! ServerConfig::whereKey((int) $configId)->exists()) {
$this->warnOnce("Server config {$configId} not found -- runs will import without a config link.");
$configId = null;
}
if ($gpuName !== null && $gpu === null && ! in_array(strtolower((string) $gpuName), ['unknown', 'hosted', 'cloud'], true)) {
$this->warnOnce("GPU '{$gpuName}' is not in the catalog (run {$source['run_id']}) — it can be confirmed on the website later.");
@ -155,13 +172,14 @@ private function createRun(array $source, ?User $user): BenchmarkRun
'n_ctx' => $source['n_ctx'] ?? null,
'slots' => $source['slots'] ?? null,
'gpu' => $gpuName,
'gpu_count' => $source['gpu_count'] ?? null,
'gpu_count' => $gpuCount,
'gpu_id' => $gpu?->id,
'gpu_source' => $this->option('gpu') ? ($source['gpu_source'] ?? 'backfill-manual') : ($source['gpu_source'] ?? null),
'host' => $source['host'] ?? null,
'client' => $source['client'] ?? null,
'endpoint' => $source['endpoint'] ?? null,
'label' => ($source['label'] ?? '') !== '' ? $source['label'] : null,
'label' => $label,
'config_id' => $configId,
'config' => $source['config'] ?? null,
'image_digest' => $source['image_digest'] ?? null,
'validators' => $this->decodeJson($source['validators'] ?? null),

147
app/Console/Commands/SessionSummary.php
View file

@ -4,6 +4,7 @@
use App\Support\Suites;
use Illuminate\Console\Command;
use Illuminate\Support\Collection;
use Illuminate\Support\Facades\DB;
use Symfony\Component\Console\Attribute\AsCommand;
@ -16,30 +17,42 @@ public function handle(): int
$runs = DB::table('benchmark_runs')
->leftJoin('users', 'users.id', '=', 'benchmark_runs.user_id')
->leftJoin('server_configs', 'server_configs.id', '=', 'benchmark_runs.config_id')
->orderByDesc('ran_at')
->orderByDesc('benchmark_runs.id')
->get([
'benchmark_runs.*',
'users.name as runner_name',
'server_configs.name as config_name',
]);
foreach ($runs as $run) {
// Visibility participates in the grouping key so private runs
// form their own band — the guest view can then filter bands by
// visibility without any per-request recomputation.
// Grouping key: user + GPU + model + quant + visibility + engine
// + harness identity + named server config. Free-text labels are
// intentionally NOT part of identity: rerunning psychbench with
// the same actual settings belongs in the same bubble even when
// the user wrote a different run note.
$key = implode('|', [
$run->label ?? '-',
$run->model_id ?? '-',
$run->model_quant ?? '-',
$run->gpu ?? '-',
$run->gpu_id ?? 'raw:'.$run->gpu,
$run->gpu_count ?? '-',
$run->user_id ?? '-',
$run->visibility,
$run->engine ?? '-',
$run->engine_build ?: '-',
// Older direct runs stored empty harness fields; newer
// depthsweep writes the explicit equivalent. Normalize so
// semantically identical direct runs share a session.
$run->harness ?: 'direct',
$run->harness_ver ?: '-',
$run->harness_mode ?: 'one-shot',
$run->config_id ?? 'none',
]);
$sessions[$key] ??= [
'key' => $key,
'label' => $run->label,
'labels' => collect([$run->label])->filter(),
'model_id' => $run->model_id,
'model_quant' => $run->model_quant,
'gpu' => $run->gpu,
@ -47,6 +60,8 @@ public function handle(): int
'runner_name' => $run->runner_name,
'user_id' => $run->user_id,
'visibility' => $run->visibility,
'engine' => $run->engine,
'config_name' => $run->config_name,
'ran_at' => $run->ran_at,
'passed' => 0,
'failed' => 0,
@ -54,12 +69,16 @@ public function handle(): int
'runs' => collect(),
];
$agg = $this->aggregateRun($run);
if ($run->label !== null && $run->label !== '') {
$sessions[$key]['labels']->push($run->label);
}
$sessions[$key]['passed'] += $agg['passed'];
$sessions[$key]['failed'] += $agg['failed'];
$sessions[$key]['tokens'] += $agg['total_tokens'];
$sessions[$key]['runs']->push($agg + ['suite' => $run->suite, 'level' => $agg['level']]);
$aggs = $this->aggregateRun($run);
$sessions[$key]['passed'] += $aggs->sum('passed');
$sessions[$key]['failed'] += $aggs->sum('failed');
$sessions[$key]['tokens'] += $aggs->sum('total_tokens');
$sessions[$key]['runs'] = $sessions[$key]['runs']->merge($aggs);
}
DB::transaction(function () use ($sessions) {
@ -68,7 +87,11 @@ public function handle(): int
foreach (array_values($sessions) as $bandIndex => $band) {
$summaryId = DB::table('session_summaries')->insertGetId([
'session_key' => $band['key'],
'label' => $band['label'],
// Keep a label only when every grouped run agrees. A
// mixed label is a note, not a configuration split.
'label' => $band['labels']->unique()->count() === 1
? $band['labels']->first()
: null,
'model_id' => $band['model_id'],
'model_quant' => $band['model_quant'],
'gpu' => $band['gpu'],
@ -76,6 +99,8 @@ public function handle(): int
'runner_name' => $band['runner_name'],
'user_id' => $band['user_id'],
'visibility' => $band['visibility'],
'engine' => $band['engine'],
'config_name' => $band['config_name'],
'ran_at' => $band['ran_at'],
'total_passed' => $band['passed'],
'total_failed' => $band['failed'],
@ -85,7 +110,7 @@ public function handle(): int
'updated_at' => now(),
]);
foreach ($band['runs'] as $suiteIndex => $row) {
foreach ($band['runs']->values() as $suiteIndex => $row) {
DB::table('session_suite_rows')->insert([
'session_summary_id' => $summaryId,
'benchmark_run_id' => $row['run_pk'],
@ -96,6 +121,7 @@ public function handle(): int
'failed' => $row['failed'],
'avg_user_tps' => $row['avg_user_tps'],
'weighted_user_tps' => $row['weighted_user_tps'],
'gen_tps' => $row['gen_tps'],
'total_tokens' => $row['total_tokens'],
'sequence' => $suiteIndex,
'created_at' => now(),
@ -111,67 +137,124 @@ public function handle(): int
}
/**
* Aggregate one run's suite table: pass/fail counts, simple + weighted
* user tok/s, generated tokens, and the run's thinking level.
* Aggregate one run. Returns a collection: normally one row, but a
* depthsweep run expands into one row per thinking level — each level's
* rows aggregate independently so the band renders level-by-level.
*
* @return Collection<int, array>
*/
private function aggregateRun(object $run): array
private function aggregateRun(object $run)
{
$model = Suites::modelFor($run->suite);
if ($model === null) {
return [
'run_pk' => $run->id,
'run_id' => $run->run_id,
'level' => null,
'passed' => 0,
'failed' => 0,
'avg_user_tps' => null,
'weighted_user_tps' => null,
'total_tokens' => 0,
];
return collect([$this->emptyAgg($run)]);
}
$table = $model::make()->getTable();
if ($run->suite === 'depthsweep') {
// One suite row per thinking level; levels with no rows are
// skipped. The level badge is the identity of the row.
$levelOrder = ['off' => 0, 'low' => 1, 'medium' => 2, 'xhigh' => 3];
return collect(DB::table($table)
->where('benchmark_run_id', $run->id)
->whereNotNull('level')
->groupBy('level')
->get([
'level',
DB::raw('count(*) filter (where ok) as passed'),
DB::raw('count(*) filter (where not ok) as failed'),
DB::raw('avg(nullif(ttft, 0)) as avg_ttft'),
DB::raw('avg(nullif(wall, 0)) as avg_wall'),
DB::raw('sum(total) as total_tokens'),
DB::raw('sum(user_tps * total) / nullif(sum(total), 0) as weighted_user_tps'),
]))
->sortBy(fn ($agg) => $levelOrder[$agg->level] ?? 99)
->values()
->map(fn ($agg) => [
'run_pk' => $run->id,
'run_id' => $run->run_id,
'suite' => $run->suite,
'level' => $agg->level,
'passed' => (int) $agg->passed,
'failed' => (int) $agg->failed,
// depthsweep has no decode metrics: speed stays blank
'avg_user_tps' => null,
'weighted_user_tps' => $this->roundOrNull($agg->weighted_user_tps),
'gen_tps' => null,
'total_tokens' => (int) ($agg->total_tokens ?? 0),
'avg_ttft' => $this->roundOrNull($agg->avg_ttft),
'avg_wall' => $this->roundOrNull($agg->avg_wall),
]);
}
if ($run->suite === 'scenariobench') {
$agg = DB::table($table)->where('benchmark_run_id', $run->id)
->selectRaw('count(*) filter (where ok) as passed')
->selectRaw('count(*) filter (where not ok) as failed')
->selectRaw('sum(gen_tokens) as total_tokens')
->selectRaw('sum(gen_tokens) / nullif(sum(wall), 0) as gen_tps')
->first();
return [
return collect([[
'run_pk' => $run->id,
'run_id' => $run->run_id,
'suite' => $run->suite,
'level' => $this->runLevel($table, $run->id),
'passed' => (int) ($agg->passed ?? 0),
'failed' => (int) ($agg->failed ?? 0),
'avg_user_tps' => null,
'weighted_user_tps' => null,
'gen_tps' => $this->roundOrNull($agg->gen_tps),
'total_tokens' => (int) ($agg->total_tokens ?? 0),
];
]]);
}
$agg = DB::table($table)->where('benchmark_run_id', $run->id)
->selectRaw('count(*) filter (where ok) as passed')
->selectRaw('count(*) filter (where not ok) as failed')
->selectRaw('avg(user_tps) as avg_user_tps')
->selectRaw('sum(user_tps * total) filter (where total > 0) / nullif(sum(total) filter (where total > 0), 0) as weighted_user_tps')
->selectRaw('sum(user_tps * total) / nullif(sum(total), 0) as weighted_user_tps')
->selectRaw('sum(total) as total_tokens')
->first();
return [
return collect([[
'run_pk' => $run->id,
'run_id' => $run->run_id,
'suite' => $run->suite,
'level' => $this->runLevel($table, $run->id),
'passed' => (int) ($agg->passed ?? 0),
'failed' => (int) ($agg->failed ?? 0),
'avg_user_tps' => $agg->avg_user_tps !== null ? round((float) $agg->avg_user_tps, 1) : null,
'weighted_user_tps' => $agg->weighted_user_tps !== null ? round((float) $agg->weighted_user_tps, 1) : null,
'avg_user_tps' => $this->roundOrNull($agg->avg_user_tps),
'weighted_user_tps' => $this->roundOrNull($agg->weighted_user_tps),
'gen_tps' => null,
'total_tokens' => (int) ($agg->total_tokens ?? 0),
]]);
}
private function emptyAgg(object $run): array
{
return [
'run_pk' => $run->id,
'run_id' => $run->run_id,
'suite' => $run->suite,
'level' => null,
'passed' => 0,
'failed' => 0,
'avg_user_tps' => null,
'weighted_user_tps' => null,
'gen_tps' => null,
'total_tokens' => 0,
];
}
private function roundOrNull($value, int $precision = 1): ?float
{
return $value === null ? null : round((float) $value, $precision);
}
/**
* The run's thinking level: the distinct level across its result rows
* (a run is single-level; falls back to null when mixed or empty).

20
app/Models/BenchmarkRun.php
View file

@ -28,7 +28,7 @@ class BenchmarkRun extends Model
'run_id', 'ran_at', 'suite', 'run_key',
'model_id', 'model_quant', 'model_quant_raw', 'model_path',
'engine', 'engine_build', 'n_ctx', 'slots',
'gpu', 'gpu_count', 'gpu_id', 'gpu_source', 'host', 'client', 'endpoint',
'gpu', 'gpu_count', 'gpu_id', 'gpu_source', 'config_id', 'host', 'client', 'endpoint',
'label', 'config', 'image_digest', 'validators', 'suite_versions',
'harness', 'harness_ver', 'harness_mode', 'user_id',
'visibility', 'notes',
@ -41,7 +41,7 @@ class BenchmarkRun extends Model
* catalog is owner behavior (it links identity, it does not alter
* benchmark metrics).
*/
public const OVERLAY_FIELDS = ['visibility', 'notes', 'gpu', 'gpu_id'];
public const OVERLAY_FIELDS = ['visibility', 'notes', 'gpu', 'gpu_id', 'config_id'];
protected function casts(): array
{
@ -76,6 +76,11 @@ public function user(): BelongsTo
return $this->belongsTo(User::class);
}
public function serverConfig(): BelongsTo
{
return $this->belongsTo(ServerConfig::class, 'config_id');
}
public function softbenchResults(): HasMany
{
return $this->hasMany(SoftbenchResult::class);
@ -108,9 +113,10 @@ public function depthsweepResults(): HasMany
/**
* The suite-specific result rows for this run, via the registry. Returns
* an empty collection for suites this run does not have.
* an empty collection for suites this run does not have. An optional
* level filter narrows to one thinking level (depthsweep drill-in).
*/
public function suiteResults(): Collection
public function suiteResults(?string $level = null): Collection
{
$model = Suites::modelFor($this->suite);
@ -120,7 +126,11 @@ public function suiteResults(): Collection
$relation = Suites::runRelation($this->suite);
return $this->{$relation}()->orderBy('task')->orderBy('id')->get();
return $this->{$relation}()
->when($level !== null && $level !== '', fn ($q) => $q->where('level', $level))
->orderBy('task')
->orderBy('id')
->get();
}
public function isOwnedBy(?User $user): bool

22
app/Models/ServerConfig.php Normal file
View file

@ -0,0 +1,22 @@
<?php
namespace App\Models;
use Illuminate\Database\Eloquent\Model;
use Illuminate\Database\Eloquent\Relations\BelongsTo;
use Illuminate\Database\Eloquent\Relations\HasMany;
class ServerConfig extends Model
{
protected $fillable = ['user_id', 'name', 'engine', 'config_yaml'];
public function user(): BelongsTo
{
return $this->belongsTo(User::class);
}
public function runs(): HasMany
{
return $this->hasMany(BenchmarkRun::class);
}
}

42
database/migrations/2026_09_19_000003_create_server_configs.php Normal file
View file

@ -0,0 +1,42 @@
<?php
use Illuminate\Database\Migrations\Migration;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Support\Facades\Schema;
return new class extends Migration
{
/**
* Named, user-owned server configs ("llama.cpp DFlash spec-decode, 131k
* ctx"). The full harness args that /props cannot expose live here;
* runs link via config_id and the identity joins the session grouping
* key — same hardware + model but a different draft model or
* reasoning-budget regroups into its own session.
*/
public function up(): void
{
Schema::create('server_configs', function (Blueprint $table) {
$table->id();
$table->foreignId('user_id')->constrained()->cascadeOnDelete();
$table->string('name', 128);
$table->string('engine', 64)->default('llama.cpp');
$table->text('config_yaml')->comment('full harness args (YAML blob or argv list)');
$table->timestamps();
$table->unique(['user_id', 'name']);
});
Schema::table('benchmark_runs', function (Blueprint $table) {
$table->foreignId('config_id')->nullable()->after('gpu_source')
->constrained('server_configs')->nullOnDelete();
});
}
public function down(): void
{
Schema::table('benchmark_runs', function (Blueprint $table) {
$table->dropConstrainedForeignId('config_id');
});
Schema::dropIfExists('server_configs');
}
};

34
database/migrations/2026_09_19_000004_v23_summary_columns.php Normal file
View file

@ -0,0 +1,34 @@
<?php
use Illuminate\Database\Migrations\Migration;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Support\Facades\Schema;
return new class extends Migration
{
/**
* v2.3 summary columns: engine + config identity on bands (grouping and
* display), generated-tok/s on scenariobench suite rows.
*/
public function up(): void
{
Schema::table('session_summaries', function (Blueprint $table) {
$table->string('engine', 64)->nullable()->after('gpu_count');
$table->string('config_name', 128)->nullable()->after('engine');
});
Schema::table('session_suite_rows', function (Blueprint $table) {
$table->double('gen_tps')->nullable()->after('weighted_user_tps');
});
}
public function down(): void
{
Schema::table('session_suite_rows', function (Blueprint $table) {
$table->dropColumn('gen_tps');
});
Schema::table('session_summaries', function (Blueprint $table) {
$table->dropColumn(['engine', 'config_name']);
});
}
};

2
resources/views/components/layouts/app/sidebar.blade.php
View file

@ -68,6 +68,7 @@ class="flex h-full w-full items-center justify-center rounded-lg bg-neutral-200
<flux:menu.radio.group>
<flux:menu.item href="/settings/profile" icon="cog" wire:navigate>Settings</flux:menu.item>
<flux:menu.item href="{{ route('settings.server-configs') }}" icon="cpu" wire:navigate>Server Configs</flux:menu.item>
</flux:menu.radio.group>
<flux:menu.separator />
@ -122,6 +123,7 @@ class="flex h-full w-full items-center justify-center rounded-lg bg-neutral-200
<flux:menu.radio.group>
<flux:menu.item href="/settings/profile" icon="cog" wire:navigate>Settings</flux:menu.item>
<flux:menu.item href="{{ route('settings.server-configs') }}" icon="cpu" wire:navigate>Server Configs</flux:menu.item>
</flux:menu.radio.group>
<flux:menu.separator />

19
resources/views/livewire/runs/show.blade.php
View file

@ -12,9 +12,13 @@
public ?string $runId = null;
/** Depth-sweep drill-in filter: only this thinking level's rows. */
public ?string $level = null;
public function mount(string $run): void
{
$this->runId = $run;
$this->level = request()->query('level');
}
#[Computed]
@ -101,7 +105,11 @@ private function authorizeOwner(): void
<div><span class="text-zinc-500">{{ __('Context') }}</span><div class="font-mono">{{ number_format($this->run->n_ctx ?? 0) }}</div></div>
<div><span class="text-zinc-500">{{ __('Slots') }}</span><div class="font-mono">{{ $this->run->slots ?? '—' }}</div></div>
<div><span class="text-zinc-500">{{ __('GPU') }}</span><div>{{ ($this->run->gpu_count ? $this->run->gpu_count.'x ' : '').($this->run->gpu ?? 'unknown') }}</div></div>
<div><span class="text-zinc-500">{{ __('Inference host') }}</span><div><flux:badge size="xs" color="sky">{{ $this->run->engine ?? 'unknown' }}</flux:badge></div></div>
<div class="col-span-2"><span class="text-zinc-500">{{ __('Image digest') }}</span><div class="break-all font-mono text-xs">{{ $this->run->image_digest ?? '—' }}</div></div>
@if ($this->run->serverConfig)
<div class="col-span-2"><span class="text-zinc-500">{{ __('Server config') }}</span><div><flux:badge size="xs" color="amber">{{ $this->run->serverConfig->name }}</flux:badge></div></div>
@endif
@if ($this->run->suite_versions)
<div class="col-span-2"><span class="text-zinc-500">{{ __('Suite versions') }}</span><div class="break-all font-mono text-xs">{{ json_encode($this->run->suite_versions) }}</div></div>
@endif
@ -170,6 +178,13 @@ private function authorizeOwner(): void
</flux:table.rows>
</flux:table>
@elseif ($this->run->suite === 'depthsweep')
@if ($this->level)
<div class="mb-3 flex items-center gap-2">
<flux:text size="sm">{{ __('Thinking level') }}:</flux:text>
<flux:badge size="xs" color="sky">{{ $this->level }}</flux:badge>
<flux:button size="xs" variant="ghost" icon="x-mark" href="{{ route('runs.show', $this->run->run_id) }}" wire:navigate>{{ __('show all levels') }}</flux:button>
</div>
@endif
<flux:table>
<flux:table.columns>
<flux:table.column>{{ __('Depth') }}</flux:table.column>
@ -183,7 +198,7 @@ private function authorizeOwner(): void
</flux:table.columns>
<flux:table.rows>
@foreach ($this->run->suiteResults() as $result)
@foreach ($this->run->suiteResults($this->level) as $result)
<flux:table.row :key="$result->id">
<flux:table.cell><flux:badge size="xs" color="sky">{{ $result->section }}</flux:badge></flux:table.cell>
<flux:table.cell>{{ $result->task }}</flux:table.cell>
@ -266,4 +281,4 @@ private function authorizeOwner(): void
@endif
</div>
</div>
</div>
</div>

140
resources/views/livewire/settings/server-configs.blade.php Normal file
View file

@ -0,0 +1,140 @@
<?php
use App\Models\ServerConfig;
use Flux\Flux;
use Illuminate\Validation\Rule;
use Livewire\Attributes\Layout;
use Livewire\Component;
new #[Layout('components.layouts.app')] class extends Component {
public ?int $editingId = null;
public string $name = '';
public string $engine = 'llama.cpp';
public string $config_yaml = '';
public function create(): void
{
$this->reset(['editingId', 'name', 'config_yaml']);
$this->engine = 'llama.cpp';
$this->modal('config-form')->show();
}
public function edit(int $id): void
{
$config = ServerConfig::where('user_id', auth()->id())->findOrFail($id);
$this->editingId = $config->id;
$this->name = $config->name;
$this->engine = $config->engine;
$this->config_yaml = $config->config_yaml;
$this->modal('config-form')->show();
}
public function save(): void
{
$validated = $this->validate([
'name' => ['required', 'string', 'max:128',
Rule::unique('server_configs', 'name')->where('user_id', auth()->id())->ignore($this->editingId)],
'engine' => ['required', 'string', 'max:64'],
'config_yaml' => ['nullable', 'string', 'max:65535'],
]);
ServerConfig::updateOrCreate(
['id' => $this->editingId, 'user_id' => auth()->id()],
$validated
);
$this->modal('config-form')->close();
Flux::toast(variant: 'success', text: __('Server config saved.'));
// Config identity is part of the session grouping key — re-summarize
// so linked runs regroup immediately.
Illuminate\Support\Facades\Artisan::call('festbench:summary');
}
public function delete(int $id): void
{
ServerConfig::where('user_id', auth()->id())->whereKey($id)->firstOrFail()->delete();
Flux::toast(variant: 'success', text: __('Server config deleted.'));
Illuminate\Support\Facades\Artisan::call('festbench:summary');
}
public function with(): array
{
return [
'configs' => ServerConfig::where('user_id', auth()->id())->orderBy('name')->get(),
];
}
};
?>
<div class="flex flex-col gap-6">
<div class="flex items-center justify-between">
<div>
<flux:heading size="xl">{{ __('Server Configs') }}</flux:heading>
<flux:subheading>{{ __('Named harness configs. Runs link to them; the config identity keeps differently-tuned sessions in their own groups.') }}</flux:subheading>
</div>
<flux:button variant="primary" icon="plus" wire:click="create">{{ __('New config') }}</flux:button>
</div>
@if ($configs->isNotEmpty())
<flux:table>
<flux:table.columns>
<flux:table.column>{{ __('Name') }}</flux:table.column>
<flux:table.column>{{ __('Engine') }}</flux:table.column>
<flux:table.column class="text-right">{{ __('Linked runs') }}</flux:table.column>
<flux:table.column></flux:table.column>
</flux:table.columns>
<flux:table.rows>
@foreach ($configs as $config)
<flux:table.row :key="$config->id">
<flux:table.cell>{{ $config->name }}</flux:table.cell>
<flux:table.cell><flux:badge size="xs" color="sky">{{ $config->engine }}</flux:badge></flux:table.cell>
<flux:table.cell class="text-right font-mono text-sm">{{ $config->runs()->count() }}</flux:table.cell>
<flux:table.cell>
<div class="flex justify-end gap-1">
<flux:button size="xs" variant="ghost" icon="pencil-square" wire:click="edit({{ $config->id }})">{{ __('Edit') }}</flux:button>
<flux:button size="xs" variant="ghost" color="danger" icon="trash-2" wire:click="delete({{ $config->id }})">{{ __('Delete') }}</flux:button>
</div>
</flux:table.cell>
</flux:table.row>
@endforeach
</flux:table.rows>
</flux:table>
@else
<flux:callout variant="brand">
<flux:callout.heading>{{ __('No server configs yet') }}</flux:callout.heading>
<flux:callout.text>{{ __('Save your llama.cpp / vllm args as a named config so runs group by their tuning.') }}</flux:callout.text>
</flux:callout>
@endif
<flux:modal name="config-form" class="w-2xl">
<div class="flex flex-col gap-4">
<div>
<flux:heading size="lg">{{ $editingId ? __('Edit config') : __('New config') }}</flux:heading>
<flux:subheading>{{ __('Paste your harness args — the full server invocation (YAML or argv list).') }}</flux:subheading>
</div>
<flux:input wire:model="name" :label="__('Name')" placeholder="{{ __('llama.cpp DFlash spec-decode, 131k ctx') }}" />
<flux:select wire:model="engine" :label="__('Engine')">
@foreach (['llama.cpp', 'vllm', 'unknown', 'other'] as $engine)
<flux:select.option value="{{ $engine }}">{{ $engine }}</flux:select.option>
@endforeach
</flux:select>
<flux:textarea wire:model="config_yaml" :label="__('Harness args')" rows="12" class="font-mono text-xs" placeholder="--host 0.0.0.0 --port 8000 -hf model:Q5_K_M -ngl 99 ..." />
<div class="flex justify-end gap-2">
<flux:button variant="ghost" wire:click="$toggle('config-form')">{{ __('Cancel') }}</flux:button>
<flux:button variant="primary" wire:click="save">{{ __('Save config') }}</flux:button>
</div>
</div>
</flux:modal>
</div>

11
resources/views/livewire/summary/index.blade.php
View file

@ -151,6 +151,12 @@ class="w-64"
@if ($session->band->model_quant)
<flux:badge size="xs" color="zinc">{{ $session->band->model_quant }}</flux:badge>
@endif
@if ($session->band->engine)
<flux:badge size="xs" color="sky">{{ $session->band->engine }}</flux:badge>
@endif
@if ($session->band->config_name)
<flux:badge size="xs" color="amber">{{ $session->band->config_name }}</flux:badge>
@endif
<flux:spacer />
<span class="text-sm text-zinc-500">{{ $session->band->runner_name ?? 'anonymous' }}</span>
<span class="text-xs text-zinc-400">{{ optional($session->band->ran_at)->format('M j Y') }}</span>
@ -168,7 +174,7 @@ class="w-64"
</div>
<div class="divide-y divide-zinc-100 dark:divide-zinc-800">
@foreach ($session->suites as $row)
<a href="{{ route('runs.show', $row->run_id) }}" wire:navigate
<a href="{{ route('runs.show', $row->run_id) }}{{ $row->level && $row->suite === 'depthsweep' ? '?level='.$row->level : '' }}" wire:navigate
class="flex flex-wrap items-center gap-x-4 gap-y-1 px-4 py-2.5 hover:bg-zinc-50 dark:hover:bg-zinc-800/60">
<flux:badge size="xs">{{ $row->suite }}</flux:badge>
@if ($row->level)
@ -181,6 +187,9 @@ class="flex flex-wrap items-center gap-x-4 gap-y-1 px-4 py-2.5 hover:bg-zinc-50
@if ($row->avg_user_tps !== null)
<span class="font-mono text-sm">{{ number_format($row->avg_user_tps, 1) }} <span class="text-xs text-zinc-400">avg</span></span>
<span class="font-mono text-sm">{{ number_format($row->weighted_user_tps, 1) }} <span class="text-xs text-zinc-400">wtd tok/s</span></span>
@elseif ($row->gen_tps !== null)
<span class="font-mono text-sm">{{ number_format($row->gen_tps, 1) }} <span class="text-xs text-zinc-400">gen tok/s</span></span>
<span class="font-mono text-sm text-zinc-400">—</span>
@endif
<span class="font-mono text-sm">{{ number_format($row->total_tokens) }} <span class="text-xs text-zinc-400">tokens</span></span>
<span class="ms-auto text-xs text-zinc-400">{{ __('per-task detail') }} →</span>

1
routes/web.php
View file

@ -21,6 +21,7 @@
Volt::route('settings/profile', 'settings.profile')->name('settings.profile');
Volt::route('settings/password', 'settings.password')->name('settings.password');
Volt::route('settings/appearance', 'settings.appearance')->name('settings.appearance');
Route::livewire('settings/server-configs', 'settings.server-configs')->name('settings.server-configs');
});
require __DIR__.'/auth.php';

95
tests/Feature/ServerConfigAndSummaryTest.php Normal file
View file

@ -0,0 +1,95 @@
<?php
use App\Models\BenchmarkRun;
use App\Models\DepthsweepResult;
use App\Models\ServerConfig;
use App\Models\SoftbenchResult;
use App\Models\User;
use Illuminate\Support\Facades\Artisan;
use Illuminate\Support\Facades\DB;
test('engine and server config split otherwise identical session bands', function () {
$owner = User::factory()->create();
$first = ServerConfig::create([
'user_id' => $owner->id,
'name' => 'DFlash Q4 draft',
'engine' => 'llama.cpp',
'config_yaml' => '--spec-draft-hf z-lab/Qwen-DFlash:Q4_K_M',
]);
$second = ServerConfig::create([
'user_id' => $owner->id,
'name' => 'No draft',
'engine' => 'llama.cpp',
'config_yaml' => '--no-speculative',
]);
foreach ([$first, $second] as $config) {
$run = BenchmarkRun::factory()->create([
'suite' => 'softbench',
'user_id' => $owner->id,
'label' => 'same test label',
'model_id' => 'same/model',
'model_quant' => 'Q5_K_M',
'gpu' => 'AMD Radeon AI PRO R9700',
'gpu_count' => 1,
'engine' => 'llama.cpp',
'config_id' => $config->id,
]);
SoftbenchResult::factory()->create(['benchmark_run_id' => $run->id]);
}
Artisan::call('festbench:summary');
expect(DB::table('session_summaries')->count())->toBe(2)
->and(DB::table('session_summaries')->pluck('engine')->unique()->all())->toBe(['llama.cpp'])
->and(DB::table('session_summaries')->pluck('config_name')->sort()->values()->all())
->toBe(['DFlash Q4 draft', 'No draft']);
});
test('depthsweep summary expands one run into a row for each thinking level', function () {
$run = BenchmarkRun::factory()->create(['suite' => 'depthsweep', 'visibility' => 'public']);
foreach (['off', 'low', 'medium', 'xhigh'] as $level) {
DepthsweepResult::factory()->count(2)->create([
'benchmark_run_id' => $run->id,
'level' => $level,
'total' => null,
'user_tps' => null,
]);
}
Artisan::call('festbench:summary');
$rows = DB::table('session_suite_rows')->where('suite', 'depthsweep')->orderBy('sequence')->get();
expect($rows)->toHaveCount(4)
->and($rows->pluck('level')->all())->toBe(['off', 'low', 'medium', 'xhigh']);
expect($rows->pluck('avg_user_tps')->filter())->toBeEmpty();
});
test('depthsweep run detail honors a thinking level query filter', function () {
$run = BenchmarkRun::factory()->create(['suite' => 'depthsweep', 'visibility' => 'public']);
DepthsweepResult::factory()->create([
'benchmark_run_id' => $run->id,
'level' => 'off',
'task' => 'off-only-task',
]);
DepthsweepResult::factory()->create([
'benchmark_run_id' => $run->id,
'level' => 'xhigh',
'task' => 'xhigh-only-task',
]);
$this->get(route('runs.show', $run->run_id).'?level=off')
->assertOk()
->assertSee('Thinking level')
->assertSee('off-only-task')
->assertDontSee('xhigh-only-task');
});
test('server configs settings page is available to authenticated users', function () {
$this->actingAs(User::factory()->create())
->get(route('settings.server-configs'))
->assertOk()
->assertSee('Server Configs');
});

31
tests/Feature/SessionSummaryTest.php
View file

@ -6,14 +6,15 @@
use App\Models\User;
use Illuminate\Support\Facades\Artisan;
test('festbench:summary groups runs into session bands with weighted tok/s', function () {
test('festbench:summary groups matching runs despite different labels', function () {
$me = User::factory()->create(['name' => 'peterZ']);
// One session: two softbench runs sharing label/model/gpu/user, plus a
// different-session run (different label).
// All three runs share real configuration identity. Labels are user
// notes, not grouping identity, so the different session B label must
// not fork a second bubble.
$a = BenchmarkRun::factory()->create([
'suite' => 'softbench', 'label' => 'session A', 'model_id' => 'm', 'model_quant' => 'Q5_K_M',
'gpu' => 'AMD R9700', 'gpu_count' => 1, 'user_id' => $me->id, 'visibility' => 'public',
'gpu' => 'AMD R9700', 'gpu_count' => 1, 'engine' => 'llama.cpp', 'user_id' => $me->id, 'visibility' => 'public',
'ran_at' => '2026-09-03 12:00:00',
]);
SoftbenchResult::factory()->create(['benchmark_run_id' => $a->id, 'ok' => true, 'user_tps' => 60.0, 'total' => 100, 'level' => 'off']);
@ -21,14 +22,14 @@
$b = BenchmarkRun::factory()->create([
'suite' => 'infrabench', 'label' => 'session A', 'model_id' => 'm', 'model_quant' => 'Q5_K_M',
'gpu' => 'AMD R9700', 'gpu_count' => 1, 'user_id' => $me->id, 'visibility' => 'public',
'gpu' => 'AMD R9700', 'gpu_count' => 1, 'engine' => 'llama.cpp', 'user_id' => $me->id, 'visibility' => 'public',
'ran_at' => '2026-09-03 12:05:00',
]);
InfrabenchResult::factory()->create(['benchmark_run_id' => $b->id, 'ok' => true, 'user_tps' => 40.0, 'total' => 200, 'level' => 'low']);
$c = BenchmarkRun::factory()->create([
'suite' => 'softbench', 'label' => 'session B', 'model_id' => 'm', 'model_quant' => 'Q5_K_M',
'gpu' => 'AMD R9700', 'gpu_count' => 1, 'user_id' => $me->id, 'visibility' => 'public',
'gpu' => 'AMD R9700', 'gpu_count' => 1, 'engine' => 'llama.cpp', 'user_id' => $me->id, 'visibility' => 'public',
'ran_at' => '2026-09-04 12:00:00',
]);
SoftbenchResult::factory()->create(['benchmark_run_id' => $c->id, 'ok' => true, 'user_tps' => 50.0, 'total' => 50, 'level' => 'medium']);
@ -36,20 +37,22 @@
Artisan::call('festbench:summary');
$bands = DB::table('session_summaries')->orderBy('sequence')->get();
expect($bands)->toHaveCount(2);
expect($bands)->toHaveCount(1);
$bandA = $bands->firstWhere('label', 'session A');
$bandA = $bands->first();
expect($bandA->runner_name)->toBe('peterZ')
->and($bandA->gpu)->toBe('AMD R9700')
->and($bandA->gpu_count)->toBe(1)
->and($bandA->total_passed)->toBe(2)
// Mixed labels intentionally show no single label on the band.
->and($bandA->label)->toBeNull()
->and($bandA->total_passed)->toBe(3)
->and($bandA->total_failed)->toBe(1)
->and($bandA->total_tokens)->toBe(600);
->and($bandA->total_tokens)->toBe(650);
$rowsA = DB::table('session_suite_rows')->where('session_summary_id', $bandA->id)->orderBy('sequence')->get();
expect($rowsA)->toHaveCount(2);
expect($rowsA)->toHaveCount(3);
$soft = $rowsA->firstWhere('suite', 'softbench');
$soft = $rowsA->where('suite', 'softbench')->firstWhere('level', 'off');
// simple avg of (60, 20) = 40; weighted = (60*100 + 20*300)/400 = 30
expect($soft->passed)->toBe(1)
->and($soft->failed)->toBe(1)
@ -61,8 +64,8 @@
$infra = $rowsA->firstWhere('suite', 'infrabench');
expect($infra->passed)->toBe(1)->and($infra->level)->toBe('low');
// Newest session first
expect($bands->first()->label)->toBe('session B');
$secondSoft = $rowsA->where('suite', 'softbench')->firstWhere('level', 'medium');
expect($secondSoft->passed)->toBe(1)->and($secondSoft->total_tokens)->toBe(50);
});
test('private runs form their own band and are hidden from guests', function () {