diff --git a/app/Console/Commands/FestbenchImport.php b/app/Console/Commands/FestbenchImport.php index 6ccc87a..67cdfd3 100644 --- a/app/Console/Commands/FestbenchImport.php +++ b/app/Console/Commands/FestbenchImport.php @@ -4,6 +4,7 @@ use App\Models\BenchmarkRun; use App\Models\Gpu; +use App\Models\ServerConfig; use App\Models\User; use App\Support\Suites; use Illuminate\Console\Command; @@ -23,6 +24,9 @@ class FestbenchImport extends Command {--label= : Only import runs whose label matches exactly} {--run-id= : Comma-separated list of run_ids to import} {--gpu= : Override the GPU name on imported runs (backfill or correction)} + {--gpu-count= : Override GPU count when --gpu is supplied (defaults to 1 if source count is 0)} + {--relabel= : Replace the label on imported runs (build one session from many)} + {--config-id= : Link imported runs to a server config by id} {--private : Import runs as private instead of public} {--fresh : Delete ALL existing benchmark data before importing}'; @@ -136,6 +140,19 @@ private function createRun(array $source, ?User $user): BenchmarkRun // source row keeps its provenance inside payload/run_key untouched. $gpuName = $this->option('gpu') ?? ($source['gpu'] ?? null); $gpu = Gpu::match($gpuName); + $gpuCount = $this->option('gpu-count') ?? ($source['gpu_count'] ?? null); + if ($this->option('gpu') !== null && ((int) $gpuCount) < 1) { + $gpuCount = 1; + } + + // --relabel rewrites the session glue so runs imported together form + // one band; --config-id links the named server config for grouping. + $label = $this->option('relabel') ?? (($source['label'] ?? '') !== '' ? $source['label'] : null); + $configId = $this->option('config-id'); + if ($configId !== null && ! ServerConfig::whereKey((int) $configId)->exists()) { + $this->warnOnce("Server config {$configId} not found -- runs will import without a config link."); + $configId = null; + } if ($gpuName !== null && $gpu === null && ! in_array(strtolower((string) $gpuName), ['unknown', 'hosted', 'cloud'], true)) { $this->warnOnce("GPU '{$gpuName}' is not in the catalog (run {$source['run_id']}) — it can be confirmed on the website later."); @@ -155,13 +172,14 @@ private function createRun(array $source, ?User $user): BenchmarkRun 'n_ctx' => $source['n_ctx'] ?? null, 'slots' => $source['slots'] ?? null, 'gpu' => $gpuName, - 'gpu_count' => $source['gpu_count'] ?? null, + 'gpu_count' => $gpuCount, 'gpu_id' => $gpu?->id, 'gpu_source' => $this->option('gpu') ? ($source['gpu_source'] ?? 'backfill-manual') : ($source['gpu_source'] ?? null), 'host' => $source['host'] ?? null, 'client' => $source['client'] ?? null, 'endpoint' => $source['endpoint'] ?? null, - 'label' => ($source['label'] ?? '') !== '' ? $source['label'] : null, + 'label' => $label, + 'config_id' => $configId, 'config' => $source['config'] ?? null, 'image_digest' => $source['image_digest'] ?? null, 'validators' => $this->decodeJson($source['validators'] ?? null), diff --git a/app/Console/Commands/SessionSummary.php b/app/Console/Commands/SessionSummary.php index 98516b7..e9345ef 100644 --- a/app/Console/Commands/SessionSummary.php +++ b/app/Console/Commands/SessionSummary.php @@ -4,6 +4,7 @@ use App\Support\Suites; use Illuminate\Console\Command; +use Illuminate\Support\Collection; use Illuminate\Support\Facades\DB; use Symfony\Component\Console\Attribute\AsCommand; @@ -16,30 +17,42 @@ public function handle(): int $runs = DB::table('benchmark_runs') ->leftJoin('users', 'users.id', '=', 'benchmark_runs.user_id') + ->leftJoin('server_configs', 'server_configs.id', '=', 'benchmark_runs.config_id') ->orderByDesc('ran_at') ->orderByDesc('benchmark_runs.id') ->get([ 'benchmark_runs.*', 'users.name as runner_name', + 'server_configs.name as config_name', ]); foreach ($runs as $run) { - // Visibility participates in the grouping key so private runs - // form their own band — the guest view can then filter bands by - // visibility without any per-request recomputation. + // Grouping key: user + GPU + model + quant + visibility + engine + // + harness identity + named server config. Free-text labels are + // intentionally NOT part of identity: rerunning psychbench with + // the same actual settings belongs in the same bubble even when + // the user wrote a different run note. $key = implode('|', [ - $run->label ?? '-', $run->model_id ?? '-', $run->model_quant ?? '-', - $run->gpu ?? '-', + $run->gpu_id ?? 'raw:'.$run->gpu, $run->gpu_count ?? '-', $run->user_id ?? '-', $run->visibility, + $run->engine ?? '-', + $run->engine_build ?: '-', + // Older direct runs stored empty harness fields; newer + // depthsweep writes the explicit equivalent. Normalize so + // semantically identical direct runs share a session. + $run->harness ?: 'direct', + $run->harness_ver ?: '-', + $run->harness_mode ?: 'one-shot', + $run->config_id ?? 'none', ]); $sessions[$key] ??= [ 'key' => $key, - 'label' => $run->label, + 'labels' => collect([$run->label])->filter(), 'model_id' => $run->model_id, 'model_quant' => $run->model_quant, 'gpu' => $run->gpu, @@ -47,6 +60,8 @@ public function handle(): int 'runner_name' => $run->runner_name, 'user_id' => $run->user_id, 'visibility' => $run->visibility, + 'engine' => $run->engine, + 'config_name' => $run->config_name, 'ran_at' => $run->ran_at, 'passed' => 0, 'failed' => 0, @@ -54,12 +69,16 @@ public function handle(): int 'runs' => collect(), ]; - $agg = $this->aggregateRun($run); + if ($run->label !== null && $run->label !== '') { + $sessions[$key]['labels']->push($run->label); + } - $sessions[$key]['passed'] += $agg['passed']; - $sessions[$key]['failed'] += $agg['failed']; - $sessions[$key]['tokens'] += $agg['total_tokens']; - $sessions[$key]['runs']->push($agg + ['suite' => $run->suite, 'level' => $agg['level']]); + $aggs = $this->aggregateRun($run); + + $sessions[$key]['passed'] += $aggs->sum('passed'); + $sessions[$key]['failed'] += $aggs->sum('failed'); + $sessions[$key]['tokens'] += $aggs->sum('total_tokens'); + $sessions[$key]['runs'] = $sessions[$key]['runs']->merge($aggs); } DB::transaction(function () use ($sessions) { @@ -68,7 +87,11 @@ public function handle(): int foreach (array_values($sessions) as $bandIndex => $band) { $summaryId = DB::table('session_summaries')->insertGetId([ 'session_key' => $band['key'], - 'label' => $band['label'], + // Keep a label only when every grouped run agrees. A + // mixed label is a note, not a configuration split. + 'label' => $band['labels']->unique()->count() === 1 + ? $band['labels']->first() + : null, 'model_id' => $band['model_id'], 'model_quant' => $band['model_quant'], 'gpu' => $band['gpu'], @@ -76,6 +99,8 @@ public function handle(): int 'runner_name' => $band['runner_name'], 'user_id' => $band['user_id'], 'visibility' => $band['visibility'], + 'engine' => $band['engine'], + 'config_name' => $band['config_name'], 'ran_at' => $band['ran_at'], 'total_passed' => $band['passed'], 'total_failed' => $band['failed'], @@ -85,7 +110,7 @@ public function handle(): int 'updated_at' => now(), ]); - foreach ($band['runs'] as $suiteIndex => $row) { + foreach ($band['runs']->values() as $suiteIndex => $row) { DB::table('session_suite_rows')->insert([ 'session_summary_id' => $summaryId, 'benchmark_run_id' => $row['run_pk'], @@ -96,6 +121,7 @@ public function handle(): int 'failed' => $row['failed'], 'avg_user_tps' => $row['avg_user_tps'], 'weighted_user_tps' => $row['weighted_user_tps'], + 'gen_tps' => $row['gen_tps'], 'total_tokens' => $row['total_tokens'], 'sequence' => $suiteIndex, 'created_at' => now(), @@ -111,67 +137,124 @@ public function handle(): int } /** - * Aggregate one run's suite table: pass/fail counts, simple + weighted - * user tok/s, generated tokens, and the run's thinking level. + * Aggregate one run. Returns a collection: normally one row, but a + * depthsweep run expands into one row per thinking level — each level's + * rows aggregate independently so the band renders level-by-level. + * + * @return Collection */ - private function aggregateRun(object $run): array + private function aggregateRun(object $run) { $model = Suites::modelFor($run->suite); if ($model === null) { - return [ - 'run_pk' => $run->id, - 'run_id' => $run->run_id, - 'level' => null, - 'passed' => 0, - 'failed' => 0, - 'avg_user_tps' => null, - 'weighted_user_tps' => null, - 'total_tokens' => 0, - ]; + return collect([$this->emptyAgg($run)]); } $table = $model::make()->getTable(); + if ($run->suite === 'depthsweep') { + // One suite row per thinking level; levels with no rows are + // skipped. The level badge is the identity of the row. + $levelOrder = ['off' => 0, 'low' => 1, 'medium' => 2, 'xhigh' => 3]; + + return collect(DB::table($table) + ->where('benchmark_run_id', $run->id) + ->whereNotNull('level') + ->groupBy('level') + ->get([ + 'level', + DB::raw('count(*) filter (where ok) as passed'), + DB::raw('count(*) filter (where not ok) as failed'), + DB::raw('avg(nullif(ttft, 0)) as avg_ttft'), + DB::raw('avg(nullif(wall, 0)) as avg_wall'), + DB::raw('sum(total) as total_tokens'), + DB::raw('sum(user_tps * total) / nullif(sum(total), 0) as weighted_user_tps'), + ])) + ->sortBy(fn ($agg) => $levelOrder[$agg->level] ?? 99) + ->values() + ->map(fn ($agg) => [ + 'run_pk' => $run->id, + 'run_id' => $run->run_id, + 'suite' => $run->suite, + 'level' => $agg->level, + 'passed' => (int) $agg->passed, + 'failed' => (int) $agg->failed, + // depthsweep has no decode metrics: speed stays blank + 'avg_user_tps' => null, + 'weighted_user_tps' => $this->roundOrNull($agg->weighted_user_tps), + 'gen_tps' => null, + 'total_tokens' => (int) ($agg->total_tokens ?? 0), + 'avg_ttft' => $this->roundOrNull($agg->avg_ttft), + 'avg_wall' => $this->roundOrNull($agg->avg_wall), + ]); + } + if ($run->suite === 'scenariobench') { $agg = DB::table($table)->where('benchmark_run_id', $run->id) ->selectRaw('count(*) filter (where ok) as passed') ->selectRaw('count(*) filter (where not ok) as failed') ->selectRaw('sum(gen_tokens) as total_tokens') + ->selectRaw('sum(gen_tokens) / nullif(sum(wall), 0) as gen_tps') ->first(); - return [ + return collect([[ 'run_pk' => $run->id, 'run_id' => $run->run_id, + 'suite' => $run->suite, 'level' => $this->runLevel($table, $run->id), 'passed' => (int) ($agg->passed ?? 0), 'failed' => (int) ($agg->failed ?? 0), 'avg_user_tps' => null, 'weighted_user_tps' => null, + 'gen_tps' => $this->roundOrNull($agg->gen_tps), 'total_tokens' => (int) ($agg->total_tokens ?? 0), - ]; + ]]); } $agg = DB::table($table)->where('benchmark_run_id', $run->id) ->selectRaw('count(*) filter (where ok) as passed') ->selectRaw('count(*) filter (where not ok) as failed') ->selectRaw('avg(user_tps) as avg_user_tps') - ->selectRaw('sum(user_tps * total) filter (where total > 0) / nullif(sum(total) filter (where total > 0), 0) as weighted_user_tps') + ->selectRaw('sum(user_tps * total) / nullif(sum(total), 0) as weighted_user_tps') ->selectRaw('sum(total) as total_tokens') ->first(); - return [ + return collect([[ 'run_pk' => $run->id, 'run_id' => $run->run_id, + 'suite' => $run->suite, 'level' => $this->runLevel($table, $run->id), 'passed' => (int) ($agg->passed ?? 0), 'failed' => (int) ($agg->failed ?? 0), - 'avg_user_tps' => $agg->avg_user_tps !== null ? round((float) $agg->avg_user_tps, 1) : null, - 'weighted_user_tps' => $agg->weighted_user_tps !== null ? round((float) $agg->weighted_user_tps, 1) : null, + 'avg_user_tps' => $this->roundOrNull($agg->avg_user_tps), + 'weighted_user_tps' => $this->roundOrNull($agg->weighted_user_tps), + 'gen_tps' => null, 'total_tokens' => (int) ($agg->total_tokens ?? 0), + ]]); + } + + private function emptyAgg(object $run): array + { + return [ + 'run_pk' => $run->id, + 'run_id' => $run->run_id, + 'suite' => $run->suite, + 'level' => null, + 'passed' => 0, + 'failed' => 0, + 'avg_user_tps' => null, + 'weighted_user_tps' => null, + 'gen_tps' => null, + 'total_tokens' => 0, ]; } + private function roundOrNull($value, int $precision = 1): ?float + { + return $value === null ? null : round((float) $value, $precision); + } + /** * The run's thinking level: the distinct level across its result rows * (a run is single-level; falls back to null when mixed or empty). diff --git a/app/Models/BenchmarkRun.php b/app/Models/BenchmarkRun.php index cf3f9c7..6210561 100644 --- a/app/Models/BenchmarkRun.php +++ b/app/Models/BenchmarkRun.php @@ -28,7 +28,7 @@ class BenchmarkRun extends Model 'run_id', 'ran_at', 'suite', 'run_key', 'model_id', 'model_quant', 'model_quant_raw', 'model_path', 'engine', 'engine_build', 'n_ctx', 'slots', - 'gpu', 'gpu_count', 'gpu_id', 'gpu_source', 'host', 'client', 'endpoint', + 'gpu', 'gpu_count', 'gpu_id', 'gpu_source', 'config_id', 'host', 'client', 'endpoint', 'label', 'config', 'image_digest', 'validators', 'suite_versions', 'harness', 'harness_ver', 'harness_mode', 'user_id', 'visibility', 'notes', @@ -41,7 +41,7 @@ class BenchmarkRun extends Model * catalog is owner behavior (it links identity, it does not alter * benchmark metrics). */ - public const OVERLAY_FIELDS = ['visibility', 'notes', 'gpu', 'gpu_id']; + public const OVERLAY_FIELDS = ['visibility', 'notes', 'gpu', 'gpu_id', 'config_id']; protected function casts(): array { @@ -76,6 +76,11 @@ public function user(): BelongsTo return $this->belongsTo(User::class); } + public function serverConfig(): BelongsTo + { + return $this->belongsTo(ServerConfig::class, 'config_id'); + } + public function softbenchResults(): HasMany { return $this->hasMany(SoftbenchResult::class); @@ -108,9 +113,10 @@ public function depthsweepResults(): HasMany /** * The suite-specific result rows for this run, via the registry. Returns - * an empty collection for suites this run does not have. + * an empty collection for suites this run does not have. An optional + * level filter narrows to one thinking level (depthsweep drill-in). */ - public function suiteResults(): Collection + public function suiteResults(?string $level = null): Collection { $model = Suites::modelFor($this->suite); @@ -120,7 +126,11 @@ public function suiteResults(): Collection $relation = Suites::runRelation($this->suite); - return $this->{$relation}()->orderBy('task')->orderBy('id')->get(); + return $this->{$relation}() + ->when($level !== null && $level !== '', fn ($q) => $q->where('level', $level)) + ->orderBy('task') + ->orderBy('id') + ->get(); } public function isOwnedBy(?User $user): bool diff --git a/app/Models/ServerConfig.php b/app/Models/ServerConfig.php new file mode 100644 index 0000000..ba3dcdf --- /dev/null +++ b/app/Models/ServerConfig.php @@ -0,0 +1,22 @@ +belongsTo(User::class); + } + + public function runs(): HasMany + { + return $this->hasMany(BenchmarkRun::class); + } +} diff --git a/database/migrations/2026_09_19_000003_create_server_configs.php b/database/migrations/2026_09_19_000003_create_server_configs.php new file mode 100644 index 0000000..6102a3a --- /dev/null +++ b/database/migrations/2026_09_19_000003_create_server_configs.php @@ -0,0 +1,42 @@ +id(); + $table->foreignId('user_id')->constrained()->cascadeOnDelete(); + $table->string('name', 128); + $table->string('engine', 64)->default('llama.cpp'); + $table->text('config_yaml')->comment('full harness args (YAML blob or argv list)'); + $table->timestamps(); + + $table->unique(['user_id', 'name']); + }); + + Schema::table('benchmark_runs', function (Blueprint $table) { + $table->foreignId('config_id')->nullable()->after('gpu_source') + ->constrained('server_configs')->nullOnDelete(); + }); + } + + public function down(): void + { + Schema::table('benchmark_runs', function (Blueprint $table) { + $table->dropConstrainedForeignId('config_id'); + }); + Schema::dropIfExists('server_configs'); + } +}; diff --git a/database/migrations/2026_09_19_000004_v23_summary_columns.php b/database/migrations/2026_09_19_000004_v23_summary_columns.php new file mode 100644 index 0000000..b86b3bc --- /dev/null +++ b/database/migrations/2026_09_19_000004_v23_summary_columns.php @@ -0,0 +1,34 @@ +string('engine', 64)->nullable()->after('gpu_count'); + $table->string('config_name', 128)->nullable()->after('engine'); + }); + + Schema::table('session_suite_rows', function (Blueprint $table) { + $table->double('gen_tps')->nullable()->after('weighted_user_tps'); + }); + } + + public function down(): void + { + Schema::table('session_suite_rows', function (Blueprint $table) { + $table->dropColumn('gen_tps'); + }); + Schema::table('session_summaries', function (Blueprint $table) { + $table->dropColumn(['engine', 'config_name']); + }); + } +}; diff --git a/resources/views/components/layouts/app/sidebar.blade.php b/resources/views/components/layouts/app/sidebar.blade.php index 473f9fa..6caf4ef 100644 --- a/resources/views/components/layouts/app/sidebar.blade.php +++ b/resources/views/components/layouts/app/sidebar.blade.php @@ -68,6 +68,7 @@ class="flex h-full w-full items-center justify-center rounded-lg bg-neutral-200 Settings + Server Configs @@ -122,6 +123,7 @@ class="flex h-full w-full items-center justify-center rounded-lg bg-neutral-200 Settings + Server Configs diff --git a/resources/views/livewire/runs/show.blade.php b/resources/views/livewire/runs/show.blade.php index e3e5df0..4fdcb4d 100644 --- a/resources/views/livewire/runs/show.blade.php +++ b/resources/views/livewire/runs/show.blade.php @@ -12,9 +12,13 @@ public ?string $runId = null; + /** Depth-sweep drill-in filter: only this thinking level's rows. */ + public ?string $level = null; + public function mount(string $run): void { $this->runId = $run; + $this->level = request()->query('level'); } #[Computed] @@ -101,7 +105,11 @@ private function authorizeOwner(): void
{{ __('Context') }}
{{ number_format($this->run->n_ctx ?? 0) }}
{{ __('Slots') }}
{{ $this->run->slots ?? '—' }}
{{ __('GPU') }}
{{ ($this->run->gpu_count ? $this->run->gpu_count.'x ' : '').($this->run->gpu ?? 'unknown') }}
+
{{ __('Inference host') }}
{{ $this->run->engine ?? 'unknown' }}
{{ __('Image digest') }}
{{ $this->run->image_digest ?? '—' }}
+ @if ($this->run->serverConfig) +
{{ __('Server config') }}
{{ $this->run->serverConfig->name }}
+ @endif @if ($this->run->suite_versions)
{{ __('Suite versions') }}
{{ json_encode($this->run->suite_versions) }}
@endif @@ -170,6 +178,13 @@ private function authorizeOwner(): void @elseif ($this->run->suite === 'depthsweep') + @if ($this->level) +
+ {{ __('Thinking level') }}: + {{ $this->level }} + {{ __('show all levels') }} +
+ @endif {{ __('Depth') }} @@ -183,7 +198,7 @@ private function authorizeOwner(): void - @foreach ($this->run->suiteResults() as $result) + @foreach ($this->run->suiteResults($this->level) as $result) {{ $result->section }} {{ $result->task }} @@ -266,4 +281,4 @@ private function authorizeOwner(): void @endif - \ No newline at end of file + diff --git a/resources/views/livewire/settings/server-configs.blade.php b/resources/views/livewire/settings/server-configs.blade.php new file mode 100644 index 0000000..56f44f8 --- /dev/null +++ b/resources/views/livewire/settings/server-configs.blade.php @@ -0,0 +1,140 @@ +reset(['editingId', 'name', 'config_yaml']); + $this->engine = 'llama.cpp'; + $this->modal('config-form')->show(); + } + + public function edit(int $id): void + { + $config = ServerConfig::where('user_id', auth()->id())->findOrFail($id); + + $this->editingId = $config->id; + $this->name = $config->name; + $this->engine = $config->engine; + $this->config_yaml = $config->config_yaml; + + $this->modal('config-form')->show(); + } + + public function save(): void + { + $validated = $this->validate([ + 'name' => ['required', 'string', 'max:128', + Rule::unique('server_configs', 'name')->where('user_id', auth()->id())->ignore($this->editingId)], + 'engine' => ['required', 'string', 'max:64'], + 'config_yaml' => ['nullable', 'string', 'max:65535'], + ]); + + ServerConfig::updateOrCreate( + ['id' => $this->editingId, 'user_id' => auth()->id()], + $validated + ); + + $this->modal('config-form')->close(); + Flux::toast(variant: 'success', text: __('Server config saved.')); + + // Config identity is part of the session grouping key — re-summarize + // so linked runs regroup immediately. + Illuminate\Support\Facades\Artisan::call('festbench:summary'); + } + + public function delete(int $id): void + { + ServerConfig::where('user_id', auth()->id())->whereKey($id)->firstOrFail()->delete(); + + Flux::toast(variant: 'success', text: __('Server config deleted.')); + Illuminate\Support\Facades\Artisan::call('festbench:summary'); + } + + public function with(): array + { + return [ + 'configs' => ServerConfig::where('user_id', auth()->id())->orderBy('name')->get(), + ]; + } +}; +?> + +
+
+
+ {{ __('Server Configs') }} + {{ __('Named harness configs. Runs link to them; the config identity keeps differently-tuned sessions in their own groups.') }} +
+ {{ __('New config') }} +
+ + @if ($configs->isNotEmpty()) + + + {{ __('Name') }} + {{ __('Engine') }} + {{ __('Linked runs') }} + + + + + @foreach ($configs as $config) + + {{ $config->name }} + {{ $config->engine }} + {{ $config->runs()->count() }} + +
+ {{ __('Edit') }} + {{ __('Delete') }} +
+
+
+ @endforeach +
+
+ @else + + {{ __('No server configs yet') }} + {{ __('Save your llama.cpp / vllm args as a named config so runs group by their tuning.') }} + + @endif + + +
+
+ {{ $editingId ? __('Edit config') : __('New config') }} + {{ __('Paste your harness args — the full server invocation (YAML or argv list).') }} +
+ + + + + @foreach (['llama.cpp', 'vllm', 'unknown', 'other'] as $engine) + {{ $engine }} + @endforeach + + + + +
+ {{ __('Cancel') }} + {{ __('Save config') }} +
+
+
+
\ No newline at end of file diff --git a/resources/views/livewire/summary/index.blade.php b/resources/views/livewire/summary/index.blade.php index a1c836d..365cfa0 100644 --- a/resources/views/livewire/summary/index.blade.php +++ b/resources/views/livewire/summary/index.blade.php @@ -151,6 +151,12 @@ class="w-64" @if ($session->band->model_quant) {{ $session->band->model_quant }} @endif + @if ($session->band->engine) + {{ $session->band->engine }} + @endif + @if ($session->band->config_name) + {{ $session->band->config_name }} + @endif {{ $session->band->runner_name ?? 'anonymous' }} {{ optional($session->band->ran_at)->format('M j Y') }} @@ -168,7 +174,7 @@ class="w-64"
@foreach ($session->suites as $row) - {{ $row->suite }} @if ($row->level) @@ -181,6 +187,9 @@ class="flex flex-wrap items-center gap-x-4 gap-y-1 px-4 py-2.5 hover:bg-zinc-50 @if ($row->avg_user_tps !== null) {{ number_format($row->avg_user_tps, 1) }} avg {{ number_format($row->weighted_user_tps, 1) }} wtd tok/s + @elseif ($row->gen_tps !== null) + {{ number_format($row->gen_tps, 1) }} gen tok/s + — @endif {{ number_format($row->total_tokens) }} tokens {{ __('per-task detail') }} → diff --git a/routes/web.php b/routes/web.php index 26b8f1f..6b26c46 100644 --- a/routes/web.php +++ b/routes/web.php @@ -21,6 +21,7 @@ Volt::route('settings/profile', 'settings.profile')->name('settings.profile'); Volt::route('settings/password', 'settings.password')->name('settings.password'); Volt::route('settings/appearance', 'settings.appearance')->name('settings.appearance'); + Route::livewire('settings/server-configs', 'settings.server-configs')->name('settings.server-configs'); }); require __DIR__.'/auth.php'; diff --git a/tests/Feature/ServerConfigAndSummaryTest.php b/tests/Feature/ServerConfigAndSummaryTest.php new file mode 100644 index 0000000..2441331 --- /dev/null +++ b/tests/Feature/ServerConfigAndSummaryTest.php @@ -0,0 +1,95 @@ +create(); + $first = ServerConfig::create([ + 'user_id' => $owner->id, + 'name' => 'DFlash Q4 draft', + 'engine' => 'llama.cpp', + 'config_yaml' => '--spec-draft-hf z-lab/Qwen-DFlash:Q4_K_M', + ]); + $second = ServerConfig::create([ + 'user_id' => $owner->id, + 'name' => 'No draft', + 'engine' => 'llama.cpp', + 'config_yaml' => '--no-speculative', + ]); + + foreach ([$first, $second] as $config) { + $run = BenchmarkRun::factory()->create([ + 'suite' => 'softbench', + 'user_id' => $owner->id, + 'label' => 'same test label', + 'model_id' => 'same/model', + 'model_quant' => 'Q5_K_M', + 'gpu' => 'AMD Radeon AI PRO R9700', + 'gpu_count' => 1, + 'engine' => 'llama.cpp', + 'config_id' => $config->id, + ]); + SoftbenchResult::factory()->create(['benchmark_run_id' => $run->id]); + } + + Artisan::call('festbench:summary'); + + expect(DB::table('session_summaries')->count())->toBe(2) + ->and(DB::table('session_summaries')->pluck('engine')->unique()->all())->toBe(['llama.cpp']) + ->and(DB::table('session_summaries')->pluck('config_name')->sort()->values()->all()) + ->toBe(['DFlash Q4 draft', 'No draft']); +}); + +test('depthsweep summary expands one run into a row for each thinking level', function () { + $run = BenchmarkRun::factory()->create(['suite' => 'depthsweep', 'visibility' => 'public']); + + foreach (['off', 'low', 'medium', 'xhigh'] as $level) { + DepthsweepResult::factory()->count(2)->create([ + 'benchmark_run_id' => $run->id, + 'level' => $level, + 'total' => null, + 'user_tps' => null, + ]); + } + + Artisan::call('festbench:summary'); + + $rows = DB::table('session_suite_rows')->where('suite', 'depthsweep')->orderBy('sequence')->get(); + + expect($rows)->toHaveCount(4) + ->and($rows->pluck('level')->all())->toBe(['off', 'low', 'medium', 'xhigh']); + expect($rows->pluck('avg_user_tps')->filter())->toBeEmpty(); +}); + +test('depthsweep run detail honors a thinking level query filter', function () { + $run = BenchmarkRun::factory()->create(['suite' => 'depthsweep', 'visibility' => 'public']); + DepthsweepResult::factory()->create([ + 'benchmark_run_id' => $run->id, + 'level' => 'off', + 'task' => 'off-only-task', + ]); + DepthsweepResult::factory()->create([ + 'benchmark_run_id' => $run->id, + 'level' => 'xhigh', + 'task' => 'xhigh-only-task', + ]); + + $this->get(route('runs.show', $run->run_id).'?level=off') + ->assertOk() + ->assertSee('Thinking level') + ->assertSee('off-only-task') + ->assertDontSee('xhigh-only-task'); +}); + +test('server configs settings page is available to authenticated users', function () { + $this->actingAs(User::factory()->create()) + ->get(route('settings.server-configs')) + ->assertOk() + ->assertSee('Server Configs'); +}); diff --git a/tests/Feature/SessionSummaryTest.php b/tests/Feature/SessionSummaryTest.php index f8b24d4..52f3126 100644 --- a/tests/Feature/SessionSummaryTest.php +++ b/tests/Feature/SessionSummaryTest.php @@ -6,14 +6,15 @@ use App\Models\User; use Illuminate\Support\Facades\Artisan; -test('festbench:summary groups runs into session bands with weighted tok/s', function () { +test('festbench:summary groups matching runs despite different labels', function () { $me = User::factory()->create(['name' => 'peterZ']); - // One session: two softbench runs sharing label/model/gpu/user, plus a - // different-session run (different label). + // All three runs share real configuration identity. Labels are user + // notes, not grouping identity, so the different session B label must + // not fork a second bubble. $a = BenchmarkRun::factory()->create([ 'suite' => 'softbench', 'label' => 'session A', 'model_id' => 'm', 'model_quant' => 'Q5_K_M', - 'gpu' => 'AMD R9700', 'gpu_count' => 1, 'user_id' => $me->id, 'visibility' => 'public', + 'gpu' => 'AMD R9700', 'gpu_count' => 1, 'engine' => 'llama.cpp', 'user_id' => $me->id, 'visibility' => 'public', 'ran_at' => '2026-09-03 12:00:00', ]); SoftbenchResult::factory()->create(['benchmark_run_id' => $a->id, 'ok' => true, 'user_tps' => 60.0, 'total' => 100, 'level' => 'off']); @@ -21,14 +22,14 @@ $b = BenchmarkRun::factory()->create([ 'suite' => 'infrabench', 'label' => 'session A', 'model_id' => 'm', 'model_quant' => 'Q5_K_M', - 'gpu' => 'AMD R9700', 'gpu_count' => 1, 'user_id' => $me->id, 'visibility' => 'public', + 'gpu' => 'AMD R9700', 'gpu_count' => 1, 'engine' => 'llama.cpp', 'user_id' => $me->id, 'visibility' => 'public', 'ran_at' => '2026-09-03 12:05:00', ]); InfrabenchResult::factory()->create(['benchmark_run_id' => $b->id, 'ok' => true, 'user_tps' => 40.0, 'total' => 200, 'level' => 'low']); $c = BenchmarkRun::factory()->create([ 'suite' => 'softbench', 'label' => 'session B', 'model_id' => 'm', 'model_quant' => 'Q5_K_M', - 'gpu' => 'AMD R9700', 'gpu_count' => 1, 'user_id' => $me->id, 'visibility' => 'public', + 'gpu' => 'AMD R9700', 'gpu_count' => 1, 'engine' => 'llama.cpp', 'user_id' => $me->id, 'visibility' => 'public', 'ran_at' => '2026-09-04 12:00:00', ]); SoftbenchResult::factory()->create(['benchmark_run_id' => $c->id, 'ok' => true, 'user_tps' => 50.0, 'total' => 50, 'level' => 'medium']); @@ -36,20 +37,22 @@ Artisan::call('festbench:summary'); $bands = DB::table('session_summaries')->orderBy('sequence')->get(); - expect($bands)->toHaveCount(2); + expect($bands)->toHaveCount(1); - $bandA = $bands->firstWhere('label', 'session A'); + $bandA = $bands->first(); expect($bandA->runner_name)->toBe('peterZ') ->and($bandA->gpu)->toBe('AMD R9700') ->and($bandA->gpu_count)->toBe(1) - ->and($bandA->total_passed)->toBe(2) + // Mixed labels intentionally show no single label on the band. + ->and($bandA->label)->toBeNull() + ->and($bandA->total_passed)->toBe(3) ->and($bandA->total_failed)->toBe(1) - ->and($bandA->total_tokens)->toBe(600); + ->and($bandA->total_tokens)->toBe(650); $rowsA = DB::table('session_suite_rows')->where('session_summary_id', $bandA->id)->orderBy('sequence')->get(); - expect($rowsA)->toHaveCount(2); + expect($rowsA)->toHaveCount(3); - $soft = $rowsA->firstWhere('suite', 'softbench'); + $soft = $rowsA->where('suite', 'softbench')->firstWhere('level', 'off'); // simple avg of (60, 20) = 40; weighted = (60*100 + 20*300)/400 = 30 expect($soft->passed)->toBe(1) ->and($soft->failed)->toBe(1) @@ -61,8 +64,8 @@ $infra = $rowsA->firstWhere('suite', 'infrabench'); expect($infra->passed)->toBe(1)->and($infra->level)->toBe('low'); - // Newest session first - expect($bands->first()->label)->toBe('session B'); + $secondSoft = $rowsA->where('suite', 'softbench')->firstWhere('level', 'medium'); + expect($secondSoft->passed)->toBe(1)->and($secondSoft->total_tokens)->toBe(50); }); test('private runs form their own band and are hidden from guests', function () {