pch/festbench-web
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Split band total into Devlogic and Sysops; remove config badge

Browse source 
Replace the single Band total with two adjacent metric groups:
- Devlogic total: softbench, prosebench, psychbench, depthsweep
  (all non-infra, non-scenario suites)
- Sysops total: infrabench + scenariobench

Each group independently shows geo-mean tok/s, total tokens, and
pass/fail counts. Scenariobench contributes per-observation
gen_tokens/wall rates to the sysops geo-mean.

Remove the server-config badge (e.g. 'llama.cpp DFlash spec-decode,
131k ctx, fa on') from the front-page card header. Label, quant, and
engine badges remain.

Migration drops the old total_* columns and adds devlogic_* and
sysops_* columns to session_summaries.
This commit is contained in:
PeterChrz 2026-10-04 23:38:13 -04:00
parent 9bf17dba6f
commit 7e57fb5e16
Signed by untrusted user who does not match committer: pch
GPG key ID: 8F0826ECF7302C63
4 changed files with 190 additions and 55 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

70
app/Console/Commands/SessionSummary.php
View file

@ -69,11 +69,19 @@ public function handle(): int
'engine' => $run->engine,
'config_name' => $run->config_name,
'ran_at' => $run->ran_at,
'passed' => 0,
'failed' => 0,
'tokens' => 0,
'ln_sum' => 0.0,
'ln_n' => 0,
// Devlogic totals: softbench, prosebench, psychbench,
// depthsweep — everything except infrabench and scenariobench.
'dl_passed' => 0,
'dl_failed' => 0,
'dl_tokens' => 0,
'dl_ln_sum' => 0.0,
'dl_ln_n' => 0,
// Sysops totals: infrabench and scenariobench.
'so_passed' => 0,
'so_failed' => 0,
'so_tokens' => 0,
'so_ln_sum' => 0.0,
'so_ln_n' => 0,
'runs' => collect(),
];
@ -88,25 +96,39 @@ public function handle(): int
$aggs = $this->aggregateRun($run);
$sessions[$key]['passed'] += $aggs->sum('passed');
$sessions[$key]['failed'] += $aggs->sum('failed');
$sessions[$key]['tokens'] += $aggs->sum('total_tokens');
$sessions[$key]['ln_sum'] += $aggs->sum('ln_sum');
$sessions[$key]['ln_n'] += $aggs->sum('ln_n');
// Route the aggregation to the correct total bucket.
$isSysops = $run->suite === 'infrabench';
$sessions[$key]['runs'] = $sessions[$key]['runs']->merge($aggs);
if ($isSysops) {
$sessions[$key]['so_passed'] += $aggs->sum('passed');
$sessions[$key]['so_failed'] += $aggs->sum('failed');
$sessions[$key]['so_tokens'] += $aggs->sum('total_tokens');
$sessions[$key]['so_ln_sum'] += $aggs->sum('ln_sum');
$sessions[$key]['so_ln_n'] += $aggs->sum('ln_n');
} else {
$sessions[$key]['dl_passed'] += $aggs->sum('passed');
$sessions[$key]['dl_failed'] += $aggs->sum('failed');
$sessions[$key]['dl_tokens'] += $aggs->sum('total_tokens');
$sessions[$key]['dl_ln_sum'] += $aggs->sum('ln_sum');
$sessions[$key]['dl_ln_n'] += $aggs->sum('ln_n');
}
}
// Second pass: emit one aggregated scenariobench row per band,
// covering all scenario runs in that band.
// covering all scenario runs in that band. Scenariobench is sysops.
foreach ($sessions as $key => $band) {
$scenarioRuns = $band['scenario_runs'] ?? [];
if (empty($scenarioRuns)) {
continue;
}
$agg = $this->aggregateScenarioBand($scenarioRuns);
$sessions[$key]['passed'] += $agg['passed'];
$sessions[$key]['failed'] += $agg['failed'];
$sessions[$key]['tokens'] += $agg['total_tokens'];
$sessions[$key]['so_passed'] += $agg['passed'];
$sessions[$key]['so_failed'] += $agg['failed'];
$sessions[$key]['so_tokens'] += $agg['total_tokens'];
$sessions[$key]['so_ln_sum'] += $agg['ln_sum'] ?? 0;
$sessions[$key]['so_ln_n'] += $agg['ln_n'] ?? 0;
$sessions[$key]['runs'] = $band['runs']->merge([$agg]);
}
@ -127,11 +149,17 @@ public function handle(): int
'engine' => $band['engine'],
'config_name' => $band['config_name'],
'ran_at' => $band['ran_at'],
'total_passed' => $band['passed'],
'total_failed' => $band['failed'],
'total_tokens' => $band['tokens'],
'total_tps' => $band['ln_n'] > 0
? round(exp($band['ln_sum'] / $band['ln_n']), 1)
'devlogic_passed' => $band['dl_passed'],
'devlogic_failed' => $band['dl_failed'],
'devlogic_tokens' => $band['dl_tokens'],
'devlogic_tps' => $band['dl_ln_n'] > 0
? round(exp($band['dl_ln_sum'] / $band['dl_ln_n']), 1)
: null,
'sysops_passed' => $band['so_passed'],
'sysops_failed' => $band['so_failed'],
'sysops_tokens' => $band['so_tokens'],
'sysops_tps' => $band['so_ln_n'] > 0
? round(exp($band['so_ln_sum'] / $band['so_ln_n']), 1)
: null,
'sequence' => $bandIndex,
'created_at' => now(),
@ -287,6 +315,8 @@ private function aggregateScenarioBand(array $runs): array
->selectRaw('count(*) filter (where not ok) as failed')
->selectRaw('sum(gen_tokens) as total_tokens')
->selectRaw('sum(gen_tokens) / nullif(sum(wall), 0) as gen_tps')
->selectRaw('sum(ln(gen_tokens / nullif(wall, 0))) filter (where gen_tokens > 0 and wall > 0) as ln_sum')
->selectRaw('count(*) filter (where gen_tokens > 0 and wall > 0) as ln_n')
->first();
// Thinking level: keep it only when every scenario row in the band
@ -306,6 +336,8 @@ private function aggregateScenarioBand(array $runs): array
'weighted_user_tps' => null,
'gen_tps' => $this->roundOrNull($agg->gen_tps),
'total_tokens' => (int) ($agg->total_tokens ?? 0),
'ln_sum' => (float) ($agg->ln_sum ?? 0),
'ln_n' => (int) ($agg->ln_n ?? 0),
];
}

46
database/migrations/2026_10_04_000001_split_band_totals.php Normal file
View file

@ -0,0 +1,46 @@
<?php
use Illuminate\Database\Migrations\Migration;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Support\Facades\Schema;
return new class extends Migration
{
public function up(): void
{
Schema::table('session_summaries', function (Blueprint $table) {
$table->decimal('devlogic_tps', 10, 1)->nullable()
->comment('Geo-mean tok/s across devlogic suites (softbench, prosebench, psychbench, depthsweep, …)');
$table->unsignedInteger('devlogic_tokens')->default(0);
$table->unsignedInteger('devlogic_passed')->default(0);
$table->unsignedInteger('devlogic_failed')->default(0);
$table->decimal('sysops_tps', 10, 1)->nullable()
->comment('Geo-mean tok/s across sysops suites (infrabench, scenariobench)');
$table->unsignedInteger('sysops_tokens')->default(0);
$table->unsignedInteger('sysops_passed')->default(0);
$table->unsignedInteger('sysops_failed')->default(0);
});
Schema::table('session_summaries', function (Blueprint $table) {
$table->dropColumn(['total_tps', 'total_tokens', 'total_passed', 'total_failed']);
});
}
public function down(): void
{
Schema::table('session_summaries', function (Blueprint $table) {
$table->decimal('total_tps', 10, 1)->nullable()->after('ran_at');
$table->unsignedInteger('total_tokens')->default(0);
$table->unsignedInteger('total_passed')->default(0);
$table->unsignedInteger('total_failed')->default(0);
});
Schema::table('session_summaries', function (Blueprint $table) {
$table->dropColumn([
'devlogic_tps', 'devlogic_tokens', 'devlogic_passed', 'devlogic_failed',
'sysops_tps', 'sysops_tokens', 'sysops_passed', 'sysops_failed',
]);
});
}
};

56
resources/views/livewire/summary/index.blade.php
View file

@ -160,29 +160,47 @@ class="w-64"
@if ($session->band->engine)
<flux:badge size="xs" color="sky">{{ $session->band->engine }}</flux:badge>
@endif
@if ($session->band->config_name)
<flux:badge size="xs" color="amber">{{ $session->band->config_name }}</flux:badge>
@endif
<flux:spacer />
<span class="text-sm text-zinc-500">{{ $session->band->runner_name ?? 'anonymous' }}</span>
<span class="text-xs text-zinc-400">{{ optional($session->band->ran_at)->format('M j Y') }}</span>
</div>
{{-- Band total: geometric-mean tok/s across decode-suite observations + total tokens --}}
@if ($session->band->total_tps !== null)
<div class="flex flex-wrap items-center gap-x-4 gap-y-1 border-b border-zinc-200 px-4 py-2 dark:border-zinc-700">
<span class="text-xs font-semibold uppercase tracking-wider text-zinc-400">{{ __('Band total') }}</span>
<span class="text-sm">
<span class="font-mono font-semibold">{{ number_format($session->band->total_tps, 1) }}</span>
<span class="text-xs text-zinc-400">{{ __('tok/s') }}</span>
</span>
<span class="text-sm">
<span class="font-mono font-semibold">{{ number_format($session->band->total_tokens) }}</span>
<span class="text-xs text-zinc-400">{{ __('tokens') }}</span>
</span>
<span class="text-sm">
<flux:badge size="xs" color="emerald">✔ {{ $session->band->total_passed }}</flux:badge>
<flux:badge size="xs" color="{{ $session->band->total_failed > 0 ? 'rose' : 'zinc' }}">✘ {{ $session->band->total_failed }}</flux:badge>
</span>
{{-- Dual totals: Devlogic (soft/prose/psych/depthsweep) + Sysops (infra/scenario) --}}
@if ($session->band->devlogic_tps !== null || $session->band->sysops_tps !== null)
<div class="flex flex-wrap items-center gap-x-6 gap-y-1 border-b border-zinc-200 px-4 py-2 dark:border-zinc-700">
@if ($session->band->devlogic_tps !== null)
<div class="flex items-center gap-x-3">
<span class="text-xs font-semibold uppercase tracking-wider text-zinc-400">{{ __('Devlogic total') }}</span>
<span class="text-sm">
<span class="font-mono font-semibold">{{ number_format($session->band->devlogic_tps, 1) }}</span>
<span class="text-xs text-zinc-400">{{ __('tok/s') }}</span>
</span>
<span class="text-sm">
<span class="font-mono font-semibold">{{ number_format($session->band->devlogic_tokens) }}</span>
<span class="text-xs text-zinc-400">{{ __('tokens') }}</span>
</span>
<span class="text-sm">
<flux:badge size="xs" color="emerald">✔ {{ $session->band->devlogic_passed }}</flux:badge>
<flux:badge size="xs" color="{{ $session->band->devlogic_failed > 0 ? 'rose' : 'zinc' }}">✘ {{ $session->band->devlogic_failed }}</flux:badge>
</span>
</div>
@endif
@if ($session->band->sysops_tps !== null)
<div class="flex items-center gap-x-3">
<span class="text-xs font-semibold uppercase tracking-wider text-zinc-400">{{ __('Sysops total') }}</span>
<span class="text-sm">
<span class="font-mono font-semibold">{{ number_format($session->band->sysops_tps, 1) }}</span>
<span class="text-xs text-zinc-400">{{ __('tok/s') }}</span>
</span>
<span class="text-sm">
<span class="font-mono font-semibold">{{ number_format($session->band->sysops_tokens) }}</span>
<span class="text-xs text-zinc-400">{{ __('tokens') }}</span>
</span>
<span class="text-sm">
<flux:badge size="xs" color="emerald">✔ {{ $session->band->sysops_passed }}</flux:badge>
<flux:badge size="xs" color="{{ $session->band->sysops_failed > 0 ? 'rose' : 'zinc' }}">✘ {{ $session->band->sysops_failed }}</flux:badge>
</span>
</div>
@endif
</div>
@endif
{{-- Column titles aligned over the suite rows (md+ only) --}}

73
tests/Feature/SessionSummaryTest.php
View file

@ -2,16 +2,14 @@
use App\Models\BenchmarkRun;
use App\Models\InfrabenchResult;
use App\Models\ScenarioRun;
use App\Models\SoftbenchResult;
use App\Models\User;
use Illuminate\Support\Facades\Artisan;
test('festbench:summary groups matching runs despite different labels', function () {
test('festbench:summary splits runs by label into separate bands', function () {
$me = User::factory()->create(['name' => 'peterZ']);
// All three runs share real configuration identity. Labels are user
// notes, not grouping identity, so the different session B label must
// not fork a second bubble.
$a = BenchmarkRun::factory()->create([
'suite' => 'softbench', 'label' => 'session A', 'model_id' => 'm', 'model_quant' => 'Q5_K_M',
'gpu' => 'AMD R9700', 'gpu_count' => 1, 'engine' => 'llama.cpp', 'user_id' => $me->id, 'visibility' => 'public',
@ -36,24 +34,20 @@
Artisan::call('festbench:summary');
// Different labels → separate bands.
$bands = DB::table('session_summaries')->orderBy('sequence')->get();
expect($bands)->toHaveCount(1);
expect($bands)->toHaveCount(2);
$bandA = $bands->first();
$bandA = $bands->firstWhere('label', 'session A');
expect($bandA->runner_name)->toBe('peterZ')
->and($bandA->gpu)->toBe('AMD R9700')
->and($bandA->gpu_count)->toBe(1)
// Mixed labels intentionally show no single label on the band.
->and($bandA->label)->toBeNull()
->and($bandA->total_passed)->toBe(3)
->and($bandA->total_failed)->toBe(1)
->and($bandA->total_tokens)->toBe(650);
->and($bandA->label)->toBe('session A');
$rowsA = DB::table('session_suite_rows')->where('session_summary_id', $bandA->id)->orderBy('sequence')->get();
expect($rowsA)->toHaveCount(3);
expect($rowsA)->toHaveCount(2);
$soft = $rowsA->where('suite', 'softbench')->firstWhere('level', 'off');
// simple avg of (60, 20) = 40; weighted = (60*100 + 20*300)/400 = 30
$soft = $rowsA->firstWhere('suite', 'softbench');
expect($soft->passed)->toBe(1)
->and($soft->failed)->toBe(1)
->and($soft->avg_user_tps)->toBe(40.0)
@ -63,9 +57,55 @@
$infra = $rowsA->firstWhere('suite', 'infrabench');
expect($infra->passed)->toBe(1)->and($infra->level)->toBe('low');
});
$secondSoft = $rowsA->where('suite', 'softbench')->firstWhere('level', 'medium');
expect($secondSoft->passed)->toBe(1)->and($secondSoft->total_tokens)->toBe(50);
test('devlogic and sysops totals are split by suite category', function () {
$me = User::factory()->create();
$soft = BenchmarkRun::factory()->create([
'suite' => 'softbench', 'label' => 'dual', 'model_id' => 'm', 'model_quant' => 'Q5_K_M',
'gpu' => 'GPU1', 'gpu_count' => 1, 'engine' => 'llama.cpp', 'user_id' => $me->id, 'visibility' => 'public',
'ran_at' => '2026-09-03 12:00:00',
]);
SoftbenchResult::factory()->create(['benchmark_run_id' => $soft->id, 'ok' => true, 'user_tps' => 50.0, 'total' => 100, 'level' => 'off']);
SoftbenchResult::factory()->create(['benchmark_run_id' => $soft->id, 'ok' => false, 'user_tps' => 10.0, 'total' => 200, 'level' => 'off']);
$infra = BenchmarkRun::factory()->create([
'suite' => 'infrabench', 'label' => 'dual', 'model_id' => 'm', 'model_quant' => 'Q5_K_M',
'gpu' => 'GPU1', 'gpu_count' => 1, 'engine' => 'llama.cpp', 'user_id' => $me->id, 'visibility' => 'public',
'ran_at' => '2026-09-03 12:05:00',
]);
InfrabenchResult::factory()->create(['benchmark_run_id' => $infra->id, 'ok' => true, 'user_tps' => 30.0, 'total' => 150, 'level' => 'low']);
$scen = BenchmarkRun::factory()->create([
'suite' => 'scenariobench', 'label' => 'dual', 'model_id' => 'm', 'model_quant' => 'Q5_K_M',
'gpu' => 'GPU1', 'gpu_count' => 1, 'engine' => 'llama.cpp', 'user_id' => $me->id, 'visibility' => 'public',
'ran_at' => '2026-09-03 12:10:00',
]);
ScenarioRun::factory()->create(['benchmark_run_id' => $scen->id, 'ok' => true, 'gen_tokens' => 5000, 'wall' => 100.0]);
ScenarioRun::factory()->create(['benchmark_run_id' => $scen->id, 'ok' => false, 'gen_tokens' => 3000, 'wall' => 200.0]);
Artisan::call('festbench:summary');
$band = DB::table('session_summaries')->where('label', 'dual')->first();
// Devlogic: softbench only — 2 observations (50, 10).
// geo-mean = exp((ln(50)+ln(10))/2) = sqrt(500) ≈ 22.4
expect($band->devlogic_passed)->toBe(1)
->and($band->devlogic_failed)->toBe(1)
->and($band->devlogic_tokens)->toBe(300)
->and($band->devlogic_tps)->toBe(22.4);
// Sysops: infrabench (1 obs: user_tps=30, tokens=150) +
// scenariobench (2 obs: gen_tokens/wall = 50.0 and 15.0, tokens=8000).
// Total sysops tokens = 150 + 8000 = 8150.
expect($band->sysops_passed)->toBe(2)
->and($band->sysops_failed)->toBe(1)
->and($band->sysops_tokens)->toBe(8150);
// sysops_tps is a geo-mean of 3 positive values — just verify it is
// populated and roughly in range.
expect($band->sysops_tps)->not->toBeNull();
expect($band->sysops_tps)->toBeGreaterThan(10.0)->toBeLessThan(50.0);
});
test('private runs form their own band and are hidden from guests', function () {
@ -86,7 +126,6 @@
expect(DB::table('session_summaries')->count())->toBe(2)
->and(DB::table('session_summaries')->where('visibility', 'private')->count())->toBe(1);
// Guest home shows the public band only; owner sees both
$this->get(route('home'))->assertOk()->assertSee('pub');
$bands = DB::table('session_summaries')->pluck('visibility');