pch/festbench-web
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Add geometric-mean band total (tok/s + tokens) under each band header

Browse source
This commit is contained in:
PeterChrz 2026-09-30 15:47:01 -04:00
parent f334538866
commit 8c9e512fc2
Signed by untrusted user who does not match committer: pch
GPG key ID: 8F0826ECF7302C63
3 changed files with 68 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

28
app/Console/Commands/SessionSummary.php
View file

@ -66,6 +66,8 @@ public function handle(): int
'passed' => 0,
'failed' => 0,
'tokens' => 0,
'ln_sum' => 0.0,
'ln_n' => 0,
'runs' => collect(),
];
@ -87,6 +89,8 @@ public function handle(): int
$sessions[$key]['passed'] += $aggs->sum('passed');
$sessions[$key]['failed'] += $aggs->sum('failed');
$sessions[$key]['tokens'] += $aggs->sum('total_tokens');
$sessions[$key]['ln_sum'] += $aggs->sum('ln_sum');
$sessions[$key]['ln_n'] += $aggs->sum('ln_n');
$sessions[$key]['runs'] = $sessions[$key]['runs']->merge($aggs);
}
@ -101,6 +105,8 @@ public function handle(): int
$sessions[$key]['passed'] += $agg['passed'];
$sessions[$key]['failed'] += $agg['failed'];
$sessions[$key]['tokens'] += $agg['total_tokens'];
$sessions[$key]['ln_sum'] += $agg['ln_sum'];
$sessions[$key]['ln_n'] += $agg['ln_n'];
$sessions[$key]['runs'] = $band['runs']->merge([$agg]);
}
@ -128,6 +134,9 @@ public function handle(): int
'total_passed' => $band['passed'],
'total_failed' => $band['failed'],
'total_tokens' => $band['tokens'],
'total_tps' => $band['ln_n'] > 0
? round(exp($band['ln_sum'] / $band['ln_n']), 1)
: null,
'sequence' => $bandIndex,
'created_at' => now(),
'updated_at' => now(),
@ -193,6 +202,8 @@ private function aggregateRun(object $run)
DB::raw('avg(nullif(wall, 0)) as avg_wall'),
DB::raw('sum(total) as total_tokens'),
DB::raw('sum(user_tps * total) / nullif(sum(total), 0) as weighted_user_tps'),
DB::raw('sum(ln(user_tps)) filter (where user_tps > 0) as ln_sum'),
DB::raw('count(*) filter (where user_tps > 0) as ln_n'),
]))
->sortBy(fn ($agg) => $levelOrder[$agg->level] ?? 99)
->values()
@ -203,13 +214,14 @@ private function aggregateRun(object $run)
'level' => $agg->level,
'passed' => (int) $agg->passed,
'failed' => (int) $agg->failed,
// depthsweep has no decode metrics: speed stays blank
'avg_user_tps' => null,
'weighted_user_tps' => $this->roundOrNull($agg->weighted_user_tps),
'gen_tps' => null,
'total_tokens' => (int) ($agg->total_tokens ?? 0),
'avg_ttft' => $this->roundOrNull($agg->avg_ttft),
'avg_wall' => $this->roundOrNull($agg->avg_wall),
'ln_sum' => (float) ($agg->ln_sum ?? 0),
'ln_n' => (int) ($agg->ln_n ?? 0),
]);
}
@ -219,6 +231,8 @@ private function aggregateRun(object $run)
->selectRaw('count(*) filter (where not ok) as failed')
->selectRaw('sum(gen_tokens) as total_tokens')
->selectRaw('sum(gen_tokens) / nullif(sum(wall), 0) as gen_tps')
->selectRaw('sum(ln(gen_tokens::float8 / nullif(wall, 0))) filter (where wall > 0 and gen_tokens > 0) as ln_sum')
->selectRaw('count(*) filter (where wall > 0 and gen_tokens > 0) as ln_n')
->first();
return collect([[
@ -232,6 +246,8 @@ private function aggregateRun(object $run)
'weighted_user_tps' => null,
'gen_tps' => $this->roundOrNull($agg->gen_tps),
'total_tokens' => (int) ($agg->total_tokens ?? 0),
'ln_sum' => (float) ($agg->ln_sum ?? 0),
'ln_n' => (int) ($agg->ln_n ?? 0),
]]);
}
@ -241,6 +257,8 @@ private function aggregateRun(object $run)
->selectRaw('avg(user_tps) as avg_user_tps')
->selectRaw('sum(user_tps * total) / nullif(sum(total), 0) as weighted_user_tps')
->selectRaw('sum(total) as total_tokens')
->selectRaw('sum(ln(user_tps)) filter (where user_tps > 0) as ln_sum')
->selectRaw('count(*) filter (where user_tps > 0) as ln_n')
->first();
return collect([[
@ -254,6 +272,8 @@ private function aggregateRun(object $run)
'weighted_user_tps' => $this->roundOrNull($agg->weighted_user_tps),
'gen_tps' => null,
'total_tokens' => (int) ($agg->total_tokens ?? 0),
'ln_sum' => (float) ($agg->ln_sum ?? 0),
'ln_n' => (int) ($agg->ln_n ?? 0),
]]);
}
@ -275,6 +295,8 @@ private function aggregateScenarioBand(array $runs): array
->selectRaw('count(*) filter (where not ok) as failed')
->selectRaw('sum(gen_tokens) as total_tokens')
->selectRaw('sum(gen_tokens) / nullif(sum(wall), 0) as gen_tps')
->selectRaw('sum(ln(gen_tokens::float8 / nullif(wall, 0))) filter (where wall > 0 and gen_tokens > 0) as ln_sum')
->selectRaw('count(*) filter (where wall > 0 and gen_tokens > 0) as ln_n')
->first();
// Thinking level: keep it only when every scenario row in the band
@ -294,6 +316,8 @@ private function aggregateScenarioBand(array $runs): array
'weighted_user_tps' => null,
'gen_tps' => $this->roundOrNull($agg->gen_tps),
'total_tokens' => (int) ($agg->total_tokens ?? 0),
'ln_sum' => (float) ($agg->ln_sum ?? 0),
'ln_n' => (int) ($agg->ln_n ?? 0),
];
}
@ -310,6 +334,8 @@ private function emptyAgg(object $run): array
'weighted_user_tps' => null,
'gen_tps' => null,
'total_tokens' => 0,
'ln_sum' => 0.0,
'ln_n' => 0,
];
}

23
database/migrations/2026_09_30_000001_add_total_tps_to_session_summaries.php Normal file
View file

@ -0,0 +1,23 @@
<?php
use Illuminate\Database\Migrations\Migration;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Support\Facades\Schema;
return new class extends Migration
{
public function up(): void
{
Schema::table('session_summaries', function (Blueprint $table) {
$table->decimal('total_tps', 10, 1)->nullable()->after('total_tokens')
->comment('Geometric mean of per-task tok/s across all observations in the band (exp(avg(ln(tps))))');
});
}
public function down(): void
{
Schema::table('session_summaries', function (Blueprint $table) {
$table->dropColumn('total_tps');
});
}
};

18
resources/views/livewire/summary/index.blade.php
View file

@ -167,6 +167,24 @@ class="w-64"
<span class="text-sm text-zinc-500">{{ $session->band->runner_name ?? 'anonymous' }}</span>
<span class="text-xs text-zinc-400">{{ optional($session->band->ran_at)->format('M j Y') }}</span>
</div>
{{-- Band total: geometric-mean tok/s across all observations + total tokens --}}
@if ($session->band->total_tps !== null)
<div class="flex flex-wrap items-center gap-x-4 gap-y-1 border-b border-zinc-200 px-4 py-2 dark:border-zinc-700">
<span class="text-xs font-semibold uppercase tracking-wider text-zinc-400">{{ __('Band total') }}</span>
<span class="text-sm">
<span class="font-mono font-semibold">{{ number_format($session->band->total_tps, 1) }}</span>
<span class="text-xs text-zinc-400">{{ __('tok/s (geo)') }}</span>
</span>
<span class="text-sm">
<span class="font-mono font-semibold">{{ number_format($session->band->total_tokens) }}</span>
<span class="text-xs text-zinc-400">{{ __('tokens') }}</span>
</span>
<span class="text-sm">
<flux:badge size="xs" color="emerald">✔ {{ $session->band->total_passed }}</flux:badge>
<flux:badge size="xs" color="{{ $session->band->total_failed > 0 ? 'rose' : 'zinc' }}">✘ {{ $session->band->total_failed }}</flux:badge>
</span>
</div>
@endif
{{-- Column titles aligned over the suite rows (md+ only) --}}
<div class="hidden px-4 pt-2 md:grid md:grid-cols-[7.5rem_4.5rem_7rem_15rem_6rem_1fr] md:gap-x-4 md:items-center">
<span class="text-[10px] font-semibold uppercase tracking-wider text-zinc-400">{{ __('Benchmark') }}</span>