From 8c9e512fc2f2a41055e5661c18e6f23f7794c8df Mon Sep 17 00:00:00 2001 From: PeterChrz Date: Wed, 30 Sep 2026 15:47:01 -0400 Subject: [PATCH] Add geometric-mean band total (tok/s + tokens) under each band header --- app/Console/Commands/SessionSummary.php | 28 ++++++++++++++++++- ...001_add_total_tps_to_session_summaries.php | 23 +++++++++++++++ .../views/livewire/summary/index.blade.php | 18 ++++++++++++ 3 files changed, 68 insertions(+), 1 deletion(-) create mode 100644 database/migrations/2026_09_30_000001_add_total_tps_to_session_summaries.php diff --git a/app/Console/Commands/SessionSummary.php b/app/Console/Commands/SessionSummary.php index daf9266..2ccf819 100644 --- a/app/Console/Commands/SessionSummary.php +++ b/app/Console/Commands/SessionSummary.php @@ -66,6 +66,8 @@ public function handle(): int 'passed' => 0, 'failed' => 0, 'tokens' => 0, + 'ln_sum' => 0.0, + 'ln_n' => 0, 'runs' => collect(), ]; @@ -87,6 +89,8 @@ public function handle(): int $sessions[$key]['passed'] += $aggs->sum('passed'); $sessions[$key]['failed'] += $aggs->sum('failed'); $sessions[$key]['tokens'] += $aggs->sum('total_tokens'); + $sessions[$key]['ln_sum'] += $aggs->sum('ln_sum'); + $sessions[$key]['ln_n'] += $aggs->sum('ln_n'); $sessions[$key]['runs'] = $sessions[$key]['runs']->merge($aggs); } @@ -101,6 +105,8 @@ public function handle(): int $sessions[$key]['passed'] += $agg['passed']; $sessions[$key]['failed'] += $agg['failed']; $sessions[$key]['tokens'] += $agg['total_tokens']; + $sessions[$key]['ln_sum'] += $agg['ln_sum']; + $sessions[$key]['ln_n'] += $agg['ln_n']; $sessions[$key]['runs'] = $band['runs']->merge([$agg]); } @@ -128,6 +134,9 @@ public function handle(): int 'total_passed' => $band['passed'], 'total_failed' => $band['failed'], 'total_tokens' => $band['tokens'], + 'total_tps' => $band['ln_n'] > 0 + ? round(exp($band['ln_sum'] / $band['ln_n']), 1) + : null, 'sequence' => $bandIndex, 'created_at' => now(), 'updated_at' => now(), @@ -193,6 +202,8 @@ private function aggregateRun(object $run) DB::raw('avg(nullif(wall, 0)) as avg_wall'), DB::raw('sum(total) as total_tokens'), DB::raw('sum(user_tps * total) / nullif(sum(total), 0) as weighted_user_tps'), + DB::raw('sum(ln(user_tps)) filter (where user_tps > 0) as ln_sum'), + DB::raw('count(*) filter (where user_tps > 0) as ln_n'), ])) ->sortBy(fn ($agg) => $levelOrder[$agg->level] ?? 99) ->values() @@ -203,13 +214,14 @@ private function aggregateRun(object $run) 'level' => $agg->level, 'passed' => (int) $agg->passed, 'failed' => (int) $agg->failed, - // depthsweep has no decode metrics: speed stays blank 'avg_user_tps' => null, 'weighted_user_tps' => $this->roundOrNull($agg->weighted_user_tps), 'gen_tps' => null, 'total_tokens' => (int) ($agg->total_tokens ?? 0), 'avg_ttft' => $this->roundOrNull($agg->avg_ttft), 'avg_wall' => $this->roundOrNull($agg->avg_wall), + 'ln_sum' => (float) ($agg->ln_sum ?? 0), + 'ln_n' => (int) ($agg->ln_n ?? 0), ]); } @@ -219,6 +231,8 @@ private function aggregateRun(object $run) ->selectRaw('count(*) filter (where not ok) as failed') ->selectRaw('sum(gen_tokens) as total_tokens') ->selectRaw('sum(gen_tokens) / nullif(sum(wall), 0) as gen_tps') + ->selectRaw('sum(ln(gen_tokens::float8 / nullif(wall, 0))) filter (where wall > 0 and gen_tokens > 0) as ln_sum') + ->selectRaw('count(*) filter (where wall > 0 and gen_tokens > 0) as ln_n') ->first(); return collect([[ @@ -232,6 +246,8 @@ private function aggregateRun(object $run) 'weighted_user_tps' => null, 'gen_tps' => $this->roundOrNull($agg->gen_tps), 'total_tokens' => (int) ($agg->total_tokens ?? 0), + 'ln_sum' => (float) ($agg->ln_sum ?? 0), + 'ln_n' => (int) ($agg->ln_n ?? 0), ]]); } @@ -241,6 +257,8 @@ private function aggregateRun(object $run) ->selectRaw('avg(user_tps) as avg_user_tps') ->selectRaw('sum(user_tps * total) / nullif(sum(total), 0) as weighted_user_tps') ->selectRaw('sum(total) as total_tokens') + ->selectRaw('sum(ln(user_tps)) filter (where user_tps > 0) as ln_sum') + ->selectRaw('count(*) filter (where user_tps > 0) as ln_n') ->first(); return collect([[ @@ -254,6 +272,8 @@ private function aggregateRun(object $run) 'weighted_user_tps' => $this->roundOrNull($agg->weighted_user_tps), 'gen_tps' => null, 'total_tokens' => (int) ($agg->total_tokens ?? 0), + 'ln_sum' => (float) ($agg->ln_sum ?? 0), + 'ln_n' => (int) ($agg->ln_n ?? 0), ]]); } @@ -275,6 +295,8 @@ private function aggregateScenarioBand(array $runs): array ->selectRaw('count(*) filter (where not ok) as failed') ->selectRaw('sum(gen_tokens) as total_tokens') ->selectRaw('sum(gen_tokens) / nullif(sum(wall), 0) as gen_tps') + ->selectRaw('sum(ln(gen_tokens::float8 / nullif(wall, 0))) filter (where wall > 0 and gen_tokens > 0) as ln_sum') + ->selectRaw('count(*) filter (where wall > 0 and gen_tokens > 0) as ln_n') ->first(); // Thinking level: keep it only when every scenario row in the band @@ -294,6 +316,8 @@ private function aggregateScenarioBand(array $runs): array 'weighted_user_tps' => null, 'gen_tps' => $this->roundOrNull($agg->gen_tps), 'total_tokens' => (int) ($agg->total_tokens ?? 0), + 'ln_sum' => (float) ($agg->ln_sum ?? 0), + 'ln_n' => (int) ($agg->ln_n ?? 0), ]; } @@ -310,6 +334,8 @@ private function emptyAgg(object $run): array 'weighted_user_tps' => null, 'gen_tps' => null, 'total_tokens' => 0, + 'ln_sum' => 0.0, + 'ln_n' => 0, ]; } diff --git a/database/migrations/2026_09_30_000001_add_total_tps_to_session_summaries.php b/database/migrations/2026_09_30_000001_add_total_tps_to_session_summaries.php new file mode 100644 index 0000000..f233dd4 --- /dev/null +++ b/database/migrations/2026_09_30_000001_add_total_tps_to_session_summaries.php @@ -0,0 +1,23 @@ +decimal('total_tps', 10, 1)->nullable()->after('total_tokens') + ->comment('Geometric mean of per-task tok/s across all observations in the band (exp(avg(ln(tps))))'); + }); + } + + public function down(): void + { + Schema::table('session_summaries', function (Blueprint $table) { + $table->dropColumn('total_tps'); + }); + } +}; \ No newline at end of file diff --git a/resources/views/livewire/summary/index.blade.php b/resources/views/livewire/summary/index.blade.php index 50078b2..a7c1403 100644 --- a/resources/views/livewire/summary/index.blade.php +++ b/resources/views/livewire/summary/index.blade.php @@ -167,6 +167,24 @@ class="w-64" {{ $session->band->runner_name ?? 'anonymous' }} {{ optional($session->band->ran_at)->format('M j Y') }} + {{-- Band total: geometric-mean tok/s across all observations + total tokens --}} + @if ($session->band->total_tps !== null) +
+ {{ __('Band total') }} + + {{ number_format($session->band->total_tps, 1) }} + {{ __('tok/s (geo)') }} + + + {{ number_format($session->band->total_tokens) }} + {{ __('tokens') }} + + + ✔ {{ $session->band->total_passed }} + ✘ {{ $session->band->total_failed }} + +
+ @endif {{-- Column titles aligned over the suite rows (md+ only) --}}