Add geometric-mean band total (tok/s + tokens) under each band header
This commit is contained in:
parent
f334538866
commit
8c9e512fc2
3 changed files with 68 additions and 1 deletions
|
|
@ -66,6 +66,8 @@ public function handle(): int
|
||||||
'passed' => 0,
|
'passed' => 0,
|
||||||
'failed' => 0,
|
'failed' => 0,
|
||||||
'tokens' => 0,
|
'tokens' => 0,
|
||||||
|
'ln_sum' => 0.0,
|
||||||
|
'ln_n' => 0,
|
||||||
'runs' => collect(),
|
'runs' => collect(),
|
||||||
];
|
];
|
||||||
|
|
||||||
|
|
@ -87,6 +89,8 @@ public function handle(): int
|
||||||
$sessions[$key]['passed'] += $aggs->sum('passed');
|
$sessions[$key]['passed'] += $aggs->sum('passed');
|
||||||
$sessions[$key]['failed'] += $aggs->sum('failed');
|
$sessions[$key]['failed'] += $aggs->sum('failed');
|
||||||
$sessions[$key]['tokens'] += $aggs->sum('total_tokens');
|
$sessions[$key]['tokens'] += $aggs->sum('total_tokens');
|
||||||
|
$sessions[$key]['ln_sum'] += $aggs->sum('ln_sum');
|
||||||
|
$sessions[$key]['ln_n'] += $aggs->sum('ln_n');
|
||||||
$sessions[$key]['runs'] = $sessions[$key]['runs']->merge($aggs);
|
$sessions[$key]['runs'] = $sessions[$key]['runs']->merge($aggs);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -101,6 +105,8 @@ public function handle(): int
|
||||||
$sessions[$key]['passed'] += $agg['passed'];
|
$sessions[$key]['passed'] += $agg['passed'];
|
||||||
$sessions[$key]['failed'] += $agg['failed'];
|
$sessions[$key]['failed'] += $agg['failed'];
|
||||||
$sessions[$key]['tokens'] += $agg['total_tokens'];
|
$sessions[$key]['tokens'] += $agg['total_tokens'];
|
||||||
|
$sessions[$key]['ln_sum'] += $agg['ln_sum'];
|
||||||
|
$sessions[$key]['ln_n'] += $agg['ln_n'];
|
||||||
$sessions[$key]['runs'] = $band['runs']->merge([$agg]);
|
$sessions[$key]['runs'] = $band['runs']->merge([$agg]);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -128,6 +134,9 @@ public function handle(): int
|
||||||
'total_passed' => $band['passed'],
|
'total_passed' => $band['passed'],
|
||||||
'total_failed' => $band['failed'],
|
'total_failed' => $band['failed'],
|
||||||
'total_tokens' => $band['tokens'],
|
'total_tokens' => $band['tokens'],
|
||||||
|
'total_tps' => $band['ln_n'] > 0
|
||||||
|
? round(exp($band['ln_sum'] / $band['ln_n']), 1)
|
||||||
|
: null,
|
||||||
'sequence' => $bandIndex,
|
'sequence' => $bandIndex,
|
||||||
'created_at' => now(),
|
'created_at' => now(),
|
||||||
'updated_at' => now(),
|
'updated_at' => now(),
|
||||||
|
|
@ -193,6 +202,8 @@ private function aggregateRun(object $run)
|
||||||
DB::raw('avg(nullif(wall, 0)) as avg_wall'),
|
DB::raw('avg(nullif(wall, 0)) as avg_wall'),
|
||||||
DB::raw('sum(total) as total_tokens'),
|
DB::raw('sum(total) as total_tokens'),
|
||||||
DB::raw('sum(user_tps * total) / nullif(sum(total), 0) as weighted_user_tps'),
|
DB::raw('sum(user_tps * total) / nullif(sum(total), 0) as weighted_user_tps'),
|
||||||
|
DB::raw('sum(ln(user_tps)) filter (where user_tps > 0) as ln_sum'),
|
||||||
|
DB::raw('count(*) filter (where user_tps > 0) as ln_n'),
|
||||||
]))
|
]))
|
||||||
->sortBy(fn ($agg) => $levelOrder[$agg->level] ?? 99)
|
->sortBy(fn ($agg) => $levelOrder[$agg->level] ?? 99)
|
||||||
->values()
|
->values()
|
||||||
|
|
@ -203,13 +214,14 @@ private function aggregateRun(object $run)
|
||||||
'level' => $agg->level,
|
'level' => $agg->level,
|
||||||
'passed' => (int) $agg->passed,
|
'passed' => (int) $agg->passed,
|
||||||
'failed' => (int) $agg->failed,
|
'failed' => (int) $agg->failed,
|
||||||
// depthsweep has no decode metrics: speed stays blank
|
|
||||||
'avg_user_tps' => null,
|
'avg_user_tps' => null,
|
||||||
'weighted_user_tps' => $this->roundOrNull($agg->weighted_user_tps),
|
'weighted_user_tps' => $this->roundOrNull($agg->weighted_user_tps),
|
||||||
'gen_tps' => null,
|
'gen_tps' => null,
|
||||||
'total_tokens' => (int) ($agg->total_tokens ?? 0),
|
'total_tokens' => (int) ($agg->total_tokens ?? 0),
|
||||||
'avg_ttft' => $this->roundOrNull($agg->avg_ttft),
|
'avg_ttft' => $this->roundOrNull($agg->avg_ttft),
|
||||||
'avg_wall' => $this->roundOrNull($agg->avg_wall),
|
'avg_wall' => $this->roundOrNull($agg->avg_wall),
|
||||||
|
'ln_sum' => (float) ($agg->ln_sum ?? 0),
|
||||||
|
'ln_n' => (int) ($agg->ln_n ?? 0),
|
||||||
]);
|
]);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -219,6 +231,8 @@ private function aggregateRun(object $run)
|
||||||
->selectRaw('count(*) filter (where not ok) as failed')
|
->selectRaw('count(*) filter (where not ok) as failed')
|
||||||
->selectRaw('sum(gen_tokens) as total_tokens')
|
->selectRaw('sum(gen_tokens) as total_tokens')
|
||||||
->selectRaw('sum(gen_tokens) / nullif(sum(wall), 0) as gen_tps')
|
->selectRaw('sum(gen_tokens) / nullif(sum(wall), 0) as gen_tps')
|
||||||
|
->selectRaw('sum(ln(gen_tokens::float8 / nullif(wall, 0))) filter (where wall > 0 and gen_tokens > 0) as ln_sum')
|
||||||
|
->selectRaw('count(*) filter (where wall > 0 and gen_tokens > 0) as ln_n')
|
||||||
->first();
|
->first();
|
||||||
|
|
||||||
return collect([[
|
return collect([[
|
||||||
|
|
@ -232,6 +246,8 @@ private function aggregateRun(object $run)
|
||||||
'weighted_user_tps' => null,
|
'weighted_user_tps' => null,
|
||||||
'gen_tps' => $this->roundOrNull($agg->gen_tps),
|
'gen_tps' => $this->roundOrNull($agg->gen_tps),
|
||||||
'total_tokens' => (int) ($agg->total_tokens ?? 0),
|
'total_tokens' => (int) ($agg->total_tokens ?? 0),
|
||||||
|
'ln_sum' => (float) ($agg->ln_sum ?? 0),
|
||||||
|
'ln_n' => (int) ($agg->ln_n ?? 0),
|
||||||
]]);
|
]]);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -241,6 +257,8 @@ private function aggregateRun(object $run)
|
||||||
->selectRaw('avg(user_tps) as avg_user_tps')
|
->selectRaw('avg(user_tps) as avg_user_tps')
|
||||||
->selectRaw('sum(user_tps * total) / nullif(sum(total), 0) as weighted_user_tps')
|
->selectRaw('sum(user_tps * total) / nullif(sum(total), 0) as weighted_user_tps')
|
||||||
->selectRaw('sum(total) as total_tokens')
|
->selectRaw('sum(total) as total_tokens')
|
||||||
|
->selectRaw('sum(ln(user_tps)) filter (where user_tps > 0) as ln_sum')
|
||||||
|
->selectRaw('count(*) filter (where user_tps > 0) as ln_n')
|
||||||
->first();
|
->first();
|
||||||
|
|
||||||
return collect([[
|
return collect([[
|
||||||
|
|
@ -254,6 +272,8 @@ private function aggregateRun(object $run)
|
||||||
'weighted_user_tps' => $this->roundOrNull($agg->weighted_user_tps),
|
'weighted_user_tps' => $this->roundOrNull($agg->weighted_user_tps),
|
||||||
'gen_tps' => null,
|
'gen_tps' => null,
|
||||||
'total_tokens' => (int) ($agg->total_tokens ?? 0),
|
'total_tokens' => (int) ($agg->total_tokens ?? 0),
|
||||||
|
'ln_sum' => (float) ($agg->ln_sum ?? 0),
|
||||||
|
'ln_n' => (int) ($agg->ln_n ?? 0),
|
||||||
]]);
|
]]);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -275,6 +295,8 @@ private function aggregateScenarioBand(array $runs): array
|
||||||
->selectRaw('count(*) filter (where not ok) as failed')
|
->selectRaw('count(*) filter (where not ok) as failed')
|
||||||
->selectRaw('sum(gen_tokens) as total_tokens')
|
->selectRaw('sum(gen_tokens) as total_tokens')
|
||||||
->selectRaw('sum(gen_tokens) / nullif(sum(wall), 0) as gen_tps')
|
->selectRaw('sum(gen_tokens) / nullif(sum(wall), 0) as gen_tps')
|
||||||
|
->selectRaw('sum(ln(gen_tokens::float8 / nullif(wall, 0))) filter (where wall > 0 and gen_tokens > 0) as ln_sum')
|
||||||
|
->selectRaw('count(*) filter (where wall > 0 and gen_tokens > 0) as ln_n')
|
||||||
->first();
|
->first();
|
||||||
|
|
||||||
// Thinking level: keep it only when every scenario row in the band
|
// Thinking level: keep it only when every scenario row in the band
|
||||||
|
|
@ -294,6 +316,8 @@ private function aggregateScenarioBand(array $runs): array
|
||||||
'weighted_user_tps' => null,
|
'weighted_user_tps' => null,
|
||||||
'gen_tps' => $this->roundOrNull($agg->gen_tps),
|
'gen_tps' => $this->roundOrNull($agg->gen_tps),
|
||||||
'total_tokens' => (int) ($agg->total_tokens ?? 0),
|
'total_tokens' => (int) ($agg->total_tokens ?? 0),
|
||||||
|
'ln_sum' => (float) ($agg->ln_sum ?? 0),
|
||||||
|
'ln_n' => (int) ($agg->ln_n ?? 0),
|
||||||
];
|
];
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -310,6 +334,8 @@ private function emptyAgg(object $run): array
|
||||||
'weighted_user_tps' => null,
|
'weighted_user_tps' => null,
|
||||||
'gen_tps' => null,
|
'gen_tps' => null,
|
||||||
'total_tokens' => 0,
|
'total_tokens' => 0,
|
||||||
|
'ln_sum' => 0.0,
|
||||||
|
'ln_n' => 0,
|
||||||
];
|
];
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,23 @@
|
||||||
|
<?php
|
||||||
|
|
||||||
|
use Illuminate\Database\Migrations\Migration;
|
||||||
|
use Illuminate\Database\Schema\Blueprint;
|
||||||
|
use Illuminate\Support\Facades\Schema;
|
||||||
|
|
||||||
|
return new class extends Migration
|
||||||
|
{
|
||||||
|
public function up(): void
|
||||||
|
{
|
||||||
|
Schema::table('session_summaries', function (Blueprint $table) {
|
||||||
|
$table->decimal('total_tps', 10, 1)->nullable()->after('total_tokens')
|
||||||
|
->comment('Geometric mean of per-task tok/s across all observations in the band (exp(avg(ln(tps))))');
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
public function down(): void
|
||||||
|
{
|
||||||
|
Schema::table('session_summaries', function (Blueprint $table) {
|
||||||
|
$table->dropColumn('total_tps');
|
||||||
|
});
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
@ -167,6 +167,24 @@ class="w-64"
|
||||||
<span class="text-sm text-zinc-500">{{ $session->band->runner_name ?? 'anonymous' }}</span>
|
<span class="text-sm text-zinc-500">{{ $session->band->runner_name ?? 'anonymous' }}</span>
|
||||||
<span class="text-xs text-zinc-400">{{ optional($session->band->ran_at)->format('M j Y') }}</span>
|
<span class="text-xs text-zinc-400">{{ optional($session->band->ran_at)->format('M j Y') }}</span>
|
||||||
</div>
|
</div>
|
||||||
|
{{-- Band total: geometric-mean tok/s across all observations + total tokens --}}
|
||||||
|
@if ($session->band->total_tps !== null)
|
||||||
|
<div class="flex flex-wrap items-center gap-x-4 gap-y-1 border-b border-zinc-200 px-4 py-2 dark:border-zinc-700">
|
||||||
|
<span class="text-xs font-semibold uppercase tracking-wider text-zinc-400">{{ __('Band total') }}</span>
|
||||||
|
<span class="text-sm">
|
||||||
|
<span class="font-mono font-semibold">{{ number_format($session->band->total_tps, 1) }}</span>
|
||||||
|
<span class="text-xs text-zinc-400">{{ __('tok/s (geo)') }}</span>
|
||||||
|
</span>
|
||||||
|
<span class="text-sm">
|
||||||
|
<span class="font-mono font-semibold">{{ number_format($session->band->total_tokens) }}</span>
|
||||||
|
<span class="text-xs text-zinc-400">{{ __('tokens') }}</span>
|
||||||
|
</span>
|
||||||
|
<span class="text-sm">
|
||||||
|
<flux:badge size="xs" color="emerald">✔ {{ $session->band->total_passed }}</flux:badge>
|
||||||
|
<flux:badge size="xs" color="{{ $session->band->total_failed > 0 ? 'rose' : 'zinc' }}">✘ {{ $session->band->total_failed }}</flux:badge>
|
||||||
|
</span>
|
||||||
|
</div>
|
||||||
|
@endif
|
||||||
{{-- Column titles aligned over the suite rows (md+ only) --}}
|
{{-- Column titles aligned over the suite rows (md+ only) --}}
|
||||||
<div class="hidden px-4 pt-2 md:grid md:grid-cols-[7.5rem_4.5rem_7rem_15rem_6rem_1fr] md:gap-x-4 md:items-center">
|
<div class="hidden px-4 pt-2 md:grid md:grid-cols-[7.5rem_4.5rem_7rem_15rem_6rem_1fr] md:gap-x-4 md:items-center">
|
||||||
<span class="text-[10px] font-semibold uppercase tracking-wider text-zinc-400">{{ __('Benchmark') }}</span>
|
<span class="text-[10px] font-semibold uppercase tracking-wider text-zinc-400">{{ __('Benchmark') }}</span>
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue