348 lines
14 KiB
PHP
348 lines
14 KiB
PHP
<?php
|
|
|
|
namespace App\Console\Commands;
|
|
|
|
use App\Support\Suites;
|
|
use Illuminate\Console\Command;
|
|
use Illuminate\Support\Collection;
|
|
use Illuminate\Support\Facades\DB;
|
|
use Symfony\Component\Console\Attribute\AsCommand;
|
|
|
|
#[AsCommand(name: 'festbench:summary', description: 'Rebuild the session-grouped home summary tables')]
|
|
class SessionSummary extends Command
|
|
{
|
|
public function handle(): int
|
|
{
|
|
$sessions = [];
|
|
|
|
$runs = DB::table('benchmark_runs')
|
|
->leftJoin('users', 'users.id', '=', 'benchmark_runs.user_id')
|
|
->leftJoin('server_configs', 'server_configs.id', '=', 'benchmark_runs.config_id')
|
|
->orderByDesc('ran_at')
|
|
->orderByDesc('benchmark_runs.id')
|
|
->get([
|
|
'benchmark_runs.*',
|
|
'users.name as runner_name',
|
|
'server_configs.name as config_name',
|
|
]);
|
|
|
|
foreach ($runs as $run) {
|
|
// Grouping key: user + GPU + model + quant + visibility + engine
|
|
// + harness identity + named server config. Free-text labels are
|
|
// intentionally NOT part of identity: rerunning psychbench with
|
|
// the same actual settings belongs in the same bubble even when
|
|
// the user wrote a different run note.
|
|
$key = implode('|', [
|
|
$run->model_id ?? '-',
|
|
$run->model_quant ?? '-',
|
|
$run->gpu_id ?? 'raw:'.$run->gpu,
|
|
$run->gpu_count ?? '-',
|
|
$run->user_id ?? '-',
|
|
$run->visibility,
|
|
$run->engine ?? '-',
|
|
$run->engine_build ?: '-',
|
|
// Older direct runs stored empty harness fields; newer
|
|
// depthsweep writes the explicit equivalent. Normalize so
|
|
// semantically identical direct runs share a session.
|
|
$run->harness ?: 'direct',
|
|
$run->harness_ver ?: '-',
|
|
$run->harness_mode ?: 'one-shot',
|
|
$run->config_id ?? 'none',
|
|
]);
|
|
|
|
$sessions[$key] ??= [
|
|
'key' => $key,
|
|
'labels' => collect([$run->label])->filter(),
|
|
'model_id' => $run->model_id,
|
|
'model_quant' => $run->model_quant,
|
|
'gpu' => $run->gpu,
|
|
'gpu_count' => $run->gpu_count,
|
|
'runner_name' => $run->runner_name,
|
|
'user_id' => $run->user_id,
|
|
'visibility' => $run->visibility,
|
|
'engine' => $run->engine,
|
|
'config_name' => $run->config_name,
|
|
'ran_at' => $run->ran_at,
|
|
'passed' => 0,
|
|
'failed' => 0,
|
|
'tokens' => 0,
|
|
'ln_sum' => 0.0,
|
|
'ln_n' => 0,
|
|
'runs' => collect(),
|
|
];
|
|
|
|
if ($run->label !== null && $run->label !== '') {
|
|
$sessions[$key]['labels']->push($run->label);
|
|
}
|
|
|
|
// Scenariobench is aggregated at band level (one row for all
|
|
// scenario runs in the band) rather than per-run, so collect the
|
|
// run rows and defer to a second pass below.
|
|
if ($run->suite === 'scenariobench') {
|
|
$sessions[$key]['scenario_runs'][] = $run;
|
|
|
|
continue;
|
|
}
|
|
|
|
$aggs = $this->aggregateRun($run);
|
|
|
|
$sessions[$key]['passed'] += $aggs->sum('passed');
|
|
$sessions[$key]['failed'] += $aggs->sum('failed');
|
|
$sessions[$key]['tokens'] += $aggs->sum('total_tokens');
|
|
$sessions[$key]['ln_sum'] += $aggs->sum('ln_sum');
|
|
$sessions[$key]['ln_n'] += $aggs->sum('ln_n');
|
|
$sessions[$key]['runs'] = $sessions[$key]['runs']->merge($aggs);
|
|
}
|
|
|
|
// Second pass: emit one aggregated scenariobench row per band,
|
|
// covering all scenario runs in that band.
|
|
foreach ($sessions as $key => $band) {
|
|
$scenarioRuns = $band['scenario_runs'] ?? [];
|
|
if (empty($scenarioRuns)) {
|
|
continue;
|
|
}
|
|
$agg = $this->aggregateScenarioBand($scenarioRuns);
|
|
$sessions[$key]['passed'] += $agg['passed'];
|
|
$sessions[$key]['failed'] += $agg['failed'];
|
|
$sessions[$key]['tokens'] += $agg['total_tokens'];
|
|
$sessions[$key]['runs'] = $band['runs']->merge([$agg]);
|
|
}
|
|
|
|
DB::transaction(function () use ($sessions) {
|
|
DB::table('session_summaries')->truncate();
|
|
|
|
foreach (array_values($sessions) as $bandIndex => $band) {
|
|
$summaryId = DB::table('session_summaries')->insertGetId([
|
|
'session_key' => $band['key'],
|
|
// Keep a label only when every grouped run agrees. A
|
|
// mixed label is a note, not a configuration split.
|
|
'label' => $band['labels']->unique()->count() === 1
|
|
? $band['labels']->first()
|
|
: null,
|
|
'model_id' => $band['model_id'],
|
|
'model_quant' => $band['model_quant'],
|
|
'gpu' => $band['gpu'],
|
|
'gpu_count' => $band['gpu_count'],
|
|
'runner_name' => $band['runner_name'],
|
|
'user_id' => $band['user_id'],
|
|
'visibility' => $band['visibility'],
|
|
'engine' => $band['engine'],
|
|
'config_name' => $band['config_name'],
|
|
'ran_at' => $band['ran_at'],
|
|
'total_passed' => $band['passed'],
|
|
'total_failed' => $band['failed'],
|
|
'total_tokens' => $band['tokens'],
|
|
'total_tps' => $band['ln_n'] > 0
|
|
? round(exp($band['ln_sum'] / $band['ln_n']), 1)
|
|
: null,
|
|
'sequence' => $bandIndex,
|
|
'created_at' => now(),
|
|
'updated_at' => now(),
|
|
]);
|
|
|
|
foreach ($band['runs']->values() as $suiteIndex => $row) {
|
|
DB::table('session_suite_rows')->insert([
|
|
'session_summary_id' => $summaryId,
|
|
'benchmark_run_id' => $row['run_pk'],
|
|
'run_id' => $row['run_id'],
|
|
'suite' => $row['suite'],
|
|
'level' => $row['level'],
|
|
'passed' => $row['passed'],
|
|
'failed' => $row['failed'],
|
|
'avg_user_tps' => $row['avg_user_tps'],
|
|
'weighted_user_tps' => $row['weighted_user_tps'],
|
|
'gen_tps' => $row['gen_tps'],
|
|
'total_tokens' => $row['total_tokens'],
|
|
'sequence' => $suiteIndex,
|
|
'created_at' => now(),
|
|
'updated_at' => now(),
|
|
]);
|
|
}
|
|
}
|
|
});
|
|
|
|
$this->info('session summaries rebuilt: '.count($sessions).' session(s).');
|
|
|
|
return self::SUCCESS;
|
|
}
|
|
|
|
/**
|
|
* Aggregate one run. Returns a collection: normally one row, but a
|
|
* depthsweep run expands into one row per thinking level — each level's
|
|
* rows aggregate independently so the band renders level-by-level.
|
|
*
|
|
* @return Collection<int, array>
|
|
*/
|
|
private function aggregateRun(object $run)
|
|
{
|
|
$model = Suites::modelFor($run->suite);
|
|
|
|
if ($model === null) {
|
|
return collect([$this->emptyAgg($run)]);
|
|
}
|
|
|
|
$table = $model::make()->getTable();
|
|
|
|
if ($run->suite === 'depthsweep') {
|
|
// One suite row per thinking level; levels with no rows are
|
|
// skipped. The level badge is the identity of the row.
|
|
$levelOrder = ['off' => 0, 'low' => 1, 'medium' => 2, 'xhigh' => 3];
|
|
|
|
return collect(DB::table($table)
|
|
->where('benchmark_run_id', $run->id)
|
|
->whereNotNull('level')
|
|
->groupBy('level')
|
|
->get([
|
|
'level',
|
|
DB::raw('count(*) filter (where ok) as passed'),
|
|
DB::raw('count(*) filter (where not ok) as failed'),
|
|
DB::raw('avg(nullif(ttft, 0)) as avg_ttft'),
|
|
DB::raw('avg(nullif(wall, 0)) as avg_wall'),
|
|
DB::raw('sum(total) as total_tokens'),
|
|
DB::raw('sum(user_tps * total) / nullif(sum(total), 0) as weighted_user_tps'),
|
|
DB::raw('sum(ln(user_tps)) filter (where user_tps > 0) as ln_sum'),
|
|
DB::raw('count(*) filter (where user_tps > 0) as ln_n'),
|
|
]))
|
|
->sortBy(fn ($agg) => $levelOrder[$agg->level] ?? 99)
|
|
->values()
|
|
->map(fn ($agg) => [
|
|
'run_pk' => $run->id,
|
|
'run_id' => $run->run_id,
|
|
'suite' => $run->suite,
|
|
'level' => $agg->level,
|
|
'passed' => (int) $agg->passed,
|
|
'failed' => (int) $agg->failed,
|
|
'avg_user_tps' => null,
|
|
'weighted_user_tps' => $this->roundOrNull($agg->weighted_user_tps),
|
|
'gen_tps' => null,
|
|
'total_tokens' => (int) ($agg->total_tokens ?? 0),
|
|
'avg_ttft' => $this->roundOrNull($agg->avg_ttft),
|
|
'avg_wall' => $this->roundOrNull($agg->avg_wall),
|
|
'ln_sum' => (float) ($agg->ln_sum ?? 0),
|
|
'ln_n' => (int) ($agg->ln_n ?? 0),
|
|
]);
|
|
}
|
|
|
|
if ($run->suite === 'scenariobench') {
|
|
$agg = DB::table($table)->where('benchmark_run_id', $run->id)
|
|
->selectRaw('count(*) filter (where ok) as passed')
|
|
->selectRaw('count(*) filter (where not ok) as failed')
|
|
->selectRaw('sum(gen_tokens) as total_tokens')
|
|
->selectRaw('sum(gen_tokens) / nullif(sum(wall), 0) as gen_tps')
|
|
->first();
|
|
|
|
return collect([[
|
|
'run_pk' => $run->id,
|
|
'run_id' => $run->run_id,
|
|
'suite' => $run->suite,
|
|
'level' => $this->runLevel($table, $run->id),
|
|
'passed' => (int) ($agg->passed ?? 0),
|
|
'failed' => (int) ($agg->failed ?? 0),
|
|
'avg_user_tps' => null,
|
|
'weighted_user_tps' => null,
|
|
'gen_tps' => $this->roundOrNull($agg->gen_tps),
|
|
'total_tokens' => (int) ($agg->total_tokens ?? 0),
|
|
]]);
|
|
}
|
|
|
|
$agg = DB::table($table)->where('benchmark_run_id', $run->id)
|
|
->selectRaw('count(*) filter (where ok) as passed')
|
|
->selectRaw('count(*) filter (where not ok) as failed')
|
|
->selectRaw('avg(user_tps) as avg_user_tps')
|
|
->selectRaw('sum(user_tps * total) / nullif(sum(total), 0) as weighted_user_tps')
|
|
->selectRaw('sum(total) as total_tokens')
|
|
->selectRaw('sum(ln(user_tps)) filter (where user_tps > 0) as ln_sum')
|
|
->selectRaw('count(*) filter (where user_tps > 0) as ln_n')
|
|
->first();
|
|
|
|
return collect([[
|
|
'run_pk' => $run->id,
|
|
'run_id' => $run->run_id,
|
|
'suite' => $run->suite,
|
|
'level' => $this->runLevel($table, $run->id),
|
|
'passed' => (int) ($agg->passed ?? 0),
|
|
'failed' => (int) ($agg->failed ?? 0),
|
|
'avg_user_tps' => $this->roundOrNull($agg->avg_user_tps),
|
|
'weighted_user_tps' => $this->roundOrNull($agg->weighted_user_tps),
|
|
'gen_tps' => null,
|
|
'total_tokens' => (int) ($agg->total_tokens ?? 0),
|
|
'ln_sum' => (float) ($agg->ln_sum ?? 0),
|
|
'ln_n' => (int) ($agg->ln_n ?? 0),
|
|
]]);
|
|
}
|
|
|
|
/**
|
|
* Aggregate all scenariobench runs in a band into one summary row.
|
|
* Aggregates passed/failed/tokens and generated tok/s across every
|
|
* scenario run that belongs to the band.
|
|
*
|
|
* @param array<int, object> $runs
|
|
*/
|
|
private function aggregateScenarioBand(array $runs): array
|
|
{
|
|
$runPks = array_map(fn ($r) => $r->id, $runs);
|
|
$first = $runs[0];
|
|
|
|
$agg = DB::table('scenario_runs')
|
|
->whereIn('benchmark_run_id', $runPks)
|
|
->selectRaw('count(*) filter (where ok) as passed')
|
|
->selectRaw('count(*) filter (where not ok) as failed')
|
|
->selectRaw('sum(gen_tokens) as total_tokens')
|
|
->selectRaw('sum(gen_tokens) / nullif(sum(wall), 0) as gen_tps')
|
|
->first();
|
|
|
|
// Thinking level: keep it only when every scenario row in the band
|
|
// agrees — matches the per-run runLevel() rule, applied at band scope.
|
|
$level = DB::table('scenario_runs')
|
|
->whereIn('benchmark_run_id', $runPks)
|
|
->distinct()->pluck('level')->filter()->values();
|
|
|
|
return [
|
|
'run_pk' => $first->id,
|
|
'run_id' => $first->run_id,
|
|
'suite' => 'scenariobench',
|
|
'level' => $level->count() === 1 ? $level->first() : null,
|
|
'passed' => (int) ($agg->passed ?? 0),
|
|
'failed' => (int) ($agg->failed ?? 0),
|
|
'avg_user_tps' => null,
|
|
'weighted_user_tps' => null,
|
|
'gen_tps' => $this->roundOrNull($agg->gen_tps),
|
|
'total_tokens' => (int) ($agg->total_tokens ?? 0),
|
|
];
|
|
}
|
|
|
|
private function emptyAgg(object $run): array
|
|
{
|
|
return [
|
|
'run_pk' => $run->id,
|
|
'run_id' => $run->run_id,
|
|
'suite' => $run->suite,
|
|
'level' => null,
|
|
'passed' => 0,
|
|
'failed' => 0,
|
|
'avg_user_tps' => null,
|
|
'weighted_user_tps' => null,
|
|
'gen_tps' => null,
|
|
'total_tokens' => 0,
|
|
'ln_sum' => 0.0,
|
|
'ln_n' => 0,
|
|
];
|
|
}
|
|
|
|
private function roundOrNull($value, int $precision = 1): ?float
|
|
{
|
|
return $value === null ? null : round((float) $value, $precision);
|
|
}
|
|
|
|
/**
|
|
* The run's thinking level: the distinct level across its result rows
|
|
* (a run is single-level; falls back to null when mixed or empty).
|
|
*/
|
|
private function runLevel(string $table, int $runPk): ?string
|
|
{
|
|
$level = DB::table($table)->where('benchmark_run_id', $runPk)
|
|
->distinct()->pluck('level')->filter()->values();
|
|
|
|
return $level->count() === 1 ? $level->first() : null;
|
|
}
|
|
}
|