ac941fdd27
The headline totals were using a per-task geometric mean, which gave disproportionate weight to short responses with low effective user_tps (e.g. 2-token SoftBench answers at ~1.2 tok/s). This produced misleadingly low Devlogic totals (20.5 for asus_210w) when the displayed suite speeds were all in the 40-50 tok/s range. Replace geo-mean accumulators (ln_sum/ln_n) with token-weighted accumulators (rate_token_sum = sum(user_tps × tokens)). The headline total is now rate_token_sum / total_tokens, matching the methodology already used by each displayed suite row. For ScenarioBench, the suite aggregate gen_tps (sum(gen_tokens)/sum(wall)) is weighted by the suite's total generated tokens, keeping the headline total consistent with the displayed ScenarioBench row. Results for the 210W bands: asus_210w: devlogic 20.5 → 47.0, sysops 19.5 → 10.6 sapphire_210w: devlogic 19.6 → 43.7, sysops 8.3 → 8.8
375 lines
16 KiB
PHP
375 lines
16 KiB
PHP
<?php
|
||
|
||
namespace App\Console\Commands;
|
||
|
||
use App\Support\Suites;
|
||
use Illuminate\Console\Command;
|
||
use Illuminate\Support\Collection;
|
||
use Illuminate\Support\Facades\DB;
|
||
use Symfony\Component\Console\Attribute\AsCommand;
|
||
|
||
#[AsCommand(name: 'festbench:summary', description: 'Rebuild the session-grouped home summary tables')]
|
||
class SessionSummary extends Command
|
||
{
|
||
public function handle(): int
|
||
{
|
||
$sessions = [];
|
||
|
||
$runs = DB::table('benchmark_runs')
|
||
->leftJoin('users', 'users.id', '=', 'benchmark_runs.user_id')
|
||
->leftJoin('server_configs', 'server_configs.id', '=', 'benchmark_runs.config_id')
|
||
->orderByDesc('ran_at')
|
||
->orderByDesc('benchmark_runs.id')
|
||
->get([
|
||
'benchmark_runs.*',
|
||
'users.name as runner_name',
|
||
'server_configs.name as config_name',
|
||
]);
|
||
|
||
foreach ($runs as $run) {
|
||
// Grouping key: user + GPU + model + quant + visibility + engine
|
||
// + harness identity + named server config + label. The label is
|
||
// the run-wide experiment/config variant (e.g. a power-cap
|
||
// setting), so different labels are different sessions even when
|
||
// every other dimension matches. Normalize null/empty to a
|
||
// single sentinel so unlabeled runs still group together.
|
||
$normalizedLabel = $run->label !== null && $run->label !== ''
|
||
? $run->label
|
||
: 'unlabeled';
|
||
|
||
$key = implode('|', [
|
||
$run->model_id ?? '-',
|
||
$run->model_quant ?? '-',
|
||
$run->gpu_id ?? 'raw:'.$run->gpu,
|
||
$run->gpu_count ?? '-',
|
||
$run->user_id ?? '-',
|
||
$run->visibility,
|
||
$run->engine ?? '-',
|
||
$run->engine_build ?: '-',
|
||
// Older direct runs stored empty harness fields; newer
|
||
// depthsweep writes the explicit equivalent. Normalize so
|
||
// semantically identical direct runs share a session.
|
||
$run->harness ?: 'direct',
|
||
$run->harness_ver ?: '-',
|
||
$run->harness_mode ?: 'one-shot',
|
||
$run->config_id ?? 'none',
|
||
$normalizedLabel,
|
||
]);
|
||
|
||
$sessions[$key] ??= [
|
||
'key' => $key,
|
||
'label' => $run->label,
|
||
'model_id' => $run->model_id,
|
||
'model_quant' => $run->model_quant,
|
||
'gpu' => $run->gpu,
|
||
'gpu_count' => $run->gpu_count,
|
||
'runner_name' => $run->runner_name,
|
||
'user_id' => $run->user_id,
|
||
'visibility' => $run->visibility,
|
||
'engine' => $run->engine,
|
||
'config_name' => $run->config_name,
|
||
'ran_at' => $run->ran_at,
|
||
// Devlogic totals: softbench, prosebench, psychbench,
|
||
// depthsweep — everything except infrabench and scenariobench.
|
||
// Token-weighted: sum(user_tps × tokens) / sum(tokens).
|
||
'dl_passed' => 0,
|
||
'dl_failed' => 0,
|
||
'dl_tokens' => 0,
|
||
'dl_rate_sum' => 0.0,
|
||
// Sysops totals: infrabench and scenariobench.
|
||
// Token-weighted at the suite level: each suite's aggregate
|
||
// rate is weighted by that suite's total tokens.
|
||
'so_passed' => 0,
|
||
'so_failed' => 0,
|
||
'so_tokens' => 0,
|
||
'so_rate_sum' => 0.0,
|
||
'runs' => collect(),
|
||
];
|
||
|
||
// Scenariobench is aggregated at band level (one row for all
|
||
// scenario runs in the band) rather than per-run, so collect the
|
||
// run rows and defer to a second pass below.
|
||
if ($run->suite === 'scenariobench') {
|
||
$sessions[$key]['scenario_runs'][] = $run;
|
||
|
||
continue;
|
||
}
|
||
|
||
$aggs = $this->aggregateRun($run);
|
||
|
||
// Route the aggregation to the correct total bucket.
|
||
$isSysops = $run->suite === 'infrabench';
|
||
|
||
$sessions[$key]['runs'] = $sessions[$key]['runs']->merge($aggs);
|
||
|
||
if ($isSysops) {
|
||
$sessions[$key]['so_passed'] += $aggs->sum('passed');
|
||
$sessions[$key]['so_failed'] += $aggs->sum('failed');
|
||
$sessions[$key]['so_tokens'] += $aggs->sum('total_tokens');
|
||
$sessions[$key]['so_rate_sum'] += $aggs->sum('rate_token_sum');
|
||
} else {
|
||
$sessions[$key]['dl_passed'] += $aggs->sum('passed');
|
||
$sessions[$key]['dl_failed'] += $aggs->sum('failed');
|
||
$sessions[$key]['dl_tokens'] += $aggs->sum('total_tokens');
|
||
$sessions[$key]['dl_rate_sum'] += $aggs->sum('rate_token_sum');
|
||
}
|
||
}
|
||
|
||
// Second pass: emit one aggregated scenariobench row per band,
|
||
// covering all scenario runs in that band. Scenariobench is sysops.
|
||
foreach ($sessions as $key => $band) {
|
||
$scenarioRuns = $band['scenario_runs'] ?? [];
|
||
if (empty($scenarioRuns)) {
|
||
continue;
|
||
}
|
||
$agg = $this->aggregateScenarioBand($scenarioRuns);
|
||
$sessions[$key]['so_passed'] += $agg['passed'];
|
||
$sessions[$key]['so_failed'] += $agg['failed'];
|
||
$sessions[$key]['so_tokens'] += $agg['total_tokens'];
|
||
$sessions[$key]['so_rate_sum'] += $agg['rate_token_sum'] ?? 0;
|
||
$sessions[$key]['runs'] = $band['runs']->merge([$agg]);
|
||
}
|
||
|
||
DB::transaction(function () use ($sessions) {
|
||
DB::table('session_summaries')->truncate();
|
||
|
||
foreach (array_values($sessions) as $bandIndex => $band) {
|
||
$summaryId = DB::table('session_summaries')->insertGetId([
|
||
'session_key' => $band['key'],
|
||
'label' => $band['label'],
|
||
'model_id' => $band['model_id'],
|
||
'model_quant' => $band['model_quant'],
|
||
'gpu' => $band['gpu'],
|
||
'gpu_count' => $band['gpu_count'],
|
||
'runner_name' => $band['runner_name'],
|
||
'user_id' => $band['user_id'],
|
||
'visibility' => $band['visibility'],
|
||
'engine' => $band['engine'],
|
||
'config_name' => $band['config_name'],
|
||
'ran_at' => $band['ran_at'],
|
||
'devlogic_passed' => $band['dl_passed'],
|
||
'devlogic_failed' => $band['dl_failed'],
|
||
'devlogic_tokens' => $band['dl_tokens'],
|
||
'devlogic_tps' => $band['dl_tokens'] > 0
|
||
? round($band['dl_rate_sum'] / $band['dl_tokens'], 1)
|
||
: null,
|
||
'sysops_passed' => $band['so_passed'],
|
||
'sysops_failed' => $band['so_failed'],
|
||
'sysops_tokens' => $band['so_tokens'],
|
||
'sysops_tps' => $band['so_tokens'] > 0
|
||
? round($band['so_rate_sum'] / $band['so_tokens'], 1)
|
||
: null,
|
||
'sequence' => $bandIndex,
|
||
'created_at' => now(),
|
||
'updated_at' => now(),
|
||
]);
|
||
|
||
foreach ($band['runs']->values() as $suiteIndex => $row) {
|
||
DB::table('session_suite_rows')->insert([
|
||
'session_summary_id' => $summaryId,
|
||
'benchmark_run_id' => $row['run_pk'],
|
||
'run_id' => $row['run_id'],
|
||
'suite' => $row['suite'],
|
||
'level' => $row['level'],
|
||
'passed' => $row['passed'],
|
||
'failed' => $row['failed'],
|
||
'avg_user_tps' => $row['avg_user_tps'],
|
||
'weighted_user_tps' => $row['weighted_user_tps'],
|
||
'gen_tps' => $row['gen_tps'],
|
||
'total_tokens' => $row['total_tokens'],
|
||
'sequence' => $suiteIndex,
|
||
'created_at' => now(),
|
||
'updated_at' => now(),
|
||
]);
|
||
}
|
||
}
|
||
});
|
||
|
||
$this->info('session summaries rebuilt: '.count($sessions).' session(s).');
|
||
|
||
return self::SUCCESS;
|
||
}
|
||
|
||
/**
|
||
* Aggregate one run. Returns a collection: normally one row, but a
|
||
* depthsweep run expands into one row per thinking level — each level's
|
||
* rows aggregate independently so the band renders level-by-level.
|
||
*
|
||
* @return Collection<int, array>
|
||
*/
|
||
private function aggregateRun(object $run)
|
||
{
|
||
$model = Suites::modelFor($run->suite);
|
||
|
||
if ($model === null) {
|
||
return collect([$this->emptyAgg($run)]);
|
||
}
|
||
|
||
$table = $model::make()->getTable();
|
||
|
||
if ($run->suite === 'depthsweep') {
|
||
// One suite row per thinking level; levels with no rows are
|
||
// skipped. The level badge is the identity of the row.
|
||
$levelOrder = ['off' => 0, 'low' => 1, 'medium' => 2, 'xhigh' => 3];
|
||
|
||
return collect(DB::table($table)
|
||
->where('benchmark_run_id', $run->id)
|
||
->whereNotNull('level')
|
||
->groupBy('level')
|
||
->get([
|
||
'level',
|
||
DB::raw('count(*) filter (where ok) as passed'),
|
||
DB::raw('count(*) filter (where not ok) as failed'),
|
||
DB::raw('avg(nullif(ttft, 0)) as avg_ttft'),
|
||
DB::raw('avg(nullif(wall, 0)) as avg_wall'),
|
||
DB::raw('sum(total) as total_tokens'),
|
||
DB::raw('sum(user_tps * total) / nullif(sum(total), 0) as weighted_user_tps'),
|
||
DB::raw('sum(user_tps * total) as rate_token_sum'),
|
||
]))
|
||
->sortBy(fn ($agg) => $levelOrder[$agg->level] ?? 99)
|
||
->values()
|
||
->map(fn ($agg) => [
|
||
'run_pk' => $run->id,
|
||
'run_id' => $run->run_id,
|
||
'suite' => $run->suite,
|
||
'level' => $agg->level,
|
||
'passed' => (int) $agg->passed,
|
||
'failed' => (int) $agg->failed,
|
||
'avg_user_tps' => null,
|
||
'weighted_user_tps' => $this->roundOrNull($agg->weighted_user_tps),
|
||
'gen_tps' => null,
|
||
'total_tokens' => (int) ($agg->total_tokens ?? 0),
|
||
'avg_ttft' => $this->roundOrNull($agg->avg_ttft),
|
||
'avg_wall' => $this->roundOrNull($agg->avg_wall),
|
||
'rate_token_sum' => (float) ($agg->rate_token_sum ?? 0),
|
||
]);
|
||
}
|
||
|
||
if ($run->suite === 'scenariobench') {
|
||
$agg = DB::table($table)->where('benchmark_run_id', $run->id)
|
||
->selectRaw('count(*) filter (where ok) as passed')
|
||
->selectRaw('count(*) filter (where not ok) as failed')
|
||
->selectRaw('sum(gen_tokens) as total_tokens')
|
||
->selectRaw('sum(gen_tokens) / nullif(sum(wall), 0) as gen_tps')
|
||
->first();
|
||
|
||
$genTps = (float) ($agg->gen_tps ?? 0);
|
||
$tokens = (int) ($agg->total_tokens ?? 0);
|
||
|
||
return collect([[
|
||
'run_pk' => $run->id,
|
||
'run_id' => $run->run_id,
|
||
'suite' => $run->suite,
|
||
'level' => $this->runLevel($table, $run->id),
|
||
'passed' => (int) ($agg->passed ?? 0),
|
||
'failed' => (int) ($agg->failed ?? 0),
|
||
'avg_user_tps' => null,
|
||
'weighted_user_tps' => null,
|
||
'gen_tps' => $this->roundOrNull($agg->gen_tps),
|
||
'total_tokens' => $tokens,
|
||
'rate_token_sum' => $genTps * $tokens,
|
||
]]);
|
||
}
|
||
|
||
$agg = DB::table($table)->where('benchmark_run_id', $run->id)
|
||
->selectRaw('count(*) filter (where ok) as passed')
|
||
->selectRaw('count(*) filter (where not ok) as failed')
|
||
->selectRaw('avg(user_tps) as avg_user_tps')
|
||
->selectRaw('sum(user_tps * total) / nullif(sum(total), 0) as weighted_user_tps')
|
||
->selectRaw('sum(total) as total_tokens')
|
||
->selectRaw('sum(user_tps * total) as rate_token_sum')
|
||
->first();
|
||
|
||
return collect([[
|
||
'run_pk' => $run->id,
|
||
'run_id' => $run->run_id,
|
||
'suite' => $run->suite,
|
||
'level' => $this->runLevel($table, $run->id),
|
||
'passed' => (int) ($agg->passed ?? 0),
|
||
'failed' => (int) ($agg->failed ?? 0),
|
||
'avg_user_tps' => $this->roundOrNull($agg->avg_user_tps),
|
||
'weighted_user_tps' => $this->roundOrNull($agg->weighted_user_tps),
|
||
'gen_tps' => null,
|
||
'total_tokens' => (int) ($agg->total_tokens ?? 0),
|
||
'rate_token_sum' => (float) ($agg->rate_token_sum ?? 0),
|
||
]]);
|
||
}
|
||
|
||
/**
|
||
* Aggregate all scenariobench runs in a band into one summary row.
|
||
* Aggregates passed/failed/tokens and generated tok/s across every
|
||
* scenario run that belongs to the band.
|
||
*
|
||
* @param array<int, object> $runs
|
||
*/
|
||
private function aggregateScenarioBand(array $runs): array
|
||
{
|
||
$runPks = array_map(fn ($r) => $r->id, $runs);
|
||
$first = $runs[0];
|
||
|
||
$agg = DB::table('scenario_runs')
|
||
->whereIn('benchmark_run_id', $runPks)
|
||
->selectRaw('count(*) filter (where ok) as passed')
|
||
->selectRaw('count(*) filter (where not ok) as failed')
|
||
->selectRaw('sum(gen_tokens) as total_tokens')
|
||
->selectRaw('sum(gen_tokens) / nullif(sum(wall), 0) as gen_tps')
|
||
->first();
|
||
|
||
$genTps = (float) ($agg->gen_tps ?? 0);
|
||
$tokens = (int) ($agg->total_tokens ?? 0);
|
||
|
||
// Thinking level: keep it only when every scenario row in the band
|
||
// agrees — matches the per-run runLevel() rule, applied at band scope.
|
||
$level = DB::table('scenario_runs')
|
||
->whereIn('benchmark_run_id', $runPks)
|
||
->distinct()->pluck('level')->filter()->values();
|
||
|
||
return [
|
||
'run_pk' => $first->id,
|
||
'run_id' => $first->run_id,
|
||
'suite' => 'scenariobench',
|
||
'level' => $level->count() === 1 ? $level->first() : null,
|
||
'passed' => (int) ($agg->passed ?? 0),
|
||
'failed' => (int) ($agg->failed ?? 0),
|
||
'avg_user_tps' => null,
|
||
'weighted_user_tps' => null,
|
||
'gen_tps' => $this->roundOrNull($agg->gen_tps),
|
||
'total_tokens' => $tokens,
|
||
'rate_token_sum' => $genTps * $tokens,
|
||
];
|
||
}
|
||
|
||
private function emptyAgg(object $run): array
|
||
{
|
||
return [
|
||
'run_pk' => $run->id,
|
||
'run_id' => $run->run_id,
|
||
'suite' => $run->suite,
|
||
'level' => null,
|
||
'passed' => 0,
|
||
'failed' => 0,
|
||
'avg_user_tps' => null,
|
||
'weighted_user_tps' => null,
|
||
'gen_tps' => null,
|
||
'total_tokens' => 0,
|
||
'rate_token_sum' => 0.0,
|
||
];
|
||
}
|
||
|
||
private function roundOrNull($value, int $precision = 1): ?float
|
||
{
|
||
return $value === null ? null : round((float) $value, $precision);
|
||
}
|
||
|
||
/**
|
||
* The run's thinking level: the distinct level across its result rows
|
||
* (a run is single-level; falls back to null when mixed or empty).
|
||
*/
|
||
private function runLevel(string $table, int $runPk): ?string
|
||
{
|
||
$level = DB::table($table)->where('benchmark_run_id', $runPk)
|
||
->distinct()->pluck('level')->filter()->values();
|
||
|
||
return $level->count() === 1 ? $level->first() : null;
|
||
}
|
||
}
|