leftJoin('users', 'users.id', '=', 'benchmark_runs.user_id') ->leftJoin('server_configs', 'server_configs.id', '=', 'benchmark_runs.config_id') ->orderByDesc('ran_at') ->orderByDesc('benchmark_runs.id') ->get([ 'benchmark_runs.*', 'users.name as runner_name', 'server_configs.name as config_name', ]); foreach ($runs as $run) { // Grouping key: user + GPU + model + quant + visibility + engine // + harness identity + named server config. Free-text labels are // intentionally NOT part of identity: rerunning psychbench with // the same actual settings belongs in the same bubble even when // the user wrote a different run note. $key = implode('|', [ $run->model_id ?? '-', $run->model_quant ?? '-', $run->gpu_id ?? 'raw:'.$run->gpu, $run->gpu_count ?? '-', $run->user_id ?? '-', $run->visibility, $run->engine ?? '-', $run->engine_build ?: '-', // Older direct runs stored empty harness fields; newer // depthsweep writes the explicit equivalent. Normalize so // semantically identical direct runs share a session. $run->harness ?: 'direct', $run->harness_ver ?: '-', $run->harness_mode ?: 'one-shot', $run->config_id ?? 'none', ]); $sessions[$key] ??= [ 'key' => $key, 'labels' => collect([$run->label])->filter(), 'model_id' => $run->model_id, 'model_quant' => $run->model_quant, 'gpu' => $run->gpu, 'gpu_count' => $run->gpu_count, 'runner_name' => $run->runner_name, 'user_id' => $run->user_id, 'visibility' => $run->visibility, 'engine' => $run->engine, 'config_name' => $run->config_name, 'ran_at' => $run->ran_at, 'passed' => 0, 'failed' => 0, 'tokens' => 0, 'runs' => collect(), ]; if ($run->label !== null && $run->label !== '') { $sessions[$key]['labels']->push($run->label); } $aggs = $this->aggregateRun($run); $sessions[$key]['passed'] += $aggs->sum('passed'); $sessions[$key]['failed'] += $aggs->sum('failed'); $sessions[$key]['tokens'] += $aggs->sum('total_tokens'); $sessions[$key]['runs'] = $sessions[$key]['runs']->merge($aggs); } DB::transaction(function () use ($sessions) { DB::table('session_summaries')->truncate(); foreach (array_values($sessions) as $bandIndex => $band) { $summaryId = DB::table('session_summaries')->insertGetId([ 'session_key' => $band['key'], // Keep a label only when every grouped run agrees. A // mixed label is a note, not a configuration split. 'label' => $band['labels']->unique()->count() === 1 ? $band['labels']->first() : null, 'model_id' => $band['model_id'], 'model_quant' => $band['model_quant'], 'gpu' => $band['gpu'], 'gpu_count' => $band['gpu_count'], 'runner_name' => $band['runner_name'], 'user_id' => $band['user_id'], 'visibility' => $band['visibility'], 'engine' => $band['engine'], 'config_name' => $band['config_name'], 'ran_at' => $band['ran_at'], 'total_passed' => $band['passed'], 'total_failed' => $band['failed'], 'total_tokens' => $band['tokens'], 'sequence' => $bandIndex, 'created_at' => now(), 'updated_at' => now(), ]); foreach ($band['runs']->values() as $suiteIndex => $row) { DB::table('session_suite_rows')->insert([ 'session_summary_id' => $summaryId, 'benchmark_run_id' => $row['run_pk'], 'run_id' => $row['run_id'], 'suite' => $row['suite'], 'level' => $row['level'], 'passed' => $row['passed'], 'failed' => $row['failed'], 'avg_user_tps' => $row['avg_user_tps'], 'weighted_user_tps' => $row['weighted_user_tps'], 'gen_tps' => $row['gen_tps'], 'total_tokens' => $row['total_tokens'], 'sequence' => $suiteIndex, 'created_at' => now(), 'updated_at' => now(), ]); } } }); $this->info('session summaries rebuilt: '.count($sessions).' session(s).'); return self::SUCCESS; } /** * Aggregate one run. Returns a collection: normally one row, but a * depthsweep run expands into one row per thinking level — each level's * rows aggregate independently so the band renders level-by-level. * * @return Collection */ private function aggregateRun(object $run) { $model = Suites::modelFor($run->suite); if ($model === null) { return collect([$this->emptyAgg($run)]); } $table = $model::make()->getTable(); if ($run->suite === 'depthsweep') { // One suite row per thinking level; levels with no rows are // skipped. The level badge is the identity of the row. $levelOrder = ['off' => 0, 'low' => 1, 'medium' => 2, 'xhigh' => 3]; return collect(DB::table($table) ->where('benchmark_run_id', $run->id) ->whereNotNull('level') ->groupBy('level') ->get([ 'level', DB::raw('count(*) filter (where ok) as passed'), DB::raw('count(*) filter (where not ok) as failed'), DB::raw('avg(nullif(ttft, 0)) as avg_ttft'), DB::raw('avg(nullif(wall, 0)) as avg_wall'), DB::raw('sum(total) as total_tokens'), DB::raw('sum(user_tps * total) / nullif(sum(total), 0) as weighted_user_tps'), ])) ->sortBy(fn ($agg) => $levelOrder[$agg->level] ?? 99) ->values() ->map(fn ($agg) => [ 'run_pk' => $run->id, 'run_id' => $run->run_id, 'suite' => $run->suite, 'level' => $agg->level, 'passed' => (int) $agg->passed, 'failed' => (int) $agg->failed, // depthsweep has no decode metrics: speed stays blank 'avg_user_tps' => null, 'weighted_user_tps' => $this->roundOrNull($agg->weighted_user_tps), 'gen_tps' => null, 'total_tokens' => (int) ($agg->total_tokens ?? 0), 'avg_ttft' => $this->roundOrNull($agg->avg_ttft), 'avg_wall' => $this->roundOrNull($agg->avg_wall), ]); } if ($run->suite === 'scenariobench') { $agg = DB::table($table)->where('benchmark_run_id', $run->id) ->selectRaw('count(*) filter (where ok) as passed') ->selectRaw('count(*) filter (where not ok) as failed') ->selectRaw('sum(gen_tokens) as total_tokens') ->selectRaw('sum(gen_tokens) / nullif(sum(wall), 0) as gen_tps') ->first(); return collect([[ 'run_pk' => $run->id, 'run_id' => $run->run_id, 'suite' => $run->suite, 'level' => $this->runLevel($table, $run->id), 'passed' => (int) ($agg->passed ?? 0), 'failed' => (int) ($agg->failed ?? 0), 'avg_user_tps' => null, 'weighted_user_tps' => null, 'gen_tps' => $this->roundOrNull($agg->gen_tps), 'total_tokens' => (int) ($agg->total_tokens ?? 0), ]]); } $agg = DB::table($table)->where('benchmark_run_id', $run->id) ->selectRaw('count(*) filter (where ok) as passed') ->selectRaw('count(*) filter (where not ok) as failed') ->selectRaw('avg(user_tps) as avg_user_tps') ->selectRaw('sum(user_tps * total) / nullif(sum(total), 0) as weighted_user_tps') ->selectRaw('sum(total) as total_tokens') ->first(); return collect([[ 'run_pk' => $run->id, 'run_id' => $run->run_id, 'suite' => $run->suite, 'level' => $this->runLevel($table, $run->id), 'passed' => (int) ($agg->passed ?? 0), 'failed' => (int) ($agg->failed ?? 0), 'avg_user_tps' => $this->roundOrNull($agg->avg_user_tps), 'weighted_user_tps' => $this->roundOrNull($agg->weighted_user_tps), 'gen_tps' => null, 'total_tokens' => (int) ($agg->total_tokens ?? 0), ]]); } private function emptyAgg(object $run): array { return [ 'run_pk' => $run->id, 'run_id' => $run->run_id, 'suite' => $run->suite, 'level' => null, 'passed' => 0, 'failed' => 0, 'avg_user_tps' => null, 'weighted_user_tps' => null, 'gen_tps' => null, 'total_tokens' => 0, ]; } private function roundOrNull($value, int $precision = 1): ?float { return $value === null ? null : round((float) $value, $precision); } /** * The run's thinking level: the distinct level across its result rows * (a run is single-level; falls back to null when mixed or empty). */ private function runLevel(string $table, int $runPk): ?string { $level = DB::table($table)->where('benchmark_run_id', $runPk) ->distinct()->pluck('level')->filter()->values(); return $level->count() === 1 ? $level->first() : null; } }