leftJoin('users', 'users.id', '=', 'benchmark_runs.user_id') ->orderByDesc('ran_at') ->orderByDesc('benchmark_runs.id') ->get([ 'benchmark_runs.*', 'users.name as runner_name', ]); foreach ($runs as $run) { // Visibility participates in the grouping key so private runs // form their own band — the guest view can then filter bands by // visibility without any per-request recomputation. $key = implode('|', [ $run->label ?? '-', $run->model_id ?? '-', $run->model_quant ?? '-', $run->gpu ?? '-', $run->gpu_count ?? '-', $run->user_id ?? '-', $run->visibility, ]); $sessions[$key] ??= [ 'key' => $key, 'label' => $run->label, 'model_id' => $run->model_id, 'model_quant' => $run->model_quant, 'gpu' => $run->gpu, 'gpu_count' => $run->gpu_count, 'runner_name' => $run->runner_name, 'user_id' => $run->user_id, 'visibility' => $run->visibility, 'ran_at' => $run->ran_at, 'passed' => 0, 'failed' => 0, 'tokens' => 0, 'runs' => collect(), ]; $agg = $this->aggregateRun($run); $sessions[$key]['passed'] += $agg['passed']; $sessions[$key]['failed'] += $agg['failed']; $sessions[$key]['tokens'] += $agg['total_tokens']; $sessions[$key]['runs']->push($agg + ['suite' => $run->suite, 'level' => $agg['level']]); } DB::transaction(function () use ($sessions) { DB::table('session_summaries')->truncate(); foreach (array_values($sessions) as $bandIndex => $band) { $summaryId = DB::table('session_summaries')->insertGetId([ 'session_key' => $band['key'], 'label' => $band['label'], 'model_id' => $band['model_id'], 'model_quant' => $band['model_quant'], 'gpu' => $band['gpu'], 'gpu_count' => $band['gpu_count'], 'runner_name' => $band['runner_name'], 'user_id' => $band['user_id'], 'visibility' => $band['visibility'], 'ran_at' => $band['ran_at'], 'total_passed' => $band['passed'], 'total_failed' => $band['failed'], 'total_tokens' => $band['tokens'], 'sequence' => $bandIndex, 'created_at' => now(), 'updated_at' => now(), ]); foreach ($band['runs'] as $suiteIndex => $row) { DB::table('session_suite_rows')->insert([ 'session_summary_id' => $summaryId, 'benchmark_run_id' => $row['run_pk'], 'run_id' => $row['run_id'], 'suite' => $row['suite'], 'level' => $row['level'], 'passed' => $row['passed'], 'failed' => $row['failed'], 'avg_user_tps' => $row['avg_user_tps'], 'weighted_user_tps' => $row['weighted_user_tps'], 'total_tokens' => $row['total_tokens'], 'sequence' => $suiteIndex, 'created_at' => now(), 'updated_at' => now(), ]); } } }); $this->info('session summaries rebuilt: '.count($sessions).' session(s).'); return self::SUCCESS; } /** * Aggregate one run's suite table: pass/fail counts, simple + weighted * user tok/s, generated tokens, and the run's thinking level. */ private function aggregateRun(object $run): array { $model = Suites::modelFor($run->suite); if ($model === null) { return [ 'run_pk' => $run->id, 'run_id' => $run->run_id, 'level' => null, 'passed' => 0, 'failed' => 0, 'avg_user_tps' => null, 'weighted_user_tps' => null, 'total_tokens' => 0, ]; } $table = $model::make()->getTable(); if ($run->suite === 'scenariobench') { $agg = DB::table($table)->where('benchmark_run_id', $run->id) ->selectRaw('count(*) filter (where ok) as passed') ->selectRaw('count(*) filter (where not ok) as failed') ->selectRaw('sum(gen_tokens) as total_tokens') ->first(); return [ 'run_pk' => $run->id, 'run_id' => $run->run_id, 'level' => $this->runLevel($table, $run->id), 'passed' => (int) ($agg->passed ?? 0), 'failed' => (int) ($agg->failed ?? 0), 'avg_user_tps' => null, 'weighted_user_tps' => null, 'total_tokens' => (int) ($agg->total_tokens ?? 0), ]; } $agg = DB::table($table)->where('benchmark_run_id', $run->id) ->selectRaw('count(*) filter (where ok) as passed') ->selectRaw('count(*) filter (where not ok) as failed') ->selectRaw('avg(user_tps) as avg_user_tps') ->selectRaw('sum(user_tps * total) filter (where total > 0) / nullif(sum(total) filter (where total > 0), 0) as weighted_user_tps') ->selectRaw('sum(total) as total_tokens') ->first(); return [ 'run_pk' => $run->id, 'run_id' => $run->run_id, 'level' => $this->runLevel($table, $run->id), 'passed' => (int) ($agg->passed ?? 0), 'failed' => (int) ($agg->failed ?? 0), 'avg_user_tps' => $agg->avg_user_tps !== null ? round((float) $agg->avg_user_tps, 1) : null, 'weighted_user_tps' => $agg->weighted_user_tps !== null ? round((float) $agg->weighted_user_tps, 1) : null, 'total_tokens' => (int) ($agg->total_tokens ?? 0), ]; } /** * The run's thinking level: the distinct level across its result rows * (a run is single-level; falls back to null when mixed or empty). */ private function runLevel(string $table, int $runPk): ?string { $level = DB::table($table)->where('benchmark_run_id', $runPk) ->distinct()->pluck('level')->filter()->values(); return $level->count() === 1 ? $level->first() : null; } }