festbench-web/app/Console/Commands/SessionSummary.php

<?php

namespace App\Console\Commands;

use App\Support\Suites;
use Illuminate\Console\Command;
use Illuminate\Support\Facades\DB;
use Symfony\Component\Console\Attribute\AsCommand;

#[AsCommand(name: 'festbench:summary', description: 'Rebuild the session-grouped home summary tables')]
class SessionSummary extends Command
{
    public function handle(): int
    {
        $sessions = [];

        $runs = DB::table('benchmark_runs')
            ->leftJoin('users', 'users.id', '=', 'benchmark_runs.user_id')
            ->orderByDesc('ran_at')
            ->orderByDesc('benchmark_runs.id')
            ->get([
                'benchmark_runs.*',
                'users.name as runner_name',
            ]);

        foreach ($runs as $run) {
            // Visibility participates in the grouping key so private runs
            // form their own band — the guest view can then filter bands by
            // visibility without any per-request recomputation.
            $key = implode('|', [
                $run->label ?? '-',
                $run->model_id ?? '-',
                $run->model_quant ?? '-',
                $run->gpu ?? '-',
                $run->gpu_count ?? '-',
                $run->user_id ?? '-',
                $run->visibility,
            ]);

            $sessions[$key] ??= [
                'key' => $key,
                'label' => $run->label,
                'model_id' => $run->model_id,
                'model_quant' => $run->model_quant,
                'gpu' => $run->gpu,
                'gpu_count' => $run->gpu_count,
                'runner_name' => $run->runner_name,
                'user_id' => $run->user_id,
                'visibility' => $run->visibility,
                'ran_at' => $run->ran_at,
                'passed' => 0,
                'failed' => 0,
                'tokens' => 0,
                'runs' => collect(),
            ];

            $agg = $this->aggregateRun($run);

            $sessions[$key]['passed'] += $agg['passed'];
            $sessions[$key]['failed'] += $agg['failed'];
            $sessions[$key]['tokens'] += $agg['total_tokens'];
            $sessions[$key]['runs']->push($agg + ['suite' => $run->suite, 'level' => $agg['level']]);
        }

        DB::transaction(function () use ($sessions) {
            DB::table('session_summaries')->truncate();

            foreach (array_values($sessions) as $bandIndex => $band) {
                $summaryId = DB::table('session_summaries')->insertGetId([
                    'session_key' => $band['key'],
                    'label' => $band['label'],
                    'model_id' => $band['model_id'],
                    'model_quant' => $band['model_quant'],
                    'gpu' => $band['gpu'],
                    'gpu_count' => $band['gpu_count'],
                    'runner_name' => $band['runner_name'],
                    'user_id' => $band['user_id'],
                    'visibility' => $band['visibility'],
                    'ran_at' => $band['ran_at'],
                    'total_passed' => $band['passed'],
                    'total_failed' => $band['failed'],
                    'total_tokens' => $band['tokens'],
                    'sequence' => $bandIndex,
                    'created_at' => now(),
                    'updated_at' => now(),
                ]);

                foreach ($band['runs'] as $suiteIndex => $row) {
                    DB::table('session_suite_rows')->insert([
                        'session_summary_id' => $summaryId,
                        'benchmark_run_id' => $row['run_pk'],
                        'run_id' => $row['run_id'],
                        'suite' => $row['suite'],
                        'level' => $row['level'],
                        'passed' => $row['passed'],
                        'failed' => $row['failed'],
                        'avg_user_tps' => $row['avg_user_tps'],
                        'weighted_user_tps' => $row['weighted_user_tps'],
                        'total_tokens' => $row['total_tokens'],
                        'sequence' => $suiteIndex,
                        'created_at' => now(),
                        'updated_at' => now(),
                    ]);
                }
            }
        });

        $this->info('session summaries rebuilt: '.count($sessions).' session(s).');

        return self::SUCCESS;
    }

    /**
     * Aggregate one run's suite table: pass/fail counts, simple + weighted
     * user tok/s, generated tokens, and the run's thinking level.
     */
    private function aggregateRun(object $run): array
    {
        $model = Suites::modelFor($run->suite);

        if ($model === null) {
            return [
                'run_pk' => $run->id,
                'run_id' => $run->run_id,
                'level' => null,
                'passed' => 0,
                'failed' => 0,
                'avg_user_tps' => null,
                'weighted_user_tps' => null,
                'total_tokens' => 0,
            ];
        }

        $table = $model::make()->getTable();

        if ($run->suite === 'scenariobench') {
            $agg = DB::table($table)->where('benchmark_run_id', $run->id)
                ->selectRaw('count(*) filter (where ok) as passed')
                ->selectRaw('count(*) filter (where not ok) as failed')
                ->selectRaw('sum(gen_tokens) as total_tokens')
                ->first();

            return [
                'run_pk' => $run->id,
                'run_id' => $run->run_id,
                'level' => $this->runLevel($table, $run->id),
                'passed' => (int) ($agg->passed ?? 0),
                'failed' => (int) ($agg->failed ?? 0),
                'avg_user_tps' => null,
                'weighted_user_tps' => null,
                'total_tokens' => (int) ($agg->total_tokens ?? 0),
            ];
        }

        $agg = DB::table($table)->where('benchmark_run_id', $run->id)
            ->selectRaw('count(*) filter (where ok) as passed')
            ->selectRaw('count(*) filter (where not ok) as failed')
            ->selectRaw('avg(user_tps) as avg_user_tps')
            ->selectRaw('sum(user_tps * total) filter (where total > 0) / nullif(sum(total) filter (where total > 0), 0) as weighted_user_tps')
            ->selectRaw('sum(total) as total_tokens')
            ->first();

        return [
            'run_pk' => $run->id,
            'run_id' => $run->run_id,
            'level' => $this->runLevel($table, $run->id),
            'passed' => (int) ($agg->passed ?? 0),
            'failed' => (int) ($agg->failed ?? 0),
            'avg_user_tps' => $agg->avg_user_tps !== null ? round((float) $agg->avg_user_tps, 1) : null,
            'weighted_user_tps' => $agg->weighted_user_tps !== null ? round((float) $agg->weighted_user_tps, 1) : null,
            'total_tokens' => (int) ($agg->total_tokens ?? 0),
        ];
    }

    /**
     * The run's thinking level: the distinct level across its result rows
     * (a run is single-level; falls back to null when mixed or empty).
     */
    private function runLevel(string $table, int $runPk): ?string
    {
        $level = DB::table($table)->where('benchmark_run_id', $runPk)
            ->distinct()->pluck('level')->filter()->values();

        return $level->count() === 1 ? $level->first() : null;
    }
}