festbench-web/app/Console/Commands/SessionSummary.php

<?php

namespace App\Console\Commands;

use App\Support\Suites;
use Illuminate\Console\Command;
use Illuminate\Support\Collection;
use Illuminate\Support\Facades\DB;
use Symfony\Component\Console\Attribute\AsCommand;

#[AsCommand(name: 'festbench:summary', description: 'Rebuild the session-grouped home summary tables')]
class SessionSummary extends Command
{
    public function handle(): int
    {
        $sessions = [];

        $runs = DB::table('benchmark_runs')
            ->leftJoin('users', 'users.id', '=', 'benchmark_runs.user_id')
            ->leftJoin('server_configs', 'server_configs.id', '=', 'benchmark_runs.config_id')
            ->orderByDesc('ran_at')
            ->orderByDesc('benchmark_runs.id')
            ->get([
                'benchmark_runs.*',
                'users.name as runner_name',
                'server_configs.name as config_name',
            ]);

        foreach ($runs as $run) {
            // Grouping key: user + GPU + model + quant + visibility + engine
            // + harness identity + named server config. Free-text labels are
            // intentionally NOT part of identity: rerunning psychbench with
            // the same actual settings belongs in the same bubble even when
            // the user wrote a different run note.
            $key = implode('|', [
                $run->model_id ?? '-',
                $run->model_quant ?? '-',
                $run->gpu_id ?? 'raw:'.$run->gpu,
                $run->gpu_count ?? '-',
                $run->user_id ?? '-',
                $run->visibility,
                $run->engine ?? '-',
                $run->engine_build ?: '-',
                // Older direct runs stored empty harness fields; newer
                // depthsweep writes the explicit equivalent. Normalize so
                // semantically identical direct runs share a session.
                $run->harness ?: 'direct',
                $run->harness_ver ?: '-',
                $run->harness_mode ?: 'one-shot',
                $run->config_id ?? 'none',
            ]);

            $sessions[$key] ??= [
                'key' => $key,
                'labels' => collect([$run->label])->filter(),
                'model_id' => $run->model_id,
                'model_quant' => $run->model_quant,
                'gpu' => $run->gpu,
                'gpu_count' => $run->gpu_count,
                'runner_name' => $run->runner_name,
                'user_id' => $run->user_id,
                'visibility' => $run->visibility,
                'engine' => $run->engine,
                'config_name' => $run->config_name,
                'ran_at' => $run->ran_at,
                'passed' => 0,
                'failed' => 0,
                'tokens' => 0,
                'ln_sum' => 0.0,
                'ln_n' => 0,
                'runs' => collect(),
            ];

            if ($run->label !== null && $run->label !== '') {
                $sessions[$key]['labels']->push($run->label);
            }

            // Scenariobench is aggregated at band level (one row for all
            // scenario runs in the band) rather than per-run, so collect the
            // run rows and defer to a second pass below.
            if ($run->suite === 'scenariobench') {
                $sessions[$key]['scenario_runs'][] = $run;

                continue;
            }

            $aggs = $this->aggregateRun($run);

            $sessions[$key]['passed'] += $aggs->sum('passed');
            $sessions[$key]['failed'] += $aggs->sum('failed');
            $sessions[$key]['tokens'] += $aggs->sum('total_tokens');
            $sessions[$key]['ln_sum'] += $aggs->sum('ln_sum');
            $sessions[$key]['ln_n'] += $aggs->sum('ln_n');
            $sessions[$key]['runs'] = $sessions[$key]['runs']->merge($aggs);
        }

        // Second pass: emit one aggregated scenariobench row per band,
        // covering all scenario runs in that band.
        foreach ($sessions as $key => $band) {
            $scenarioRuns = $band['scenario_runs'] ?? [];
            if (empty($scenarioRuns)) {
                continue;
            }
            $agg = $this->aggregateScenarioBand($scenarioRuns);
            $sessions[$key]['passed'] += $agg['passed'];
            $sessions[$key]['failed'] += $agg['failed'];
            $sessions[$key]['tokens'] += $agg['total_tokens'];
            $sessions[$key]['ln_sum'] += $agg['ln_sum'];
            $sessions[$key]['ln_n'] += $agg['ln_n'];
            $sessions[$key]['runs'] = $band['runs']->merge([$agg]);
        }

        DB::transaction(function () use ($sessions) {
            DB::table('session_summaries')->truncate();

            foreach (array_values($sessions) as $bandIndex => $band) {
                $summaryId = DB::table('session_summaries')->insertGetId([
                    'session_key' => $band['key'],
                    // Keep a label only when every grouped run agrees. A
                    // mixed label is a note, not a configuration split.
                    'label' => $band['labels']->unique()->count() === 1
                        ? $band['labels']->first()
                        : null,
                    'model_id' => $band['model_id'],
                    'model_quant' => $band['model_quant'],
                    'gpu' => $band['gpu'],
                    'gpu_count' => $band['gpu_count'],
                    'runner_name' => $band['runner_name'],
                    'user_id' => $band['user_id'],
                    'visibility' => $band['visibility'],
                    'engine' => $band['engine'],
                    'config_name' => $band['config_name'],
                    'ran_at' => $band['ran_at'],
                    'total_passed' => $band['passed'],
                    'total_failed' => $band['failed'],
                    'total_tokens' => $band['tokens'],
                    'total_tps' => $band['ln_n'] > 0
                        ? round(exp($band['ln_sum'] / $band['ln_n']), 1)
                        : null,
                    'sequence' => $bandIndex,
                    'created_at' => now(),
                    'updated_at' => now(),
                ]);

                foreach ($band['runs']->values() as $suiteIndex => $row) {
                    DB::table('session_suite_rows')->insert([
                        'session_summary_id' => $summaryId,
                        'benchmark_run_id' => $row['run_pk'],
                        'run_id' => $row['run_id'],
                        'suite' => $row['suite'],
                        'level' => $row['level'],
                        'passed' => $row['passed'],
                        'failed' => $row['failed'],
                        'avg_user_tps' => $row['avg_user_tps'],
                        'weighted_user_tps' => $row['weighted_user_tps'],
                        'gen_tps' => $row['gen_tps'],
                        'total_tokens' => $row['total_tokens'],
                        'sequence' => $suiteIndex,
                        'created_at' => now(),
                        'updated_at' => now(),
                    ]);
                }
            }
        });

        $this->info('session summaries rebuilt: '.count($sessions).' session(s).');

        return self::SUCCESS;
    }

    /**
     * Aggregate one run. Returns a collection: normally one row, but a
     * depthsweep run expands into one row per thinking level — each level's
     * rows aggregate independently so the band renders level-by-level.
     *
     * @return Collection<int, array>
     */
    private function aggregateRun(object $run)
    {
        $model = Suites::modelFor($run->suite);

        if ($model === null) {
            return collect([$this->emptyAgg($run)]);
        }

        $table = $model::make()->getTable();

        if ($run->suite === 'depthsweep') {
            // One suite row per thinking level; levels with no rows are
            // skipped. The level badge is the identity of the row.
            $levelOrder = ['off' => 0, 'low' => 1, 'medium' => 2, 'xhigh' => 3];

            return collect(DB::table($table)
                ->where('benchmark_run_id', $run->id)
                ->whereNotNull('level')
                ->groupBy('level')
                ->get([
                    'level',
                    DB::raw('count(*) filter (where ok) as passed'),
                    DB::raw('count(*) filter (where not ok) as failed'),
                    DB::raw('avg(nullif(ttft, 0)) as avg_ttft'),
                    DB::raw('avg(nullif(wall, 0)) as avg_wall'),
                    DB::raw('sum(total) as total_tokens'),
                    DB::raw('sum(user_tps * total) / nullif(sum(total), 0) as weighted_user_tps'),
                    DB::raw('sum(ln(user_tps)) filter (where user_tps > 0) as ln_sum'),
                    DB::raw('count(*) filter (where user_tps > 0) as ln_n'),
                ]))
                ->sortBy(fn ($agg) => $levelOrder[$agg->level] ?? 99)
                ->values()
                ->map(fn ($agg) => [
                    'run_pk' => $run->id,
                    'run_id' => $run->run_id,
                    'suite' => $run->suite,
                    'level' => $agg->level,
                    'passed' => (int) $agg->passed,
                    'failed' => (int) $agg->failed,
                    'avg_user_tps' => null,
                    'weighted_user_tps' => $this->roundOrNull($agg->weighted_user_tps),
                    'gen_tps' => null,
                    'total_tokens' => (int) ($agg->total_tokens ?? 0),
                    'avg_ttft' => $this->roundOrNull($agg->avg_ttft),
                    'avg_wall' => $this->roundOrNull($agg->avg_wall),
                    'ln_sum' => (float) ($agg->ln_sum ?? 0),
                    'ln_n' => (int) ($agg->ln_n ?? 0),
                ]);
        }

        if ($run->suite === 'scenariobench') {
            $agg = DB::table($table)->where('benchmark_run_id', $run->id)
                ->selectRaw('count(*) filter (where ok) as passed')
                ->selectRaw('count(*) filter (where not ok) as failed')
                ->selectRaw('sum(gen_tokens) as total_tokens')
                ->selectRaw('sum(gen_tokens) / nullif(sum(wall), 0) as gen_tps')
                ->selectRaw('sum(ln(gen_tokens::float8 / nullif(wall, 0))) filter (where wall > 0 and gen_tokens > 0) as ln_sum')
                ->selectRaw('count(*) filter (where wall > 0 and gen_tokens > 0) as ln_n')
                ->first();

            return collect([[
                'run_pk' => $run->id,
                'run_id' => $run->run_id,
                'suite' => $run->suite,
                'level' => $this->runLevel($table, $run->id),
                'passed' => (int) ($agg->passed ?? 0),
                'failed' => (int) ($agg->failed ?? 0),
                'avg_user_tps' => null,
                'weighted_user_tps' => null,
                'gen_tps' => $this->roundOrNull($agg->gen_tps),
                'total_tokens' => (int) ($agg->total_tokens ?? 0),
                'ln_sum' => (float) ($agg->ln_sum ?? 0),
                'ln_n' => (int) ($agg->ln_n ?? 0),
            ]]);
        }

        $agg = DB::table($table)->where('benchmark_run_id', $run->id)
            ->selectRaw('count(*) filter (where ok) as passed')
            ->selectRaw('count(*) filter (where not ok) as failed')
            ->selectRaw('avg(user_tps) as avg_user_tps')
            ->selectRaw('sum(user_tps * total) / nullif(sum(total), 0) as weighted_user_tps')
            ->selectRaw('sum(total) as total_tokens')
            ->selectRaw('sum(ln(user_tps)) filter (where user_tps > 0) as ln_sum')
            ->selectRaw('count(*) filter (where user_tps > 0) as ln_n')
            ->first();

        return collect([[
            'run_pk' => $run->id,
            'run_id' => $run->run_id,
            'suite' => $run->suite,
            'level' => $this->runLevel($table, $run->id),
            'passed' => (int) ($agg->passed ?? 0),
            'failed' => (int) ($agg->failed ?? 0),
            'avg_user_tps' => $this->roundOrNull($agg->avg_user_tps),
            'weighted_user_tps' => $this->roundOrNull($agg->weighted_user_tps),
            'gen_tps' => null,
            'total_tokens' => (int) ($agg->total_tokens ?? 0),
            'ln_sum' => (float) ($agg->ln_sum ?? 0),
            'ln_n' => (int) ($agg->ln_n ?? 0),
        ]]);
    }

    /**
     * Aggregate all scenariobench runs in a band into one summary row.
     * Aggregates passed/failed/tokens and generated tok/s across every
     * scenario run that belongs to the band.
     *
     * @param  array<int, object>  $runs
     */
    private function aggregateScenarioBand(array $runs): array
    {
        $runPks = array_map(fn ($r) => $r->id, $runs);
        $first = $runs[0];

        $agg = DB::table('scenario_runs')
            ->whereIn('benchmark_run_id', $runPks)
            ->selectRaw('count(*) filter (where ok) as passed')
            ->selectRaw('count(*) filter (where not ok) as failed')
            ->selectRaw('sum(gen_tokens) as total_tokens')
            ->selectRaw('sum(gen_tokens) / nullif(sum(wall), 0) as gen_tps')
            ->selectRaw('sum(ln(gen_tokens::float8 / nullif(wall, 0))) filter (where wall > 0 and gen_tokens > 0) as ln_sum')
            ->selectRaw('count(*) filter (where wall > 0 and gen_tokens > 0) as ln_n')
            ->first();

        // Thinking level: keep it only when every scenario row in the band
        // agrees — matches the per-run runLevel() rule, applied at band scope.
        $level = DB::table('scenario_runs')
            ->whereIn('benchmark_run_id', $runPks)
            ->distinct()->pluck('level')->filter()->values();

        return [
            'run_pk' => $first->id,
            'run_id' => $first->run_id,
            'suite' => 'scenariobench',
            'level' => $level->count() === 1 ? $level->first() : null,
            'passed' => (int) ($agg->passed ?? 0),
            'failed' => (int) ($agg->failed ?? 0),
            'avg_user_tps' => null,
            'weighted_user_tps' => null,
            'gen_tps' => $this->roundOrNull($agg->gen_tps),
            'total_tokens' => (int) ($agg->total_tokens ?? 0),
            'ln_sum' => (float) ($agg->ln_sum ?? 0),
            'ln_n' => (int) ($agg->ln_n ?? 0),
        ];
    }

    private function emptyAgg(object $run): array
    {
        return [
            'run_pk' => $run->id,
            'run_id' => $run->run_id,
            'suite' => $run->suite,
            'level' => null,
            'passed' => 0,
            'failed' => 0,
            'avg_user_tps' => null,
            'weighted_user_tps' => null,
            'gen_tps' => null,
            'total_tokens' => 0,
            'ln_sum' => 0.0,
            'ln_n' => 0,
        ];
    }

    private function roundOrNull($value, int $precision = 1): ?float
    {
        return $value === null ? null : round((float) $value, $precision);
    }

    /**
     * The run's thinking level: the distinct level across its result rows
     * (a run is single-level; falls back to null when mixed or empty).
     */
    private function runLevel(string $table, int $runPk): ?string
    {
        $level = DB::table($table)->where('benchmark_run_id', $runPk)
            ->distinct()->pluck('level')->filter()->values();

        return $level->count() === 1 ? $level->first() : null;
    }
}