festbench-web/tests/Feature/SessionSummaryTest.php

<?php

use App\Models\BenchmarkRun;
use App\Models\InfrabenchResult;
use App\Models\ScenarioRun;
use App\Models\SoftbenchResult;
use App\Models\User;
use Illuminate\Support\Facades\Artisan;
use Livewire\Volt\Volt;

test('festbench:summary splits runs by label into separate bands', function () {
    $me = User::factory()->create(['name' => 'peterZ']);

    $a = BenchmarkRun::factory()->create([
        'suite' => 'softbench', 'label' => 'session A', 'model_id' => 'm', 'model_quant' => 'Q5_K_M',
        'gpu' => 'AMD R9700', 'gpu_count' => 1, 'engine' => 'llama.cpp', 'user_id' => $me->id, 'visibility' => 'public',
        'ran_at' => '2026-09-03 12:00:00',
    ]);
    SoftbenchResult::factory()->create(['benchmark_run_id' => $a->id, 'ok' => true, 'user_tps' => 60.0, 'total' => 100, 'level' => 'off']);
    SoftbenchResult::factory()->create(['benchmark_run_id' => $a->id, 'ok' => false, 'user_tps' => 20.0, 'total' => 300, 'level' => 'off']);

    $b = BenchmarkRun::factory()->create([
        'suite' => 'infrabench', 'label' => 'session A', 'model_id' => 'm', 'model_quant' => 'Q5_K_M',
        'gpu' => 'AMD R9700', 'gpu_count' => 1, 'engine' => 'llama.cpp', 'user_id' => $me->id, 'visibility' => 'public',
        'ran_at' => '2026-09-03 12:05:00',
    ]);
    InfrabenchResult::factory()->create(['benchmark_run_id' => $b->id, 'ok' => true, 'user_tps' => 40.0, 'total' => 200, 'level' => 'low']);

    $c = BenchmarkRun::factory()->create([
        'suite' => 'softbench', 'label' => 'session B', 'model_id' => 'm', 'model_quant' => 'Q5_K_M',
        'gpu' => 'AMD R9700', 'gpu_count' => 1, 'engine' => 'llama.cpp', 'user_id' => $me->id, 'visibility' => 'public',
        'ran_at' => '2026-09-04 12:00:00',
    ]);
    SoftbenchResult::factory()->create(['benchmark_run_id' => $c->id, 'ok' => true, 'user_tps' => 50.0, 'total' => 50, 'level' => 'medium']);

    Artisan::call('festbench:summary');

    // Different labels → separate bands.
    $bands = DB::table('session_summaries')->orderBy('sequence')->get();
    expect($bands)->toHaveCount(2);

    $bandA = $bands->firstWhere('label', 'session A');
    expect($bandA->runner_name)->toBe('peterZ')
        ->and($bandA->gpu)->toBe('AMD R9700')
        ->and($bandA->gpu_count)->toBe(1)
        ->and($bandA->label)->toBe('session A');

    $rowsA = DB::table('session_suite_rows')->where('session_summary_id', $bandA->id)->orderBy('sequence')->get();
    expect($rowsA)->toHaveCount(2);

    $soft = $rowsA->firstWhere('suite', 'softbench');
    expect($soft->passed)->toBe(1)
        ->and($soft->failed)->toBe(1)
        ->and($soft->avg_user_tps)->toBe(40.0)
        ->and($soft->weighted_user_tps)->toBe(30.0)
        ->and($soft->total_tokens)->toBe(400)
        ->and($soft->level)->toBe('off');

    $infra = $rowsA->firstWhere('suite', 'infrabench');
    expect($infra->passed)->toBe(1)->and($infra->level)->toBe('low');
});

test('devlogic and sysops totals are split by suite category', function () {
    $me = User::factory()->create();

    $soft = BenchmarkRun::factory()->create([
        'suite' => 'softbench', 'label' => 'dual', 'model_id' => 'm', 'model_quant' => 'Q5_K_M',
        'gpu' => 'GPU1', 'gpu_count' => 1, 'engine' => 'llama.cpp', 'user_id' => $me->id, 'visibility' => 'public',
        'ran_at' => '2026-09-03 12:00:00',
    ]);
    SoftbenchResult::factory()->create(['benchmark_run_id' => $soft->id, 'ok' => true, 'user_tps' => 50.0, 'total' => 100, 'level' => 'off']);
    SoftbenchResult::factory()->create(['benchmark_run_id' => $soft->id, 'ok' => false, 'user_tps' => 10.0, 'total' => 200, 'level' => 'off']);

    $infra = BenchmarkRun::factory()->create([
        'suite' => 'infrabench', 'label' => 'dual', 'model_id' => 'm', 'model_quant' => 'Q5_K_M',
        'gpu' => 'GPU1', 'gpu_count' => 1, 'engine' => 'llama.cpp', 'user_id' => $me->id, 'visibility' => 'public',
        'ran_at' => '2026-09-03 12:05:00',
    ]);
    InfrabenchResult::factory()->create(['benchmark_run_id' => $infra->id, 'ok' => true, 'user_tps' => 30.0, 'total' => 150, 'level' => 'low']);

    $scen = BenchmarkRun::factory()->create([
        'suite' => 'scenariobench', 'label' => 'dual', 'model_id' => 'm', 'model_quant' => 'Q5_K_M',
        'gpu' => 'GPU1', 'gpu_count' => 1, 'engine' => 'llama.cpp', 'user_id' => $me->id, 'visibility' => 'public',
        'ran_at' => '2026-09-03 12:10:00',
    ]);
    ScenarioRun::factory()->create(['benchmark_run_id' => $scen->id, 'ok' => true, 'gen_tokens' => 5000, 'wall' => 100.0]);
    ScenarioRun::factory()->create(['benchmark_run_id' => $scen->id, 'ok' => false, 'gen_tokens' => 3000, 'wall' => 200.0]);

    Artisan::call('festbench:summary');

    $band = DB::table('session_summaries')->where('label', 'dual')->first();

    // Devlogic: softbench only — 2 tasks: user_tps=50 (100 tokens),
    // user_tps=10 (200 tokens).
    // Token-weighted: (50×100 + 10×200) / (100+200) = 7000/300 ≈ 23.3
    expect($band->devlogic_passed)->toBe(1)
        ->and($band->devlogic_failed)->toBe(1)
        ->and($band->devlogic_tokens)->toBe(300)
        ->and($band->devlogic_tps)->toBe(23.3);

    // Sysops: infrabench (1 task: user_tps=30, tokens=150 → rate_sum=4500)
    // + scenariobench (gen_tps = 8000/300 = 26.67, tokens=8000 →
    //   rate_sum = 26.67×8000 ≈ 213333.33).
    // Total sysops tokens = 150 + 8000 = 8150.
    // Token-weighted: (4500 + 213333.33) / 8150 ≈ 26.7
    expect($band->sysops_passed)->toBe(2)
        ->and($band->sysops_failed)->toBe(1)
        ->and($band->sysops_tokens)->toBe(8150);
    expect($band->sysops_tps)->not->toBeNull();
    expect($band->sysops_tps)->toBeGreaterThan(20.0)->toBeLessThan(30.0);
});

test('private runs form their own band and are hidden from guests', function () {
    $me = User::factory()->create();
    BenchmarkRun::factory()->create([
        'suite' => 'softbench', 'label' => 'pub', 'model_id' => 'm', 'visibility' => 'public',
        'gpu' => 'g', 'gpu_count' => 1, 'user_id' => $me->id,
    ]);
    SoftbenchResult::factory()->create(['benchmark_run_id' => BenchmarkRun::latest('id')->first()->id]);

    BenchmarkRun::factory()->create([
        'suite' => 'softbench', 'label' => 'pub', 'model_id' => 'm', 'visibility' => 'private',
        'gpu' => 'g', 'gpu_count' => 1, 'user_id' => $me->id,
    ]);

    Artisan::call('festbench:summary');

    expect(DB::table('session_summaries')->count())->toBe(2)
        ->and(DB::table('session_summaries')->where('visibility', 'private')->count())->toBe(1);

    $this->get(route('home'))->assertOk()->assertSee('pub');

    $bands = DB::table('session_summaries')->pluck('visibility');
    expect($bands)->toContain('public')->toContain('private');
});

test('home renders the session summary and links into per-task detail', function () {
    $run = BenchmarkRun::factory()->create([
        'suite' => 'softbench', 'label' => 'rendered session', 'visibility' => 'public',
        'model_id' => 'render/model', 'gpu' => 'AMD R9700', 'gpu_count' => 2,
    ]);
    SoftbenchResult::factory()->create(['benchmark_run_id' => $run->id, 'ok' => true, 'level' => 'off']);

    Artisan::call('festbench:summary');

    $this->get(route('home'))->assertOk()
        ->assertSee('rendered session')
        ->assertSee('render/model')
        ->assertSee('AMD R9700')
        ->assertSee(route('runs.show', $run->run_id))
        ->assertSee('All task rows');
});

test('results page still lists flat task rows', function () {
    $run = BenchmarkRun::factory()->create(['suite' => 'softbench', 'visibility' => 'public']);
    SoftbenchResult::factory()->create(['benchmark_run_id' => $run->id, 'task' => 'flat-task']);

    $this->get(route('results'))->assertOk()
        ->assertSee('All Task Rows')
        ->assertSee('flat-task');
});

test('multi-filter: suite filter returns bands containing any selected suite', function () {
    $me = User::factory()->create();

    $soft = BenchmarkRun::factory()->create([
        'suite' => 'softbench', 'label' => 'soft-only', 'model_id' => 'm', 'model_quant' => 'Q5_K_M',
        'gpu' => 'GPU1', 'gpu_count' => 1, 'engine' => 'llama.cpp', 'user_id' => $me->id, 'visibility' => 'public',
        'ran_at' => '2026-09-03 12:00:00',
    ]);
    SoftbenchResult::factory()->create(['benchmark_run_id' => $soft->id, 'ok' => true, 'user_tps' => 50.0, 'total' => 100, 'level' => 'off']);

    $infra = BenchmarkRun::factory()->create([
        'suite' => 'infrabench', 'label' => 'infra-only', 'model_id' => 'm', 'model_quant' => 'Q5_K_M',
        'gpu' => 'GPU1', 'gpu_count' => 1, 'engine' => 'llama.cpp', 'user_id' => $me->id, 'visibility' => 'public',
        'ran_at' => '2026-09-03 12:05:00',
    ]);
    InfrabenchResult::factory()->create(['benchmark_run_id' => $infra->id, 'ok' => true, 'user_tps' => 30.0, 'total' => 100, 'level' => 'low']);

    Artisan::call('festbench:summary');

    // No filter: both bands visible.
    Volt::test('summary.index')->assertSee('soft-only')->assertSee('infra-only');

    // Filter to infrabench: only the infra band should match.
    Volt::test('summary.index', ['filters' => [
        'gpu' => [], 'model' => [], 'quant' => [], 'engine' => [], 'suite' => ['infrabench'],
    ]])
        ->assertDontSee('soft-only')
        ->assertSee('infra-only');
});

test('multi-filter: AND across categories, OR within a category', function () {
    $me = User::factory()->create();

    // Band A: GPU1 + model-A + Q5_K_M + llama.cpp
    $a = BenchmarkRun::factory()->create([
        'suite' => 'softbench', 'label' => 'band-a', 'model_id' => 'model-A', 'model_quant' => 'Q5_K_M',
        'gpu' => 'GPU1', 'gpu_count' => 1, 'engine' => 'llama.cpp', 'user_id' => $me->id, 'visibility' => 'public',
        'ran_at' => '2026-09-03 12:00:00',
    ]);
    SoftbenchResult::factory()->create(['benchmark_run_id' => $a->id, 'ok' => true, 'user_tps' => 50.0, 'total' => 100, 'level' => 'off']);

    // Band B: GPU2 + model-B + UD-Q4_K_M + ollama-cloud
    $b = BenchmarkRun::factory()->create([
        'suite' => 'softbench', 'label' => 'band-b', 'model_id' => 'model-B', 'model_quant' => 'UD-Q4_K_M',
        'gpu' => 'GPU2', 'gpu_count' => 1, 'engine' => 'ollama-cloud', 'user_id' => $me->id, 'visibility' => 'public',
        'ran_at' => '2026-09-04 12:00:00',
    ]);
    SoftbenchResult::factory()->create(['benchmark_run_id' => $b->id, 'ok' => true, 'user_tps' => 40.0, 'total' => 100, 'level' => 'off']);

    Artisan::call('festbench:summary');

    // GPU filter alone: OR within GPU → both bands.
    Volt::test('summary.index', ['filters' => [
        'gpu' => ['GPU1', 'GPU2'], 'model' => [], 'quant' => [], 'engine' => [], 'suite' => [],
    ]])
        ->assertSee('band-a')
        ->assertSee('band-b');

    // GPU + model: AND across → only matching band.
    Volt::test('summary.index', ['filters' => [
        'gpu' => ['GPU1', 'GPU2'], 'model' => ['model-A'], 'quant' => [], 'engine' => [], 'suite' => [],
    ]])
        ->assertSee('band-a')
        ->assertDontSee('band-b');

    // Engine filter: only llama.cpp band.
    Volt::test('summary.index', ['filters' => [
        'gpu' => [], 'model' => [], 'quant' => [], 'engine' => ['llama.cpp'], 'suite' => [],
    ]])
        ->assertSee('band-a')
        ->assertDontSee('band-b');

    // Quant filter: only UD-Q4_K_M band.
    Volt::test('summary.index', ['filters' => [
        'gpu' => [], 'model' => [], 'quant' => ['UD-Q4_K_M'], 'engine' => [], 'suite' => [],
    ]])
        ->assertDontSee('band-a')
        ->assertSee('band-b');
});

test('multi-filter: multiple suites use OR for band inclusion', function () {
    $me = User::factory()->create();

    // Band with only softbench.
    $soft = BenchmarkRun::factory()->create([
        'suite' => 'softbench', 'label' => 'has-soft', 'model_id' => 'm', 'model_quant' => 'Q5_K_M',
        'gpu' => 'GPU1', 'gpu_count' => 1, 'engine' => 'llama.cpp', 'user_id' => $me->id, 'visibility' => 'public',
        'ran_at' => '2026-09-03 12:00:00',
    ]);
    SoftbenchResult::factory()->create(['benchmark_run_id' => $soft->id, 'ok' => true, 'user_tps' => 50.0, 'total' => 100, 'level' => 'off']);

    // Band with only infrabench.
    $infra = BenchmarkRun::factory()->create([
        'suite' => 'infrabench', 'label' => 'has-infra', 'model_id' => 'm', 'model_quant' => 'Q5_K_M',
        'gpu' => 'GPU1', 'gpu_count' => 1, 'engine' => 'llama.cpp', 'user_id' => $me->id, 'visibility' => 'public',
        'ran_at' => '2026-09-03 12:05:00',
    ]);
    InfrabenchResult::factory()->create(['benchmark_run_id' => $infra->id, 'ok' => true, 'user_tps' => 30.0, 'total' => 100, 'level' => 'low']);

    Artisan::call('festbench:summary');

    // Both suites selected → both bands visible (OR within suite).
    Volt::test('summary.index', ['filters' => [
        'gpu' => [], 'model' => [], 'quant' => [], 'engine' => [], 'suite' => ['softbench', 'infrabench'],
    ]])
        ->assertSee('has-soft')
        ->assertSee('has-infra');
});

test('toggleFilter adds and removes a value from a filter array', function () {
    $component = Volt::test('summary.index');

    $component->call('toggleFilter', 'suite', 'infrabench')
        ->assertSet('filters.suite', ['infrabench']);

    $component->call('toggleFilter', 'suite', 'infrabench')
        ->assertSet('filters.suite', []);
});

test('suite filter hides non-selected suite rows and category totals within matching bands', function () {
    $me = User::factory()->create();

    // One band containing both softbench (devlogic) and infrabench (sysops).
    $soft = BenchmarkRun::factory()->create([
        'run_id' => 'RUN-SOFT-001',
        'suite' => 'softbench', 'label' => 'mixed-band', 'model_id' => 'm', 'model_quant' => 'Q5_K_M',
        'gpu' => 'GPU1', 'gpu_count' => 1, 'engine' => 'llama.cpp', 'user_id' => $me->id, 'visibility' => 'public',
        'ran_at' => '2026-09-03 12:00:00',
    ]);
    SoftbenchResult::factory()->create(['benchmark_run_id' => $soft->id, 'ok' => true, 'user_tps' => 50.0, 'total' => 100, 'level' => 'off']);

    $infra = BenchmarkRun::factory()->create([
        'run_id' => 'RUN-INFRA-001',
        'suite' => 'infrabench', 'label' => 'mixed-band', 'model_id' => 'm', 'model_quant' => 'Q5_K_M',
        'gpu' => 'GPU1', 'gpu_count' => 1, 'engine' => 'llama.cpp', 'user_id' => $me->id, 'visibility' => 'public',
        'ran_at' => '2026-09-03 12:05:00',
    ]);
    InfrabenchResult::factory()->create(['benchmark_run_id' => $infra->id, 'ok' => true, 'user_tps' => 30.0, 'total' => 100, 'level' => 'low']);

    Artisan::call('festbench:summary');

    // No suite filter: both suite rows and both category totals visible.
    Volt::test('summary.index')
        ->assertSee('mixed-band')
        ->assertSee('RUN-SOFT-001')
        ->assertSee('RUN-INFRA-001')
        ->assertSee('Devlogic total')
        ->assertSee('Sysops total');

    // Filter to infrabench: band still shows, but only the infrabench row
    // is rendered and both category total strips are hidden.
    Volt::test('summary.index', ['filters' => [
        'gpu' => [], 'model' => [], 'quant' => [], 'engine' => [], 'suite' => ['infrabench'],
    ]])
        ->assertSee('mixed-band')
        ->assertSee('RUN-INFRA-001')
        ->assertDontSee('RUN-SOFT-001')
        ->assertDontSee('Devlogic total')
        ->assertDontSee('Sysops total');
});

test('clearing the suite filter restores all suite rows and category totals', function () {
    $me = User::factory()->create();

    $soft = BenchmarkRun::factory()->create([
        'run_id' => 'RUN-SOFT-002',
        'suite' => 'softbench', 'label' => 'mixed-band', 'model_id' => 'm', 'model_quant' => 'Q5_K_M',
        'gpu' => 'GPU1', 'gpu_count' => 1, 'engine' => 'llama.cpp', 'user_id' => $me->id, 'visibility' => 'public',
        'ran_at' => '2026-09-03 12:00:00',
    ]);
    SoftbenchResult::factory()->create(['benchmark_run_id' => $soft->id, 'ok' => true, 'user_tps' => 50.0, 'total' => 100, 'level' => 'off']);

    $infra = BenchmarkRun::factory()->create([
        'run_id' => 'RUN-INFRA-002',
        'suite' => 'infrabench', 'label' => 'mixed-band', 'model_id' => 'm', 'model_quant' => 'Q5_K_M',
        'gpu' => 'GPU1', 'gpu_count' => 1, 'engine' => 'llama.cpp', 'user_id' => $me->id, 'visibility' => 'public',
        'ran_at' => '2026-09-03 12:05:00',
    ]);
    InfrabenchResult::factory()->create(['benchmark_run_id' => $infra->id, 'ok' => true, 'user_tps' => 30.0, 'total' => 100, 'level' => 'low']);

    Artisan::call('festbench:summary');

    $component = Volt::test('summary.index', ['filters' => [
        'gpu' => [], 'model' => [], 'quant' => [], 'engine' => [], 'suite' => ['infrabench'],
    ]])
        ->assertSee('RUN-INFRA-002')
        ->assertDontSee('RUN-SOFT-002');

    $component->call('resetFilters')
        ->assertSee('RUN-SOFT-002')
        ->assertSee('RUN-INFRA-002')
        ->assertSee('Devlogic total')
        ->assertSee('Sysops total');
});