Compare commits
17 commits
10d4e0aef8
...
700bdae959
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
700bdae959 | ||
|
|
895a764b88 | ||
|
|
75337a80a7 | ||
|
|
179a678b71 | ||
|
|
ac941fdd27 | ||
|
|
9b9150dd6b | ||
|
|
7e57fb5e16 | ||
|
|
9bf17dba6f | ||
|
|
f96fd1ad77 | ||
|
|
6c19822235 | ||
|
|
8c9e512fc2 | ||
|
|
f334538866 | ||
|
|
4bfa1c75ea | ||
|
|
dc2e78bff0 | ||
|
|
163a16a386 | ||
|
|
8ab06133a5 | ||
|
|
18ffb713c9 |
9 changed files with 617 additions and 143 deletions
|
|
@ -51,8 +51,8 @@ private function addGpu(): int
|
||||||
|
|
||||||
$vendor = $this->option('vendor') ?? $this->guessVendor($name);
|
$vendor = $this->option('vendor') ?? $this->guessVendor($name);
|
||||||
|
|
||||||
if (! in_array($vendor, ['amd', 'nvidia', 'intel', 'other'], true)) {
|
if (! in_array($vendor, ['amd', 'nvidia', 'intel', 'cloud', 'other'], true)) {
|
||||||
$this->error('--vendor must be amd, nvidia, intel or other.');
|
$this->error('--vendor must be amd, nvidia, intel, cloud or other.');
|
||||||
|
|
||||||
return self::FAILURE;
|
return self::FAILURE;
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -28,10 +28,15 @@ public function handle(): int
|
||||||
|
|
||||||
foreach ($runs as $run) {
|
foreach ($runs as $run) {
|
||||||
// Grouping key: user + GPU + model + quant + visibility + engine
|
// Grouping key: user + GPU + model + quant + visibility + engine
|
||||||
// + harness identity + named server config. Free-text labels are
|
// + harness identity + named server config + label. The label is
|
||||||
// intentionally NOT part of identity: rerunning psychbench with
|
// the run-wide experiment/config variant (e.g. a power-cap
|
||||||
// the same actual settings belongs in the same bubble even when
|
// setting), so different labels are different sessions even when
|
||||||
// the user wrote a different run note.
|
// every other dimension matches. Normalize null/empty to a
|
||||||
|
// single sentinel so unlabeled runs still group together.
|
||||||
|
$normalizedLabel = $run->label !== null && $run->label !== ''
|
||||||
|
? $run->label
|
||||||
|
: 'unlabeled';
|
||||||
|
|
||||||
$key = implode('|', [
|
$key = implode('|', [
|
||||||
$run->model_id ?? '-',
|
$run->model_id ?? '-',
|
||||||
$run->model_quant ?? '-',
|
$run->model_quant ?? '-',
|
||||||
|
|
@ -48,11 +53,12 @@ public function handle(): int
|
||||||
$run->harness_ver ?: '-',
|
$run->harness_ver ?: '-',
|
||||||
$run->harness_mode ?: 'one-shot',
|
$run->harness_mode ?: 'one-shot',
|
||||||
$run->config_id ?? 'none',
|
$run->config_id ?? 'none',
|
||||||
|
$normalizedLabel,
|
||||||
]);
|
]);
|
||||||
|
|
||||||
$sessions[$key] ??= [
|
$sessions[$key] ??= [
|
||||||
'key' => $key,
|
'key' => $key,
|
||||||
'labels' => collect([$run->label])->filter(),
|
'label' => $run->label,
|
||||||
'model_id' => $run->model_id,
|
'model_id' => $run->model_id,
|
||||||
'model_quant' => $run->model_quant,
|
'model_quant' => $run->model_quant,
|
||||||
'gpu' => $run->gpu,
|
'gpu' => $run->gpu,
|
||||||
|
|
@ -63,16 +69,23 @@ public function handle(): int
|
||||||
'engine' => $run->engine,
|
'engine' => $run->engine,
|
||||||
'config_name' => $run->config_name,
|
'config_name' => $run->config_name,
|
||||||
'ran_at' => $run->ran_at,
|
'ran_at' => $run->ran_at,
|
||||||
'passed' => 0,
|
// Devlogic totals: softbench, prosebench, psychbench,
|
||||||
'failed' => 0,
|
// depthsweep — everything except infrabench and scenariobench.
|
||||||
'tokens' => 0,
|
// Token-weighted: sum(user_tps × tokens) / sum(tokens).
|
||||||
|
'dl_passed' => 0,
|
||||||
|
'dl_failed' => 0,
|
||||||
|
'dl_tokens' => 0,
|
||||||
|
'dl_rate_sum' => 0.0,
|
||||||
|
// Sysops totals: infrabench and scenariobench.
|
||||||
|
// Token-weighted at the suite level: each suite's aggregate
|
||||||
|
// rate is weighted by that suite's total tokens.
|
||||||
|
'so_passed' => 0,
|
||||||
|
'so_failed' => 0,
|
||||||
|
'so_tokens' => 0,
|
||||||
|
'so_rate_sum' => 0.0,
|
||||||
'runs' => collect(),
|
'runs' => collect(),
|
||||||
];
|
];
|
||||||
|
|
||||||
if ($run->label !== null && $run->label !== '') {
|
|
||||||
$sessions[$key]['labels']->push($run->label);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Scenariobench is aggregated at band level (one row for all
|
// Scenariobench is aggregated at band level (one row for all
|
||||||
// scenario runs in the band) rather than per-run, so collect the
|
// scenario runs in the band) rather than per-run, so collect the
|
||||||
// run rows and defer to a second pass below.
|
// run rows and defer to a second pass below.
|
||||||
|
|
@ -84,23 +97,36 @@ public function handle(): int
|
||||||
|
|
||||||
$aggs = $this->aggregateRun($run);
|
$aggs = $this->aggregateRun($run);
|
||||||
|
|
||||||
$sessions[$key]['passed'] += $aggs->sum('passed');
|
// Route the aggregation to the correct total bucket.
|
||||||
$sessions[$key]['failed'] += $aggs->sum('failed');
|
$isSysops = $run->suite === 'infrabench';
|
||||||
$sessions[$key]['tokens'] += $aggs->sum('total_tokens');
|
|
||||||
$sessions[$key]['runs'] = $sessions[$key]['runs']->merge($aggs);
|
$sessions[$key]['runs'] = $sessions[$key]['runs']->merge($aggs);
|
||||||
|
|
||||||
|
if ($isSysops) {
|
||||||
|
$sessions[$key]['so_passed'] += $aggs->sum('passed');
|
||||||
|
$sessions[$key]['so_failed'] += $aggs->sum('failed');
|
||||||
|
$sessions[$key]['so_tokens'] += $aggs->sum('total_tokens');
|
||||||
|
$sessions[$key]['so_rate_sum'] += $aggs->sum('rate_token_sum');
|
||||||
|
} else {
|
||||||
|
$sessions[$key]['dl_passed'] += $aggs->sum('passed');
|
||||||
|
$sessions[$key]['dl_failed'] += $aggs->sum('failed');
|
||||||
|
$sessions[$key]['dl_tokens'] += $aggs->sum('total_tokens');
|
||||||
|
$sessions[$key]['dl_rate_sum'] += $aggs->sum('rate_token_sum');
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Second pass: emit one aggregated scenariobench row per band,
|
// Second pass: emit one aggregated scenariobench row per band,
|
||||||
// covering all scenario runs in that band.
|
// covering all scenario runs in that band. Scenariobench is sysops.
|
||||||
foreach ($sessions as $key => $band) {
|
foreach ($sessions as $key => $band) {
|
||||||
$scenarioRuns = $band['scenario_runs'] ?? [];
|
$scenarioRuns = $band['scenario_runs'] ?? [];
|
||||||
if (empty($scenarioRuns)) {
|
if (empty($scenarioRuns)) {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
$agg = $this->aggregateScenarioBand($scenarioRuns);
|
$agg = $this->aggregateScenarioBand($scenarioRuns);
|
||||||
$sessions[$key]['passed'] += $agg['passed'];
|
$sessions[$key]['so_passed'] += $agg['passed'];
|
||||||
$sessions[$key]['failed'] += $agg['failed'];
|
$sessions[$key]['so_failed'] += $agg['failed'];
|
||||||
$sessions[$key]['tokens'] += $agg['total_tokens'];
|
$sessions[$key]['so_tokens'] += $agg['total_tokens'];
|
||||||
|
$sessions[$key]['so_rate_sum'] += $agg['rate_token_sum'] ?? 0;
|
||||||
$sessions[$key]['runs'] = $band['runs']->merge([$agg]);
|
$sessions[$key]['runs'] = $band['runs']->merge([$agg]);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -110,11 +136,7 @@ public function handle(): int
|
||||||
foreach (array_values($sessions) as $bandIndex => $band) {
|
foreach (array_values($sessions) as $bandIndex => $band) {
|
||||||
$summaryId = DB::table('session_summaries')->insertGetId([
|
$summaryId = DB::table('session_summaries')->insertGetId([
|
||||||
'session_key' => $band['key'],
|
'session_key' => $band['key'],
|
||||||
// Keep a label only when every grouped run agrees. A
|
'label' => $band['label'],
|
||||||
// mixed label is a note, not a configuration split.
|
|
||||||
'label' => $band['labels']->unique()->count() === 1
|
|
||||||
? $band['labels']->first()
|
|
||||||
: null,
|
|
||||||
'model_id' => $band['model_id'],
|
'model_id' => $band['model_id'],
|
||||||
'model_quant' => $band['model_quant'],
|
'model_quant' => $band['model_quant'],
|
||||||
'gpu' => $band['gpu'],
|
'gpu' => $band['gpu'],
|
||||||
|
|
@ -125,9 +147,18 @@ public function handle(): int
|
||||||
'engine' => $band['engine'],
|
'engine' => $band['engine'],
|
||||||
'config_name' => $band['config_name'],
|
'config_name' => $band['config_name'],
|
||||||
'ran_at' => $band['ran_at'],
|
'ran_at' => $band['ran_at'],
|
||||||
'total_passed' => $band['passed'],
|
'devlogic_passed' => $band['dl_passed'],
|
||||||
'total_failed' => $band['failed'],
|
'devlogic_failed' => $band['dl_failed'],
|
||||||
'total_tokens' => $band['tokens'],
|
'devlogic_tokens' => $band['dl_tokens'],
|
||||||
|
'devlogic_tps' => $band['dl_tokens'] > 0
|
||||||
|
? round($band['dl_rate_sum'] / $band['dl_tokens'], 1)
|
||||||
|
: null,
|
||||||
|
'sysops_passed' => $band['so_passed'],
|
||||||
|
'sysops_failed' => $band['so_failed'],
|
||||||
|
'sysops_tokens' => $band['so_tokens'],
|
||||||
|
'sysops_tps' => $band['so_tokens'] > 0
|
||||||
|
? round($band['so_rate_sum'] / $band['so_tokens'], 1)
|
||||||
|
: null,
|
||||||
'sequence' => $bandIndex,
|
'sequence' => $bandIndex,
|
||||||
'created_at' => now(),
|
'created_at' => now(),
|
||||||
'updated_at' => now(),
|
'updated_at' => now(),
|
||||||
|
|
@ -193,6 +224,7 @@ private function aggregateRun(object $run)
|
||||||
DB::raw('avg(nullif(wall, 0)) as avg_wall'),
|
DB::raw('avg(nullif(wall, 0)) as avg_wall'),
|
||||||
DB::raw('sum(total) as total_tokens'),
|
DB::raw('sum(total) as total_tokens'),
|
||||||
DB::raw('sum(user_tps * total) / nullif(sum(total), 0) as weighted_user_tps'),
|
DB::raw('sum(user_tps * total) / nullif(sum(total), 0) as weighted_user_tps'),
|
||||||
|
DB::raw('sum(user_tps * total) as rate_token_sum'),
|
||||||
]))
|
]))
|
||||||
->sortBy(fn ($agg) => $levelOrder[$agg->level] ?? 99)
|
->sortBy(fn ($agg) => $levelOrder[$agg->level] ?? 99)
|
||||||
->values()
|
->values()
|
||||||
|
|
@ -203,13 +235,13 @@ private function aggregateRun(object $run)
|
||||||
'level' => $agg->level,
|
'level' => $agg->level,
|
||||||
'passed' => (int) $agg->passed,
|
'passed' => (int) $agg->passed,
|
||||||
'failed' => (int) $agg->failed,
|
'failed' => (int) $agg->failed,
|
||||||
// depthsweep has no decode metrics: speed stays blank
|
|
||||||
'avg_user_tps' => null,
|
'avg_user_tps' => null,
|
||||||
'weighted_user_tps' => $this->roundOrNull($agg->weighted_user_tps),
|
'weighted_user_tps' => $this->roundOrNull($agg->weighted_user_tps),
|
||||||
'gen_tps' => null,
|
'gen_tps' => null,
|
||||||
'total_tokens' => (int) ($agg->total_tokens ?? 0),
|
'total_tokens' => (int) ($agg->total_tokens ?? 0),
|
||||||
'avg_ttft' => $this->roundOrNull($agg->avg_ttft),
|
'avg_ttft' => $this->roundOrNull($agg->avg_ttft),
|
||||||
'avg_wall' => $this->roundOrNull($agg->avg_wall),
|
'avg_wall' => $this->roundOrNull($agg->avg_wall),
|
||||||
|
'rate_token_sum' => (float) ($agg->rate_token_sum ?? 0),
|
||||||
]);
|
]);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -221,6 +253,9 @@ private function aggregateRun(object $run)
|
||||||
->selectRaw('sum(gen_tokens) / nullif(sum(wall), 0) as gen_tps')
|
->selectRaw('sum(gen_tokens) / nullif(sum(wall), 0) as gen_tps')
|
||||||
->first();
|
->first();
|
||||||
|
|
||||||
|
$genTps = (float) ($agg->gen_tps ?? 0);
|
||||||
|
$tokens = (int) ($agg->total_tokens ?? 0);
|
||||||
|
|
||||||
return collect([[
|
return collect([[
|
||||||
'run_pk' => $run->id,
|
'run_pk' => $run->id,
|
||||||
'run_id' => $run->run_id,
|
'run_id' => $run->run_id,
|
||||||
|
|
@ -231,7 +266,8 @@ private function aggregateRun(object $run)
|
||||||
'avg_user_tps' => null,
|
'avg_user_tps' => null,
|
||||||
'weighted_user_tps' => null,
|
'weighted_user_tps' => null,
|
||||||
'gen_tps' => $this->roundOrNull($agg->gen_tps),
|
'gen_tps' => $this->roundOrNull($agg->gen_tps),
|
||||||
'total_tokens' => (int) ($agg->total_tokens ?? 0),
|
'total_tokens' => $tokens,
|
||||||
|
'rate_token_sum' => $genTps * $tokens,
|
||||||
]]);
|
]]);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -241,6 +277,7 @@ private function aggregateRun(object $run)
|
||||||
->selectRaw('avg(user_tps) as avg_user_tps')
|
->selectRaw('avg(user_tps) as avg_user_tps')
|
||||||
->selectRaw('sum(user_tps * total) / nullif(sum(total), 0) as weighted_user_tps')
|
->selectRaw('sum(user_tps * total) / nullif(sum(total), 0) as weighted_user_tps')
|
||||||
->selectRaw('sum(total) as total_tokens')
|
->selectRaw('sum(total) as total_tokens')
|
||||||
|
->selectRaw('sum(user_tps * total) as rate_token_sum')
|
||||||
->first();
|
->first();
|
||||||
|
|
||||||
return collect([[
|
return collect([[
|
||||||
|
|
@ -254,6 +291,7 @@ private function aggregateRun(object $run)
|
||||||
'weighted_user_tps' => $this->roundOrNull($agg->weighted_user_tps),
|
'weighted_user_tps' => $this->roundOrNull($agg->weighted_user_tps),
|
||||||
'gen_tps' => null,
|
'gen_tps' => null,
|
||||||
'total_tokens' => (int) ($agg->total_tokens ?? 0),
|
'total_tokens' => (int) ($agg->total_tokens ?? 0),
|
||||||
|
'rate_token_sum' => (float) ($agg->rate_token_sum ?? 0),
|
||||||
]]);
|
]]);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -277,17 +315,27 @@ private function aggregateScenarioBand(array $runs): array
|
||||||
->selectRaw('sum(gen_tokens) / nullif(sum(wall), 0) as gen_tps')
|
->selectRaw('sum(gen_tokens) / nullif(sum(wall), 0) as gen_tps')
|
||||||
->first();
|
->first();
|
||||||
|
|
||||||
|
$genTps = (float) ($agg->gen_tps ?? 0);
|
||||||
|
$tokens = (int) ($agg->total_tokens ?? 0);
|
||||||
|
|
||||||
|
// Thinking level: keep it only when every scenario row in the band
|
||||||
|
// agrees — matches the per-run runLevel() rule, applied at band scope.
|
||||||
|
$level = DB::table('scenario_runs')
|
||||||
|
->whereIn('benchmark_run_id', $runPks)
|
||||||
|
->distinct()->pluck('level')->filter()->values();
|
||||||
|
|
||||||
return [
|
return [
|
||||||
'run_pk' => $first->id,
|
'run_pk' => $first->id,
|
||||||
'run_id' => $first->run_id,
|
'run_id' => $first->run_id,
|
||||||
'suite' => 'scenariobench',
|
'suite' => 'scenariobench',
|
||||||
'level' => null,
|
'level' => $level->count() === 1 ? $level->first() : null,
|
||||||
'passed' => (int) ($agg->passed ?? 0),
|
'passed' => (int) ($agg->passed ?? 0),
|
||||||
'failed' => (int) ($agg->failed ?? 0),
|
'failed' => (int) ($agg->failed ?? 0),
|
||||||
'avg_user_tps' => null,
|
'avg_user_tps' => null,
|
||||||
'weighted_user_tps' => null,
|
'weighted_user_tps' => null,
|
||||||
'gen_tps' => $this->roundOrNull($agg->gen_tps),
|
'gen_tps' => $this->roundOrNull($agg->gen_tps),
|
||||||
'total_tokens' => (int) ($agg->total_tokens ?? 0),
|
'total_tokens' => $tokens,
|
||||||
|
'rate_token_sum' => $genTps * $tokens,
|
||||||
];
|
];
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -304,6 +352,7 @@ private function emptyAgg(object $run): array
|
||||||
'weighted_user_tps' => null,
|
'weighted_user_tps' => null,
|
||||||
'gen_tps' => null,
|
'gen_tps' => null,
|
||||||
'total_tokens' => 0,
|
'total_tokens' => 0,
|
||||||
|
'rate_token_sum' => 0.0,
|
||||||
];
|
];
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,23 @@
|
||||||
|
<?php
|
||||||
|
|
||||||
|
use Illuminate\Database\Migrations\Migration;
|
||||||
|
use Illuminate\Database\Schema\Blueprint;
|
||||||
|
use Illuminate\Support\Facades\Schema;
|
||||||
|
|
||||||
|
return new class extends Migration
|
||||||
|
{
|
||||||
|
public function up(): void
|
||||||
|
{
|
||||||
|
Schema::table('session_summaries', function (Blueprint $table) {
|
||||||
|
$table->decimal('total_tps', 10, 1)->nullable()->after('total_tokens')
|
||||||
|
->comment('Geometric mean of per-task tok/s across all observations in the band (exp(avg(ln(tps))))');
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
public function down(): void
|
||||||
|
{
|
||||||
|
Schema::table('session_summaries', function (Blueprint $table) {
|
||||||
|
$table->dropColumn('total_tps');
|
||||||
|
});
|
||||||
|
}
|
||||||
|
};
|
||||||
46
database/migrations/2026_10_04_000001_split_band_totals.php
Normal file
46
database/migrations/2026_10_04_000001_split_band_totals.php
Normal file
|
|
@ -0,0 +1,46 @@
|
||||||
|
<?php
|
||||||
|
|
||||||
|
use Illuminate\Database\Migrations\Migration;
|
||||||
|
use Illuminate\Database\Schema\Blueprint;
|
||||||
|
use Illuminate\Support\Facades\Schema;
|
||||||
|
|
||||||
|
return new class extends Migration
|
||||||
|
{
|
||||||
|
public function up(): void
|
||||||
|
{
|
||||||
|
Schema::table('session_summaries', function (Blueprint $table) {
|
||||||
|
$table->decimal('devlogic_tps', 10, 1)->nullable()
|
||||||
|
->comment('Geo-mean tok/s across devlogic suites (softbench, prosebench, psychbench, depthsweep, …)');
|
||||||
|
$table->unsignedInteger('devlogic_tokens')->default(0);
|
||||||
|
$table->unsignedInteger('devlogic_passed')->default(0);
|
||||||
|
$table->unsignedInteger('devlogic_failed')->default(0);
|
||||||
|
|
||||||
|
$table->decimal('sysops_tps', 10, 1)->nullable()
|
||||||
|
->comment('Geo-mean tok/s across sysops suites (infrabench, scenariobench)');
|
||||||
|
$table->unsignedInteger('sysops_tokens')->default(0);
|
||||||
|
$table->unsignedInteger('sysops_passed')->default(0);
|
||||||
|
$table->unsignedInteger('sysops_failed')->default(0);
|
||||||
|
});
|
||||||
|
|
||||||
|
Schema::table('session_summaries', function (Blueprint $table) {
|
||||||
|
$table->dropColumn(['total_tps', 'total_tokens', 'total_passed', 'total_failed']);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
public function down(): void
|
||||||
|
{
|
||||||
|
Schema::table('session_summaries', function (Blueprint $table) {
|
||||||
|
$table->decimal('total_tps', 10, 1)->nullable()->after('ran_at');
|
||||||
|
$table->unsignedInteger('total_tokens')->default(0);
|
||||||
|
$table->unsignedInteger('total_passed')->default(0);
|
||||||
|
$table->unsignedInteger('total_failed')->default(0);
|
||||||
|
});
|
||||||
|
|
||||||
|
Schema::table('session_summaries', function (Blueprint $table) {
|
||||||
|
$table->dropColumn([
|
||||||
|
'devlogic_tps', 'devlogic_tokens', 'devlogic_passed', 'devlogic_failed',
|
||||||
|
'sysops_tps', 'sysops_tokens', 'sysops_passed', 'sysops_failed',
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
@ -25,6 +25,19 @@ public function run(): void
|
||||||
['name' => 'NVIDIA H100', 'vendor' => 'nvidia'],
|
['name' => 'NVIDIA H100', 'vendor' => 'nvidia'],
|
||||||
['name' => 'NVIDIA A100', 'vendor' => 'nvidia'],
|
['name' => 'NVIDIA A100', 'vendor' => 'nvidia'],
|
||||||
['name' => 'Intel Arc A770', 'vendor' => 'intel'],
|
['name' => 'Intel Arc A770', 'vendor' => 'intel'],
|
||||||
|
['name' => 'Sapphire AMD Radeon AI PRO R9700', 'vendor' => 'amd'],
|
||||||
|
['name' => 'ASUS AMD Radeon AI PRO R9700', 'vendor' => 'amd'],
|
||||||
|
['name' => 'Ollama Cloud', 'vendor' => 'cloud'],
|
||||||
|
['name' => 'OpenRouter', 'vendor' => 'cloud'],
|
||||||
|
['name' => 'Anthropic', 'vendor' => 'cloud'],
|
||||||
|
['name' => 'OpenAI', 'vendor' => 'cloud'],
|
||||||
|
['name' => 'Google Gemini', 'vendor' => 'cloud'],
|
||||||
|
['name' => 'Groq', 'vendor' => 'cloud'],
|
||||||
|
['name' => 'Mistral', 'vendor' => 'cloud'],
|
||||||
|
['name' => 'Cerebras', 'vendor' => 'cloud'],
|
||||||
|
['name' => 'DeepSeek', 'vendor' => 'cloud'],
|
||||||
|
['name' => 'Together AI', 'vendor' => 'cloud'],
|
||||||
|
['name' => 'Fireworks AI', 'vendor' => 'cloud'],
|
||||||
];
|
];
|
||||||
|
|
||||||
foreach ($gpus as $gpu) {
|
foreach ($gpus as $gpu) {
|
||||||
|
|
|
||||||
|
|
@ -166,25 +166,29 @@ class="w-64"
|
||||||
clearable
|
clearable
|
||||||
/>
|
/>
|
||||||
|
|
||||||
<flux:select wire:model.live="filters.suite" :label="__('Suite')" placeholder="{{ __('All suites') }}" size="sm" class="w-44">
|
<flux:select wire:model.live="filters.suite" :label="__('Suite')" size="sm" class="w-44">
|
||||||
|
<flux:select.option value="">{{ __('All suites') }}</flux:select.option>
|
||||||
@foreach ($this->suites as $suite)
|
@foreach ($this->suites as $suite)
|
||||||
<flux:select.option value="{{ $suite }}">{{ $suite }}</flux:select.option>
|
<flux:select.option value="{{ $suite }}">{{ $suite }}</flux:select.option>
|
||||||
@endforeach
|
@endforeach
|
||||||
</flux:select>
|
</flux:select>
|
||||||
|
|
||||||
<flux:select wire:model.live="filters.level" :label="__('Level')" placeholder="{{ __('All levels') }}" size="sm" class="w-36">
|
<flux:select wire:model.live="filters.level" :label="__('Level')" size="sm" class="w-36">
|
||||||
|
<flux:select.option value="">{{ __('All levels') }}</flux:select.option>
|
||||||
@foreach (['off', 'low', 'medium', 'xhigh'] as $level)
|
@foreach (['off', 'low', 'medium', 'xhigh'] as $level)
|
||||||
<flux:select.option value="{{ $level }}">{{ $level }}</flux:select.option>
|
<flux:select.option value="{{ $level }}">{{ $level }}</flux:select.option>
|
||||||
@endforeach
|
@endforeach
|
||||||
</flux:select>
|
</flux:select>
|
||||||
|
|
||||||
<flux:select wire:model.live="filters.model" :label="__('Model')" placeholder="{{ __('All models') }}" size="sm" class="w-56">
|
<flux:select wire:model.live="filters.model" :label="__('Model')" size="sm" class="w-56">
|
||||||
|
<flux:select.option value="">{{ __('All models') }}</flux:select.option>
|
||||||
@foreach ($this->models as $model)
|
@foreach ($this->models as $model)
|
||||||
<flux:select.option value="{{ $model }}">{{ $model }}</flux:select.option>
|
<flux:select.option value="{{ $model }}">{{ $model }}</flux:select.option>
|
||||||
@endforeach
|
@endforeach
|
||||||
</flux:select>
|
</flux:select>
|
||||||
|
|
||||||
<flux:select wire:model.live="filters.result" :label="__('Result')" placeholder="{{ __('Any') }}" size="sm" class="w-32">
|
<flux:select wire:model.live="filters.result" :label="__('Result')" size="sm" class="w-32">
|
||||||
|
<flux:select.option value="">{{ __('Any') }}</flux:select.option>
|
||||||
<flux:select.option value="1">{{ __('Pass') }}</flux:select.option>
|
<flux:select.option value="1">{{ __('Pass') }}</flux:select.option>
|
||||||
<flux:select.option value="0">{{ __('Fail') }}</flux:select.option>
|
<flux:select.option value="0">{{ __('Fail') }}</flux:select.option>
|
||||||
</flux:select>
|
</flux:select>
|
||||||
|
|
|
||||||
|
|
@ -7,13 +7,14 @@
|
||||||
use Livewire\Attributes\Layout;
|
use Livewire\Attributes\Layout;
|
||||||
use Livewire\Component;
|
use Livewire\Component;
|
||||||
|
|
||||||
new #[Layout('components.layouts.app')] class extends Component {
|
new #[Layout('components.layouts.app')] class extends Component
|
||||||
|
{
|
||||||
use IncludesSorting;
|
use IncludesSorting;
|
||||||
|
|
||||||
private const SORTABLE = [
|
private const SORTABLE = [
|
||||||
'scenario', 'level', 'ok', 'outcome',
|
'scenario', 'level', 'ok', 'outcome',
|
||||||
'wall', 'turns', 'commands', 'submissions', 'rejected',
|
'wall', 'turns', 'commands', 'submissions', 'rejected',
|
||||||
't_first_sub', 't_pass', 'gen_tokens',
|
't_first_sub', 't_pass', 'gen_tokens', 'gen_tps',
|
||||||
];
|
];
|
||||||
|
|
||||||
public ?int $sessionId = null;
|
public ?int $sessionId = null;
|
||||||
|
|
@ -55,7 +56,16 @@ public function runs()
|
||||||
->where('engine', $band->engine)
|
->where('engine', $band->engine)
|
||||||
->where(function ($q) use ($band) {
|
->where(function ($q) use ($band) {
|
||||||
$q->whereNull('config_id')
|
$q->whereNull('config_id')
|
||||||
->orWhere('config_id', $this->configIdFromName($band->config_name));
|
->orWhere('config_id', $this->configIdFromName($band->config_name));
|
||||||
|
})
|
||||||
|
->where(function ($q) use ($band) {
|
||||||
|
if ($band->label !== null && $band->label !== '') {
|
||||||
|
$q->where('label', $band->label);
|
||||||
|
} else {
|
||||||
|
$q->where(function ($inner) {
|
||||||
|
$inner->whereNull('label')->orWhere('label', '');
|
||||||
|
});
|
||||||
|
}
|
||||||
})
|
})
|
||||||
->orderBy('ran_at')
|
->orderBy('ran_at')
|
||||||
->get();
|
->get();
|
||||||
|
|
@ -78,11 +88,16 @@ public function results()
|
||||||
$column = in_array($this->sortBy, self::SORTABLE, true) ? $this->sortBy : 'scenario';
|
$column = in_array($this->sortBy, self::SORTABLE, true) ? $this->sortBy : 'scenario';
|
||||||
$direction = $this->sortDirection === 'asc' ? 'asc' : 'desc';
|
$direction = $this->sortDirection === 'asc' ? 'asc' : 'desc';
|
||||||
|
|
||||||
return DB::table('scenario_runs')
|
$query = DB::table('scenario_runs')
|
||||||
->whereIn('benchmark_run_id', $runIds)
|
->whereIn('benchmark_run_id', $runIds);
|
||||||
->orderBy($column, $direction)
|
|
||||||
->orderBy('id')
|
if ($column === 'gen_tps') {
|
||||||
->get();
|
$query->orderByRaw('gen_tokens / nullif(wall, 0) '.$direction);
|
||||||
|
} else {
|
||||||
|
$query->orderBy($column, $direction);
|
||||||
|
}
|
||||||
|
|
||||||
|
return $query->orderBy('id')->get();
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
?>
|
?>
|
||||||
|
|
@ -120,6 +135,7 @@ public function results()
|
||||||
<flux:table.column sortable :sorted="$sortBy === 't_first_sub'" :direction="$sortDirection" wire:click="sort('t_first_sub')" class="text-center">{{ __('t first sub') }}</flux:table.column>
|
<flux:table.column sortable :sorted="$sortBy === 't_first_sub'" :direction="$sortDirection" wire:click="sort('t_first_sub')" class="text-center">{{ __('t first sub') }}</flux:table.column>
|
||||||
<flux:table.column sortable :sorted="$sortBy === 't_pass'" :direction="$sortDirection" wire:click="sort('t_pass')" class="text-center">{{ __('t pass') }}</flux:table.column>
|
<flux:table.column sortable :sorted="$sortBy === 't_pass'" :direction="$sortDirection" wire:click="sort('t_pass')" class="text-center">{{ __('t pass') }}</flux:table.column>
|
||||||
<flux:table.column sortable :sorted="$sortBy === 'gen_tokens'" :direction="$sortDirection" wire:click="sort('gen_tokens')" class="text-center">{{ __('Gen tokens') }}</flux:table.column>
|
<flux:table.column sortable :sorted="$sortBy === 'gen_tokens'" :direction="$sortDirection" wire:click="sort('gen_tokens')" class="text-center">{{ __('Gen tokens') }}</flux:table.column>
|
||||||
|
<flux:table.column sortable :sorted="$sortBy === 'gen_tps'" :direction="$sortDirection" wire:click="sort('gen_tps')" class="text-center">{{ __('Gen tok/s') }}</flux:table.column>
|
||||||
</flux:table.columns>
|
</flux:table.columns>
|
||||||
|
|
||||||
<flux:table.rows>
|
<flux:table.rows>
|
||||||
|
|
@ -145,6 +161,7 @@ public function results()
|
||||||
<flux:table.cell class="text-center font-mono text-sm">{{ number_format($result->t_first_sub, 1) }}</flux:table.cell>
|
<flux:table.cell class="text-center font-mono text-sm">{{ number_format($result->t_first_sub, 1) }}</flux:table.cell>
|
||||||
<flux:table.cell class="text-center font-mono text-sm">{{ $result->t_pass !== null ? number_format($result->t_pass, 1) : '—' }}</flux:table.cell>
|
<flux:table.cell class="text-center font-mono text-sm">{{ $result->t_pass !== null ? number_format($result->t_pass, 1) : '—' }}</flux:table.cell>
|
||||||
<flux:table.cell class="text-center font-mono text-sm">{{ number_format($result->gen_tokens) }}</flux:table.cell>
|
<flux:table.cell class="text-center font-mono text-sm">{{ number_format($result->gen_tokens) }}</flux:table.cell>
|
||||||
|
<flux:table.cell class="text-center font-mono text-sm">{{ $result->wall > 0 && $result->gen_tokens > 0 ? number_format($result->gen_tokens / $result->wall, 1) : '—' }}</flux:table.cell>
|
||||||
</flux:table.row>
|
</flux:table.row>
|
||||||
@endforeach
|
@endforeach
|
||||||
</flux:table.rows>
|
</flux:table.rows>
|
||||||
|
|
|
||||||
|
|
@ -6,21 +6,57 @@
|
||||||
use Livewire\Component;
|
use Livewire\Component;
|
||||||
use Livewire\WithPagination;
|
use Livewire\WithPagination;
|
||||||
|
|
||||||
new class extends Component {
|
new class extends Component
|
||||||
|
{
|
||||||
use WithPagination;
|
use WithPagination;
|
||||||
|
|
||||||
public array $filters = [
|
public array $filters = [
|
||||||
'search' => null,
|
'gpu' => [],
|
||||||
'model' => null,
|
'model' => [],
|
||||||
'suite' => null,
|
'quant' => [],
|
||||||
|
'engine' => [],
|
||||||
|
'suite' => [],
|
||||||
];
|
];
|
||||||
|
|
||||||
|
public function updatedFilters(): void
|
||||||
|
{
|
||||||
|
$this->resetPage();
|
||||||
|
}
|
||||||
|
|
||||||
public function resetFilters(): void
|
public function resetFilters(): void
|
||||||
{
|
{
|
||||||
$this->filters = ['search' => null, 'model' => null, 'suite' => null];
|
$this->filters = [
|
||||||
|
'gpu' => [],
|
||||||
|
'model' => [],
|
||||||
|
'quant' => [],
|
||||||
|
'engine' => [],
|
||||||
|
'suite' => [],
|
||||||
|
];
|
||||||
$this->resetPage();
|
$this->resetPage();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public function toggleFilter(string $field, string $value): void
|
||||||
|
{
|
||||||
|
if (! array_key_exists($field, $this->filters)) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
$this->filters[$field] = array_values(array_unique(
|
||||||
|
in_array($value, $this->filters[$field], true)
|
||||||
|
? array_diff($this->filters[$field], [$value])
|
||||||
|
: array_merge($this->filters[$field], [$value])
|
||||||
|
));
|
||||||
|
|
||||||
|
$this->resetPage();
|
||||||
|
}
|
||||||
|
|
||||||
|
public function filterLabel(string $field): string
|
||||||
|
{
|
||||||
|
$count = count($this->filters[$field] ?? []);
|
||||||
|
|
||||||
|
return $count === 0 ? '' : ' ('.$count.')';
|
||||||
|
}
|
||||||
|
|
||||||
#[Computed]
|
#[Computed]
|
||||||
public function sessions()
|
public function sessions()
|
||||||
{
|
{
|
||||||
|
|
@ -62,34 +98,64 @@ private function bandQuery($user)
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
->when($this->filters['search'], fn ($q, $search) => $q->where(function ($q) use ($search) {
|
->when($this->filters['gpu'], fn ($q, array $gpus) => $q->whereIn('gpu', $gpus))
|
||||||
$q->where('label', 'like', "%{$search}%")
|
->when($this->filters['model'], fn ($q, array $models) => $q->whereIn('model_id', $models))
|
||||||
->orWhere('model_id', 'like', "%{$search}%")
|
->when($this->filters['quant'], fn ($q, array $quants) => $q->whereIn('model_quant', $quants))
|
||||||
->orWhere('gpu', 'like', "%{$search}%")
|
->when($this->filters['engine'], fn ($q, array $engines) => $q->whereIn('engine', $engines))
|
||||||
->orWhere('runner_name', 'like', "%{$search}%");
|
->when($this->filters['suite'], fn ($q, array $suites) => $q->whereExists(function ($sub) use ($suites) {
|
||||||
}))
|
|
||||||
->when($this->filters['model'], fn ($q, $model) => $q->where('model_id', $model))
|
|
||||||
// Suite filter: keep bands containing the chosen suite
|
|
||||||
->when($this->filters['suite'], fn ($q, $suite) => $q->whereExists(function ($sub) use ($suite) {
|
|
||||||
$sub->select(\Illuminate\Support\Facades\DB::raw(1))
|
$sub->select(\Illuminate\Support\Facades\DB::raw(1))
|
||||||
->from('session_suite_rows')
|
->from('session_suite_rows')
|
||||||
->whereColumn('session_suite_rows.session_summary_id', 'session_summaries.id')
|
->whereColumn('session_suite_rows.session_summary_id', 'session_summaries.id')
|
||||||
->where('session_suite_rows.suite', $suite);
|
->whereIn('session_suite_rows.suite', $suites);
|
||||||
}))
|
}))
|
||||||
->orderByDesc('ran_at')
|
->orderByDesc('ran_at')
|
||||||
->orderBy('sequence');
|
->orderBy('sequence');
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[Computed]
|
||||||
|
public function gpus(): array
|
||||||
|
{
|
||||||
|
return Cache::remember('summary:gpus', 300, function () {
|
||||||
|
return \Illuminate\Support\Facades\DB::table('session_summaries')
|
||||||
|
->where('visibility', 'public')
|
||||||
|
->whereNotNull('gpu')
|
||||||
|
->distinct()->orderBy('gpu')->pluck('gpu')->all();
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
#[Computed]
|
#[Computed]
|
||||||
public function models(): array
|
public function models(): array
|
||||||
{
|
{
|
||||||
return Cache::remember('summary:models', 300, function () {
|
return Cache::remember('summary:models', 300, function () {
|
||||||
return \Illuminate\Support\Facades\DB::table('session_summaries')
|
return \Illuminate\Support\Facades\DB::table('session_summaries')
|
||||||
->where('visibility', 'public')
|
->where('visibility', 'public')
|
||||||
|
->whereNotNull('model_id')
|
||||||
->distinct()->orderBy('model_id')->pluck('model_id')->all();
|
->distinct()->orderBy('model_id')->pluck('model_id')->all();
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[Computed]
|
||||||
|
public function quants(): array
|
||||||
|
{
|
||||||
|
return Cache::remember('summary:quants', 300, function () {
|
||||||
|
return \Illuminate\Support\Facades\DB::table('session_summaries')
|
||||||
|
->where('visibility', 'public')
|
||||||
|
->whereNotNull('model_quant')
|
||||||
|
->distinct()->orderBy('model_quant')->pluck('model_quant')->all();
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
#[Computed]
|
||||||
|
public function engines(): array
|
||||||
|
{
|
||||||
|
return Cache::remember('summary:engines', 300, function () {
|
||||||
|
return \Illuminate\Support\Facades\DB::table('session_summaries')
|
||||||
|
->where('visibility', 'public')
|
||||||
|
->whereNotNull('engine')
|
||||||
|
->distinct()->orderBy('engine')->pluck('engine')->all();
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
#[Computed]
|
#[Computed]
|
||||||
public function suites(): array
|
public function suites(): array
|
||||||
{
|
{
|
||||||
|
|
@ -106,33 +172,50 @@ public function suites(): array
|
||||||
<div>
|
<div>
|
||||||
<div class="flex items-center justify-between">
|
<div class="flex items-center justify-between">
|
||||||
<div>
|
<div>
|
||||||
<flux:heading size="xl">{{ __('Benchmark Sessions') }}</flux:heading>
|
<flux:heading size="xl">{{ __('FestBench - Benchmarks') }}</flux:heading>
|
||||||
<flux:subheading>{{ __('GPU, model, suite scores — click a suite line for per-task detail.') }}</flux:subheading>
|
<flux:subheading>{{ __('GPU, model, suite scores — click a suite line for per-task detail.') }}</flux:subheading>
|
||||||
</div>
|
</div>
|
||||||
<flux:button variant="ghost" icon="table-cells" href="{{ route('results') }}" wire:navigate>{{ __('All task rows') }}</flux:button>
|
<flux:button variant="ghost" icon="table-cells" href="{{ route('results') }}" wire:navigate>{{ __('All task rows') }}</flux:button>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<div class="mt-6 flex flex-wrap items-end gap-3">
|
<div class="mt-6 flex flex-wrap items-end gap-3">
|
||||||
<flux:input
|
@php
|
||||||
wire:model.live.debounce.300ms="filters.search"
|
$filterGroups = [
|
||||||
:label="__('Search')"
|
['field' => 'gpu', 'label' => __('GPU'), 'options' => $this->gpus],
|
||||||
:placeholder="__('Session, model, GPU or runner…')"
|
['field' => 'model', 'label' => __('Model'), 'options' => $this->models],
|
||||||
size="sm"
|
['field' => 'quant', 'label' => __('Quant'), 'options' => $this->quants],
|
||||||
class="w-64"
|
['field' => 'engine', 'label' => __('Inference engine'), 'options' => $this->engines],
|
||||||
clearable
|
['field' => 'suite', 'label' => __('Suite'), 'options' => $this->suites],
|
||||||
/>
|
];
|
||||||
|
@endphp
|
||||||
|
|
||||||
<flux:select wire:model.live="filters.model" :label="__('Model')" placeholder="{{ __('All models') }}" size="sm" class="w-56">
|
@foreach ($filterGroups as $group)
|
||||||
@foreach ($this->models as $model)
|
<flux:dropdown position="bottom" align="start">
|
||||||
<flux:select.option value="{{ $model }}">{{ $model }}</flux:select.option>
|
<flux:button size="sm" variant="outline" class="justify-between {{ count($this->filters[$group['field']]) > 0 ? 'border-sky-400 text-sky-600 dark:border-sky-500 dark:text-sky-400' : '' }}">
|
||||||
@endforeach
|
{{ $group['label'] }}{{ $this->filterLabel($group['field']) }}
|
||||||
</flux:select>
|
<flux:icon.chevron-down variant="mini" />
|
||||||
|
</flux:button>
|
||||||
|
|
||||||
<flux:select wire:model.live="filters.suite" :label="__('Suite')" placeholder="{{ __('All suites') }}" size="sm" class="w-44">
|
<flux:menu class="min-w-48 max-h-72 overflow-y-auto p-1">
|
||||||
@foreach ($this->suites as $suite)
|
@foreach ($group['options'] as $option)
|
||||||
<flux:select.option value="{{ $suite }}">{{ $suite }}</flux:select.option>
|
@php $checked = in_array($option, $this->filters[$group['field']], true); @endphp
|
||||||
@endforeach
|
<flux:menu.item
|
||||||
</flux:select>
|
wire:click="toggleFilter('{{ $group['field'] }}', '{{ addslashes($option) }}')"
|
||||||
|
class="cursor-pointer"
|
||||||
|
>
|
||||||
|
<span class="flex items-center gap-2">
|
||||||
|
@if ($checked)
|
||||||
|
<flux:icon.check variant="mini" class="size-4 text-sky-500" />
|
||||||
|
@else
|
||||||
|
<span class="size-4"></span>
|
||||||
|
@endif
|
||||||
|
<span class="text-sm">{{ $option }}</span>
|
||||||
|
</span>
|
||||||
|
</flux:menu.item>
|
||||||
|
@endforeach
|
||||||
|
</flux:menu>
|
||||||
|
</flux:dropdown>
|
||||||
|
@endforeach
|
||||||
|
|
||||||
<flux:button size="sm" variant="ghost" wire:click="resetFilters" icon="x-mark">{{ __('Clear') }}</flux:button>
|
<flux:button size="sm" variant="ghost" wire:click="resetFilters" icon="x-mark">{{ __('Clear') }}</flux:button>
|
||||||
</div>
|
</div>
|
||||||
|
|
@ -150,58 +233,137 @@ class="w-64"
|
||||||
<span class="text-sm">{{ $session->band->model_id ?? 'unknown model' }}</span>
|
<span class="text-sm">{{ $session->band->model_id ?? 'unknown model' }}</span>
|
||||||
@if ($session->band->label)
|
@if ($session->band->label)
|
||||||
<flux:badge size="xs" color="amber">{{ $session->band->label }}</flux:badge>
|
<flux:badge size="xs" color="amber">{{ $session->band->label }}</flux:badge>
|
||||||
@else
|
@endif
|
||||||
@if ($session->band->model_quant)
|
@if ($session->band->model_quant)
|
||||||
<flux:badge size="xs" color="zinc">{{ $session->band->model_quant }}</flux:badge>
|
<flux:badge size="xs" color="zinc">{{ $session->band->model_quant }}</flux:badge>
|
||||||
@endif
|
@endif
|
||||||
@if ($session->band->engine)
|
@if ($session->band->engine)
|
||||||
<flux:badge size="xs" color="sky">{{ $session->band->engine }}</flux:badge>
|
<flux:badge size="xs" color="sky">{{ $session->band->engine }}</flux:badge>
|
||||||
@endif
|
|
||||||
@if ($session->band->config_name)
|
|
||||||
<flux:badge size="xs" color="amber">{{ $session->band->config_name }}</flux:badge>
|
|
||||||
@endif
|
|
||||||
@endif
|
@endif
|
||||||
<flux:spacer />
|
<flux:spacer />
|
||||||
<span class="text-sm text-zinc-500">{{ $session->band->runner_name ?? 'anonymous' }}</span>
|
<span class="text-sm text-zinc-500">{{ $session->band->runner_name ?? 'anonymous' }}</span>
|
||||||
<span class="text-xs text-zinc-400">{{ optional($session->band->ran_at)->format('M j Y') }}</span>
|
<span class="text-xs text-zinc-400">{{ optional($session->band->ran_at)->format('M j Y') }}</span>
|
||||||
</div>
|
</div>
|
||||||
{{-- Column titles aligned over the suite rows (md+ only) --}}
|
@php
|
||||||
<div class="hidden px-4 pt-2 md:flex flex-wrap items-center gap-x-4 gap-y-1">
|
$devlogicSuites = ['softbench', 'prosebench', 'psychbench', 'depthsweep'];
|
||||||
<span class="w-24 text-[10px] font-semibold uppercase tracking-wider text-zinc-400">{{ __('Benchmark') }}</span>
|
$sysopsSuites = ['infrabench', 'scenariobench'];
|
||||||
<span class="w-14 text-[10px] font-semibold uppercase tracking-wider text-zinc-400">{{ __('Thinking') }}</span>
|
$devlogicRows = $session->suites->filter(fn ($r) => in_array($r->suite, $devlogicSuites));
|
||||||
<span class="w-24 text-[10px] font-semibold uppercase tracking-wider text-zinc-400">{{ __('Result') }}</span>
|
$sysopsRows = $session->suites->filter(fn ($r) => in_array($r->suite, $sysopsSuites));
|
||||||
<span class="w-40 text-[10px] font-semibold uppercase tracking-wider text-zinc-400">{{ __('Speed (tok/s)') }} <span class="font-normal normal-case">avg · wtd</span></span>
|
@endphp
|
||||||
<span class="w-20 text-[10px] font-semibold uppercase tracking-wider text-zinc-400">{{ __('Tokens') }}</span>
|
{{-- Devlogic section: total strip + column headers + suite rows --}}
|
||||||
</div>
|
@if ($devlogicRows->isNotEmpty() && $session->band->devlogic_tps !== null)
|
||||||
<div class="divide-y divide-zinc-100 dark:divide-zinc-800">
|
<div class="flex flex-wrap items-center gap-x-4 gap-y-1 border-l-2 border-sky-400 bg-sky-50/50 px-4 py-2 dark:border-sky-500 dark:bg-sky-950/20">
|
||||||
@foreach ($session->suites as $row)
|
<span class="text-xs font-semibold uppercase tracking-wider text-sky-600 dark:text-sky-400">{{ __('Devlogic total') }}</span>
|
||||||
@php
|
<span class="text-sm">
|
||||||
$detailUrl = $row->suite === 'scenariobench'
|
<span class="font-mono font-semibold">{{ number_format($session->band->devlogic_tps, 1) }}</span>
|
||||||
? route('sessions.scenarios', $session->band->id)
|
<span class="text-xs text-zinc-400">{{ __('tok/s') }}</span>
|
||||||
: route('runs.show', $row->run_id) . ($row->level && $row->suite === 'depthsweep' ? '?level=' . $row->level : '');
|
</span>
|
||||||
@endphp
|
<span class="text-sm">
|
||||||
<a href="{{ $detailUrl }}" wire:navigate
|
<span class="font-mono font-semibold">{{ number_format($session->band->devlogic_tokens) }}</span>
|
||||||
class="flex flex-wrap items-center gap-x-4 gap-y-1 px-4 py-2.5 hover:bg-zinc-50 dark:hover:bg-zinc-800/60">
|
<span class="text-xs text-zinc-400">{{ __('tokens') }}</span>
|
||||||
<flux:badge size="xs">{{ $row->suite }}</flux:badge>
|
</span>
|
||||||
@if ($row->level)
|
<span class="text-sm">
|
||||||
<flux:badge size="xs" color="sky">{{ $row->level }}</flux:badge>
|
<flux:badge size="xs" color="emerald">✔ {{ $session->band->devlogic_passed }}</flux:badge>
|
||||||
@endif
|
<flux:badge size="xs" color="{{ $session->band->devlogic_failed > 0 ? 'rose' : 'zinc' }}">✘ {{ $session->band->devlogic_failed }}</flux:badge>
|
||||||
<span class="flex items-center gap-1.5">
|
</span>
|
||||||
<flux:badge size="xs" color="emerald">✔ {{ $row->passed }}</flux:badge>
|
</div>
|
||||||
<flux:badge size="xs" color="{{ $row->failed > 0 ? 'rose' : 'zinc' }}">✘ {{ $row->failed }}</flux:badge>
|
<div class="hidden px-4 pt-2 md:grid md:grid-cols-[7.5rem_4.5rem_7rem_15rem_6rem_1fr] md:gap-x-4 md:items-center">
|
||||||
</span>
|
<span class="text-[10px] font-semibold uppercase tracking-wider text-zinc-400">{{ __('Benchmark') }}</span>
|
||||||
@if ($row->avg_user_tps !== null)
|
<span class="text-[10px] font-semibold uppercase tracking-wider text-zinc-400">{{ __('Thinking') }}</span>
|
||||||
<span class="font-mono text-sm">{{ number_format($row->avg_user_tps, 1) }} <span class="text-xs text-zinc-400">avg</span></span>
|
<span class="text-[10px] font-semibold uppercase tracking-wider text-zinc-400">{{ __('Result') }}</span>
|
||||||
<span class="font-mono text-sm">{{ number_format($row->weighted_user_tps, 1) }} <span class="text-xs text-zinc-400">wtd tok/s</span></span>
|
<span class="text-[10px] font-semibold uppercase tracking-wider text-zinc-400">{{ __('Speed (tok/s)') }} <span class="font-normal normal-case">{{ __('weighted average') }}</span></span>
|
||||||
@elseif ($row->gen_tps !== null)
|
<span class="text-[10px] font-semibold uppercase tracking-wider text-zinc-400">{{ __('Tokens') }}</span>
|
||||||
<span class="font-mono text-sm">{{ number_format($row->gen_tps, 1) }} <span class="text-xs text-zinc-400">gen tok/s</span></span>
|
<span></span>
|
||||||
<span class="font-mono text-sm text-zinc-400">—</span>
|
</div>
|
||||||
@endif
|
<div class="divide-y divide-zinc-100 dark:divide-zinc-800">
|
||||||
<span class="font-mono text-sm">{{ number_format($row->total_tokens) }} <span class="text-xs text-zinc-400">tokens</span></span>
|
@foreach ($devlogicRows as $row)
|
||||||
<span class="ms-auto text-xs text-zinc-400">{{ __('per-task detail') }} →</span>
|
@php
|
||||||
</a>
|
$detailUrl = $row->suite === 'scenariobench'
|
||||||
@endforeach
|
? route('sessions.scenarios', $session->band->id)
|
||||||
</div>
|
: route('runs.show', $row->run_id) . ($row->level && $row->suite === 'depthsweep' ? '?level=' . $row->level : '');
|
||||||
|
@endphp
|
||||||
|
<a href="{{ $detailUrl }}" wire:navigate
|
||||||
|
class="flex flex-wrap items-center gap-x-4 gap-y-1 px-4 py-2.5 hover:bg-zinc-50 dark:hover:bg-zinc-800/60 md:grid md:grid-cols-[7.5rem_4.5rem_7rem_15rem_6rem_1fr] md:gap-x-4 md:items-center">
|
||||||
|
<span><flux:badge size="xs">{{ $row->suite }}</flux:badge></span>
|
||||||
|
<span>
|
||||||
|
@if ($row->level)
|
||||||
|
<flux:badge size="xs" color="sky">{{ $row->level }}</flux:badge>
|
||||||
|
@endif
|
||||||
|
</span>
|
||||||
|
<span class="flex items-center gap-1.5">
|
||||||
|
<flux:badge size="xs" color="emerald">✔ {{ $row->passed }}</flux:badge>
|
||||||
|
<flux:badge size="xs" color="{{ $row->failed > 0 ? 'rose' : 'zinc' }}">✘ {{ $row->failed }}</flux:badge>
|
||||||
|
</span>
|
||||||
|
<span>
|
||||||
|
@if ($row->weighted_user_tps !== null)
|
||||||
|
<span class="font-mono text-sm">{{ number_format($row->weighted_user_tps, 1) }}</span>
|
||||||
|
@elseif ($row->gen_tps !== null)
|
||||||
|
<span class="font-mono text-sm">{{ number_format($row->gen_tps, 1) }} <span class="text-xs text-zinc-400">{{ __('gen') }}</span></span>
|
||||||
|
@endif
|
||||||
|
</span>
|
||||||
|
<span class="font-mono text-sm">{{ number_format($row->total_tokens) }}</span>
|
||||||
|
<span class="text-xs text-zinc-400 md:justify-self-end">{{ __('per-task detail') }} →</span>
|
||||||
|
</a>
|
||||||
|
@endforeach
|
||||||
|
</div>
|
||||||
|
@endif
|
||||||
|
{{-- Sysops section: total strip + column headers + suite rows --}}
|
||||||
|
@if ($sysopsRows->isNotEmpty() && $session->band->sysops_tps !== null)
|
||||||
|
<div class="flex flex-wrap items-center gap-x-4 gap-y-1 border-l-2 border-amber-400 bg-amber-50/50 px-4 py-2 dark:border-amber-500 dark:bg-amber-950/20">
|
||||||
|
<span class="text-xs font-semibold uppercase tracking-wider text-amber-600 dark:text-amber-400">{{ __('Sysops total') }}</span>
|
||||||
|
<span class="text-sm">
|
||||||
|
<span class="font-mono font-semibold">{{ number_format($session->band->sysops_tps, 1) }}</span>
|
||||||
|
<span class="text-xs text-zinc-400">{{ __('tok/s') }}</span>
|
||||||
|
</span>
|
||||||
|
<span class="text-sm">
|
||||||
|
<span class="font-mono font-semibold">{{ number_format($session->band->sysops_tokens) }}</span>
|
||||||
|
<span class="text-xs text-zinc-400">{{ __('tokens') }}</span>
|
||||||
|
</span>
|
||||||
|
<span class="text-sm">
|
||||||
|
<flux:badge size="xs" color="emerald">✔ {{ $session->band->sysops_passed }}</flux:badge>
|
||||||
|
<flux:badge size="xs" color="{{ $session->band->sysops_failed > 0 ? 'rose' : 'zinc' }}">✘ {{ $session->band->sysops_failed }}</flux:badge>
|
||||||
|
</span>
|
||||||
|
</div>
|
||||||
|
<div class="hidden px-4 pt-2 md:grid md:grid-cols-[7.5rem_4.5rem_7rem_15rem_6rem_1fr] md:gap-x-4 md:items-center">
|
||||||
|
<span class="text-[10px] font-semibold uppercase tracking-wider text-zinc-400">{{ __('Benchmark') }}</span>
|
||||||
|
<span class="text-[10px] font-semibold uppercase tracking-wider text-zinc-400">{{ __('Thinking') }}</span>
|
||||||
|
<span class="text-[10px] font-semibold uppercase tracking-wider text-zinc-400">{{ __('Result') }}</span>
|
||||||
|
<span class="text-[10px] font-semibold uppercase tracking-wider text-zinc-400">{{ __('Speed (tok/s)') }} <span class="font-normal normal-case">{{ __('weighted average') }}</span></span>
|
||||||
|
<span class="text-[10px] font-semibold uppercase tracking-wider text-zinc-400">{{ __('Tokens') }}</span>
|
||||||
|
<span></span>
|
||||||
|
</div>
|
||||||
|
<div class="divide-y divide-zinc-100 dark:divide-zinc-800">
|
||||||
|
@foreach ($sysopsRows as $row)
|
||||||
|
@php
|
||||||
|
$detailUrl = $row->suite === 'scenariobench'
|
||||||
|
? route('sessions.scenarios', $session->band->id)
|
||||||
|
: route('runs.show', $row->run_id) . ($row->level && $row->suite === 'depthsweep' ? '?level=' . $row->level : '');
|
||||||
|
@endphp
|
||||||
|
<a href="{{ $detailUrl }}" wire:navigate
|
||||||
|
class="flex flex-wrap items-center gap-x-4 gap-y-1 px-4 py-2.5 hover:bg-zinc-50 dark:hover:bg-zinc-800/60 md:grid md:grid-cols-[7.5rem_4.5rem_7rem_15rem_6rem_1fr] md:gap-x-4 md:items-center">
|
||||||
|
<span><flux:badge size="xs">{{ $row->suite }}</flux:badge></span>
|
||||||
|
<span>
|
||||||
|
@if ($row->level)
|
||||||
|
<flux:badge size="xs" color="sky">{{ $row->level }}</flux:badge>
|
||||||
|
@endif
|
||||||
|
</span>
|
||||||
|
<span class="flex items-center gap-1.5">
|
||||||
|
<flux:badge size="xs" color="emerald">✔ {{ $row->passed }}</flux:badge>
|
||||||
|
<flux:badge size="xs" color="{{ $row->failed > 0 ? 'rose' : 'zinc' }}">✘ {{ $row->failed }}</flux:badge>
|
||||||
|
</span>
|
||||||
|
<span>
|
||||||
|
@if ($row->weighted_user_tps !== null)
|
||||||
|
<span class="font-mono text-sm">{{ number_format($row->weighted_user_tps, 1) }}</span>
|
||||||
|
@elseif ($row->gen_tps !== null)
|
||||||
|
<span class="font-mono text-sm">{{ number_format($row->gen_tps, 1) }} <span class="text-xs text-zinc-400">{{ __('gen') }}</span></span>
|
||||||
|
@endif
|
||||||
|
</span>
|
||||||
|
<span class="font-mono text-sm">{{ number_format($row->total_tokens) }}</span>
|
||||||
|
<span class="text-xs text-zinc-400 md:justify-self-end">{{ __('per-task detail') }} →</span>
|
||||||
|
</a>
|
||||||
|
@endforeach
|
||||||
|
</div>
|
||||||
|
@endif
|
||||||
</flux:card>
|
</flux:card>
|
||||||
@endforeach
|
@endforeach
|
||||||
</div>
|
</div>
|
||||||
|
|
|
||||||
|
|
@ -2,16 +2,15 @@
|
||||||
|
|
||||||
use App\Models\BenchmarkRun;
|
use App\Models\BenchmarkRun;
|
||||||
use App\Models\InfrabenchResult;
|
use App\Models\InfrabenchResult;
|
||||||
|
use App\Models\ScenarioRun;
|
||||||
use App\Models\SoftbenchResult;
|
use App\Models\SoftbenchResult;
|
||||||
use App\Models\User;
|
use App\Models\User;
|
||||||
use Illuminate\Support\Facades\Artisan;
|
use Illuminate\Support\Facades\Artisan;
|
||||||
|
use Livewire\Volt\Volt;
|
||||||
|
|
||||||
test('festbench:summary groups matching runs despite different labels', function () {
|
test('festbench:summary splits runs by label into separate bands', function () {
|
||||||
$me = User::factory()->create(['name' => 'peterZ']);
|
$me = User::factory()->create(['name' => 'peterZ']);
|
||||||
|
|
||||||
// All three runs share real configuration identity. Labels are user
|
|
||||||
// notes, not grouping identity, so the different session B label must
|
|
||||||
// not fork a second bubble.
|
|
||||||
$a = BenchmarkRun::factory()->create([
|
$a = BenchmarkRun::factory()->create([
|
||||||
'suite' => 'softbench', 'label' => 'session A', 'model_id' => 'm', 'model_quant' => 'Q5_K_M',
|
'suite' => 'softbench', 'label' => 'session A', 'model_id' => 'm', 'model_quant' => 'Q5_K_M',
|
||||||
'gpu' => 'AMD R9700', 'gpu_count' => 1, 'engine' => 'llama.cpp', 'user_id' => $me->id, 'visibility' => 'public',
|
'gpu' => 'AMD R9700', 'gpu_count' => 1, 'engine' => 'llama.cpp', 'user_id' => $me->id, 'visibility' => 'public',
|
||||||
|
|
@ -36,24 +35,20 @@
|
||||||
|
|
||||||
Artisan::call('festbench:summary');
|
Artisan::call('festbench:summary');
|
||||||
|
|
||||||
|
// Different labels → separate bands.
|
||||||
$bands = DB::table('session_summaries')->orderBy('sequence')->get();
|
$bands = DB::table('session_summaries')->orderBy('sequence')->get();
|
||||||
expect($bands)->toHaveCount(1);
|
expect($bands)->toHaveCount(2);
|
||||||
|
|
||||||
$bandA = $bands->first();
|
$bandA = $bands->firstWhere('label', 'session A');
|
||||||
expect($bandA->runner_name)->toBe('peterZ')
|
expect($bandA->runner_name)->toBe('peterZ')
|
||||||
->and($bandA->gpu)->toBe('AMD R9700')
|
->and($bandA->gpu)->toBe('AMD R9700')
|
||||||
->and($bandA->gpu_count)->toBe(1)
|
->and($bandA->gpu_count)->toBe(1)
|
||||||
// Mixed labels intentionally show no single label on the band.
|
->and($bandA->label)->toBe('session A');
|
||||||
->and($bandA->label)->toBeNull()
|
|
||||||
->and($bandA->total_passed)->toBe(3)
|
|
||||||
->and($bandA->total_failed)->toBe(1)
|
|
||||||
->and($bandA->total_tokens)->toBe(650);
|
|
||||||
|
|
||||||
$rowsA = DB::table('session_suite_rows')->where('session_summary_id', $bandA->id)->orderBy('sequence')->get();
|
$rowsA = DB::table('session_suite_rows')->where('session_summary_id', $bandA->id)->orderBy('sequence')->get();
|
||||||
expect($rowsA)->toHaveCount(3);
|
expect($rowsA)->toHaveCount(2);
|
||||||
|
|
||||||
$soft = $rowsA->where('suite', 'softbench')->firstWhere('level', 'off');
|
$soft = $rowsA->firstWhere('suite', 'softbench');
|
||||||
// simple avg of (60, 20) = 40; weighted = (60*100 + 20*300)/400 = 30
|
|
||||||
expect($soft->passed)->toBe(1)
|
expect($soft->passed)->toBe(1)
|
||||||
->and($soft->failed)->toBe(1)
|
->and($soft->failed)->toBe(1)
|
||||||
->and($soft->avg_user_tps)->toBe(40.0)
|
->and($soft->avg_user_tps)->toBe(40.0)
|
||||||
|
|
@ -63,9 +58,56 @@
|
||||||
|
|
||||||
$infra = $rowsA->firstWhere('suite', 'infrabench');
|
$infra = $rowsA->firstWhere('suite', 'infrabench');
|
||||||
expect($infra->passed)->toBe(1)->and($infra->level)->toBe('low');
|
expect($infra->passed)->toBe(1)->and($infra->level)->toBe('low');
|
||||||
|
});
|
||||||
|
|
||||||
$secondSoft = $rowsA->where('suite', 'softbench')->firstWhere('level', 'medium');
|
test('devlogic and sysops totals are split by suite category', function () {
|
||||||
expect($secondSoft->passed)->toBe(1)->and($secondSoft->total_tokens)->toBe(50);
|
$me = User::factory()->create();
|
||||||
|
|
||||||
|
$soft = BenchmarkRun::factory()->create([
|
||||||
|
'suite' => 'softbench', 'label' => 'dual', 'model_id' => 'm', 'model_quant' => 'Q5_K_M',
|
||||||
|
'gpu' => 'GPU1', 'gpu_count' => 1, 'engine' => 'llama.cpp', 'user_id' => $me->id, 'visibility' => 'public',
|
||||||
|
'ran_at' => '2026-09-03 12:00:00',
|
||||||
|
]);
|
||||||
|
SoftbenchResult::factory()->create(['benchmark_run_id' => $soft->id, 'ok' => true, 'user_tps' => 50.0, 'total' => 100, 'level' => 'off']);
|
||||||
|
SoftbenchResult::factory()->create(['benchmark_run_id' => $soft->id, 'ok' => false, 'user_tps' => 10.0, 'total' => 200, 'level' => 'off']);
|
||||||
|
|
||||||
|
$infra = BenchmarkRun::factory()->create([
|
||||||
|
'suite' => 'infrabench', 'label' => 'dual', 'model_id' => 'm', 'model_quant' => 'Q5_K_M',
|
||||||
|
'gpu' => 'GPU1', 'gpu_count' => 1, 'engine' => 'llama.cpp', 'user_id' => $me->id, 'visibility' => 'public',
|
||||||
|
'ran_at' => '2026-09-03 12:05:00',
|
||||||
|
]);
|
||||||
|
InfrabenchResult::factory()->create(['benchmark_run_id' => $infra->id, 'ok' => true, 'user_tps' => 30.0, 'total' => 150, 'level' => 'low']);
|
||||||
|
|
||||||
|
$scen = BenchmarkRun::factory()->create([
|
||||||
|
'suite' => 'scenariobench', 'label' => 'dual', 'model_id' => 'm', 'model_quant' => 'Q5_K_M',
|
||||||
|
'gpu' => 'GPU1', 'gpu_count' => 1, 'engine' => 'llama.cpp', 'user_id' => $me->id, 'visibility' => 'public',
|
||||||
|
'ran_at' => '2026-09-03 12:10:00',
|
||||||
|
]);
|
||||||
|
ScenarioRun::factory()->create(['benchmark_run_id' => $scen->id, 'ok' => true, 'gen_tokens' => 5000, 'wall' => 100.0]);
|
||||||
|
ScenarioRun::factory()->create(['benchmark_run_id' => $scen->id, 'ok' => false, 'gen_tokens' => 3000, 'wall' => 200.0]);
|
||||||
|
|
||||||
|
Artisan::call('festbench:summary');
|
||||||
|
|
||||||
|
$band = DB::table('session_summaries')->where('label', 'dual')->first();
|
||||||
|
|
||||||
|
// Devlogic: softbench only — 2 tasks: user_tps=50 (100 tokens),
|
||||||
|
// user_tps=10 (200 tokens).
|
||||||
|
// Token-weighted: (50×100 + 10×200) / (100+200) = 7000/300 ≈ 23.3
|
||||||
|
expect($band->devlogic_passed)->toBe(1)
|
||||||
|
->and($band->devlogic_failed)->toBe(1)
|
||||||
|
->and($band->devlogic_tokens)->toBe(300)
|
||||||
|
->and($band->devlogic_tps)->toBe(23.3);
|
||||||
|
|
||||||
|
// Sysops: infrabench (1 task: user_tps=30, tokens=150 → rate_sum=4500)
|
||||||
|
// + scenariobench (gen_tps = 8000/300 = 26.67, tokens=8000 →
|
||||||
|
// rate_sum = 26.67×8000 ≈ 213333.33).
|
||||||
|
// Total sysops tokens = 150 + 8000 = 8150.
|
||||||
|
// Token-weighted: (4500 + 213333.33) / 8150 ≈ 26.7
|
||||||
|
expect($band->sysops_passed)->toBe(2)
|
||||||
|
->and($band->sysops_failed)->toBe(1)
|
||||||
|
->and($band->sysops_tokens)->toBe(8150);
|
||||||
|
expect($band->sysops_tps)->not->toBeNull();
|
||||||
|
expect($band->sysops_tps)->toBeGreaterThan(20.0)->toBeLessThan(30.0);
|
||||||
});
|
});
|
||||||
|
|
||||||
test('private runs form their own band and are hidden from guests', function () {
|
test('private runs form their own band and are hidden from guests', function () {
|
||||||
|
|
@ -86,7 +128,6 @@
|
||||||
expect(DB::table('session_summaries')->count())->toBe(2)
|
expect(DB::table('session_summaries')->count())->toBe(2)
|
||||||
->and(DB::table('session_summaries')->where('visibility', 'private')->count())->toBe(1);
|
->and(DB::table('session_summaries')->where('visibility', 'private')->count())->toBe(1);
|
||||||
|
|
||||||
// Guest home shows the public band only; owner sees both
|
|
||||||
$this->get(route('home'))->assertOk()->assertSee('pub');
|
$this->get(route('home'))->assertOk()->assertSee('pub');
|
||||||
|
|
||||||
$bands = DB::table('session_summaries')->pluck('visibility');
|
$bands = DB::table('session_summaries')->pluck('visibility');
|
||||||
|
|
@ -118,3 +159,122 @@
|
||||||
->assertSee('All Task Rows')
|
->assertSee('All Task Rows')
|
||||||
->assertSee('flat-task');
|
->assertSee('flat-task');
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test('multi-filter: suite filter returns bands containing any selected suite', function () {
|
||||||
|
$me = User::factory()->create();
|
||||||
|
|
||||||
|
$soft = BenchmarkRun::factory()->create([
|
||||||
|
'suite' => 'softbench', 'label' => 'soft-only', 'model_id' => 'm', 'model_quant' => 'Q5_K_M',
|
||||||
|
'gpu' => 'GPU1', 'gpu_count' => 1, 'engine' => 'llama.cpp', 'user_id' => $me->id, 'visibility' => 'public',
|
||||||
|
'ran_at' => '2026-09-03 12:00:00',
|
||||||
|
]);
|
||||||
|
SoftbenchResult::factory()->create(['benchmark_run_id' => $soft->id, 'ok' => true, 'user_tps' => 50.0, 'total' => 100, 'level' => 'off']);
|
||||||
|
|
||||||
|
$infra = BenchmarkRun::factory()->create([
|
||||||
|
'suite' => 'infrabench', 'label' => 'infra-only', 'model_id' => 'm', 'model_quant' => 'Q5_K_M',
|
||||||
|
'gpu' => 'GPU1', 'gpu_count' => 1, 'engine' => 'llama.cpp', 'user_id' => $me->id, 'visibility' => 'public',
|
||||||
|
'ran_at' => '2026-09-03 12:05:00',
|
||||||
|
]);
|
||||||
|
InfrabenchResult::factory()->create(['benchmark_run_id' => $infra->id, 'ok' => true, 'user_tps' => 30.0, 'total' => 100, 'level' => 'low']);
|
||||||
|
|
||||||
|
Artisan::call('festbench:summary');
|
||||||
|
|
||||||
|
// No filter: both bands visible.
|
||||||
|
Volt::test('summary.index')->assertSee('soft-only')->assertSee('infra-only');
|
||||||
|
|
||||||
|
// Filter to infrabench: only the infra band should match.
|
||||||
|
Volt::test('summary.index', ['filters' => [
|
||||||
|
'gpu' => [], 'model' => [], 'quant' => [], 'engine' => [], 'suite' => ['infrabench'],
|
||||||
|
]])
|
||||||
|
->assertDontSee('soft-only')
|
||||||
|
->assertSee('infra-only');
|
||||||
|
});
|
||||||
|
|
||||||
|
test('multi-filter: AND across categories, OR within a category', function () {
|
||||||
|
$me = User::factory()->create();
|
||||||
|
|
||||||
|
// Band A: GPU1 + model-A + Q5_K_M + llama.cpp
|
||||||
|
$a = BenchmarkRun::factory()->create([
|
||||||
|
'suite' => 'softbench', 'label' => 'band-a', 'model_id' => 'model-A', 'model_quant' => 'Q5_K_M',
|
||||||
|
'gpu' => 'GPU1', 'gpu_count' => 1, 'engine' => 'llama.cpp', 'user_id' => $me->id, 'visibility' => 'public',
|
||||||
|
'ran_at' => '2026-09-03 12:00:00',
|
||||||
|
]);
|
||||||
|
SoftbenchResult::factory()->create(['benchmark_run_id' => $a->id, 'ok' => true, 'user_tps' => 50.0, 'total' => 100, 'level' => 'off']);
|
||||||
|
|
||||||
|
// Band B: GPU2 + model-B + UD-Q4_K_M + ollama-cloud
|
||||||
|
$b = BenchmarkRun::factory()->create([
|
||||||
|
'suite' => 'softbench', 'label' => 'band-b', 'model_id' => 'model-B', 'model_quant' => 'UD-Q4_K_M',
|
||||||
|
'gpu' => 'GPU2', 'gpu_count' => 1, 'engine' => 'ollama-cloud', 'user_id' => $me->id, 'visibility' => 'public',
|
||||||
|
'ran_at' => '2026-09-04 12:00:00',
|
||||||
|
]);
|
||||||
|
SoftbenchResult::factory()->create(['benchmark_run_id' => $b->id, 'ok' => true, 'user_tps' => 40.0, 'total' => 100, 'level' => 'off']);
|
||||||
|
|
||||||
|
Artisan::call('festbench:summary');
|
||||||
|
|
||||||
|
// GPU filter alone: OR within GPU → both bands.
|
||||||
|
Volt::test('summary.index', ['filters' => [
|
||||||
|
'gpu' => ['GPU1', 'GPU2'], 'model' => [], 'quant' => [], 'engine' => [], 'suite' => [],
|
||||||
|
]])
|
||||||
|
->assertSee('band-a')
|
||||||
|
->assertSee('band-b');
|
||||||
|
|
||||||
|
// GPU + model: AND across → only matching band.
|
||||||
|
Volt::test('summary.index', ['filters' => [
|
||||||
|
'gpu' => ['GPU1', 'GPU2'], 'model' => ['model-A'], 'quant' => [], 'engine' => [], 'suite' => [],
|
||||||
|
]])
|
||||||
|
->assertSee('band-a')
|
||||||
|
->assertDontSee('band-b');
|
||||||
|
|
||||||
|
// Engine filter: only llama.cpp band.
|
||||||
|
Volt::test('summary.index', ['filters' => [
|
||||||
|
'gpu' => [], 'model' => [], 'quant' => [], 'engine' => ['llama.cpp'], 'suite' => [],
|
||||||
|
]])
|
||||||
|
->assertSee('band-a')
|
||||||
|
->assertDontSee('band-b');
|
||||||
|
|
||||||
|
// Quant filter: only UD-Q4_K_M band.
|
||||||
|
Volt::test('summary.index', ['filters' => [
|
||||||
|
'gpu' => [], 'model' => [], 'quant' => ['UD-Q4_K_M'], 'engine' => [], 'suite' => [],
|
||||||
|
]])
|
||||||
|
->assertDontSee('band-a')
|
||||||
|
->assertSee('band-b');
|
||||||
|
});
|
||||||
|
|
||||||
|
test('multi-filter: multiple suites use OR for band inclusion', function () {
|
||||||
|
$me = User::factory()->create();
|
||||||
|
|
||||||
|
// Band with only softbench.
|
||||||
|
$soft = BenchmarkRun::factory()->create([
|
||||||
|
'suite' => 'softbench', 'label' => 'has-soft', 'model_id' => 'm', 'model_quant' => 'Q5_K_M',
|
||||||
|
'gpu' => 'GPU1', 'gpu_count' => 1, 'engine' => 'llama.cpp', 'user_id' => $me->id, 'visibility' => 'public',
|
||||||
|
'ran_at' => '2026-09-03 12:00:00',
|
||||||
|
]);
|
||||||
|
SoftbenchResult::factory()->create(['benchmark_run_id' => $soft->id, 'ok' => true, 'user_tps' => 50.0, 'total' => 100, 'level' => 'off']);
|
||||||
|
|
||||||
|
// Band with only infrabench.
|
||||||
|
$infra = BenchmarkRun::factory()->create([
|
||||||
|
'suite' => 'infrabench', 'label' => 'has-infra', 'model_id' => 'm', 'model_quant' => 'Q5_K_M',
|
||||||
|
'gpu' => 'GPU1', 'gpu_count' => 1, 'engine' => 'llama.cpp', 'user_id' => $me->id, 'visibility' => 'public',
|
||||||
|
'ran_at' => '2026-09-03 12:05:00',
|
||||||
|
]);
|
||||||
|
InfrabenchResult::factory()->create(['benchmark_run_id' => $infra->id, 'ok' => true, 'user_tps' => 30.0, 'total' => 100, 'level' => 'low']);
|
||||||
|
|
||||||
|
Artisan::call('festbench:summary');
|
||||||
|
|
||||||
|
// Both suites selected → both bands visible (OR within suite).
|
||||||
|
Volt::test('summary.index', ['filters' => [
|
||||||
|
'gpu' => [], 'model' => [], 'quant' => [], 'engine' => [], 'suite' => ['softbench', 'infrabench'],
|
||||||
|
]])
|
||||||
|
->assertSee('has-soft')
|
||||||
|
->assertSee('has-infra');
|
||||||
|
});
|
||||||
|
|
||||||
|
test('toggleFilter adds and removes a value from a filter array', function () {
|
||||||
|
$component = Volt::test('summary.index');
|
||||||
|
|
||||||
|
$component->call('toggleFilter', 'suite', 'infrabench')
|
||||||
|
->assertSet('filters.suite', ['infrabench']);
|
||||||
|
|
||||||
|
$component->call('toggleFilter', 'suite', 'infrabench')
|
||||||
|
->assertSet('filters.suite', []);
|
||||||
|
});
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue