From ac941fdd27dd33f37a0f8ce50082e62c4e8914bb Mon Sep 17 00:00:00 2001
From: PeterChrz <pc@stampedepress.org>
Date: Mon, 5 Oct 2026 00:00:19 -0400
Subject: [PATCH] fix: switch devlogic/sysops totals from geo-mean to
 token-weighted average
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The headline totals were using a per-task geometric mean, which gave
disproportionate weight to short responses with low effective user_tps
(e.g. 2-token SoftBench answers at ~1.2 tok/s). This produced misleadingly
low Devlogic totals (20.5 for asus_210w) when the displayed suite speeds
were all in the 40-50 tok/s range.

Replace geo-mean accumulators (ln_sum/ln_n) with token-weighted
accumulators (rate_token_sum = sum(user_tps × tokens)). The headline total
is now rate_token_sum / total_tokens, matching the methodology already used
by each displayed suite row.

For ScenarioBench, the suite aggregate gen_tps (sum(gen_tokens)/sum(wall))
is weighted by the suite's total generated tokens, keeping the headline
total consistent with the displayed ScenarioBench row.

Results for the 210W bands:
  asus_210w:    devlogic 20.5 → 47.0, sysops 19.5 → 10.6
  sapphire_210w: devlogic 19.6 → 43.7, sysops 8.3 → 8.8
---
 app/Console/Commands/SessionSummary.php | 57 ++++++++++++-------------
 tests/Feature/SessionSummaryTest.php    | 17 ++++----
 2 files changed, 36 insertions(+), 38 deletions(-)

diff --git a/app/Console/Commands/SessionSummary.php b/app/Console/Commands/SessionSummary.php
index e83b116..7c2d810 100644
--- a/app/Console/Commands/SessionSummary.php
+++ b/app/Console/Commands/SessionSummary.php
@@ -71,17 +71,18 @@ public function handle(): int
                 'ran_at' => $run->ran_at,
                 // Devlogic totals: softbench, prosebench, psychbench,
                 // depthsweep — everything except infrabench and scenariobench.
+                // Token-weighted: sum(user_tps × tokens) / sum(tokens).
                 'dl_passed' => 0,
                 'dl_failed' => 0,
                 'dl_tokens' => 0,
-                'dl_ln_sum' => 0.0,
-                'dl_ln_n' => 0,
+                'dl_rate_sum' => 0.0,
                 // Sysops totals: infrabench and scenariobench.
+                // Token-weighted at the suite level: each suite's aggregate
+                // rate is weighted by that suite's total tokens.
                 'so_passed' => 0,
                 'so_failed' => 0,
                 'so_tokens' => 0,
-                'so_ln_sum' => 0.0,
-                'so_ln_n' => 0,
+                'so_rate_sum' => 0.0,
                 'runs' => collect(),
             ];
 
@@ -105,14 +106,12 @@ public function handle(): int
                 $sessions[$key]['so_passed'] += $aggs->sum('passed');
                 $sessions[$key]['so_failed'] += $aggs->sum('failed');
                 $sessions[$key]['so_tokens'] += $aggs->sum('total_tokens');
-                $sessions[$key]['so_ln_sum'] += $aggs->sum('ln_sum');
-                $sessions[$key]['so_ln_n'] += $aggs->sum('ln_n');
+                $sessions[$key]['so_rate_sum'] += $aggs->sum('rate_token_sum');
             } else {
                 $sessions[$key]['dl_passed'] += $aggs->sum('passed');
                 $sessions[$key]['dl_failed'] += $aggs->sum('failed');
                 $sessions[$key]['dl_tokens'] += $aggs->sum('total_tokens');
-                $sessions[$key]['dl_ln_sum'] += $aggs->sum('ln_sum');
-                $sessions[$key]['dl_ln_n'] += $aggs->sum('ln_n');
+                $sessions[$key]['dl_rate_sum'] += $aggs->sum('rate_token_sum');
             }
         }
 
@@ -127,8 +126,7 @@ public function handle(): int
             $sessions[$key]['so_passed'] += $agg['passed'];
             $sessions[$key]['so_failed'] += $agg['failed'];
             $sessions[$key]['so_tokens'] += $agg['total_tokens'];
-            $sessions[$key]['so_ln_sum'] += $agg['ln_sum'] ?? 0;
-            $sessions[$key]['so_ln_n'] += $agg['ln_n'] ?? 0;
+            $sessions[$key]['so_rate_sum'] += $agg['rate_token_sum'] ?? 0;
             $sessions[$key]['runs'] = $band['runs']->merge([$agg]);
         }
 
@@ -152,14 +150,14 @@ public function handle(): int
                     'devlogic_passed' => $band['dl_passed'],
                     'devlogic_failed' => $band['dl_failed'],
                     'devlogic_tokens' => $band['dl_tokens'],
-                    'devlogic_tps' => $band['dl_ln_n'] > 0
-                        ? round(exp($band['dl_ln_sum'] / $band['dl_ln_n']), 1)
+                    'devlogic_tps' => $band['dl_tokens'] > 0
+                        ? round($band['dl_rate_sum'] / $band['dl_tokens'], 1)
                         : null,
                     'sysops_passed' => $band['so_passed'],
                     'sysops_failed' => $band['so_failed'],
                     'sysops_tokens' => $band['so_tokens'],
-                    'sysops_tps' => $band['so_ln_n'] > 0
-                        ? round(exp($band['so_ln_sum'] / $band['so_ln_n']), 1)
+                    'sysops_tps' => $band['so_tokens'] > 0
+                        ? round($band['so_rate_sum'] / $band['so_tokens'], 1)
                         : null,
                     'sequence' => $bandIndex,
                     'created_at' => now(),
@@ -226,8 +224,7 @@ private function aggregateRun(object $run)
                     DB::raw('avg(nullif(wall, 0)) as avg_wall'),
                     DB::raw('sum(total) as total_tokens'),
                     DB::raw('sum(user_tps * total) / nullif(sum(total), 0) as weighted_user_tps'),
-                    DB::raw('sum(ln(user_tps)) filter (where user_tps > 0) as ln_sum'),
-                    DB::raw('count(*) filter (where user_tps > 0) as ln_n'),
+                    DB::raw('sum(user_tps * total) as rate_token_sum'),
                 ]))
                 ->sortBy(fn ($agg) => $levelOrder[$agg->level] ?? 99)
                 ->values()
@@ -244,8 +241,7 @@ private function aggregateRun(object $run)
                     'total_tokens' => (int) ($agg->total_tokens ?? 0),
                     'avg_ttft' => $this->roundOrNull($agg->avg_ttft),
                     'avg_wall' => $this->roundOrNull($agg->avg_wall),
-                    'ln_sum' => (float) ($agg->ln_sum ?? 0),
-                    'ln_n' => (int) ($agg->ln_n ?? 0),
+                    'rate_token_sum' => (float) ($agg->rate_token_sum ?? 0),
                 ]);
         }
 
@@ -257,6 +253,9 @@ private function aggregateRun(object $run)
                 ->selectRaw('sum(gen_tokens) / nullif(sum(wall), 0) as gen_tps')
                 ->first();
 
+            $genTps = (float) ($agg->gen_tps ?? 0);
+            $tokens = (int) ($agg->total_tokens ?? 0);
+
             return collect([[
                 'run_pk' => $run->id,
                 'run_id' => $run->run_id,
@@ -267,7 +266,8 @@ private function aggregateRun(object $run)
                 'avg_user_tps' => null,
                 'weighted_user_tps' => null,
                 'gen_tps' => $this->roundOrNull($agg->gen_tps),
-                'total_tokens' => (int) ($agg->total_tokens ?? 0),
+                'total_tokens' => $tokens,
+                'rate_token_sum' => $genTps * $tokens,
             ]]);
         }
 
@@ -277,8 +277,7 @@ private function aggregateRun(object $run)
             ->selectRaw('avg(user_tps) as avg_user_tps')
             ->selectRaw('sum(user_tps * total) / nullif(sum(total), 0) as weighted_user_tps')
             ->selectRaw('sum(total) as total_tokens')
-            ->selectRaw('sum(ln(user_tps)) filter (where user_tps > 0) as ln_sum')
-            ->selectRaw('count(*) filter (where user_tps > 0) as ln_n')
+            ->selectRaw('sum(user_tps * total) as rate_token_sum')
             ->first();
 
         return collect([[
@@ -292,8 +291,7 @@ private function aggregateRun(object $run)
             'weighted_user_tps' => $this->roundOrNull($agg->weighted_user_tps),
             'gen_tps' => null,
             'total_tokens' => (int) ($agg->total_tokens ?? 0),
-            'ln_sum' => (float) ($agg->ln_sum ?? 0),
-            'ln_n' => (int) ($agg->ln_n ?? 0),
+            'rate_token_sum' => (float) ($agg->rate_token_sum ?? 0),
         ]]);
     }
 
@@ -315,10 +313,11 @@ private function aggregateScenarioBand(array $runs): array
             ->selectRaw('count(*) filter (where not ok) as failed')
             ->selectRaw('sum(gen_tokens) as total_tokens')
             ->selectRaw('sum(gen_tokens) / nullif(sum(wall), 0) as gen_tps')
-            ->selectRaw('sum(ln(gen_tokens / nullif(wall, 0))) filter (where gen_tokens > 0 and wall > 0) as ln_sum')
-            ->selectRaw('count(*) filter (where gen_tokens > 0 and wall > 0) as ln_n')
             ->first();
 
+        $genTps = (float) ($agg->gen_tps ?? 0);
+        $tokens = (int) ($agg->total_tokens ?? 0);
+
         // Thinking level: keep it only when every scenario row in the band
         // agrees — matches the per-run runLevel() rule, applied at band scope.
         $level = DB::table('scenario_runs')
@@ -335,9 +334,8 @@ private function aggregateScenarioBand(array $runs): array
             'avg_user_tps' => null,
             'weighted_user_tps' => null,
             'gen_tps' => $this->roundOrNull($agg->gen_tps),
-            'total_tokens' => (int) ($agg->total_tokens ?? 0),
-            'ln_sum' => (float) ($agg->ln_sum ?? 0),
-            'ln_n' => (int) ($agg->ln_n ?? 0),
+            'total_tokens' => $tokens,
+            'rate_token_sum' => $genTps * $tokens,
         ];
     }
 
@@ -354,8 +352,7 @@ private function emptyAgg(object $run): array
             'weighted_user_tps' => null,
             'gen_tps' => null,
             'total_tokens' => 0,
-            'ln_sum' => 0.0,
-            'ln_n' => 0,
+            'rate_token_sum' => 0.0,
         ];
     }
 
diff --git a/tests/Feature/SessionSummaryTest.php b/tests/Feature/SessionSummaryTest.php
index eb3efa2..dd2dd6b 100644
--- a/tests/Feature/SessionSummaryTest.php
+++ b/tests/Feature/SessionSummaryTest.php
@@ -89,23 +89,24 @@
 
     $band = DB::table('session_summaries')->where('label', 'dual')->first();
 
-    // Devlogic: softbench only — 2 observations (50, 10).
-    // geo-mean = exp((ln(50)+ln(10))/2) = sqrt(500) ≈ 22.4
+    // Devlogic: softbench only — 2 tasks: user_tps=50 (100 tokens),
+    // user_tps=10 (200 tokens).
+    // Token-weighted: (50×100 + 10×200) / (100+200) = 7000/300 ≈ 23.3
     expect($band->devlogic_passed)->toBe(1)
         ->and($band->devlogic_failed)->toBe(1)
         ->and($band->devlogic_tokens)->toBe(300)
-        ->and($band->devlogic_tps)->toBe(22.4);
+        ->and($band->devlogic_tps)->toBe(23.3);
 
-    // Sysops: infrabench (1 obs: user_tps=30, tokens=150) +
-    // scenariobench (2 obs: gen_tokens/wall = 50.0 and 15.0, tokens=8000).
+    // Sysops: infrabench (1 task: user_tps=30, tokens=150 → rate_sum=4500)
+    // + scenariobench (gen_tps = 8000/300 = 26.67, tokens=8000 →
+    //   rate_sum = 26.67×8000 ≈ 213333.33).
     // Total sysops tokens = 150 + 8000 = 8150.
+    // Token-weighted: (4500 + 213333.33) / 8150 ≈ 26.7
     expect($band->sysops_passed)->toBe(2)
         ->and($band->sysops_failed)->toBe(1)
         ->and($band->sysops_tokens)->toBe(8150);
-    // sysops_tps is a geo-mean of 3 positive values — just verify it is
-    // populated and roughly in range.
     expect($band->sysops_tps)->not->toBeNull();
-    expect($band->sysops_tps)->toBeGreaterThan(10.0)->toBeLessThan(50.0);
+    expect($band->sysops_tps)->toBeGreaterThan(20.0)->toBeLessThan(30.0);
 });
 
 test('private runs form their own band and are hidden from guests', function () {