b005fb1ef6
Some checks are pending
release / release (push) Waiting to run
Package layout (zubble/), vendored benchlib.stream_chat, LocalEnv, minimal CLI, 38 pure-logic tests, and a tag-triggered Forgejo Actions release pipeline that builds a single-file PyInstaller executable.
339 lines
14 KiB
Python
339 lines
14 KiB
Python
#!/usr/bin/env python3
|
|
"""agent — structured turn loop for the zubble agent.
|
|
|
|
Integrates protocol.py (JSON schema), ledger.py (plan-state memory),
|
|
doom.py (escape-hatch detectors), and benchlib.stream_chat (streaming
|
|
HTTP client) into a turn loop that replaces scenariobench's flat-text
|
|
rendering.
|
|
|
|
The loop:
|
|
1. Build prompt: system_prompt (protocol) + extra_messages (history)
|
|
+ per-turn user message (session memory + time remaining)
|
|
2. Call stream_chat with StreamRepetitionDetector as watcher
|
|
3. Handle in-stream aborts (L1 keep+note, L2 retry, L3 kill)
|
|
4. Parse JSON response via protocol.parse_response (1 repair attempt)
|
|
5. Feed parsed action to ActionRepetitionDetector (3→note, 4→kill)
|
|
6. Execute action via env (run/submit/ask/plan)
|
|
7. Record in ledger, append harness response to messages
|
|
8. On kill: fresh conversation (reset messages, keep ledger),
|
|
enforce reset cap (3 → doom_loop)
|
|
"""
|
|
import time
|
|
|
|
from zubble import benchlib
|
|
from zubble import protocol
|
|
from zubble.ledger import Ledger
|
|
from zubble.doom import (
|
|
ActionRepetitionDetector, ActionSignal,
|
|
StreamRepetitionDetector, Level,
|
|
InsuranceCounter, ResetCap,
|
|
NOTE_ACTION_REPEAT, NOTE_STREAM_L1, NOTE_STREAM_L2,
|
|
NOTE_KILL_RESET, NOTE_DOOM_LOOP,
|
|
)
|
|
|
|
_PER_CALL_TIMEOUT = {4000: 240, 8000: 240, 16000: 480, 32000: 480}
|
|
|
|
|
|
def run_agent_loop(base, env, brief, login, deadline, level, level_kwargs,
|
|
max_turns=200, maxtok=32000, show=False, name="",
|
|
proto=None):
|
|
"""Run the structured agent loop for one scenario.
|
|
|
|
``proto`` is a ``protocol.Protocol`` instance supplying the
|
|
scenario-specific framing (role, opening text, deliverable wording,
|
|
verifier feedback). Defaults to ``protocol.REPAIR`` (the original
|
|
Kubernetes repair scenario) for backward compatibility.
|
|
|
|
Returns a telemetry dict with the same keys as scenariobench.run_one's
|
|
inner loop: turns, commands, submissions, rejected, repeats, passed,
|
|
outcome, detail, gen_tok, vis_tok, rea_tok, t_first_sub, n_first_sub,
|
|
t_pass, n_pass, verdicts, messages, exchange.
|
|
"""
|
|
if proto is None:
|
|
proto = protocol.REPAIR
|
|
|
|
ledger = Ledger()
|
|
action_det = ActionRepetitionDetector()
|
|
insurance = InsuranceCounter()
|
|
reset_cap = ResetCap()
|
|
|
|
initial_content = proto.initial_message(
|
|
env.intro, brief, login, env.report_path)
|
|
messages = [{"role": "user", "content": initial_content}]
|
|
|
|
turns = commands = submissions = rejected = repeats = 0
|
|
parse_failures = 0
|
|
gen_tok = vis_tok = rea_tok = 0
|
|
passed = False
|
|
outcome = "timeout"
|
|
detail = "ran out of time"
|
|
t_first_sub = n_first_sub = t_pass = n_pass = None
|
|
verdicts = []
|
|
exchange = []
|
|
t0 = time.monotonic()
|
|
nudged = False
|
|
l2_retried = False
|
|
per_call_timeout = _PER_CALL_TIMEOUT.get(maxtok, 480)
|
|
|
|
def _fresh_context(reason):
|
|
"""Reset messages for a fresh conversation, preserving the ledger."""
|
|
nonlocal l2_retried
|
|
messages.clear()
|
|
messages.append({"role": "user", "content": proto.reset_message(
|
|
env.intro, brief, login, env.report_path,
|
|
NOTE_KILL_RESET, ledger.render_summary())})
|
|
action_det.reset()
|
|
l2_retried = False
|
|
|
|
while time.monotonic() < deadline:
|
|
if turns >= max_turns:
|
|
detail = f"hit the {max_turns} turn cap"
|
|
outcome = "turn_cap"
|
|
break
|
|
turns += 1
|
|
left = int(deadline - time.monotonic())
|
|
|
|
if not nudged and (left < (deadline - t0) * 0.3 or turns > max_turns * 0.7):
|
|
nudged = True
|
|
messages.append({"role": "user", "content":
|
|
f"[{left}s remaining] {proto.time_nudge}"})
|
|
|
|
session_memory = ledger.render_summary()
|
|
prompt = f"## SESSION MEMORY\n{session_memory}\n\n[{left}s remaining]"
|
|
|
|
stream_det = StreamRepetitionDetector()
|
|
|
|
def watcher(kind, text):
|
|
return stream_det.feed(kind, text) != Level.NONE
|
|
|
|
timeout = max(30, min(left, per_call_timeout))
|
|
try:
|
|
text, _reasoning, perf = benchlib.stream_chat(
|
|
base, prompt, level_kwargs,
|
|
maxtok=maxtok, timeout=timeout,
|
|
system_prompt=proto.system_prompt(),
|
|
response_format=protocol.RESPONSE_FORMAT,
|
|
extra_messages=messages,
|
|
watcher=watcher,
|
|
)
|
|
except Exception as e:
|
|
detail = f"{type(e).__name__}: {e}"
|
|
outcome = "error"
|
|
break
|
|
|
|
if time.monotonic() >= deadline:
|
|
detail = "ran out of time mid-turn"
|
|
outcome = "timeout"
|
|
break
|
|
|
|
if perf:
|
|
gen_tok += perf.get("total", 0) or 0
|
|
vis_tok += perf.get("visible", 0) or 0
|
|
rea_tok += perf.get("reasoning", 0) or 0
|
|
|
|
if show:
|
|
print(f" --- turn {turns} ---\n{text[:600]}", flush=True)
|
|
|
|
# ── In-stream abort handling ────────────────────────────────
|
|
if perf.get("finish") == "aborted":
|
|
lvl = stream_det.level
|
|
streak = stream_det.streak
|
|
|
|
if lvl == Level.L3:
|
|
if reset_cap.register_reset():
|
|
outcome = "doom_loop"
|
|
detail = NOTE_DOOM_LOOP.format(resets=reset_cap.count)
|
|
print(f" [{name}] doom_loop: {reset_cap.count} resets exceeded")
|
|
break
|
|
ledger.record_abandoned(
|
|
"in-stream doom loop (L3)",
|
|
reason=f"repeated 50-token block {streak} times")
|
|
_fresh_context("L3 kill")
|
|
print(f" [{name}] fresh context (L3 kill, streak={streak})")
|
|
continue
|
|
|
|
if lvl == Level.L2 and not l2_retried:
|
|
if insurance.register_abort():
|
|
if reset_cap.register_reset():
|
|
outcome = "doom_loop"
|
|
detail = NOTE_DOOM_LOOP.format(resets=reset_cap.count)
|
|
break
|
|
ledger.record_abandoned(
|
|
"in-stream doom loop (insurance)",
|
|
reason=f"insurance tripwire: {insurance.count} aborts")
|
|
_fresh_context("insurance kill")
|
|
continue
|
|
l2_retried = True
|
|
messages.append({"role": "user", "content":
|
|
NOTE_STREAM_L2.format(streak=streak)})
|
|
print(f" [{name}] L2 abort (streak={streak}), retrying turn")
|
|
continue
|
|
|
|
# L1, or L2 when already retried → keep partial, inject note
|
|
if insurance.register_abort():
|
|
if reset_cap.register_reset():
|
|
outcome = "doom_loop"
|
|
detail = NOTE_DOOM_LOOP.format(resets=reset_cap.count)
|
|
break
|
|
ledger.record_abandoned(
|
|
"in-stream doom loop (insurance)",
|
|
reason=f"insurance tripwire: {insurance.count} aborts")
|
|
_fresh_context("insurance kill")
|
|
continue
|
|
messages.append({"role": "assistant", "content": text})
|
|
messages.append({"role": "user", "content":
|
|
NOTE_STREAM_L1.format(streak=streak)})
|
|
l2_retried = False
|
|
print(f" [{name}] L1 abort (streak={streak}), keeping prefix")
|
|
continue
|
|
|
|
# ── Parse JSON response ─────────────────────────────────────
|
|
try:
|
|
obj = protocol.parse_response(text)
|
|
except protocol.ProtocolError as e:
|
|
parse_failures += 1
|
|
print(f" [{name}] PARSE FAIL (turn {turns}): {e}", flush=True)
|
|
messages.append({"role": "assistant", "content": text})
|
|
messages.append({"role": "user", "content": proto.retry_nudge(e)})
|
|
try:
|
|
text2, _r2, perf2 = benchlib.stream_chat(
|
|
base, prompt, level_kwargs,
|
|
maxtok=maxtok, timeout=timeout,
|
|
system_prompt=proto.system_prompt(),
|
|
response_format=protocol.RESPONSE_FORMAT,
|
|
extra_messages=messages,
|
|
)
|
|
if perf2:
|
|
gen_tok += perf2.get("total", 0) or 0
|
|
vis_tok += perf2.get("visible", 0) or 0
|
|
rea_tok += perf2.get("reasoning", 0) or 0
|
|
obj = protocol.parse_response(text2)
|
|
text = text2
|
|
except protocol.ProtocolError as e2:
|
|
parse_failures += 1
|
|
print(f" [{name}] PARSE FAIL repair failed: {e2}", flush=True)
|
|
messages.append({"role": "assistant", "content": text2 if text2 else ""})
|
|
outcome = "parse_error"
|
|
detail = f"JSON parse failed after one repair: {e2}"
|
|
break
|
|
except Exception as e2:
|
|
outcome = "error"
|
|
detail = f"parse repair {type(e2).__name__}: {e2}"
|
|
break
|
|
|
|
messages.append({"role": "assistant", "content": text})
|
|
l2_retried = False
|
|
|
|
# ── Action-level repetition detection ───────────────────────
|
|
sig = action_det.feed(obj)
|
|
if sig == ActionSignal.KILL:
|
|
if reset_cap.register_reset():
|
|
outcome = "doom_loop"
|
|
detail = NOTE_DOOM_LOOP.format(resets=reset_cap.count)
|
|
print(f" [{name}] doom_loop: action kill, cap exceeded")
|
|
break
|
|
canonical = repr({k: v for k, v in obj.items() if k != "reasoning"})
|
|
ledger.record_abandoned(
|
|
f"action doom loop: {obj.get('action', '?')} {obj.get('command', '')[:80]}",
|
|
reason=f"repeated identically {action_det.count} times")
|
|
_fresh_context("action kill")
|
|
print(f" [{name}] fresh context (action kill, count={action_det.count})")
|
|
continue
|
|
|
|
action_note = ""
|
|
if sig == ActionSignal.NOTE:
|
|
action_note = NOTE_ACTION_REPEAT.format(n=action_det.count)
|
|
print(f" [{name}] action repeat note (count={action_det.count})")
|
|
|
|
# ── Execute the action ──────────────────────────────────────
|
|
action = obj.get("action")
|
|
|
|
if action == "plan":
|
|
steps = obj.get("steps", [])
|
|
ledger.set_plan(steps)
|
|
harness_resp = (
|
|
f"Plan recorded with {len(steps)} steps. "
|
|
f"Execute step 1 now."
|
|
)
|
|
if action_note:
|
|
harness_resp += f"\n{action_note}"
|
|
messages.append({"role": "user", "content": harness_resp})
|
|
continue
|
|
|
|
if action == "run":
|
|
cmd = obj.get("command", "")
|
|
step_id = obj.get("plan_step_id")
|
|
|
|
div = ledger.check_divergence(step_id, cmd)
|
|
if div:
|
|
action_note = (action_note + "\n" + div).strip()
|
|
|
|
crc, cout = env.exec(cmd, timeout=90)
|
|
commands += 1
|
|
exchange.append({"turn": turns, "cmd": cmd, "exit": crc,
|
|
"out": cout[:8000]})
|
|
ledger.record_action(step_id, cmd, crc, cout)
|
|
|
|
harness_resp = proto.render_action_result(
|
|
"run", crc, cout, note=action_note)
|
|
messages.append({"role": "user", "content": harness_resp})
|
|
continue
|
|
|
|
if action == "submit":
|
|
submissions += 1
|
|
if t_first_sub is None:
|
|
t_first_sub = round(time.monotonic() - t0, 1)
|
|
n_first_sub = turns
|
|
|
|
report_text = obj.get("report", "")
|
|
vrc, vout = env.verify()
|
|
verdicts.append({
|
|
"n": submissions,
|
|
"at": round(time.monotonic() - t0, 1),
|
|
"accepted": vrc == 0,
|
|
"oracle": " ".join(vout.split())[:1200],
|
|
})
|
|
print(f" [{name}] submission {submissions} @ "
|
|
f"{int(time.monotonic()-t0)}s -> "
|
|
f"{'ACCEPTED' if vrc == 0 else 'REJECTED'}", flush=True)
|
|
|
|
if vrc == 0:
|
|
passed = True
|
|
outcome = "passed"
|
|
t_pass = round(time.monotonic() - t0, 1)
|
|
n_pass = turns
|
|
detail = f"passed on submission {submissions}"
|
|
break
|
|
|
|
rejected += 1
|
|
harness_resp = proto.render_action_result(
|
|
"submit", vrc, vout, note=action_note)
|
|
messages.append({"role": "user", "content": harness_resp})
|
|
continue
|
|
|
|
if action == "ask":
|
|
harness_resp = proto.render_action_result(
|
|
"ask", 0, "", note=action_note)
|
|
messages.append({"role": "user", "content": harness_resp})
|
|
continue
|
|
|
|
if action == "report":
|
|
harness_resp = (
|
|
"Report noted. Submit when you are ready for verification."
|
|
)
|
|
if action_note:
|
|
harness_resp += f"\n{action_note}"
|
|
messages.append({"role": "user", "content": harness_resp})
|
|
continue
|
|
|
|
secs = time.monotonic() - t0
|
|
return dict(
|
|
turns=turns, commands=commands, submissions=submissions,
|
|
rejected=rejected, repeats=repeats, passed=passed,
|
|
outcome=outcome, detail=detail,
|
|
gen_tok=gen_tok, vis_tok=vis_tok, rea_tok=rea_tok,
|
|
t_first_sub=t_first_sub, n_first_sub=n_first_sub,
|
|
t_pass=t_pass, n_pass=n_pass, verdicts=verdicts,
|
|
messages=messages, exchange=exchange, seconds=round(secs, 1),
|
|
parse_failures=parse_failures,
|
|
)
|