b005fb1ef6
Some checks are pending
release / release (push) Waiting to run
Package layout (zubble/), vendored benchlib.stream_chat, LocalEnv, minimal CLI, 38 pure-logic tests, and a tag-triggered Forgejo Actions release pipeline that builds a single-file PyInstaller executable.
67 lines
No EOL
2.4 KiB
Python
67 lines
No EOL
2.4 KiB
Python
#!/usr/bin/env python3
|
|
"""env — environment abstractions for the zubble agent loop.
|
|
|
|
An Environment provides the surface the agent loop needs to interact
|
|
with the world:
|
|
|
|
intro str — shell/environment description shown to the model
|
|
report_path str — where the agent writes its deliverable
|
|
exec(cmd, timeout) -> (exit_code, output)
|
|
verify() -> (exit_code, output) # oracle / grader
|
|
|
|
LocalEnv is the simplest concrete Environment: each ``exec()`` call
|
|
runs in a fresh subprocess on the local machine. The scenario supplies
|
|
``intro``, ``report_path``, and a ``verify_fn`` callable at construction
|
|
time. This is the building block for container-based scenarios — a
|
|
container variant would override ``exec`` to shell into the container.
|
|
"""
|
|
import subprocess
|
|
|
|
|
|
class LocalEnv:
|
|
"""Execute commands via local subprocess, stateless between calls.
|
|
|
|
Parameters
|
|
----------
|
|
intro : str
|
|
Description of the environment shown to the model as the first
|
|
line of the opening message.
|
|
report_path : str
|
|
File path the agent is told to write its deliverable to.
|
|
verify_fn : callable, optional
|
|
``verify_fn() -> (exit_code, output)``. If omitted, verify()
|
|
always returns ``(0, "accepted")`` — useful for tests.
|
|
"""
|
|
|
|
def __init__(self, intro="You have a shell on this machine.",
|
|
report_path="/work/output.md", verify_fn=None):
|
|
self.intro = intro
|
|
self.report_path = report_path
|
|
self._verify_fn = verify_fn
|
|
|
|
def exec(self, cmd, timeout=90):
|
|
"""Run *cmd* in a fresh subprocess shell.
|
|
|
|
Returns (exit_code, combined_output). Output is truncated to
|
|
8000 chars to match the scenariobench convention.
|
|
"""
|
|
try:
|
|
r = subprocess.run(
|
|
cmd, shell=True, capture_output=True, text=True,
|
|
timeout=timeout,
|
|
)
|
|
out = (r.stdout or "") + (r.stderr or "")
|
|
return r.returncode, out[:8000]
|
|
except subprocess.TimeoutExpired:
|
|
return 124, f"[command timed out after {timeout}s]"
|
|
except Exception as e:
|
|
return 1, f"[exec error: {e}]"
|
|
|
|
def verify(self):
|
|
"""Call the scenario's verification oracle.
|
|
|
|
Returns (exit_code, output). Exit code 0 means accepted.
|
|
"""
|
|
if self._verify_fn:
|
|
return self._verify_fn()
|
|
return (0, "accepted") |