#!/usr/bin/env python3 import json, os, shutil, statistics, subprocess, threading, time from http.server import ThreadingHTTPServer, SimpleHTTPRequestHandler from pathlib import Path ROOT = Path('/tmp/agent-browser-bench') SITE = ROOT / 'site' AB = Path('/tmp/agent-browser-source/cli/target/release/agent-browser') PWCLI = Path('/tmp/playwright-1.62/node_modules/playwright/cli.js') CHROME = Path('/tmp/playwright-browsers/chromium_headless_shell-1234/chrome-headless-shell-linux64/chrome-headless-shell') class QuietHandler(SimpleHTTPRequestHandler): def log_message(self, *_): pass handler = lambda *a, **kw: QuietHandler(*a, directory=str(SITE), **kw) server = ThreadingHTTPServer(('127.0.0.1', 0), handler) threading.Thread(target=server.serve_forever, daemon=True).start() base = f'http://127.0.0.1:{server.server_port}' url = f'{base}/index.html' env = os.environ.copy() env.update({ 'AGENT_BROWSER_EXECUTABLE_PATH': str(CHROME), 'NO_COLOR': '1', 'PLAYWRIGHT_BROWSERS_PATH': '/tmp/playwright-browsers', }) configs = { 'playwright_cli_default': (ROOT / 'pwcli.config.json', ROOT / 'pwcli-output', 'bench-default'), 'playwright_cli_tuned': (ROOT / 'pwcli-tuned.config.json', ROOT / 'pwcli-tuned-output', 'bench-tuned'), } must = { 'open': ['Browser Bench', 'Open details', 'Search', 'Submit'], 'click': ['Navigation complete'], 'fill': ['Results for: rust browser'], } labels = { 'open': 'open + snapshot', 'click': 'open + click + snapshot', 'fill': 'open + fill + submit + snapshot', } def exec_cmd(cmd, *, e=env, timeout=45): return subprocess.run(cmd, env=e, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, timeout=timeout) def check_proc(p, cmd): if p.returncode: raise RuntimeError(f'failed {cmd}\nstdout={p.stdout}\nstderr={p.stderr}') return p.stdout + p.stderr def latest_snapshot(outdir): files = sorted(outdir.rglob('*.yml'), key=lambda p: p.stat().st_mtime_ns) if not files: raise RuntimeError(f'no Playwright CLI snapshot in {outdir}') return files[-1].read_text() def run_ab(workflow, warm=False): session = 'bench-ab-warm' if warm else 'bench-ab-cold' e = env.copy(); e['AGENT_BROWSER_SESSION'] = session commands = [f'open {url}'] if workflow == 'click': commands += ['click #details'] elif workflow == 'fill': commands += ['fill #query rust browser', 'click #submit'] commands += ['snapshot'] if not warm: commands += ['close'] p = exec_cmd([str(AB), 'batch', '--bail', *commands], e=e) out = check_proc(p, commands) if not all(x in out for x in must[workflow]): raise RuntimeError(f'bad agent-browser end state {workflow}:\n{out}') return out def pwcli_cmd(session, *args): return ['node', str(PWCLI), 'cli', f'-s={session}', *args] def run_pwcli(tool, workflow, warm=False): config, outdir, session = configs[tool] shutil.rmtree(outdir, ignore_errors=True) outdir.mkdir(parents=True, exist_ok=True) outputs = [] if warm: commands = [pwcli_cmd(session, 'goto', url)] else: commands = [pwcli_cmd(session, 'open', url, '--config', str(config))] if workflow == 'click': commands += [pwcli_cmd(session, 'click', '#details')] elif workflow == 'fill': commands += [pwcli_cmd(session, 'fill', '#query', 'rust browser'), pwcli_cmd(session, 'click', '#submit')] if not warm: commands += [pwcli_cmd(session, 'close')] for cmd in commands: outputs.append(check_proc(exec_cmd(cmd), cmd)) snap = latest_snapshot(outdir) combined = '\n'.join(outputs) + '\n' + snap if not all(x in combined for x in must[workflow]): raise RuntimeError(f'bad {tool} end state {workflow}:\n{combined}') return combined def run_pwlib(workflow): p = exec_cmd(['node', str(ROOT / 'pw_current_cold.mjs'), url, workflow]) out = check_proc(p, workflow) if not all(x in out for x in must[workflow]): raise RuntimeError(f'bad Playwright library end state {workflow}:\n{out}') return out def timed(fn): t0 = time.perf_counter_ns(); out = fn(); return (time.perf_counter_ns() - t0) / 1e6, out def summary(xs): ys = sorted(xs) return { 'n': len(xs), 'median_ms': round(statistics.median(xs), 1), 'mean_ms': round(statistics.mean(xs), 1), 'min_ms': round(min(xs), 1), 'max_ms': round(max(xs), 1), 'p95_ms': round(ys[min(len(ys)-1, int(.95*len(ys)))], 1), 'runs_ms': [round(x, 1) for x in xs], } def close_all(): for session in ('bench-ab-cold', 'bench-ab-warm'): e = env.copy(); e['AGENT_BROWSER_SESSION'] = session exec_cmd([str(AB), 'close', '--all'], e=e) exec_cmd(['node', str(PWCLI), 'cli', 'close-all']) result = { 'setup': { 'agent_browser': check_proc(exec_cmd([str(AB), '--version']), 'version').strip(), 'agent_browser_commit': '6dcea79b4b567a5671f1e1164807204f69542a5c', 'playwright': json.load(open('/tmp/playwright-1.62/node_modules/playwright/package.json'))['version'], 'chrome': check_proc(exec_cmd([str(CHROME), '--version']), 'chrome version').strip(), 'cold_runs': 10, 'cold_warmups': 2, 'warm_runs': 20, 'warm_warmups': 3, 'note': 'Deterministic localhost site. Same Chrome executable. Cold means fresh browser with each tool daemon already available; warm means one persistent browser. Playwright CLI measured at default 500ms settle and tuned 0ms settle.' }, 'cold': {}, 'warm': {} } try: close_all() tools = ['agent_browser', 'playwright_cli_default', 'playwright_cli_tuned', 'playwright_library'] runners = { 'agent_browser': lambda w: run_ab(w, False), 'playwright_cli_default': lambda w: run_pwcli('playwright_cli_default', w, False), 'playwright_cli_tuned': lambda w: run_pwcli('playwright_cli_tuned', w, False), 'playwright_library': run_pwlib, } for workflow in ('open', 'click', 'fill'): for tool in tools: for _ in range(2): runners[tool](workflow); time.sleep(.35) samples = {t: [] for t in tools} for i in range(10): order = tools[i % len(tools):] + tools[:i % len(tools)] for tool in order: ms, _ = timed(lambda t=tool: runners[t](workflow)) samples[tool].append(ms) time.sleep(.4) result['cold'][labels[workflow]] = {t: summary(samples[t]) for t in tools} # Start persistent browser sessions outside timing. eab = env.copy(); eab['AGENT_BROWSER_SESSION'] = 'bench-ab-warm' check_proc(exec_cmd([str(AB), 'open', url], e=eab), 'ab warm open') for tool, (config, outdir, session) in configs.items(): shutil.rmtree(outdir, ignore_errors=True); outdir.mkdir(parents=True, exist_ok=True) check_proc(exec_cmd(pwcli_cmd(session, 'open', url, '--config', str(config))), f'{tool} warm open') warm_runners = { 'agent_browser': lambda w: run_ab(w, True), 'playwright_cli_default': lambda w: run_pwcli('playwright_cli_default', w, True), 'playwright_cli_tuned': lambda w: run_pwcli('playwright_cli_tuned', w, True), } for workflow in ('open', 'click', 'fill'): for tool in warm_runners: for _ in range(3): warm_runners[tool](workflow) samples = {t: [] for t in warm_runners} for i in range(20): order = list(warm_runners)[i % 3:] + list(warm_runners)[:i % 3] for tool in order: ms, _ = timed(lambda t=tool: warm_runners[t](workflow)) samples[tool].append(ms) result['warm'][labels[workflow]] = {t: summary(samples[t]) for t in warm_runners} p = exec_cmd(['node', str(ROOT / 'pw_current_warm.mjs'), base, '20'], timeout=90) pw_warm_raw = json.loads(check_proc(p, 'pw warm')) for workflow in ('open', 'click', 'fill'): result['warm'][labels[workflow]]['playwright_library'] = summary(pw_warm_raw[workflow]) finally: close_all() server.shutdown() out = ROOT / 'current-v0331-vs-pw162-results.json' out.write_text(json.dumps(result, indent=2) + '\n') print(json.dumps(result, indent=2))