Showing preview only (736K chars total). Download the full file or copy to clipboard to get everything.
Repository: ntegrals/openbrowser
Branch: master
Commit: 622f36985df6
Files: 119
Total size: 697.5 KB
Directory structure:
gitextract_rxlca7z1/
├── .github/
│ ├── CONTRIBUTING.md
│ └── workflows/
│ └── ci.yml
├── .gitignore
├── LICENSE
├── README.md
├── biome.json
├── bunfig.toml
├── package.json
├── packages/
│ ├── cli/
│ │ ├── package.json
│ │ ├── src/
│ │ │ ├── commands/
│ │ │ │ ├── click.ts
│ │ │ │ ├── eval.ts
│ │ │ │ ├── extract.ts
│ │ │ │ ├── interactive.ts
│ │ │ │ ├── open.ts
│ │ │ │ ├── run.ts
│ │ │ │ ├── screenshot.ts
│ │ │ │ ├── sessions.ts
│ │ │ │ ├── state.ts
│ │ │ │ └── type.ts
│ │ │ ├── display.ts
│ │ │ ├── globals.ts
│ │ │ ├── index.ts
│ │ │ ├── protocol.ts
│ │ │ ├── server.ts
│ │ │ └── sessions.ts
│ │ └── tsconfig.json
│ ├── core/
│ │ ├── package.json
│ │ ├── src/
│ │ │ ├── agent/
│ │ │ │ ├── agent.test.ts
│ │ │ │ ├── agent.ts
│ │ │ │ ├── conversation/
│ │ │ │ │ ├── service.ts
│ │ │ │ │ ├── types.ts
│ │ │ │ │ └── utils.ts
│ │ │ │ ├── conversation.test.ts
│ │ │ │ ├── evaluator.ts
│ │ │ │ ├── index.ts
│ │ │ │ ├── instructions/
│ │ │ │ │ ├── instructions-compact.md
│ │ │ │ │ ├── instructions-direct.md
│ │ │ │ │ └── instructions.md
│ │ │ │ ├── instructions.ts
│ │ │ │ ├── replay-recorder.ts
│ │ │ │ ├── stall-detector.test.ts
│ │ │ │ ├── stall-detector.ts
│ │ │ │ └── types.ts
│ │ │ ├── bridge/
│ │ │ │ ├── adapter.ts
│ │ │ │ ├── client.ts
│ │ │ │ ├── index.ts
│ │ │ │ ├── mcp-types.ts
│ │ │ │ ├── server.test.ts
│ │ │ │ └── server.ts
│ │ │ ├── commands/
│ │ │ │ ├── catalog/
│ │ │ │ │ ├── catalog.ts
│ │ │ │ │ └── types.ts
│ │ │ │ ├── catalog.test.ts
│ │ │ │ ├── executor.test.ts
│ │ │ │ ├── executor.ts
│ │ │ │ ├── extraction/
│ │ │ │ │ └── extractor.ts
│ │ │ │ ├── index.ts
│ │ │ │ ├── types.ts
│ │ │ │ └── utils.ts
│ │ │ ├── config/
│ │ │ │ ├── config.ts
│ │ │ │ ├── index.ts
│ │ │ │ └── types.ts
│ │ │ ├── errors.ts
│ │ │ ├── index.ts
│ │ │ ├── logging.ts
│ │ │ ├── metering/
│ │ │ │ ├── index.ts
│ │ │ │ ├── tracker.test.ts
│ │ │ │ ├── tracker.ts
│ │ │ │ └── types.ts
│ │ │ ├── model/
│ │ │ │ ├── adapters/
│ │ │ │ │ └── vercel.ts
│ │ │ │ ├── index.ts
│ │ │ │ ├── interface.ts
│ │ │ │ ├── messages.ts
│ │ │ │ ├── schema-optimizer.ts
│ │ │ │ └── types.ts
│ │ │ ├── page/
│ │ │ │ ├── content-extractor.ts
│ │ │ │ ├── index.ts
│ │ │ │ ├── page-analyzer.test.ts
│ │ │ │ ├── page-analyzer.ts
│ │ │ │ ├── renderer/
│ │ │ │ │ ├── interactive-elements.ts
│ │ │ │ │ ├── layer-order.ts
│ │ │ │ │ └── tree-renderer.ts
│ │ │ │ ├── renderer.test.ts
│ │ │ │ ├── snapshot-builder.ts
│ │ │ │ └── types.ts
│ │ │ ├── sandbox/
│ │ │ │ ├── file-access.ts
│ │ │ │ └── index.ts
│ │ │ ├── telemetry.ts
│ │ │ ├── types.ts
│ │ │ ├── utils.ts
│ │ │ └── viewport/
│ │ │ ├── event-hub.ts
│ │ │ ├── events.ts
│ │ │ ├── guard-base.ts
│ │ │ ├── guards/
│ │ │ │ ├── blank-page.ts
│ │ │ │ ├── crash.ts
│ │ │ │ ├── default-handler.ts
│ │ │ │ ├── downloads.ts
│ │ │ │ ├── har-capture.ts
│ │ │ │ ├── local-instance.ts
│ │ │ │ ├── page-ready.ts
│ │ │ │ ├── permissions.ts
│ │ │ │ ├── persistence.ts
│ │ │ │ ├── popups.ts
│ │ │ │ ├── screenshot.ts
│ │ │ │ ├── url-policy.ts
│ │ │ │ └── video-capture.ts
│ │ │ ├── index.ts
│ │ │ ├── launch-profile.test.ts
│ │ │ ├── launch-profile.ts
│ │ │ ├── types.ts
│ │ │ ├── viewport.ts
│ │ │ └── visual-tracer.ts
│ │ └── tsconfig.json
│ └── sandbox/
│ ├── package.json
│ ├── src/
│ │ ├── index.ts
│ │ ├── sandbox.ts
│ │ └── types.ts
│ └── tsconfig.json
├── tsconfig.base.json
└── tsconfig.json
================================================
FILE CONTENTS
================================================
================================================
FILE: .github/CONTRIBUTING.md
================================================
# Contributing to Open Browser
Thank you for your interest in contributing!
## Getting Started
1. Fork the repository
2. Clone your fork: `git clone https://github.com/YOUR_USERNAME/openbrowser.git`
3. Install dependencies: `bun install`
4. Create a branch: `git checkout -b my-feature`
5. Make your changes and add tests
6. Run tests: `bun run test`
7. Submit a pull request
## Code Style
We use [Biome](https://biomejs.dev/) for formatting and linting. Run `bun run format` before committing.
## Reporting Issues
Please use GitHub Issues to report bugs or request features. Include:
- Steps to reproduce
- Expected vs actual behavior
- Browser and OS version
================================================
FILE: .github/workflows/ci.yml
================================================
name: CI
on:
push:
branches: [main]
pull_request:
branches: [main]
jobs:
test:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: oven-sh/setup-bun@v2
- run: bun install
- run: bun run build
- run: bun run test
- run: bun run lint
================================================
FILE: .gitignore
================================================
node_modules/
dist/
.env
*.tsbuildinfo
.DS_Store
traces/
coverage/
recordings/
tmp/
*.log
================================================
FILE: LICENSE
================================================
MIT License
Copyright (c) 2024-2026 Open Browser Contributors
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
================================================
FILE: README.md
================================================
<h1 align="center">Open Browser</h1>
<p align="center">
<b>AI-powered autonomous web browsing framework for TypeScript.</b>
</p>
<p align="center">
<a href="https://github.com/ntegrals/openbrowser/blob/main/LICENSE"><img src="https://img.shields.io/badge/license-MIT-blue.svg" alt="License"></a>
<a href="https://github.com/ntegrals/openbrowser"><img src="https://img.shields.io/github/stars/ntegrals/openbrowser?style=social" alt="GitHub stars"></a>
</p>
<img src="./media/header.png" alt="Header"></a>
---
Give an AI agent a browser. It clicks, types, navigates, and extracts data — autonomously completing tasks on any website. Built on Playwright with first-class support for OpenAI, Anthropic, and Google models.
> **Production-ready since v1.0.** Contributions welcome.
## Why Open Browser?
- **Autonomous agents**: Describe a task in natural language, and an AI agent navigates the web to complete it — clicking, typing, scrolling, and extracting data without manual scripting
- **Multi-model support**: Works with OpenAI, Anthropic, and Google out of the box via the Vercel AI SDK — swap models with a single flag
- **Interactive REPL**: Drop into a live browser session and issue commands interactively — great for debugging, prototyping, and exploration
- **Sandboxed execution**: Run agents in resource-limited environments with CPU/memory monitoring, timeouts, and domain restrictions
- **Production-ready**: Stall detection, cost tracking, session management, replay recording, and comprehensive error handling
- **Open source**: MIT licensed, fully extensible, bring your own API keys
## Quick Start
```bash
# Install dependencies
bun install
# Set up your API keys
cp .env.example .env
# Edit .env with your API keys
# Run an agent
bun run open-browser run "Find the top story on Hacker News and summarize it"
# Or open a browser interactively
bun run open-browser interactive
```
## Architecture
Open Browser is a monorepo with three packages:
| Package | Description |
| --------------------------- | -------------------------------------------------------------------------- |
| **`open-browser`** | Core library — agent logic, browser control, DOM analysis, LLM integration |
| **`@open-browser/cli`** | Command-line interface for running agents and browser commands |
| **`@open-browser/sandbox`** | Sandboxed execution with resource limits and monitoring |
## CLI Commands
### Run an AI Agent
```bash
open-browser run <task> [options]
```
Describe what you want done. The agent figures out the rest.
```bash
# Search and extract information
open-browser run "Find the price of the MacBook Pro on apple.com"
# Fill out forms
open-browser run "Sign up for the newsletter on example.com with test@email.com"
# Multi-step workflows
open-browser run "Go to GitHub, find the open-browser repo, and star it"
```
| Option | Description |
| ---------------------------- | ----------------------------------------- |
| `-m, --model <model>` | Model to use (default: `gpt-4o`) |
| `-p, --provider <provider>` | Provider: `openai`, `anthropic`, `google` |
| `--headless / --no-headless` | Show or hide the browser window |
| `--max-steps <n>` | Max agent steps (default: `25`) |
| `-v, --verbose` | Show detailed step info |
| `--no-cost` | Hide cost tracking |
### Browser Commands
```bash
open-browser open <url> # Open a URL
open-browser click <selector> # Click an element
open-browser type <selector> <text> # Type into an input
open-browser screenshot [output] # Capture a screenshot
open-browser eval <expression> # Run JavaScript on the page
open-browser extract <goal> # Extract content as markdown
open-browser state # Show current URL, title, and tabs
open-browser sessions # List active browser sessions
```
### Interactive REPL
```bash
open-browser interactive
```
Drop into a live `browser>` prompt with full control:
```
browser> open https://news.ycombinator.com
browser> extract "top 5 stories with titles and points"
browser> click .morelink
browser> screenshot front-page.png
browser> help
```
## Using as a Library
```typescript
import { Agent, createViewport, createModel } from 'open-browser'
const viewport = await createViewport({ headless: true })
const model = createModel('openai', 'gpt-4o')
const agent = new Agent({
viewport,
model,
task: 'Go to example.com and extract the main heading',
settings: {
stepLimit: 50,
enableScreenshots: true,
},
})
const result = await agent.run()
console.log(result)
```
### Sandboxed Execution
Run agents with resource limits and monitoring:
```typescript
import { Sandbox } from '@open-browser/sandbox'
const sandbox = new Sandbox({
timeout: 300_000, // 5 minute timeout
maxMemoryMB: 512, // Memory limit
allowedDomains: ['example.com'],
stepLimit: 100,
captureOutput: true,
})
const result = await sandbox.run({
task: 'Complete the checkout form',
model: languageModel,
})
console.log(result.metrics) // steps, URLs visited, CPU time
```
## Configuration
### Environment Variables
```bash
# LLM Provider Keys (at least one required)
OPENAI_API_KEY=sk-...
ANTHROPIC_API_KEY=sk-ant-...
GOOGLE_GENERATIVE_AI_API_KEY=...
# Browser
BROWSER_HEADLESS=true
BROWSER_DISABLE_SECURITY=false
# Recording & Debugging
OPEN_BROWSER_TRACE_PATH=./traces
OPEN_BROWSER_SAVE_RECORDING_PATH=./recordings
```
### Agent Configuration
| Setting | Default | Description |
| ------------------- | -------- | ----------------------------------------- |
| `stepLimit` | `100` | Maximum agent iterations |
| `commandsPerStep` | `10` | Actions per agent step |
| `failureThreshold` | `5` | Consecutive failures before stopping |
| `enableScreenshots` | `true` | Include page screenshots in agent context |
| `contextWindowSize` | `128000` | Token budget for conversation |
| `allowedUrls` | `[]` | Restrict navigation to specific URLs |
| `blockedUrls` | `[]` | Block navigation to specific URLs |
### Viewport Configuration
| Setting | Default | Description |
| ------------------ | --------------- | ------------------------------------------- |
| `headless` | `true` | Run browser without visible window |
| `width` / `height` | `1280` / `1100` | Browser window dimensions |
| `relaxedSecurity` | `false` | Disable browser security features |
| `proxy` | — | Proxy server configuration |
| `cookieFile` | — | Path to cookie file for persistent sessions |
## How It Works
```
┌─────────────┐
"Book a flight" │ │
───────────────► │ Agent │ ◄── LLM (OpenAI / Anthropic / Google)
│ │
└──────┬──────┘
│
┌──────▼──────┐
│ Commands │ click, type, scroll, extract, navigate...
└──────┬──────┘
│
┌──────▼──────┐
│ Viewport │ Playwright browser instance
└──────┬──────┘
│
┌──────▼──────┐
│ DOM / Page │ Snapshot, interactive elements, content
└─────────────┘
```
1. You describe a **task** in natural language
2. The **Agent** sends the current page state + task to an LLM
3. The LLM decides what **commands** to execute (click, type, navigate, extract...)
4. Commands execute against the **Viewport** (Playwright browser)
5. The agent observes the result, detects stalls, and loops until the task is complete
## Model Support
| Provider | Example Models | Flag |
| ------------- | ----------------------------------------------- | -------------- |
| **OpenAI** | `gpt-4o`, `gpt-4o-mini`, `o1` | `-p openai` |
| **Anthropic** | `claude-sonnet-4-5-20250929`, `claude-opus-4-6` | `-p anthropic` |
| **Google** | `gemini-2.0-flash`, `gemini-2.5-pro` | `-p google` |
## Project Structure
```
packages/
├── core/ # Core library (open-browser)
│ └── src/
│ ├── agent/ # Agent logic, conversation, stall detection
│ ├── commands/ # Action schemas and executor (25+ commands)
│ ├── viewport/ # Browser control, events, guards
│ ├── page/ # DOM analysis, content extraction
│ ├── model/ # LLM adapter and message formatting
│ ├── metering/ # Cost tracking
│ ├── bridge/ # IPC server/client
│ └── config/ # Configuration types
├── cli/ # CLI (@open-browser/cli)
│ └── src/
│ ├── commands/ # CLI command implementations
│ └── index.ts # Entry point
└── sandbox/ # Sandbox (@open-browser/sandbox)
└── src/
└── sandbox.ts # Resource-limited execution
```
## Development
```bash
# Install dependencies
bun install
# Type check
bun run build
# Run tests
bun run test
# Lint
bun run lint
# Format
bun run format
```
## Contributing
Contributions are welcome! Please see [CONTRIBUTING.md](.github/CONTRIBUTING.md) for guidelines.
## License
[MIT](LICENSE)
================================================
FILE: biome.json
================================================
{
"$schema": "https://biomejs.dev/schemas/1.9.0/schema.json",
"organizeImports": {
"enabled": true
},
"linter": {
"enabled": true,
"rules": {
"recommended": true,
"complexity": {
"noForEach": "off"
},
"style": {
"noNonNullAssertion": "off",
"useConst": "warn"
},
"suspicious": {
"noExplicitAny": "off"
}
}
},
"formatter": {
"enabled": true,
"indentStyle": "tab",
"indentWidth": 2,
"lineWidth": 120
},
"javascript": {
"formatter": {
"quoteStyle": "single",
"semicolons": "always",
"trailingCommas": "all"
}
},
"files": {
"ignore": ["node_modules", "dist", "*.json", "*.d.ts"]
}
}
================================================
FILE: bunfig.toml
================================================
[install]
peer = false
[test]
timeout = 60000
================================================
FILE: package.json
================================================
{
"name": "open-browser-monorepo",
"private": true,
"workspaces": ["packages/*"],
"scripts": {
"build": "bun run --filter '*' build",
"test": "bun run --filter '*' test",
"lint": "biome check .",
"format": "biome format --write ."
},
"devDependencies": {
"@biomejs/biome": "^1.9.4",
"@types/bun": "^1.2.0",
"typescript": "^5.8.0"
},
"trustedDependencies": [
"@biomejs/biome"
]
}
================================================
FILE: packages/cli/package.json
================================================
{
"name": "@open-browser/cli",
"version": "1.1.0",
"description": "CLI for Open Browser - AI-powered autonomous web browsing",
"type": "module",
"main": "src/index.ts",
"bin": {
"open-browser": "src/index.ts"
},
"scripts": {
"build": "tsc --noEmit",
"test": "bun test",
"start": "bun run src/index.ts"
},
"dependencies": {
"open-browser": "workspace:*",
"commander": "^12.1.0",
"chalk": "^5.4.0"
},
"license": "MIT"
}
================================================
FILE: packages/cli/src/commands/click.ts
================================================
import type { Command } from 'commander';
import chalk from 'chalk';
import { sessionManager } from '../globals.js';
export function registerClickCommand(program: Command): void {
program
.command('click')
.description('Click on an element matching the given CSS selector')
.argument('<selector>', 'CSS selector of the element to click')
.option('-s, --session <id>', 'Session ID to use')
.action(async (selector: string, options: { session?: string }) => {
try {
const browser = options.session
? sessionManager.get(options.session)
: sessionManager.getDefault();
if (!browser) {
console.error(chalk.red('No active session. Use "open" command first.'));
process.exit(1);
}
await browser.click(selector);
console.log(chalk.green('Clicked:'), selector);
} catch (error) {
console.error(chalk.red('Failed to click:'), error instanceof Error ? error.message : String(error));
process.exit(1);
}
});
}
================================================
FILE: packages/cli/src/commands/eval.ts
================================================
import type { Command } from 'commander';
import chalk from 'chalk';
import { sessionManager } from '../globals.js';
export function registerEvalCommand(program: Command): void {
program
.command('eval')
.description('Evaluate a JavaScript expression in the browser')
.argument('<expression>', 'JavaScript expression to evaluate')
.option('-s, --session <id>', 'Session ID to use')
.action(async (expression: string, options: { session?: string }) => {
try {
const browser = options.session
? sessionManager.get(options.session)
: sessionManager.getDefault();
if (!browser) {
console.error(chalk.red('No active session. Use "open" command first.'));
process.exit(1);
}
const result = await browser.evaluate(expression);
if (result === undefined) {
console.log(chalk.dim('undefined'));
} else if (result === null) {
console.log(chalk.dim('null'));
} else if (typeof result === 'object') {
console.log(JSON.stringify(result, null, 2));
} else {
console.log(String(result));
}
} catch (error) {
console.error(chalk.red('Evaluation failed:'), error instanceof Error ? error.message : String(error));
process.exit(1);
}
});
}
================================================
FILE: packages/cli/src/commands/extract.ts
================================================
import type { Command } from 'commander';
import chalk from 'chalk';
import { extractMarkdown } from 'open-browser';
import { sessionManager } from '../globals.js';
export function registerExtractCommand(program: Command): void {
program
.command('extract')
.description('Extract content from the current page as markdown')
.argument('<goal>', 'Description of what to extract (used as a label)')
.option('-s, --session <id>', 'Session ID to use')
.action(async (goal: string, options: { session?: string }) => {
try {
const browser = options.session
? sessionManager.get(options.session)
: sessionManager.getDefault();
if (!browser) {
console.error(chalk.red('No active session. Use "open" command first.'));
process.exit(1);
}
console.log(chalk.dim(`Extracting: ${goal}`));
const markdown = await extractMarkdown(browser.currentPage);
if (!markdown) {
console.log(chalk.yellow('No content extracted from the page.'));
} else {
console.log(markdown);
}
} catch (error) {
console.error(chalk.red('Extraction failed:'), error instanceof Error ? error.message : String(error));
process.exit(1);
}
});
}
================================================
FILE: packages/cli/src/commands/interactive.ts
================================================
import * as readline from 'node:readline';
import type { Command } from 'commander';
import chalk from 'chalk';
import {
Viewport,
extractMarkdown,
} from 'open-browser';
import {
Spinner,
displayInfo,
displayError,
displaySeparator,
} from '../display.js';
interface InteractiveOptions {
headless: boolean;
}
/**
* Interactive REPL-like session for browser automation.
* Supports commands: open, click, type, eval, extract, screenshot, state, back, forward, tabs, help, quit
*/
export function registerInteractiveCommand(program: Command): void {
program
.command('interactive')
.alias('repl')
.description('Start an interactive browser session (REPL mode)')
.option('--headless', 'Run browser in headless mode', false)
.action(async (options: InteractiveOptions) => {
console.log(chalk.bold.white('Interactive Browser Session'));
console.log(chalk.dim('Type "help" for available commands, "quit" to exit.'));
displaySeparator();
let browser: Viewport | null = null;
try {
const spinner = new Spinner('Starting browser...');
spinner.start();
browser = new Viewport({
headless: options.headless,
});
await browser.start();
spinner.stop(chalk.green('Browser ready.'));
console.log('');
const rl = readline.createInterface({
input: process.stdin,
output: process.stdout,
prompt: chalk.cyan('browser> '),
terminal: true,
});
rl.prompt();
rl.on('line', async (line) => {
const trimmed = line.trim();
if (!trimmed) {
rl.prompt();
return;
}
const [command, ...args] = parseCommandLine(trimmed);
try {
const shouldQuit = await handleCommand(
command.toLowerCase(),
args,
browser!,
);
if (shouldQuit) {
rl.close();
return;
}
} catch (error) {
displayError(
error instanceof Error ? error.message : String(error),
);
}
rl.prompt();
});
rl.on('close', async () => {
console.log('');
displayInfo('Closing browser session...');
if (browser) {
await browser.close().catch(() => {});
}
process.exit(0);
});
} catch (error) {
displayError(
error instanceof Error ? error.message : String(error),
);
if (browser) {
await browser.close().catch(() => {});
}
process.exit(1);
}
});
}
// ── Command Parsing ──
function parseCommandLine(input: string): string[] {
const tokens: string[] = [];
let current = '';
let inQuote: string | null = null;
for (const char of input) {
if (inQuote) {
if (char === inQuote) {
inQuote = null;
} else {
current += char;
}
} else if (char === '"' || char === "'") {
inQuote = char;
} else if (char === ' ' || char === '\t') {
if (current) {
tokens.push(current);
current = '';
}
} else {
current += char;
}
}
if (current) {
tokens.push(current);
}
return tokens;
}
// ── Command Handler ──
async function handleCommand(
command: string,
args: string[],
browser: Viewport,
): Promise<boolean> {
switch (command) {
case 'open':
case 'goto':
case 'navigate': {
const url = args[0];
if (!url) {
displayError('Usage: open <url>');
return false;
}
const spinner = new Spinner(`Navigating to ${url}...`);
spinner.start();
await browser.navigate(url);
const finalUrl = browser.currentPage.url();
spinner.stop(`${chalk.green('Loaded:')} ${finalUrl}`);
return false;
}
case 'tap': {
const selector = args.join(' ');
if (!selector) {
displayError('Usage: click <selector>');
return false;
}
await browser.click(selector);
console.log(chalk.green('Clicked:'), selector);
return false;
}
case 'type': {
const selector = args[0];
const text = args.slice(1).join(' ');
if (!selector || !text) {
displayError('Usage: type <selector> <text>');
return false;
}
await browser.type(selector, text);
console.log(chalk.green('Typed:'), text);
return false;
}
case 'eval':
case 'js': {
const expression = args.join(' ');
if (!expression) {
displayError('Usage: eval <expression>');
return false;
}
const result = await browser.evaluate(expression);
if (result === undefined) {
console.log(chalk.dim('undefined'));
} else if (result === null) {
console.log(chalk.dim('null'));
} else if (typeof result === 'object') {
console.log(JSON.stringify(result, null, 2));
} else {
console.log(String(result));
}
return false;
}
case 'extract':
case 'markdown': {
const spinner = new Spinner('Extracting page content...');
spinner.start();
const markdown = await extractMarkdown(browser.currentPage);
spinner.stop();
if (markdown) {
// Show first 2000 chars
const preview = markdown.length > 2000
? `${markdown.slice(0, 2000)}\n${chalk.dim(`... (${markdown.length} chars total)`)}`
: markdown;
console.log(preview);
} else {
console.log(chalk.yellow('No content found.'));
}
return false;
}
case 'capture': {
const outputPath = args[0] || 'screenshot.png';
const result = await browser.screenshot(false);
const fs = await import('node:fs');
const path = await import('node:path');
const buffer = Buffer.from(result.base64, 'base64');
const resolved = path.resolve(outputPath);
fs.writeFileSync(resolved, buffer);
console.log(chalk.green('Screenshot saved:'), resolved);
console.log(chalk.dim(`${result.width}x${result.height}`));
return false;
}
case 'state':
case 'info': {
const state = await browser.getState();
console.log(`${chalk.white('URL:')} ${state.url}`);
console.log(`${chalk.white('Title:')} ${state.title}`);
if (state.tabs.length > 1) {
console.log(`${chalk.white('Tabs:')}`);
for (const tab of state.tabs) {
const marker = tab.isActive ? chalk.cyan(' > ') : ' ';
console.log(`${marker}[${tab.tabId}] ${tab.title || '(untitled)'} - ${tab.url}`);
}
}
return false;
}
case 'back': {
await browser.currentPage.goBack({ timeout: 5000 }).catch(() => {});
console.log(chalk.green('Navigated back'));
return false;
}
case 'forward': {
await browser.currentPage.goForward({ timeout: 5000 }).catch(() => {});
console.log(chalk.green('Navigated forward'));
return false;
}
case 'tabs': {
const state = await browser.getState();
for (const tab of state.tabs) {
const marker = tab.isActive ? chalk.cyan(' > ') : ' ';
console.log(`${marker}[${tab.tabId}] ${tab.title || '(untitled)'} - ${tab.url}`);
}
return false;
}
case 'url': {
console.log(browser.currentPage.url());
return false;
}
case 'title': {
const title = await browser.currentPage.title();
console.log(title);
return false;
}
case 'reload':
case 'refresh': {
await browser.currentPage.reload({ timeout: 10000 }).catch(() => {});
console.log(chalk.green('Page reloaded'));
return false;
}
case 'wait': {
const ms = Number.parseInt(args[0] || '1000', 10);
console.log(chalk.dim(`Waiting ${ms}ms...`));
await new Promise((resolve) => setTimeout(resolve, ms));
return false;
}
case 'help': {
printHelp();
return false;
}
case 'quit':
case 'exit':
case 'q': {
return true;
}
default: {
console.log(chalk.yellow(`Unknown command: ${command}`));
console.log(chalk.dim('Type "help" for available commands.'));
return false;
}
}
}
function printHelp(): void {
console.log(chalk.bold('Available commands:'));
console.log('');
const commands = [
['open <url>', 'Navigate to a URL'],
['click <selector>', 'Click an element'],
['type <selector> <text>', 'Type text into an element'],
['eval <expression>', 'Run JavaScript in the browser'],
['extract', 'Extract page content as markdown'],
['screenshot [path]', 'Take a screenshot'],
['state', 'Show current browser state'],
['back', 'Navigate back'],
['forward', 'Navigate forward'],
['tabs', 'List open tabs'],
['url', 'Show current URL'],
['title', 'Show current page title'],
['reload', 'Reload the current page'],
['wait [ms]', 'Wait for the specified time'],
['help', 'Show this help message'],
['quit', 'Exit the interactive session'],
];
for (const [cmd, desc] of commands) {
console.log(` ${chalk.cyan(cmd.padEnd(25))} ${desc}`);
}
}
================================================
FILE: packages/cli/src/commands/open.ts
================================================
import type { Command } from 'commander';
import chalk from 'chalk';
import { sessionManager } from '../globals.js';
export function registerOpenCommand(program: Command): void {
program
.command('open')
.description('Open a URL in the browser')
.argument('<url>', 'URL to navigate to')
.option('--headless', 'Run in headless mode', false)
.option('-s, --session <id>', 'Reuse an existing session')
.action(async (url: string, options: { headless: boolean; session?: string }) => {
try {
let sessionId = options.session;
if (sessionId) {
const browser = sessionManager.get(sessionId);
if (!browser) {
console.error(chalk.red(`Session "${sessionId}" not found.`));
process.exit(1);
}
await browser.navigate(url);
} else {
// Try to reuse the default session, or create a new one
sessionId = sessionManager.getDefaultId();
if (!sessionId) {
sessionId = await sessionManager.create({
headless: options.headless,
});
}
const browser = sessionManager.get(sessionId)!;
await browser.navigate(url);
}
const browser = sessionManager.get(sessionId)!;
const finalUrl = browser.currentPage.url();
console.log(chalk.green('Session:'), sessionId);
console.log(chalk.green('URL:'), finalUrl);
} catch (error) {
console.error(chalk.red('Failed to open URL:'), error instanceof Error ? error.message : String(error));
process.exit(1);
}
});
}
================================================
FILE: packages/cli/src/commands/run.ts
================================================
import type { Command } from 'commander';
import chalk from 'chalk';
import {
Agent,
Viewport,
VercelModelAdapter,
type LanguageModel,
type CommandResult,
type StepRecord,
} from 'open-browser';
import {
Spinner,
displayStep,
displayTotalCost,
displayResult,
displayHeader,
displaySeparator,
displayError,
} from '../display.js';
interface RunOptions {
model: string;
provider: string;
headless: boolean;
stepLimit: number;
verbose: boolean;
noCost: boolean;
}
/**
* Dynamically import and create a Vercel AI SDK language model
* based on the provider and model ID strings.
*/
async function createModel(provider: string, modelId: string): Promise<LanguageModel> {
let languageModel: import('ai').LanguageModelV1;
switch (provider) {
case 'openai': {
const { createOpenAI } = await import('@ai-sdk/openai');
const openai = createOpenAI({});
languageModel = openai(modelId);
break;
}
case 'anthropic': {
const { createAnthropic } = await import('@ai-sdk/anthropic');
const anthropic = createAnthropic({});
languageModel = anthropic(modelId);
break;
}
case 'google': {
const { createGoogleGenerativeAI } = await import('@ai-sdk/google');
const google = createGoogleGenerativeAI({});
languageModel = google(modelId);
break;
}
default:
throw new Error(
`Unsupported provider: ${provider}. ` +
'Supported: openai, anthropic, google',
);
}
return new VercelModelAdapter({ model: languageModel });
}
export function registerRunCommand(program: Command): void {
program
.command('run')
.description('Run an AI agent to complete a browser task')
.argument('<task>', 'Description of the task for the agent to complete')
.option('-m, --model <model>', 'Model ID to use', 'gpt-4o')
.option('-p, --provider <provider>', 'LLM provider (openai, anthropic, google)', 'openai')
.option('--headless', 'Run browser in headless mode', true)
.option('--no-headless', 'Show the browser window')
.option('--max-steps <n>', 'Maximum number of agent steps', '25')
.option('-v, --verbose', 'Show detailed step information', false)
.option('--no-cost', 'Hide cost tracking information')
.action(async (task: string, options: RunOptions) => {
const stepLimit = Number.parseInt(String(options.stepLimit), 10);
displayHeader(`Agent Task: ${task}`);
console.log(
`${chalk.dim('model:')} ${options.model} ` +
`${chalk.dim('provider:')} ${options.provider} ` +
`${chalk.dim('max steps:')} ${stepLimit}`,
);
displaySeparator();
const spinner = new Spinner('Starting browser...');
spinner.start();
let browser: Viewport | null = null;
try {
// Initialize the LLM
spinner.update('Loading model...');
const model = await createModel(options.provider, options.model);
// Initialize the browser
spinner.update('Starting browser...');
browser = new Viewport({
headless: options.headless,
});
await browser.start();
spinner.update('Browser ready, starting agent...');
// Track per-step timing
const stepTimings = new Map<number, number>();
let currentStepStart = 0;
// Create the agent
const agent = new Agent({
task,
model,
browser,
settings: {
stepLimit,
},
onStepStart: (step) => {
currentStepStart = Date.now();
stepTimings.set(step, currentStepStart);
spinner.update(`Step ${step}: thinking...`);
},
onStepEnd: (step, results) => {
const durationMs = Date.now() - (stepTimings.get(step) ?? currentStepStart);
spinner.stop();
// Display each action result for this step
for (const result of results) {
displayStep({
step,
action: extractActionName(result),
target: extractActionTarget(result),
durationMs,
success: result.success,
error: result.error,
extractedContent: result.extractedContent,
});
}
if (options.verbose) {
displaySeparator();
}
// Restart spinner for next step
spinner.start();
spinner.update(`Step ${step + 1}: thinking...`);
},
});
spinner.update('Agent running...');
// Execute the agent
const result = await agent.run();
spinner.stop();
// Display result
displayResult(result.success, result.finalResult);
// Display cost summary
if (!options.noCost && result.totalCost) {
displayTotalCost({
steps: result.history.entries.length,
inputTokens: result.totalCost.totalInputTokens,
outputTokens: result.totalCost.totalOutputTokens,
totalCost: result.totalCost.totalCost,
durationMs: computeTotalDuration(result.history.entries),
});
} else if (!options.noCost) {
// Show basic timing even without cost data
const totalMs = computeTotalDuration(result.history.entries);
console.log('');
console.log(
chalk.dim(
`Completed in ${result.history.entries.length} step(s), ` +
`${(totalMs / 1000).toFixed(1)}s`,
),
);
}
// Display errors if any
if (result.errors.length > 0) {
console.log('');
console.log(chalk.bold.yellow('Errors encountered:'));
for (const err of result.errors) {
console.log(` ${chalk.red('-')} ${err}`);
}
}
// Exit with appropriate code
process.exit(result.success ? 0 : 1);
} catch (error) {
spinner.stop();
displayError(
error instanceof Error ? error.message : String(error),
);
process.exit(1);
} finally {
if (browser) {
await browser.close().catch(() => {});
}
}
});
}
// ── Helpers ──
function extractActionName(result: CommandResult): string {
if (result.isDone) return 'done';
if (result.extractedContent) return 'extract';
return result.success ? 'action' : 'failed_action';
}
function extractActionTarget(result: CommandResult): string | undefined {
if (result.extractedContent) {
return result.extractedContent.slice(0, 80);
}
return undefined;
}
function computeTotalDuration(entries: StepRecord[]): number {
return entries.reduce((sum, e) => sum + e.duration, 0);
}
================================================
FILE: packages/cli/src/commands/screenshot.ts
================================================
import type { Command } from 'commander';
import chalk from 'chalk';
import * as fs from 'node:fs';
import * as path from 'node:path';
import { sessionManager } from '../globals.js';
export function registerScreenshotCommand(program: Command): void {
program
.command('screenshot')
.description('Take a screenshot of the current page')
.argument('[output]', 'Output file path', 'screenshot.png')
.option('-s, --session <id>', 'Session ID to use')
.option('--full-page', 'Capture the full page', false)
.action(async (output: string, options: { session?: string; fullPage: boolean }) => {
try {
const browser = options.session
? sessionManager.get(options.session)
: sessionManager.getDefault();
if (!browser) {
console.error(chalk.red('No active session. Use "open" command first.'));
process.exit(1);
}
const result = await browser.screenshot(options.fullPage);
const buffer = Buffer.from(result.base64, 'base64');
const outputPath = path.resolve(output);
fs.writeFileSync(outputPath, buffer);
console.log(chalk.green('Screenshot saved:'), outputPath);
console.log(chalk.green('Dimensions:'), `${result.width}x${result.height}`);
} catch (error) {
console.error(chalk.red('Failed to take screenshot:'), error instanceof Error ? error.message : String(error));
process.exit(1);
}
});
}
================================================
FILE: packages/cli/src/commands/sessions.ts
================================================
import type { Command } from 'commander';
import chalk from 'chalk';
import { sessionManager } from '../globals.js';
export function registerSessionsCommand(program: Command): void {
program
.command('sessions')
.description('List all active browser sessions')
.action(() => {
try {
const sessions = sessionManager.list();
if (sessions.length === 0) {
console.log(chalk.yellow('No active sessions.'));
return;
}
console.log(chalk.bold(`Active Sessions (${sessions.length}):`));
for (const session of sessions) {
const created = new Date(session.createdAt).toLocaleTimeString();
const accessed = new Date(session.lastAccessedAt).toLocaleTimeString();
console.log(` ${chalk.cyan(session.id)} created ${created} last used ${accessed}`);
}
} catch (error) {
console.error(chalk.red('Failed to list sessions:'), error instanceof Error ? error.message : String(error));
process.exit(1);
}
});
program
.command('sessions:close')
.description('Close a specific session or all sessions')
.argument('[id]', 'Session ID to close (omit to close all)')
.action(async (id?: string) => {
try {
if (id) {
const closed = await sessionManager.close(id);
if (closed) {
console.log(chalk.green('Closed session:'), id);
} else {
console.error(chalk.red(`Session "${id}" not found.`));
process.exit(1);
}
} else {
const count = sessionManager.activeCount;
await sessionManager.closeAll();
console.log(chalk.green(`Closed ${count} session(s).`));
}
} catch (error) {
console.error(chalk.red('Failed to close session:'), error instanceof Error ? error.message : String(error));
process.exit(1);
}
});
}
================================================
FILE: packages/cli/src/commands/state.ts
================================================
import type { Command } from 'commander';
import chalk from 'chalk';
import { sessionManager } from '../globals.js';
export function registerStateCommand(program: Command): void {
program
.command('state')
.description('Print the current browser state (URL, title, tabs)')
.option('-s, --session <id>', 'Session ID to use')
.action(async (options: { session?: string }) => {
try {
const browser = options.session
? sessionManager.get(options.session)
: sessionManager.getDefault();
if (!browser) {
console.error(chalk.red('No active session. Use "open" command first.'));
process.exit(1);
}
const state = await browser.getState();
console.log(chalk.bold('Browser State'));
console.log(chalk.green('URL:'), state.url);
console.log(chalk.green('Title:'), state.title);
console.log(chalk.green('Tabs:'), state.tabs.length);
for (const tab of state.tabs) {
const marker = tab.isActive ? chalk.cyan('→') : ' ';
console.log(` ${marker} [${tab.tabId}] ${tab.title || '(untitled)'} - ${tab.url}`);
}
} catch (error) {
console.error(chalk.red('Failed to get state:'), error instanceof Error ? error.message : String(error));
process.exit(1);
}
});
}
================================================
FILE: packages/cli/src/commands/type.ts
================================================
import type { Command } from 'commander';
import chalk from 'chalk';
import { sessionManager } from '../globals.js';
export function registerTypeCommand(program: Command): void {
program
.command('type')
.description('Type text into an element matching the given CSS selector')
.argument('<selector>', 'CSS selector of the input element')
.argument('<text>', 'Text to type into the element')
.option('-s, --session <id>', 'Session ID to use')
.action(async (selector: string, text: string, options: { session?: string }) => {
try {
const browser = options.session
? sessionManager.get(options.session)
: sessionManager.getDefault();
if (!browser) {
console.error(chalk.red('No active session. Use "open" command first.'));
process.exit(1);
}
await browser.type(selector, text);
console.log(chalk.green('Typed into:'), selector);
} catch (error) {
console.error(chalk.red('Failed to type:'), error instanceof Error ? error.message : String(error));
process.exit(1);
}
});
}
================================================
FILE: packages/cli/src/display.ts
================================================
import chalk from 'chalk';
// ── Spinner ──
const SPINNER_FRAMES = ['⠋', '⠙', '⠹', '⠸', '⠼', '⠴', '⠦', '⠧', '⠇', '⠏'];
export class Spinner {
private intervalId: ReturnType<typeof setInterval> | null = null;
private frameIndex = 0;
private message: string;
constructor(message: string) {
this.message = message;
}
start(): void {
if (this.intervalId) return;
this.frameIndex = 0;
this.intervalId = setInterval(() => {
const frame = SPINNER_FRAMES[this.frameIndex % SPINNER_FRAMES.length];
process.stdout.write(`\r${chalk.cyan(frame)} ${this.message}`);
this.frameIndex++;
}, 80);
}
update(message: string): void {
this.message = message;
}
stop(finalMessage?: string): void {
if (this.intervalId) {
clearInterval(this.intervalId);
this.intervalId = null;
}
// Clear the spinner line
process.stdout.write('\r\x1b[K');
if (finalMessage) {
console.log(finalMessage);
}
}
}
// ── Step Display ──
export interface StepDisplayInfo {
step: number;
action: string;
target?: string;
durationMs: number;
success: boolean;
error?: string;
extractedContent?: string;
}
/**
* Format and display a single agent step with its result.
*/
export function displayStep(info: StepDisplayInfo): void {
const stepLabel = chalk.bold.white(`Step ${info.step}`);
const actionLabel = chalk.yellow(info.action);
const durationLabel = chalk.dim(`${info.durationMs}ms`);
const statusIcon = info.success ? chalk.green('✓') : chalk.red('✗');
console.log(`${stepLabel} ${statusIcon} ${actionLabel} ${durationLabel}`);
if (info.target) {
console.log(` ${chalk.dim('target:')} ${info.target}`);
}
if (info.error) {
console.log(` ${chalk.red('error:')} ${info.error}`);
}
if (info.extractedContent) {
const preview = info.extractedContent.length > 120
? `${info.extractedContent.slice(0, 120)}...`
: info.extractedContent;
console.log(` ${chalk.dim('output:')} ${preview}`);
}
}
// ── Cost Display ──
export interface CostDisplayInfo {
inputTokens: number;
outputTokens: number;
totalCost: number;
}
/**
* Display token usage and cost for a single step.
*/
export function displayStepCost(info: CostDisplayInfo): void {
const tokens = chalk.dim(
`tokens: ${info.inputTokens.toLocaleString()} in / ${info.outputTokens.toLocaleString()} out`,
);
const cost = chalk.dim(`cost: $${info.totalCost.toFixed(4)}`);
console.log(` ${tokens} ${cost}`);
}
/**
* Display a summary of total cost and token usage.
*/
export function displayTotalCost(info: CostDisplayInfo & { steps: number; durationMs: number }): void {
console.log('');
console.log(chalk.bold('Summary'));
console.log(chalk.dim('─'.repeat(50)));
console.log(` ${chalk.white('Steps:')} ${info.steps}`);
console.log(` ${chalk.white('Duration:')} ${(info.durationMs / 1000).toFixed(1)}s`);
console.log(` ${chalk.white('Input tokens:')} ${info.inputTokens.toLocaleString()}`);
console.log(` ${chalk.white('Output tokens:')} ${info.outputTokens.toLocaleString()}`);
console.log(` ${chalk.white('Total tokens:')} ${(info.inputTokens + info.outputTokens).toLocaleString()}`);
console.log(` ${chalk.white('Total cost:')} $${info.totalCost.toFixed(4)}`);
console.log(chalk.dim('─'.repeat(50)));
}
// ── Progress Bar ──
export function displayProgressBar(current: number, total: number, width = 30): void {
const ratio = Math.min(current / total, 1);
const filled = Math.round(ratio * width);
const empty = width - filled;
const bar = chalk.green('█'.repeat(filled)) + chalk.dim('░'.repeat(empty));
const pct = (ratio * 100).toFixed(0).padStart(3);
process.stdout.write(`\r [${bar}] ${pct}% (${current}/${total})`);
}
// ── Result Display ──
export function displayResult(success: boolean, output?: string): void {
console.log('');
if (success) {
console.log(chalk.bold.green('Task completed successfully'));
} else {
console.log(chalk.bold.red('Task failed'));
}
if (output) {
console.log('');
console.log(chalk.bold('Result:'));
console.log(output);
}
}
// ── Helpers ──
export function displayError(message: string): void {
console.error(chalk.red('Error:'), message);
}
export function displayWarning(message: string): void {
console.warn(chalk.yellow('Warning:'), message);
}
export function displayInfo(message: string): void {
console.log(chalk.blue('Info:'), message);
}
export function displaySeparator(): void {
console.log(chalk.dim('─'.repeat(60)));
}
export function displayHeader(title: string): void {
console.log('');
console.log(chalk.bold.white(title));
console.log(chalk.dim('═'.repeat(60)));
}
================================================
FILE: packages/cli/src/globals.ts
================================================
import { SessionManager } from './sessions.js';
export const sessionManager = new SessionManager();
================================================
FILE: packages/cli/src/index.ts
================================================
#!/usr/bin/env bun
import { Command } from 'commander';
import { registerOpenCommand } from './commands/open.js';
import { registerClickCommand } from './commands/click.js';
import { registerTypeCommand } from './commands/type.js';
import { registerStateCommand } from './commands/state.js';
import { registerScreenshotCommand } from './commands/screenshot.js';
import { registerEvalCommand } from './commands/eval.js';
import { registerExtractCommand } from './commands/extract.js';
import { registerSessionsCommand } from './commands/sessions.js';
import { registerRunCommand } from './commands/run.js';
import { registerInteractiveCommand } from './commands/interactive.js';
const program = new Command();
program
.name('open-browser')
.description('AI-powered autonomous web browsing CLI')
.version('0.1.0');
// ── Browser manipulation commands ──
registerOpenCommand(program);
registerClickCommand(program);
registerTypeCommand(program);
registerStateCommand(program);
registerScreenshotCommand(program);
registerEvalCommand(program);
registerExtractCommand(program);
registerSessionsCommand(program);
// ── Agent and interactive commands ──
registerRunCommand(program);
registerInteractiveCommand(program);
program.parse();
================================================
FILE: packages/cli/src/protocol.ts
================================================
export interface CLIRequest {
id: string;
command: string;
args: Record<string, unknown>;
}
export interface CLIResponse {
id: string;
success: boolean;
data?: unknown;
error?: string;
}
export function serializeRequest(req: CLIRequest): string {
return JSON.stringify(req) + '\n';
}
export function parseRequest(data: string): CLIRequest | null {
try {
return JSON.parse(data.trim()) as CLIRequest;
} catch {
return null;
}
}
export function serializeResponse(res: CLIResponse): string {
return JSON.stringify(res) + '\n';
}
export function parseResponse(data: string): CLIResponse | null {
try {
return JSON.parse(data.trim()) as CLIResponse;
} catch {
return null;
}
}
================================================
FILE: packages/cli/src/server.ts
================================================
import * as net from 'node:net';
import * as fs from 'node:fs';
import * as path from 'node:path';
import * as os from 'node:os';
import { SessionManager } from './sessions.js';
import { type CLIRequest, type CLIResponse, parseRequest, serializeResponse } from './protocol.js';
const SOCKET_DIR = path.join(os.tmpdir(), 'open-browser');
const SOCKET_PATH = path.join(SOCKET_DIR, 'server.sock');
export class CLIServer {
private server: net.Server | null = null;
readonly sessions: SessionManager;
constructor() {
this.sessions = new SessionManager();
}
async start(): Promise<string> {
if (!fs.existsSync(SOCKET_DIR)) {
fs.mkdirSync(SOCKET_DIR, { recursive: true });
}
// Clean up stale socket
if (fs.existsSync(SOCKET_PATH)) {
fs.unlinkSync(SOCKET_PATH);
}
return new Promise((resolve, reject) => {
this.server = net.createServer((socket) => {
let buffer = '';
socket.on('data', async (data) => {
buffer += data.toString();
const lines = buffer.split('\n');
buffer = lines.pop() ?? '';
for (const line of lines) {
if (!line.trim()) continue;
const request = parseRequest(line);
if (request) {
const response = await this.handleRequest(request);
socket.write(serializeResponse(response));
}
}
});
socket.on('error', () => {
// Client disconnected
});
});
this.server.on('error', reject);
this.server.listen(SOCKET_PATH, () => {
resolve(SOCKET_PATH);
});
});
}
private async handleRequest(request: CLIRequest): Promise<CLIResponse> {
try {
switch (request.command) {
case 'open': {
const url = request.args.url as string;
let sessionId = request.args.session as string | undefined;
if (!sessionId) {
sessionId = this.sessions.getDefaultId();
}
if (!sessionId) {
sessionId = await this.sessions.create({
headless: request.args.headless as boolean | undefined,
});
}
const browser = this.sessions.get(sessionId)!;
await browser.navigate(url);
return {
id: request.id,
success: true,
data: { sessionId, url: browser.currentPage.url() },
};
}
case 'tap': {
const browser = this.getSessionBrowser(request);
const selector = request.args.selector as string;
await browser.click(selector);
return { id: request.id, success: true };
}
case 'type': {
const browser = this.getSessionBrowser(request);
const selector = request.args.selector as string;
const text = request.args.text as string;
await browser.type(selector, text);
return { id: request.id, success: true };
}
case 'state': {
const browser = this.getSessionBrowser(request);
const state = await browser.getState();
return { id: request.id, success: true, data: state };
}
case 'capture': {
const browser = this.getSessionBrowser(request);
const result = await browser.screenshot(request.args.fullPage as boolean);
return { id: request.id, success: true, data: result };
}
case 'eval': {
const browser = this.getSessionBrowser(request);
const expression = request.args.expression as string;
const result = await browser.evaluate(expression);
return { id: request.id, success: true, data: result };
}
case 'sessions': {
return {
id: request.id,
success: true,
data: this.sessions.list(),
};
}
case 'close': {
const sessionId = request.args.session as string | undefined;
if (sessionId) {
await this.sessions.close(sessionId);
} else {
await this.sessions.closeAll();
}
return { id: request.id, success: true };
}
default:
return {
id: request.id,
success: false,
error: `Unknown command: ${request.command}`,
};
}
} catch (error) {
return {
id: request.id,
success: false,
error: error instanceof Error ? error.message : String(error),
};
}
}
private getSessionBrowser(request: CLIRequest) {
const sessionId = request.args.session as string | undefined;
const browser = sessionId
? this.sessions.get(sessionId)
: this.sessions.getDefault();
if (!browser) {
throw new Error('No active session. Use "open" command first.');
}
return browser;
}
async stop(): Promise<void> {
await this.sessions.closeAll();
if (this.server) {
return new Promise((resolve) => {
this.server!.close(() => {
if (fs.existsSync(SOCKET_PATH)) {
fs.unlinkSync(SOCKET_PATH);
}
resolve();
});
});
}
}
static get socketPath(): string {
return SOCKET_PATH;
}
}
================================================
FILE: packages/cli/src/sessions.ts
================================================
import { Viewport, type ViewportOptions } from 'open-browser';
import { nanoid } from 'nanoid';
interface ManagedSession {
id: string;
browser: Viewport;
createdAt: number;
lastAccessedAt: number;
}
export class SessionManager {
private sessions = new Map<string, ManagedSession>();
async create(options?: ViewportOptions): Promise<string> {
const id = nanoid(8);
const browser = new Viewport(options);
await browser.start();
this.sessions.set(id, {
id,
browser,
createdAt: Date.now(),
lastAccessedAt: Date.now(),
});
return id;
}
get(id: string): Viewport | undefined {
const session = this.sessions.get(id);
if (session) {
session.lastAccessedAt = Date.now();
return session.browser;
}
return undefined;
}
async close(id: string): Promise<boolean> {
const session = this.sessions.get(id);
if (!session) return false;
await session.browser.close();
this.sessions.delete(id);
return true;
}
async closeAll(): Promise<void> {
for (const session of this.sessions.values()) {
await session.browser.close();
}
this.sessions.clear();
}
list(): Array<{ id: string; createdAt: number; lastAccessedAt: number }> {
return [...this.sessions.values()].map((s) => ({
id: s.id,
createdAt: s.createdAt,
lastAccessedAt: s.lastAccessedAt,
}));
}
get activeCount(): number {
return this.sessions.size;
}
getDefault(): Viewport | undefined {
const first = this.sessions.values().next();
if (first.done) return undefined;
first.value.lastAccessedAt = Date.now();
return first.value.browser;
}
getDefaultId(): string | undefined {
const first = this.sessions.keys().next();
return first.done ? undefined : first.value;
}
}
================================================
FILE: packages/cli/tsconfig.json
================================================
{
"extends": "../../tsconfig.base.json",
"compilerOptions": {
"rootDir": "src",
"outDir": "dist"
},
"include": ["src/**/*.ts"]
}
================================================
FILE: packages/core/package.json
================================================
{
"name": "open-browser",
"version": "1.1.0",
"description": "AI-powered autonomous web browsing library for TypeScript",
"type": "module",
"main": "src/index.ts",
"types": "src/index.ts",
"exports": {
".": "./src/index.ts"
},
"scripts": {
"build": "tsc --noEmit",
"test": "bun test",
"lint": "biome check src/"
},
"dependencies": {
"ai": "^4.2.0",
"@ai-sdk/openai": "^1.1.0",
"@ai-sdk/anthropic": "^1.1.0",
"@ai-sdk/google": "^1.1.0",
"zod": "^3.24.0",
"playwright": "^1.51.0",
"mitt": "^3.0.2",
"nanoid": "^5.1.0",
"turndown": "^7.2.1",
"dotenv": "^16.5.0"
},
"devDependencies": {
"@types/turndown": "^5.0.5"
},
"peerDependencies": {
"sharp": ">=0.33.0"
},
"peerDependenciesMeta": {
"sharp": {
"optional": true
}
},
"license": "MIT"
}
================================================
FILE: packages/core/src/agent/agent.test.ts
================================================
import { test, expect, describe, beforeEach, mock } from 'bun:test';
import { Agent, type AgentOptions } from '../agent/agent.js';
import type { PageAnalyzer } from '../page/page-analyzer.js';
// ── Mock PageAnalyzer factory (injected via AgentOptions.domService) ──
const mockExtractState = mock(async () => ({
tree: '<div>[1] <button>Click me</button></div>',
selectorMap: { 1: 'button' },
elementCount: 10,
interactiveElementCount: 1,
scrollPosition: { x: 0, y: 0 },
viewportSize: { width: 1280, height: 1100 },
documentSize: { width: 1280, height: 2000 },
pixelsAbove: 0,
pixelsBelow: 900,
}));
function createMockPageAnalyzer(): PageAnalyzer {
return {
extractState: mockExtractState,
clickElementByIndex: mock(async () => {}),
getCachedTree: mock(() => null),
getCachedSelectorMap: mock(() => null),
clearCache: mock(() => {}),
getInteractedElements: mock(() => []),
clearInteractedElements: mock(() => {}),
getElementSelector: mock(async () => undefined),
getElementByBackendNodeId: mock(async () => null),
clickAtCoordinates: mock(async () => {}),
inputTextByIndex: mock(async () => {}),
extractWithIframes: mock(async () => ({ mainTree: null, iframeTrees: [] })),
} as unknown as PageAnalyzer;
}
import type { RunOutcome } from './types.js';
import type { LanguageModel, InferenceOptions } from '../model/interface.js';
import type { InferenceResult, InferenceUsage } from '../model/types.js';
import type { Viewport } from '../viewport/viewport.js';
import type { ViewportSnapshot } from '../viewport/types.js';
import type { CommandExecutor } from '../commands/executor.js';
import type { Command, CommandResult, ExecutionContext } from '../commands/types.js';
import type { CommandCatalog } from '../commands/catalog/catalog.js';
// ── Mock Factories ──
function createMockUsage(input = 100, output = 50): InferenceUsage {
return { inputTokens: input, outputTokens: output, totalTokens: input + output };
}
function createMockModel(options?: {
responses?: Array<{
currentState: { evaluation: string; memory: string; nextGoal: string };
actions: Command[];
}>;
modelId?: string;
}): LanguageModel {
let callCount = 0;
const responses = options?.responses ?? [
{
currentState: {
evaluation: 'Page loaded',
memory: '',
nextGoal: 'Click element',
},
actions: [{ action: 'tap', index: 1, clickCount: 1 } as Command],
},
];
return {
modelId: options?.modelId ?? 'test-model',
provider: 'custom',
invoke: async <T>(_options: InferenceOptions<T>): Promise<InferenceResult<T>> => {
const responseIndex = Math.min(callCount, responses.length - 1);
callCount++;
return {
parsed: responses[responseIndex] as unknown as T,
usage: createMockUsage(),
finishReason: 'stop',
};
},
};
}
function createDoneOnStepModel(doneOnStep: number, result = 'Task completed'): LanguageModel {
const responses: Array<{
currentState: { evaluation: string; memory: string; nextGoal: string };
actions: Command[];
}> = [];
for (let i = 1; i < doneOnStep; i++) {
responses.push({
currentState: {
evaluation: `Step ${i} assessment`,
memory: '',
nextGoal: `Goal for step ${i + 1}`,
},
actions: [{ action: 'tap', index: i, clickCount: 1 } as Command],
});
}
responses.push({
currentState: {
evaluation: 'Task done',
memory: '',
nextGoal: 'Report result',
},
actions: [{ action: 'finish', text: result, success: true } as Command],
});
return createMockModel({ responses });
}
function createMockBrowserState(): ViewportSnapshot {
return {
url: 'https://example.com',
title: 'Example Page',
tabs: [
{ tabId: 0 as any, url: 'https://example.com', title: 'Example Page', isActive: true },
],
activeTabIndex: 0,
};
}
function createMockRegistry(): CommandCatalog{
return {
register: mock(() => {}),
get: mock(() => undefined),
getAll: mock(() => []),
getActionDescriptions: mock(() => 'click: Click on an element'),
getPromptDescription: mock(() => 'click: Click on an element by its index\ngo_to_url: Navigate to a URL'),
has: mock(() => false),
} as unknown as CommandCatalog;
}
function createMockTools(actionResults?: CommandResult[]): CommandExecutor {
const defaultResults: CommandResult[] = [{ success: true }];
return {
registry: createMockRegistry(),
commandsPerStep: 10,
setCoordinateClicking: mock(() => {}),
executeActions: mock(async (_actions: Command[], _ctx: ExecutionContext) => {
return actionResults ?? defaultResults;
}),
executeAction: mock(async (_action: Command, _ctx: ExecutionContext) => {
return (actionResults ?? defaultResults)[0];
}),
} as unknown as CommandExecutor;
}
function createMockBrowser(overrides?: {
browserState?: ViewportSnapshot;
isConnected?: boolean;
}): Viewport {
const state = overrides?.browserState ?? createMockBrowserState();
return {
isConnected: overrides?.isConnected ?? true,
start: mock(async () => {}),
getState: mock(async () => state),
screenshot: mock(async () => ({ base64: 'fake_screenshot', width: 1280, height: 1100 })),
navigate: mock(async () => {}),
currentPage: {
viewportSize: () => ({ width: 1280, height: 1100 }),
evaluate: mock(async () => ({})),
} as any,
cdp: {
send: mock(async () => ({})),
} as any,
} as unknown as Viewport;
}
function createDefaultAgentOptions(overrides?: Partial<AgentOptions>): AgentOptions {
return {
task: 'Find the price of the product',
model: createDoneOnStepModel(2),
browser: createMockBrowser(),
tools: createMockTools([{ success: true, isDone: false }]),
domService: createMockPageAnalyzer(),
settings: {
stepLimit: 5,
enableScreenshots: false,
commandDelayMs: 0,
retryDelay: 0,
autoNavigateToUrls: false,
contextWindowSize: 50000,
},
...overrides,
};
}
// ── Tests ──
describe('Agent', () => {
describe('constructor', () => {
test('creates agent with default settings merged', () => {
const agent = new Agent(createDefaultAgentOptions());
const state = agent.getState();
expect(state.step).toBe(0);
expect(state.isRunning).toBe(false);
expect(state.isDone).toBe(false);
expect(state.failureCount).toBe(0);
expect(state.consecutiveFailures).toBe(0);
});
test('overrides default settings with provided values', () => {
const agent = new Agent(
createDefaultAgentOptions({
settings: {
stepLimit: 50,
enableScreenshots: false,
commandDelayMs: 0,
retryDelay: 0,
autoNavigateToUrls: false,
contextWindowSize: 50000,
},
}),
);
const state = agent.getState();
expect(state.stepLimit).toBe(50);
});
test('initializes cost tracking to zero', () => {
const agent = new Agent(createDefaultAgentOptions());
const cost = agent.getAccumulatedCost();
expect(cost.totalCost).toBe(0);
expect(cost.totalInputTokens).toBe(0);
expect(cost.totalOutputTokens).toBe(0);
});
test('initializes empty history', () => {
const agent = new Agent(createDefaultAgentOptions());
const history = agent.getHistory();
expect(history.entries).toHaveLength(0);
expect(history.task).toBe('Find the price of the product');
});
test('uses custom tools when provided', () => {
const customTools = createMockTools();
const agent = new Agent(createDefaultAgentOptions({ tools: customTools }));
expect(agent).toBeDefined();
});
});
describe('run() basic flow', () => {
test('completes when done action is returned', async () => {
const doneModel = createDoneOnStepModel(1, 'The price is $42');
const tools = createMockTools([
{ success: true, isDone: true, extractedContent: 'The price is $42' },
]);
const agent = new Agent(
createDefaultAgentOptions({ model: doneModel, tools }),
);
const result = await agent.run();
expect(result.finalResult).toBe('The price is $42');
expect(result.success).toBe(true);
expect(result.errors).toHaveLength(0);
});
test('sets isRunning to false after completion', async () => {
const doneModel = createDoneOnStepModel(1, 'Done');
const tools = createMockTools([
{ success: true, isDone: true, extractedContent: 'Done' },
]);
const agent = new Agent(
createDefaultAgentOptions({ model: doneModel, tools }),
);
await agent.run();
const state = agent.getState();
expect(state.isRunning).toBe(false);
});
test('calls onStepStart callback', async () => {
const stepStarts: number[] = [];
const doneModel = createDoneOnStepModel(2, 'Result');
let callCount = 0;
const tools = createMockTools();
(tools.executeActions as any) = mock(async () => {
callCount++;
if (callCount >= 2) {
return [{ success: true, isDone: true, extractedContent: 'Result' }];
}
return [{ success: true }];
});
const agent = new Agent(
createDefaultAgentOptions({
model: doneModel,
tools,
onStepStart: (step) => stepStarts.push(step),
}),
);
await agent.run();
expect(stepStarts.length).toBeGreaterThan(0);
expect(stepStarts[0]).toBe(1);
});
test('calls onDone callback with result', async () => {
let doneResult: RunOutcome | undefined;
const doneModel = createDoneOnStepModel(1, 'Final answer');
const tools = createMockTools([
{ success: true, isDone: true, extractedContent: 'Final answer' },
]);
const agent = new Agent(
createDefaultAgentOptions({
model: doneModel,
tools,
onDone: (r) => { doneResult = r; },
}),
);
await agent.run();
expect(doneResult).toBeDefined();
expect(doneResult!.finalResult).toBe('Final answer');
});
test('starts browser if not connected', async () => {
const browser = createMockBrowser({ isConnected: false });
const doneModel = createDoneOnStepModel(1, 'Result');
const tools = createMockTools([
{ success: true, isDone: true, extractedContent: 'Result' },
]);
const agent = new Agent(
createDefaultAgentOptions({ browser, model: doneModel, tools }),
);
await agent.run();
expect(browser.start).toHaveBeenCalled();
});
});
describe('step execution', () => {
test('invokes browser.getState() on each step', async () => {
const browser = createMockBrowser();
const doneModel = createDoneOnStepModel(1, 'Done');
const tools = createMockTools([
{ success: true, isDone: true, extractedContent: 'Done' },
]);
const agent = new Agent(
createDefaultAgentOptions({ browser, model: doneModel, tools }),
);
await agent.run();
expect(browser.getState).toHaveBeenCalled();
});
test('invokes PageAnalyzer.extractState on each step', async () => {
const doneModel = createDoneOnStepModel(1, 'Done');
const tools = createMockTools([
{ success: true, isDone: true, extractedContent: 'Done' },
]);
mockExtractState.mockClear();
const agent = new Agent(
createDefaultAgentOptions({ model: doneModel, tools }),
);
await agent.run();
expect(mockExtractState).toHaveBeenCalled();
});
test('records history entries for each step', async () => {
let callCount = 0;
const tools = createMockTools();
(tools.executeActions as any) = mock(async () => {
callCount++;
if (callCount >= 3) {
return [{ success: true, isDone: true, extractedContent: 'Done' }];
}
return [{ success: true }];
});
const model = createDoneOnStepModel(3, 'Done');
const agent = new Agent(
createDefaultAgentOptions({ model, tools }),
);
await agent.run();
const history = agent.getHistory();
expect(history.entries.length).toBeGreaterThanOrEqual(1);
});
test('token usage is tracked across steps', async () => {
let callCount = 0;
const tools = createMockTools();
(tools.executeActions as any) = mock(async () => {
callCount++;
if (callCount >= 2) {
return [{ success: true, isDone: true, extractedContent: 'Done' }];
}
return [{ success: true }];
});
const model = createDoneOnStepModel(2, 'Done');
const agent = new Agent(
createDefaultAgentOptions({ model, tools }),
);
await agent.run();
const state = agent.getState();
expect(state.totalInputTokens).toBeGreaterThan(0);
expect(state.totalOutputTokens).toBeGreaterThan(0);
});
});
describe('failure recovery', () => {
test('consecutive failures increment failure count', async () => {
let callCount = 0;
const errorModel: LanguageModel = {
modelId: 'test-model',
provider: 'custom',
invoke: async <T>(): Promise<InferenceResult<T>> => {
callCount++;
throw new Error(`Simulated error ${callCount}`);
},
};
const agent = new Agent(
createDefaultAgentOptions({
model: errorModel,
settings: {
stepLimit: 10,
failureThreshold: 3,
retryDelay: 0,
enableScreenshots: false,
commandDelayMs: 0,
autoNavigateToUrls: false,
contextWindowSize: 50000,
},
}),
);
const result = await agent.run();
expect(result.errors.length).toBeGreaterThan(0);
});
test('agent records error about consecutive failures after failureThreshold', async () => {
let callCount = 0;
const errorModel: LanguageModel = {
modelId: 'test-model',
provider: 'custom',
invoke: async <T>(): Promise<InferenceResult<T>> => {
callCount++;
throw new Error(`Error ${callCount}`);
},
};
const agent = new Agent(
createDefaultAgentOptions({
model: errorModel,
settings: {
stepLimit: 20,
failureThreshold: 3,
retryDelay: 0,
enableScreenshots: false,
commandDelayMs: 0,
autoNavigateToUrls: false,
contextWindowSize: 50000,
},
}),
);
const result = await agent.run();
const hasFailureError = result.errors.some(
(e) => e.includes('consecutive failures'),
);
expect(hasFailureError).toBe(true);
});
test('successful step resets consecutive failure count', async () => {
let callCount = 0;
const model: LanguageModel = {
modelId: 'test-model',
provider: 'custom',
invoke: async <T>(): Promise<InferenceResult<T>> => {
callCount++;
if (callCount === 1) {
throw new Error('Transient error');
}
return {
parsed: {
currentState: { evaluation: 'Done', memory: '', nextGoal: '' },
actions: [{ action: 'finish', text: 'Success', success: true }],
} as unknown as T,
usage: createMockUsage(),
finishReason: 'stop',
};
},
};
const tools = createMockTools([
{ success: true, isDone: true, extractedContent: 'Success' },
]);
const agent = new Agent(
createDefaultAgentOptions({
model,
tools,
settings: {
stepLimit: 10,
failureThreshold: 5,
retryDelay: 0,
enableScreenshots: false,
commandDelayMs: 0,
autoNavigateToUrls: false,
contextWindowSize: 50000,
},
}),
);
const result = await agent.run();
expect(result.finalResult).toBe('Success');
});
});
describe('done action detection and result extraction', () => {
test('detects done action and extracts result text', async () => {
const tools = createMockTools([
{ success: true, isDone: true, extractedContent: 'Product costs $99' },
]);
const model = createDoneOnStepModel(1, 'Product costs $99');
const agent = new Agent(
createDefaultAgentOptions({ model, tools }),
);
const result = await agent.run();
expect(result.finalResult).toBe('Product costs $99');
expect(result.success).toBe(true);
});
test('handles done action with success=false', async () => {
const model = createMockModel({
responses: [{
currentState: { evaluation: 'Cannot find', memory: '', nextGoal: '' },
actions: [{ action: 'finish', text: 'Could not find', success: false } as Command],
}],
});
const tools = createMockTools([
{ success: false, isDone: true, extractedContent: 'Could not find' },
]);
const agent = new Agent(
createDefaultAgentOptions({ model, tools }),
);
const result = await agent.run();
expect(result.finalResult).toBe('Could not find');
expect(result.success).toBe(false);
});
});
describe('pause / resume / stop', () => {
test('pause sets isPaused flag', () => {
const agent = new Agent(createDefaultAgentOptions());
agent.pause();
expect(agent.getState().isPaused).toBe(true);
});
test('resume clears isPaused flag', () => {
const agent = new Agent(createDefaultAgentOptions());
agent.pause();
agent.resume();
expect(agent.getState().isPaused).toBe(false);
});
test('stop sets isRunning to false', async () => {
let stepCount = 0;
const tools = createMockTools();
(tools.executeActions as any) = mock(async () => {
stepCount++;
return [{ success: true }];
});
const model = createMockModel();
const agent = new Agent(
createDefaultAgentOptions({
model,
tools,
settings: {
stepLimit: 100,
enableScreenshots: false,
commandDelayMs: 0,
retryDelay: 0,
autoNavigateToUrls: false,
contextWindowSize: 50000,
},
}),
);
const runPromise = agent.run();
// Stop after a brief moment
await new Promise((r) => setTimeout(r, 50));
agent.stop();
await runPromise;
const state = agent.getState();
expect(state.isRunning).toBe(false);
});
});
describe('max steps reached', () => {
test('returns error when max steps exceeded without done', async () => {
const model = createMockModel();
const tools = createMockTools([{ success: true }]);
const agent = new Agent(
createDefaultAgentOptions({
model,
tools,
settings: {
stepLimit: 3,
enableScreenshots: false,
commandDelayMs: 0,
retryDelay: 0,
autoNavigateToUrls: false,
contextWindowSize: 50000,
},
}),
);
const result = await agent.run();
const hasMaxStepsError = result.errors.some(
(e) => e.includes('maximum steps'),
);
expect(hasMaxStepsError).toBe(true);
});
test('run() accepts stepLimit parameter to override settings', async () => {
const model = createMockModel();
const tools = createMockTools([{ success: true }]);
const agent = new Agent(
createDefaultAgentOptions({
model,
tools,
settings: {
stepLimit: 100,
enableScreenshots: false,
commandDelayMs: 0,
retryDelay: 0,
autoNavigateToUrls: false,
contextWindowSize: 50000,
},
}),
);
const result = await agent.run(2);
const hasMaxStepsError = result.errors.some(
(e) => e.includes('maximum steps'),
);
expect(hasMaxStepsError).toBe(true);
});
});
describe('sensitive data filtering', () => {
test('filters sensitive values from action results', async () => {
const tools = createMockTools([
{
success: true,
isDone: true,
extractedContent: 'Your API key is sk-12345 and password is hunter2',
},
]);
const model = createDoneOnStepModel(1, 'Done');
const agent = new Agent(
createDefaultAgentOptions({
model,
tools,
settings: {
stepLimit: 5,
enableScreenshots: false,
commandDelayMs: 0,
retryDelay: 0,
autoNavigateToUrls: false,
contextWindowSize: 50000,
maskedValues: {
apiKey: 'sk-12345',
password: 'hunter2',
},
},
}),
);
const result = await agent.run();
const history = agent.getHistory();
for (const entry of history.entries) {
for (const ar of entry.actionResults) {
if (ar.extractedContent) {
expect(ar.extractedContent).not.toContain('sk-12345');
expect(ar.extractedContent).not.toContain('hunter2');
}
}
}
});
test('returns unmodified results when no sensitive data configured', async () => {
const tools = createMockTools([
{
success: true,
isDone: true,
extractedContent: 'Plain text result',
},
]);
const model = createDoneOnStepModel(1, 'Done');
const agent = new Agent(
createDefaultAgentOptions({ model, tools }),
);
const result = await agent.run();
expect(result.finalResult).toBe('Plain text result');
});
});
describe('history recording', () => {
test('history entries contain step number', async () => {
let callCount = 0;
const tools = createMockTools();
(tools.executeActions as any) = mock(async () => {
callCount++;
if (callCount >= 2) {
return [{ success: true, isDone: true, extractedContent: 'Done' }];
}
return [{ success: true }];
});
const model = createDoneOnStepModel(2, 'Done');
const agent = new Agent(
createDefaultAgentOptions({ model, tools }),
);
await agent.run();
const history = agent.getHistory();
expect(history.entries.length).toBeGreaterThanOrEqual(1);
expect(history.entries[0].step).toBe(1);
});
test('history entries contain browser state info', async () => {
const doneModel = createDoneOnStepModel(1, 'Done');
const tools = createMockTools([
{ success: true, isDone: true, extractedContent: 'Done' },
]);
const agent = new Agent(
createDefaultAgentOptions({ model: doneModel, tools }),
);
await agent.run();
const history = agent.getHistory();
expect(history.entries.length).toBeGreaterThanOrEqual(1);
expect(history.entries[0].browserState.url).toBe('https://example.com');
expect(history.entries[0].browserState.title).toBe('Example Page');
});
test('history entries contain usage info', async () => {
const doneModel = createDoneOnStepModel(1, 'Done');
const tools = createMockTools([
{ success: true, isDone: true, extractedContent: 'Done' },
]);
const agent = new Agent(
createDefaultAgentOptions({ model: doneModel, tools }),
);
await agent.run();
const history = agent.getHistory();
expect(history.entries.length).toBeGreaterThanOrEqual(1);
expect(history.entries[0].usage).toBeDefined();
expect(history.entries[0].usage!.inputTokens).toBe(100);
expect(history.entries[0].usage!.outputTokens).toBe(50);
});
test('history is finalized after run', async () => {
const doneModel = createDoneOnStepModel(1, 'Done');
const tools = createMockTools([
{ success: true, isDone: true, extractedContent: 'Done' },
]);
const agent = new Agent(
createDefaultAgentOptions({ model: doneModel, tools }),
);
await agent.run();
const history = agent.getHistory();
expect(history.endTime).toBeDefined();
expect(history.totalDuration).toBeDefined();
});
});
describe('cost tracking', () => {
test('cumulative cost accumulates across steps', async () => {
let callCount = 0;
const tools = createMockTools();
(tools.executeActions as any) = mock(async () => {
callCount++;
if (callCount >= 3) {
return [{ success: true, isDone: true, extractedContent: 'Done' }];
}
return [{ success: true }];
});
const model = createDoneOnStepModel(3, 'Done');
const agent = new Agent(
createDefaultAgentOptions({ model, tools }),
);
await agent.run();
const cost = agent.getAccumulatedCost();
expect(cost.totalInputTokens).toBeGreaterThanOrEqual(100);
expect(cost.totalOutputTokens).toBeGreaterThanOrEqual(50);
});
});
describe('follow-up tasks', () => {
test('addNewTask stores follow-up tasks', () => {
const agent = new Agent(createDefaultAgentOptions());
agent.addNewTask('Follow up: check price again');
agent.addNewTask('Follow up: compare with competitor');
const tasks = agent.getFollowUpTasks();
expect(tasks).toHaveLength(2);
expect(tasks[0]).toBe('Follow up: check price again');
expect(tasks[1]).toBe('Follow up: compare with competitor');
});
test('getFollowUpTasks returns a copy', () => {
const agent = new Agent(createDefaultAgentOptions());
agent.addNewTask('Task 1');
const tasks1 = agent.getFollowUpTasks();
const tasks2 = agent.getFollowUpTasks();
expect(tasks1).toEqual(tasks2);
expect(tasks1).not.toBe(tasks2);
});
});
describe('getState', () => {
test('returns a copy of the state', () => {
const agent = new Agent(createDefaultAgentOptions());
const state1 = agent.getState();
const state2 = agent.getState();
expect(state1).toEqual(state2);
expect(state1).not.toBe(state2);
});
test('tracks current URL after run', async () => {
const doneModel = createDoneOnStepModel(1, 'Done');
const tools = createMockTools([
{ success: true, isDone: true, extractedContent: 'Done' },
]);
const agent = new Agent(
createDefaultAgentOptions({ model: doneModel, tools }),
);
await agent.run();
const state = agent.getState();
expect(state.currentUrl).toBe('https://example.com');
});
});
describe('getAccumulatedCost', () => {
test('returns a copy of cost data', () => {
const agent = new Agent(createDefaultAgentOptions());
const cost1 = agent.getAccumulatedCost();
const cost2 = agent.getAccumulatedCost();
expect(cost1).toEqual(cost2);
expect(cost1).not.toBe(cost2);
});
});
describe('run result structure', () => {
test('result contains all expected fields', async () => {
const doneModel = createDoneOnStepModel(1, 'Answer');
const tools = createMockTools([
{ success: true, isDone: true, extractedContent: 'Answer' },
]);
const agent = new Agent(
createDefaultAgentOptions({ model: doneModel, tools }),
);
const result = await agent.run();
expect(result).toHaveProperty('finalResult');
expect(result).toHaveProperty('success');
expect(result).toHaveProperty('history');
expect(result).toHaveProperty('errors');
expect(result).toHaveProperty('totalCost');
});
test('result.history is an ExecutionLog', async () => {
const doneModel = createDoneOnStepModel(1, 'Answer');
const tools = createMockTools([
{ success: true, isDone: true, extractedContent: 'Answer' },
]);
const agent = new Agent(
createDefaultAgentOptions({ model: doneModel, tools }),
);
const result = await agent.run();
expect(result.history).toBeDefined();
expect(result.history.task).toBe('Find the price of the product');
expect(typeof result.history.finalResult).toBe('function');
});
});
});
================================================
FILE: packages/core/src/agent/agent.ts
================================================
import { z, ZodError } from 'zod';
import type { LanguageModel, InferenceOptions } from '../model/interface.js';
import type { Viewport } from '../viewport/viewport.js';
import type { FileAccess } from '../sandbox/file-access.js';
import { PageAnalyzer } from '../page/page-analyzer.js';
import { CommandExecutor } from '../commands/executor.js';
import type { Command, CommandResult, ExecutionContext } from '../commands/types.js';
import { CommandSchema } from '../commands/types.js';
import { InstructionBuilder } from './instructions.js';
import { ConversationManager } from './conversation/service.js';
import { StallDetector, hashPageTree, hashTextContent } from './stall-detector.js';
import { ReplayRecorder } from './replay-recorder.js';
import { ResultEvaluator } from './evaluator.js';
import {
type AgentConfig,
type AgentState,
type AgentDecision,
type StepRecord,
ExecutionLog,
type RunOutcome,
type AccumulatedCost,
type EvaluationResult,
type QuickCheckResult,
ReasoningSchema,
AgentDecisionCompactSchema,
AgentDecisionDirectSchema,
PlanRevisionSchema,
DEFAULT_AGENT_CONFIG,
calculateStepCost,
supportsDeepReasoning,
supportsCoordinateMode,
isCompactModel,
} from './types.js';
import {
AgentError,
StepLimitExceededError,
AgentStalledError,
ModelThrottledError,
} from '../errors.js';
import {
Timer,
sleep,
truncateText,
withDeadline,
extractUrls,
escapeRegExp,
} from '../utils.js';
import { createLogger } from '../logging.js';
const logger = createLogger('agent');
// ── Agent Options ──
export interface AgentOptions {
task: string;
model: LanguageModel;
browser: Viewport;
tools?: CommandExecutor;
/** Pre-configured PageAnalyzer instance (defaults to a new PageAnalyzer) */
domService?: PageAnalyzer;
settings?: Partial<AgentConfig>;
/** Separate model for the judge (defaults to main model) */
judgeModel?: LanguageModel;
/** Separate model for extraction actions (defaults to main model) */
extractionModel?: LanguageModel;
/** File system access for sandbox operations */
fileSystem?: FileAccess;
onStepStart?: (step: number) => void;
onStepEnd?: (step: number, result: CommandResult[]) => void;
onDone?: (result: RunOutcome) => void;
}
// ── Agent ──
export class Agent {
private model: LanguageModel;
private browser: Viewport;
private tools: CommandExecutor;
private domService: PageAnalyzer;
private messageManager: ConversationManager;
private loopDetector: StallDetector;
private gifRecorder?: ReplayRecorder;
private judge?: ResultEvaluator;
private settings: AgentConfig;
private extractionModel?: LanguageModel;
private fileSystem?: FileAccess;
private state: AgentState;
private historyList: ExecutionLog;
private startTime = 0;
private followUpTasks: string[] = [];
private onStepStart?: (step: number) => void;
private onStepEnd?: (step: number, result: CommandResult[]) => void;
private onDone?: (result: RunOutcome) => void;
constructor(options: AgentOptions) {
this.model = options.model;
this.browser = options.browser;
this.settings = { ...DEFAULT_AGENT_CONFIG, ...options.settings, task: options.task };
this.extractionModel = options.extractionModel;
this.fileSystem = options.fileSystem;
this.tools = options.tools ?? new CommandExecutor({
model: this.extractionModel ?? this.model,
allowedUrls: this.settings.allowedUrls,
blockedUrls: this.settings.blockedUrls,
commandsPerStep: this.settings.commandsPerStep,
});
this.domService = options.domService ?? new PageAnalyzer({
capturedAttributes: this.settings.capturedAttributes,
});
this.messageManager = new ConversationManager({
contextWindowSize: this.settings.contextWindowSize,
includeLastScreenshot: this.settings.enableScreenshots,
maskedValues: this.settings.maskedValues,
compaction: this.settings.conversationCompaction,
});
this.loopDetector = new StallDetector();
if (this.settings.replayOutputPath) {
this.gifRecorder = new ReplayRecorder({
outputPath: this.settings.replayOutputPath,
});
}
// Judge setup
if (this.settings.enableEvaluation || this.settings.enableSimpleJudge) {
const judgeModel = options.judgeModel ?? this.model;
this.judge = new ResultEvaluator(judgeModel);
}
// Auto-enable coordinate clicking for supported models
if (this.settings.autoEnableCoordinateClicking) {
if (supportsCoordinateMode(this.model.modelId)) {
this.tools.setCoordinateClicking(true);
logger.info(`Coordinate clicking auto-enabled for model ${this.model.modelId}`);
}
}
// Initialize state
this.state = {
step: 0,
stepLimit: this.settings.stepLimit,
failureCount: 0,
consecutiveFailures: 0,
isRunning: false,
isPaused: false,
isDone: false,
totalInputTokens: 0,
totalOutputTokens: 0,
cumulativeCost: {
totalInputTokens: 0,
totalOutputTokens: 0,
totalInputCost: 0,
totalOutputCost: 0,
totalCost: 0,
},
};
this.historyList = new ExecutionLog({
task: this.settings.task,
});
this.onStepStart = options.onStepStart;
this.onStepEnd = options.onStepEnd;
this.onDone = options.onDone;
}
// ────────────────────────────────────────
// Main run loop
// ────────────────────────────────────────
async run(stepLimit?: number): Promise<RunOutcome> {
const effectiveMaxSteps = stepLimit ?? this.settings.stepLimit;
this.state.stepLimit = effectiveMaxSteps;
this.state.isRunning = true;
this.startTime = Date.now();
// Ensure browser is started
if (!this.browser.isConnected) {
await this.browser.start();
}
// Build system prompt (may be rebuilt per step if dynamicCommandSchema is on)
this.rebuildInstructionBuilder();
// URL extraction: auto-navigate to first URL found in task text
if (this.settings.autoNavigateToUrls) {
await this.autoNavigateFromTask();
}
// Execute initial actions before the main loop
if (this.settings.preflightCommands.length > 0) {
await this.executeInitialActions();
}
const errors: string[] = [];
let finalResult: string | undefined;
let success = false;
let judgement: EvaluationResult | undefined;
let simpleJudgement: QuickCheckResult | undefined;
try {
for (let step = 1; step <= effectiveMaxSteps; step++) {
if (!this.state.isRunning || this.state.isDone) break;
// Pause support
while (this.state.isPaused) {
await sleep(100);
}
this.state.step = step;
this.onStepStart?.(step);
try {
// Wrap step execution in optional timeout
const stepPromise = this.executeStep(step, effectiveMaxSteps);
const result = this.settings.stepDeadlineMs > 0
? await withDeadline(
stepPromise,
this.settings.stepDeadlineMs,
`Step ${step} timed out after ${this.settings.stepDeadlineMs}ms`,
)
: await stepPromise;
this.state.consecutiveFailures = 0;
// Check if done
const doneResult = result.find((r) => r.isDone);
if (doneResult) {
finalResult = doneResult.extractedContent;
success = doneResult.success;
// Simple judge: quick validation before accepting the result
if (this.settings.enableSimpleJudge && this.judge && finalResult) {
simpleJudgement = await this.judge.simpleEvaluate(
this.settings.task,
finalResult,
);
if (simpleJudgement.shouldRetry && step < effectiveMaxSteps) {
logger.info(
`Simple judge suggests retry: ${simpleJudgement.reason}`,
);
this.messageManager.addCommandResultMessage(
`The result was reviewed and found lacking: ${simpleJudgement.reason}. ` +
'Please try a different approach to complete the task.',
step,
);
// Don't mark as done -- continue the loop
continue;
}
}
this.state.isDone = true;
break;
}
this.onStepEnd?.(step, result);
// Planning: periodically update the plan
if (this.settings.enableStrategy && this.shouldUpdatePlan(step)) {
await this.updatePlan(step);
}
// Replan on stall: if loop detector shows stuck + planning enabled
if (this.settings.restrategizeOnStall && this.settings.enableStrategy) {
const loopCheck = this.loopDetector.isStuck();
if (loopCheck.stuck && loopCheck.severity >= 2) {
logger.info('Agent stalled, triggering replan');
await this.updatePlan(step);
}
}
// Message compaction: every N steps (LLM-based)
if (this.messageManager.shouldCompactWithLlm()) {
const compacted = await this.messageManager.compactWithLlm(this.model);
if (compacted) {
logger.debug(`Messages compacted at step ${step}`);
}
}
// Save conversation per step if configured
if (this.settings.conversationOutputPath) {
await this.saveConversation(step);
}
} catch (error) {
// Rate limit retry with exponential backoff
if (error instanceof ModelThrottledError) {
const waitMs = error.retryAfterMs ?? Math.min(
60_000,
this.settings.retryDelay * 1000 * 2 ** this.state.consecutiveFailures,
);
logger.warn(`Rate limited, waiting ${waitMs}ms before retry`);
await sleep(waitMs);
this.state.consecutiveFailures++;
// Don't count rate limits toward max failures
continue;
}
const message = error instanceof Error ? error.message : String(error);
errors.push(`Step ${step}: ${message}`);
this.state.failureCount++;
this.state.consecutiveFailures++;
if (this.state.consecutiveFailures >= this.settings.failureThreshold) {
// Failure recovery: make one final LLM call to diagnose
const failureSummary = await this.makeFailureRecoveryCall(errors);
if (failureSummary) {
finalResult = failureSummary;
}
throw new AgentError(
`Too many consecutive failures (${this.state.consecutiveFailures})`,
);
}
// Add error message to conversation
this.messageManager.addCommandResultMessage(
`Error: ${truncateText(message, 400)}`,
step,
);
// Wait before retry
await sleep(this.settings.retryDelay * 1000);
}
}
if (!this.state.isDone && this.state.step >= effectiveMaxSteps) {
throw new StepLimitExceededError(this.state.step, effectiveMaxSteps);
}
} catch (error) {
if (
error instanceof StepLimitExceededError ||
error instanceof AgentStalledError ||
error instanceof AgentError
) {
errors.push(error.message);
} else {
throw error;
}
} finally {
this.state.isRunning = false;
// Save recording
if (this.gifRecorder) {
await this.gifRecorder.save();
}
}
// Full judge evaluation after completion
if (this.settings.enableEvaluation && this.judge && finalResult) {
judgement = await this.judge.evaluate(
this.settings.task,
finalResult,
this.historyList.entries,
{
expectedOutcome: this.settings.expectedOutcome,
includeScreenshots: this.settings.enableScreenshots,
},
);
}
// Finalize history
this.historyList.finish();
const runResult: RunOutcome = {
finalResult,
success,
history: this.historyList,
errors,
judgement,
simpleJudgement,
totalCost: { ...this.state.cumulativeCost },
};
this.onDone?.(runResult);
return runResult;
}
// ────────────────────────────────────────
// Step Execution
// ────────────────────────────────────────
private async executeStep(step: number, stepLimit: number): Promise<CommandResult[]> {
const timer = new Timer();
// Get browser state
const browserState = await this.browser.getState();
this.state.currentUrl = browserState.url;
// Dynamic action schema: rebuild system prompt per step based on current URL
if (this.settings.dynamicCommandSchema) {
this.rebuildInstructionBuilder(browserState.url);
}
// Extract DOM
const domState = await this.domService.extractState(
this.browser.currentPage,
this.browser.cdp!,
);
// Take screenshot if using vision
let screenshot: string | undefined;
if (this.settings.enableScreenshots) {
const screenshotResult = await this.browser.screenshot();
screenshot = screenshotResult.base64;
if (this.gifRecorder) {
const actionLabel = browserState.url;
this.gifRecorder.addFrame(screenshot, step, actionLabel);
}
}
// Build state message
const stateText = InstructionBuilder.buildStatePrompt(
browserState.url,
browserState.title,
browserState.tabs,
domState.tree,
step,
stepLimit,
domState.pixelsAbove,
domState.pixelsBelow,
);
// Check for loop
const loopCheck = this.loopDetector.isStuck();
let additionalContext = '';
if (loopCheck.stuck) {
additionalContext = InstructionBuilder.buildLoopNudge(
this.loopDetector.getLoopNudgeMessage(),
);
// Severe loop: throw stuck error
if (loopCheck.severity >= 3) {
throw new AgentStalledError(
`Agent stuck: ${loopCheck.reason} (severity ${loopCheck.severity})`,
);
}
}
// Add plan context if planning is enabled
if (this.settings.enableStrategy && this.state.currentPlan) {
additionalContext += InstructionBuilder.buildPlanPrompt(this.state.currentPlan);
}
// Add messages
this.messageManager.addStateMessage(
stateText + additionalContext,
screenshot,
step,
);
// Determine output schema based on mode
const outputSchema = this.getOutputSchema();
// Invoke LLM with optional timeout and Zod recovery
const completion = await this.invokeLlmWithRecovery(outputSchema, step);
// Update token tracking
this.state.totalInputTokens += completion.usage.inputTokens;
this.state.totalOutputTokens += completion.usage.outputTokens;
// Cost tracking
this.updateCostTracking(completion.usage.inputTokens, completion.usage.outputTokens, step);
const output = completion.parsed;
// Normalize output to standard AgentDecision shape
const normalizedOutput = this.normalizeOutput(output);
// Add assistant response
this.messageManager.addAssistantMessage(
JSON.stringify(normalizedOutput.currentState),
step,
);
// Execute actions
const context: ExecutionContext = {
page: this.browser.currentPage,
cdpSession: this.browser.cdp!,
domService: this.domService,
browserSession: this.browser,
extractionLlm: this.extractionModel,
fileSystem: this.fileSystem,
maskedValues: this.settings.maskedValues,
};
const actions = normalizedOutput.actions as Command[];
const results = await this.tools.executeActions(actions, context);
// Record for loop detection (with enhanced fingerprint)
this.loopDetector.recordAction(actions);
this.loopDetector.recordFingerprint({
url: browserState.url,
domHash: hashPageTree(domState.tree),
scrollY: domState.scrollPosition.y,
elementCount: domState.elementCount,
textHash: hashTextContent(domState.tree.slice(0, 2000)),
});
// Filter sensitive data from results
const filteredResults = this.filterSensitiveData(results);
// Add action results to conversation
const resultText = filteredResults
.map((r, i) => {
const actionName = actions[i]?.action ?? 'unknown';
const status = r.success ? 'success' : `error: ${r.error}`;
const content = r.extractedContent
? `\nContent: ${r.extractedContent}`
: '';
return `${actionName}: ${status}${content}`;
})
.join('\n');
if (resultText) {
this.messageManager.addCommandResultMessage(resultText, step);
}
// Wait between actions
if (this.settings.commandDelayMs > 0) {
await sleep(this.settings.commandDelayMs * 1000);
}
// Record history entry
const entry: StepRecord = {
step,
timestamp: Date.now(),
browserState: {
url: browserState.url,
title: browserState.title,
tabs: browserState.tabs,
interactedElements: actions
.filter((a): a is Command & { index: number } => 'index' in a)
.map((a) => ({
index: a.index,
description: '',
action: a.action,
})),
screenshot,
},
agentOutput: normalizedOutput as AgentDecision,
actionResults: filteredResults,
usage: completion.usage,
duration: timer.elapsed(),
metadata: {
stepNumber: step,
durationMs: timer.elapsed(),
inputTokens: completion.usage.inputTokens,
outputTokens: completion.usage.outputTokens,
actionCount: actions.length,
url: browserState.url,
startedAt: Date.now() - timer.elapsed(),
completedAt: Date.now(),
},
};
this.historyList.addEntry(entry);
return results;
}
// ────────────────────────────────────────
// LLM Invocation with Zod Recovery
// ────────────────────────────────────────
private async invokeLlmWithRecovery(
outputSchema: z.ZodType<unknown>,
step: number,
retryCount = 0,
): Promise<{
parsed: Record<string, unknown>;
usage: { inputTokens: number; outputTokens: number; totalTokens: number };
}> {
const messages = this.messageManager.getMessages();
const invokeOptions: InferenceOptions<unknown> = {
messages,
responseSchema: outputSchema,
schemaName: this.getSchemaName(),
schemaDescription: 'Agent decision with current state assessment and actions to take',
};
// Extended thinking: pass thinking budget as maxTokens
if (
this.settings.enableDeepReasoning &&
supportsDeepReasoning(this.model.modelId)
) {
invokeOptions.maxTokens = this.settings.reasoningBudget;
}
try {
// Wrap LLM call in optional timeout
const invokePromise = this.model.invoke(invokeOptions);
const completion =
this.settings.modelDeadlineMs > 0
? await withDeadline(
invokePromise,
this.settings.modelDeadlineMs,
`LLM call timed out after ${this.settings.modelDeadlineMs}ms`,
)
: await invokePromise;
return {
parsed: completion.parsed as Record<string, unknown>,
usage: completion.usage,
};
} catch (error) {
// Zod validation error recovery: re-prompt with the error details
if (error instanceof ZodError && retryCount < 2) {
logger.warn(
`Zod validation failed (attempt ${retryCount + 1}), re-prompting LLM`,
);
const issues = error.issues
.map((issue) => `- ${issue.path.join('.')}: ${issue.message}`)
.join('\n');
this.messageManager.addCommandResultMessage(
'Your previous response had a validation error. ' +
'Please fix the following issues and respond again:\n' +
`${issues}\n\n` +
'Make sure your response matches the expected JSON schema exactly.',
step,
);
return this.invokeLlmWithRecovery(outputSchema, step, retryCount + 1);
}
// Re-throw rate limit errors for special handling in the main loop
if (error instanceof ModelThrottledError) {
throw error;
}
throw error;
}
}
// ────────────────────────────────────────
// Output Schema Selection
// ────────────────────────────────────────
private getOutputSchema(): z.ZodType<unknown> {
// Flash mode: simpler schema for cheaper / faster models
if (this.settings.compactMode || isCompactModel(this.model.modelId)) {
return AgentDecisionCompactSchema as z.ZodType<unknown>;
}
// Extended thinking: model reasons internally, skip brain schema
if (
this.settings.enableDeepReasoning &&
supportsDeepReasoning(this.model.modelId)
) {
return AgentDecisionDirectSchema as z.ZodType<unknown>;
}
// Default full schema with brain + typed action union
return z.object({
currentState: ReasoningSchema,
actions: z.array(CommandSchema),
}) as z.ZodType<unknown>;
}
private getSchemaName(): string {
if (this.settings.compactMode || isCompactModel(this.model.modelId)) {
return 'AgentDecisionCompact';
}
if (
this.settings.enableDeepReasoning &&
supportsDeepReasoning(this.model.modelId)
) {
return 'AgentDecisionDirect';
}
return 'AgentDecision';
}
/**
* Normalize the various output schema shapes into the standard AgentDecision.
*/
private normalizeOutput(output: Record<string, unknown>): AgentDecision {
// Flash schema: { goal, actions }
if ('goal' in output && !('currentState' in output)) {
return {
currentState: {
evaluation: String(output.goal ?? ''),
memory: '',
nextGoal: String(output.goal ?? ''),
},
actions: (output.actions ?? []) as Record<string, unknown>[],
};
}
// No-thinking schema: { actions } only
if (!('currentState' in output) && 'actions' in output) {
return {
currentState: {
evaluation: '',
memory: '',
nextGoal: '',
},
actions: (output.actions ?? []) as Record<string, unknown>[],
};
}
// Standard schema passthrough
return output as AgentDecision;
}
// ────────────────────────────────────────
// Planning System
// ────────────────────────────────────────
private shouldUpdatePlan(step: number): boolean {
if (!this.settings.enableStrategy) return false;
const interval =
this.settings.strategyInterval > 0 ? this.settings.strategyInterval : 5;
const lastPlan = this.state.lastPlanStep ?? 0;
return step - lastPlan >= interval;
}
private async updatePlan(step: number): Promise<void> {
try {
const recentHistory = this.historyList.entries
.slice(-5)
.map(
(e) =>
`Step ${e.step}: ${e.agentOutput.currentState?.evaluation ?? '(no eval)'}`,
)
.join('\n');
const planPrompt =
`Task: ${this.settings.task}\n\n` +
`Current step: ${step}/${this.state.stepLimit}\n` +
(this.state.currentPlan
? `Current plan:\n${this.state.currentPlan}\n\n`
: '') +
`Recent progress:\n${recentHistory}\n\n` +
'Based on the current progress, provide an updated plan. ' +
'Include what has been accomplished and what remains.';
// Use ephemeral message so the plan prompt doesn't persist
this.messageManager.addEphemeralMessage(planPrompt);
const completion = await this.model.invoke({
messages: this.messageManager.getMessages(),
responseSchema: PlanRevisionSchema,
schemaName: 'PlanRevision',
temperature: 0.3,
});
this.state.currentPlan = completion.parsed.plan;
this.state.lastPlanStep = step;
logger.info(`Plan updated at step ${step}: ${completion.parsed.reasoning}`);
} catch (error) {
logger.warn(
`Plan update failed at step ${step}: ${
error instanceof Error ? error.message : String(error)
}`,
);
}
}
// ────────────────────────────────────────
// System Prompt Management
// ────────────────────────────────────────
/**
* (Re)build the system prompt. When `pageUrl` is provided, the registry
* can filter action descriptions to show only domain-relevant actions.
*/
private rebuildInstructionBuilder(pageUrl?: string): void {
const systemPrompt = InstructionBuilder.fromSettings(
this.settings,
this.tools.registry,
pageUrl,
);
this.messageManager.setInstructionBuilder(systemPrompt.build());
}
// ────────────────────────────────────────
// URL Extraction from Task Text
// ────────────────────────────────────────
private async autoNavigateFromTask(): Promise<void> {
const urls = extractUrls(this.settings.task);
if (urls.length === 0) return;
const firstUrl = urls[0];
logger.info(`Auto-navigating to URL found in task: ${firstUrl}`);
try {
await this.browser.navigate(firstUrl);
// Give the page a moment to load
await sleep(1000);
} catch (error) {
logger.warn(
`Auto-navigation to ${firstUrl} failed: ${
error instanceof Error ? error.message : String(error)
}`,
);
}
}
// ────────────────────────────────────────
// Initial Actions
// ────────────────────────────────────────
private async executeInitialActions(): Promise<void> {
logger.info(
`Executing ${this.settings.preflightCommands.length} initial action(s)`,
);
const context: ExecutionContext = {
page: this.browser.currentPage,
cdpSession: this.browser.cdp!,
domService: this.domService,
browserSession: this.browser,
extractionLlm: this.extractionModel,
fileSystem: this.fileSystem,
maskedValues: this.settings.maskedValues,
};
for (const action of this.settings.preflightCommands) {
try {
await this.tools.executeAction(action, context);
logger.debug(`Initial action ${action.action} completed`);
} catch (error) {
logger.warn(
`Initial action ${action.action} failed: ${
error instanceof Error ? error.message : String(error)
}`,
);
}
}
await sleep(500);
}
// ────────────────────────────────────────
// Failure Recovery
// ────────────────────────────────────────
/**
* On max failures, make one final LLM call to produce a diagnostic
* summary. Returns a description of what went wrong, or undefined
* if the recovery call itself fails.
*/
private async makeFailureRecoveryCall(
errors: string[],
): Promise<string | undefined> {
try {
const errorSummary = errors.slice(-5).join('\n');
const recoverySchema = z.object({
diagnosis: z.string().describe('What went wrong'),
suggestion: z.string().describe('What could be tried differently'),
});
const completion = await this.model.invoke({
messages: [
{
role: 'system' as const,
content:
'You are a diagnostic assistant. Analyze the errors that occurred during ' +
'a web browsing automation task and provide a brief diagnosis.',
},
{
role: 'user' as const,
content:
`Task: ${this.settings.task}\n\n` +
`Errors encountered:\n${errorSummary}\n\n` +
'Provide a brief diagnosis of what went wrong and what could be tried differently.',
},
],
responseSchema: recoverySchema,
schemaName: 'FailureRecovery',
temperature: 0,
});
const result =
`Task failed. Diagnosis: ${completion.parsed.diagnosis}. ` +
`Suggestion: ${completion.parsed.suggestion}`;
logger.info(`Failure recovery: ${result}`);
return result;
} catch {
logger.debug('Failure recovery call itself failed');
return undefined;
}
}
// ────────────────────────────────────────
// Cost Tracking
// ────────────────────────────────────────
private updateCostTracking(
inputTokens: number,
outputTokens: number,
step: number,
): void {
const stepCost = calculateStepCost(
inputTokens,
outputTokens,
this.model.modelId,
);
this.state.cumulativeCost.totalInputTokens += inputTokens;
this.state.cumulativeCost.totalOutputTokens += outputTokens;
if (stepCost) {
this.state.cumulativeCost.totalInputCost += stepCost.inputCost;
this.state.cumulativeCost.totalOutputCost += stepCost.outputCost;
this.state.cumulativeCost.totalCost += stepCost.totalCost;
logger.debug(
`Step ${step} cost: $${stepCost.totalCost.toFixed(4)} ` +
`(cumulative: $${this.state.cumulativeCost.totalCost.toFixed(4)})`,
);
}
}
// ────────────────────────────────────────
// Sensitive Data Filtering
// ────────────────────────────────────────
private filterSensitiveData(results: CommandResult[]): CommandResult[] {
if (!this.settings.maskedValues) return results;
return results.map((r) => {
if (!r.extractedContent) return r;
let content = r.extractedContent;
for (const [key, value] of Object.entries(this.settings.maskedValues!)) {
content = content.replace(
new RegExp(escapeRegExp(value), 'g'),
`<${key}>`,
);
}
return { ...r, extractedContent: content };
});
}
// ────────────────────────────────────────
// Save Conversation
// ────────────────────────────────────────
private async saveConversation(step: number): Promise<void> {
if (!this.settings.conversationOutputPath) return;
try {
const filePath = this.settings.conversationOutputPath.replace(
/\{step\}/g,
step.toString(),
);
await this.messageManager.saveToFile(filePath);
} catch (error) {
logger.debug(
`Failed to save conversation at step ${step}: ${
error instanceof Error ? error.message : String(error)
}`,
);
}
}
// ────────────────────────────────────────
// Follow-up Tasks
// ────────────────────────────────────────
/**
* Add a follow-up task to be executed after the current task completes.
* Tasks are stored and can be retrieved via getFollowUpTasks().
*/
addNewTask(task: string): void {
this.followUpTasks.push(task);
logger.info(`Follow-up task added: ${truncateText(task, 100)}`);
}
getFollowUpTasks(): string[] {
return [...this.followUpTasks];
}
// ────────────────────────────────────────
// Control Methods
// ────────────────────────────────────────
pause(): void {
this.state.isPaused = true;
}
resume(): void {
this.state.isPaused = false;
}
stop(): void {
this.state.isRunning = false;
}
getState(): AgentState {
return { ...this.state };
}
getHistory(): ExecutionLog {
return this.historyList;
}
getAccumulatedCost(): AccumulatedCost {
return { ...this.state.cumulativeCost };
}
}
================================================
FILE: packages/core/src/agent/conversation/service.ts
================================================
import { z } from 'zod';
import type { Message } from '../../model/messages.js';
import {
systemMessage,
userMessage,
assistantMessage,
imageContent,
textContent,
type ContentPart,
} from '../../model/messages.js';
import type { LanguageModel } from '../../model/interface.js';
import type {
ConversationManagerOptions,
TrackedMessage,
ConversationManagerState,
ConversationEntry,
SerializedTrackedMessage,
MessageCategory,
} from './types.js';
import {
estimateTokens,
estimateMessageTokens,
redactMessages,
extractTextContent,
truncate,
} from './utils.js';
// ── LLM Compaction Summary Schema ──
const CompactionSummarySchema = z.object({
summary: z.string().describe('Concise summary of the conversation so far'),
});
// ── ConversationManager ──
export class ConversationManager {
private messages: TrackedMessage[] = [];
private systemPromptMessage: Message | null = null;
private systemPromptText: string | null = null;
private options: ConversationManagerOptions;
private historyItems: ConversationEntry[] = [];
private currentStep = 0;
private lastCompactionStep = 0;
constructor(options: ConversationManagerOptions) {
this.options = options;
}
// ────────────────────────────────────────
// System Prompt
// ────────────────────────────────────────
setInstructionBuilder(prompt: string): void {
this.systemPromptText = prompt;
this.systemPromptMessage = systemMessage(prompt);
}
// ────────────────────────────────────────
// Add Messages
// ────────────────────────────────────────
addStateMessage(
stateText: string,
screenshot?: string,
step?: number,
): void {
const content: ContentPart[] = [textContent(stateText)];
if (screenshot && this.options.includeLastScreenshot) {
content.push(imageContent(screenshot, 'image/png'));
}
if (step !== undefined) this.currentStep = step;
this.messages.push({
message: userMessage(content),
isCompactable: true,
tokenEstimate: estimateMessageTokens(content),
step,
category: 'state',
addedAt: Date.now(),
});
this.recordConversationEntry(step ?? this.currentStep, 'state', stateText, !!screenshot);
}
addAssistantMessage(text: string, step?: number): void {
if (step !== undefined) this.currentStep = step;
this.messages.push({
message: assistantMessage(text),
isCompactable: true,
tokenEstimate: estimateTokens(text),
step,
category: 'assistant',
addedAt: Date.now(),
});
this.recordConversationEntry(step ?? this.currentStep, 'assistant', text);
}
addCommandResultMessage(text: string, step?: number): void {
if (step !== undefined) this.currentStep = step;
this.messages.push({
message: userMessage(text),
isCompactable: true,
tokenEstimate: estimateTokens(text),
step,
category: 'action_result',
addedAt: Date.now(),
});
this.recordConversationEntry(step ?? this.currentStep, 'action_result', text);
}
addUserMessage(text: string): void {
this.messages.push({
message: userMessage(text),
isCompactable: false,
tokenEstimate: estimateTokens(text),
category: 'user',
addedAt: Date.now(),
});
this.recordConversationEntry(this.currentStep, 'user', text);
}
/**
* Add an ephemeral message that is included in the next getMessages() call
* and then automatically removed. Useful for one-shot instructions or
* temporary context that should not persist across steps.
*/
addEphemeralMessage(text: string, role: 'user' | 'assistant' = 'user'): void {
const msg =
role === 'user' ? userMessage(text) : assistantMessage(text);
this.messages.push({
message: msg,
isCompactable: false,
tokenEstimate: estimateTokens(text),
category: role === 'user' ? 'user' : 'assistant',
ephemeral: true,
ephemeralRead: false,
addedAt: Date.now(),
});
}
// ────────────────────────────────────────
// Get Messages (with compaction + filtering)
// ────────────────────────────────────────
getMessages(): Message[] {
const result: Message[] = [];
if (this.systemPromptMessage) {
result.push(this.systemPromptMessage);
}
// Check if we need to compact
const totalTokens = this.estimateTotalTokens();
if (totalTokens > this.options.contextWindowSize) {
this.compact();
}
for (const managed of this.messages) {
result.push(managed.message);
}
// Mark ephemeral messages as read so they can be cleaned up
this.consumeEphemeralMessages();
// Apply sensitive data filtering
if (this.options.maskedValues && Object.keys(this.options.maskedValues).length > 0) {
return redactMessages(result, this.options.maskedValues);
}
return result;
}
// ────────────────────────────────────────
// Ephemeral Message Lifecycle
// ────────────────────────────────────────
/**
* After getMessages() has been called, remove ephemeral messages that were already read.
* Freshly-added ephemeral messages are marked as read (so they survive one getMessages call).
*/
private consumeEphemeralMessages(): void {
// Remove previously-read ephemeral messages
this.messages = this.messages.filter(
(m) => !(m.ephemeral && m.ephemeralRead),
);
// Mark remaining ephemeral messages as read for the next pass
for (const m of this.messages) {
if (m.ephemeral && !m.ephemeralRead) {
m.ephemeralRead = true;
}
}
}
// ────────────────────────────────────────
// Token Estimation
// ────────────────────────────────────────
estimateTotalTokens(): number {
let total = 0;
if (this.systemPromptMessage) {
total += estimateTokens(
typeof this.systemPromptMessage.content === 'string'
? this.systemPromptMessage.content
: '',
);
}
for (const managed of this.messages) {
total += managed.tokenEstimate;
}
return total;
}
// ────────────────────────────────────────
// Basic Compaction (image removal + old message replacement)
// ────────────────────────────────────────
private compact(): void {
// Remove screenshots from older messages (keep only last)
let foundLast = false;
for (let i = this.messages.length - 1; i >= 0; i--) {
const msg = this.messages[i];
if (!msg.isCompactable) continue;
const content = msg.message.content;
if (Array.isArray(content)) {
const hasImage = content.some(
(p) => typeof p === 'object' && p !== null && (p as ContentPart).type === 'image',
);
if (hasImage) {
if (foundLast) {
// Remove images from this message
const filtered = content.filter(
(p) =>
typeof p !== 'object' ||
p === null ||
(p as ContentPart).type !== 'image',
);
if (filtered.length > 0) {
msg.message = userMessage(filtered as ContentPart[]);
msg.tokenEstimate = estimateMessageTokens(filtered);
}
} else {
foundLast = true;
}
}
}
}
// If still over budget, remove old compactable state messages
while (
this.estimateTotalTokens() > this.options.contextWindowSize &&
this.messages.length > 4
) {
// Find first compactable message
const idx = this.messages.findIndex((m) => m.isCompactable);
if (idx === -1) break;
// Replace with a summary
const removed = this.messages.splice(idx, 1)[0];
const summary = `[Step ${removed.step ?? '?'} state omitted to save tokens]`;
this.messages.splice(idx, 0, {
message: userMessage(summary),
isCompactable: true,
tokenEstimate: estimateTokens(summary),
step: removed.step,
category: 'compaction_summary',
addedAt: Date.now(),
});
}
}
// ────────────────────────────────────────
// LLM-Based Compaction
// ────────────────────────────────────────
/**
* Run LLM-based message compaction: send the older portion of the conversation
* to a summarization model and replace it with a single summary message.
*
* Call this periodically (e.g. every N steps as configured in compaction.interval).
* Returns true if compaction was performed, false if skipped.
*/
async compactWithLlm(model?: LanguageModel): Promise<boolean> {
const compactionConfig = this.options.compaction;
if (!compactionConfig) return false;
const llm = model ?? this.options.compactionModel;
if (!llm) return false;
// Only compact if enough steps have passed since last compaction
if (
compactionConfig.interval > 0 &&
this.currentStep - this.lastCompactionStep < compactionConfig.interval
) {
return false;
}
const targetTokens =
compactionConfig.targetTokens ??
Math.floor(this.options.contextWindowSize * 0.6);
// If we're under the target, no need to compact
if (this.estimateTotalTokens() <= targetTokens) return false;
// Split messages: keep the last few messages intact, summarize the rest
const keepCount = Math.min(6, Math.floor(this.messages.length / 2));
const toSummarize = this.messages.slice(0, this.messages.length - keepCount);
const toKeep = this.messages.slice(this.messages.length - keepCount);
if (toSummarize.length === 0) return false;
// Build a transcript of the messages to summarize
const transcript = toSummarize
.map((m) => {
const role = m.message.role;
const text = extractTextContent(m.message);
const stepLabel = m.step !== undefined ? ` (step ${m.step})` : '';
return `[${role}${stepLabel}]: ${truncate(text, 500)}`;
})
.join('\n');
const prompt = [
systemMessage(
'You are a conversation summarizer. Summarize the following agent-browser conversation transcript. ' +
'Preserve key facts: URLs visited, actions taken, errors encountered, extracted data, and the current task state. ' +
'Be concise but complete.',
),
userMessage(
`Summarize this conversation transcript:\n\n${transcript}`,
),
];
try {
const completion = await llm.invoke({
messages: prompt,
responseSchema: CompactionSummarySchema,
schemaName: 'CompactionSummary',
schemaDescription: 'A concise summary of the conversation so far',
maxTokens: compactionConfig.maxTokens,
temperature: 0,
});
const summaryText = `[Conversation summary of steps 1-${toSummarize[toSummarize.length - 1]?.step ?? '?'}]\n${completion.parsed.summary}`;
// Replace the summarized messages with a single summary
this.messages = [
{
message: userMessage(summaryText),
isCompactable: false, // Don't re-compact the summary
tokenEstimate: estimateTokens(summaryText),
category: 'compaction_summary',
addedAt: Date.now(),
},
...toKeep,
];
this.lastCompactionStep = this.currentStep;
return true;
} catch {
// If LLM compaction fails, fall back to basic compaction silently
return false;
}
}
/**
* Check whether LLM compaction should run at the current step.
* This is a convenience check; the caller can use it to decide whether
* to call compactWithLlm().
*/
shouldCompactWithLlm(): boolean {
const config = this.options.compaction;
if (!config || config.interval <= 0) return false;
return (
this.currentStep - this.lastCompactionStep >= config.interval &&
this.estimateTotalTokens() > (config.targetTokens ?? this.options.contextWindowSize * 0.6)
);
}
// ────────────────────────────────────────
// History Items & Description
// ────────────────────────────────────────
private recordConversationEntry(
step: number,
category: MessageCategory,
content: string,
hasScreenshot?: boolean,
): void {
this.historyItems.push({
step,
category,
summary: truncate(content, 120),
content: truncate(content, 2000),
hasScreenshot,
timestamp: Date.now(),
});
}
/**
* Build a human-readable description of the agent's history,
* with "N steps omitted" truncation for long histories.
*
* @param stepLimitShown Maximum number of steps to show in full detail.
* If the history is longer, middle steps are replaced with a "N steps omitted" line.
*/
agentHistoryDescription(stepLimitShown = 10): string {
// Group history items by step
const byStep = new Map<number, ConversationEntry[]>();
for (const item of this.historyItems) {
const existing = byStep.get(item.step);
if (existing) {
existing.push(item);
} else {
byStep.set(item.step, [item]);
}
}
const stepNumbers = [...byStep.keys()].sort((a, b) => a - b);
if (stepNumbers.length === 0) return '(no history)';
const lines: string[] = [];
if (stepNumbers.length <= stepLimitShown) {
// Show all steps
for (const stepNum of stepNumbers) {
lines.push(this.formatStepDescription(stepNum, byStep.get(stepNum)!));
}
} else {
// Show first few, omitted middle, last few
const headCount = Math.ceil(stepLimitShown / 2);
const tailCount = stepLimitShown - headCount;
const headSteps = stepNumbers.slice(0, headCount);
const tailSteps = stepNumbers.slice(stepNumbers.length - tailCount);
const omittedCount = stepNumbers.length - headCount - tailCount;
for (const stepNum of headSteps) {
lines.push(this.formatStepDescription(stepNum, byStep.get(stepNum)!));
}
lines.push(` ... (${omittedCount} steps omitted) ...`);
for (const stepNum of tailSteps) {
lines.push(this.formatStepDescription(stepNum, byStep.get(stepNum)!));
}
}
return lines.join('\n');
}
private formatStepDescription(step: number, items: ConversationEntry[]): string {
const parts = items.map((item) => {
const prefix = item.category === 'state' ? 'State' :
item.category === 'assistant' ? 'Agent' :
item.category === 'action_result' ? 'Result' :
item.category === 'user' ? 'User' : item.category;
return `${prefix}: ${item.summary}`;
});
return `Step ${step}:\n ${parts.join('\n ')}`;
}
/** Get all recorded history items. */
getConversationEntrys(): readonly ConversationEntry[] {
return this.historyItems;
}
// ────────────────────────────────────────
// Save / Load (Conversation Persistence)
// ────────────────────────────────────────
/**
* Serialize the current state to a persistence-friendly snapshot.
* Screenshots are stripped (replaced with placeholder text) to keep size manageable.
*/
save(): ConversationManagerState {
const serialized: SerializedTrackedMessage[] = this.messages.map((m) => ({
role: m.message.role,
content: extractTextContent(m.message),
isCompactable: m.isCompactable,
tokenEstimate: m.tokenEstimate,
step: m.step,
category: m.category,
}));
return {
systemPrompt: this.systemPromptText,
messages: serialized,
historyItems: [...this.historyItems],
currentStep: this.currentStep,
};
}
/**
* Restore the ConversationManager from a previously saved state.
* This replaces all current messages and history.
*/
load(state: ConversationManagerState): void {
if (state.systemPrompt) {
this.setInstructionBuilder(state.systemPrompt);
} else {
this.systemPromptMessage = null;
this.systemPromptText = null;
}
this.messages = state.messages.map((s) => ({
message:
s.role === 'assistant'
? assistantMessage(s.content)
: userMessage(s.content),
isCompactable: s.isCompactable,
tokenEstimate: s.tokenEstimate,
step: s.step,
category: s.category,
addedAt: Date.now(),
}));
this.historyItems = [...state.historyItems];
this.currentStep = state.currentStep;
}
/**
* Save the conversation state to a JSON file.
*/
async saveToFile(filePath: string): Promise<string> {
const { writeFile, mkdir } = await import('node:fs/promises');
const { dirname } = await import('node:path');
await mkdir(dirname(filePath), { recursive: true });
const json = JSON.stringify(this.save(), null, 2);
await writeFile(filePath, json, 'utf-8');
return filePath;
}
/**
* Load conversation state from a JSON file.
*/
async loadFromFile(filePath: string): Promise<void> {
const { readFile } = await import('node:fs/promises');
const raw = await readFile(filePath, 'utf-8');
const state = JSON.parse(raw) as ConversationManagerState;
this.load(state);
}
// ────────────────────────────────────────
// Accessors
// ────────────────────────────────────────
get messageCount(): number {
return this.messages.length + (this.systemPromptMessage ? 1 : 0);
}
get step(): number {
return this.currentStep;
}
clear(): void {
this.messages = [];
this.historyItems = [];
this.currentStep = 0;
this.lastCompactionStep = 0;
}
/**
* Remove all messages but preserve history items and step counter.
* Useful when restarting message context without losing the history summary.
*/
resetMessages(): void {
this.messages = [];
this.lastCompactionStep = 0;
}
}
================================================
FILE: packages/core/src/agent/conversation/types.ts
================================================
import type { Message } from '../../model/messages.js';
import type { CompactionPolicy } from '../types.js';
import type { LanguageModel } from '../../model/interface.js';
// ── Message Manager Options ──
export interface ConversationManagerOptions {
contextWindowSize: number;
estimateTokens?: (text: string) => number;
includeLastScreenshot: boolean;
/** Sensitive key-value pairs to mask in outgoing messages. */
maskedValues?: Record<string, string>;
/** LLM-based compaction configuration. */
compaction?: CompactionPolicy;
/** LanguageModel used for LLM-based compaction. Ignored if compaction is not set. */
compactionModel?: LanguageModel;
}
// ── Managed Message ──
export type MessageCategory =
| 'system'
| 'state'
| 'action_result'
| 'assistant'
| 'user'
| 'compaction_summary';
export interface TrackedMessage {
message: Message;
isCompactable: boolean;
tokenEstimate: number;
step?: number;
/** Semantic category for structured history tracking. */
category?: MessageCategory;
/** When true, this message is only included on the next getMessages() call then removed. */
ephemeral?: boolean;
/** When true, this message has already been read (consumed) in an ephemeral pass. */
ephemeralRead?: boolean;
/** Timestamp when this message was added. */
addedAt?: number;
}
// ── History Item ──
/**
* A structured entry in the agent's conversation history, richer than TrackedMessage.
* Used for building human-readable summaries and for save/load.
*/
export interface ConversationEntry {
/** Step number this item belongs to. */
step: number;
/** Category of this history item. */
category: MessageCategory;
/** Brief human-readable summary of this item (e.g. "Clicked element 5" or "Navigated to google.com"). */
summary: string;
/** The full text content (truncated for large payloads). */
content?: string;
/** Whether this item included a screenshot. */
hasScreenshot?: boolean;
/** Timestamp. */
timestamp: number;
}
// ── Message Manager State (persistence) ──
/**
* Serializable snapshot of the ConversationManager for save/load.
*/
export interface ConversationManagerState {
systemPrompt: string | null;
messages: SerializedTrackedMessage[];
historyItems: ConversationEntry[];
/** Step count at the time of snapshot. */
currentStep: number;
}
/**
* Serializable form of TrackedMessage (Message content may contain base64
* screenshots, which are replaced with placeholders during serialization).
*/
export interface SerializedTrackedMessage {
role: string;
content: string;
isCompactable: boolean;
tokenEstimate: number;
step?: number;
category?: MessageCategory;
}
================================================
FILE: packages/core/src/agent/conversation/utils.ts
================================================
import type { Message } from '../../model/messages.js';
import type { ContentPart } from '../../model/messages.js';
/**
* Rough token estimation: ~4 characters per token.
*/
export function estimateTokens(text: string): number {
return Math.ceil(text.length / 4);
}
export function estimateMessageTokens(content: string | unknown[]): number {
if (typeof content === 'string') {
return estimateTokens(content);
}
let total = 0;
for (const part of content) {
if (typeof part === 'object' && part !== null) {
const p = part as Record<string, unknown>;
if (p.type === 'text' && typeof p.text === 'string') {
total += estimateTokens(p.text);
} else if (p.type === 'image') {
total += 1000; // Approximate cost for an image
}
}
}
return total;
}
// ── Sensitive Data Filtering ──
const MASK = '***';
/**
* Replace all occurrences of each sensitive value in `text` with a mask.
* Keys are used only for logging context; values are the secrets to redact.
*/
export function redactSensitiveValues(
text: string,
maskedValues: Record<string, string>,
): string {
let result = text;
for (const [_key, value] of Object.entries(maskedValues)) {
if (!value) continue;
// Escape regex special characters in the value
const escaped = value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
result = result.replace(new RegExp(escaped, 'g'), MASK);
}
return result;
}
/**
* Deep-filter a Message, masking any sensitive values found in text content.
* Returns a new message (does not mutate the original).
*/
export function redactMessage(
message: Message,
maskedValues: Record<string, string>,
): Message {
const entries = Object.entries(maskedValues);
if (entries.length === 0) return message;
const content = message.content;
if (typeof content === 'string') {
return {
...message,
content: redactSensitiveValues(content, maskedValues),
} as Message;
}
if (Array.isArray(content)) {
const filtered = (content as ContentPart[]).map((part) => {
if (part.type === 'text') {
return {
...part,
text: redactSensitiveValues(part.text, maskedValues),
};
}
// Images are left as-is (binary data)
return part;
});
return {
...message,
content: filtered,
} as Message;
}
return message;
}
/**
* Filter an array of Messages, masking sensitive data in each.
*/
export function redactMessages(
messages: Message[],
maskedValues: Record<string, string>,
): Message[] {
if (Object.keys(maskedValues).length === 0) return messages;
return messages.map((m) => redactMessage(m, maskedValues));
}
/**
* Extract the text content from a Message as a plain string.
* For multi-part content, concatenates all text parts.
*/
export function extractTextContent(message: Message): string {
const content = message.content;
if (typeof content === 'string') return content;
if (Array.isArray(content)) {
return (content as ContentPart[])
.filter((p): p is Extract<ContentPart, { type: 'text' }> => p.type === 'text')
.map((p) => p.text)
.join('\n');
}
return '';
}
/**
* Truncate a string to maxLen characters, appending an ellipsis if truncated.
*/
export function truncate(text: string, maxLen: number): string {
if (text.length <= maxLen) return text;
return `${text.slice(0, maxLen - 3)}...`;
}
================================================
FILE: packages/core/src/agent/conversation.test.ts
================================================
import { test, expect, describe, beforeEach } from 'bun:test';
import { ConversationManager } from './conversation/service.js';
import type { ConversationManagerOptions } from './conversation/types.js';
import type { LanguageModel, InferenceOptions } from '../model/interface.js';
import type { InferenceResult } from '../model/types.js';
// ── Helpers ──
function createManager(
overrides: Partial<ConversationManagerOptions> = {},
): ConversationManager {
return new ConversationManager({
contextWindowSize: 10000,
includeLastScreenshot: true,
...overrides,
});
}
function createMockModel(summary = 'Summary of the conversation'): LanguageModel {
return {
modelId: 'test-model',
provider: 'custom',
invoke: async <T>(_options: InferenceOptions<T>): Promise<InferenceResult<T>> => {
return {
parsed: { summary } as unknown as T,
usage: { inputTokens: 100, outputTokens: 50, totalTokens: 150 },
finishReason: 'stop',
};
},
};
}
// ── Tests ──
describe('ConversationManager', () => {
let mm: ConversationManager;
beforeEach(() => {
mm = createManager();
});
describe('system prompt', () => {
test('setInstructionBuilder stores the system prompt', () => {
mm.setInstructionBuilder('You are a helpful assistant');
const messages = mm.getMessages();
expect(messages[0]).toEqual({
role: 'system',
content: 'You are a helpful assistant',
});
});
test('system prompt appears first in getMessages', () => {
mm.setInstructionBuilder('System');
mm.addStateMessage('State text', undefined, 1);
const messages = mm.getMessages();
expect(messages[0].role).toBe('system');
expect(messages[1].role).toBe('user');
});
test('changing system prompt replaces the previous one', () => {
mm.setInstructionBuilder('First');
mm.setInstructionBuilder('Second');
const messages = mm.getMessages();
const systemMessages = messages.filter((m) => m.role === 'system');
expect(systemMessages).toHaveLength(1);
expect(systemMessages[0].content).toBe('Second');
});
});
describe('addStateMessage', () => {
test('adds a user message with state text', () => {
mm.addStateMessage('Page state info', undefined, 1);
const messages = mm.getMessages();
expect(messages).toHaveLength(1);
expect(messages[0].role).toBe('user');
});
test('includes screenshot when provided and vision enabled', () => {
mm.addStateMessage('State', 'base64screenshot', 1);
const messages = mm.getMessages();
const content = messages[0].content;
expect(Array.isArray(content)).toBe(true);
if (Array.isArray(content)) {
expect(content).toHaveLength(2);
expect(content[0]).toEqual({ type: 'text', text: 'State' });
expect(content[1]).toHaveProperty('type', 'image');
}
});
test('excludes screenshot when vision disabled', () => {
const noVision = createManager({ includeLastScreenshot: false });
noVision.addStateMessage('State', 'base64screenshot', 1);
const messages = noVision.getMessages();
const content = messages[0].content;
// Content should be text-only array
expect(Array.isArray(content)).toBe(true);
if (Array.isArray(content)) {
expect(content).toHaveLength(1);
expect(content[0]).toHaveProperty('type', 'text');
}
});
test('updates messageCount', () => {
expect(mm.messageCount).toBe(0);
mm.addStateMessage('State 1', undefined, 1);
expect(mm.messageCount).toBe(1);
mm.addStateMessage('State 2', undefined, 2);
expect(mm.messageCount).toBe(2);
});
});
describe('addAssistantMessage', () => {
test('adds an assistant role message', () => {
mm.addAssistantMessage('Agent response', 1);
const messages = mm.getMessages();
expect(messages[0].role).toBe('assistant');
expect(messages[0].content).toBe('Agent response');
});
});
describe('addCommandResultMessage', () => {
test('adds a user role message for action results', () => {
mm.addCommandResultMessage('click: success', 1);
const messages = mm.getMessages();
expect(messages[0].role).toBe('user');
expect(messages[0].content).toBe('click: success');
});
});
describe('getMessages ordering', () => {
test('returns messages in correct order', () => {
mm.setInstructionBuilder('System prompt');
mm.addStateMessage('State text', undefined, 1);
mm.addAssistantMessage('Agent thought', 1);
mm.addCommandResultMessage('Action result', 1);
const messages = mm.getMessages();
expect(messages).toHaveLength(4);
expect(messages[0].role).toBe('system');
expect(messages[1].role).toBe('user');
expect(messages[2].role).toBe('assistant');
expect(messages[3].role).toBe('user');
});
});
describe('compaction - screenshot removal', () => {
test('removes old screenshots when over token budget, keeps last', () => {
// 3 screenshots: each ~1000 tokens for image + ~2 for text = ~3006 total.
// Budget of 1500: after removing 2 old screenshots (saving 2000),
// total becomes ~1006 < 1500, so compact exits successfully.
const small = createManager({ contextWindowSize: 1500 });
small.addStateMessage('State 1', 'screenshot1', 1);
small.addStateMessage('State 2', 'screenshot2', 2);
small.addStateMessage('State 3', 'screenshot3', 3);
const messages = small.getMessages();
// After compaction, older screenshots should be removed
// The last message should still have its image
const lastMessage = messages[messages.length - 1];
const lastContent = lastMessage.content;
expect(Array.isArray(lastContent)).toBe(true);
if (Array.isArray(lastContent)) {
const hasImage = lastContent.some(
(p: any) => typeof p === 'object' && p.type === 'image',
);
expect(hasImage).toBe(true);
// Older messages should have had their images removed
const firstMsg = messages[0];
const firstContent = firstMsg.content;
if (Array.isArray(firstContent)) {
const firstHasImage = firstContent.some(
(p: any) => typeof p === 'object' && p.type === 'image',
);
expect(firstHasImage).toBe(false);
}
}
});
});
describe('compaction - token budget behavior', () => {
test('does not trigger compaction when under budget', () => {
// Budget of 10000 means no compaction needed for a few messages
const large = createManager({ contextWindowSize: 10000, includeLastScreenshot: false });
large.addStateMessage('Short state', undefined, 1);
large.addAssistantMessage('Short response', 1);
const messages = large.getMessages();
// No summaries should be present
const summaryMessages = messages.filter(
(m) =>
typeof m.content === 'string' &&
m.content.includes('omitted to save tokens'),
);
expect(summaryMessages).toHaveLength(0);
});
test('estimateTotalTokens reflects actual message content', () => {
const mm2 = createManager({ contextWindowSize: 100000, includeLastScreenshot: false });
mm2.addStateMessage('A'.repeat(400), undefined, 1); // ~100 tokens
mm2.addStateMessage('B'.repeat(800), undefined, 2); // ~200 tokens
const total = mm2.estimateTotalTokens();
// Total should be roughly 300 tokens for 1200 chars
expect(total).toBeGreaterThanOrEqual(250);
expect(total).toBeLessThanOrEqual(400);
});
});
describe('token estimation', () => {
test('estimateTotalTokens includes system prompt', () => {
mm.setInstructionBuilder('System prompt text');
const tokensWithSystem = mm.estimateTotalTokens();
expect(tokensWithSystem).toBeGreaterThan(0);
});
test('estimateTotalTokens grows with messages', () => {
const before = mm.estimateTotalTokens();
mm.addStateMessage('Some state text', undefined, 1);
const after = mm.estimateTotalTokens();
expect(after).toBeGreaterThan(before);
});
test('estimateTotalTokens counts images as ~1000 tokens', () => {
mm.addStateMessage('Text', 'screenshot', 1);
const tokens = mm.estimateTotalTokens();
// Text ~4 chars = 1 token, plus ~1000 for image
expect(tokens).toBeGreaterThanOrEqual(1000);
});
});
describe('history items', () => {
test('records history for each added message', () => {
mm.addStateMessage('State text', undefined, 1);
mm.addAssistantMessage('Agent response', 1);
mm.addCommandResultMessage('Result text', 1);
const items = mm.getConversationEntrys();
expect(items).toHaveLength(3);
expect(items[0].category).toBe('state');
expect(items[1].category).toBe('assistant');
expect(items[2].category).toBe('action_result');
});
test('history items include step number', () => {
mm.addStateMessage('State', undefined, 5);
const items = mm.getConversationEntrys();
expect(items[0].step).toBe(5);
});
test('history items include truncated summary', () => {
const longText = 'a'.repeat(200);
mm.addStateMessage(longText, undefined, 1);
const items = mm.getConversationEntrys();
// Summary should be truncated to 120 chars
expect(items[0].summary.length).toBeLessThanOrEqual(123); // 120 + '...'
});
test('history items track screenshot presence', () => {
mm.addStateMessage('State', 'screenshot_data', 1);
const items = mm.getConversationEntrys();
expect(items[0].hasScreenshot).toBe(true);
});
});
describe('agentHistoryDescription', () => {
test('returns "(no history)" when empty', () => {
expect(mm.agentHistoryDescription()).toBe('(no history)');
});
test('shows all steps when under stepLimitShown', () => {
mm.addStateMessage('State 1', undefined, 1);
mm.addAssistantMessage('Agent 1', 1);
mm.addStateMessage('State 2', undefined, 2);
mm.addAssistantMessage('Agent 2', 2);
const desc = mm.agentHistoryDescription(10);
expect(desc).toContain('Step 1:');
expect(desc).toContain('Step 2:');
});
test('truncates with "steps omitted" when exceeding stepLimitShown', () => {
for (let i = 1; i <= 20; i++) {
mm.addStateMessage(`State ${i}`, undefined, i);
mm.addAssistantMessage(`Agent ${i}`, i);
}
const desc = mm.agentHistoryDescription(4);
expect(desc).toContain('steps omitted');
// Should show first 2 and last 2 steps
expect(desc).toContain('Step 1:');
expect(desc).toContain('Step 2:');
expect(desc).toContain('Step 19:');
expect(desc).toContain('Step 20:');
});
test('includes category prefixes in description', () => {
mm.addStateMessage('Page loaded', undefined, 1);
mm.addAssistantMessage('Clicking button', 1);
mm.addCommandResultMessage('click: success', 1);
const desc = mm.agentHistoryDescription();
expect(desc).toContain('State:');
expect(desc).toContain('Agent:');
expect(desc).toContain('Result:');
});
});
describe('ephemeral messages', () => {
test('ephemeral message appears in first getMessages call', () => {
mm.addEphemeralMessage('Temporary instruction');
const messages = mm.getMessages();
const found = messages.some(
(m) => typeof m.content === 'string' && m.content === 'Temporary instruction',
);
expect(found).toBe(true);
});
test('ephemeral message is removed after being consumed', () => {
mm.addEphemeralMessage('Temp');
// First call: message is present and gets marked as read
const first = mm.getMessages();
expect(first.some((m) => typeof m.content === 'string' && m.content === 'Temp')).toBe(true);
// Second call: message is still in result (removal happens after building result),
// then gets removed during consumeEphemeralMessages
const second = mm.getMessages();
// Third call: message is now actually gone from this.messages
const third = mm.getMessages();
const found = third.some(
(m) => typeof m.content === 'string' && m.content === 'Temp',
);
expect(found).toBe(false);
});
test('ephemeral message with assistant role', () => {
mm.addEphemeralMessage('Agent thought', 'assistant');
const messages = mm.getMessages();
const found = messages.find(
(m) => m.role === 'assistant' && m.content === 'Agent thought',
);
expect(found).toBeDefined();
});
test('multiple ephemeral messages all appear then get cleaned up', () => {
mm.addEphemeralMessage('Temp 1');
mm.addEphemeralMessage('Temp 2');
// First call: both present, marked as read
const first = mm.getMessages();
expect(first).toHaveLength(2);
// Second call: still in result (removal after build), then removed
mm.getMessages();
// Third call: messages have been removed
const third = mm.getMessages();
expect(third).toHaveLength(0);
});
});
describe('save / load round-trip', () => {
test('save and load preserves system prompt', () => {
mm.setInstructionBuilder('My system prompt');
mm.addStateMessage('State 1', undefined, 1);
const saved = mm.save();
const restored = createManager();
restored.load(saved);
const messages = restored.getMessages();
expect(messages[0].role).toBe('system');
expect(messages[0].content).toBe('My system prompt');
});
test('save and load preserves messages', () => {
mm.addStateMessage('State 1', undefined, 1);
mm.addAssistantMessage('Response 1', 1);
mm.addCommandResultMessage('Result 1', 1);
const saved = mm.save();
const restored = createManager();
restored.load(saved);
const messages = restored.getMessages();
expect(messages).toHaveLength(3);
expect(messages[0].role).toBe('user');
expect(messages[1].role).toBe('assistant');
expect(messages[2].role).toBe('user');
});
test('save and load preserves history items', () => {
mm.addStateMessage('State 1', undefined, 1);
mm.addAssistantMessage('Response 1', 1);
const saved = mm.save();
const restored = createManager();
restored.load(saved);
const items = restored.getConversationEntrys();
expect(items).toHaveLength(2);
expect(items[0].category).toBe('state');
expect(items[1].category).toBe('assistant');
});
test('save and load preserves currentStep', () => {
mm.addStateMessage('State', undefined, 7);
const saved = mm.save();
expect(saved.currentStep).toBe(7);
const restored = createManager();
restored.load(saved);
expect(restored.step).toBe(7);
});
test('save strips screenshots (text only in serialized form)', () => {
mm.addStateMessage('State with screenshot', 'base64data', 1);
const saved = mm.save();
// Serialized content should be text-only, no base64
for (const msg of saved.messages) {
expect(msg.content).not.toContain('base64data');
}
});
test('load with null system prompt clears system prompt', () => {
mm.setInstructionBuilder('Initial prompt');
const saved = mm.save();
saved.systemPrompt = null;
mm.load(saved);
const messages = mm.getMessages();
const hasSystem = messages.some((m) => m.role === 'system');
expect(hasSystem).toBe(false);
});
});
describe('sensitive data filtering', () => {
test('masks sensitive values in outgoing messages', () => {
const sensitive = createManager({
maskedValues: { password: 'secret123', apiKey: 'key-abc' },
});
sensitive.addStateMessage('Login with password secret123', undefined, 1);
sensitive.addAssistantMessage('Using key-abc to authenticate', 1);
const messages = sensitive.getMessages();
// Text should have been masked
const stateMsg = messages[0];
if (typeof stateMsg.content === 'string') {
expect(stateMsg.content).not.toContain('secret123');
expect(stateMsg.content).toContain('***');
} else if (Array.isArray(stateMsg.content)) {
const textPart = stateMsg.content.find((p: any) => p.type === 'text');
expect((textPart as any).text).not.toContain('secret123');
}
const assistantMsg = messages[1];
if (typeof assistantMsg.content === 'string') {
expect(assistantMsg.content).not.toContain('key-abc');
expect(assistantMsg.content).toContain('***');
}
});
test('no filtering when maskedValues is empty', () => {
const noSensitive = createManager({ maskedValues: {} });
noSensitive.addStateMessage('Plain text with secret123', undefined, 1);
const messages = noSensitive.getMessages();
const content = messages[0].content;
if (Array.isArray(content)) {
const textPart = content.find((p: any) => p.type === 'text');
expect((textPart as any).text).toContain('secret123');
}
});
test('no filtering when maskedValues is not set', () => {
mm.addStateMessage('Text with sensitive data', undefined, 1);
const messages = mm.getMessages();
const content = messages[0].content;
if (Array.isArray(content)) {
const textPart = content.find((p: any) => p.type === 'text');
expect((textPart as any).text).toContain('sensitive data');
}
});
});
describe('LLM-based compaction', () => {
test('shouldCompactWithLlm returns false when no compaction config', () => {
expect(mm.shouldCompactWithLlm()).toBe(false);
});
test('shouldCompactWithLlm returns false when interval not reached', () => {
const withCompaction = createManager({
compaction: { interval: 10, maxTokens: 500 },
});
// Only 1 message, interval not reached
withCompaction.addStateMessage('State', undefined, 1);
expect(withCompaction.shouldCompactWithLlm()).toBe(false);
});
test('compactWithLlm returns false without a model', async () => {
const withCompaction = createManager({
contextWindowSize: 100000,
includeLastScreenshot: false,
compaction: { interval: 1, maxTokens: 500, targetTokens: 10 },
});
// Add enough messages so estimateTotalTokens > targetTokens (10)
for (let i = 1; i <= 5; i++) {
withCompaction.addStateMessage('x'.repeat(100), undefined, i);
}
const result = await withCompaction.compactWithLlm();
expect(result).toBe(false);
});
test('compactWithLlm performs compaction with model', async () => {
const model = createMockModel('Summarized: visited pages and clicked buttons');
// Use large contextWindowSize so getMessages() doesn't trigger basic compact(),
// but low targetTokens so the LLM compaction decides to run.
const longText = 'A'.repeat(500);
const withCompaction = createManager({
contextWindowSize: 100000,
includeLastScreenshot: false,
compaction: { interval: 1, maxTokens: 500, targetTokens: 500 },
});
// Add lots of messages to exceed targetTokens (500).
// Each 500-char message = ~125 tokens. 10 messages = ~1250 tokens > 500.
for (let i = 1; i <= 10; i++) {
withCompaction.addStateMessage(`${longText} step ${i}`, undefined, i);
withCompaction.addAssistantMessage(`${longText} response ${i}`, i);
}
const result = await withCompaction.compactWithLlm(model);
expect(result).toBe(true);
// After compaction, message count should be reduced
const messages = withCompaction.getMessages();
expect(messages.length).toBeLessThan(20);
// First message should be the summary
const firstContent = messages[0].content;
expect(typeof firstContent).toBe('string');
expect(firstContent as string).toContain('Conversation summary');
});
});
describe('clear and resetMessages', () => {
test('clear removes all messages and history', () => {
mm.setInstructionBuilder('System');
mm.addStateMessage('State', undefined, 1);
mm.addAssistantMessage('Response', 1);
mm.clear();
expect(mm.messageCount).toBe(1); // system prompt still present via setInstructionBuilder
expect(mm.getConversationEntrys()).toHaveLength(0);
expect(mm.step).toBe(0);
});
test('resetMessages removes messages but preserves history', () => {
mm.addStateMessage('State', undefined, 1);
mm.addAssistantMessage('Response', 1);
const historyBefore = mm.getConversationEntrys().length;
mm.resetMessages();
// Messages cleared
const messages = mm.getMessages();
expect(messages).toHaveLength(0);
// History preserved
expect(mm.getConversationEntrys()).toHaveLength(historyBefore);
});
});
describe('messageCount', () => {
test('includes system prompt in count', () => {
mm.setInstructionBuilder('System');
expect(mm.messageCount).toBe(1);
mm.addStateMessage('State', undefined, 1);
expect(mm.messageCount).toBe(2);
});
test('does not count system prompt when not set', () => {
expect(mm.messageCount).toBe(0);
mm.addStateMessage('State', undefined, 1);
expect(mm.messageCount).toBe(1);
});
});
describe('step tracking', () => {
test('step reflects the most recent step from added messages', () => {
mm.addStateMessage('State 1', undefined, 1);
expect(mm.step).toBe(1);
mm.addStateMessage('State 5', undefined, 5);
expect(mm.step).toBe(5);
});
});
});
================================================
FILE: packages/core/src/agent/evaluator.ts
================================================
import type { LanguageModel } from '../model/interface.js';
import type { Message, ContentPart } from '../model/messages.js';
import { systemMessage, userMessage, imageContent, textContent } from '../model/messages.js';
import {
EvaluationResultSchema,
QuickCheckResultSchema,
type EvaluationResult,
type QuickCheckResult,
type StepRecord,
} from './types.js';
import { createLogger } from '../logging.js';
const logger = createLogger('judge');
// ── Judge System Prompts ──
const JUDGE_SYSTEM_PROMPT = `You are an expert task completion judge. Your job is to evaluate whether a web browser automation agent completed its assigned task successfully.
You will be provided with:
1. The task description
2. A history of steps the agent took (including actions and their results)
3. Screenshots from during execution (if available)
4. Optionally, ground truth information about the expected result
Evaluate thoroughly:
- Did the agent actually complete the task, or just claim to?
- Is the extracted information correct and complete?
- Did the agent handle errors and edge cases appropriately?
- Was the agent stuck at any point without recovery?
If ground truth is provided, compare the agent's result against it.
Be strict but fair. Partial completions should be marked with lower confidence.`;
const SIMPLE_JUDGE_SYSTEM_PROMPT = `You are a quick-check validator for web browser automation results.
Given a task and the agent's final result, determine if the result appears correct.
Be concise. Focus on whether the result directly answers/completes the task.`;
export class ResultEvaluator {
private model: LanguageModel;
constructor(model: LanguageModel) {
this.model = model;
}
/**
* Full evaluation with step history, screenshots, and optional ground truth.
* Provides detailed verdict with failure analysis.
*/
async evaluate(
task: string,
result: string,
history: StepRecord[],
options?: {
expectedOutcome?: string;
includeScreenshots?: boolean;
},
): Promise<EvaluationResult> {
const messages = constructEvaluatorMessages(task, result, history, options);
try {
const completion = await this.model.invoke({
messages,
responseSchema: EvaluationResultSchema,
schemaName: 'EvaluationResult',
temperature: 0,
});
logger.info(
`Judge verdict: complete=${completion.parsed.isComplete}, ` +
`confidence=${completion.parsed.confidence}, ` +
`verdict=${completion.parsed.verdict ?? 'n/a'}`,
);
return completion.parsed;
} catch (error) {
logger.error('Judge evaluation failed', error);
return {
isComplete: false,
reason: `Judge evaluation failed: ${error instanceof Error ? error.message : String(error)}`,
confidence: 0,
verdict: 'unknown',
};
}
}
/**
* Lightweight always-on validation.
* Quick pass/fail check without detailed history analysis.
* Useful for running after every "done" action to catch obvious errors.
*/
async simpleEvaluate(
task: string,
result: string,
): Promise<QuickCheckResult> {
const messages = constructQuickCheckMessages(task, result);
try {
const completion = await this.model.invoke({
messages,
responseSchema: QuickCheckResultSchema,
schemaName: 'QuickCheckResult',
temperature: 0,
});
logger.debug(
`Simple judge: passed=${completion.parsed.passed}, reason=${completion.parsed.reason}`,
);
return completion.parsed;
} catch (error) {
logger.error('Simple judge evaluation failed', error);
return {
passed: true, // Default to pass on error to avoid blocking
reason: `Simple judge failed: ${error instanceof Error ? error.message : String(error)}`,
shouldRetry: false,
};
}
}
}
// ── Message Construction ──
/**
* Build the full message array for detailed judge evaluation.
* Includes step-by-step history, screenshots (if enabled), and ground truth.
*/
export function constructEvaluatorMessages(
task: string,
result: string,
history: StepRecord[],
options?: {
expectedOutcome?: string;
includeScreenshots?: boolean;
},
): Message[] {
const messages: Message[] = [
systemMessage(JUDGE_SYSTEM_PROMPT),
];
// Build the evaluation prompt
const parts: string[] = [];
parts.push(`## Task\n${task}`);
parts.push(`## Agent's Final Result\n${result}`);
// Step history summary
if (history.length > 0) {
const stepSummaries: string[] = [];
for (const entry of history) {
const actions = entry.agentOutput.actions
.map((a) => {
const actionObj = a as Record<string, unknown>;
return actionObj.action ?? 'unknown';
})
.join(', ');
const results = entry.actionResults
.map((r) => {
if (r.isDone) return `DONE: ${r.extractedContent?.slice(0, 200) ?? ''}`;
if (r.error) return `ERROR: ${r.error.slice(0, 150)}`;
if (r.extractedContent) return `OK: ${r.extractedContent.slice(0, 150)}`;
return r.success ? 'OK' : 'FAILED';
})
.join('; ');
const evaluation = entry.agentOutput.currentState?.evaluation ?? '';
stepSummaries.push(
`Step ${entry.step} [${entry.browserState.url}]:\n` +
` Eval: ${evaluation.slice(0, 200)}\n` +
` Actions: ${actions}\n` +
` Results: ${results}`,
);
}
parts.push(`## Step History (${history.length} steps)\n${stepSummaries.join('\n\n')}`);
}
// Ground truth
if (options?.expectedOutcome) {
parts.push(
`## Ground Truth (Expected Result)\n${options.expectedOutcome}\n\n` +
'Compare the agent\'s result against this ground truth carefully.',
);
}
parts.push(
'## Instructions\n' +
'Evaluate the task completion. Provide:\n' +
'- isComplete: whether the task was fully completed\n' +
'- reason: detailed explanation\n' +
'- confidence: 0-1 score\n' +
'- verdict: "success", "partial", "failed", or "unknown"\n' +
'- failureReason: if failed, explain why\n' +
'- impossibleTask: true if the task appears impossible\n' +
'- reachedCaptcha: true if a CAPTCHA blocked progress',
);
// If screenshots are requested and available, include the last few
if (options?.includeScreenshots) {
const screenshotEntries = history
.filter((e) => e.browserState.screenshot)
.slice(-3); // Last 3 screenshots
if (screenshotEntries.length > 0) {
const content: ContentPart[] = [
textContent(`${parts.join('\n\n')}\n\nBelow are screenshots from the agent's execution:`),
];
for (const entry of screenshotEntries) {
if (entry.browserState.screenshot) {
content.push(
textContent(`Screenshot from step ${entry.step} (${entry.browserState.url}):`),
);
content.push(imageContent(entry.browserState.screenshot));
}
}
messages.push(userMessage(content));
return messages;
}
}
messages.push(userMessage(parts.join('\n\n')));
return messages;
}
/**
* Build messages for lightweight simple judge evaluation.
* Only includes task and result -- no history or screenshots.
*/
export function constructQuickCheckMessages(
task: string,
result: string,
): Message[] {
return [
systemMessage(SIMPLE_JUDGE_SYSTEM_PROMPT),
userMessage(
`Task: ${task}\n\n` +
`Agent's Result: ${result}\n\n` +
'Does this result correctly complete the task? ' +
'If not, should the agent retry with a different approach?',
),
];
}
================================================
FILE: packages/core/src/agent/index.ts
================================================
export { Agent, type AgentOptions } from '../agent/agent.js';
export {
InstructionBuilder,
StepPromptBuilder,
buildCommandDescriptions,
buildContextualCommands,
buildExtractionInstructionBuilder,
buildExtractionUserPrompt,
clearTemplateCache,
type PromptTemplate,
type InstructionBuilderOptions,
type StepInfo,
type StepPromptBuilderOptions,
} from './instructions.js';
export { ConversationManager } from './conversation/service.js';
export {
StallDetector,
hashPageTree,
hashTextContent,
type PageSignature,
type StallDetectorConfig,
type StallCheckResult,
} from './stall-detector.js';
export {
ResultEvaluator,
constructEvaluatorMessages,
constructQuickCheckMessages,
} from './evaluator.js';
export { ReplayRecorder, type ReplayRecorderOptions } from './replay-recorder.js';
export {
type AgentConfig,
type AgentState,
type AgentDecision,
type AgentDecisionCompact,
type AgentDecisionDirect,
type StepRecord,
ExecutionLog,
type RunOutcome,
type Reasoning,
type PlanStep,
type EvaluationResult,
type QuickCheckResult,
type CompactionPolicy,
type StepTelemetry,
type ExtractedVariable,
type AccumulatedCost,
type StepCostBreakdown,
type PricingTable,
type PlanRevision,
AgentDecisionSchema,
AgentDecisionCompactSchema,
AgentDecisionDirectSchema,
ReasoningSchema,
EvaluationResultSchema,
QuickCheckResultSchema,
PlanStepSchema,
StrategyPlanSchema,
PlanRevisionSchema,
PRICING_TABLE,
calculateStepCost,
supportsDeepReasoning,
supportsCoordinateMode,
isCompactModel,
DEFAULT_AGENT_CONFIG,
} from './types.js';
export type {
ConversationManagerOptions,
TrackedMessage,
ConversationManagerState,
ConversationEntry,
SerializedTrackedMessage,
MessageCategory,
} from './conversation/types.js';
export {
estimateTokens,
estimateMessageTokens,
redactSensitiveValues,
redactMessage,
redactMessages,
extractTextContent,
truncate,
} from './conversation/utils.js';
================================================
FILE: packages/core/src/agent/instructions/instructions-compact.md
================================================
You are an AI agent that controls a web browser to complete tasks. You operate in an iterative loop: observe, decide, act, repeat.
Your task: {{task}}
<language_settings>Default: English. Match the task's language.</language_settings>
<browser_state>
Elements: `[index]<type>text</type>`. Only `[indexed]` elements are interactive. Indentation = child. `*[` = new element.
</browser_state>
<rules>
- Only interact with elements that have a numeric [index]
- If research is needed, open a **new tab** instead of reusing the current one
- If the page changes after an input action, analyze new elements (e.g., suggestions) before proceeding
- If an action sequence was interrupted, complete remaining actions in the next step
- For autocomplete fields: type text, WAIT for suggestions, click the correct one or press Enter
- Handle popups/modals/cookie banners immediately before other actions
- If blocked by captcha/login/403, try alternative approaches rather than retrying
- ALWAYS look for filter/sort options FIRST when the task specifies criteria
- Detect unproductive loops: if same URL for 3+ steps without progress, change approach
</rules>
<action_rules>
Maximum {{maxActionsPerStep}} actions per step. If the page changes after an action, remaining actions are skipped.
Check browser state each step to verify your previous action succeeded.
When chaining actions, never take consequential actions (form submissions, critical button clicks) without confirming changes occurred.
</action_rules>
<available_actions>
{{actionDescriptions}}
</available_actions>
<efficiency>
Combine actions when sensible. Do not predict actions that do not apply to the current page.
**Recommended combinations:**
- `input_text` + `click` -> Fill field and submit
- `input_text` + `input_text` -> Fill multiple fields
- `click` + `click` -> Multi-step flows (when page does not navigate between clicks)
Do not chain actions that change browser state multiple times (e.g., click then navigate). Always have one clear goal per step.
</efficiency>
<output>
Respond with valid JSON:
```json
{
"currentState": {
"evaluation": "One-sentence analysis of last action. State success, failure, or uncertain.",
"memory": "1-3 sentences: progress tracking, data found, approaches tried.",
"nextGoal": "Next immediate goal in one clear sentence."
},
"actions": [{"action_name": {"param": "value"}}]
}
```
Action list should NEVER be empty.
</output>
<task_completion>
Call `done` when:
- Task is fully completed
- Reached max steps (even if incomplete)
- Absolutely impossible to continue
Set `success=true` ONLY if the full task is completed. Put ALL findings in the `text` field.
Before calling done with success=true: re-read the task, verify every requirement is met, confirm actions completed via page state, ensure no data was fabricated.
</task_completion>
<error_recovery>
1. Verify state using screenshot as ground truth
2. Handle blocking popups/overlays first
3. If element not found, scroll to reveal more content
4. If action fails 2-3 times, try alternative approach
5. If blocked by login/captcha/403, try alternative sites
6. If stuck in a loop, acknowledge and change strategy
</error_recovery>
================================================
FILE: packages/core/src/agent/instructions/instructions-direct.md
================================================
You are an AI agent that controls a web browser to complete tasks. You operate in an iterative loop: observe the current page state, decide on actions, execute them, and repeat until the task is done.
Your task: {{task}}
<capabilities>
You excel at:
1. Navigating complex websites and extracting precise information
2. Automating form submissions and interactive web actions
3. Gathering and organizing information across multiple pages
4. Operating effectively in an iterative agent loop
5. Adapting strategies when encountering obstacles
</capabilities>
<language_settings>
- Default working language: **English**
- Always respond in the same language as the task description
</language_settings>
<input>
At every step, your input will consist of:
1. **Agent history**: A chronological event stream including your previous actions and their results.
2. **Browser state**: Current URL, open tabs, interactive elements indexed for actions, and visible page content.
3. **Screenshot** (when vision is enabled): A screenshot of the current page with bounding boxes around interactive elements.
</input>
<browser_state>
Browser state is given as:
- **Current URL**: The URL of the page you are currently viewing.
- **Open Tabs**: Open tabs with their IDs.
- **Interactive Elements**: All interactive elements in the format `[index]<type>text</type>` where:
- `index`: Numeric identifier for interaction
- `type`: HTML element type (button, input, etc.)
- `text`: Element description
Important notes:
- Only elements with numeric indexes in `[]` are interactive
- Indentation (with tab) means the element is a child of the element above
- Elements tagged with `*[` are **new** interactive elements that appeared since the last step
- Pure text elements without `[]` are not interactive
</browser_state>
<screenshot>
If vision is enabled, you will receive a screenshot of the current page with bounding boxes around interactive elements.
- This is your **ground truth**: use it to evaluate your progress
- If an interactive element has no text in browser_state, its index is at the top center of its bounding box
</screenshot>
<rules>
Strictly follow these rules while using the browser:
- Only interact with elements that have a numeric `[index]`
- Only use indexes that are explicitly provided
- If research is needed, open a **new tab** instead of reusing the current one
- If the page changes after an action, analyze new elements before proceeding
- By default, only elements in the visible viewport are listed
- If the page is not fully loaded, use the wait action
- Use extract_content only if information is NOT visible in browser_state
- extract_content is expensive - do NOT call it multiple times on the same page
- If you fill an input field and your action sequence is interrupted, something changed (e.g., suggestions appeared)
- Complete any remaining actions from interrupted sequences in the next step
- For autocomplete fields: type text, WAIT for suggestions, click the correct one or press Enter
- If the task specifies criteria (price, rating, location, etc.), look for filter/sort options FIRST
- Handle popups, modals, cookie banners immediately before other actions
- If blocked by captcha/login/403, try alternative approaches
- Detect loops: if same URL for 3+ steps without progress, change approach
- Do not log in unless the task requires it and you have credentials
</rules>
<output_format>
## Output Format
Respond with:
1. **currentState**: Your assessment including:
- `evaluation`: Assessment of how the last action went
- `memory`: Important information to remember
- `nextGoal`: The next immediate goal
2. **actions**: A list of actions to execute (max {{maxActionsPerStep}} per step)
</output_format>
<action_rules>
Maximum {{maxActionsPerStep}} actions per step, executed sequentially.
- If the page changes after an action, remaining actions are skipped and you get the new state.
- Check browser state each step to verify your previous action achieved its goal.
- When chaining actions, never take consequential actions without confirming changes occurred.
</action_rules>
<available_actions>
{{actionDescriptions}}
</available_actions>
<efficiency>
Combine actions when sensible. Do not predict actions that do not apply to the current page.
**Recommended combinations:**
- `input_text` + `input_text` + `click` -> Fill multiple fields then submit
- `input_text` + `send_keys` -> Fill a field and press Enter
- `scroll` + `scroll` -> Scroll further down
Do not try multiple paths in one step. Have one clear goal per step.
Place page-changing actions **last** in your action list.
</efficiency>
<reasoning>
Be clear and concise in your decision-making:
1. Analyze the last action result - state success, failure, or uncertain
2. Analyze browser state and screenshot to understand current position
3. If stuck, consider alternative approaches
4. Store concise, actionable context in memory
5. State your next immediate goal clearly
</reasoning>
<task_completion>
Call `done` when:
- Task is fully completed
- Reached max steps (even if incomplete)
- Absolutely impossible to continue
Rules:
- Set `success=true` ONLY if the full task is completed
- Put ALL relevant findings in the `text` field
- Call `done` as a single action - never combine with other actions
**Before calling done with success=true, verify:**
1. Re-read the original task and check every requirement
2. Verify correct count, filters, format
3. Confirm actions completed via page state/screenshot
4. Ensure no fabricated data
5. If anything is unmet or uncertain, set success to false
</task_completion>
<error_recovery>
When encountering errors:
1. Verify state using screenshot as ground truth
2. Check for blocking popups/overlays
3. If element not found, scroll to reveal content
4. If action fails 2-3 times, try alternative approach
5. If blocked by login/captcha/403, try alternative sites
6. If page structure differs from expected, re-analyze and adapt
7. If stuck in loop, acknowledge in memory and change strategy
8. If max_steps approaching, prioritize most important parts
</error_recovery>
<examples>
**Good evaluation examples:**
- "Successfully navigated to the product page and found the target information. Verdict: Success"
- "Failed to input text into the search bar - element not visible. Verdict: Failure"
**Good memory examples:**
- "Visited 2 of 5 target websites. Collected pricing from Amazon ($39.99) and eBay ($42.00). Still need Walmart, Target, Best Buy."
- "Search returned results but no filter applied. User wants items under $50 with 4+ stars. Will apply price filter first."
**Good next goal examples:**
- "Click 'Add to Cart' to proceed with purchase flow."
- "Apply price filter to narrow results to items under $50."
</examples>
<critical_reminders>
1. ALWAYS verify action success using screenshot/browser state
2. ALWAYS handle popups/modals before other actions
3. ALWAYS apply filters when task specifies criteria
4. NEVER repeat failing actions more than 2-3 times
5. NEVER assume success without verification
6. Track progress in memory to avoid loops
7. Match requested output format exactly
8. Be efficient - combine actions when possible
</critical_reminders>
================================================
FILE: packages/core/src/agent/instructions/instructions.md
================================================
You are an AI agent that controls a web browser to complete tasks. You operate in an iterative loop: observe the current page state, decide on actions, execute them, and repeat until the task is done.
Your task: {{task}}
<capabilities>
You excel at:
1. Navigating complex websites and extracting precise information
2. Automating form submissions and interactive web actions
3. Gathering and organizing information across multiple pages
4. Operating effectively in an iterative agent loop
5. Adapting strategies when encountering obstacles
</capabilities>
<language_settings>
- Default working language: **English**
- Always respond in the same language as the task description
</language_settings>
<input>
At every step, your input will consist of:
1. **Agent history**: A chronological event stream including your previous actions and their results.
2. **Browser state**: Current URL, open tabs, interactive elements indexed for actions, and visible page content.
3. **Screenshot** (when vision is enabled): A screenshot of the current page with bounding boxes around interactive elements.
</input>
<browser_state>
Browser state is given as:
- **Current URL**: The URL of the page you are currently viewing.
- **Open Tabs**: Open tabs with their IDs.
- **Interactive Elements**: All interactive elements in the format `[index]<type>text</type>` where:
- `index`: Numeric identifier for interaction
- `type`: HTML element type (button, input, etc.)
- `text`: Element description
Examples:
```
[33]<div>User form</div>
*[35]<button aria-label='Submit form'>Submit</button>
```
Important notes:
- Only elements with numeric indexes in `[]` are interactive
- Indentation (with tab) means the element is a child of the element above
- Elements tagged with `*[` are **new** interactive elements that appeared since the last step. Your previous actions caused that change. Consider if you need to interact with them.
- Pure text elements without `[]` are not interactive
</browser_state>
<screenshot>
If vision is enabled, you will receive a screenshot of the current page with bounding boxes around interactive elements.
- This is your **ground truth**: use it to evaluate your progress
- If an interactive element has no text in browser_state, its index is written at the top center of its bounding box in the screenshot
- Use the screenshot action if you need more visual information
</screenshot>
<rules>
Strictly follow these rules while using the browser:
**Element Interaction:**
- Only interact with elements that have a numeric `[index]` assigned
- Only use indexes that are explicitly provided in the current browser state
- If a page changes after an action (e.g., input text triggers suggestions), analyze new elements before proceeding
**Navigation:**
- If research is needed, open a **new tab** instead of reusing the current one
- By default, only elements in the visible viewport are listed
- If the page is not fully loaded, use the wait action
**Content Extraction:**
- Use extract_content on specific pages to gather structured information from the entire page, including parts not currently visible
- Only call extract_content if the information is NOT already visible in browser_state - prefer using text directly from browser_state
- extract_content is expensive - do NOT call it multiple times with the same query on the same page
**Input Handling:**
- If you fill an input field and your action sequence is interrupted, something likely changed (e.g., suggestions appeared)
- If the action sequence was interrupted in a previous step, complete any remaining actions that were not executed
- For autocomplete/combobox fields: type your text, then WAIT for suggestions in the next step. If suggestions appear (marked with `*[`), click the correct one. If none appear, press Enter.
- After input, you may need to press Enter, click a search button, or select from a dropdown
**Filters and Criteria:**
- If the task includes specific criteria (product type, rating, price, location, etc.), ALWAYS look for filter/sort options FIRST before browsing results
**Error Recovery:**
- If a captcha appears, attempt solving it. If blocked after 3-4 steps, try alternative approaches or report the limitation
- Handle popups, modals, cookie banners, and overlays immediately before other actions
- If you encounter access denied (403), bot detection, or rate limiting, do NOT retry the same URL repeatedly - try alternatives
- Detect and break out of unproductive loops: if you are on the same URL for 3+ steps without progress, or the same action fails 2-3 times, try a different approach
**Authentication:**
- Do not log into a page unless required by the task and you have credentials
</rules>
<output_format>
## Output Format
Respond with:
1. **currentState**: Your assessment of the current state including:
- `evaluation`: Assessment of how the last action went
- `memory`: Important information to remember (progress, data found, approaches tried)
- `nextGoal`: The next immediate goal to pursue
2. **actions**: A list of actions to execute (max {{maxActionsPerStep}} per step)
</output_format>
<action_rules>
You are allowed to use a maximum of {{maxActionsPerStep}} actions per step.
Multiple actions execute sequentially (one after another).
- If the page changes after an action, remaining actions are automatically skipped and you get the new state.
- Check the browser state each step to verify your previous action achieved its goal.
</action_rules>
<available_actions>
{{actionDescriptions}}
</available_actions>
<efficiency>
You can output multiple actions in one step. Be efficient where it makes sense, but do not predict actions that do not make sense for the current page.
**Action categories:**
- **Page-changing (always last):** navigate, search_google, go_back, switch_tab - these always change the page. Remaining actions after them are skipped automatically.
- **Potentially page-changing:** click (on links/buttons that navigate) - monitored at runtime; if the page changes, remaining actions are skipped.
- **Safe to chain:** input_text, scroll, extract_content, find_elements - these do not change the page and can be freely combined.
**Recommended combinations:**
- `input_text` + `input_text` + `click` -> Fill multiple form fields then submit
- `input_text` + `send_keys` -> Fill a field and press Enter
- `scroll` + `scroll` -> Scroll further down the page
Do not try multiple different paths in one step. Always have one clear goal per step.
Place any page-changing action **last** in your action list.
</efficiency>
<reasoning>
You must reason systematically at every step:
1. Analyze the most recent action result - clearly state success, failure, or uncertainty. Never assume success without verification.
2. Analyze browser state, screenshot, and history to understand current position relative to the task.
3. If stuck (same actions repeated without progress), consider alternative approaches.
4. Decide what concise, actionable context should be stored in memory.
5. State your next immediate goal clearly.
</reasoning>
<task_completion>
You must use the `done` action when:
- You have fully completed the task
- You reach the final allowed step, even if the task is incomplete
- It is absolutely impossible to continue
Rules for `done`:
- Set `success` to `true` only if the FULL task has been completed
- If any part is missing, incomplete, or uncertain, set `success` to `false`
- Put ALL relevant findings in the `text` field
- You are ONLY allowed to call `done` as a single action - never combine it with other actions
**Before calling done with success=true, verify:**
1. Re-read the original task and list every concrete requirement
2. Check each requirement against your results (correct count, filters applied, format matched)
3. Verify actions actually completed (check page state/screenshot)
4. Ensure no data was fabricated - every fact must come from pages you visited
5. If ANY requirement is unmet or uncertain, set success to false
</task_completion>
<budget_management>
- When you reach 75% of your step budget, critically evaluate whether you can complete the full task in remaining steps
- If completion is unlikely, shift strategy: focus on highest-value remaining items and consolidate results
- For large multi-item tasks, estimate per-item cost from the first few items and prioritize if the task will exceed your budget
</budget_management>
<error_recovery>
When encountering errors or unexpected states:
1. Verify the current state using screenshot as ground truth
2. Check if a popup, modal, or overlay is blocking interaction
3. If an element is not found, scroll to reveal more content
4. If an action fails repeatedly (2-3 times), try an alternative approach
5. If blocked by login/captcha/403, consider alternative sites or search engines
6. If the page structure is different than expected, re-analyze and adapt
7. If stuck in a loop, explicitly acknowledge it in memory and change strategy
8. If max_steps is approaching, prioritize completing the most important parts
</error_recovery>
<examples>
**Good evaluation examples:**
- "Successfully navigated to the product page and found the target information. Verdict: Success"
- "Failed to input text into the search bar - element not visible. Verdict: Failure"
**Good memory examples:**
- "Visited 2 of 5 target websites. Collected pricing data from Amazon ($39.99) and eBay ($42.00). Still need Walmart, Target, Best Buy."
- "Search returned results but no filter applied yet. User wants items under $50 with 4+ stars. Will apply price filter first."
- "Captcha appeared twice on this site. Will try alternative approach via search engine."
**Good next goal examples:**
- "Click the 'Add to Cart' button to proceed with the purchase flow."
- "Apply price filter to narrow results to items under $50."
- "Close the popup blocking the main content."
</examples>
<critical_reminders>
1. ALWAYS verify action success using screenshot/browser state before proceeding
2. ALWAYS handle popups/modals/cookie banners before other actions
3. ALWAYS apply filters when the task specifies criteria
4. NEVER repeat the same failing action more than 2-3 times
5. NEVER assume success without verification
6. Track progress in memory to avoid loops
7. Match the task's requested output format exactly
8. Be efficient - combine actions when possible but verify between major steps
</critical_reminders>
================================================
FILE: packages/core/src/agent/instructions.ts
================================================
import { readFileSync } from 'node:fs';
import { resolve, dirname } from 'node:path';
import { fileURLToPath } from 'node:url';
import type { AgentConfig } from './types.js';
import type { ViewportSnapshot, TabDescriptor } from '../viewport/types.js';
import type { CommandCatalog } from '../commands/catalog/catalog.js';
import type { ContentPart } from '../model/messages.js';
import { textContent, imageContent } from '../model/messages.js';
import { isNewTabPage, sanitizeSurrogates, dedent } from '../utils.js';
// ── Template types ──
export type PromptTemplate = 'default' | 'flash' | 'no-thinking';
export interface InstructionBuilderOptions {
/** Maximum actions the agent can take per step. */
commandsPerStep: number;
/** Override the entire system prompt with a custom string. */
overrideInstructionBuilder?: string;
/** Append additional instructions to the system prompt. */
extendInstructionBuilder?: string;
/** Which template variant to use. Defaults to 'default'. */
template?: PromptTemplate;
/** Whether to include sensitive-data warnings. */
hasSensitiveData?: boolean;
}
export interface StepInfo {
step: number;
stepLimit: number;
}
export interface StepPromptBuilderOptions {
browserState: ViewportSnapshot;
task: string;
stepInfo?: StepInfo;
actionDescriptions?: string;
pageFilteredActions?: string;
agentHistoryDescription?: string;
maskedValues?: string;
planDescription?: string;
screenshots?: string[];
enableScreenshots?: boolean;
maxElementsLength?: number;
}
// ── Template loading ──
/**
* Directory containing the .md system prompt templates.
* Resolved relative to this file's location so it works regardless of
* the current working directory or whether the package is installed.
*/
const TEMPLATES_DIR = resolve(dirname(fileURLToPath(import.meta.url)), 'instructions');
/** Cache loaded templates so we only hit the filesystem once per variant. */
const templateCache = new Map<string, string>();
/**
* Map from PromptTemplate variant to the corresponding filename.
*/
const TEMPLATE_FILES: Record<PromptTemplate, string> = {
default: 'instructions.md',
flash: 'instructions-compact.md',
'no-thinking': 'instructions-direct.md',
};
/**
* Load a system-prompt template from disk. Results are cached.
*
* @param variant - Which prompt template to load.
* @returns The raw template string with `{{variable}}` placeholders.
* @throws If the template file cannot be read.
*/
function loadTemplate(variant: PromptTemplate): string {
const cached = templateCache.get(variant);
if (cached !== undefined) return cached;
const filename = TEMPLATE_FILES[variant];
const filepath = resolve(TEMPLATES_DIR, filename);
try {
const content = readFileSync(filepath, 'utf-8');
templateCache.set(variant, content);
return content;
} catch (error) {
const message = error instanceof Error ? error.message : String(error);
throw new Error(`Failed to load system prompt template "${filename}": ${message}`);
}
}
/**
* Interpolate `{{key}}` placeholders in a template string.
* Unmatched placeholders are left as-is so downstream code can detect them.
*/
function interpolate(template: string, variables: Record<string, string>): string {
return template.replace(/\{\{(\w+)\}\}/g, (match, key: string) => {
return key in variables ? variables[key] : match;
});
}
/**
* Clear the template cache. Useful for testing or hot-reloading.
*/
export function clearTemplateCache(): void {
templateCache.clear();
}
// ── InstructionBuilder ──
/**
* Builds the system prompt for the browser automation agent.
*
* In the simplest case it loads a `.md` template from the `system-prompts/`
* directory and interpolates variables like `{{task}}`, `{{commandsPerStep}}`,
* and `{{actionDescriptions}}`.
*
* The class also exposes static helpers for building per-step state messages,
* action results, and other ancillary prompt fragments that are injected as
* user messages during the agent loop.
*/
export class InstructionBuilder {
private options: InstructionBuilderOptions;
private actionDescriptions: string;
constructor(options: InstructionBuilderOptions, actionDescriptions: string) {
this.options = options;
this.actionDescriptions = actionDescriptions;
}
/**
* Build and return the complete system prompt string.
*
* If `overrideInstructionBuilder` is set, it is returned verbatim (after
* optional extension). Otherwise, the appropriate `.md` template is
* loaded and interpolated with the current settings.
*/
build(): string {
if (this.options.overrideInstructionBuilder) {
let prompt = this.options.overrideInstructionBuilder;
if (this.options.extendInstructionBuilder) {
prompt += `\n${this.options.extendInstructionBuilder}`;
}
return prompt;
}
const variant = this.options.template ?? 'default';
const template = loadTemplate(variant);
const variables: Record<string, string> = {
task: '(set per-step in user messages)',
commandsPerStep: String(this.options.commandsPerStep),
actionDescriptions: this.actionDescriptions,
};
let prompt = interpolate(template, variables);
if (this.options.extendInstructionBuilder) {
prompt += `\n${this.options.extendInstructionBuilder}`;
}
return prompt;
}
/**
* Convenience: create a InstructionBuilder from AgentConfig + a CommandCatalog.
* Pulls action descriptions directly from the registry, optionally
* filtered by the current page URL.
*/
static fromSettings(settings: AgentConfig, registry: CommandCatalog, pageUrl?: string): InstructionBuilder {
const descriptions = registry.getPromptDescription(pageUrl);
return new InstructionBuilder(
{
commandsPerStep: settings.commandsPerStep,
overrideInstructionBuilder: settings.overrideInstructionBuilder,
extendInstructionBuilder: settings.extendInstructionBuilder,
hasSensitiveData: settings.maskedValues !== undefined,
},
descriptions,
);
}
// ── Static prompt fragment builders ──
static buildTaskPrompt(task: string): string {
return `Your current task: ${task}`;
}
static buildStatePrompt(
url: string,
title: string,
tabs: Array<{ url: string; title: string; isActive: boolean }>,
domTree: string,
step: number,
stepLimit: number,
pixelsAbove?: number,
pixelsBelow?: number,
): string {
const parts: string[] = [];
parts.push(`[Step ${step}/${stepLimit}]`);
parts.push(`Current URL: ${url}`);
parts.push(`Page Title: ${title}`);
if (tabs.length > 1) {
const tabList = tabs
.map((t, i) => ` [${i}] ${t.isActive ? '(active) ' : ''}${t.title} - ${t.url}`)
.join('\n');
parts.push(`Open Tabs:\n${tabList}`);
}
if (pixelsAbove !== undefined && pixelsAbove > 0) {
parts.push(`Scroll position: ${pixelsAbove}px from top`);
}
if (pixelsBelow !== undefined && pixelsBelow > 0) {
parts.push(`${pixelsBelow}px of content below the visible area`);
}
parts.push(`\nPage content:\n${domTree}`);
return parts.join('\n');
}
static buildCommandResultPrompt(results: Array<{ action: string; result: string }>): string {
if (results.length === 0) return '';
const formatted = results
.map((r) => `Action: ${r.action}\nResult: ${r.result}`)
.join('\n---\n');
return `Previous action results:\n${formatted}`;
}
static buildLoopNudge(message: string): string {
return `\nIMPORTANT: ${message}`;
}
static buildPlanPrompt(currentPlan: string): string {
return `\nCurrent plan:\n${currentPlan}`;
}
}
// ── StepPromptBuilder ──
/**
* Constructs the per-step user message for the agent.
*
* Each step of the agent loop sends a user message containing:
* - The current browser state (URL, tabs, interactive elements)
* - Scroll position and page boundaries
* - Agent history summary
* - Step information (step N of M)
* - Optionally: screenshots, sensitive data warnings, plan description
* - Optionally: page-specific action descriptions
*
* The message can be returned as a plain string or as a multipart content
* array (text + images) when vision is enabled.
*/
export class StepPromptBuilder {
private browserState: ViewportSnapshot;
private task: string;
private stepInfo?: StepInfo;
private actionDescriptions?: string;
private pageFilteredActions?: string;
private agentHistoryDescription?: string;
private maskedValues?: string;
private planDescription?: string;
private screenshots: string[];
private enableScreenshots: boolean;
private maxElementsLength: number;
constructor(options: StepPromptBuilderOptions) {
this.browserState = options.browserState;
this.task = options.task;
this.stepInfo = options.stepInfo;
this.actionDescriptions = options.actionDescriptions;
this.pageFilteredActions = options.pageFilteredActions;
this.agentHistoryDescription = options.agentHistoryDescription;
this.maskedValues = options.maskedValues;
this.planDescription = options.planDescription;
this.screenshots = options.screenshots ?? [];
this.enableScreenshots = options.enableScreenshots ?? false;
this.maxElementsLength = options.maxElementsLength ?? 40_000;
}
/**
* Build the user message content.
*
* When vision is disabled (or no screenshots are available), returns a
* single string. When vision is enabled and screenshots exist, returns
* a `ContentPart[]` array interleaving text and image parts.
*/
getUserMessage(): string | ContentPart[] {
// Skip screenshots on step 0 for new-tab pages with a single tab
let effectiveVision = this.enableScreenshots;
if (
isNewTabPage(this.browserState.url) &&
this.stepInfo?.step === 0 &&
this.browserState.tabs.length <= 1
) {
effectiveVision = false;
}
const stateDescription = this.buildStateDescription();
if (effectiveVision && this.screenshots.length > 0) {
const parts: ContentPart[] = [textContent(stateDescription)];
for (let i = 0; i < this.screenshots.length; i++) {
const label =
i === this.screenshots.length - 1 ? 'Current screenshot:' : 'Previous screenshot:';
parts.push(textContent(label));
parts.push(imageContent(this.screenshots[i], 'image/png'));
}
return parts;
}
return stateDescription;
}
/**
* Build the complete text description of the current state.
* This includes agent history, agent state (task, step info, plan),
* and browser state (URL, tabs, elements, scroll position).
*/
private buildStateDescription(): string {
const sections: string[] = [];
// Agent history
sections.push(this.buildAgentHistorySection());
// Agent state (task, step info, plan, sensitive data)
sections.push(this.buildAgentStateSection());
// Browser state (URL, tabs, elements)
sections.push(this.buildBrowserStateSection());
// Page-specific actions (if any domain-filtered actions apply)
if (this.pageFilteredActions) {
sections.push(
`<page_specific_actions>\n${this.pageFilteredActions}\n</page_specific_actions>`,
);
}
// Sanitize surrogates to prevent JSON serialization issues
return sanitizeSurrogates(sections.join('\n\n'));
}
private buildAgentHistorySection(): string {
const history = this.agentHistoryDescription?.trim() ?? '';
return `<agent_history>\n${history}\n</agent_history>`;
}
private buildAgentStateSection(): string {
const parts: string[] = [];
parts.push(`<user_request>\n${this.task}\n</user_request>`);
if (this.planDescription) {
parts.push(`<plan>\n${this.planDescription}\n</plan>`);
}
if (this.maskedValues) {
parts.push(`<sensitive_data>${this.maskedValues}</sensitive_data>`);
}
if (this.stepInfo) {
const today = new Date().toISOString().slice(0, 10);
parts.push(
`<step_info>Step ${this.stepInfo.step + 1} of ${this.stepInfo.stepLimit} | Today: ${today}</step_info>`,
);
}
return `<agent_state>\n${parts.join('\n')}\n</agent_state>`;
}
private buildBrowserStateSection(): string {
const parts: string[] = [];
// Tabs
const tabsText = this.buildTabsText();
if (tabsText) {
parts.push(tabsText);
}
// Scroll / page info
const pageInfo = this.buildPageInfoText();
if (pageInfo) {
parts.push(pageInfo);
}
// Interactive elements
parts.push(this.buildElementsText());
return `<browser_state>\n${parts.join('\n')}\n</browser_state>`;
}
private buildTabsText(): string {
const { tabs, url, title } = this.browserState;
if (tabs.length === 0) return '';
// Try to identify the current tab
const currentCandidates = tabs.filter((t) => t.url === url && t.title === title);
const currentTabId =
currentCandidates.length === 1 ? currentCandidates[0].tabId : undefined;
const lines: string[] = [];
if (currentTabId) {
lines.push(`Current tab: ${String(currentTabId).slice(-4)}`);
}
lines.push('Available tabs:');
for (const tab of tabs) {
lines.push(`Tab ${String(tab.tabId).slice(-4)}: ${tab.url} - ${tab.title.slice(0, 30)}`);
}
return lines.join('\n');
}
private buildPageInfoText(): string {
const { pixelsAbove, pixelsBelow } = this.browserState;
const parts: string[] = [];
if (pixelsAbove !== undefined && pixelsAbove > 0) {
// Estimate "pages above" assuming ~900px viewport height
const pagesAbove = (pixelsAbove / 900).toFixed(1);
parts.push(`${pagesAbove} pages abo
gitextract_rxlca7z1/ ├── .github/ │ ├── CONTRIBUTING.md │ └── workflows/ │ └── ci.yml ├── .gitignore ├── LICENSE ├── README.md ├── biome.json ├── bunfig.toml ├── package.json ├── packages/ │ ├── cli/ │ │ ├── package.json │ │ ├── src/ │ │ │ ├── commands/ │ │ │ │ ├── click.ts │ │ │ │ ├── eval.ts │ │ │ │ ├── extract.ts │ │ │ │ ├── interactive.ts │ │ │ │ ├── open.ts │ │ │ │ ├── run.ts │ │ │ │ ├── screenshot.ts │ │ │ │ ├── sessions.ts │ │ │ │ ├── state.ts │ │ │ │ └── type.ts │ │ │ ├── display.ts │ │ │ ├── globals.ts │ │ │ ├── index.ts │ │ │ ├── protocol.ts │ │ │ ├── server.ts │ │ │ └── sessions.ts │ │ └── tsconfig.json │ ├── core/ │ │ ├── package.json │ │ ├── src/ │ │ │ ├── agent/ │ │ │ │ ├── agent.test.ts │ │ │ │ ├── agent.ts │ │ │ │ ├── conversation/ │ │ │ │ │ ├── service.ts │ │ │ │ │ ├── types.ts │ │ │ │ │ └── utils.ts │ │ │ │ ├── conversation.test.ts │ │ │ │ ├── evaluator.ts │ │ │ │ ├── index.ts │ │ │ │ ├── instructions/ │ │ │ │ │ ├── instructions-compact.md │ │ │ │ │ ├── instructions-direct.md │ │ │ │ │ └── instructions.md │ │ │ │ ├── instructions.ts │ │ │ │ ├── replay-recorder.ts │ │ │ │ ├── stall-detector.test.ts │ │ │ │ ├── stall-detector.ts │ │ │ │ └── types.ts │ │ │ ├── bridge/ │ │ │ │ ├── adapter.ts │ │ │ │ ├── client.ts │ │ │ │ ├── index.ts │ │ │ │ ├── mcp-types.ts │ │ │ │ ├── server.test.ts │ │ │ │ └── server.ts │ │ │ ├── commands/ │ │ │ │ ├── catalog/ │ │ │ │ │ ├── catalog.ts │ │ │ │ │ └── types.ts │ │ │ │ ├── catalog.test.ts │ │ │ │ ├── executor.test.ts │ │ │ │ ├── executor.ts │ │ │ │ ├── extraction/ │ │ │ │ │ └── extractor.ts │ │ │ │ ├── index.ts │ │ │ │ ├── types.ts │ │ │ │ └── utils.ts │ │ │ ├── config/ │ │ │ │ ├── config.ts │ │ │ │ ├── index.ts │ │ │ │ └── types.ts │ │ │ ├── errors.ts │ │ │ ├── index.ts │ │ │ ├── logging.ts │ │ │ ├── metering/ │ │ │ │ ├── index.ts │ │ │ │ ├── tracker.test.ts │ │ │ │ ├── tracker.ts │ │ │ │ └── types.ts │ │ │ ├── model/ │ │ │ │ ├── adapters/ │ │ │ │ │ └── vercel.ts │ │ │ │ ├── index.ts │ │ │ │ ├── interface.ts │ │ │ │ ├── messages.ts │ │ │ │ ├── schema-optimizer.ts │ │ │ │ └── types.ts │ │ │ ├── page/ │ │ │ │ ├── content-extractor.ts │ │ │ │ ├── index.ts │ │ │ │ ├── page-analyzer.test.ts │ │ │ │ ├── page-analyzer.ts │ │ │ │ ├── renderer/ │ │ │ │ │ ├── interactive-elements.ts │ │ │ │ │ ├── layer-order.ts │ │ │ │ │ └── tree-renderer.ts │ │ │ │ ├── renderer.test.ts │ │ │ │ ├── snapshot-builder.ts │ │ │ │ └── types.ts │ │ │ ├── sandbox/ │ │ │ │ ├── file-access.ts │ │ │ │ └── index.ts │ │ │ ├── telemetry.ts │ │ │ ├── types.ts │ │ │ ├── utils.ts │ │ │ └── viewport/ │ │ │ ├── event-hub.ts │ │ │ ├── events.ts │ │ │ ├── guard-base.ts │ │ │ ├── guards/ │ │ │ │ ├── blank-page.ts │ │ │ │ ├── crash.ts │ │ │ │ ├── default-handler.ts │ │ │ │ ├── downloads.ts │ │ │ │ ├── har-capture.ts │ │ │ │ ├── local-instance.ts │ │ │ │ ├── page-ready.ts │ │ │ │ ├── permissions.ts │ │ │ │ ├── persistence.ts │ │ │ │ ├── popups.ts │ │ │ │ ├── screenshot.ts │ │ │ │ ├── url-policy.ts │ │ │ │ └── video-capture.ts │ │ │ ├── index.ts │ │ │ ├── launch-profile.test.ts │ │ │ ├── launch-profile.ts │ │ │ ├── types.ts │ │ │ ├── viewport.ts │ │ │ └── visual-tracer.ts │ │ └── tsconfig.json │ └── sandbox/ │ ├── package.json │ ├── src/ │ │ ├── index.ts │ │ ├── sandbox.ts │ │ └── types.ts │ └── tsconfig.json ├── tsconfig.base.json └── tsconfig.json
SYMBOL INDEX (969 symbols across 86 files)
FILE: packages/cli/src/commands/click.ts
function registerClickCommand (line 5) | function registerClickCommand(program: Command): void {
FILE: packages/cli/src/commands/eval.ts
function registerEvalCommand (line 5) | function registerEvalCommand(program: Command): void {
FILE: packages/cli/src/commands/extract.ts
function registerExtractCommand (line 6) | function registerExtractCommand(program: Command): void {
FILE: packages/cli/src/commands/interactive.ts
type InteractiveOptions (line 15) | interface InteractiveOptions {
function registerInteractiveCommand (line 23) | function registerInteractiveCommand(program: Command): void {
function parseCommandLine (line 107) | function parseCommandLine(input: string): string[] {
function handleCommand (line 140) | async function handleCommand(
function printHelp (line 316) | function printHelp(): void {
FILE: packages/cli/src/commands/open.ts
function registerOpenCommand (line 5) | function registerOpenCommand(program: Command): void {
FILE: packages/cli/src/commands/run.ts
type RunOptions (line 21) | interface RunOptions {
function createModel (line 34) | async function createModel(provider: string, modelId: string): Promise<L...
function registerRunCommand (line 66) | function registerRunCommand(program: Command): void {
function extractActionName (line 210) | function extractActionName(result: CommandResult): string {
function extractActionTarget (line 216) | function extractActionTarget(result: CommandResult): string | undefined {
function computeTotalDuration (line 223) | function computeTotalDuration(entries: StepRecord[]): number {
FILE: packages/cli/src/commands/screenshot.ts
function registerScreenshotCommand (line 7) | function registerScreenshotCommand(program: Command): void {
FILE: packages/cli/src/commands/sessions.ts
function registerSessionsCommand (line 5) | function registerSessionsCommand(program: Command): void {
FILE: packages/cli/src/commands/state.ts
function registerStateCommand (line 5) | function registerStateCommand(program: Command): void {
FILE: packages/cli/src/commands/type.ts
function registerTypeCommand (line 5) | function registerTypeCommand(program: Command): void {
FILE: packages/cli/src/display.ts
constant SPINNER_FRAMES (line 5) | const SPINNER_FRAMES = ['⠋', '⠙', '⠹', '⠸', '⠼', '⠴', '⠦', '⠧', '⠇', '⠏'];
class Spinner (line 7) | class Spinner {
method constructor (line 12) | constructor(message: string) {
method start (line 16) | start(): void {
method update (line 27) | update(message: string): void {
method stop (line 31) | stop(finalMessage?: string): void {
type StepDisplayInfo (line 46) | interface StepDisplayInfo {
function displayStep (line 59) | function displayStep(info: StepDisplayInfo): void {
type CostDisplayInfo (line 85) | interface CostDisplayInfo {
function displayStepCost (line 94) | function displayStepCost(info: CostDisplayInfo): void {
function displayTotalCost (line 105) | function displayTotalCost(info: CostDisplayInfo & { steps: number; durat...
function displayProgressBar (line 120) | function displayProgressBar(current: number, total: number, width = 30):...
function displayResult (line 131) | function displayResult(success: boolean, output?: string): void {
function displayError (line 148) | function displayError(message: string): void {
function displayWarning (line 152) | function displayWarning(message: string): void {
function displayInfo (line 156) | function displayInfo(message: string): void {
function displaySeparator (line 160) | function displaySeparator(): void {
function displayHeader (line 164) | function displayHeader(title: string): void {
FILE: packages/cli/src/protocol.ts
type CLIRequest (line 1) | interface CLIRequest {
type CLIResponse (line 7) | interface CLIResponse {
function serializeRequest (line 14) | function serializeRequest(req: CLIRequest): string {
function parseRequest (line 18) | function parseRequest(data: string): CLIRequest | null {
function serializeResponse (line 26) | function serializeResponse(res: CLIResponse): string {
function parseResponse (line 30) | function parseResponse(data: string): CLIResponse | null {
FILE: packages/cli/src/server.ts
constant SOCKET_DIR (line 8) | const SOCKET_DIR = path.join(os.tmpdir(), 'open-browser');
constant SOCKET_PATH (line 9) | const SOCKET_PATH = path.join(SOCKET_DIR, 'server.sock');
class CLIServer (line 11) | class CLIServer {
method constructor (line 15) | constructor() {
method start (line 19) | async start(): Promise<string> {
method handleRequest (line 60) | private async handleRequest(request: CLIRequest): Promise<CLIResponse> {
method getSessionBrowser (line 155) | private getSessionBrowser(request: CLIRequest) {
method stop (line 168) | async stop(): Promise<void> {
method socketPath (line 183) | static get socketPath(): string {
FILE: packages/cli/src/sessions.ts
type ManagedSession (line 4) | interface ManagedSession {
class SessionManager (line 11) | class SessionManager {
method create (line 14) | async create(options?: ViewportOptions): Promise<string> {
method get (line 29) | get(id: string): Viewport | undefined {
method close (line 38) | async close(id: string): Promise<boolean> {
method closeAll (line 47) | async closeAll(): Promise<void> {
method list (line 54) | list(): Array<{ id: string; createdAt: number; lastAccessedAt: number ...
method activeCount (line 62) | get activeCount(): number {
method getDefault (line 66) | getDefault(): Viewport | undefined {
method getDefaultId (line 73) | getDefaultId(): string | undefined {
FILE: packages/core/src/agent/agent.test.ts
function createMockPageAnalyzer (line 19) | function createMockPageAnalyzer(): PageAnalyzer {
function createMockUsage (line 46) | function createMockUsage(input = 100, output = 50): InferenceUsage {
function createMockModel (line 50) | function createMockModel(options?: {
function createDoneOnStepModel (line 84) | function createDoneOnStepModel(doneOnStep: number, result = 'Task comple...
function createMockBrowserState (line 113) | function createMockBrowserState(): ViewportSnapshot {
function createMockRegistry (line 124) | function createMockRegistry(): CommandCatalog{
function createMockTools (line 135) | function createMockTools(actionResults?: CommandResult[]): CommandExecut...
function createMockBrowser (line 150) | function createMockBrowser(overrides?: {
function createDefaultAgentOptions (line 171) | function createDefaultAgentOptions(overrides?: Partial<AgentOptions>): A...
FILE: packages/core/src/agent/agent.ts
type AgentOptions (line 54) | interface AgentOptions {
class Agent (line 75) | class Agent {
method constructor (line 97) | constructor(options: AgentOptions) {
method run (line 177) | async run(stepLimit?: number): Promise<RunOutcome> {
method executeStep (line 390) | private async executeStep(step: number, stepLimit: number): Promise<Co...
method invokeLlmWithRecovery (line 574) | private async invokeLlmWithRecovery(
method getOutputSchema (line 650) | private getOutputSchema(): z.ZodType<unknown> {
method getSchemaName (line 671) | private getSchemaName(): string {
method normalizeOutput (line 687) | private normalizeOutput(output: Record<string, unknown>): AgentDecision {
method shouldUpdatePlan (line 720) | private shouldUpdatePlan(step: number): boolean {
method updatePlan (line 728) | private async updatePlan(step: number): Promise<void> {
method rebuildInstructionBuilder (line 779) | private rebuildInstructionBuilder(pageUrl?: string): void {
method autoNavigateFromTask (line 792) | private async autoNavigateFromTask(): Promise<void> {
method executeInitialActions (line 816) | private async executeInitialActions(): Promise<void> {
method makeFailureRecoveryCall (line 856) | private async makeFailureRecoveryCall(
method updateCostTracking (line 903) | private updateCostTracking(
method filterSensitiveData (line 933) | private filterSensitiveData(results: CommandResult[]): CommandResult[] {
method saveConversation (line 955) | private async saveConversation(step: number): Promise<void> {
method addNewTask (line 981) | addNewTask(task: string): void {
method getFollowUpTasks (line 986) | getFollowUpTasks(): string[] {
method pause (line 994) | pause(): void {
method resume (line 998) | resume(): void {
method stop (line 1002) | stop(): void {
method getState (line 1006) | getState(): AgentState {
method getHistory (line 1010) | getHistory(): ExecutionLog {
method getAccumulatedCost (line 1014) | getAccumulatedCost(): AccumulatedCost {
FILE: packages/core/src/agent/conversation.test.ts
function createManager (line 9) | function createManager(
function createMockModel (line 19) | function createMockModel(summary = 'Summary of the conversation'): Langu...
FILE: packages/core/src/agent/conversation/service.ts
class ConversationManager (line 36) | class ConversationManager {
method constructor (line 45) | constructor(options: ConversationManagerOptions) {
method setInstructionBuilder (line 53) | setInstructionBuilder(prompt: string): void {
method addStateMessage (line 62) | addStateMessage(
method addAssistantMessage (line 87) | addAssistantMessage(text: string, step?: number): void {
method addCommandResultMessage (line 102) | addCommandResultMessage(text: string, step?: number): void {
method addUserMessage (line 117) | addUserMessage(text: string): void {
method addEphemeralMessage (line 134) | addEphemeralMessage(text: string, role: 'user' | 'assistant' = 'user')...
method getMessages (line 153) | getMessages(): Message[] {
method consumeEphemeralMessages (line 189) | private consumeEphemeralMessages(): void {
method estimateTotalTokens (line 207) | estimateTotalTokens(): number {
method compact (line 226) | private compact(): void {
method compactWithLlm (line 292) | async compactWithLlm(model?: LanguageModel): Promise<boolean> {
method shouldCompactWithLlm (line 379) | shouldCompactWithLlm(): boolean {
method recordConversationEntry (line 392) | private recordConversationEntry(
method agentHistoryDescription (line 415) | agentHistoryDescription(stepLimitShown = 10): string {
method formatStepDescription (line 459) | private formatStepDescription(step: number, items: ConversationEntry[]...
method getConversationEntrys (line 471) | getConversationEntrys(): readonly ConversationEntry[] {
method save (line 483) | save(): ConversationManagerState {
method load (line 505) | load(state: ConversationManagerState): void {
method saveToFile (line 532) | async saveToFile(filePath: string): Promise<string> {
method loadFromFile (line 544) | async loadFromFile(filePath: string): Promise<void> {
method messageCount (line 555) | get messageCount(): number {
method step (line 559) | get step(): number {
method clear (line 563) | clear(): void {
method resetMessages (line 574) | resetMessages(): void {
FILE: packages/core/src/agent/conversation/types.ts
type ConversationManagerOptions (line 7) | interface ConversationManagerOptions {
type MessageCategory (line 21) | type MessageCategory =
type TrackedMessage (line 29) | interface TrackedMessage {
type ConversationEntry (line 50) | interface ConversationEntry {
type ConversationManagerState (line 70) | interface ConversationManagerState {
type SerializedTrackedMessage (line 82) | interface SerializedTrackedMessage {
FILE: packages/core/src/agent/conversation/utils.ts
function estimateTokens (line 7) | function estimateTokens(text: string): number {
function estimateMessageTokens (line 11) | function estimateMessageTokens(content: string | unknown[]): number {
constant MASK (line 32) | const MASK = '***';
function redactSensitiveValues (line 38) | function redactSensitiveValues(
function redactMessage (line 56) | function redactMessage(
function redactMessages (line 95) | function redactMessages(
function extractTextContent (line 107) | function extractTextContent(message: Message): string {
function truncate (line 122) | function truncate(text: string, maxLen: number): string {
FILE: packages/core/src/agent/evaluator.ts
constant JUDGE_SYSTEM_PROMPT (line 17) | const JUDGE_SYSTEM_PROMPT = `You are an expert task completion judge. Yo...
constant SIMPLE_JUDGE_SYSTEM_PROMPT (line 35) | const SIMPLE_JUDGE_SYSTEM_PROMPT = `You are a quick-check validator for ...
class ResultEvaluator (line 39) | class ResultEvaluator {
method constructor (line 42) | constructor(model: LanguageModel) {
method evaluate (line 50) | async evaluate(
method simpleEvaluate (line 92) | async simpleEvaluate(
function constructEvaluatorMessages (line 128) | function constructEvaluatorMessages(
function constructQuickCheckMessages (line 231) | function constructQuickCheckMessages(
FILE: packages/core/src/agent/instructions.ts
type PromptTemplate (line 14) | type PromptTemplate = 'default' | 'flash' | 'no-thinking';
type InstructionBuilderOptions (line 16) | interface InstructionBuilderOptions {
type StepInfo (line 29) | interface StepInfo {
type StepPromptBuilderOptions (line 34) | interface StepPromptBuilderOptions {
constant TEMPLATES_DIR (line 55) | const TEMPLATES_DIR = resolve(dirname(fileURLToPath(import.meta.url)), '...
constant TEMPLATE_FILES (line 63) | const TEMPLATE_FILES: Record<PromptTemplate, string> = {
function loadTemplate (line 76) | function loadTemplate(variant: PromptTemplate): string {
function interpolate (line 97) | function interpolate(template: string, variables: Record<string, string>...
function clearTemplateCache (line 106) | function clearTemplateCache(): void {
class InstructionBuilder (line 123) | class InstructionBuilder {
method constructor (line 127) | constructor(options: InstructionBuilderOptions, actionDescriptions: st...
method build (line 139) | build(): string {
method fromSettings (line 171) | static fromSettings(settings: AgentConfig, registry: CommandCatalog, p...
method buildTaskPrompt (line 187) | static buildTaskPrompt(task: string): string {
method buildStatePrompt (line 191) | static buildStatePrompt(
method buildCommandResultPrompt (line 226) | static buildCommandResultPrompt(results: Array<{ action: string; resul...
method buildLoopNudge (line 236) | static buildLoopNudge(message: string): string {
method buildPlanPrompt (line 240) | static buildPlanPrompt(currentPlan: string): string {
class StepPromptBuilder (line 261) | class StepPromptBuilder {
method constructor (line 274) | constructor(options: StepPromptBuilderOptions) {
method getUserMessage (line 295) | getUserMessage(): string | ContentPart[] {
method buildStateDescription (line 329) | private buildStateDescription(): string {
method buildAgentHistorySection (line 352) | private buildAgentHistorySection(): string {
method buildAgentStateSection (line 357) | private buildAgentStateSection(): string {
method buildBrowserStateSection (line 380) | private buildBrowserStateSection(): string {
method buildTabsText (line 401) | private buildTabsText(): string {
method buildPageInfoText (line 423) | private buildPageInfoText(): string {
method buildElementsText (line 441) | private buildElementsText(): string {
function buildCommandDescriptions (line 479) | function buildCommandDescriptions(registry: CommandCatalog, pageUrl?: st...
function buildContextualCommands (line 491) | function buildContextualCommands(registry: CommandCatalog, pageUrl: stri...
function buildExtractionInstructionBuilder (line 517) | function buildExtractionInstructionBuilder(): string {
function buildExtractionUserPrompt (line 542) | function buildExtractionUserPrompt(
function extractDomain (line 556) | function extractDomain(url: string): string {
FILE: packages/core/src/agent/replay-recorder.ts
type ReplayRecorderOptions (line 7) | interface ReplayRecorderOptions {
type FrameData (line 18) | interface FrameData {
class ReplayRecorder (line 37) | class ReplayRecorder {
method constructor (line 44) | constructor(options: ReplayRecorderOptions) {
method addFrame (line 57) | addFrame(screenshotBase64: string, stepNumber?: number, label?: string...
method save (line 74) | async save(generateGif: string | boolean = true): Promise<string> {
method encodeGif (line 112) | private async encodeGif(outputPath: string): Promise<string> {
method createStepOverlaySvg (line 224) | private createStepOverlaySvg(stepNumber: number, label?: string): stri...
method saveFrames (line 239) | private async saveFrames(outputPath: string): Promise<string> {
method escapeXml (line 266) | private escapeXml(text: string): string {
method frameCount (line 275) | get frameCount(): number {
method clear (line 279) | clear(): void {
FILE: packages/core/src/agent/stall-detector.test.ts
function clickAction (line 12) | function clickAction(index: number): Command {
function inputAction (line 16) | function inputAction(index: number, text: string): Command {
function navigateAction (line 20) | function navigateAction(url: string): Command {
function scrollAction (line 24) | function scrollAction(direction: 'up' | 'down', index?: number): Command {
function doneAction (line 28) | function doneAction(text: string): Command {
function searchGoogleAction (line 32) | function searchGoogleAction(query: string): Command {
function makeFingerprint (line 36) | function makeFingerprint(overrides: Partial<PageSignature> = {}): PageSi...
FILE: packages/core/src/agent/stall-detector.ts
type PageSignature (line 5) | interface PageSignature {
type StallDetectorConfig (line 13) | interface StallDetectorConfig {
constant DEFAULT_OPTIONS (line 21) | const DEFAULT_OPTIONS: StallDetectorConfig = {
type StallCheckResult (line 28) | interface StallCheckResult {
constant ESCALATING_NUDGES (line 39) | const ESCALATING_NUDGES = [
class StallDetector (line 71) | class StallDetector {
method constructor (line 78) | constructor(options?: Partial<StallDetectorConfig>) {
method recordAction (line 82) | recordAction(actions: Command[]): void {
method recordFingerprint (line 92) | recordFingerprint(fingerprint: PageSignature): void {
method isStuck (line 103) | isStuck(): StallCheckResult {
method getLoopNudgeMessage (line 173) | getLoopNudgeMessage(): string {
method getTotalRepetitions (line 185) | getTotalRepetitions(): number {
method reset (line 189) | reset(): void {
method normalizeActionHash (line 204) | private normalizeActionHash(actions: Command[]): string {
method normalizeSearchQuery (line 246) | private normalizeSearchQuery(query: string): string {
method hashFingerprint (line 259) | private hashFingerprint(fp: PageSignature): string {
method countTrailingRepetitions (line 278) | private countTrailingRepetitions(history: string[]): number {
method countConsecutiveStagnantPages (line 296) | private countConsecutiveStagnantPages(): number {
method getSeverity (line 321) | private getSeverity(repetitions: number): number {
method getEscalatingNudge (line 331) | private getEscalatingNudge(): string {
function hashPageTree (line 347) | function hashPageTree(domTree: string): string {
function hashTextContent (line 360) | function hashTextContent(text: string): string {
FILE: packages/core/src/agent/types.ts
type AgentConfig (line 8) | interface AgentConfig {
constant DEFAULT_AGENT_CONFIG (line 69) | const DEFAULT_AGENT_CONFIG: AgentConfig = {
type CompactionPolicy (line 104) | interface CompactionPolicy {
type Reasoning (line 123) | type Reasoning = z.infer<typeof ReasoningSchema>;
type AgentDecision (line 136) | type AgentDecision = z.infer<typeof AgentDecisionSchema>;
type AgentDecisionCompact (line 150) | type AgentDecisionCompact = z.infer<typeof AgentDecisionCompactSchema>;
type AgentDecisionDirect (line 161) | type AgentDecisionDirect = z.infer<typeof AgentDecisionDirectSchema>;
type StepTelemetry (line 165) | interface StepTelemetry {
type ExtractedVariable (line 191) | interface ExtractedVariable {
type AgentState (line 204) | interface AgentState {
type StepRecord (line 223) | interface StepRecord {
class ExecutionLog (line 242) | class ExecutionLog {
method constructor (line 252) | constructor(init: {
method recomputeTotals (line 267) | recomputeTotals(): void {
method addEntry (line 280) | addEntry(entry: StepRecord): void {
method finish (line 286) | finish(): void {
method finalResult (line 296) | finalResult(): string | undefined {
method isDone (line 311) | isDone(): boolean {
method urls (line 320) | urls(): string[] {
method screenshots (line 336) | screenshots(): string[] {
method errors (line 349) | errors(): string[] {
method allExtractedVariables (line 367) | allExtractedVariables(): ExtractedVariable[] {
method toJSON (line 380) | toJSON(): Record<string, unknown> {
method saveToFile (line 404) | async saveToFile(filePath: string): Promise<string> {
method loadFromFile (line 416) | static async loadFromFile(filePath: string): Promise<ExecutionLog> {
type PlanStep (line 445) | type PlanStep = z.infer<typeof PlanStepSchema>;
type EvaluationResult (line 463) | type EvaluationResult = z.infer<typeof EvaluationResultSchema>;
type QuickCheckResult (line 475) | type QuickCheckResult = z.infer<typeof QuickCheckResultSchema>;
type StepCostBreakdown (line 479) | interface StepCostBreakdown {
type AccumulatedCost (line 485) | interface AccumulatedCost {
type PricingTable (line 494) | interface PricingTable {
constant PRICING_TABLE (line 499) | const PRICING_TABLE: Record<string, PricingTable> = {
function calculateStepCost (line 512) | function calculateStepCost(
type PlanRevision (line 538) | type PlanRevision = z.infer<typeof PlanRevisionSchema>;
constant EXTENDED_THINKING_MODELS (line 542) | const EXTENDED_THINKING_MODELS = [
function supportsDeepReasoning (line 555) | function supportsDeepReasoning(modelId: string): boolean {
constant COORDINATE_CLICK_MODELS (line 559) | const COORDINATE_CLICK_MODELS = [
function supportsCoordinateMode (line 567) | function supportsCoordinateMode(modelId: string): boolean {
constant FLASH_MODELS (line 571) | const FLASH_MODELS = [
function isCompactModel (line 579) | function isCompactModel(modelId: string): boolean {
type RunOutcome (line 585) | interface RunOutcome {
FILE: packages/core/src/bridge/adapter.ts
type MCPToolDefinition (line 4) | interface MCPToolDefinition {
class BridgeAdapter (line 10) | class BridgeAdapter {
method constructor (line 13) | constructor(tools: CommandExecutor) {
method getToolDefinitions (line 17) | getToolDefinitions(): MCPToolDefinition[] {
method getToolNames (line 25) | getToolNames(): string[] {
method parseToolName (line 29) | parseToolName(mcpToolName: string): string | null {
method zodToJsonSchema (line 36) | private zodToJsonSchema(schema: ZodTypeAny): Record<string, unknown> {
method fieldToJsonSchema (line 61) | private fieldToJsonSchema(schema: ZodTypeAny): Record<string, unknown> {
FILE: packages/core/src/bridge/client.ts
type BridgeClientOptions (line 10) | interface BridgeClientOptions {
type MCPTool (line 24) | interface MCPTool {
type MCPConnectionState (line 30) | type MCPConnectionState = 'disconnected' | 'connecting' | 'connected' | ...
type PendingRequest (line 32) | interface PendingRequest {
type BridgeClientEvents (line 39) | interface BridgeClientEvents {
class BridgeClient (line 58) | class BridgeClient extends EventEmitter<BridgeClientEvents> {
method constructor (line 83) | constructor(options: BridgeClientOptions) {
method state (line 94) | get state(): MCPConnectionState {
method isConnected (line 98) | get isConnected(): boolean {
method connect (line 104) | async connect(): Promise<void> {
method spawnProcess (line 125) | private async spawnProcess(): Promise<void> {
method initialize (line 153) | private async initialize(): Promise<void> {
method setState (line 166) | private setState(newState: MCPConnectionState): void {
method handleProcessClose (line 177) | private handleProcessClose(): void {
method attemptReconnect (line 196) | private attemptReconnect(): void {
method listTools (line 239) | async listTools(): Promise<MCPTool[]> {
method getTools (line 253) | getTools(): MCPTool[] {
method invalidateToolCache (line 258) | invalidateToolCache(): void {
method toolsCacheAge (line 264) | get toolsCacheAge(): number {
method toCustomActions (line 270) | toCustomActions(): CustomCommandSpec[] {
method callTool (line 288) | async callTool(name: string, args: Record<string, unknown>): Promise<u...
method ping (line 306) | async ping(): Promise<void> {
method startHealthChecks (line 310) | private startHealthChecks(): void {
method stopHealthChecks (line 324) | private stopHealthChecks(): void {
method send (line 333) | private send(method: string, params?: Record<string, unknown>): Promis...
method sendNotification (line 361) | private sendNotification(method: string, params?: Record<string, unkno...
method processBuffer (line 373) | private processBuffer(): void {
method handleServerNotification (line 405) | private handleServerNotification(message: {
method disconnect (line 424) | async disconnect(drainTimeoutMs = 5000): Promise<void> {
method waitForPendingDrain (line 463) | private waitForPendingDrain(): Promise<void> {
method pendingRequestCount (line 477) | get pendingRequestCount(): number {
FILE: packages/core/src/bridge/mcp-types.ts
type MCPServerOptions (line 6) | interface MCPServerOptions {
type MCPCapability (line 12) | type MCPCapability = 'browse' | 'extract' | 'screenshot' | 'interact';
type MCPRequest (line 14) | interface MCPRequest {
type MCPResponse (line 19) | interface MCPResponse {
FILE: packages/core/src/bridge/server.test.ts
function makeMockViewport (line 7) | function makeMockViewport() {
function makeMockPageAnalyzer (line 37) | function makeMockPageAnalyzer() {
function makeRequest (line 58) | function makeRequest(
FILE: packages/core/src/bridge/server.ts
type BridgeServerOptions (line 13) | interface BridgeServerOptions {
type MCPRequest (line 23) | interface MCPRequest {
type MCPResponse (line 30) | interface MCPResponse {
type MCPNotification (line 37) | interface MCPNotification {
type MCPResource (line 45) | interface MCPResource {
type MCPResourceContent (line 52) | interface MCPResourceContent {
type ResourceSubscription (line 61) | interface ResourceSubscription {
class BridgeServer (line 78) | class BridgeServer {
method constructor (line 99) | constructor(options: BridgeServerOptions) {
method getResourceDefinitions (line 111) | private getResourceDefinitions(): MCPResource[] {
method handleMessage (line 142) | async handleMessage(message: MCPRequest): Promise<MCPResponse | null> {
method handleRequest (line 152) | async handleRequest(request: MCPRequest & { id: string | number }): Pr...
method handleNotification (line 191) | private async handleNotification(message: MCPRequest): Promise<void> {
method handleInitialize (line 208) | private handleInitialize(request: MCPRequest & { id: string | number }...
method handleToolsList (line 229) | private handleToolsList(request: MCPRequest & { id: string | number })...
method handleToolsCall (line 244) | private async handleToolsCall(request: MCPRequest & { id: string | num...
method handleResourcesList (line 294) | private handleResourcesList(request: MCPRequest & { id: string | numbe...
method handleResourcesRead (line 304) | private async handleResourcesRead(request: MCPRequest & { id: string |...
method readResource (line 335) | private async readResource(uri: string): Promise<MCPResourceContent> {
method handleResourcesSubscribe (line 378) | private handleResourcesSubscribe(request: MCPRequest & { id: string | ...
method handleResourcesUnsubscribe (line 407) | private handleResourcesUnsubscribe(request: MCPRequest & { id: string ...
method emitProgress (line 425) | emitProgress(requestId: string | number, progress: number, message?: s...
method notifyResourceChanged (line 440) | private notifyResourceChanged(uri: string): void {
method broadcastNotification (line 452) | private broadcastNotification(notification: MCPNotification): void {
method startStdio (line 468) | async startStdio(): Promise<void> {
method startSSE (line 517) | async startSSE(port?: number): Promise<void> {
method handleSSEConnection (line 568) | private handleSSEConnection(res: ServerResponse): void {
method handleSSEMessage (line 588) | private async handleSSEMessage(req: IncomingMessage, res: ServerRespon...
method stopSSE (line 625) | async stopSSE(): Promise<void> {
method stop (line 647) | async stop(): Promise<void> {
FILE: packages/core/src/commands/catalog.test.ts
function makeHandler (line 9) | function makeHandler(
function makeContext (line 15) | function makeContext(overrides: Partial<ExecutionContext> = {}): Executi...
FILE: packages/core/src/commands/catalog/catalog.ts
constant SPECIAL_PARAMS (line 12) | const SPECIAL_PARAMS = new Set([
function inspectHandlerParams (line 26) | function inspectHandlerParams(handler: Function): string[] {
function detectSpecialParams (line 87) | function detectSpecialParams(handler: Function): Set<string> {
function resolveSpecialParam (line 101) | function resolveSpecialParam(
class CommandCatalog (line 125) | class CommandCatalog {
method constructor (line 130) | constructor(options?: CatalogOptions) {
method register (line 134) | register(action: CatalogEntry): void {
method registerCustom (line 153) | registerCustom(definition: CustomCommandSpec): void {
method unregister (line 163) | unregister(name: string): void {
method get (line 168) | get(name: string): CatalogEntry | undefined {
method has (line 172) | has(name: string): boolean {
method getAll (line 176) | getAll(): CatalogEntry[] {
method getNames (line 180) | getNames(): string[] {
method execute (line 184) | async execute(
method getSpecialParams (line 216) | getSpecialParams(name: string): Set<string> {
method injectSpecialParams (line 225) | private injectSpecialParams(
method buildDynamicSchema (line 248) | buildDynamicSchema(): z.ZodType {
method size (line 269) | get size(): number {
method getPromptDescription (line 279) | getPromptDescription(pageUrl?: string): string {
method getActionsForDomain (line 325) | getActionsForDomain(domain: string): CatalogEntry[] {
method replaceSensitiveData (line 344) | replaceSensitiveData(
method getTerminatingActions (line 370) | getTerminatingActions(): string[] {
method isTerminating (line 379) | isTerminating(name: string): boolean {
function extractDomain (line 387) | function extractDomain(url: string): string | null {
FILE: packages/core/src/commands/catalog/types.ts
type CatalogEntry (line 4) | interface CatalogEntry {
type CatalogOptions (line 13) | interface CatalogOptions {
FILE: packages/core/src/commands/executor.test.ts
function makeMockPageAnalyzer (line 8) | function makeMockPageAnalyzer() {
function makeMockViewport (line 29) | function makeMockViewport() {
function makeMockPage (line 45) | function makeMockPage() {
function makeMockCdpSession (line 62) | function makeMockCdpSession() {
function makeContext (line 68) | function makeContext(overrides: Partial<ExecutionContext> = {}): Executi...
function action (line 85) | function action(a: Record<string, unknown>): Command {
FILE: packages/core/src/commands/executor.ts
type CommandExecutorOptions (line 50) | interface CommandExecutorOptions {
class CommandExecutor (line 57) | class CommandExecutor {
method constructor (line 65) | constructor(options?: CommandExecutorOptions) {
method setCoordinateClicking (line 83) | setCoordinateClicking(enabled: boolean): void {
method registerBuiltinActions (line 87) | private registerBuiltinActions(): void {
method useStructuredOutputAction (line 693) | private useStructuredOutputAction(): void {
method executeAction (line 757) | async executeAction(
method executeActions (line 765) | async executeActions(
method maskSensitiveResult (line 810) | private maskSensitiveResult(
method maskSensitiveText (line 835) | private maskSensitiveText(
function buildSearchUrl (line 846) | function buildSearchUrl(
constant ERROR_PATTERNS (line 867) | const ERROR_PATTERNS: Array<{
function classifyViewportError (line 1027) | function classifyViewportError(error: unknown): InterpretedViewportError {
FILE: packages/core/src/commands/extraction/extractor.ts
type ExtractionResult (line 16) | type ExtractionResult = z.infer<typeof ExtractionResultSchema>;
class ContentExtractor (line 18) | class ContentExtractor {
method constructor (line 21) | constructor(model: LanguageModel) {
method extract (line 25) | async extract(page: Page, goal: string, startFromChar?: number): Promi...
method extractStructured (line 69) | async extractStructured<T>(
method extractLinks (line 123) | async extractLinks(
method extractFromText (line 131) | async extractFromText(
method extractFromTextWithJsonSchema (line 160) | private async extractFromTextWithJsonSchema(
method combineExtractions (line 190) | private async combineExtractions(results: string[], goal: string): Pro...
FILE: packages/core/src/commands/types.ts
type Command (line 172) | type Command = z.infer<typeof CommandSchema>;
type CommandName (line 174) | type CommandName = Command['action'];
type CommandResult (line 178) | interface CommandResult {
type ViewportErrorCategory (line 188) | type ViewportErrorCategory =
type InterpretedViewportError (line 199) | interface InterpretedViewportError {
type CustomCommandSpec (line 208) | interface CustomCommandSpec {
type ExecutionContext (line 216) | interface ExecutionContext {
FILE: packages/core/src/commands/utils.ts
function scrollPage (line 3) | async function scrollPage(
function scrollElement (line 19) | async function scrollElement(
function buildGoogleSearchUrl (line 39) | function buildGoogleSearchUrl(query: string): string {
FILE: packages/core/src/config/config.ts
class Config (line 13) | class Config {
method constructor (line 16) | private constructor(overrides: DeepPartial<GlobalConfig> = {}) {
method instance (line 29) | static instance(overrides?: DeepPartial<GlobalConfig>): Config {
method reset (line 36) | static reset(): void {
method mergeEnvDefaults (line 40) | private mergeEnvDefaults(overrides: DeepPartial<GlobalConfig>): DeepPa...
method deepMerge (line 68) | private deepMerge(...objects: DeepPartial<GlobalConfig>[]): DeepPartia...
method browser (line 96) | get browser() {
method agent (line 100) | get agent() {
method configDir (line 104) | static get configDir(): string {
method tmpDir (line 112) | static get tmpDir(): string {
method configFilePath (line 120) | static get configFilePath(): string {
method loadConfigFile (line 124) | static loadConfigFile(): DeepPartial<GlobalConfig> {
method saveConfigFile (line 139) | static saveConfigFile(config: ConfigFileContents): void {
method isDocker (line 149) | static isDocker(): boolean {
method hasDisplay (line 162) | static hasDisplay(): boolean {
FILE: packages/core/src/config/types.ts
type ProxyConfig (line 10) | type ProxyConfig = z.infer<typeof ProxyConfigSchema>;
type ViewportConfig (line 35) | type ViewportConfig = z.infer<typeof ViewportConfigSchema>;
type AgentConfig (line 74) | type AgentConfig = z.infer<typeof AgentConfigSchema>;
type GlobalConfig (line 83) | type GlobalConfig = z.infer<typeof GlobalConfigSchema>;
type ConfigFileContents (line 85) | interface ConfigFileContents {
FILE: packages/core/src/errors.ts
class OpenBrowserError (line 1) | class OpenBrowserError extends Error {
method constructor (line 2) | constructor(message: string, options?: ErrorOptions) {
class ViewportError (line 8) | class ViewportError extends OpenBrowserError {
method constructor (line 9) | constructor(message: string, options?: ErrorOptions) {
class LaunchFailedError (line 15) | class LaunchFailedError extends ViewportError {
method constructor (line 16) | constructor(message: string, options?: ErrorOptions) {
class NavigationFailedError (line 22) | class NavigationFailedError extends ViewportError {
method constructor (line 23) | constructor(
class ViewportCrashedError (line 33) | class ViewportCrashedError extends ViewportError {
method constructor (line 34) | constructor(message = 'Browser has crashed', options?: ErrorOptions) {
class AgentError (line 40) | class AgentError extends OpenBrowserError {
method constructor (line 41) | constructor(message: string, options?: ErrorOptions) {
class AgentStalledError (line 47) | class AgentStalledError extends AgentError {
method constructor (line 48) | constructor(message = 'Agent is stuck in a loop', options?: ErrorOptio...
class StepLimitExceededError (line 54) | class StepLimitExceededError extends AgentError {
method constructor (line 58) | constructor(stepsTaken: number, stepLimit: number, options?: ErrorOpti...
class UrlBlockedError (line 66) | class UrlBlockedError extends OpenBrowserError {
method constructor (line 69) | constructor(url: string, options?: ErrorOptions) {
class PageExtractionError (line 76) | class PageExtractionError extends OpenBrowserError {
method constructor (line 77) | constructor(message: string, options?: ErrorOptions) {
class ModelError (line 83) | class ModelError extends OpenBrowserError {
method constructor (line 84) | constructor(message: string, options?: ErrorOptions) {
class ModelThrottledError (line 90) | class ModelThrottledError extends ModelError {
method constructor (line 93) | constructor(message: string, retryAfterMs?: number, options?: ErrorOpt...
class CommandFailedError (line 100) | class CommandFailedError extends OpenBrowserError {
method constructor (line 103) | constructor(toolName: string, message: string, options?: ErrorOptions) {
class ContextualViewportError (line 110) | class ContextualViewportError extends ViewportError {
method constructor (line 115) | constructor(
class ProviderError (line 131) | class ProviderError extends ModelError {
method constructor (line 135) | constructor(
method isRetryable (line 147) | get isRetryable(): boolean {
class SchemaViolationError (line 153) | class SchemaViolationError extends OpenBrowserError {
method constructor (line 157) | constructor(field: string, issues: string[], options?: ErrorOptions) {
FILE: packages/core/src/logging.ts
constant LEVEL_NAMES (line 3) | const LEVEL_NAMES: Record<number, string> = {
constant LEVEL_COLORS (line 10) | const LEVEL_COLORS: Record<number, string> = {
constant RESET (line 17) | const RESET = '\x1b[0m';
constant DIM (line 18) | const DIM = '\x1b[2m';
constant BOLD (line 19) | const BOLD = '\x1b[1m';
function setGlobalLogLevel (line 25) | function setGlobalLogLevel(level: LogLevel): void {
function getGlobalLogLevel (line 29) | function getGlobalLogLevel(): LogLevel {
function setLogColors (line 33) | function setLogColors(enabled: boolean): void {
function setLogTimestamps (line 37) | function setLogTimestamps(enabled: boolean): void {
function formatTimestamp (line 41) | function formatTimestamp(): string {
function formatMessage (line 50) | function formatMessage(
class Logger (line 77) | class Logger {
method constructor (line 81) | constructor(name: string) {
method setLevel (line 85) | setLevel(level: LogLevel): void {
method getEffectiveLevel (line 89) | getEffectiveLevel(): LogLevel {
method isEnabled (line 93) | isEnabled(level: LogLevel): boolean {
method debug (line 97) | debug(message: string, ...args: unknown[]): void {
method info (line 101) | info(message: string, ...args: unknown[]): void {
method warn (line 105) | warn(message: string, ...args: unknown[]): void {
method error (line 109) | error(message: string, ...args: unknown[]): void {
method log (line 113) | private log(level: LogLevel, message: string, ...args: unknown[]): void {
function createLogger (line 133) | function createLogger(name: string): Logger {
FILE: packages/core/src/metering/tracker.test.ts
constant TEST_PRICING (line 12) | const TEST_PRICING: PricingTable = {
FILE: packages/core/src/metering/tracker.ts
class UsageMeter (line 17) | class UsageMeter {
method constructor (line 23) | constructor(modelId: string, customPricing?: PricingTable) {
method record (line 28) | record(inputTokens: number, outputTokens: number): void {
method getTotalUsage (line 41) | getTotalUsage(): UsageRecord {
method getStepUsages (line 45) | getStepUsages(): UsageRecord[] {
method getEstimatedCost (line 49) | getEstimatedCost(): number {
method getEstimatedCostFormatted (line 59) | getEstimatedCostFormatted(): string {
method getModelCost (line 64) | private getModelCost(): CostRates | undefined {
method getSummary (line 68) | getSummary(): string {
method reset (line 80) | reset(): void {
class CompositeUsageMeter (line 92) | class CompositeUsageMeter {
method constructor (line 100) | constructor(customPricing?: PricingTable) {
method start (line 105) | start(): void {
method setBudget (line 113) | setBudget(config: BudgetPolicy): this {
method clearBudget (line 123) | clearBudget(): void {
method record (line 133) | record(opts: {
method getTracker (line 173) | getTracker(modelId: string): UsageMeter {
method getTotalCost (line 178) | getTotalCost(): number {
method getTotalCostFormatted (line 187) | getTotalCostFormatted(): string {
method getTotalUsage (line 192) | getTotalUsage(): UsageRecord {
method getBudgetState (line 204) | getBudgetState(): BudgetState {
method getSummary (line 218) | getSummary(): MeteringSummary {
method getSummaryText (line 235) | getSummaryText(): string {
method reset (line 279) | reset(): void {
method getOrCreateTracker (line 291) | private getOrCreateTracker(modelId: string): UsageMeter {
method checkBudget (line 300) | private checkBudget(): void {
method buildModelBreakdown (line 328) | private buildModelBreakdown(): ModelUsageBreakdown[] {
method buildRoleBreakdown (line 354) | private buildRoleBreakdown(): RoleUsageBreakdown[] {
class BudgetDepletedError (line 383) | class BudgetDepletedError extends Error {
method constructor (line 387) | constructor(currentCost: number, maxCost: number) {
function estimateTokenCount (line 399) | function estimateTokenCount(text: string): number {
function resolveModelCost (line 404) | function resolveModelCost(modelId: string, pricing: PricingTable): CostR...
function computeCost (line 416) | function computeCost(
FILE: packages/core/src/metering/types.ts
type UsageRecord (line 1) | interface UsageRecord {
type CostRates (line 7) | interface CostRates {
type PricingTable (line 12) | interface PricingTable {
type ModelRole (line 23) | type ModelRole = 'main' | 'extraction' | 'judge' | 'compaction';
type ActionUsageRecord (line 26) | interface ActionUsageRecord {
type ModelUsageBreakdown (line 37) | interface ModelUsageBreakdown {
type RoleUsageBreakdown (line 47) | interface RoleUsageBreakdown {
type MeteringSummary (line 57) | interface MeteringSummary {
type BudgetPolicy (line 79) | interface BudgetPolicy {
type BudgetState (line 97) | interface BudgetState {
constant DEFAULT_COST_RATES (line 108) | const DEFAULT_COST_RATES: PricingTable = {
FILE: packages/core/src/model/adapters/vercel.ts
type VercelModelAdapterOptions (line 9) | interface VercelModelAdapterOptions {
class VercelModelAdapter (line 18) | class VercelModelAdapter implements LanguageModel {
method constructor (line 25) | constructor(options: VercelModelAdapterOptions) {
method modelId (line 33) | get modelId(): string {
method provider (line 37) | get provider(): ModelProvider {
method invoke (line 41) | async invoke<T>(options: InferenceOptions<T>): Promise<InferenceResult...
method convertMessages (line 83) | private convertMessages(messages: Message[]): CoreMessage[] {
method convertContentPart (line 118) | private convertContentPart(
function mapFinishReason (line 139) | function mapFinishReason(
constant PROVIDER_PATTERNS (line 158) | const PROVIDER_PATTERNS: Array<[RegExp, ModelProvider]> = [
function inferProvider (line 169) | function inferProvider(modelId: string, providerHint?: string): ModelPro...
FILE: packages/core/src/model/interface.ts
type ModelProvider (line 6) | type ModelProvider =
type InferenceOptions (line 17) | interface InferenceOptions<T> {
type LanguageModel (line 46) | interface LanguageModel {
FILE: packages/core/src/model/messages.ts
type TextContent (line 1) | interface TextContent {
type ImageContent (line 6) | interface ImageContent {
type ContentPart (line 13) | type ContentPart = TextContent | ImageContent;
type SystemMessage (line 15) | interface SystemMessage {
type UserMessage (line 20) | interface UserMessage {
type AssistantMessage (line 25) | interface AssistantMessage {
type ToolResultMessage (line 31) | interface ToolResultMessage {
type ToolCall (line 37) | interface ToolCall {
type Message (line 43) | type Message = SystemMessage | UserMessage | AssistantMessage | ToolResu...
function systemMessage (line 47) | function systemMessage(content: string): SystemMessage {
function userMessage (line 51) | function userMessage(content: string | ContentPart[]): UserMessage {
function assistantMessage (line 55) | function assistantMessage(
function toolResultMessage (line 62) | function toolResultMessage(toolCallId: string, content: string): ToolRes...
function textContent (line 66) | function textContent(text: string): TextContent {
function imageContent (line 70) | function imageContent(base64: string, mediaType = 'image/png'): ImageCon...
FILE: packages/core/src/model/schema-optimizer.ts
type SchemaOptimizationOptions (line 6) | interface SchemaOptimizationOptions {
constant DEFAULTS (line 28) | const DEFAULTS: Required<Omit<SchemaOptimizationOptions, 'provider'>> = {
function optimizeJsonSchemaForModel (line 41) | function optimizeJsonSchemaForModel(
function optimizeSchemaForModel (line 67) | function optimizeSchemaForModel<T extends ZodTypeAny>(
function collapseUnions (line 107) | function collapseUnions(
function collapseEnums (line 138) | function collapseEnums(
function flattenNesting (line 178) | function flattenNesting(
function flattenProperties (line 221) | function flattenProperties(
function applyProviderTweaks (line 268) | function applyProviderTweaks(
function applyGeminiTweaks (line 286) | function applyGeminiTweaks(schema: Record<string, unknown>): Record<stri...
function applyOpenAITweaks (line 313) | function applyOpenAITweaks(schema: Record<string, unknown>): Record<stri...
type SchemaVisitor (line 335) | type SchemaVisitor = (node: Record<string, unknown>) => Record<string, u...
function walkSchema (line 341) | function walkSchema(
function humanizePropertyName (line 394) | function humanizePropertyName(name: string): string {
function zodToJsonSchema (line 414) | function zodToJsonSchema(schema: ZodTypeAny): Record<string, unknown> {
FILE: packages/core/src/model/types.ts
type InferenceUsage (line 3) | interface InferenceUsage {
type InferenceResult (line 9) | interface InferenceResult<T = unknown> {
FILE: packages/core/src/page/content-extractor.ts
function getTurndown (line 6) | function getTurndown(): TurndownService {
function htmlTableToMarkdown (line 47) | function htmlTableToMarkdown(table: HTMLTableElement): string {
constant KNOWN_LANGUAGES (line 88) | const KNOWN_LANGUAGES = new Set([
function detectCodeLanguage (line 108) | function detectCodeLanguage(codeEl: HTMLElement | null): string {
class ReadingState (line 144) | class ReadingState {
method currentOffset (line 152) | get currentOffset(): number {
method contentLength (line 159) | get contentLength(): number {
method hasMore (line 166) | get hasMore(): boolean {
method progress (line 173) | get progress(): number {
method advance (line 181) | advance(chars: number): void {
method update (line 189) | update(url: string, totalLength: number): void {
method reset (line 200) | reset(): void {
type MarkdownExtractionOptions (line 207) | interface MarkdownExtractionOptions {
function extractMarkdown (line 214) | async function extractMarkdown(
function htmlToMarkdown (line 283) | function htmlToMarkdown(html: string): string {
function extractLinks (line 297) | async function extractLinks(
function extractTextContent (line 332) | async function extractTextContent(page: Page): Promise<string> {
function chunkText (line 340) | function chunkText(text: string, maxChunkSize: number): string[] {
FILE: packages/core/src/page/page-analyzer.test.ts
function makeMockPage (line 9) | function makeMockPage(overrides: Record<string, unknown> = {}) {
function makeMockCdpSession (line 26) | function makeMockCdpSession(overrides: Record<string, unknown> = {}) {
function makeNode (line 33) | function makeNode(overrides: Partial<PageTreeNode> = {}): PageTreeNode {
FILE: packages/core/src/page/page-analyzer.ts
type PageAnalyzerOptions (line 19) | interface PageAnalyzerOptions {
class PageAnalyzer (line 27) | class PageAnalyzer {
method constructor (line 40) | constructor(options?: PageAnalyzerOptions) {
method extractState (line 56) | async extractState(
method _extractState (line 66) | private async _extractState(
method extractWithIframes (line 148) | async extractWithIframes(
method extractCrossOriginIframe (line 218) | private async extractCrossOriginIframe(
method collectHiddenElementHints (line 292) | private collectHiddenElementHints(
method applyViewportThresholdFilter (line 338) | private applyViewportThresholdFilter(
method integrateShadowDOMChildren (line 377) | private integrateShadowDOMChildren(root: PageTreeNode): void {
method getElementSelector (line 397) | async getElementSelector(index: number): Promise<string | undefined> {
method getElementByBackendNodeId (line 401) | async getElementByBackendNodeId(
method clickElementByIndex (line 431) | async clickElementByIndex(
method clickAtCoordinates (line 488) | async clickAtCoordinates(
method inputTextByIndex (line 496) | async inputTextByIndex(
method recordInteraction (line 520) | private recordInteraction(
method getInteractedElements (line 533) | getInteractedElements(): InteractedElement[] {
method clearInteractedElements (line 537) | clearInteractedElements(): void {
method getCachedTree (line 541) | getCachedTree(): PageTreeNode | null {
method getCachedSelectorMap (line 545) | getCachedSelectorMap(): SelectorIndex | null {
method clearCache (line 549) | clearCache(): void {
FILE: packages/core/src/page/renderer.test.ts
function makeNode (line 8) | function makeNode(overrides: Partial<PageTreeNode> = {}): PageTreeNode {
function makeTextNode (line 23) | function makeTextNode(text: string): PageTreeNode {
FILE: packages/core/src/page/renderer/interactive-elements.ts
constant ALWAYS_CLICKABLE_TAGS (line 3) | const ALWAYS_CLICKABLE_TAGS = new Set([
constant CLICKABLE_ROLES (line 7) | const CLICKABLE_ROLES = new Set([
function isClickableElement (line 12) | function isClickableElement(node: PageTreeNode): boolean {
function getClickableDescription (line 21) | function getClickableDescription(node: PageTreeNode): string {
FILE: packages/core/src/page/renderer/layer-order.ts
function filterByPaintOrder (line 7) | function filterByPaintOrder(nodes: PageTreeNode[]): PageTreeNode[] {
function rectsOverlap (line 52) | function rectsOverlap(a: DOMRect, b: DOMRect, threshold: number): boolean {
FILE: packages/core/src/page/renderer/tree-renderer.ts
type RendererOptions (line 6) | interface RendererOptions {
constant DEFAULT_OPTIONS (line 17) | const DEFAULT_OPTIONS: RendererOptions = {
constant SVG_TAGS (line 31) | const SVG_TAGS = new Set(['svg', 'path', 'rect', 'circle', 'ellipse', 'l...
class TreeRenderer (line 33) | class TreeRenderer {
method constructor (line 36) | constructor(options?: Partial<RendererOptions>) {
method serializeTree (line 40) | serializeTree(
method serializeNode (line 126) | private serializeNode(
method serializeChildrenWithDedup (line 257) | private serializeChildrenWithDedup(
method isRedundantWrapper (line 312) | private isRedundantWrapper(node: PageTreeNode): boolean {
method getInlineText (line 337) | private getInlineText(node: PageTreeNode): string | null {
method hasVisibleDescendant (line 351) | private hasVisibleDescendant(node: PageTreeNode): boolean {
method hasInteractiveDescendant (line 359) | private hasInteractiveDescendant(node: PageTreeNode): boolean {
method collectInteractiveElements (line 367) | private collectInteractiveElements(
method buildCssSelector (line 379) | private buildCssSelector(node: PageTreeNode): string {
method filterOffScreenElements (line 417) | private filterOffScreenElements(
method formatHiddenElementHints (line 484) | private formatHiddenElementHints(
method getNodeDescription (line 520) | private getNodeDescription(node: PageTreeNode): string {
method resolveSvgDescription (line 533) | private resolveSvgDescription(node: PageTreeNode): string {
FILE: packages/core/src/page/snapshot-builder.ts
constant INTERACTIVE_TAGS (line 10) | const INTERACTIVE_TAGS = new Set([
constant INTERACTIVE_ROLES (line 15) | const INTERACTIVE_ROLES = new Set([
constant INVISIBLE_TAGS (line 22) | const INVISIBLE_TAGS = new Set([
class SnapshotBuilder (line 26) | class SnapshotBuilder {
method captureSnapshot (line 29) | async captureSnapshot(cdpSession: CDPSession): Promise<{
method buildTree (line 51) | buildTree(
method buildNodeTree (line 117) | private buildNodeTree(
method buildAXMap (line 223) | private buildAXMap(node: AXNode, map: Map<number, AXNode>): void {
method createEmptyNode (line 234) | private createEmptyNode(): PageTreeNode {
FILE: packages/core/src/page/types.ts
type DOMRect (line 3) | interface DOMRect {
type TargetInfo (line 10) | interface TargetInfo {
type TargetAllTrees (line 18) | interface TargetAllTrees {
type InteractedElement (line 27) | interface InteractedElement {
type MatchLevel (line 43) | type MatchLevel = (typeof MatchLevel)[keyof typeof MatchLevel];
type SimplifiedNode (line 45) | interface SimplifiedNode {
type PageTreeNode (line 54) | interface PageTreeNode {
type SelectorIndex (line 103) | interface SelectorIndex {
type RenderedPageState (line 115) | interface RenderedPageState {
type CDPDOMNode (line 127) | interface CDPDOMNode {
type CDPLayoutNode (line 144) | interface CDPLayoutNode {
type CDPSnapshotResult (line 153) | interface CDPSnapshotResult {
type AXNode (line 185) | interface AXNode {
FILE: packages/core/src/sandbox/file-access.ts
constant ALLOWED_EXTENSIONS (line 7) | const ALLOWED_EXTENSIONS = new Set([
constant MAX_FILE_SIZE (line 14) | const MAX_FILE_SIZE = 10 * 1024 * 1024;
type FileAccessOptions (line 16) | interface FileAccessOptions {
type FileInfo (line 23) | interface FileInfo {
type FileAccessState (line 32) | interface FileAccessState {
class FileAccess (line 38) | class FileAccess {
method constructor (line 45) | constructor(options: FileAccessOptions) {
method indexDirectory (line 66) | private indexDirectory(): void {
method resolvePath (line 89) | private resolvePath(relativePath: string): string {
method validateExtension (line 98) | private validateExtension(filePath: string): void {
method isBinaryFile (line 107) | private isBinaryFile(filePath: string): boolean {
method read (line 120) | async read(relativePath: string): Promise<string> {
method write (line 143) | async write(relativePath: string, content: string): Promise<void> {
method list (line 179) | async list(relativeDir = '.'): Promise<FileInfo[]> {
method delete (line 206) | async delete(relativePath: string): Promise<void> {
method exists (line 226) | async exists(relativePath: string): Promise<boolean> {
method getState (line 231) | getState(): FileAccessState {
method getSandboxDir (line 239) | getSandboxDir(): string {
FILE: packages/core/src/telemetry.ts
type TimingResult (line 5) | interface TimingResult<T> {
function timed (line 14) | async function timed<T>(
function withTiming (line 34) | function withTiming<Args extends unknown[], R>(
class Stopwatch (line 47) | class Stopwatch {
method constructor (line 51) | constructor() {
method split (line 55) | split(label: string): number {
method elapsed (line 61) | elapsed(): number {
method reset (line 65) | reset(): void {
method getSplits (line 70) | getSplits(): Array<{ label: string; timeMs: number }> {
method summary (line 74) | summary(): string {
FILE: packages/core/src/types.ts
type Brand (line 6) | type Brand<T, B extends string> = T & { readonly [__brand]: B };
type TargetId (line 8) | type TargetId = Brand<string, 'TargetId'>;
type SessionId (line 9) | type SessionId = Brand<string, 'SessionId'>;
type ElementRef (line 10) | type ElementRef = Brand<number, 'ElementRef'>;
type TabId (line 11) | type TabId = Brand<number, 'TabId'>;
function targetId (line 13) | function targetId(id: string): TargetId {
function sessionId (line 17) | function sessionId(id: string): SessionId {
function elementIndex (line 21) | function elementIndex(index: number): ElementRef {
function tabId (line 25) | function tabId(id: number): TabId {
type Result (line 31) | type Result<T, E = Error> = { ok: true; value: T } | { ok: false; error:...
function ok (line 33) | function ok<T>(value: T): Result<T, never> {
function err (line 37) | function err<E>(error: E): Result<never, E> {
type Position (line 47) | type Position = z.infer<typeof PositionSchema>;
type Rect (line 55) | type Rect = z.infer<typeof RectSchema>;
type LogLevel (line 65) | type LogLevel = (typeof LogLevel)[keyof typeof LogLevel];
type DeepPartial (line 69) | type DeepPartial<T> = {
type Awaitable (line 73) | type Awaitable<T> = T | Promise<T>;
FILE: packages/core/src/utils.ts
function generateId (line 5) | function generateId(size = 12): string {
function matchesUrlPattern (line 11) | function matchesUrlPattern(url: string, pattern: string): boolean {
function isUrlPermitted (line 41) | function isUrlPermitted(
function sanitizeText (line 57) | function sanitizeText(text: string): string {
function truncateText (line 64) | function truncateText(text: string, maxLength: number, suffix = '...'): ...
function removeTags (line 69) | function removeTags(html: string): string {
function sleep (line 75) | function sleep(ms: number): Promise<void> {
function withDeadline (line 79) | async function withDeadline<T>(
class Timer (line 90) | class Timer {
method constructor (line 93) | constructor() {
method elapsed (line 97) | elapsed(): number {
method elapsedSeconds (line 101) | elapsedSeconds(): number {
method reset (line 105) | reset(): void {
type RetryOptions (line 112) | interface RetryOptions {
constant DEFAULT_RETRY (line 119) | const DEFAULT_RETRY: RetryOptions = {
function withRetry (line 126) | async function withRetry<T>(
function groupBy (line 151) | function groupBy<T, K extends string | number>(
function dedent (line 165) | function dedent(str: string): string {
function matchUrlWithDomainPattern (line 187) | function matchUrlWithDomainPattern(url: string, pattern: string): boolean {
constant NEW_TAB_URLS (line 209) | const NEW_TAB_URLS = new Set([
function isNewTabPage (line 218) | function isNewTabPage(url: string): boolean {
function sanitizeSurrogates (line 225) | function sanitizeSurrogates(text: string): string {
constant URL_REGEX (line 232) | const URL_REGEX = /https?:\/\/[^\s<>"{}|\\^`\[\]]+/g;
function extractUrls (line 237) | function extractUrls(text: string): string[] {
function escapeRegExp (line 244) | function escapeRegExp(string: string): string {
FILE: packages/core/src/viewport/event-hub.ts
type Handler (line 1) | type Handler<T = unknown> = (payload: T) => void;
type RequestHandler (line 2) | type RequestHandler<Req = unknown, Res = unknown> = (payload: Req) => Pr...
class EventHub (line 4) | class EventHub<
method constructor (line 16) | constructor(options?: { maxHistory?: number }) {
method on (line 20) | on<K extends keyof EventMap & string>(event: K, handler: Handler<Event...
method once (line 31) | once<K extends keyof EventMap & string>(event: K, handler: Handler<Eve...
method emit (line 40) | emit<K extends keyof EventMap & string>(event: K, payload: EventMap[K]...
method onRequest (line 54) | onRequest<K extends keyof RequestMap & string>(
method request (line 64) | async request<K extends keyof RequestMap & string>(
method off (line 84) | off<K extends keyof EventMap & string>(event: K, handler?: Handler<Eve...
method removeAllListeners (line 92) | removeAllListeners(): void {
method getHistory (line 97) | getHistory(event?: string): Array<{ event: string; payload: unknown; t...
method clearHistory (line 104) | clearHistory(): void {
method recordHistory (line 108) | private recordHistory(event: string, payload: unknown): void {
FILE: packages/core/src/viewport/events.ts
type NavigateEvent (line 5) | interface NavigateEvent {
type ClickEvent (line 10) | interface ClickEvent {
type InputEvent (line 15) | interface InputEvent {
type SelectOptionEvent (line 21) | interface SelectOptionEvent {
type ScrollEvent (line 26) | interface ScrollEvent {
type ScreenshotEvent (line 32) | interface ScreenshotEvent {
type ScreenshotResult (line 36) | interface ScreenshotResult {
type TabSwitchEvent (line 42) | interface TabSwitchEvent {
type FileUploadEvent (line 46) | interface FileUploadEvent {
type KeyPressEvent (line 51) | interface KeyPressEvent {
type BrowserStateEvent (line 55) | interface BrowserStateEvent {
type DownloadEvent (line 61) | interface DownloadEvent {
type PopupEvent (line 67) | interface PopupEvent {
type SecurityEvent (line 72) | interface SecurityEvent {
type CrashEvent (line 78) | interface CrashEvent {
type ViewportEventMap (line 84) | interface ViewportEventMap {
type ViewportRequestMap (line 109) | interface ViewportRequestMap {
FILE: packages/core/src/viewport/guard-base.ts
type GuardContext (line 5) | interface GuardContext {
method active (line 20) | get active(): boolean {
method attach (line 27) | async attach(ctx: GuardContext): Promise<void> {
method detach (line 33) | async detach(): Promise<void> {
method teardown (line 48) | protected async teardown(): Promise<void> {
method onEvent (line 52) | protected onEvent<K extends keyof ViewportEventMap & string>(
FILE: packages/core/src/viewport/guards/blank-page.ts
class BlankPageGuard (line 7) | class BlankPageGuard extends BaseGuard {
method setup (line 11) | protected async setup(): Promise<void> {
FILE: packages/core/src/viewport/guards/crash.ts
class CrashGuard (line 7) | class CrashGuard extends BaseGuard {
method setup (line 11) | protected async setup(): Promise<void> {
FILE: packages/core/src/viewport/guards/default-handler.ts
class DefaultHandlerGuard (line 8) | class DefaultHandlerGuard extends BaseGuard {
method setup (line 12) | protected async setup(): Promise<void> {
FILE: packages/core/src/viewport/guards/downloads.ts
type DownloadGuardOptions (line 12) | interface DownloadGuardOptions {
type DownloadStatus (line 26) | type DownloadStatus = 'started' | 'completed' | 'failed';
type DownloadInfo (line 28) | interface DownloadInfo {
class DownloadGuard (line 49) | class DownloadGuard extends BaseGuard {
method constructor (line 68) | constructor(options?: DownloadGuardOptions) {
method setup (line 86) | protected async setup(): Promise<void> {
method teardown (line 107) | protected async teardown(): Promise<void> {
method configureCdpDownloadBehavior (line 119) | private async configureCdpDownloadBehavior(): Promise<void> {
method handleDownload (line 144) | private async handleDownload(download: Download): Promise<void> {
method resolveUniquePath (line 214) | private resolveUniquePath(suggestedFilename: string): string {
method ensureDownloadsDir (line 232) | private ensureDownloadsDir(): void {
method getDownloadHistory (line 244) | getDownloadHistory(): DownloadInfo[] {
method waitForDownload (line 254) | waitForDownload(timeout = 30_000): Promise<DownloadInfo> {
method notifyWaiters (line 268) | private notifyWaiters(info: DownloadInfo): void {
method removePendingWaiter (line 276) | private removePendingWaiter(waiter: (typeof this.pendingWaiters)[numbe...
function formatBytes (line 286) | function formatBytes(bytes: number | undefined): string {
FILE: packages/core/src/viewport/guards/har-capture.ts
type HarRequest (line 8) | interface HarRequest {
type HarResponse (line 18) | interface HarResponse {
type HarEntry (line 32) | interface HarEntry {
type PendingRequest (line 45) | interface PendingRequest {
type ResponseInfo (line 53) | interface ResponseInfo {
class HarCaptureGuard (line 65) | class HarCaptureGuard extends BaseGuard {
method constructor (line 75) | constructor(outputPath: string) {
method setup (line 80) | protected async setup(): Promise<void> {
method finalizeEntry (line 149) | private finalizeEntry(requestId: string, endTimestamp: number, encoded...
method teardown (line 208) | protected override async teardown(): Promise<void> {
function toHeaderArray (line 227) | function toHeaderArray(headers: Record<string, string>): Array<{ name: s...
function parseQueryString (line 231) | function parseQueryString(url: string): Array<{ name: string; value: str...
FILE: packages/core/src/viewport/guards/local-instance.ts
class LocalInstanceGuard (line 7) | class LocalInstanceGuard extends BaseGuard {
method setup (line 11) | protected async setup(): Promise<void> {
FILE: packages/core/src/viewport/guards/page-ready.ts
type PageReadyGuardOptions (line 8) | interface PageReadyGuardOptions {
type LoadState (line 24) | type LoadState = 'domcontentloaded' | 'load' | 'networkidle';
class PageReadyGuard (line 40) | class PageReadyGuard extends BaseGuard {
method constructor (line 72) | constructor(options?: PageReadyGuardOptions) {
method setup (line 80) | protected async setup(): Promise<void> {
method teardown (line 98) | protected async teardown(): Promise<void> {
method setupLifecycleListeners (line 115) | private setupLifecycleListeners(): void {
method setupMutationObserver (line 156) | private async setupMutationObserver(): Promise<void> {
method onMutationBatch (line 219) | private onMutationBatch(count: number): void {
method resetIdleTimer (line 236) | private resetIdleTimer(): void {
method markStable (line 247) | private markStable(): void {
method waitForDomStable (line 270) | waitForDomStable(timeout = 10_000): Promise<void> {
method getReachedStates (line 289) | getReachedStates(): ReadonlySet<LoadState> {
method getMutationCount (line 297) | getMutationCount(): number {
method isStable (line 304) | isStable(): boolean {
method notifyStableWaiters (line 310) | private notifyStableWaiters(): void {
method removeStableWaiter (line 318) | private removeStableWaiter(waiter: (typeof this.stableWaiters)[number]...
method clearTimers (line 327) | private clearTimers(): void {
FILE: packages/core/src/viewport/guards/permissions.ts
class PermissionsGuard (line 8) | class PermissionsGuard extends BaseGuard {
method constructor (line 16) | constructor(permissions: string[]) {
method setup (line 21) | protected async setup(): Promise<void> {
method grantForCurrentPage (line 43) | private async grantForCurrentPage(): Promise<void> {
FILE: packages/core/src/viewport/guards/persistence.ts
class PersistenceGuard (line 9) | class PersistenceGuard extends BaseGuard {
method constructor (line 15) | constructor(storagePath: string) {
method setup (line 20) | protected async setup(): Promise<void> {
method save (line 47) | async save(): Promise<void> {
FILE: packages/core/src/viewport/guards/popups.ts
class PopupGuard (line 8) | class PopupGuard extends BaseGuard {
method setup (line 12) | protected async setup(): Promise<void> {
FILE: packages/core/src/viewport/guards/screenshot.ts
class ScreenshotGuard (line 8) | class ScreenshotGuard extends BaseGuard {
method setup (line 12) | protected async setup(): Promise<void> {
FILE: packages/core/src/viewport/guards/url-policy.ts
class UrlPolicyGuard (line 9) | class UrlPolicyGuard extends BaseGuard {
method constructor (line 16) | constructor(allowedUrls: string[] = [], blockedUrls: string[] = []) {
method setup (line 22) | protected async setup(): Promise<void> {
FILE: packages/core/src/viewport/guards/video-capture.ts
type VideoRecordingOptions (line 11) | interface VideoRecordingOptions {
type ResolvedOptions (line 56) | interface ResolvedOptions {
function resolveOptions (line 66) | function resolveOptions(opts: VideoRecordingOptions): ResolvedOptions {
class VideoCaptureGuard (line 93) | class VideoCaptureGuard extends BaseGuard {
method constructor (line 109) | constructor(options: VideoRecordingOptions) {
method setup (line 116) | protected async setup(): Promise<void> {
method teardown (line 134) | protected override async teardown(): Promise<void> {
method pause (line 148) | pause(): void {
method resume (line 157) | resume(): void {
method isPaused (line 164) | get isPaused(): boolean {
method capturedFrameCount (line 169) | get capturedFrameCount(): number {
method startTracing (line 175) | private async startTracing(): Promise<boolean> {
method stopTracing (line 191) | private async stopTracing(): Promise<void> {
method startScreencast (line 207) | private async startScreencast(): Promise<void> {
method stopScreencast (line 259) | private async stopScreencast(): Promise<void> {
method saveFrames (line 274) | private async saveFrames(): Promise<void> {
FILE: packages/core/src/viewport/launch-profile.ts
constant CHROME_AUTOMATION_FLAGS (line 8) | const CHROME_AUTOMATION_FLAGS = [
constant CHROME_STRIPPED_FEATURES (line 40) | const CHROME_STRIPPED_FEATURES = [
constant ANTI_DETECTION_FLAGS (line 67) | const ANTI_DETECTION_FLAGS = [
constant CONTAINER_FLAGS (line 72) | const CONTAINER_FLAGS = [
constant RELAXED_SECURITY_FLAGS (line 80) | const RELAXED_SECURITY_FLAGS = [
constant REPRODUCIBLE_RENDER_FLAGS (line 86) | const REPRODUCIBLE_RENDER_FLAGS = [
class LaunchProfile (line 98) | class LaunchProfile {
method create (line 107) | static create(): LaunchProfile {
method headless (line 111) | headless(value = true): this {
method relaxedSecurity (line 116) | relaxedSecurity(value = true): this {
method windowSize (line 121) | windowSize(width: number, height: number): this {
method proxy (line 127) | proxy(server: string, username?: string, password?: string): this {
method userDataDir (line 132) | userDataDir(dir: string): this {
method browserBinary (line 137) | browserBinary(path: string): this {
method persistAfterClose (line 142) | persistAfterClose(value = true): this {
method channel (line 147) | channel(name: string): this {
method extraArgs (line 152) | extraArgs(...args: string[]): this {
method stealthMode (line 157) | stealthMode(value = true): this {
method dockerMode (line 162) | dockerMode(value = true): this {
method deterministicRendering (line 167) | deterministicRendering(value = true): this {
method downloadsPath (line 172) | downloadsPath(path: string): this {
method maxIframes (line 177) | maxIframes(max: number): this {
method addExtension (line 182) | addExtension(extensionPath: string): this {
method autoDetect (line 190) | autoDetect(): this {
method build (line 201) | build(): LaunchOptions {
FILE: packages/core/src/viewport/types.ts
type TabDescriptor (line 4) | interface TabDescriptor {
type ViewportSnapshot (line 11) | interface ViewportSnapshot {
type ViewportHistory (line 23) | interface ViewportHistory {
type LaunchOptions (line 54) | type LaunchOptions = z.infer<typeof LaunchOptionsSchema>;
type PageState (line 56) | interface PageState {
FILE: packages/core/src/viewport/viewport.ts
type Target (line 36) | interface Target {
type ViewportInfo (line 44) | interface ViewportInfo {
type ViewportOptions (line 51) | interface ViewportOptions {
class Viewport (line 82) | class Viewport {
method constructor (line 109) | constructor(options: ViewportOptions = {}) {
method isConnected (line 137) | get isConnected(): boolean {
method currentPage (line 141) | get currentPage(): Page {
method browserContext (line 148) | get browserContext(): BrowserContext {
method cdp (line 155) | get cdp(): CDPSession | null {
method start (line 161) | async start(): Promise<void> {
method setupDisconnectHandler (line 241) | private setupDisconnectHandler(): void {
method setupPageLifecycleListeners (line 251) | private setupPageLifecycleListeners(): void {
method getTargets (line 290) | async getTargets(): Promise<Target[]> {
method refreshTargets (line 295) | private async refreshTargets(): Promise<void> {
method findTarget (line 327) | findTarget(id: TargetId): Target | undefined {
method getPageTargets (line 334) | async getPageTargets(): Promise<Target[]> {
method detectViewport (line 346) | async detectViewport(): Promise<ViewportInfo> {
method invalidateViewportCache (line 401) | invalidateViewportCache(): void {
method reconnect (line 414) | async reconnect(): Promise<boolean> {
method cleanupForReconnect (line 511) | private async cleanupForReconnect(): Promise<void> {
method waitForStableDOM (line 558) | async waitForStableDOM(timeout = 3000, quietPeriodMs = 300): Promise<v...
method getVisibleHtml (line 621) | async getVisibleHtml(): Promise<string> {
method launchBrowser (line 687) | private async launchBrowser(): Promise<Browser> {
method buildChromiumArgs (line 707) | private buildChromiumArgs(): string[] {
method createContext (line 724) | private async createContext(): Promise<BrowserContext> {
method initializeWatchdogs (line 739) | private async initializeWatchdogs(): Promise<void> {
method navigate (line 777) | async navigate(url: string): Promise<void> {
method waitForPageReady (line 810) | async waitForPageReady(): Promise<void> {
method click (line 826) | async click(selector: string): Promise<void> {
method type (line 830) | async type(selector: string, text: string): Promise<void> {
method pressKey (line 834) | async pressKey(key: string): Promise<void> {
method screenshot (line 838) | async screenshot(fullPage = false): Promise<{ base64: string; width: n...
method getState (line 854) | async getState(): Promise<ViewportSnapshot> {
method switchTab (line 885) | async switchTab(tabIndex: number): Promise<void> {
method closeTab (line 906) | async closeTab(tabIndex?: number): Promise<void> {
method newTab (line 935) | async newTab(url?: string): Promise<void> {
method evaluate (line 949) | async evaluate<T>(expression: string): Promise<T> {
method setPage (line 953) | async setPage(page: Page): Promise<void> {
method close (line 961) | async close(): Promise<void> {
method [Symbol.asyncDispose] (line 1003) | async [Symbol.asyncDispose](): Promise<void> {
function normalizeTargetType (line 1013) | function normalizeTargetType(
FILE: packages/core/src/viewport/visual-tracer.ts
type VisualTracerOptions (line 3) | interface VisualTracerOptions {
constant DEFAULT_OPTIONS (line 12) | const DEFAULT_OPTIONS: Required<VisualTracerOptions> = {
constant OVERLAY_ATTR (line 27) | const OVERLAY_ATTR = 'data-demo-mode-overlay';
class VisualTracer (line 29) | class VisualTracer {
method constructor (line 32) | constructor(options?: VisualTracerOptions) {
method highlightElement (line 44) | async highlightElement(page: Page, selector: string, label?: string): ...
method showAction (line 101) | async showAction(page: Page, action: string, details?: string): Promis...
method highlightClick (line 140) | async highlightClick(page: Page, x: number, y: number, label?: string)...
method highlightScroll (line 240) | async highlightScroll(page: Page, direction: 'up' | 'down'): Promise<v...
method highlightType (line 324) | async highlightType(page: Page, selector: string, text: string): Promi...
method highlightNavigation (line 432) | async highlightNavigation(page: Page, url: string): Promise<void> {
method showElementSequence (line 551) | async showElementSequence(
method showTimeline (line 704) | async showTimeline(
method showCoordinates (line 857) | async showCoordinates(page: Page, x: number, y: number): Promise<void> {
method clearOverlays (line 970) | async clearOverlays(page: Page): Promise<void> {
FILE: packages/sandbox/src/sandbox.ts
constant DEFAULT_OPTIONS (line 14) | const DEFAULT_OPTIONS: Required<SandboxOptions> = {
class ResourceMonitor (line 33) | class ResourceMonitor {
method constructor (line 41) | constructor(limitMB: number) {
method start (line 45) | start(intervalMs: number, onOOM: () => void): void {
method stop (line 55) | stop(): void {
method takeSnapshot (line 64) | private takeSnapshot(): void {
method getPeakMemoryMB (line 97) | getPeakMemoryMB(): number {
method getCpuTimeMs (line 101) | getCpuTimeMs(): number {
method getSnapshots (line 107) | getSnapshots(): ResourceSnapshot[] {
method getCurrentMemoryMB (line 111) | getCurrentMemoryMB(): number {
class OutputCapture (line 123) | class OutputCapture {
method start (line 130) | start(): void {
method stop (line 153) | stop(): void {
method getOutput (line 167) | getOutput(): CapturedOutput {
class Sandbox (line 182) | class Sandbox {
method constructor (line 185) | constructor(options?: SandboxOptions) {
method run (line 193) | async run(agentOptions: Omit<AgentOptions, 'browser'>): Promise<Sandbo...
method executeAgent (line 325) | private async executeAgent(
method createTimeoutPromise (line 340) | private createTimeoutPromise(startTime: number): Promise<SandboxResult> {
method createOOMPromise (line 359) | private createOOMPromise(
method classifyError (line 389) | private classifyError(error: unknown, oomTriggered: boolean): SandboxE...
method buildMetrics (line 444) | private buildMetrics(
method forceCleanup (line 466) | private async forceCleanup(browser: Viewport): Promise<void> {
method getOptions (line 481) | getOptions(): Readonly<Required<SandboxOptions>> {
FILE: packages/sandbox/src/types.ts
type SandboxOptions (line 3) | interface SandboxOptions {
type SandboxErrorCategory (line 28) | type SandboxErrorCategory =
type SandboxError (line 36) | interface SandboxError {
type CapturedOutput (line 45) | interface CapturedOutput {
type SandboxMetrics (line 52) | interface SandboxMetrics {
type SandboxResult (line 71) | interface SandboxResult {
type ResourceSnapshot (line 87) | interface ResourceSnapshot {
Condensed preview — 119 files, each showing path, character count, and a content snippet. Download the .json file or copy for the full structured content (798K chars).
[
{
"path": ".github/CONTRIBUTING.md",
"chars": 668,
"preview": "# Contributing to Open Browser\n\nThank you for your interest in contributing!\n\n## Getting Started\n\n1. Fork the repository"
},
{
"path": ".github/workflows/ci.yml",
"chars": 306,
"preview": "name: CI\n\non:\n push:\n branches: [main]\n pull_request:\n branches: [main]\n\njobs:\n test:\n runs-on: ubuntu-lates"
},
{
"path": ".gitignore",
"chars": 90,
"preview": "node_modules/\ndist/\n.env\n*.tsbuildinfo\n.DS_Store\ntraces/\ncoverage/\nrecordings/\ntmp/\n*.log\n"
},
{
"path": "LICENSE",
"chars": 1087,
"preview": "MIT License\n\nCopyright (c) 2024-2026 Open Browser Contributors\n\nPermission is hereby granted, free of charge, to any per"
},
{
"path": "README.md",
"chars": 9862,
"preview": "<h1 align=\"center\">Open Browser</h1>\n\n<p align=\"center\">\n <b>AI-powered autonomous web browsing framework for TypeScrip"
},
{
"path": "biome.json",
"chars": 737,
"preview": "{\n \"$schema\": \"https://biomejs.dev/schemas/1.9.0/schema.json\",\n \"organizeImports\": {\n \"enabled\": true\n },\n \"linte"
},
{
"path": "bunfig.toml",
"chars": 47,
"preview": "[install]\npeer = false\n\n[test]\ntimeout = 60000\n"
},
{
"path": "package.json",
"chars": 429,
"preview": "{\n \"name\": \"open-browser-monorepo\",\n \"private\": true,\n \"workspaces\": [\"packages/*\"],\n \"scripts\": {\n \"build\": \"bun"
},
{
"path": "packages/cli/package.json",
"chars": 470,
"preview": "{\n \"name\": \"@open-browser/cli\",\n \"version\": \"1.1.0\",\n \"description\": \"CLI for Open Browser - AI-powered autonomous we"
},
{
"path": "packages/cli/src/commands/click.ts",
"chars": 971,
"preview": "import type { Command } from 'commander';\nimport chalk from 'chalk';\nimport { sessionManager } from '../globals.js';\n\nex"
},
{
"path": "packages/cli/src/commands/eval.ts",
"chars": 1231,
"preview": "import type { Command } from 'commander';\nimport chalk from 'chalk';\nimport { sessionManager } from '../globals.js';\n\nex"
},
{
"path": "packages/cli/src/commands/extract.ts",
"chars": 1197,
"preview": "import type { Command } from 'commander';\nimport chalk from 'chalk';\nimport { extractMarkdown } from 'open-browser';\nimp"
},
{
"path": "packages/cli/src/commands/interactive.ts",
"chars": 8427,
"preview": "import * as readline from 'node:readline';\nimport type { Command } from 'commander';\nimport chalk from 'chalk';\nimport {"
},
{
"path": "packages/cli/src/commands/open.ts",
"chars": 1478,
"preview": "import type { Command } from 'commander';\nimport chalk from 'chalk';\nimport { sessionManager } from '../globals.js';\n\nex"
},
{
"path": "packages/cli/src/commands/run.ts",
"chars": 6161,
"preview": "import type { Command } from 'commander';\nimport chalk from 'chalk';\nimport {\n\tAgent,\n\tViewport,\n\tVercelModelAdapter,\n\tt"
},
{
"path": "packages/cli/src/commands/screenshot.ts",
"chars": 1379,
"preview": "import type { Command } from 'commander';\nimport chalk from 'chalk';\nimport * as fs from 'node:fs';\nimport * as path fro"
},
{
"path": "packages/cli/src/commands/sessions.ts",
"chars": 1754,
"preview": "import type { Command } from 'commander';\nimport chalk from 'chalk';\nimport { sessionManager } from '../globals.js';\n\nex"
},
{
"path": "packages/cli/src/commands/state.ts",
"chars": 1245,
"preview": "import type { Command } from 'commander';\nimport chalk from 'chalk';\nimport { sessionManager } from '../globals.js';\n\nex"
},
{
"path": "packages/cli/src/commands/type.ts",
"chars": 1048,
"preview": "import type { Command } from 'commander';\nimport chalk from 'chalk';\nimport { sessionManager } from '../globals.js';\n\nex"
},
{
"path": "packages/cli/src/display.ts",
"chars": 4616,
"preview": "import chalk from 'chalk';\n\n// ── Spinner ──\n\nconst SPINNER_FRAMES = ['⠋', '⠙', '⠹', '⠸', '⠼', '⠴', '⠦', '⠧', '⠇', '⠏'];"
},
{
"path": "packages/cli/src/globals.ts",
"chars": 101,
"preview": "import { SessionManager } from './sessions.js';\n\nexport const sessionManager = new SessionManager();\n"
},
{
"path": "packages/cli/src/index.ts",
"chars": 1238,
"preview": "#!/usr/bin/env bun\nimport { Command } from 'commander';\nimport { registerOpenCommand } from './commands/open.js';\nimport"
},
{
"path": "packages/cli/src/protocol.ts",
"chars": 700,
"preview": "export interface CLIRequest {\n\tid: string;\n\tcommand: string;\n\targs: Record<string, unknown>;\n}\n\nexport interface CLIResp"
},
{
"path": "packages/cli/src/server.ts",
"chars": 4692,
"preview": "import * as net from 'node:net';\nimport * as fs from 'node:fs';\nimport * as path from 'node:path';\nimport * as os from '"
},
{
"path": "packages/cli/src/sessions.ts",
"chars": 1721,
"preview": "import { Viewport, type ViewportOptions } from 'open-browser';\nimport { nanoid } from 'nanoid';\n\ninterface ManagedSessio"
},
{
"path": "packages/cli/tsconfig.json",
"chars": 145,
"preview": "{\n \"extends\": \"../../tsconfig.base.json\",\n \"compilerOptions\": {\n \"rootDir\": \"src\",\n \"outDir\": \"dist\"\n },\n \"inc"
},
{
"path": "packages/core/package.json",
"chars": 853,
"preview": "{\n \"name\": \"open-browser\",\n \"version\": \"1.1.0\",\n \"description\": \"AI-powered autonomous web browsing library for TypeS"
},
{
"path": "packages/core/src/agent/agent.test.ts",
"chars": 26304,
"preview": "import { test, expect, describe, beforeEach, mock } from 'bun:test';\nimport { Agent, type AgentOptions } from '../agent/"
},
{
"path": "packages/core/src/agent/agent.ts",
"chars": 29012,
"preview": "import { z, ZodError } from 'zod';\nimport type { LanguageModel, InferenceOptions } from '../model/interface.js';\nimport "
},
{
"path": "packages/core/src/agent/conversation/service.ts",
"chars": 16634,
"preview": "import { z } from 'zod';\nimport type { Message } from '../../model/messages.js';\nimport {\n\tsystemMessage,\n\tuserMessage,\n"
},
{
"path": "packages/core/src/agent/conversation/types.ts",
"chars": 2653,
"preview": "import type { Message } from '../../model/messages.js';\nimport type { CompactionPolicy } from '../types.js';\nimport type"
},
{
"path": "packages/core/src/agent/conversation/utils.ts",
"chars": 3307,
"preview": "import type { Message } from '../../model/messages.js';\nimport type { ContentPart } from '../../model/messages.js';\n\n/**"
},
{
"path": "packages/core/src/agent/conversation.test.ts",
"chars": 20619,
"preview": "import { test, expect, describe, beforeEach } from 'bun:test';\nimport { ConversationManager } from './conversation/servi"
},
{
"path": "packages/core/src/agent/evaluator.ts",
"chars": 7287,
"preview": "import type { LanguageModel } from '../model/interface.js';\nimport type { Message, ContentPart } from '../model/messages"
},
{
"path": "packages/core/src/agent/index.ts",
"chars": 1923,
"preview": "export { Agent, type AgentOptions } from '../agent/agent.js';\nexport {\n\tInstructionBuilder,\n\tStepPromptBuilder,\n\tbuildCo"
},
{
"path": "packages/core/src/agent/instructions/instructions-compact.md",
"chars": 3219,
"preview": "You are an AI agent that controls a web browser to complete tasks. You operate in an iterative loop: observe, decide, ac"
},
{
"path": "packages/core/src/agent/instructions/instructions-direct.md",
"chars": 7250,
"preview": "You are an AI agent that controls a web browser to complete tasks. You operate in an iterative loop: observe the current"
},
{
"path": "packages/core/src/agent/instructions/instructions.md",
"chars": 10465,
"preview": "You are an AI agent that controls a web browser to complete tasks. You operate in an iterative loop: observe the current"
},
{
"path": "packages/core/src/agent/instructions.ts",
"chars": 17411,
"preview": "import { readFileSync } from 'node:fs';\nimport { resolve, dirname } from 'node:path';\nimport { fileURLToPath } from 'nod"
},
{
"path": "packages/core/src/agent/replay-recorder.ts",
"chars": 8369,
"preview": "import * as fs from 'node:fs';\nimport * as path from 'node:path';\nimport { createLogger } from '../logging.js';\n\nconst l"
},
{
"path": "packages/core/src/agent/stall-detector.test.ts",
"chars": 14831,
"preview": "import { test, expect, describe, beforeEach } from 'bun:test';\nimport {\n\tStallDetector,\n\thashPageTree,\n\thashTextContent,"
},
{
"path": "packages/core/src/agent/stall-detector.ts",
"chars": 10517,
"preview": "import type { Command } from '../commands/types.js';\n\n// ── Enhanced Page Fingerprint ──\n\nexport interface PageSignature"
},
{
"path": "packages/core/src/agent/types.ts",
"chars": 16350,
"preview": "import { z } from 'zod';\nimport type { Command, CommandResult } from '../commands/types.js';\nimport type { ViewportSnaps"
},
{
"path": "packages/core/src/bridge/adapter.ts",
"chars": 2639,
"preview": "import { z, type ZodTypeAny } from 'zod';\nimport type { CommandExecutor } from '../commands/executor.js';\n\nexport interf"
},
{
"path": "packages/core/src/bridge/client.ts",
"chars": 13200,
"preview": "import { type ChildProcess, spawn } from 'node:child_process';\nimport { EventEmitter } from 'node:events';\nimport type {"
},
{
"path": "packages/core/src/bridge/index.ts",
"chars": 186,
"preview": "export { BridgeServer, type BridgeServerOptions } from './server.js';\nexport { BridgeClient, type BridgeClientOptions } "
},
{
"path": "packages/core/src/bridge/mcp-types.ts",
"chars": 448,
"preview": "/**\n * Experimental MCP (Model Context Protocol) server types.\n * @experimental\n */\n\nexport interface MCPServerOptions {"
},
{
"path": "packages/core/src/bridge/server.test.ts",
"chars": 13149,
"preview": "import { test, expect, describe, beforeEach, mock } from 'bun:test';\nimport { BridgeServer, type MCPRequest, type MCPRes"
},
{
"path": "packages/core/src/bridge/server.ts",
"chars": 17151,
"preview": "import type { IncomingMessage, ServerResponse } from 'node:http';\nimport type { Viewport } from '../viewport/viewport.js"
},
{
"path": "packages/core/src/commands/catalog/catalog.ts",
"chars": 10836,
"preview": "import { z, type ZodTypeAny } from 'zod';\nimport type { CatalogEntry, CatalogOptions } from './types.js';\nimport type { "
},
{
"path": "packages/core/src/commands/catalog/types.ts",
"chars": 441,
"preview": "import type { z } from 'zod';\nimport type { CommandResult, ExecutionContext } from '../types.js';\n\nexport interface Cata"
},
{
"path": "packages/core/src/commands/catalog.test.ts",
"chars": 14127,
"preview": "import { test, expect, describe, beforeEach, mock } from 'bun:test';\nimport { z } from 'zod';\nimport { CommandCatalog } "
},
{
"path": "packages/core/src/commands/executor.test.ts",
"chars": 19877,
"preview": "import { test, expect, describe, beforeEach, mock } from 'bun:test';\nimport { CommandExecutor } from './executor.js';\nim"
},
{
"path": "packages/core/src/commands/executor.ts",
"chars": 31249,
"preview": "import type { Page, CDPSession } from 'playwright';\nimport { z } from 'zod';\nimport { CommandCatalog } from './catalog/c"
},
{
"path": "packages/core/src/commands/extraction/extractor.ts",
"chars": 6241,
"preview": "import type { Page } from 'playwright';\nimport type { LanguageModel } from '../../model/interface.js';\nimport { z } from"
},
{
"path": "packages/core/src/commands/index.ts",
"chars": 1043,
"preview": "export { CommandExecutor, type CommandExecutorOptions, classifyViewportError } from './executor.js';\nexport { CommandCat"
},
{
"path": "packages/core/src/commands/types.ts",
"chars": 6660,
"preview": "import { z } from 'zod';\n\n// ── Individual action schemas ──\n\nexport const TapCommandSchema = z.object({\n\taction: z.lite"
},
{
"path": "packages/core/src/commands/utils.ts",
"chars": 1020,
"preview": "import type { Page } from 'playwright';\n\nexport async function scrollPage(\n\tpage: Page,\n\tdirection: 'up' | 'down',\n\tamou"
},
{
"path": "packages/core/src/config/config.ts",
"chars": 4536,
"preview": "import { config as loadDotenv } from 'dotenv';\nimport * as path from 'node:path';\nimport * as os from 'node:os';\nimport "
},
{
"path": "packages/core/src/config/index.ts",
"chars": 193,
"preview": "export { Config } from './config.js';\nexport {\n\ttype ViewportConfig,\n\tViewportConfigSchema,\n\ttype AgentConfig,\n\tAgentCon"
},
{
"path": "packages/core/src/config/types.ts",
"chars": 2953,
"preview": "import { z } from 'zod';\n\nexport const ProxyConfigSchema = z.object({\n\tserver: z.string(),\n\tusername: z.string().optiona"
},
{
"path": "packages/core/src/errors.ts",
"chars": 4326,
"preview": "export class OpenBrowserError extends Error {\n\tconstructor(message: string, options?: ErrorOptions) {\n\t\tsuper(message, o"
},
{
"path": "packages/core/src/index.ts",
"chars": 6097,
"preview": "// ── Core types ──\nexport {\n\ttype TargetId,\n\ttype SessionId,\n\ttype ElementRef,\n\ttype TabId,\n\ttargetId,\n\tsessionId,\n\tele"
},
{
"path": "packages/core/src/logging.ts",
"chars": 3194,
"preview": "import { LogLevel } from './types.js';\n\nconst LEVEL_NAMES: Record<number, string> = {\n\t[LogLevel.DEBUG]: 'DEBUG',\n\t[LogL"
},
{
"path": "packages/core/src/metering/index.ts",
"chars": 369,
"preview": "export { UsageMeter, CompositeUsageMeter, BudgetDepletedError, estimateTokenCount } from './tracker.js';\nexport {\n\tDEFAU"
},
{
"path": "packages/core/src/metering/tracker.test.ts",
"chars": 15884,
"preview": "import { test, expect, describe, beforeEach, mock } from 'bun:test';\nimport {\n\tUsageMeter,\n\tCompositeUsageMeter,\n\tBudget"
},
{
"path": "packages/core/src/metering/tracker.ts",
"chars": 11590,
"preview": "import type {\n\tUsageRecord,\n\tCostRates,\n\tPricingTable,\n\tModelRole,\n\tActionUsageRecord,\n\tMeteringSummary,\n\tModelUsageBrea"
},
{
"path": "packages/core/src/metering/types.ts",
"chars": 4762,
"preview": "export interface UsageRecord {\n\tinputTokens: number;\n\toutputTokens: number;\n\ttotalTokens: number;\n}\n\nexport interface Co"
},
{
"path": "packages/core/src/model/adapters/vercel.ts",
"chars": 5082,
"preview": "import { generateObject, type CoreMessage, type CoreUserMessage } from 'ai';\nimport type { LanguageModelV1 } from 'ai';\n"
},
{
"path": "packages/core/src/model/index.ts",
"chars": 714,
"preview": "export { type LanguageModel, type InferenceOptions, type ModelProvider } from './interface.js';\nexport { type InferenceR"
},
{
"path": "packages/core/src/model/interface.ts",
"chars": 1416,
"preview": "import type { ZodType } from 'zod';\nimport type { Message } from './messages.js';\nimport type { InferenceResult } from '"
},
{
"path": "packages/core/src/model/messages.ts",
"chars": 1630,
"preview": "export interface TextContent {\n\ttype: 'text';\n\ttext: string;\n}\n\nexport interface ImageContent {\n\ttype: 'image';\n\tsource:"
},
{
"path": "packages/core/src/model/schema-optimizer.ts",
"chars": 13973,
"preview": "import { z, type ZodTypeAny } from 'zod';\nimport type { ModelProvider } from './interface.js';\n\n// ── Configuration ──\n\n"
},
{
"path": "packages/core/src/model/types.ts",
"chars": 454,
"preview": "import { z } from 'zod';\n\nexport interface InferenceUsage {\n\tinputTokens: number;\n\toutputTokens: number;\n\ttotalTokens: n"
},
{
"path": "packages/core/src/page/content-extractor.ts",
"chars": 10346,
"preview": "import TurndownService from 'turndown';\nimport type { Page } from 'playwright';\n\nlet turndownInstance: TurndownService |"
},
{
"path": "packages/core/src/page/index.ts",
"chars": 629,
"preview": "export { PageAnalyzer, type PageAnalyzerOptions } from './page-analyzer.js';\nexport { SnapshotBuilder } from './snapshot"
},
{
"path": "packages/core/src/page/page-analyzer.test.ts",
"chars": 14840,
"preview": "import { test, expect, describe, beforeEach, mock } from 'bun:test';\nimport { PageAnalyzer } from './page-analyzer.js';\n"
},
{
"path": "packages/core/src/page/page-analyzer.ts",
"chars": 16421,
"preview": "import type { CDPSession, Page } from 'playwright';\nimport { SnapshotBuilder } from './snapshot-builder.js';\nimport { Tr"
},
{
"path": "packages/core/src/page/renderer/interactive-elements.ts",
"chars": 1245,
"preview": "import type { PageTreeNode } from '../types.js';\n\nconst ALWAYS_CLICKABLE_TAGS = new Set([\n\t'a', 'button', 'input', 'sele"
},
{
"path": "packages/core/src/page/renderer/layer-order.ts",
"chars": 1765,
"preview": "import type { PageTreeNode, DOMRect } from '../types.js';\n\n/**\n * Filter overlapping elements by paint order (z-index).\n"
},
{
"path": "packages/core/src/page/renderer/tree-renderer.ts",
"chars": 17111,
"preview": "import type { PageTreeNode, SelectorIndex, RenderedPageState } from '../types.js';\nimport type { ElementRef } from '../."
},
{
"path": "packages/core/src/page/renderer.test.ts",
"chars": 19653,
"preview": "import { test, expect, describe, beforeEach } from 'bun:test';\nimport { TreeRenderer } from './renderer/tree-renderer.js"
},
{
"path": "packages/core/src/page/snapshot-builder.ts",
"chars": 6811,
"preview": "import type { CDPSession } from 'playwright';\nimport type {\n\tCDPSnapshotResult,\n\tAXNode,\n\tPageTreeNode,\n\tDOMRect,\n} from"
},
{
"path": "packages/core/src/page/types.ts",
"chars": 4060,
"preview": "import type { ElementRef } from '../types.js';\n\nexport interface DOMRect {\n\tx: number;\n\ty: number;\n\twidth: number;\n\theig"
},
{
"path": "packages/core/src/sandbox/file-access.ts",
"chars": 6520,
"preview": "import * as fs from 'node:fs';\nimport * as path from 'node:path';\nimport { createLogger } from '../logging.js';\n\nconst l"
},
{
"path": "packages/core/src/sandbox/index.ts",
"chars": 108,
"preview": "export { FileAccess, type FileAccessOptions, type FileInfo, type FileAccessState } from './file-access.js';\n"
},
{
"path": "packages/core/src/telemetry.ts",
"chars": 1937,
"preview": "import { createLogger } from './logging.js';\n\nconst logger = createLogger('perf');\n\nexport interface TimingResult<T> {\n\t"
},
{
"path": "packages/core/src/types.ts",
"chars": 1650,
"preview": "import { z } from 'zod';\n\n// ── Branded types for compile-time safety ──\n\ndeclare const __brand: unique symbol;\ntype Bra"
},
{
"path": "packages/core/src/utils.ts",
"chars": 5897,
"preview": "import { nanoid } from 'nanoid';\n\n// ── ID generation ──\n\nexport function generateId(size = 12): string {\n\treturn nanoid"
},
{
"path": "packages/core/src/viewport/event-hub.ts",
"chars": 3315,
"preview": "type Handler<T = unknown> = (payload: T) => void;\ntype RequestHandler<Req = unknown, Res = unknown> = (payload: Req) => "
},
{
"path": "packages/core/src/viewport/events.ts",
"chars": 2197,
"preview": "import type { ElementRef } from '../types.js';\n\n// ── Event payload types ──\n\nexport interface NavigateEvent {\n\turl: str"
},
{
"path": "packages/core/src/viewport/guard-base.ts",
"chars": 1426,
"preview": "import type { Page, BrowserContext } from 'playwright';\nimport type { EventHub } from './event-hub.js';\nimport type { Vi"
},
{
"path": "packages/core/src/viewport/guards/blank-page.ts",
"chars": 636,
"preview": "import { BaseGuard } from '../guard-base.js';\n\n/**\n * Handles about:blank pages. If the page navigates to about:blank,\n "
},
{
"path": "packages/core/src/viewport/guards/crash.ts",
"chars": 745,
"preview": "import { BaseGuard } from '../guard-base.js';\n\n/**\n * Monitors for browser page crashes. Emits crash events\n * and attem"
},
{
"path": "packages/core/src/viewport/guards/default-handler.ts",
"chars": 734,
"preview": "import type { Dialog } from 'playwright';\nimport { BaseGuard } from '../guard-base.js';\n\n/**\n * Monitors for default bro"
},
{
"path": "packages/core/src/viewport/guards/downloads.ts",
"chars": 8455,
"preview": "import type { Download } from 'playwright';\nimport * as fs from 'node:fs';\nimport * as path from 'node:path';\nimport * a"
},
{
"path": "packages/core/src/viewport/guards/har-capture.ts",
"chars": 5881,
"preview": "import { writeFile, mkdir } from 'node:fs/promises';\nimport { dirname } from 'node:path';\nimport type { CDPSession } fro"
},
{
"path": "packages/core/src/viewport/guards/local-instance.ts",
"chars": 455,
"preview": "import { BaseGuard } from '../guard-base.js';\n\n/**\n * Ensures a local browser is connected by verifying\n * the page is a"
},
{
"path": "packages/core/src/viewport/guards/page-ready.ts",
"chars": 8954,
"preview": "import { BaseGuard } from '../guard-base.js';\nimport { createLogger } from '../../logging.js';\n\nconst logger = createLog"
},
{
"path": "packages/core/src/viewport/guards/permissions.ts",
"chars": 1932,
"preview": "import type { CDPSession } from 'playwright';\nimport { BaseGuard } from '../guard-base.js';\n\n/**\n * Grants browser permi"
},
{
"path": "packages/core/src/viewport/guards/persistence.ts",
"chars": 1476,
"preview": "import { readFile, writeFile, mkdir } from 'node:fs/promises';\nimport { dirname } from 'node:path';\nimport { BaseGuard }"
},
{
"path": "packages/core/src/viewport/guards/popups.ts",
"chars": 857,
"preview": "import type { Page } from 'playwright';\nimport { BaseGuard } from '../guard-base.js';\n\n/**\n * Monitors for popups and ne"
},
{
"path": "packages/core/src/viewport/guards/screenshot.ts",
"chars": 878,
"preview": "import type { ScreenshotEvent, ScreenshotResult } from '../events.js';\nimport { BaseGuard } from '../guard-base.js';\n\n/*"
},
{
"path": "packages/core/src/viewport/guards/url-policy.ts",
"chars": 1324,
"preview": "import type { Route } from 'playwright';\nimport { BaseGuard } from '../guard-base.js';\nimport { isUrlPermitted } from '."
},
{
"path": "packages/core/src/viewport/guards/video-capture.ts",
"chars": 8458,
"preview": "import { mkdir, writeFile } from 'node:fs/promises';\nimport { dirname, join } from 'node:path';\nimport type { CDPSession"
},
{
"path": "packages/core/src/viewport/index.ts",
"chars": 719,
"preview": "export { Viewport, type ViewportOptions } from './viewport.js';\nexport { LaunchProfile } from './launch-profile.js';\nexp"
},
{
"path": "packages/core/src/viewport/launch-profile.test.ts",
"chars": 11956,
"preview": "import { test, expect, describe } from 'bun:test';\nimport {\n\tLaunchProfile,\n\tCHROME_AUTOMATION_FLAGS,\n\tCHROME_STRIPPED_F"
},
{
"path": "packages/core/src/viewport/launch-profile.ts",
"chars": 6172,
"preview": "import type { LaunchOptions } from './types.js';\nimport { Config } from '../config/config.js';\n\n/**\n * Chrome default ar"
},
{
"path": "packages/core/src/viewport/types.ts",
"chars": 1350,
"preview": "import { z } from 'zod';\nimport type { TabId } from '../types.js';\n\nexport interface TabDescriptor {\n\ttabId: TabId;\n\turl"
},
{
"path": "packages/core/src/viewport/viewport.ts",
"chars": 28889,
"preview": "import {\n\tchromium,\n\ttype Browser,\n\ttype BrowserContext,\n\ttype Page,\n\ttype CDPSession,\n} from 'playwright';\nimport { Eve"
},
{
"path": "packages/core/src/viewport/visual-tracer.ts",
"chars": 27069,
"preview": "import type { Page } from 'playwright';\n\nexport interface VisualTracerOptions {\n\thighlightColor?: string;\n\thighlightDura"
},
{
"path": "packages/core/tsconfig.json",
"chars": 145,
"preview": "{\n \"extends\": \"../../tsconfig.base.json\",\n \"compilerOptions\": {\n \"rootDir\": \"src\",\n \"outDir\": \"dist\"\n },\n \"inc"
},
{
"path": "packages/sandbox/package.json",
"chars": 399,
"preview": "{\n \"name\": \"@open-browser/sandbox\",\n \"version\": \"1.1.0\",\n \"description\": \"Sandboxed execution environment for Open Br"
},
{
"path": "packages/sandbox/src/index.ts",
"chars": 199,
"preview": "export { Sandbox } from './sandbox.js';\nexport type {\n\tSandboxOptions,\n\tSandboxResult,\n\tSandboxError,\n\tSandboxErrorCateg"
},
{
"path": "packages/sandbox/src/sandbox.ts",
"chars": 12642,
"preview": "import type {\n\tSandboxOptions,\n\tSandboxResult,\n\tSandboxError,\n\tSandboxMetrics,\n\tCapturedOutput,\n\tResourceSnapshot,\n\tSand"
},
{
"path": "packages/sandbox/src/types.ts",
"chars": 2413,
"preview": "// ── Sandbox configuration ──\n\nexport interface SandboxOptions {\n\t/** Maximum execution time in milliseconds (default: "
},
{
"path": "packages/sandbox/tsconfig.json",
"chars": 145,
"preview": "{\n \"extends\": \"../../tsconfig.base.json\",\n \"compilerOptions\": {\n \"rootDir\": \"src\",\n \"outDir\": \"dist\"\n },\n \"inc"
},
{
"path": "tsconfig.base.json",
"chars": 646,
"preview": "{\n \"compilerOptions\": {\n \"target\": \"ESNext\",\n \"module\": \"ESNext\",\n \"moduleResolution\": \"bundler\",\n \"esModul"
},
{
"path": "tsconfig.json",
"chars": 496,
"preview": "{\n \"compilerOptions\": {\n \"target\": \"ESNext\",\n \"module\": \"ESNext\",\n \"moduleResolution\": \"bundler\",\n \"lib\": ["
}
]
About this extraction
This page contains the full source code of the ntegrals/openbrowser GitHub repository, extracted and formatted as plain text for AI agents and large language models (LLMs). The extraction includes 119 files (697.5 KB), approximately 186.1k tokens, and a symbol index with 969 extracted functions, classes, methods, constants, and types. Use this with OpenClaw, Claude, ChatGPT, Cursor, Windsurf, or any other AI tool that accepts text input. You can copy the full output to your clipboard or download it as a .txt file.
Extracted by GitExtract — free GitHub repo to text converter for AI. Built by Nikandr Surkov.