Showing preview only (853K chars total). Download the full file or copy to clipboard to get everything.
Repository: abi/screenshot-to-code
Branch: main
Commit: aaaa838548c3
Files: 228
Total size: 793.7 KB
Directory structure:
gitextract_l7qpd28l/
├── .claude/
│ └── launch.json
├── .gitattributes
├── .github/
│ ├── FUNDING.yml
│ └── ISSUE_TEMPLATE/
│ ├── bug_report.md
│ ├── custom.md
│ └── feature_request.md
├── .gitignore
├── .vscode/
│ └── settings.json
├── AGENTS.md
├── CLAUDE.md
├── Evaluation.md
├── LICENSE
├── README.md
├── TESTING.md
├── Troubleshooting.md
├── backend/
│ ├── .gitignore
│ ├── .pre-commit-config.yaml
│ ├── Dockerfile
│ ├── README.md
│ ├── agent/
│ │ ├── engine.py
│ │ ├── providers/
│ │ │ ├── __init__.py
│ │ │ ├── anthropic/
│ │ │ │ ├── __init__.py
│ │ │ │ ├── image.py
│ │ │ │ └── provider.py
│ │ │ ├── base.py
│ │ │ ├── factory.py
│ │ │ ├── gemini.py
│ │ │ ├── openai.py
│ │ │ ├── pricing.py
│ │ │ ├── token_usage.py
│ │ │ └── types.py
│ │ ├── runner.py
│ │ ├── state.py
│ │ └── tools/
│ │ ├── __init__.py
│ │ ├── definitions.py
│ │ ├── parsing.py
│ │ ├── runtime.py
│ │ ├── summaries.py
│ │ └── types.py
│ ├── codegen/
│ │ ├── __init__.py
│ │ ├── test_utils.py
│ │ └── utils.py
│ ├── config.py
│ ├── custom_types.py
│ ├── debug/
│ │ ├── DebugFileWriter.py
│ │ └── __init__.py
│ ├── evals/
│ │ ├── __init__.py
│ │ ├── config.py
│ │ ├── core.py
│ │ ├── runner.py
│ │ └── utils.py
│ ├── fs_logging/
│ │ ├── __init__.py
│ │ ├── openai_input_compare.py
│ │ ├── openai_input_formatting.py
│ │ └── openai_turn_inputs.py
│ ├── image_generation/
│ │ ├── __init__.py
│ │ ├── core.py
│ │ ├── generation.py
│ │ └── replicate.py
│ ├── llm.py
│ ├── main.py
│ ├── prompts/
│ │ ├── __init__.py
│ │ ├── create/
│ │ │ ├── __init__.py
│ │ │ ├── image.py
│ │ │ ├── text.py
│ │ │ └── video.py
│ │ ├── message_builder.py
│ │ ├── pipeline.py
│ │ ├── plan.py
│ │ ├── policies.py
│ │ ├── prompt_types.py
│ │ ├── request_parsing.py
│ │ ├── system_prompt.py
│ │ └── update/
│ │ ├── __init__.py
│ │ ├── from_file_snapshot.py
│ │ └── from_history.py
│ ├── pyproject.toml
│ ├── pyrightconfig.json
│ ├── pytest.ini
│ ├── routes/
│ │ ├── evals.py
│ │ ├── generate_code.py
│ │ ├── home.py
│ │ ├── model_choice_sets.py
│ │ └── screenshot.py
│ ├── run_evals.py
│ ├── run_image_generation_evals.py
│ ├── start.py
│ ├── tests/
│ │ ├── __init__.py
│ │ ├── test_agent_tool_runtime.py
│ │ ├── test_agent_tools.py
│ │ ├── test_batching.py
│ │ ├── test_codegen_utils.py
│ │ ├── test_evals_openai_input_compare.py
│ │ ├── test_image_generation_replicate.py
│ │ ├── test_model_selection.py
│ │ ├── test_openai_input_compare.py
│ │ ├── test_openai_provider_session.py
│ │ ├── test_openai_reasoning_parser.py
│ │ ├── test_openai_turn_input_logging.py
│ │ ├── test_parameter_extraction_stage.py
│ │ ├── test_prompt_summary.py
│ │ ├── test_prompts.py
│ │ ├── test_request_parsing.py
│ │ ├── test_screenshot.py
│ │ ├── test_status_broadcast.py
│ │ └── test_token_usage.py
│ ├── utils.py
│ ├── video/
│ │ ├── __init__.py
│ │ ├── cost_estimation.py
│ │ └── utils.py
│ └── ws/
│ ├── __init__.py
│ └── constants.py
├── blog/
│ └── evaluating-claude.md
├── design-docs/
│ ├── agent-tool-calling-flow.md
│ ├── agentic-runner-refactor.md
│ ├── commits-and-variants.md
│ ├── general.md
│ ├── images-in-update-history.md
│ ├── prompt-history-refactor.md
│ └── variant-system.md
├── docker-compose.yml
├── frontend/
│ ├── .eslintrc.cjs
│ ├── .gitignore
│ ├── Dockerfile
│ ├── components.json
│ ├── index.html
│ ├── jest.config.js
│ ├── package.json
│ ├── postcss.config.js
│ ├── src/
│ │ ├── App.tsx
│ │ ├── components/
│ │ │ ├── ImageLightbox.tsx
│ │ │ ├── ImageUpload.tsx
│ │ │ ├── ImportCodeSection.tsx
│ │ │ ├── TermsOfServiceDialog.tsx
│ │ │ ├── UpdateImageUpload.tsx
│ │ │ ├── agent/
│ │ │ │ └── AgentActivity.tsx
│ │ │ ├── commits/
│ │ │ │ ├── types.ts
│ │ │ │ └── utils.ts
│ │ │ ├── core/
│ │ │ │ ├── KeyboardShortcutBadge.tsx
│ │ │ │ ├── Spinner.tsx
│ │ │ │ ├── StackLabel.tsx
│ │ │ │ └── WorkingPulse.tsx
│ │ │ ├── evals/
│ │ │ │ ├── AllEvalsPage.tsx
│ │ │ │ ├── BestOfNEvalsPage.tsx
│ │ │ │ ├── EvalNavigation.tsx
│ │ │ │ ├── EvalsPage.tsx
│ │ │ │ ├── InputFileSelector.tsx
│ │ │ │ ├── OpenAIInputComparePage.tsx
│ │ │ │ ├── PairwiseEvalsPage.tsx
│ │ │ │ ├── RatingPicker.tsx
│ │ │ │ └── RunEvalsPage.tsx
│ │ │ ├── generate-from-text/
│ │ │ │ └── GenerateFromText.tsx
│ │ │ ├── history/
│ │ │ │ ├── HistoryDisplay.tsx
│ │ │ │ ├── utils.test.ts
│ │ │ │ └── utils.ts
│ │ │ ├── messages/
│ │ │ │ ├── OnboardingNote.tsx
│ │ │ │ ├── PicoBadge.tsx
│ │ │ │ └── TipLink.tsx
│ │ │ ├── preview/
│ │ │ │ ├── CodeMirror.tsx
│ │ │ │ ├── CodePreview.tsx
│ │ │ │ ├── CodeTab.tsx
│ │ │ │ ├── PreviewComponent.tsx
│ │ │ │ ├── PreviewPane.tsx
│ │ │ │ ├── download.ts
│ │ │ │ ├── extractHtml.ts
│ │ │ │ └── simpleHash.ts
│ │ │ ├── recording/
│ │ │ │ ├── ScreenRecorder.tsx
│ │ │ │ └── utils.ts
│ │ │ ├── select-and-edit/
│ │ │ │ └── utils.ts
│ │ │ ├── settings/
│ │ │ │ ├── GenerationSettings.tsx
│ │ │ │ ├── OutputSettingsSection.tsx
│ │ │ │ └── SettingsTab.tsx
│ │ │ ├── sidebar/
│ │ │ │ ├── IconStrip.tsx
│ │ │ │ └── Sidebar.tsx
│ │ │ ├── start-pane/
│ │ │ │ └── StartPane.tsx
│ │ │ ├── thinking/
│ │ │ │ └── ThinkingIndicator.tsx
│ │ │ ├── ui/
│ │ │ │ ├── accordion.tsx
│ │ │ │ ├── alert-dialog.tsx
│ │ │ │ ├── badge.tsx
│ │ │ │ ├── button.tsx
│ │ │ │ ├── checkbox.tsx
│ │ │ │ ├── collapsible.tsx
│ │ │ │ ├── dialog.tsx
│ │ │ │ ├── hover-card.tsx
│ │ │ │ ├── input.tsx
│ │ │ │ ├── label.tsx
│ │ │ │ ├── popover.tsx
│ │ │ │ ├── progress.tsx
│ │ │ │ ├── scroll-area.tsx
│ │ │ │ ├── select.tsx
│ │ │ │ ├── separator.tsx
│ │ │ │ ├── switch.tsx
│ │ │ │ ├── tabs.tsx
│ │ │ │ └── textarea.tsx
│ │ │ ├── unified-input/
│ │ │ │ ├── UnifiedInputPane.tsx
│ │ │ │ └── tabs/
│ │ │ │ ├── ImportTab.tsx
│ │ │ │ ├── TextTab.tsx
│ │ │ │ ├── UploadTab.tsx
│ │ │ │ └── UrlTab.tsx
│ │ │ └── variants/
│ │ │ └── Variants.tsx
│ │ ├── config.ts
│ │ ├── constants.ts
│ │ ├── generateCode.ts
│ │ ├── hooks/
│ │ │ ├── useBrowserTabIndicator.ts
│ │ │ ├── usePersistedState.ts
│ │ │ └── useThrottle.ts
│ │ ├── index.css
│ │ ├── lib/
│ │ │ ├── models.ts
│ │ │ ├── prompt-history.test.ts
│ │ │ ├── prompt-history.ts
│ │ │ ├── stacks.ts
│ │ │ ├── takeScreenshot.ts
│ │ │ └── utils.ts
│ │ ├── main.tsx
│ │ ├── setupTests.ts
│ │ ├── store/
│ │ │ ├── app-store.ts
│ │ │ └── project-store.ts
│ │ ├── tests/
│ │ │ ├── fixtures/
│ │ │ │ └── simple_page.html
│ │ │ └── qa.test.ts
│ │ ├── types.ts
│ │ ├── urls.ts
│ │ └── vite-env.d.ts
│ ├── tailwind.config.js
│ ├── tsconfig.json
│ ├── tsconfig.node.json
│ └── vite.config.ts
├── package.json
└── plan.md
================================================
FILE CONTENTS
================================================
================================================
FILE: .claude/launch.json
================================================
{
"version": "0.0.1",
"configurations": [
{
"name": "frontend",
"runtimeExecutable": "yarn",
"runtimeArgs": ["dev"],
"port": 5173,
"cwd": "frontend"
}
]
}
================================================
FILE: .gitattributes
================================================
# Auto detect text files and perform LF normalization
* text=auto
================================================
FILE: .github/FUNDING.yml
================================================
github: [abi]
================================================
FILE: .github/ISSUE_TEMPLATE/bug_report.md
================================================
---
name: Bug report
about: Create a report to help us improve
title: ''
labels: ''
assignees: ''
---
**Describe the bug**
A clear and concise description of what the bug is.
**To Reproduce**
Steps to reproduce the behavior:
1. Go to '...'
2. Click on '....'
3. Scroll down to '....'
4. See error
**Screenshots of backend AND frontend terminal logs**
If applicable, add screenshots to help explain your problem.
================================================
FILE: .github/ISSUE_TEMPLATE/custom.md
================================================
---
name: Custom issue template
about: Describe this issue template's purpose here.
title: ''
labels: ''
assignees: ''
---
================================================
FILE: .github/ISSUE_TEMPLATE/feature_request.md
================================================
---
name: Feature request
about: Suggest an idea for this project
title: ''
labels: ''
assignees: ''
---
**Is your feature request related to a problem? Please describe.**
A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
**Describe the solution you'd like**
A clear and concise description of what you want to happen.
**Describe alternatives you've considered**
A clear and concise description of any alternative solutions or features you've considered.
**Additional context**
Add any other context or screenshots about the feature request here.
================================================
FILE: .gitignore
================================================
.aider*
node_modules
# Project-related files
# Run logs
backend/run_logs/*
# Weird Docker setup related files
backend/backend/*
# Env vars
frontend/.env.local
.env
# Mac files
.DS_Store
#Rodney
.rodney
================================================
FILE: .vscode/settings.json
================================================
{
"python.analysis.typeCheckingMode": "strict",
"python.analysis.extraPaths": ["./backend"],
"python.autoComplete.extraPaths": ["./backend"]
}
================================================
FILE: AGENTS.md
================================================
# Project Agent Instructions
Python environment:
- Always use the backend Poetry virtualenv (`backend-py3.10`) for Python commands.
- Preferred invocation: `cd backend && poetry run <command>`.
- If you need to activate directly, use Poetry to discover it in the current environment:
- `cd backend && poetry env activate` (then run the `source .../bin/activate` command it prints)
Testing policy:
- Always run backend tests after every code change: `cd backend && poetry run pytest`.
- Always run type checking after every code change: `cd backend && poetry run pyright`.
- Type checking policy: no new warnings in changed files (`pyright`).
## Frontend
- Frontend: `cd frontend && yarn lint`
If changes touch both, run both sets.
## Prompt formatting
- Prefer triple-quoted strings (`"""..."""`) for multi-line prompt text.
- For interpolated multi-line prompts, prefer a single triple-quoted f-string over concatenated string fragments.
# Hosted
The hosted version is on the `hosted` branch. The `hosted` branch connects to a saas backend, which is a seperate codebase at ../screenshot-to-code-saas
================================================
FILE: CLAUDE.md
================================================
# Project Agent Instructions
Python environment:
- Always use the backend Poetry virtualenv (`backend-py3.10`) for Python commands.
- Preferred invocation: `cd backend && poetry run <command>`.
- If you need to activate directly, use Poetry to discover it in the current environment:
- `cd backend && poetry env activate` (then run the `source .../bin/activate` command it prints)
Testing policy:
- Always run backend tests after every code change: `cd backend && poetry run pytest`.
- Always run type checking after every code change: `cd backend && poetry run pyright`.
- Type checking policy: no new warnings in changed files (`pyright`).
## Frontend
- Frontend: `cd frontend && yarn lint`
If changes touch both, run both sets.
## Prompt formatting
- Prefer triple-quoted strings (`"""..."""`) for multi-line prompt text.
- For interpolated multi-line prompts, prefer a single triple-quoted f-string over concatenated string fragments.
# Hosted
The hosted version is on the `hosted` branch. The `hosted` branch connects to a saas backend, which is a seperate codebase at ../screenshot-to-code-saas
================================================
FILE: Evaluation.md
================================================
## Evaluating models and prompts
Evaluation dataset consists of 16 screenshots. A Python script for running screenshot-to-code on the dataset and a UI for rating outputs is included. With this set up, we can compare and evaluate various models and prompts.
### Running evals
- Input screenshots should be located at `backend/evals_data/inputs` and the outputs will be `backend/evals_data/outputs`. If you want to modify this, modify `EVALS_DIR` in `backend/evals/config.py`. You can download the input screenshot dataset here: TODO.
- Set a stack and model (`STACK` var, `MODEL` var) in `backend/run_evals.py`
- Run `OPENAI_API_KEY=sk-... python run_evals.py` - this runs the screenshot-to-code on the input dataset in parallel but it will still take a few minutes to complete.
- Once the script is done, you can find the outputs in `backend/evals_data/outputs`.
### Rating evals
In order to view and rate the outputs, visit your front-end at `/evals`.
- Rate each output on a scale of 1-4
- You can also print the page as PDF to share your results with others.
Generally, I run three tests for each model/prompt + stack combo and take the average score out of those tests to evaluate.
================================================
FILE: LICENSE
================================================
MIT License
Copyright (c) 2023 Abi Raja
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
================================================
FILE: README.md
================================================
# screenshot-to-code
A simple tool to convert screenshots, mockups and Figma designs into clean, functional code using AI. Now supporting Gemini 3 and Claude Opus 4.5!
https://github.com/user-attachments/assets/85b911c0-efea-4957-badb-daa97ec402ad
Supported stacks:
- HTML + Tailwind
- HTML + CSS
- React + Tailwind
- Vue + Tailwind
- Bootstrap
- Ionic + Tailwind
- SVG
Supported AI models:
- Gemini 3 Flash and Pro - Best models! (Google)
- Claude Opus 4.5 - Best model! (Anthropic)
- GPT-5.3, GPT-5.2, GPT-4.1 (OpenAI)
- Other models are available as well but we recommend using the above models.
- DALL-E 3 or Flux Schnell (using Replicate) for image generation
See the [Examples](#-examples) section below for more demos.
We have experimental support for taking a video/screen recording of a website in action and turning that into a functional prototype.
[Learn more about video here](https://github.com/abi/screenshot-to-code/wiki/Screen-Recording-to-Code).
[Follow me on Twitter for updates](https://twitter.com/_abi_).
## 🌍 Hosted Version
[Try it live on the hosted version (paid)](https://screenshottocode.com).
## 🛠 Getting Started
The app has a React/Vite frontend and a FastAPI backend.
Keys needed:
- [OpenAI API key](https://github.com/abi/screenshot-to-code/blob/main/Troubleshooting.md), Anthropic key, or Google Gemini key
- Multiple keys are recommended so you can compare results from different models
If you'd like to run the app with Ollama open source models (not recommended due to poor quality results), [follow this comment](https://github.com/abi/screenshot-to-code/issues/354#issuecomment-2435479853).
Run the backend (I use Poetry for package management - `pip install --upgrade poetry` if you don't have it):
```bash
cd backend
echo "OPENAI_API_KEY=sk-your-key" > .env
echo "ANTHROPIC_API_KEY=your-key" >> .env
echo "GEMINI_API_KEY=your-key" >> .env
poetry install
poetry env activate
# run the printed command, e.g. source /path/to/venv/bin/activate
poetry run uvicorn main:app --reload --port 7001
```
You can also set up the keys using the settings dialog on the front-end (click the gear icon after loading the frontend).
Run the frontend:
```bash
cd frontend
yarn
yarn dev
```
Open http://localhost:5173 to use the app.
If you prefer to run the backend on a different port, update VITE_WS_BACKEND_URL in `frontend/.env.local`
## Docker
If you have Docker installed on your system, in the root directory, run:
```bash
echo "OPENAI_API_KEY=sk-your-key" > .env
docker-compose up -d --build
```
The app will be up and running at http://localhost:5173. Note that you can't develop the application with this setup as the file changes won't trigger a rebuild.
## 🙋♂️ FAQs
- **I'm running into an error when setting up the backend. How can I fix it?** [Try this](https://github.com/abi/screenshot-to-code/issues/3#issuecomment-1814777959). If that still doesn't work, open an issue.
- **How do I get an OpenAI API key?** See https://github.com/abi/screenshot-to-code/blob/main/Troubleshooting.md
- **How can I configure an OpenAI proxy?** - If you're not able to access the OpenAI API directly (due to e.g. country restrictions), you can try a VPN or you can configure the OpenAI base URL to use a proxy: Set OPENAI_BASE_URL in the `backend/.env` or directly in the UI in the settings dialog. Make sure the URL has "v1" in the path so it should look like this: `https://xxx.xxxxx.xxx/v1`
- **How can I update the backend host that my front-end connects to?** - Configure VITE_HTTP_BACKEND_URL and VITE_WS_BACKEND_URL in front/.env.local For example, set VITE_HTTP_BACKEND_URL=http://124.10.20.1:7001
- **Seeing UTF-8 errors when running the backend?** - On windows, open the .env file with notepad++, then go to Encoding and select UTF-8.
- **How can I provide feedback?** For feedback, feature requests and bug reports, open an issue or ping me on [Twitter](https://twitter.com/_abi_).
## 📚 Examples
**NYTimes**
| Original | Replica |
| --------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| <img width="1238" alt="Screenshot 2023-11-20 at 12 54 03 PM" src="https://github.com/user-attachments/assets/6b0ae86c-1b0f-4598-a578-c7b62205b3e2"> | <img width="1414" alt="Screenshot 2023-11-20 at 12 59 56 PM" src="https://github.com/user-attachments/assets/981c490e-9be6-407e-8e46-2642f0ca613e"> |
**Instagram**
https://github.com/user-attachments/assets/a335a105-f9cc-40e6-ac6b-64e5390bfc21
**Hacker News**
https://github.com/user-attachments/assets/205cb5c7-9c3c-438d-acd4-26dfe6e077e5
================================================
FILE: TESTING.md
================================================
# Testing Guide
This guide explains how to run tests for the Screenshot to Code project.
## Backend Tests
The backend uses pytest for testing. All tests are located in the `backend/tests` directory.
### Prerequisites
Make sure you have Poetry installed and have installed all dependencies:
```bash
cd backend
poetry install
```
### Running Tests
#### Run all tests
```bash
cd backend
poetry run pytest
```
#### Run tests with verbose output
```bash
poetry run pytest -vv
```
#### Run a specific test file
```bash
poetry run pytest tests/test_screenshot.py
```
#### Run a specific test class
```bash
poetry run pytest tests/test_screenshot.py::TestNormalizeUrl
```
#### Run a specific test method
```bash
poetry run pytest tests/test_screenshot.py::TestNormalizeUrl::test_url_without_protocol
```
#### Run tests with coverage report
```bash
poetry run pytest --cov=routes
```
#### Run tests in parallel (requires pytest-xdist)
```bash
poetry install --with dev pytest-xdist # Install if not already installed
poetry run pytest -n auto
```
### Test Configuration
The pytest configuration is defined in `backend/pytest.ini`:
- Tests are discovered in the `tests` directory
- Test files must match the pattern `test_*.py`
- Test classes must start with `Test`
- Test functions must start with `test_`
- Verbose output and short traceback format are enabled by default
### Writing New Tests
1. Create a new test file in `backend/tests/` following the naming convention `test_<module>.py`
2. Import the functions/classes you want to test
3. Write test functions or classes following pytest conventions
Example:
```python
import pytest
from routes.screenshot import normalize_url
def test_url_normalization():
assert normalize_url("example.com") == "https://example.com"
```
================================================
FILE: Troubleshooting.md
================================================
### Getting an OpenAI API key with GPT-4 model access
You don't need a ChatGPT Pro account. Screenshot to code uses API keys from your OpenAI developer account. In order to get access to the GPT4 Vision model, log into your OpenAI account and then, follow these instructions:
1. Open [OpenAI Dashboard](https://platform.openai.com/)
1. Go to Settings > Billing
1. Click at the Add payment details
<img width="900" alt="285636868-c80deb92-ab47-45cd-988f-deee67fbd44d" src="https://github.com/abi/screenshot-to-code/assets/23818/4e0f4b77-9578-4f9a-803c-c12b1502f3d7">
4. You have to buy some credits. The minimum is $5.
5. Go to Settings > Limits and check at the bottom of the page, your current tier has to be "Tier 1" to have GPT4 access
<img width="900" alt="285636973-da38bd4d-8a78-4904-8027-ca67d729b933" src="https://github.com/abi/screenshot-to-code/assets/23818/8d07cd84-0cf9-4f88-bc00-80eba492eadf">
6. Navigate to OpenAI [api keys](https://platform.openai.com/api-keys) page and create and copy a new secret key.
7. Go to Screenshot to code and paste it in the Settings dialog under OpenAI key (gear icon). Your key is only stored in your browser. Never stored on our servers.
## Still not working?
- Some users have also reported that it can take upto 30 minutes after your credit purchase for the GPT4 vision model to be activated.
- You need to add credits to your account AND set it to renew when credits run out in order to be upgraded to Tier 1. Make sure your "Settings > Limits" page shows that you are at Tier 1.
If you've followed these steps, and it still doesn't work, feel free to open a Github issue. We only provide support for the open source version since we don't have debugging logs on the hosted version. If you're looking to use the hosted version, we recommend getting a paid subscription on screenshottocode.com
================================================
FILE: backend/.gitignore
================================================
# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]
*$py.class
# C extensions
*.so
# Distribution / packaging
.Python
build/
develop-eggs/
dist/
downloads/
eggs/
.eggs/
lib/
lib64/
parts/
sdist/
var/
wheels/
share/python-wheels/
*.egg-info/
.installed.cfg
*.egg
MANIFEST
# PyInstaller
# Usually these files are written by a python script from a template
# before PyInstaller builds the exe, so as to inject date/other infos into it.
*.manifest
*.spec
# Installer logs
pip-log.txt
pip-delete-this-directory.txt
# Unit test / coverage reports
htmlcov/
.tox/
.nox/
.coverage
.coverage.*
.cache
nosetests.xml
coverage.xml
*.cover
*.py,cover
.hypothesis/
.pytest_cache/
cover/
# Translations
*.mo
*.pot
# Django stuff:
*.log
local_settings.py
db.sqlite3
db.sqlite3-journal
# Flask stuff:
instance/
.webassets-cache
# Scrapy stuff:
.scrapy
# Sphinx documentation
docs/_build/
# PyBuilder
.pybuilder/
target/
# Jupyter Notebook
.ipynb_checkpoints
# IPython
profile_default/
ipython_config.py
# pyenv
# For a library or package, you might want to ignore these files since the code is
# intended to run in multiple environments; otherwise, check them in:
# .python-version
# pipenv
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
# However, in case of collaboration, if having platform-specific dependencies or dependencies
# having no cross-platform support, pipenv may install dependencies that don't work, or not
# install all needed dependencies.
#Pipfile.lock
# poetry
# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
# This is especially recommended for binary packages to ensure reproducibility, and is more
# commonly ignored for libraries.
# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
#poetry.lock
# PEP 582; used by e.g. github.com/David-OConnor/pyflow
__pypackages__/
# Celery stuff
celerybeat-schedule
celerybeat.pid
# SageMath parsed files
*.sage.py
# Environments
.env
.venv
env/
venv/
ENV/
env.bak/
venv.bak/
# Spyder project settings
.spyderproject
.spyproject
# Rope project settings
.ropeproject
# mkdocs documentation
/site
# mypy
.mypy_cache/
.dmypy.json
dmypy.json
# Pyre type checker
.pyre/
# pytype static type analyzer
.pytype/
# Cython debug symbols
cython_debug/
# PyCharm
# JetBrains specific template is maintainted in a separate JetBrains.gitignore that can
# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
# and can be added to the global gitignore or merged into this file. For a more nuclear
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
#.idea/
# Temporary eval output
evals_data
# Temporary video evals (Remove before merge)
video_evals
================================================
FILE: backend/.pre-commit-config.yaml
================================================
# See https://pre-commit.com for more information
# See https://pre-commit.com/hooks.html for more hooks
repos:
- repo: https://github.com/pre-commit/pre-commit-hooks
rev: v3.2.0
hooks:
# - id: end-of-file-fixer
- id: check-yaml
- id: check-added-large-files
# - repo: local
# hooks:
# - id: poetry-pytest
# name: Run pytest with Poetry
# entry: poetry run --directory backend pytest
# language: system
# pass_filenames: false
# always_run: true
# files: ^backend/
# # - id: poetry-pyright
# # name: Run pyright with Poetry
# # entry: poetry run --directory backend pyright
# # language: system
# # pass_filenames: false
# # always_run: true
# # files: ^backend/
================================================
FILE: backend/Dockerfile
================================================
FROM python:3.12.3-slim-bullseye
ENV POETRY_VERSION 1.8.0
# Install system dependencies
RUN pip install "poetry==$POETRY_VERSION"
# Set work directory
WORKDIR /app
# Copy only requirements to cache them in docker layer
COPY poetry.lock pyproject.toml /app/
# Disable the creation of virtual environments
RUN poetry config virtualenvs.create false
# Install dependencies
RUN poetry install
# Copy the current directory contents into the container at /app
COPY ./ /app/
================================================
FILE: backend/README.md
================================================
# Run the type checker
poetry run pyright
# Run tests
poetry run pytest
## Prompt Summary
Use `print_prompt_summary` from `utils.py` to quickly visualize prompts:
```python
from utils import print_prompt_summary
print_prompt_summary(prompt_messages)
```
================================================
FILE: backend/agent/engine.py
================================================
import asyncio
import uuid
from typing import Any, Awaitable, Callable, Dict, List, Optional
from openai.types.chat import ChatCompletionMessageParam
from codegen.utils import extract_html_content
from llm import Llm
from agent.providers.base import ExecutedToolCall, ProviderSession, StreamEvent
from agent.providers.factory import create_provider_session
from agent.state import AgentFileState, seed_file_state_from_messages
from agent.tools import (
AgentToolRuntime,
extract_content_from_args,
extract_path_from_args,
summarize_text,
summarize_tool_input,
)
class AgentEngine:
def __init__(
self,
send_message: Callable[
[str, Optional[str], int, Optional[Dict[str, Any]], Optional[str]],
Awaitable[None],
],
variant_index: int,
openai_api_key: Optional[str],
openai_base_url: Optional[str],
anthropic_api_key: Optional[str],
gemini_api_key: Optional[str],
should_generate_images: bool,
initial_file_state: Optional[Dict[str, str]] = None,
option_codes: Optional[List[str]] = None,
):
self.send_message = send_message
self.variant_index = variant_index
self.openai_api_key = openai_api_key
self.openai_base_url = openai_base_url
self.anthropic_api_key = anthropic_api_key
self.gemini_api_key = gemini_api_key
self.should_generate_images = should_generate_images
self.file_state = AgentFileState()
if initial_file_state and initial_file_state.get("content"):
self.file_state.path = initial_file_state.get("path") or "index.html"
self.file_state.content = initial_file_state["content"]
self.tool_runtime = AgentToolRuntime(
file_state=self.file_state,
should_generate_images=should_generate_images,
openai_api_key=openai_api_key,
openai_base_url=openai_base_url,
option_codes=option_codes,
)
self._tool_preview_lengths: Dict[str, int] = {}
def _next_event_id(self, prefix: str) -> str:
return f"{prefix}-{self.variant_index}-{uuid.uuid4().hex[:8]}"
async def _send(
self,
msg_type: str,
value: Optional[str] = None,
data: Optional[Dict[str, Any]] = None,
event_id: Optional[str] = None,
) -> None:
await self.send_message(msg_type, value, self.variant_index, data, event_id)
def _mark_preview_length(self, tool_event_id: Optional[str], length: int) -> None:
if not tool_event_id:
return
current = self._tool_preview_lengths.get(tool_event_id, 0)
if length > current:
self._tool_preview_lengths[tool_event_id] = length
async def _stream_code_preview(self, tool_event_id: Optional[str], content: str) -> None:
if not tool_event_id or not content:
return
already_sent = self._tool_preview_lengths.get(tool_event_id, 0)
total_len = len(content)
if already_sent >= total_len:
return
max_chunks = 18
min_step = 200
step = max(min_step, total_len // max_chunks)
start = already_sent if already_sent > 0 else 0
for end in range(start + step, total_len, step):
await self._send("setCode", content[:end])
self._mark_preview_length(tool_event_id, end)
await asyncio.sleep(0.01)
await self._send("setCode", content)
self._mark_preview_length(tool_event_id, total_len)
async def _handle_streamed_tool_delta(
self,
event: StreamEvent,
started_tool_ids: set[str],
streamed_lengths: Dict[str, int],
) -> None:
if event.type != "tool_call_delta":
return
if event.tool_name != "create_file":
return
if not event.tool_call_id:
return
content = extract_content_from_args(event.tool_arguments)
if content is None:
return
tool_event_id = event.tool_call_id
if tool_event_id not in started_tool_ids:
path = (
extract_path_from_args(event.tool_arguments)
or self.file_state.path
or "index.html"
)
await self._send(
"toolStart",
data={
"name": "create_file",
"input": {
"path": path,
"contentLength": len(content),
"preview": summarize_text(content, 200),
},
},
event_id=tool_event_id,
)
started_tool_ids.add(tool_event_id)
last_len = streamed_lengths.get(tool_event_id, 0)
if last_len == 0 and content:
streamed_lengths[tool_event_id] = len(content)
await self._send("setCode", content)
self._mark_preview_length(tool_event_id, len(content))
elif len(content) - last_len >= 40:
streamed_lengths[tool_event_id] = len(content)
await self._send("setCode", content)
self._mark_preview_length(tool_event_id, len(content))
async def _run_with_session(self, session: ProviderSession) -> str:
max_steps = 20
for _ in range(max_steps):
assistant_event_id = self._next_event_id("assistant")
thinking_event_id = self._next_event_id("thinking")
started_tool_ids: set[str] = set()
streamed_lengths: Dict[str, int] = {}
async def on_event(event: StreamEvent) -> None:
if event.type == "assistant_delta":
if event.text:
await self._send(
"assistant",
event.text,
event_id=assistant_event_id,
)
return
if event.type == "thinking_delta":
if event.text:
await self._send(
"thinking",
event.text,
event_id=thinking_event_id,
)
return
if event.type == "tool_call_delta":
await self._handle_streamed_tool_delta(
event,
started_tool_ids,
streamed_lengths,
)
turn = await session.stream_turn(on_event)
if not turn.tool_calls:
return await self._finalize_response(turn.assistant_text)
executed_tool_calls: List[ExecutedToolCall] = []
for tool_call in turn.tool_calls:
tool_event_id = tool_call.id or self._next_event_id("tool")
if tool_event_id not in started_tool_ids:
await self._send(
"toolStart",
data={
"name": tool_call.name,
"input": summarize_tool_input(tool_call, self.file_state),
},
event_id=tool_event_id,
)
if tool_call.name == "create_file":
content = extract_content_from_args(tool_call.arguments)
if content:
await self._stream_code_preview(tool_event_id, content)
tool_result = await self.tool_runtime.execute(tool_call)
if tool_result.updated_content:
await self._send("setCode", tool_result.updated_content)
await self._send(
"toolResult",
data={
"name": tool_call.name,
"output": tool_result.summary,
"ok": tool_result.ok,
},
event_id=tool_event_id,
)
executed_tool_calls.append(
ExecutedToolCall(tool_call=tool_call, result=tool_result)
)
session.append_tool_results(turn, executed_tool_calls)
raise Exception("Agent exceeded max tool turns")
async def run(self, model: Llm, prompt_messages: List[ChatCompletionMessageParam]) -> str:
seed_file_state_from_messages(self.file_state, prompt_messages)
session = create_provider_session(
model=model,
prompt_messages=prompt_messages,
should_generate_images=self.should_generate_images,
openai_api_key=self.openai_api_key,
openai_base_url=self.openai_base_url,
anthropic_api_key=self.anthropic_api_key,
gemini_api_key=self.gemini_api_key,
)
try:
return await self._run_with_session(session)
finally:
await session.close()
async def _finalize_response(self, assistant_text: str) -> str:
if self.file_state.content:
return self.file_state.content
html = extract_html_content(assistant_text)
if html:
self.file_state.content = html
await self._send("setCode", html)
return self.file_state.content
================================================
FILE: backend/agent/providers/__init__.py
================================================
from agent.providers.anthropic import AnthropicProviderSession, serialize_anthropic_tools
from agent.providers.base import (
EventSink,
ExecutedToolCall,
ProviderSession,
ProviderTurn,
StreamEvent,
)
from agent.providers.factory import create_provider_session
from agent.providers.gemini import GeminiProviderSession, serialize_gemini_tools
from agent.providers.openai import OpenAIProviderSession, parse_event, serialize_openai_tools
__all__ = [
"AnthropicProviderSession",
"EventSink",
"ExecutedToolCall",
"GeminiProviderSession",
"OpenAIProviderSession",
"ProviderSession",
"ProviderTurn",
"StreamEvent",
"create_provider_session",
"parse_event",
"serialize_anthropic_tools",
"serialize_gemini_tools",
"serialize_openai_tools",
]
================================================
FILE: backend/agent/providers/anthropic/__init__.py
================================================
from agent.providers.anthropic.provider import (
AnthropicProviderSession,
serialize_anthropic_tools,
_extract_anthropic_usage,
)
__all__ = [
"AnthropicProviderSession",
"serialize_anthropic_tools",
"_extract_anthropic_usage",
]
================================================
FILE: backend/agent/providers/anthropic/image.py
================================================
# pyright: reportUnknownVariableType=false
"""Claude-specific image processing.
Handles resizing and compressing images to comply with Claude's vision API
limits before sending them as base64-encoded payloads.
Comparison with official Anthropic docs
(https://docs.anthropic.com/en/docs/build-with-claude/vision):
Aligned:
- 5 MB per-image size limit matches the documented API maximum.
- Output uses the correct base64 source format (type, media_type, data).
Divergences:
- Max dimension is set to 7990 px as a safety margin; the API rejects at
8000 px. This is intentionally conservative.
- The docs note that when >20 images are sent in a single request the
per-image limit drops to 2000x2000 px. We do not enforce that stricter
limit here (the app typically sends far fewer images).
- JPEG conversion drops alpha channels, which is acceptable for website
screenshots but would degrade transparent PNGs.
Recommendation:
The docs recommend resizing to 1568 px on the long edge (~1.15 megapixels)
for optimal time-to-first-token. Images above that threshold are resized
server-side anyway, so sending larger images only adds latency and
bandwidth cost with no quality benefit. Consider lowering
CLAUDE_MAX_IMAGE_DIMENSION to 1568.
"""
import base64
import io
import time
from PIL import Image
# Hard API limit: 5 MB per image (base64-encoded).
CLAUDE_IMAGE_MAX_SIZE = 5 * 1024 * 1024
# API rejects images wider or taller than 8000 px. We use 7990 as a safety
# margin. Note: the docs recommend 1568 px for best latency (see module
# docstring).
CLAUDE_MAX_IMAGE_DIMENSION = 7990
def process_image(image_data_url: str) -> tuple[str, str]:
"""Resize / compress a data-URL image to fit Claude's vision limits.
Returns (media_type, base64_data) suitable for an ``image`` content block.
"""
media_type = image_data_url.split(";")[0].split(":")[1]
base64_data = image_data_url.split(",")[1]
image_bytes = base64.b64decode(base64_data)
img = Image.open(io.BytesIO(image_bytes))
is_under_dimension_limit = (
img.width < CLAUDE_MAX_IMAGE_DIMENSION
and img.height < CLAUDE_MAX_IMAGE_DIMENSION
)
is_under_size_limit = len(base64_data) <= CLAUDE_IMAGE_MAX_SIZE
if is_under_dimension_limit and is_under_size_limit:
return (media_type, base64_data)
start_time = time.time()
if not is_under_dimension_limit:
if img.width > img.height:
new_width = CLAUDE_MAX_IMAGE_DIMENSION
new_height = int((CLAUDE_MAX_IMAGE_DIMENSION / img.width) * img.height)
else:
new_height = CLAUDE_MAX_IMAGE_DIMENSION
new_width = int((CLAUDE_MAX_IMAGE_DIMENSION / img.height) * img.width)
img = img.resize((new_width, new_height), Image.DEFAULT_STRATEGY)
quality = 95
output = io.BytesIO()
img = img.convert("RGB")
img.save(output, format="JPEG", quality=quality)
while (
len(base64.b64encode(output.getvalue())) > CLAUDE_IMAGE_MAX_SIZE
and quality > 10
):
output = io.BytesIO()
img.save(output, format="JPEG", quality=quality)
quality -= 5
end_time = time.time()
processing_time = end_time - start_time
print(f"[CLAUDE IMAGE PROCESSING] processing time: {processing_time:.2f} seconds")
return ("image/jpeg", base64.b64encode(output.getvalue()).decode("utf-8"))
================================================
FILE: backend/agent/providers/anthropic/provider.py
================================================
# pyright: reportUnknownVariableType=false
import copy
import json
import uuid
from dataclasses import dataclass, field
from typing import Any, Dict, List, cast
from anthropic import AsyncAnthropic
from openai.types.chat import ChatCompletionMessageParam
from agent.providers.base import (
EventSink,
ExecutedToolCall,
ProviderSession,
ProviderTurn,
StreamEvent,
)
from agent.providers.anthropic.image import process_image
from agent.providers.pricing import MODEL_PRICING
from agent.providers.token_usage import TokenUsage
from agent.tools import CanonicalToolDefinition, ToolCall, parse_json_arguments
from llm import Llm
THINKING_MODELS = {
Llm.CLAUDE_4_5_SONNET_2025_09_29.value,
Llm.CLAUDE_4_5_OPUS_2025_11_01.value,
}
ADAPTIVE_THINKING_MODELS = {
Llm.CLAUDE_OPUS_4_6.value,
Llm.CLAUDE_SONNET_4_6.value,
}
def _convert_openai_messages_to_claude(
messages: List[ChatCompletionMessageParam],
) -> tuple[str, List[Dict[str, Any]]]:
cloned_messages = copy.deepcopy(messages)
system_prompt = cast(str, cloned_messages[0].get("content"))
claude_messages = [dict(message) for message in cloned_messages[1:]]
for message in claude_messages:
if not isinstance(message["content"], list):
continue
for content in message["content"]: # type: ignore
if content["type"] != "image_url":
continue
content["type"] = "image"
image_data_url = cast(str, content["image_url"]["url"])
media_type, base64_data = process_image(image_data_url)
del content["image_url"]
content["source"] = {
"type": "base64",
"media_type": media_type,
"data": base64_data,
}
return system_prompt, claude_messages
def serialize_anthropic_tools(
tools: List[CanonicalToolDefinition],
) -> List[Dict[str, Any]]:
return [
{
"name": tool.name,
"description": tool.description,
"eager_input_streaming": True,
"input_schema": copy.deepcopy(tool.parameters),
}
for tool in tools
]
@dataclass
class AnthropicParseState:
assistant_text: str = ""
tool_blocks: Dict[int, Dict[str, Any]] = field(default_factory=dict)
tool_json_buffers: Dict[int, str] = field(default_factory=dict)
async def _parse_stream_event(
event: Any,
state: AnthropicParseState,
on_event: EventSink,
) -> None:
if event.type == "content_block_start":
block = event.content_block
if getattr(block, "type", None) != "tool_use":
return
tool_id = getattr(block, "id", None) or f"tool-{uuid.uuid4().hex[:6]}"
tool_name = getattr(block, "name", None) or "unknown_tool"
args = getattr(block, "input", None)
state.tool_blocks[event.index] = {
"id": tool_id,
"name": tool_name,
}
state.tool_json_buffers[event.index] = ""
if args:
await on_event(
StreamEvent(
type="tool_call_delta",
tool_call_id=tool_id,
tool_name=tool_name,
tool_arguments=args,
)
)
return
if event.type != "content_block_delta":
return
if event.delta.type == "thinking_delta":
await on_event(StreamEvent(type="thinking_delta", text=event.delta.thinking))
return
if event.delta.type == "text_delta":
state.assistant_text += event.delta.text
await on_event(StreamEvent(type="assistant_delta", text=event.delta.text))
return
if event.delta.type != "input_json_delta":
return
partial_json = getattr(event.delta, "partial_json", None) or ""
if not partial_json:
return
buffer = state.tool_json_buffers.get(event.index, "") + partial_json
state.tool_json_buffers[event.index] = buffer
meta = state.tool_blocks.get(event.index)
if not meta:
return
await on_event(
StreamEvent(
type="tool_call_delta",
tool_call_id=meta.get("id"),
tool_name=meta.get("name"),
tool_arguments=buffer,
)
)
def _extract_tool_calls(final_message: Any) -> List[ToolCall]:
tool_calls: List[ToolCall] = []
if final_message and final_message.content:
for block in final_message.content:
if block.type != "tool_use":
continue
raw_input = getattr(block, "input", {})
args: Dict[str, Any]
if isinstance(raw_input, dict):
args = cast(Dict[str, Any], raw_input)
else:
parsed, error = parse_json_arguments(raw_input)
if error:
args = {"INVALID_JSON": str(raw_input)}
else:
args = parsed
tool_calls.append(
ToolCall(
id=block.id,
name=block.name,
arguments=args,
)
)
return tool_calls
def _extract_anthropic_usage(final_message: Any) -> TokenUsage:
"""Extract unified token usage from an Anthropic final message.
Anthropic includes thinking tokens in ``output_tokens`` so no extra
addition is needed. ``total`` is computed since the API doesn't provide it.
"""
usage = getattr(final_message, "usage", None)
if usage is None:
return TokenUsage()
input_tokens = getattr(usage, "input_tokens", 0) or 0
output_tokens = getattr(usage, "output_tokens", 0) or 0
cache_read = getattr(usage, "cache_read_input_tokens", 0) or 0
cache_write = getattr(usage, "cache_creation_input_tokens", 0) or 0
return TokenUsage(
input=input_tokens,
output=output_tokens,
cache_read=cache_read,
cache_write=cache_write,
total=input_tokens + output_tokens + cache_read + cache_write,
)
class AnthropicProviderSession(ProviderSession):
def __init__(
self,
client: AsyncAnthropic,
model: Llm,
prompt_messages: List[ChatCompletionMessageParam],
tools: List[Dict[str, Any]],
):
self._client = client
self._model = model
self._tools = tools
self._total_usage = TokenUsage()
system_prompt, claude_messages = _convert_openai_messages_to_claude(prompt_messages)
self._system_prompt = system_prompt
self._messages = claude_messages
async def stream_turn(self, on_event: EventSink) -> ProviderTurn:
stream_kwargs: Dict[str, Any] = {
"model": self._model.value,
"max_tokens": 50000,
"system": self._system_prompt,
"messages": self._messages,
"tools": self._tools,
"cache_control": {"type": "ephemeral"},
}
if self._model.value in ADAPTIVE_THINKING_MODELS:
stream_kwargs["thinking"] = {
"type": "adaptive",
}
effort = (
"high"
if self._model.value == Llm.CLAUDE_SONNET_4_6.value
else "max"
)
stream_kwargs["output_config"] = {"effort": effort}
elif self._model.value in THINKING_MODELS:
stream_kwargs["thinking"] = {
"type": "enabled",
"budget_tokens": 10000,
}
else:
stream_kwargs["temperature"] = 0.0
state = AnthropicParseState()
async with self._client.messages.stream(**stream_kwargs) as stream:
async for event in stream:
await _parse_stream_event(event, state, on_event)
final_message = await stream.get_final_message()
self._total_usage.accumulate(_extract_anthropic_usage(final_message))
tool_calls = _extract_tool_calls(final_message)
return ProviderTurn(
assistant_text=state.assistant_text,
tool_calls=tool_calls,
assistant_turn=final_message,
)
def append_tool_results(
self,
turn: ProviderTurn,
executed_tool_calls: list[ExecutedToolCall],
) -> None:
assistant_blocks: List[Dict[str, Any]] = []
if turn.assistant_text:
assistant_blocks.append({"type": "text", "text": turn.assistant_text})
for call in turn.tool_calls:
assistant_blocks.append(
{
"type": "tool_use",
"id": call.id,
"name": call.name,
"input": call.arguments,
}
)
self._messages.append({"role": "assistant", "content": assistant_blocks})
tool_result_blocks: List[Dict[str, Any]] = []
for executed in executed_tool_calls:
tool_result_blocks.append(
{
"type": "tool_result",
"tool_use_id": executed.tool_call.id,
"content": json.dumps(executed.result.result),
"is_error": not executed.result.ok,
}
)
self._messages.append({"role": "user", "content": tool_result_blocks})
async def close(self) -> None:
u = self._total_usage
model_name = self._model.value
pricing = MODEL_PRICING.get(model_name)
cost_str = f" cost=${u.cost(pricing):.4f}" if pricing else ""
cache_hit_rate_str = f" cache_hit_rate={u.cache_hit_rate_percent():.2f}%"
print(
f"[TOKEN USAGE] provider=anthropic model={model_name} | "
f"input={u.input} output={u.output} "
f"cache_read={u.cache_read} cache_write={u.cache_write} "
f"total={u.total}{cache_hit_rate_str}{cost_str}"
)
await self._client.close()
================================================
FILE: backend/agent/providers/base.py
================================================
from dataclasses import dataclass
from typing import Any, Awaitable, Callable, Literal, Optional, Protocol
from agent.tools import ToolCall, ToolExecutionResult
StreamEventType = Literal[
"assistant_delta",
"thinking_delta",
"tool_call_delta",
]
@dataclass
class StreamEvent:
type: StreamEventType
text: str = ""
tool_call_id: Optional[str] = None
tool_name: Optional[str] = None
tool_arguments: Any = None
@dataclass
class ProviderTurn:
assistant_text: str
tool_calls: list[ToolCall]
# Provider-native assistant turn object required to continue the conversation.
assistant_turn: Any = None
@dataclass
class ExecutedToolCall:
tool_call: ToolCall
result: ToolExecutionResult
EventSink = Callable[[StreamEvent], Awaitable[None]]
class ProviderSession(Protocol):
async def stream_turn(self, on_event: EventSink) -> ProviderTurn:
...
def append_tool_results(
self,
turn: ProviderTurn,
executed_tool_calls: list[ExecutedToolCall],
) -> None:
...
async def close(self) -> None:
...
================================================
FILE: backend/agent/providers/factory.py
================================================
from typing import Optional
from anthropic import AsyncAnthropic
from google import genai
from openai import AsyncOpenAI
from openai.types.chat import ChatCompletionMessageParam
from agent.providers.anthropic import AnthropicProviderSession, serialize_anthropic_tools
from agent.providers.base import ProviderSession
from agent.providers.gemini import GeminiProviderSession, serialize_gemini_tools
from agent.providers.openai import OpenAIProviderSession, serialize_openai_tools
from agent.tools import canonical_tool_definitions
from llm import ANTHROPIC_MODELS, GEMINI_MODELS, OPENAI_MODELS, Llm
def create_provider_session(
model: Llm,
prompt_messages: list[ChatCompletionMessageParam],
should_generate_images: bool,
openai_api_key: Optional[str],
openai_base_url: Optional[str],
anthropic_api_key: Optional[str],
gemini_api_key: Optional[str],
) -> ProviderSession:
canonical_tools = canonical_tool_definitions(
image_generation_enabled=should_generate_images
)
if model in OPENAI_MODELS:
if not openai_api_key:
raise Exception("OpenAI API key is missing.")
client = AsyncOpenAI(api_key=openai_api_key, base_url=openai_base_url)
return OpenAIProviderSession(
client=client,
model=model,
prompt_messages=prompt_messages,
tools=serialize_openai_tools(canonical_tools),
)
if model in ANTHROPIC_MODELS:
if not anthropic_api_key:
raise Exception("Anthropic API key is missing.")
client = AsyncAnthropic(api_key=anthropic_api_key)
return AnthropicProviderSession(
client=client,
model=model,
prompt_messages=prompt_messages,
tools=serialize_anthropic_tools(canonical_tools),
)
if model in GEMINI_MODELS:
if not gemini_api_key:
raise Exception("Gemini API key is missing.")
client = genai.Client(api_key=gemini_api_key)
return GeminiProviderSession(
client=client,
model=model,
prompt_messages=prompt_messages,
tools=serialize_gemini_tools(canonical_tools),
)
raise ValueError(f"Unsupported model: {model.value}")
================================================
FILE: backend/agent/providers/gemini.py
================================================
# pyright: reportUnknownVariableType=false
import base64
import copy
import uuid
from dataclasses import dataclass, field
from typing import Any, Dict, List, cast
from google import genai
from google.genai import types
from openai.types.chat import ChatCompletionMessageParam
from agent.providers.base import (
EventSink,
ExecutedToolCall,
ProviderSession,
ProviderTurn,
StreamEvent,
)
from agent.providers.pricing import MODEL_PRICING
from agent.providers.token_usage import TokenUsage
from agent.tools import CanonicalToolDefinition, ToolCall
from llm import Llm
DEFAULT_VIDEO_FPS = 10
def serialize_gemini_tools(tools: List[CanonicalToolDefinition]) -> List[types.Tool]:
declarations = [
types.FunctionDeclaration(
name=tool.name,
description=tool.description,
parameters_json_schema=copy.deepcopy(tool.parameters),
)
for tool in tools
]
return [types.Tool(function_declarations=declarations)]
def _get_gemini_api_model_name(model: Llm) -> str:
if model in [Llm.GEMINI_3_FLASH_PREVIEW_HIGH, Llm.GEMINI_3_FLASH_PREVIEW_MINIMAL]:
return "gemini-3-flash-preview"
if model in [
Llm.GEMINI_3_1_PRO_PREVIEW_HIGH,
Llm.GEMINI_3_1_PRO_PREVIEW_MEDIUM,
Llm.GEMINI_3_1_PRO_PREVIEW_LOW,
]:
return "gemini-3.1-pro-preview"
return model.value
def _get_thinking_level_for_model(model: Llm) -> str:
if model in [
Llm.GEMINI_3_FLASH_PREVIEW_HIGH,
Llm.GEMINI_3_1_PRO_PREVIEW_HIGH,
]:
return "high"
if model == Llm.GEMINI_3_1_PRO_PREVIEW_LOW:
return "low"
if model == Llm.GEMINI_3_1_PRO_PREVIEW_MEDIUM:
return "medium"
if model == Llm.GEMINI_3_FLASH_PREVIEW_MINIMAL:
return "minimal"
return "high"
def _extract_text_from_content(content: str | List[Dict[str, Any]]) -> str:
if isinstance(content, str):
return content
for content_part in content:
if content_part.get("type") == "text":
return content_part.get("text", "")
return ""
def _detect_mime_type_from_base64(base64_data: str) -> str | None:
try:
decoded = base64.b64decode(base64_data[:32])
if decoded[:8] == b"\x89PNG\r\n\x1a\n":
return "image/png"
if decoded[:2] == b"\xff\xd8":
return "image/jpeg"
if decoded[:6] in (b"GIF87a", b"GIF89a"):
return "image/gif"
if decoded[:4] == b"RIFF" and decoded[8:12] == b"WEBP":
return "image/webp"
if decoded[4:8] == b"ftyp":
return "video/mp4"
if decoded[:4] == b"\x1aE\xdf\xa3":
return "video/webm"
except Exception:
pass
return None
def _extract_images_from_content(content: str | List[Dict[str, Any]]) -> List[Dict[str, str]]:
if isinstance(content, str):
return []
images: List[Dict[str, str]] = []
for content_part in content:
if content_part.get("type") != "image_url":
continue
image_url = content_part["image_url"]["url"]
if image_url.startswith("data:"):
mime_type = image_url.split(";")[0].split(":")[1]
base64_data = image_url.split(",")[1]
if mime_type == "application/octet-stream":
detected_mime = _detect_mime_type_from_base64(base64_data)
if detected_mime:
mime_type = detected_mime
else:
print("Warning: Could not detect MIME type for data URL, skipping")
continue
images.append({"mime_type": mime_type, "data": base64_data})
continue
images.append({"uri": image_url})
return images
def _convert_message_to_gemini_content(
message: ChatCompletionMessageParam,
) -> types.Content:
role = message.get("role", "user")
content = message.get("content", "")
gemini_role = "model" if role == "assistant" else "user"
parts: List[types.Part | Dict[str, str]] = []
text = _extract_text_from_content(content) # type: ignore
image_data_list = _extract_images_from_content(content) # type: ignore
if text:
parts.append({"text": text})
for image_data in image_data_list:
if "data" in image_data:
mime_type = image_data["mime_type"]
media_bytes = base64.b64decode(image_data["data"])
if mime_type.startswith("video/"):
parts.append(
types.Part(
inline_data=types.Blob(data=media_bytes, mime_type=mime_type),
video_metadata=types.VideoMetadata(fps=DEFAULT_VIDEO_FPS),
media_resolution=types.PartMediaResolutionLevel.MEDIA_RESOLUTION_HIGH,
)
)
continue
parts.append(
types.Part.from_bytes(
data=media_bytes,
mime_type=mime_type,
media_resolution=types.PartMediaResolutionLevel.MEDIA_RESOLUTION_ULTRA_HIGH,
)
)
continue
if "uri" in image_data:
parts.append({"file_uri": image_data["uri"]})
return types.Content(role=gemini_role, parts=parts) # type: ignore
@dataclass
class GeminiParseState:
assistant_text: str = ""
tool_calls: List[ToolCall] = field(default_factory=list)
model_parts: List[types.Part] = field(default_factory=list)
model_role: str = "model"
def _extract_usage(chunk: types.GenerateContentResponse) -> TokenUsage | None:
"""Extract unified token usage from a Gemini streaming chunk.
Gemini reports thinking tokens separately; they are folded into ``output``
to match the unified schema used by the other providers.
``prompt_token_count`` *includes* ``cached_content_token_count``, so we
subtract cached tokens to get the non-cached input count (same approach
as the OpenAI provider).
"""
meta = chunk.usage_metadata
if meta is None:
return None
candidates = meta.candidates_token_count or 0
thoughts = meta.thoughts_token_count or 0
prompt_tokens = meta.prompt_token_count or 0
cached_tokens = meta.cached_content_token_count or 0
return TokenUsage(
input=prompt_tokens - cached_tokens,
output=candidates + thoughts,
cache_read=cached_tokens,
cache_write=0,
total=meta.total_token_count or 0,
)
async def _parse_chunk(
chunk: types.GenerateContentResponse,
state: GeminiParseState,
on_event: EventSink,
) -> None:
if not chunk.candidates:
return
candidate_content = chunk.candidates[0].content
if not candidate_content or not candidate_content.parts:
return
if candidate_content.role:
state.model_role = candidate_content.role
for part in candidate_content.parts:
# Preserve each model part as streamed so thought signatures remain attached.
state.model_parts.append(part)
if getattr(part, "thought", False) and part.text:
await on_event(StreamEvent(type="thinking_delta", text=part.text))
continue
if part.function_call:
args = part.function_call.args or {}
tool_id = part.function_call.id or f"tool-{uuid.uuid4().hex[:6]}"
tool_name = part.function_call.name or "unknown_tool"
await on_event(
StreamEvent(
type="tool_call_delta",
tool_call_id=tool_id,
tool_name=tool_name,
tool_arguments=args,
)
)
state.tool_calls.append(
ToolCall(
id=tool_id,
name=tool_name,
arguments=args,
)
)
continue
if part.text:
state.assistant_text += part.text
await on_event(StreamEvent(type="assistant_delta", text=part.text))
class GeminiProviderSession(ProviderSession):
def __init__(
self,
client: genai.Client,
model: Llm,
prompt_messages: List[ChatCompletionMessageParam],
tools: List[types.Tool],
):
self._client = client
self._model = model
self._tools = tools
self._total_usage = TokenUsage()
self._system_prompt = str(prompt_messages[0].get("content", ""))
self._contents: List[types.Content] = [
_convert_message_to_gemini_content(msg) for msg in prompt_messages[1:]
]
async def stream_turn(self, on_event: EventSink) -> ProviderTurn:
thinking_level = _get_thinking_level_for_model(self._model)
config = types.GenerateContentConfig(
temperature=1.0,
max_output_tokens=50000,
system_instruction=self._system_prompt,
thinking_config=types.ThinkingConfig(
thinking_level=cast(Any, thinking_level),
include_thoughts=True,
),
tools=self._tools,
)
stream = await self._client.aio.models.generate_content_stream(
model=_get_gemini_api_model_name(self._model),
contents=cast(Any, self._contents),
config=config,
)
state = GeminiParseState()
turn_usage: TokenUsage | None = None
async for chunk in stream:
await _parse_chunk(chunk, state, on_event)
chunk_usage = _extract_usage(chunk)
if chunk_usage is not None:
turn_usage = chunk_usage
if turn_usage is not None:
self._total_usage.accumulate(turn_usage)
assistant_turn = (
types.Content(role=state.model_role, parts=state.model_parts)
if state.model_parts
else None
)
return ProviderTurn(
assistant_text=state.assistant_text,
tool_calls=state.tool_calls,
assistant_turn=assistant_turn,
)
def append_tool_results(
self,
turn: ProviderTurn,
executed_tool_calls: list[ExecutedToolCall],
) -> None:
model_content = turn.assistant_turn
if not isinstance(model_content, types.Content) or not model_content.parts:
raise ValueError(
"Gemini step is missing model content. Cannot append tool results without the original model turn."
)
self._contents.append(model_content)
tool_result_parts: List[types.Part] = []
for executed in executed_tool_calls:
tool_result_parts.append(
types.Part.from_function_response(
name=executed.tool_call.name,
response=executed.result.result,
)
)
self._contents.append(types.Content(role="tool", parts=tool_result_parts))
async def close(self) -> None:
u = self._total_usage
model_name = _get_gemini_api_model_name(self._model)
pricing = MODEL_PRICING.get(model_name)
cost_str = f" cost=${u.cost(pricing):.4f}" if pricing else ""
cache_hit_rate_str = f" cache_hit_rate={u.cache_hit_rate_percent():.2f}%"
print(
f"[TOKEN USAGE] provider=gemini model={model_name} | "
f"input={u.input} output={u.output} "
f"cache_read={u.cache_read} cache_write={u.cache_write} "
f"total={u.total}{cache_hit_rate_str}{cost_str}"
)
================================================
FILE: backend/agent/providers/openai.py
================================================
# pyright: reportUnknownVariableType=false
import copy
import json
import uuid
from dataclasses import dataclass, field
from typing import Any, Dict, List
from openai import AsyncOpenAI
from openai.types.chat import ChatCompletionMessageParam
from agent.providers.base import (
EventSink,
ExecutedToolCall,
ProviderSession,
ProviderTurn,
StreamEvent,
)
from agent.providers.pricing import MODEL_PRICING
from agent.providers.token_usage import TokenUsage
from agent.state import ensure_str
from agent.tools import CanonicalToolDefinition, ToolCall, parse_json_arguments
from config import IS_DEBUG_ENABLED
from fs_logging.openai_turn_inputs import OpenAITurnInputLogger
from llm import Llm, get_openai_api_name, get_openai_reasoning_effort
def _convert_message_to_responses_input(
message: ChatCompletionMessageParam,
) -> Dict[str, Any]:
role = message.get("role", "user")
content = message.get("content", "")
if isinstance(content, str):
return {"role": role, "content": content}
parts: List[Dict[str, Any]] = []
if isinstance(content, list):
for part in content:
if not isinstance(part, dict):
continue
if part.get("type") == "text":
parts.append({"type": "input_text", "text": part.get("text", "")})
elif part.get("type") == "image_url":
image_url = part.get("image_url", {})
parts.append(
{
"type": "input_image",
"image_url": image_url.get("url", ""),
"detail": image_url.get("detail", "high"),
}
)
return {"role": role, "content": parts}
def _get_event_attr(event: Any, key: str, default: Any = None) -> Any:
if hasattr(event, key):
return getattr(event, key)
if isinstance(event, dict):
return event.get(key, default)
return default
def _copy_schema(schema: Dict[str, Any]) -> Dict[str, Any]:
return copy.deepcopy(schema)
def _nullable_type(type_value: Any) -> Any:
if isinstance(type_value, list):
if "null" not in type_value:
return [*type_value, "null"]
return type_value
if isinstance(type_value, str):
return [type_value, "null"]
return type_value
def _make_responses_schema_strict(schema: Dict[str, Any]) -> Dict[str, Any]:
schema_copy: Dict[str, Any] = _copy_schema(schema)
def transform(node: Dict[str, Any], in_object_property: bool = False) -> None:
node_type = node.get("type")
if node_type == "object":
node["additionalProperties"] = False
properties = node.get("properties") or {}
if isinstance(properties, dict):
node["required"] = list(properties.keys())
for prop in properties.values():
if isinstance(prop, dict):
transform(prop, in_object_property=True)
return
if node_type == "array":
if in_object_property:
node["type"] = _nullable_type(node_type)
items = node.get("items")
if isinstance(items, dict):
transform(items, in_object_property=False)
return
if in_object_property and node_type is not None:
node["type"] = _nullable_type(node_type)
transform(schema_copy, in_object_property=False)
return schema_copy
def serialize_openai_tools(
tools: List[CanonicalToolDefinition],
) -> List[Dict[str, Any]]:
serialized: List[Dict[str, Any]] = []
for tool in tools:
schema = _make_responses_schema_strict(tool.parameters)
serialized.append(
{
"type": "function",
"name": tool.name,
"description": tool.description,
"parameters": schema,
"strict": True,
}
)
return serialized
@dataclass
class OpenAIResponsesParseState:
assistant_text: str = ""
tool_calls: Dict[str, Dict[str, Any]] = field(default_factory=dict)
item_to_call_id: Dict[str, str] = field(default_factory=dict)
output_items_by_index: Dict[int, Dict[str, Any]] = field(default_factory=dict)
saw_reasoning_summary_text_delta: bool = False
last_emitted_reasoning_summary_part: str = ""
turn_usage: TokenUsage | None = None
def _extract_openai_usage(response: Any) -> TokenUsage:
"""Extract unified token usage from an OpenAI Responses ``response.completed`` event.
OpenAI includes cached tokens inside ``input_tokens``, so they are subtracted
to get the non-cached input count.
"""
usage = _get_event_attr(response, "usage")
if usage is None:
return TokenUsage()
input_tokens = _get_event_attr(usage, "input_tokens", 0) or 0
output_tokens = _get_event_attr(usage, "output_tokens", 0) or 0
total_tokens = _get_event_attr(usage, "total_tokens", 0) or 0
details = _get_event_attr(usage, "input_tokens_details") or {}
cached_tokens = _get_event_attr(details, "cached_tokens", 0) or 0
return TokenUsage(
input=input_tokens - cached_tokens,
output=output_tokens,
cache_read=cached_tokens,
cache_write=0,
total=total_tokens,
)
async def parse_event(
event: Any,
state: OpenAIResponsesParseState,
on_event: EventSink,
) -> None:
event_type = _get_event_attr(event, "type")
if event_type in (
"response.created",
"response.completed",
"response.done",
"response.output_item.done",
):
if event_type == "response.completed":
response = _get_event_attr(event, "response")
if response:
state.turn_usage = _extract_openai_usage(response)
if event_type == "response.output_item.done":
output_index = _get_event_attr(event, "output_index")
item = _get_event_attr(event, "item")
if isinstance(output_index, int) and item:
state.output_items_by_index[output_index] = item
return
if event_type == "response.output_text.delta":
delta = _get_event_attr(event, "delta", "")
if delta:
state.assistant_text += delta
await on_event(StreamEvent(type="assistant_delta", text=delta))
return
if event_type in (
"response.reasoning_text.delta",
"response.reasoning_summary_text.delta",
):
delta = _get_event_attr(event, "delta", "")
if delta:
if event_type == "response.reasoning_summary_text.delta":
state.saw_reasoning_summary_text_delta = True
await on_event(StreamEvent(type="thinking_delta", text=delta))
return
if event_type in (
"response.reasoning_summary_part.added",
"response.reasoning_summary_part.done",
):
if state.saw_reasoning_summary_text_delta:
return
part = _get_event_attr(event, "part") or {}
text = _get_event_attr(part, "text", "")
if text and text != state.last_emitted_reasoning_summary_part:
state.last_emitted_reasoning_summary_part = text
await on_event(StreamEvent(type="thinking_delta", text=text))
return
if event_type == "response.output_item.added":
item = _get_event_attr(event, "item")
item_type = _get_event_attr(item, "type") if item else None
output_index = _get_event_attr(event, "output_index")
if isinstance(output_index, int) and item:
state.output_items_by_index.setdefault(output_index, item)
if item and item_type in ("function_call", "custom_tool_call"):
item_id = _get_event_attr(item, "id")
call_id = _get_event_attr(item, "call_id") or item_id
if item_id and call_id:
state.item_to_call_id[item_id] = call_id
if call_id:
if item_id and item_id in state.tool_calls and item_id != call_id:
existing = state.tool_calls.pop(item_id)
state.tool_calls[call_id] = {
**existing,
"id": call_id,
}
args_value = _get_event_attr(item, "arguments")
if args_value is None and item_type == "custom_tool_call":
args_value = _get_event_attr(item, "input")
state.tool_calls.setdefault(
call_id,
{
"id": call_id,
"name": _get_event_attr(item, "name"),
"arguments": args_value or "",
},
)
if args_value:
await on_event(
StreamEvent(
type="tool_call_delta",
tool_call_id=call_id,
tool_name=_get_event_attr(item, "name"),
tool_arguments=args_value,
)
)
return
if event_type in (
"response.function_call_arguments.delta",
"response.mcp_call_arguments.delta",
"response.custom_tool_call_input.delta",
):
item_id = _get_event_attr(event, "item_id")
call_id = _get_event_attr(event, "call_id")
if call_id and item_id:
state.item_to_call_id[item_id] = call_id
if not call_id:
call_id = state.item_to_call_id.get(item_id) if item_id else None
if not call_id and item_id:
call_id = item_id
if not call_id:
return
entry = state.tool_calls.setdefault(
call_id,
{
"id": call_id,
"name": _get_event_attr(event, "name"),
"arguments": "",
},
)
delta_value = _get_event_attr(event, "delta")
if delta_value is None:
delta_value = _get_event_attr(event, "input")
entry["arguments"] += ensure_str(delta_value)
await on_event(
StreamEvent(
type="tool_call_delta",
tool_call_id=call_id,
tool_name=entry.get("name"),
tool_arguments=entry.get("arguments"),
)
)
return
if event_type not in (
"response.function_call_arguments.done",
"response.mcp_call_arguments.done",
"response.custom_tool_call_input.done",
):
return
item_id = _get_event_attr(event, "item_id")
call_id = _get_event_attr(event, "call_id")
if call_id and item_id:
state.item_to_call_id[item_id] = call_id
if not call_id:
call_id = state.item_to_call_id.get(item_id) if item_id else None
if not call_id and item_id:
call_id = item_id
if not call_id:
return
entry = state.tool_calls.setdefault(
call_id,
{
"id": call_id,
"name": _get_event_attr(event, "name"),
"arguments": "",
},
)
final_value = _get_event_attr(event, "arguments")
if final_value is None:
final_value = _get_event_attr(event, "input")
if final_value is None:
final_value = entry["arguments"]
entry["arguments"] = final_value
if _get_event_attr(event, "name"):
entry["name"] = _get_event_attr(event, "name")
await on_event(
StreamEvent(
type="tool_call_delta",
tool_call_id=call_id,
tool_name=entry.get("name"),
tool_arguments=entry.get("arguments"),
)
)
output_index = _get_event_attr(event, "output_index")
if (
item_id
and isinstance(output_index, int)
and isinstance(state.output_items_by_index.get(output_index), dict)
):
state.output_items_by_index[output_index] = {
**state.output_items_by_index[output_index],
"arguments": entry["arguments"],
"call_id": call_id,
"name": entry.get("name"),
}
def _build_provider_turn(state: OpenAIResponsesParseState) -> ProviderTurn:
output_items = [
state.output_items_by_index[idx]
for idx in sorted(state.output_items_by_index.keys())
if state.output_items_by_index.get(idx)
]
tool_items = [
item
for item in output_items
if isinstance(item, dict)
and item.get("type") in ("function_call", "custom_tool_call")
]
tool_calls: List[ToolCall] = []
if tool_items:
for item in tool_items:
raw_args = item.get("arguments")
if raw_args is None and item.get("type") == "custom_tool_call":
raw_args = item.get("input")
args, error = parse_json_arguments(raw_args)
if error:
args = {"INVALID_JSON": ensure_str(raw_args)}
call_id = item.get("call_id") or item.get("id")
tool_calls.append(
ToolCall(
id=call_id or f"call-{uuid.uuid4().hex[:6]}",
name=item.get("name") or "unknown_tool",
arguments=args,
)
)
else:
for entry in state.tool_calls.values():
args, error = parse_json_arguments(entry.get("arguments"))
if error:
args = {"INVALID_JSON": ensure_str(entry.get("arguments"))}
call_id = entry.get("id") or entry.get("call_id")
tool_calls.append(
ToolCall(
id=call_id or f"call-{uuid.uuid4().hex[:6]}",
name=entry.get("name") or "unknown_tool",
arguments=args,
)
)
assistant_turn: List[Dict[str, Any]] = output_items if tool_calls else []
return ProviderTurn(
assistant_text=state.assistant_text,
tool_calls=tool_calls,
assistant_turn=assistant_turn,
)
class OpenAIProviderSession(ProviderSession):
def __init__(
self,
client: AsyncOpenAI,
model: Llm,
prompt_messages: List[ChatCompletionMessageParam],
tools: List[Dict[str, Any]],
):
self._client = client
self._model = model
self._tools = tools
self._total_usage = TokenUsage()
self._turn_input_logger = OpenAITurnInputLogger(
model,
enabled=IS_DEBUG_ENABLED,
)
self._input_items: List[Dict[str, Any]] = [
_convert_message_to_responses_input(message) for message in prompt_messages
]
async def stream_turn(self, on_event: EventSink) -> ProviderTurn:
model_name = get_openai_api_name(self._model)
params: Dict[str, Any] = {
"model": model_name,
"input": self._input_items,
"tools": self._tools,
"tool_choice": "auto",
"stream": True,
"max_output_tokens": 50000,
}
if model_name == "gpt-5.4-2026-03-05":
params["prompt_cache_retention"] = "24h"
reasoning_effort = get_openai_reasoning_effort(self._model)
if reasoning_effort:
params["reasoning"] = {"effort": reasoning_effort, "summary": "auto"}
self._turn_input_logger.record_turn_input(
self._input_items,
request_payload=params,
)
state = OpenAIResponsesParseState()
stream = await self._client.responses.create(**params) # type: ignore
async for event in stream: # type: ignore
await parse_event(event, state, on_event)
if state.turn_usage is not None:
self._turn_input_logger.record_turn_usage(state.turn_usage)
self._total_usage.accumulate(state.turn_usage)
return _build_provider_turn(state)
def append_tool_results(
self,
turn: ProviderTurn,
executed_tool_calls: list[ExecutedToolCall],
) -> None:
assistant_output_items = turn.assistant_turn or []
if assistant_output_items:
self._input_items.extend(assistant_output_items)
tool_output_items: List[Dict[str, Any]] = []
for executed in executed_tool_calls:
tool_output_items.append(
{
"type": "function_call_output",
"call_id": executed.tool_call.id,
"output": json.dumps(executed.result.result),
}
)
self._input_items.extend(tool_output_items)
async def close(self) -> None:
u = self._total_usage
model_name = get_openai_api_name(self._model)
pricing = MODEL_PRICING.get(model_name)
cost_str = f" cost=${u.cost(pricing):.4f}" if pricing else ""
cache_hit_rate_str = f" cache_hit_rate={u.cache_hit_rate_percent():.2f}%"
print(
f"[TOKEN USAGE] provider=openai model={model_name} | "
f"input={u.input} output={u.output} "
f"cache_read={u.cache_read} cache_write={u.cache_write} "
f"total={u.total}{cache_hit_rate_str}{cost_str}"
)
report_path = self._turn_input_logger.write_html_report()
if report_path:
print(f"[OPENAI TURN INPUT] HTML report: {report_path}")
await self._client.close()
================================================
FILE: backend/agent/providers/pricing.py
================================================
from dataclasses import dataclass
from typing import Dict
@dataclass
class ModelPricing:
"""Per-million-token pricing in USD."""
input: float = 0.0
output: float = 0.0
cache_read: float = 0.0
cache_write: float = 0.0
# Pricing keyed by the API model name string sent to the provider.
MODEL_PRICING: Dict[str, ModelPricing] = {
# --- OpenAI ---
"gpt-4.1-2025-04-14": ModelPricing(
input=2.00, output=8.00, cache_read=0.50
),
"gpt-5.2-codex": ModelPricing(
input=1.75, output=14.00, cache_read=0.4375
),
"gpt-5.3-codex": ModelPricing(
input=1.75, output=14.00, cache_read=0.4375
),
"gpt-5.4-2026-03-05": ModelPricing(
input=2.50, output=15.00, cache_read=0.25
),
# --- Anthropic ---
"claude-sonnet-4-6": ModelPricing(
input=3.00, output=15.00, cache_read=0.30, cache_write=3.75
),
"claude-sonnet-4-5-20250929": ModelPricing(
input=3.00, output=15.00, cache_read=0.30, cache_write=3.75
),
"claude-opus-4-5-20251101": ModelPricing(
input=5.00, output=25.00, cache_read=0.50, cache_write=6.25
),
"claude-opus-4-6": ModelPricing(
input=5.00, output=25.00, cache_read=0.50, cache_write=6.25
),
# --- Gemini ---
"gemini-3-flash-preview": ModelPricing(
input=0.50, output=3.00, cache_read=0.05
),
"gemini-3-pro-preview": ModelPricing(
input=2.00, output=12.00, cache_read=0.20
),
"gemini-3.1-pro-preview": ModelPricing(
input=2.00, output=12.00, cache_read=0.20
),
}
================================================
FILE: backend/agent/providers/token_usage.py
================================================
from __future__ import annotations
from dataclasses import dataclass
from agent.providers.pricing import ModelPricing
@dataclass
class TokenUsage:
"""Unified token usage across all providers.
Log line example:
[TOKEN USAGE] provider=gemini model=... | input=1000 output=500
cache_read=200 cache_write=0 total=1700 cost=$0.0020
Fields:
input: Non-cached input tokens (billed at full input rate).
For providers whose API includes cached tokens in the
prompt count (OpenAI, Gemini), cached tokens are
subtracted so this is always *exclusive* of cache_read.
output: Output tokens including thinking/reasoning (billed at
output rate).
cache_read: Input tokens served from cache (billed at reduced rate).
cache_write: Input tokens written to cache (Anthropic only).
total: All tokens as reported by the provider API. Equals
input + cache_read + output (+ thinking for Gemini).
Total input sent to the model = input + cache_read + cache_write.
Cost = (input * input_rate + output * output_rate
+ cache_read * cache_read_rate + cache_write * cache_write_rate)
/ 1_000_000
"""
input: int = 0
output: int = 0
cache_read: int = 0
cache_write: int = 0
total: int = 0
def accumulate(self, other: TokenUsage) -> None:
self.input += other.input
self.output += other.output
self.cache_read += other.cache_read
self.cache_write += other.cache_write
self.total += other.total
def cost(self, pricing: ModelPricing) -> float:
"""Compute cost in USD using per-million-token rates."""
return (
self.input * pricing.input
+ self.output * pricing.output
+ self.cache_read * pricing.cache_read
+ self.cache_write * pricing.cache_write
) / 1_000_000
def total_input_tokens(self) -> int:
"""All input tokens, including non-cached, cache-read, and cache-write."""
return self.input + self.cache_read + self.cache_write
def cache_hit_rate_percent(self) -> float:
"""Percent of total input tokens served from cache."""
total_input = self.total_input_tokens()
if total_input == 0:
return 0.0
return (self.cache_read / total_input) * 100.0
================================================
FILE: backend/agent/providers/types.py
================================================
from agent.providers.base import (
EventSink,
ExecutedToolCall,
ProviderTurn,
StreamEvent,
)
# Backwards-compatible alias for older imports.
StepResult = ProviderTurn
__all__ = [
"EventSink",
"ExecutedToolCall",
"ProviderTurn",
"StepResult",
"StreamEvent",
]
================================================
FILE: backend/agent/runner.py
================================================
from agent.engine import AgentEngine
class Agent(AgentEngine):
pass
================================================
FILE: backend/agent/state.py
================================================
from dataclasses import dataclass
from typing import Any, List
from openai.types.chat import ChatCompletionMessageParam
from codegen.utils import extract_html_content
@dataclass
class AgentFileState:
path: str = "index.html"
content: str = ""
def ensure_str(value: Any) -> str:
if value is None:
return ""
return str(value)
def extract_text_content(message: ChatCompletionMessageParam) -> str:
content = message.get("content", "")
if isinstance(content, str):
return content
if isinstance(content, list):
for part in content:
if isinstance(part, dict) and part.get("type") == "text":
return ensure_str(part.get("text"))
return ""
def seed_file_state_from_messages(
file_state: AgentFileState,
prompt_messages: List[ChatCompletionMessageParam],
) -> None:
if file_state.content:
return
for message in reversed(prompt_messages):
if message.get("role") != "assistant":
continue
raw_text = extract_text_content(message)
if not raw_text:
continue
extracted = extract_html_content(raw_text)
file_state.content = extracted or raw_text
if not file_state.path:
file_state.path = "index.html"
return
if not prompt_messages:
return
system_message = prompt_messages[0]
if system_message.get("role") != "system":
return
system_text = extract_text_content(system_message)
markers = [
"Here is the code of the app:",
]
for marker in markers:
if marker not in system_text:
continue
raw_text = system_text.split(marker, 1)[1].strip()
extracted = extract_html_content(raw_text)
file_state.content = extracted or raw_text
if not file_state.path:
file_state.path = "index.html"
return
================================================
FILE: backend/agent/tools/__init__.py
================================================
from agent.tools.definitions import canonical_tool_definitions
from agent.tools.parsing import (
extract_content_from_args,
extract_path_from_args,
parse_json_arguments,
)
from agent.tools.runtime import AgentToolRuntime, AgentToolbox
from agent.tools.summaries import summarize_text, summarize_tool_input
from agent.tools.types import (
CanonicalToolDefinition,
ToolCall,
ToolExecutionResult,
)
__all__ = [
"AgentToolRuntime",
"AgentToolbox",
"CanonicalToolDefinition",
"ToolCall",
"ToolExecutionResult",
"canonical_tool_definitions",
"extract_content_from_args",
"extract_path_from_args",
"parse_json_arguments",
"summarize_text",
"summarize_tool_input",
]
================================================
FILE: backend/agent/tools/definitions.py
================================================
from typing import Any, Dict, List
from agent.tools.types import CanonicalToolDefinition
def _create_schema() -> Dict[str, Any]:
return {
"type": "object",
"properties": {
"path": {
"type": "string",
"description": "Path for the main HTML file. Use index.html if unsure.",
},
"content": {
"type": "string",
"description": "Full HTML for the single-file app.",
},
},
"required": ["content"],
}
def _edit_schema() -> Dict[str, Any]:
return {
"type": "object",
"properties": {
"path": {
"type": "string",
"description": "Path for the main HTML file.",
},
"old_text": {
"type": "string",
"description": "Exact text to replace. Must match the file contents.",
},
"new_text": {
"type": "string",
"description": "Replacement text.",
},
"count": {
"type": "integer",
"description": "How many occurrences to replace. Use -1 for all.",
},
"edits": {
"type": "array",
"items": {
"type": "object",
"properties": {
"old_text": {"type": "string"},
"new_text": {"type": "string"},
"count": {"type": "integer"},
},
"required": ["old_text", "new_text"],
},
},
},
}
def _image_schema() -> Dict[str, Any]:
return {
"type": "object",
"properties": {
"prompts": {
"type": "array",
"items": {
"type": "string",
"description": "Prompt describing a single image to generate.",
},
}
},
"required": ["prompts"],
}
def _remove_background_schema() -> Dict[str, Any]:
return {
"type": "object",
"properties": {
"image_urls": {
"type": "array",
"items": {
"type": "string",
"description": "URL of an image to remove the background from.",
},
},
},
"required": ["image_urls"],
}
def _retrieve_option_schema() -> Dict[str, Any]:
return {
"type": "object",
"properties": {
"option_number": {
"type": "integer",
"description": "1-based option number to retrieve (Option 1, Option 2, etc.).",
}
},
"required": ["option_number"],
}
def canonical_tool_definitions(
image_generation_enabled: bool = True,
) -> List[CanonicalToolDefinition]:
tools: List[CanonicalToolDefinition] = [
CanonicalToolDefinition(
name="create_file",
description=(
"Create the main HTML file for the app. Use exactly once to write the "
"full HTML. Returns a success message and file metadata."
),
parameters=_create_schema(),
),
CanonicalToolDefinition(
name="edit_file",
description=(
"Edit the main HTML file using exact string replacements. Do not "
"regenerate the entire file. Returns a success message plus edit "
"details, including a unified diff and first changed line."
),
parameters=_edit_schema(),
),
]
if image_generation_enabled:
tools.append(
CanonicalToolDefinition(
name="generate_images",
description=(
"Generate image URLs from prompts. Use to replace placeholder images. "
"You can pass multiple prompts at once."
),
parameters=_image_schema(),
)
)
tools.extend(
[
CanonicalToolDefinition(
name="remove_background",
description=(
"Remove the background from one or more images. You can pass multiple "
"image URLs at once. Returns URLs to the processed images with "
"transparent backgrounds."
),
parameters=_remove_background_schema(),
),
CanonicalToolDefinition(
name="retrieve_option",
description=(
"Retrieve the full HTML for a specific option (variant) so you can "
"reference it."
),
parameters=_retrieve_option_schema(),
),
]
)
return tools
================================================
FILE: backend/agent/tools/parsing.py
================================================
# pyright: reportUnknownVariableType=false
import json
from typing import Any, Dict, Optional, Tuple
from agent.state import ensure_str
def parse_json_arguments(raw_args: Any) -> Tuple[Dict[str, Any], Optional[str]]:
if isinstance(raw_args, dict):
return raw_args, None
if raw_args is None:
return {}, None
raw_text = ensure_str(raw_args).strip()
if not raw_text:
return {}, None
try:
return json.loads(raw_text), None
except json.JSONDecodeError as exc:
return {}, f"Invalid JSON arguments: {exc}"
def _strip_incomplete_escape(value: str) -> str:
if not value:
return value
trailing = 0
for ch in reversed(value):
if ch == "\\":
trailing += 1
else:
break
if trailing % 2 == 1:
return value[:-1]
return value
def _extract_partial_json_string(raw_text: str, key: str) -> Optional[str]:
if not raw_text:
return None
token = f'"{key}"'
idx = raw_text.find(token)
if idx == -1:
return None
colon = raw_text.find(":", idx + len(token))
if colon == -1:
return None
cursor = colon + 1
while cursor < len(raw_text) and raw_text[cursor].isspace():
cursor += 1
if cursor >= len(raw_text) or raw_text[cursor] != '"':
return None
start = cursor + 1
last_quote: Optional[int] = None
cursor = start
while cursor < len(raw_text):
if raw_text[cursor] == '"':
backslashes = 0
back = cursor - 1
while back >= start and raw_text[back] == "\\":
backslashes += 1
back -= 1
if backslashes % 2 == 0:
last_quote = cursor
cursor += 1
partial = raw_text[start:] if last_quote is None else raw_text[start:last_quote]
partial = _strip_incomplete_escape(partial)
if not partial:
return ""
try:
return json.loads(f'"{partial}"')
except Exception:
return (
partial.replace("\\n", "\n")
.replace("\\t", "\t")
.replace("\\r", "\r")
.replace('\\"', '"')
.replace("\\\\", "\\")
)
def extract_content_from_args(raw_args: Any) -> Optional[str]:
if isinstance(raw_args, dict):
content = raw_args.get("content")
if content is None:
return None
return ensure_str(content)
raw_text = ensure_str(raw_args)
return _extract_partial_json_string(raw_text, "content")
def extract_path_from_args(raw_args: Any) -> Optional[str]:
if isinstance(raw_args, dict):
path = raw_args.get("path")
return ensure_str(path) if path is not None else None
raw_text = ensure_str(raw_args)
return _extract_partial_json_string(raw_text, "path")
================================================
FILE: backend/agent/tools/runtime.py
================================================
# pyright: reportUnknownVariableType=false
import asyncio
import difflib
from typing import Any, Dict, List, Optional, Tuple, Union
from codegen.utils import extract_html_content
from config import REPLICATE_API_KEY
from image_generation.generation import process_tasks
from image_generation.replicate import remove_background
from agent.state import AgentFileState, ensure_str
from agent.tools.types import ToolCall, ToolExecutionResult
from agent.tools.summaries import summarize_text
class AgentToolRuntime:
def __init__(
self,
file_state: AgentFileState,
should_generate_images: bool,
openai_api_key: Optional[str],
openai_base_url: Optional[str],
option_codes: Optional[List[str]] = None,
):
self.file_state = file_state
self.should_generate_images = should_generate_images
self.openai_api_key = openai_api_key
self.openai_base_url = openai_base_url
self.option_codes = option_codes or []
async def execute(self, tool_call: ToolCall) -> ToolExecutionResult:
if "INVALID_JSON" in tool_call.arguments:
invalid_json = ensure_str(tool_call.arguments.get("INVALID_JSON"))
return ToolExecutionResult(
ok=False,
result={
"error": "Tool arguments were invalid JSON.",
"INVALID_JSON": invalid_json,
},
summary={"error": "Invalid JSON tool arguments"},
)
if tool_call.name == "create_file":
return self._create_file(tool_call.arguments)
if tool_call.name == "edit_file":
return self._edit_file(tool_call.arguments)
if tool_call.name == "generate_images":
return await self._generate_images(tool_call.arguments)
if tool_call.name == "remove_background":
return await self._remove_background(tool_call.arguments)
if tool_call.name == "retrieve_option":
return self._retrieve_option(tool_call.arguments)
return ToolExecutionResult(
ok=False,
result={"error": f"Unknown tool: {tool_call.name}"},
summary={"error": f"Unknown tool: {tool_call.name}"},
)
def _create_file(self, args: Dict[str, Any]) -> ToolExecutionResult:
path = ensure_str(args.get("path") or self.file_state.path or "index.html")
content = ensure_str(args.get("content"))
if not content:
return ToolExecutionResult(
ok=False,
result={"error": "create_file requires non-empty content"},
summary={"error": "Missing content"},
)
extracted = extract_html_content(content)
self.file_state.path = path
self.file_state.content = extracted or content
summary = {
"path": self.file_state.path,
"contentLength": len(self.file_state.content),
"preview": summarize_text(self.file_state.content, 320),
}
result = {
"content": f"Successfully created file at {self.file_state.path}.",
"details": {
"path": self.file_state.path,
"contentLength": len(self.file_state.content),
},
}
return ToolExecutionResult(
ok=True,
result=result,
summary=summary,
updated_content=self.file_state.content,
)
@staticmethod
def _generate_diff(old_content: str, new_content: str, path: str) -> Dict[str, Any]:
"""Generate a unified diff between old and new content."""
old_lines = old_content.splitlines(keepends=True)
new_lines = new_content.splitlines(keepends=True)
diff_lines = list(
difflib.unified_diff(old_lines, new_lines, fromfile=path, tofile=path)
)
diff_str = "".join(diff_lines)
first_changed_line: Optional[int] = None
for line in diff_lines:
if not line.startswith("@@"):
continue
try:
plus_part = line.split("+")[1].split("@@")[0].strip()
first_changed_line = int(plus_part.split(",")[0])
except (IndexError, ValueError):
pass
break
return {
"diff": diff_str,
"firstChangedLine": first_changed_line,
}
def _apply_single_edit(
self,
content: str,
old_text: str,
new_text: str,
count: Optional[int],
) -> Tuple[str, int]:
if old_text not in content:
return content, 0
if count is None:
replace_count = 1
elif count < 0:
replace_count = content.count(old_text)
else:
replace_count = count
updated = content.replace(old_text, new_text, replace_count)
return updated, min(replace_count, content.count(old_text))
def _edit_file(self, args: Dict[str, Any]) -> ToolExecutionResult:
if not self.file_state.content:
return ToolExecutionResult(
ok=False,
result={"error": "No file exists yet. Call create_file first."},
summary={"error": "No file to edit"},
)
edits = args.get("edits")
if not edits:
old_text = ensure_str(args.get("old_text"))
new_text = ensure_str(args.get("new_text"))
count = args.get("count")
edits = [{"old_text": old_text, "new_text": new_text, "count": count}]
if not isinstance(edits, list):
return ToolExecutionResult(
ok=False,
result={"error": "edits must be a list"},
summary={"error": "Invalid edits payload"},
)
content = self.file_state.content
original_content = content
summary_edits: List[Dict[str, Any]] = []
for edit in edits:
old_text = ensure_str(edit.get("old_text"))
new_text = ensure_str(edit.get("new_text"))
count = edit.get("count")
if not old_text:
return ToolExecutionResult(
ok=False,
result={"error": "edit_file requires old_text"},
summary={"error": "Missing old_text"},
)
content, replaced = self._apply_single_edit(content, old_text, new_text, count)
if replaced == 0:
return ToolExecutionResult(
ok=False,
result={"error": "old_text not found", "old_text": old_text},
summary={
"error": "old_text not found",
"old_text": summarize_text(old_text, 160),
},
)
summary_edits.append(
{
"old_text": summarize_text(old_text, 140),
"new_text": summarize_text(new_text, 140),
"replaced": replaced,
}
)
self.file_state.content = content
path = self.file_state.path or "index.html"
diff_info = self._generate_diff(original_content, content, path)
summary = {
"path": path,
"edits": summary_edits,
"contentLength": len(self.file_state.content),
"diff": diff_info["diff"],
"firstChangedLine": diff_info["firstChangedLine"],
}
result = {
"content": f"Successfully edited file at {path}.",
"details": {
"diff": diff_info["diff"],
"firstChangedLine": diff_info["firstChangedLine"],
},
}
return ToolExecutionResult(
ok=True,
result=result,
summary=summary,
updated_content=self.file_state.content,
)
async def _generate_images(self, args: Dict[str, Any]) -> ToolExecutionResult:
if not self.should_generate_images:
return ToolExecutionResult(
ok=False,
result={"error": "Image generation is disabled."},
summary={"error": "Image generation disabled"},
)
prompts = args.get("prompts") or []
if not isinstance(prompts, list) or not prompts:
return ToolExecutionResult(
ok=False,
result={"error": "generate_images requires a non-empty prompts list"},
summary={"error": "Missing prompts"},
)
cleaned = [prompt.strip() for prompt in prompts if isinstance(prompt, str)]
unique_prompts = list(dict.fromkeys([p for p in cleaned if p]))
if not unique_prompts:
return ToolExecutionResult(
ok=False,
result={"error": "No valid prompts provided"},
summary={"error": "No valid prompts"},
)
if REPLICATE_API_KEY:
model = "flux"
api_key = REPLICATE_API_KEY
base_url = None
else:
if not self.openai_api_key:
return ToolExecutionResult(
ok=False,
result={"error": "No API key available for image generation."},
summary={"error": "Missing image generation API key"},
)
model = "dalle3"
api_key = self.openai_api_key
base_url = self.openai_base_url
generated = await process_tasks(unique_prompts, api_key, base_url, model) # type: ignore
merged_results = {
prompt: url for prompt, url in zip(unique_prompts, generated)
}
summary_items = [
{
"prompt": prompt,
"url": url,
"status": "ok" if url else "error",
}
for prompt, url in merged_results.items()
]
result = {"images": merged_results}
summary = {"images": summary_items}
return ToolExecutionResult(ok=True, result=result, summary=summary)
async def _remove_background(self, args: Dict[str, Any]) -> ToolExecutionResult:
if not REPLICATE_API_KEY:
return ToolExecutionResult(
ok=False,
result={"error": "Background removal requires REPLICATE_API_KEY."},
summary={"error": "Missing Replicate API key"},
)
image_urls = args.get("image_urls") or []
if not isinstance(image_urls, list) or not image_urls:
return ToolExecutionResult(
ok=False,
result={
"error": "remove_background requires a non-empty image_urls list"
},
summary={"error": "Missing image_urls"},
)
cleaned = [url.strip() for url in image_urls if isinstance(url, str)]
unique_urls = list(dict.fromkeys([u for u in cleaned if u]))
if not unique_urls:
return ToolExecutionResult(
ok=False,
result={"error": "No valid image URLs provided"},
summary={"error": "No valid image_urls"},
)
batch_size = 20
raw_results: list[str | BaseException] = []
for i in range(0, len(unique_urls), batch_size):
batch = unique_urls[i : i + batch_size]
tasks = [remove_background(url, REPLICATE_API_KEY) for url in batch]
raw_results.extend(await asyncio.gather(*tasks, return_exceptions=True))
results: List[Dict[str, Any]] = []
for url, raw in zip(unique_urls, raw_results):
if isinstance(raw, BaseException):
print(f"Background removal failed for {url}: {raw}")
results.append(
{"image_url": url, "result_url": None, "status": "error"}
)
else:
results.append(
{"image_url": url, "result_url": raw, "status": "ok"}
)
summary_items = [
{
"image_url": summarize_text(r["image_url"], 100),
"result_url": r["result_url"],
"status": r["status"],
}
for r in results
]
return ToolExecutionResult(
ok=True,
result={"images": results},
summary={"images": summary_items},
)
def _retrieve_option(self, args: Dict[str, Any]) -> ToolExecutionResult:
raw_option_number = args.get("option_number")
raw_index = args.get("index")
def coerce_int(value: Any) -> Optional[int]:
if value is None:
return None
try:
return int(value)
except (TypeError, ValueError):
return None
option_number = coerce_int(raw_option_number)
index = coerce_int(raw_index)
if option_number is None and index is None:
return ToolExecutionResult(
ok=False,
result={"error": "retrieve_option requires option_number"},
summary={"error": "Missing option_number"},
)
resolved_index = index if option_number is None else option_number - 1
if resolved_index is None:
return ToolExecutionResult(
ok=False,
result={"error": "Invalid option_number"},
summary={"error": "Invalid option_number"},
)
if resolved_index < 0 or resolved_index >= len(self.option_codes):
return ToolExecutionResult(
ok=False,
result={
"error": "Option index out of range",
"option_number": resolved_index + 1,
"available": len(self.option_codes),
},
summary={
"error": "Option index out of range",
"available": len(self.option_codes),
},
)
code = ensure_str(self.option_codes[resolved_index])
if not code.strip():
return ToolExecutionResult(
ok=False,
result={
"error": "Option code is empty or unavailable",
"option_number": resolved_index + 1,
},
summary={"error": "Option code unavailable"},
)
summary = {
"option_number": resolved_index + 1,
"contentLength": len(code),
"preview": summarize_text(code, 200),
}
result = {"option_number": resolved_index + 1, "code": code}
return ToolExecutionResult(ok=True, result=result, summary=summary)
# Backwards-compatible alias for older imports.
AgentToolbox = AgentToolRuntime
================================================
FILE: backend/agent/tools/summaries.py
================================================
# pyright: reportUnknownVariableType=false
from typing import Any, Dict
from agent.state import AgentFileState, ensure_str
from agent.tools.types import ToolCall
def summarize_text(value: str, limit: int = 240) -> str:
if len(value) <= limit:
return value
return value[:limit] + "..."
def summarize_tool_input(tool_call: ToolCall, file_state: AgentFileState) -> Dict[str, Any]:
args = tool_call.arguments or {}
if tool_call.name == "create_file":
content = ensure_str(args.get("content"))
return {
"path": args.get("path") or file_state.path,
"contentLength": len(content),
"preview": summarize_text(content, 200),
}
if tool_call.name == "edit_file":
edits = args.get("edits")
if not edits:
edits = [
{
"old_text": args.get("old_text"),
"new_text": args.get("new_text"),
"count": args.get("count"),
}
]
summary_edits = []
for edit in edits if isinstance(edits, list) else []:
summary_edits.append(
{
"old_text": summarize_text(ensure_str(edit.get("old_text")), 160),
"new_text": summarize_text(ensure_str(edit.get("new_text")), 160),
"count": edit.get("count"),
}
)
return {
"path": args.get("path") or file_state.path,
"edits": summary_edits,
}
if tool_call.name == "generate_images":
prompts = args.get("prompts") or []
if isinstance(prompts, list):
return {
"count": len(prompts),
"prompts": [ensure_str(p) for p in prompts],
}
if tool_call.name == "remove_background":
image_urls = args.get("image_urls") or []
if isinstance(image_urls, list):
return {
"count": len(image_urls),
"image_urls": [ensure_str(u) for u in image_urls],
}
return {"image_urls": []}
if tool_call.name == "retrieve_option":
return {
"option_number": args.get("option_number"),
"index": args.get("index"),
}
return args
================================================
FILE: backend/agent/tools/types.py
================================================
from dataclasses import dataclass
from typing import Any, Dict, Optional
@dataclass(frozen=True)
class ToolCall:
id: str
name: str
arguments: Dict[str, Any]
@dataclass
class ToolExecutionResult:
ok: bool
result: Dict[str, Any]
summary: Dict[str, Any]
updated_content: Optional[str] = None
@dataclass(frozen=True)
class CanonicalToolDefinition:
name: str
description: str
parameters: Dict[str, Any]
================================================
FILE: backend/codegen/__init__.py
================================================
================================================
FILE: backend/codegen/test_utils.py
================================================
import unittest
from codegen.utils import extract_html_content
class TestUtils(unittest.TestCase):
def test_extract_html_content_with_html_tags(self):
text = "<html><body><p>Hello, World!</p></body></html>"
expected = "<html><body><p>Hello, World!</p></body></html>"
result = extract_html_content(text)
self.assertEqual(result, expected)
def test_extract_html_content_without_html_tags(self):
text = "No HTML content here."
expected = "No HTML content here."
result = extract_html_content(text)
self.assertEqual(result, expected)
def test_extract_html_content_with_partial_html_tags(self):
text = "<html><body><p>Hello, World!</p></body>"
expected = "<html><body><p>Hello, World!</p></body>"
result = extract_html_content(text)
self.assertEqual(result, expected)
def test_extract_html_content_with_multiple_html_tags(self):
text = "<html><body><p>First</p></body></html> Some text <html><body><p>Second</p></body></html>"
expected = "<html><body><p>First</p></body></html>"
result = extract_html_content(text)
self.assertEqual(result, expected)
## The following are tests based on actual LLM outputs
def test_extract_html_content_some_explanation_before(self):
text = """Got it! You want the song list to be displayed horizontally. I'll update the code to ensure that the song list is displayed in a horizontal layout.
Here's the updated code:
<html lang="en"><head></head><body class="bg-black text-white"></body></html>"""
expected = '<html lang="en"><head></head><body class="bg-black text-white"></body></html>'
result = extract_html_content(text)
self.assertEqual(result, expected)
def test_markdown_tags(self):
text = "```html<head></head>```"
expected = "```html<head></head>```"
result = extract_html_content(text)
self.assertEqual(result, expected)
def test_doctype_text(self):
text = '<!DOCTYPE html><html lang="en"><head></head><body></body></html>'
expected = '<html lang="en"><head></head><body></body></html>'
result = extract_html_content(text)
self.assertEqual(result, expected)
if __name__ == "__main__":
unittest.main()
================================================
FILE: backend/codegen/utils.py
================================================
import re
def extract_html_content(text: str) -> str:
file_match = re.search(
r"<file\s+path=\"[^\"]+\">\s*(.*?)\s*</file>",
text,
re.DOTALL | re.IGNORECASE,
)
if file_match:
return extract_html_content(file_match.group(1).strip())
# First, strip markdown code fences if present
text = re.sub(r'^```html?\s*\n?', '', text, flags=re.MULTILINE)
text = re.sub(r'\n?```\s*$', '', text, flags=re.MULTILINE)
# Try to find DOCTYPE + html tags together
match_with_doctype = re.search(
r"(<!DOCTYPE\s+html[^>]*>.*?<html.*?>.*?</html>)", text, re.DOTALL | re.IGNORECASE
)
if match_with_doctype:
return match_with_doctype.group(1)
# Fall back to just <html> tags
match = re.search(r"(<html.*?>.*?</html>)", text, re.DOTALL)
if match:
return match.group(1)
else:
# Otherwise, we just send the previous HTML over
print(
"[HTML Extraction] No <html> tags found in the generated content"
)
return text
================================================
FILE: backend/config.py
================================================
import os
NUM_VARIANTS = 4
NUM_VARIANTS_VIDEO = 2
# LLM-related
OPENAI_API_KEY = os.environ.get("OPENAI_API_KEY", None)
ANTHROPIC_API_KEY = os.environ.get("ANTHROPIC_API_KEY", None)
GEMINI_API_KEY = os.environ.get("GEMINI_API_KEY", None)
OPENAI_BASE_URL = os.environ.get("OPENAI_BASE_URL", None)
# Image generation (optional)
REPLICATE_API_KEY = os.environ.get("REPLICATE_API_KEY", None)
# Debugging-related
IS_DEBUG_ENABLED = bool(os.environ.get("IS_DEBUG_ENABLED", False))
DEBUG_DIR = os.environ.get("DEBUG_DIR", "")
# Set to True when running in production (on the hosted version)
# Used as a feature flag to enable or disable certain features
IS_PROD = os.environ.get("IS_PROD", False)
================================================
FILE: backend/custom_types.py
================================================
from typing import Literal
InputMode = Literal[
"image",
"video",
"text",
]
================================================
FILE: backend/debug/DebugFileWriter.py
================================================
import os
import logging
import uuid
from config import DEBUG_DIR, IS_DEBUG_ENABLED
class DebugFileWriter:
def __init__(self):
if not IS_DEBUG_ENABLED:
return
try:
self.debug_artifacts_path = os.path.expanduser(
f"{DEBUG_DIR}/{str(uuid.uuid4())}"
)
os.makedirs(self.debug_artifacts_path, exist_ok=True)
print(f"Debugging artifacts will be stored in: {self.debug_artifacts_path}")
except:
logging.error("Failed to create debug directory")
def write_to_file(self, filename: str, content: str) -> None:
try:
with open(os.path.join(self.debug_artifacts_path, filename), "w") as file:
file.write(content)
except Exception as e:
logging.error(f"Failed to write to file: {e}")
def extract_html_content(self, text: str) -> str:
return str(text.split("<html>")[-1].rsplit("</html>", 1)[0] + "</html>")
================================================
FILE: backend/debug/__init__.py
================================================
================================================
FILE: backend/evals/__init__.py
================================================
================================================
FILE: backend/evals/config.py
================================================
EVALS_DIR = "./evals_data"
================================================
FILE: backend/evals/core.py
================================================
from config import (
ANTHROPIC_API_KEY,
GEMINI_API_KEY,
OPENAI_API_KEY,
OPENAI_BASE_URL,
)
from llm import Llm, OPENAI_MODELS, ANTHROPIC_MODELS, GEMINI_MODELS
from agent.runner import Agent
from prompts.create.image import build_image_prompt_messages
from prompts.prompt_types import Stack
from openai.types.chat import ChatCompletionMessageParam
from typing import Any
async def generate_code_for_image(image_url: str, stack: Stack, model: Llm) -> str:
prompt_messages = build_image_prompt_messages(
image_data_urls=[image_url],
stack=stack,
text_prompt="",
image_generation_enabled=True,
)
async def send_message(
_: str,
__: str | None,
___: int,
____: dict[str, Any] | None = None,
_____: str | None = None,
) -> None:
# Evals do not stream tool/assistant messages to a frontend.
return None
if model in ANTHROPIC_MODELS and not ANTHROPIC_API_KEY:
raise Exception("Anthropic API key not found")
if model in GEMINI_MODELS and not GEMINI_API_KEY:
raise Exception("Gemini API key not found")
if model in OPENAI_MODELS and not OPENAI_API_KEY:
raise Exception("OpenAI API key not found")
print(f"[EVALS] Using agent runner for model: {model.value}")
runner = Agent(
send_message=send_message,
variant_index=0,
openai_api_key=OPENAI_API_KEY,
openai_base_url=OPENAI_BASE_URL,
anthropic_api_key=ANTHROPIC_API_KEY,
gemini_api_key=GEMINI_API_KEY,
should_generate_images=True,
initial_file_state=None,
option_codes=None,
)
return await runner.run(model, prompt_messages)
================================================
FILE: backend/evals/runner.py
================================================
from typing import Any, Awaitable, Callable, Coroutine, List, Optional, Tuple
import asyncio
import os
from datetime import datetime
import time
import inspect
from llm import Llm
from prompts.prompt_types import Stack
from .core import generate_code_for_image
from .utils import image_to_data_url
from .config import EVALS_DIR
MAX_EVAL_RETRIES = 2
def _resolve_eval_filenames(input_files: Optional[List[str]]) -> List[str]:
input_dir = EVALS_DIR + "/inputs"
if input_files and len(input_files) > 0:
return [os.path.basename(f) for f in input_files if f.endswith(".png")]
return [f for f in os.listdir(input_dir) if f.endswith(".png")]
def _output_html_filename(original_filename: str, attempt_idx: int) -> str:
return f"{os.path.splitext(original_filename)[0]}_{attempt_idx}.html"
def get_eval_output_subfolder(stack: Stack, model: str) -> str:
today = datetime.now().strftime("%b_%d_%Y")
output_dir = EVALS_DIR + "/outputs"
return os.path.join(output_dir, f"{today}_{model}_{stack}")
def count_pending_eval_tasks(
stack: Stack,
model: str,
input_files: Optional[List[str]] = None,
n: int = 1,
diff_mode: bool = False,
) -> Tuple[int, int]:
evals = _resolve_eval_filenames(input_files)
if not diff_mode:
return len(evals) * n, 0
output_subfolder = get_eval_output_subfolder(stack=stack, model=model)
pending_tasks = 0
skipped_existing_tasks = 0
for original_filename in evals:
for n_idx in range(n):
output_filename = _output_html_filename(original_filename, n_idx)
output_path = os.path.join(output_subfolder, output_filename)
if os.path.exists(output_path):
skipped_existing_tasks += 1
else:
pending_tasks += 1
return pending_tasks, skipped_existing_tasks
async def generate_code_and_time(
image_url: str,
stack: Stack,
model: Llm,
original_input_filename: str,
attempt_idx: int,
) -> Tuple[str, int, Optional[str], Optional[float], Optional[Exception], int]:
"""
Generates code for an image, measures the time taken, and returns identifiers
along with success/failure status.
Returns a tuple:
(original_input_filename, attempt_idx, content, duration, error_object, retries_used)
content and duration are None if an error occurs during generation.
"""
retries_used = 0
while True:
start_time = time.perf_counter()
try:
content = await generate_code_for_image(
image_url=image_url, stack=stack, model=model
)
end_time = time.perf_counter()
duration = end_time - start_time
return (
original_input_filename,
attempt_idx,
content,
duration,
None,
retries_used,
)
except Exception as e:
if retries_used >= MAX_EVAL_RETRIES:
print(
f"Error during code generation for {original_input_filename} "
f"(attempt {attempt_idx}, retries exhausted): {e}"
)
return (
original_input_filename,
attempt_idx,
None,
None,
e,
retries_used,
)
retries_used += 1
print(
f"Retrying {original_input_filename} (attempt {attempt_idx}) "
f"{retries_used}/{MAX_EVAL_RETRIES} after error: {e}"
)
async def run_image_evals(
stack: Optional[Stack] = None,
model: Optional[str] = None,
n: int = 1,
input_files: Optional[List[str]] = None,
diff_mode: bool = False,
progress_callback: Optional[Callable[[dict[str, Any]], Any | Awaitable[Any]]] = None,
) -> List[str]:
INPUT_DIR = EVALS_DIR + "/inputs"
evals = _resolve_eval_filenames(input_files)
if not stack:
raise ValueError("No stack was provided")
if not model:
raise ValueError("No model was provided")
print("User selected stack:", stack)
print("User selected model:", model)
selected_model = Llm(model)
print(f"Running evals for {selected_model.value} model")
if input_files and len(input_files) > 0:
print(f"Running on {len(evals)} selected files")
else:
print(f"Running on all {len(evals)} files in {INPUT_DIR}")
output_subfolder = get_eval_output_subfolder(
stack=stack,
model=selected_model.value,
)
os.makedirs(output_subfolder, exist_ok=True)
task_coroutines: List[
Coroutine[
Any,
Any,
Tuple[str, int, Optional[str], Optional[float], Optional[Exception], int],
]
] = []
skipped_existing_tasks = 0
for original_filename in evals:
# Handle both full paths and relative filenames
if os.path.isabs(original_filename):
filepath = original_filename
original_filename = os.path.basename(original_filename)
else:
filepath = os.path.join(INPUT_DIR, original_filename)
data_url: Optional[str] = None
for n_idx in range(n):
output_filename = _output_html_filename(original_filename, n_idx)
output_path = os.path.join(output_subfolder, output_filename)
if diff_mode and os.path.exists(output_path):
skipped_existing_tasks += 1
continue
if data_url is None:
data_url = await image_to_data_url(filepath)
current_model_for_task = (
selected_model if n_idx == 0 else Llm.GPT_4_1_2025_04_14
)
coro = generate_code_and_time(
image_url=data_url,
stack=stack,
model=current_model_for_task,
original_input_filename=original_filename,
attempt_idx=n_idx,
)
task_coroutines.append(coro)
if diff_mode and skipped_existing_tasks > 0:
print(
f"Diff mode: skipping {skipped_existing_tasks} existing outputs for "
f"{selected_model.value}"
)
print(f"Processing {len(task_coroutines)} tasks...")
total_tasks = len(task_coroutines)
completed_tasks = 0
output_files: List[str] = []
timing_data: List[str] = []
failed_tasks_log: List[str] = []
async def emit_progress(event: dict[str, Any]) -> None:
if progress_callback is None:
return
maybe_awaitable = progress_callback(event)
if inspect.isawaitable(maybe_awaitable):
await maybe_awaitable
for future in asyncio.as_completed(task_coroutines):
try:
(
task_orig_fn,
task_attempt_idx,
generated_content,
time_taken,
error_obj,
retries_used,
) = await future
completed_tasks += 1
output_html_filename_base = os.path.splitext(task_orig_fn)[0]
final_output_html_filename = (
f"{output_html_filename_base}_{task_attempt_idx}.html"
)
output_html_filepath = os.path.join(
output_subfolder, final_output_html_filename
)
if error_obj is not None:
failed_tasks_log.append(
f"Input: {task_orig_fn}, Attempt: {task_attempt_idx}, OutputFile: "
f"{final_output_html_filename}, Retries: {retries_used}, "
f"Error: Generation failed - {str(error_obj)}"
)
await emit_progress(
{
"type": "task_complete",
"completed_tasks": completed_tasks,
"total_tasks": total_tasks,
"input_file": task_orig_fn,
"attempt_idx": task_attempt_idx,
"success": False,
"error": str(error_obj),
"retries_used": retries_used,
}
)
elif generated_content is not None and time_taken is not None:
try:
with open(output_html_filepath, "w") as file:
file.write(generated_content)
timing_data.append(
f"{final_output_html_filename}: {time_taken:.2f} seconds"
)
output_files.append(final_output_html_filename)
print(
f"Successfully processed and wrote {final_output_html_filename}"
)
await emit_progress(
{
"type": "task_complete",
"completed_tasks": completed_tasks,
"total_tasks": total_tasks,
"input_file": task_orig_fn,
"attempt_idx": task_attempt_idx,
"success": True,
"output_file": final_output_html_filename,
"duration_seconds": time_taken,
"retries_used": retries_used,
}
)
except Exception as e_write:
failed_tasks_log.append(
f"Input: {task_orig_fn}, Attempt: {task_attempt_idx}, OutputFile: {final_output_html_filename}, Error: Writing to file failed - {str(e_write)}"
)
await emit_progress(
{
"type": "task_complete",
"completed_tasks": completed_tasks,
"total_tasks": total_tasks,
"input_file": task_orig_fn,
"attempt_idx": task_attempt_idx,
"success": False,
"error": str(e_write),
}
)
else:
failed_tasks_log.append(
f"Input: {task_orig_fn}, Attempt: {task_attempt_idx}, OutputFile: {final_output_html_filename}, Error: Unknown issue - content or time_taken is None without explicit error."
)
await emit_progress(
{
"type": "task_complete",
"completed_tasks": completed_tasks,
"total_tasks": total_tasks,
"input_file": task_orig_fn,
"attempt_idx": task_attempt_idx,
"success": False,
"error": "Unknown issue during task processing.",
}
)
except Exception as e_as_completed:
print(f"A task in as_completed failed unexpectedly: {e_as_completed}")
failed_tasks_log.append(
f"Critical Error: A task processing failed - {str(e_as_completed)}"
)
completed_tasks += 1
await emit_progress(
{
"type": "task_complete",
"completed_tasks": completed_tasks,
"total_tasks": total_tasks,
"input_file": "unknown",
"attempt_idx": -1,
"success": False,
"error": str(e_as_completed),
}
)
# Write timing data for successful tasks
if timing_data:
timing_file_path = os.path.join(output_subfolder, "generation_times.txt")
try:
is_new_or_empty_file = (
not os.path.exists(timing_file_path)
or os.path.getsize(timing_file_path) == 0
)
with open(timing_file_path, "a") as file:
if is_new_or_empty_file:
file.write(f"Model: {selected_model.value}\n")
elif timing_data:
file.write("\n")
file.write("\n".join(timing_data))
print(f"Timing data saved to {timing_file_path}")
except Exception as e:
print(f"Error writing timing file {timing_file_path}: {e}")
# Write log for failed tasks
if failed_tasks_log:
failed_log_path = os.path.join(output_subfolder, "failed_tasks.txt")
try:
with open(failed_log_path, "w") as file:
file.write("\n".join(failed_tasks_log))
print(f"Failed tasks log saved to {failed_log_path}")
except Exception as e:
print(f"Error writing failed tasks log {failed_log_path}: {e}")
return output_files
================================================
FILE: backend/evals/utils.py
================================================
import base64
async def image_to_data_url(filepath: str):
with open(filepath, "rb") as image_file:
encoded_string = base64.b64encode(image_file.read()).decode()
return f"data:image/png;base64,{encoded_string}"
================================================
FILE: backend/fs_logging/__init__.py
================================================
================================================
FILE: backend/fs_logging/openai_input_compare.py
================================================
import json
from dataclasses import dataclass
from typing import Any, TypeAlias, cast
from fs_logging.openai_input_formatting import (
summarize_responses_input_item,
to_serializable,
)
JSONScalar: TypeAlias = None | bool | int | float | str
JSONValue: TypeAlias = JSONScalar | list["JSONValue"] | dict[str, "JSONValue"]
@dataclass(frozen=True)
class OpenAIInputDifference:
item_index: int
path: str
left_summary: str
right_summary: str
left_value: Any
right_value: Any
@dataclass(frozen=True)
class OpenAIInputComparison:
common_prefix_items: int
left_item_count: int
right_item_count: int
difference: OpenAIInputDifference | None
def _extract_input_items(payload: Any) -> list[JSONValue]:
serialized = cast(JSONValue, to_serializable(payload))
if isinstance(serialized, list):
return serialized
if isinstance(serialized, dict):
serialized_dict = cast(dict[str, JSONValue], serialized)
input_items = serialized_dict.get("input")
if isinstance(input_items, list):
return cast(list[JSONValue], input_items)
raise ValueError("Expected a raw input array or a request payload with an 'input' list")
def _as_json_dict(value: JSONValue) -> dict[str, JSONValue]:
return cast(dict[str, JSONValue], value)
def _as_json_list(value: JSONValue) -> list[JSONValue]:
return cast(list[JSONValue], value)
def _append_dict_path(path: str, key: str) -> str:
if not path:
return key
return f"{path}.{key}"
def _append_list_path(path: str, index: int) -> str:
return f"{path}[{index}]"
def _find_first_value_difference(
left: JSONValue,
right: JSONValue,
path: str = "",
) -> tuple[str, JSONValue, JSONValue] | None:
if type(left) is not type(right):
return path, left, right
if isinstance(left, dict):
left_dict = _as_json_dict(left)
right_dict = _as_json_dict(right)
left_keys = list(left_dict.keys())
right_keys = list(right_dict.keys())
for index in range(min(len(left_keys), len(right_keys))):
left_key = left_keys[index]
right_key = right_keys[index]
if left_key != right_key:
key_path = _append_dict_path(path, left_key)
return key_path, left, right
if len(left_keys) != len(right_keys):
extra_key = (
left_keys[len(right_keys)]
if len(left_keys) > len(right_keys)
else right_keys[len(left_keys)]
)
key_path = _append_dict_path(path, extra_key)
left_value = left_dict.get(extra_key)
right_value = right_dict.get(extra_key)
return key_path, cast(JSONValue, left_value), cast(JSONValue, right_value)
for key in left_keys:
nested = _find_first_value_difference(
left_dict[key],
right_dict[key],
_append_dict_path(path, key),
)
if nested is not None:
return nested
return None
if isinstance(left, list):
left_list = _as_json_list(left)
right_list = _as_json_list(right)
for index in range(min(len(left_list), len(right_list))):
nested = _find_first_value_difference(
left_list[index],
right_list[index],
_append_list_path(path, index),
)
if nested is not None:
return nested
if len(left_list) != len(right_list):
index = min(len(left_list), len(right_list))
item_path = _append_list_path(path, index)
left_value = left_list[index] if index < len(left_list) else None
right_value = right_list[index] if index < len(right_list) else None
return item_path, left_value, right_value
return None
if left != right:
return path, left, right
return None
def compare_openai_inputs(
left_payload: Any,
right_payload: Any,
) -> OpenAIInputComparison:
left_items = _extract_input_items(left_payload)
right_items = _extract_input_items(right_payload)
common_prefix_items = 0
for index in range(min(len(left_items), len(right_items))):
left_item = left_items[index]
right_item = right_items[index]
if left_item == right_item:
common_prefix_items += 1
continue
nested_difference = _find_first_value_difference(left_item, right_item)
nested_path = "" if nested_difference is None else nested_difference[0]
path = f"input[{index}]"
if nested_path:
if nested_path.startswith("["):
path = f"{path}{nested_path}"
else:
path = f"{path}.{nested_path}"
left_value = left_item if nested_difference is None else nested_difference[1]
right_value = right_item if nested_difference is None else nested_difference[2]
return OpenAIInputComparison(
common_prefix_items=common_prefix_items,
left_item_count=len(left_items),
right_item_count=len(right_items),
difference=OpenAIInputDifference(
item_index=index,
path=path,
left_summary=summarize_responses_input_item(index, left_item),
right_summary=summarize_responses_input_item(index, right_item),
left_value=left_value,
right_value=right_value,
),
)
if len(left_items) != len(right_items):
index = min(len(left_items), len(right_items))
left_item = left_items[index] if index < len(left_items) else None
right_item = right_items[index] if index < len(right_items) else None
return OpenAIInputComparison(
common_prefix_items=common_prefix_items,
left_item_count=len(left_items),
right_item_count=len(right_items),
difference=OpenAIInputDifference(
item_index=index,
path=f"input[{index}]",
left_summary=(
summarize_responses_input_item(index, left_item)
if left_item is not None
else f"{index:02d} <missing>"
),
right_summary=(
summarize_responses_input_item(index, right_item)
if right_item is not None
else f"{index:02d} <missing>"
),
left_value=left_item,
right_value=right_item,
),
)
return OpenAIInputComparison(
common_prefix_items=common_prefix_items,
left_item_count=len(left_items),
right_item_count=len(right_items),
difference=None,
)
def format_openai_input_comparison(comparison: OpenAIInputComparison) -> str:
lines = [
"OpenAI input comparison",
f"common_prefix_items={comparison.common_prefix_items}",
f"left_item_count={comparison.left_item_count}",
f"right_item_count={comparison.right_item_count}",
]
difference = comparison.difference
if difference is None:
lines.append("difference=none")
return "\n".join(lines)
lines.extend(
[
f"first_different_item_index={difference.item_index}",
f"first_different_path={difference.path}",
f"left_summary={difference.left_summary}",
f"right_summary={difference.right_summary}",
"left_value=" + json.dumps(difference.left_value, indent=2, ensure_ascii=False),
"right_value=" + json.dumps(
difference.right_value,
indent=2,
ensure_ascii=False,
),
]
)
return "\n".join(lines)
def compare_openai_input_json_strings(
left_json: str,
right_json: str,
) -> OpenAIInputComparison:
left_payload = json.loads(left_json)
right_payload = json.loads(right_json)
return compare_openai_inputs(left_payload, right_payload)
================================================
FILE: backend/fs_logging/openai_input_formatting.py
================================================
# pyright: reportUnknownVariableType=false
import json
from typing import Any
from agent.state import ensure_str
def truncate_for_log(value: Any, max_len: int = 120) -> str:
text = ensure_str(value).replace("\n", "\\n")
if len(text) <= max_len:
return text
return f"{text[:max_len]}..."
def as_dict(value: Any) -> dict[str, Any] | None:
if isinstance(value, dict):
return value
model_dump = getattr(value, "model_dump", None)
if callable(model_dump):
dumped = model_dump()
if isinstance(dumped, dict):
return dumped
to_dict = getattr(value, "to_dict", None)
if callable(to_dict):
dumped = to_dict()
if isinstance(dumped, dict):
return dumped
dict_method = getattr(value, "dict", None)
if callable(dict_method):
dumped = dict_method()
if isinstance(dumped, dict):
return dumped
raw_dict = getattr(value, "__dict__", None)
if isinstance(raw_dict, dict):
normalized = {k: v for k, v in raw_dict.items() if not k.startswith("_")}
if normalized:
return normalized
return None
def to_serializable(value: Any) -> Any:
if value is None or isinstance(value, (bool, int, float, str)):
return value
if isinstance(value, dict):
return {ensure_str(k): to_serializable(v) for k, v in value.items()}
if isinstance(value, (list, tuple)):
return [to_serializable(v) for v in value]
value_as_dict = as_dict(value)
if value_as_dict is not None:
return to_serializable(value_as_dict)
return ensure_str(value)
def summarize_content_part(part: Any) -> str:
part_dict = as_dict(part)
if part_dict is None:
return f"{type(part).__name__}"
part_type = part_dict.get("type", "unknown")
if part_type in ("input_text", "text", "output_text", "summary_text"):
text = ensure_str(part_dict.get("text", ""))
return (
f"{part_type}(chars={len(text)} "
f"preview='{truncate_for_log(text, max_len=80)}')"
)
if part_type in ("input_image", "image_url"):
image_url_value: Any = part_dict.get("image_url", "")
detail: str | None = None
if isinstance(image_url_value, dict):
detail = ensure_str(image_url_value.get("detail", ""))
image_url_value = image_url_value.get("url", "")
else:
detail = ensure_str(part_dict.get("detail", ""))
url_text = ensure_str(image_url_value)
detail_text = detail or "-"
return (
f"{part_type}(detail={detail_text} "
f"url='{truncate_for_log(url_text, max_len=80)}')"
)
return f"{part_type}(keys={sorted(part_dict.keys())})"
def summarize_function_call_output_payload(output_text: str) -> str:
try:
parsed = json.loads(output_text)
except json.JSONDecodeError:
return (
f"output_chars={len(output_text)} "
f"preview='{truncate_for_log(output_text)}'"
)
if not isinstance(parsed, dict):
return (
f"output_type={type(parsed).__name__} "
f"preview='{truncate_for_log(parsed)}'"
)
if "error" in parsed:
error_text = ensure_str(parsed.get("error"))
return f"error='{truncate_for_log(error_text)}'"
summary_parts: list[str] = []
content_text = ensure_str(parsed.get("content"))
if content_text:
summary_parts.append(f"content='{truncate_for_log(content_text, max_len=80)}'")
details = parsed.get("details")
if isinstance(details, dict):
path = ensure_str(details.get("path"))
diff_text = details.get("diff")
if (not path) and isinstance(diff_text, str) and diff_text:
for line in diff_text.splitlines():
if line.startswith("--- "):
path = line.removeprefix("--- ").strip()
break
if path:
summary_parts.append(f"path={path}")
edits = details.get("edits")
if isinstance(edits, list):
summary_parts.append(f"edits={len(edits)}")
content_length = details.get("contentLength")
if isinstance(content_length, int):
summary_parts.append(f"content_length={content_length}")
first_changed_line = details.get("firstChangedLine")
if isinstance(first_changed_line, int):
summary_parts.append(f"first_changed_line={first_changed_line}")
if isinstance(diff_text, str) and diff_text:
diff_lines = diff_text.count("\n")
summary_parts.append(f"diff_chars={len(diff_text)}")
summary_parts.append(f"diff_lines={diff_lines}")
if not summary_parts:
summary_parts.append(f"keys={sorted(parsed.keys())}")
return " ".join(summary_parts)
def summarize_responses_input_item(index: int, item: Any) -> str:
item_dict = as_dict(item)
if item_dict is None:
return f"{index:02d} item_type={type(item).__name__}"
if "role" in item_dict:
role = ensure_str(item_dict.get("role", "unknown"))
content = item_dict.get("content", "")
if isinstance(content, str):
return (
f"{index:02d} role={role} content=str chars={len(content)} "
f"preview='{truncate_for_log(content)}'"
)
if isinstance(content, list):
part_summaries = [summarize_content_part(part) for part in content]
return (
f"{index:02d} role={role} content_parts={len(content)} "
f"[{'; '.join(part_summaries)}]"
)
return f"{index:02d} role={role} content_type={type(content).__name__}"
item_type = ensure_str(item_dict.get("type", "unknown"))
if item_type in ("function_call", "custom_tool_call"):
raw_args = (
item_dict.get("input")
if item_type == "custom_tool_call"
else item_dict.get("arguments")
)
args_text = ensure_str(raw_args or "")
call_id = item_dict.get("call_id") or item_dict.get("id")
return (
f"{index:02d} type={item_type} name={item_dict.get('name')} "
f"call_id={call_id} args_chars={len(args_text)} "
f"preview='{truncate_for_log(args_text)}'"
)
if item_type == "function_call_output":
output_text = ensure_str(item_dict.get("output", ""))
return (
f"{index:02d} type=function_call_output call_id={item_dict.get('call_id')} "
f"{summarize_function_call_output_payload(output_text)}"
)
if item_type == "message":
role = ensure_str(item_dict.get("role", "unknown"))
content = item_dict.get("content", [])
if isinstance(content, list):
part_summaries = [summarize_content_part(part) for part in content]
return (
f"{index:02d} type=message role={role} parts={len(content)} "
f"[{'; '.join(part_summaries)}]"
)
return (
f"{index:02d} type=message role={role} "
f"content_type={type(content).__name__}"
)
if item_type == "reasoning":
summary = item_dict.get("summary")
if isinstance(summary, list):
summary_parts = [summarize_content_part(part) for part in summary]
return (
f"{index:02d} type=reasoning summary_parts={len(summary)} "
f"[{'; '.join(summary_parts)}]"
)
return f"{index:02d} type=reasoning summary_type={type(summary).__name__}"
return f"{index:02d} type={item_type} keys={sorted(item_dict.keys())}"
================================================
FILE: backend/fs_logging/openai_turn_inputs.py
================================================
# pyright: reportUnknownVariableType=false
import json
import os
import uuid
from dataclasses import dataclass, field
from datetime import datetime
from html import escape
from typing import Any, Sequence
from agent.providers.pricing import MODEL_PRICING
from agent.providers.token_usage import TokenUsage
from agent.state import ensure_str
from fs_logging.openai_input_formatting import (
summarize_responses_input_item,
to_serializable,
)
from llm import Llm, get_openai_api_name
def _render_json_scalar(value: Any) -> str:
if value is None:
return "<span class='json-null'>null</span>"
if isinstance(value, bool):
return f"<span class='json-bool'>{str(value).lower()}</span>"
if isinstance(value, (int, float)):
return f"<span class='json-number'>{escape(ensure_str(value))}</span>"
text = ensure_str(value)
if "\n" not in text and len(text) <= 160:
return f"<code class='json-string'>{escape(text)}</code>"
return (
"<details class='json-string-block'>"
f"<summary>string ({len(text)} chars)</summary>"
f"<pre>{escape(text)}</pre>"
"</details>"
)
def _render_json_node(value: Any, label: str | None = None) -> str:
label_html = ""
if label is not None:
label_html = f"<span class='json-key'>{escape(label)}</span>: "
if isinstance(value, dict):
parts = [
"<details class='json-node'>",
(
f"<summary>{label_html}"
f"<span class='json-type'>object ({len(value)} keys)</span></summary>"
),
"<div class='json-children'>",
]
for child_key, child_value in value.items():
parts.append(_render_json_node(child_value, ensure_str(child_key)))
parts.append("</div>")
parts.append("</details>")
return "".join(parts)
if isinstance(value, list):
parts = [
"<details class='json-node'>",
(
f"<summary>{label_html}"
f"<span class='json-type'>array ({len(value)} items)</span></summary>"
),
"<div class='json-children'>",
]
for index, child_value in enumerate(value):
parts.append(_render_json_node(child_value, f"[{index}]"))
parts.append("</div>")
parts.append("</details>")
return "".join(parts)
return (
"<div class='json-leaf'>"
f"{label_html}{_render_json_scalar(value)}"
"</div>"
)
def _render_copy_controls(copy_target_id: str, button_label: str) -> str:
return (
"<div class='copy-controls'>"
f"<button type='button' class='copy-button' data-copy-target='{escape(copy_target_id)}'>"
f"{escape(button_label)}"
"</button>"
"<span class='copy-status' aria-live='polite'></span>"
"</div>"
)
def _log_openai_turn_input(model: Llm, turn_index: int, input_items: Sequence[Any]) -> None:
model_name = get_openai_api_name(model)
print(
f"[OPENAI TURN INPUT] model={model_name} "
f"turn={turn_index} items={len(input_items)}"
)
for index, item in enumerate(input_items):
print(
f"[OPENAI TURN INPUT] "
f"{summarize_responses_input_item(index, item)}"
)
def _is_openai_turn_input_console_enabled() -> bool:
value = os.environ.get("OPENAI_TURN_INPUT_CONSOLE", "")
return value.strip().lower() in {"1", "true", "yes", "on"}
@dataclass
class OpenAITurnInputItem:
index: int
summary: str
payload: Any
@dataclass
class OpenAITurnUsageSummary:
input_tokens: int
output_tokens: int
cache_read: int
cache_write: int
total_tokens: int
cache_hit_rate_percent: float
cost_usd: float | None
@dataclass
class OpenAITurnInputReport:
turn_index: int
items: list[OpenAITurnInputItem]
request_payload: Any | None = None
usage: OpenAITurnUsageSummary | None = None
@dataclass
class OpenAITurnInputLogger:
model: Llm
enabled: bool = False
report_id: str = field(default_factory=lambda: uuid.uuid4().hex)
_turn_index: int = 0
_turns: list[OpenAITurnInputReport] = field(default_factory=list)
def record_turn_input(
self,
input_items: Sequence[Any],
request_payload: Any | None = None,
) -> None:
if not self.enabled:
return
self._turn_index += 1
if _is_openai_turn_input_console_enabled():
_log_openai_turn_input(self.model, self._turn_index, input_items)
turn_items = [
OpenAITurnInputItem(
index=index,
summary=summarize_responses_input_item(index, item),
payload=to_serializable(item),
)
for index, item in enumerate(input_items)
]
self._turns.append(
OpenAITurnInputReport(
turn_index=self._turn_index,
items=turn_items,
request_payload=to_serializable(request_payload),
)
)
def record_turn_usage(self, usage: TokenUsage) -> None:
if not self.enabled or not self._turns:
return
pricing = MODEL_PRICING.get(get_openai_api_name(self.model))
cost_usd = usage.cost(pricing) if pricing else None
self._turns[-1].usage = OpenAITurnUsageSummary(
input_tokens=usage.input,
output_tokens=usage.output,
cache_read=usage.cache_read,
cache_write=usage.cache_write,
total_tokens=usage.total,
cache_hit_rate_percent=usage.cache_hit_rate_percent(),
cost_usd=cost_usd,
)
def write_html_report(self) -> str | None:
if not self.enabled:
return None
try:
logs_path = os.environ.get("LOGS_PATH", os.getcwd())
logs_directory = os.path.join(logs_path, "run_logs")
os.makedirs(logs_directory, exist_ok=True)
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
model_name = get_openai_api_name(self.model).replace("/", "_")
filename = (
f"openai_turn_inputs_{model_name}_{timestamp}_{self.report_id[:8]}.html"
)
filepath = os.path.join(logs_directory, filename)
with open(filepath, "w", encoding="utf-8") as f:
f.write(self._render_html_report())
return filepath
except Exception as e:
print(f"[OPENAI TURN INPUT] Failed to write HTML report: {e}")
return None
def _render_html_report(self) -> str:
model_name = get_openai_api_name(self.model)
html_parts = [
"<!DOCTYPE html>",
"<html lang='en'>",
"<head>",
" <meta charset='UTF-8' />",
" <meta name='viewport' content='width=device-width, initial-scale=1.0' />",
" <title>OpenAI Turn Input Report</title>",
" <style>",
" body { font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', sans-serif; margin: 24px; color: #111827; }",
" h1, h2, h3 { margin: 0 0 12px; }",
" .meta { margin-bottom: 18px; color: #4b5563; }",
" .turn { border: 1px solid #d1d5db; border-radius: 8px; padding: 14px; margin-bottom: 16px; }",
" table { width: 100%; border-collapse: collapse; margin-top: 8px; }",
" th, td { border: 1px solid #e5e7eb; padding: 8px; vertical-align: top; text-align: left; }",
" th { background: #f9fafb; }",
" code, pre { font-family: ui-monospace, SFMono-Regular, Menlo, monospace; font-size: 12px; }",
" details { margin-top: 10px; }",
" pre { background: #0b1020; color: #d1d5db; padding: 12px; border-radius: 8px; overflow: auto; max-height: 420px; }",
" .usage-table { width: auto; min-width: 540px; margin-top: 10px; }",
" .usage-table th { width: 180px; }",
" .usage-none { margin-top: 10px; color: #6b7280; font-style: italic; }",
" .payload-wrap { margin-top: 10px; }",
" .json-view { margin-top: 8px; padding: 10px; border: 1px solid #e5e7eb; border-radius: 8px; background: #fcfcfd; }",
" .json-node, .json-string-block { margin: 6px 0; }",
" .json-node > summary, .json-string-block > summary { cursor: pointer; color: #1f2937; }",
" .json-children { margin-left: 16px; border-left: 1px solid #e5e7eb; padding-left: 10px; }",
" .json-leaf { margin: 6px 0; }",
" .json-key { color: #7c3aed; font-family: ui-monospace, SFMono-Regular, Menlo, monospace; font-size: 12px; }",
" .json-type { color: #2563eb; font-family: ui-monospace, SFMono-Regular, Menlo, monospace; font-size: 12px; }",
" .json-number, .json-bool, .json-null { font-family: ui-monospace, SFMono-Regular, Menlo, monospace; font-size: 12px; color: #0f766e; }",
" .json-string { white-space: pre-wrap; overflow-wrap: anywhere; }",
" .copy-controls { display: flex; align-items: center; gap: 8px; margin-top: 10px; }",
" .copy-button { border: 1px solid #cbd5e1; background: #fff; color: #111827; border-radius: 6px; padding: 6px 10px; cursor: pointer; font-size: 12px; }",
" .copy-button:hover { background: #f8fafc; }",
" .copy-status { color: #2563eb; font-size: 12px; min-height: 16px; }",
" .copy-source { display: none; }",
" </style>",
"</head>",
"<body>",
" <h1>OpenAI Turn Input Report</h1>",
(
" <div class='meta'>"
f"report_id={escape(self.report_id)} | "
f"model={escape(model_name)} | turns={len(self._turns)}"
"</div>"
),
]
for turn in self._turns:
html_parts.append(" <section class='turn'>")
html_parts.append(
f" <h2>Turn {turn.turn_index} (items={len(turn.items)})</h2>"
)
if turn.request_payload is not None:
request_payload_json = json.dumps(
turn.request_payload,
indent=2,
ensure_ascii=False,
)
request_input_json: str | None = None
if isinstance(turn.request_payload, dict) and "input" in turn.request_payload:
request_input_json = json.dumps(
turn.request_payload["input"],
indent=2,
ensure_ascii=False,
)
html_parts.append(" <details class='payload-wrap' open>")
html_parts.append(" <summary>Request payload</summary>")
if request_input_json is not None:
request_input_id = f"request-input-turn-{turn.turn_index}"
html_parts.append(
" "
+ _render_copy_controls(request_input_id, "Copy input JSON")
)
html_parts.append(
f" <pre id='{escape(request_input_id)}' class='copy-source'>"
f"{escape(request_input_json)}</pre>"
)
html_parts.append(" <div class='json-view'>")
html_parts.append(_render_json_node(turn.request_payload, "root"))
html_parts.append(" </div>")
html_parts.append(" <details>")
html_parts.append(" <summary>Raw JSON payload</summary>")
html_parts.append(
f" <pre>{escape(request_payload_json)}</pre>"
)
html_parts.append(" </details>")
html_parts.append(" </details>")
if turn.usage is not None:
cost_text = "n/a"
if isinstance(turn.usage.cost_usd, (float, int)):
cost_text = f"${turn.usage.cost_usd:.4f}"
html_parts.append(" <table class='usage-table'>")
html_parts.append(
" <thead><tr><th>Metric</th><th>Value</th></tr></thead>"
)
html_parts.append(" <tbody>")
html_parts.append(
" "
f"<tr><td>Input tokens</td><td>{turn.usage.input_tokens}</td></tr>"
)
html_parts.append(
" "
f"<tr><td>Output tokens</td><td>{turn.usage.output_tokens}</td></tr>"
)
html_parts.append(
" "
f"<tr><td>Cache read</td><td>{turn.usage.cache_read}</td></tr>"
)
html_parts.append(
" "
f"<tr><td>Cache write</td><td>{turn.usage.cache_write}</td></tr>"
)
html_parts.append(
" "
f"<tr><td>Total tokens</td><td>{turn.usage.total_tokens}</td></tr>"
)
html_parts.append(
" <tr><td>Cache hit rate</td>"
f"<td>{turn.usage.cache_hit_rate_percent:.2f}%</td></tr>"
)
html_parts.append(
f" <tr><td>Cost</td><td>{escape(cost_text)}</td></tr>"
)
html_parts.append(" </tbody>")
html_parts.append(" </table>")
else:
html_parts.append(
" <div class='usage-none'>Usage unavailable for this turn.</div>"
)
html_parts.append(" <table>")
html_parts.append(
" <thead><tr><th style='width:70px'>Index</th><th>Summary</th></tr></thead>"
)
html_parts.append(" <tbody>")
for item in turn.items:
html_parts.append(
" "
f"<tr><td>{item.index:02d}</td><td><code>{escape(item.summary)}</code></td></tr>"
)
html_parts.append(" </tbody>")
html_parts.append(" </table>")
for item in turn.items:
payload_json = json.dumps(
item.payload,
indent=2,
ensure_ascii=False,
)
html_parts.append(" <details class='payload-wrap'>")
html_parts.append(
f" <summary>Item {item.index:02d} payload</summary>"
)
html_parts.append(" <div class='json-view'>")
html_parts.append(_render_json_node(item.payload, "root"))
html_parts.append(" </div>")
html_parts.append(" <details>")
html_parts.append(" <summary>Raw JSON payload</summary>")
html_parts.append(f" <pre>{escape(payload_json)}</pre>")
html_parts.append(" </details>")
html_parts.append(" </details>")
html_parts.append(" </section>")
html_parts.extend(
[
" <script>",
" document.addEventListener('click', async (event) => {",
" const target = event.target;",
" if (!(target instanceof HTMLButtonElement)) {",
" return;",
" }",
" const copyTargetId = target.dataset.copyTarget;",
" if (!copyTargetId) {",
" return;",
" }",
" const source = document.getElementById(copyTargetId);",
" const status = target.parentElement?.querySelector('.copy-status');",
" if (!source) {",
" if (status) { status.textContent = 'Missing source'; }",
" return;",
" }",
" try {",
" await navigator.clipboard.writeText(source.textContent || '');",
" if (status) { status.textContent = 'Copied'; }",
" } catch (_error) {",
" if (status) { status.textContent = 'Copy failed'; }",
" }",
" window.setTimeout(() => {",
" if (status) { status.textContent = ''; }",
" }, 1600);",
" });",
" </script>",
"</body>",
"</html>",
]
)
return "\n".join(html_parts)
================================================
FILE: backend/image_generation/__init__.py
================================================
================================================
FILE: backend/image_generation/core.py
================================================
from image_generation.generation import (
generate_image_dalle,
generate_image_replicate,
process_tasks,
)
__all__ = [
"process_tasks",
"generate_image_dalle",
"generate_image_replicate",
]
================================================
FILE: backend/image_generation/generation.py
================================================
import asyncio
import time
from typing import List, Literal, Union
from openai import AsyncOpenAI
from image_generation.replicate import call_replicate
REPLICATE_BATCH_SIZE = 20
async def process_tasks(
prompts: List[str],
api_key: str,
base_url: str | None,
model: Literal["dalle3", "flux"],
) -> List[Union[str, None]]:
start_time = time.time()
results: list[str | BaseException | None]
if model == "dalle3":
tasks = [generate_image_dalle(prompt, api_key, base_url) for prompt in prompts]
results = await asyncio.gather(*tasks, return_exceptions=True)
else:
results = []
for i in range(0, len(prompts), REPLICATE_BATCH_SIZE):
batch = prompts[i : i + REPLICATE_BATCH_SIZE]
tasks = [generate_image_replicate(p, api_key) for p in batch]
results.extend(await asyncio.gather(*tasks, return_exceptions=True))
end_time = time.time()
generation_time = end_time - start_time
print(f"Image generation time: {generation_time:.2f} seconds")
processed_results: List[Union[str, None]] = []
for result in results:
if isinstance(result, BaseException):
print(f"An exception occurred: {result}")
processed_results.append(None)
else:
processed_results.append(result)
return processed_results
async def generate_image_dalle(
prompt: str, api_key: str, base_url: str | None
) -> Union[str, None]:
client = AsyncOpenAI(api_key=api_key, base_url=base_url)
res = await client.images.generate(
model="dall-e-3",
quality="standard",
style="natural",
n=1,
size="1024x1024",
prompt=prompt,
)
await client.close()
if not res.data:
return None
return res.data[0].url
async def generate_image_replicate(prompt: str, api_key: str) -> str:
# We use Flux 2 Klein
return await call_replicate(
{
"prompt": prompt,
"aspect_ratio": "1:1",
"output_format": "png",
},
api_key,
)
================================================
FILE: backend/image_generation/replicate.py
================================================
import asyncio
import httpx
from typing import Any, Mapping, cast
REPLICATE_API_BASE_URL = "https://api.replicate.com/v1"
FLUX_MODEL_PATH = "black-forest-labs/flux-2-klein-4b"
REMOVE_BACKGROUND_VERSION = (
"a029dff38972b5fda4ec5d75d7d1cd25aeff621d2cf4946a41055d7db66b80bc"
)
POLL_INTERVAL_SECONDS = 0.1
MAX_POLLS = 100
def _build_headers(api_token: str) -> dict[str, str]:
return {
"Authorization": f"Bearer {api_token}",
"Content-Type": "application/json",
}
def _extract_prediction_id(response_json: Mapping[str, Any]) -> str:
prediction_id = response_json.get("id")
if not isinstance(prediction_id, str) or not prediction_id:
raise ValueError("Prediction ID not found in initial response.")
return prediction_id
async def _poll_prediction(
client: httpx.AsyncClient, prediction_id: str, headers: dict[str, str]
) -> dict[str, Any]:
status_check_url = f"{REPLICATE_API_BASE_URL}/predictions/{prediction_id}"
for _ in range(MAX_POLLS):
await asyncio.sleep(POLL_INTERVAL_SECONDS)
status_response = await client.get(status_check_url, headers=headers)
status_response.raise_for_status()
status_response_raw: Any = status_response.json()
if not isinstance(status_response_raw, dict):
raise ValueError("Invalid prediction status response.")
status_response_json = cast(dict[str, Any], status_response_raw)
status = status_response_json.get("status")
if status == "succeeded":
return cast(dict[str, Any], status_response_json)
if status == "error":
error_message = str(status_response_json.get("error", "Unknown error"))
raise ValueError(f"Inference errored out: {error_message}")
if status == "failed":
raise ValueError("Inference failed")
raise TimeoutError("Inference timed out")
async def _run_prediction(
endpoint_url: str, payload: dict[str, Any], api_token: str
) -> Any:
headers = _build_headers(api_token)
async with httpx.AsyncClient() as client:
try:
response = await client.post(endpoint_url, headers=headers, json=payload)
response.raise_for_status()
response_json = response.json()
if not isinstance(response_json, dict):
raise ValueError("Invalid prediction creation response.")
prediction_id = _extract_prediction_id(response_json)
final_response = await _poll_prediction(client, prediction_id, headers)
return final_response.get("output")
except httpx.HTTPStatusError as exc:
raise ValueError(f"HTTP error occurred: {exc}") from exc
except httpx.RequestError as exc:
raise ValueError(f"An error occurred while requesting: {exc}") from exc
except asyncio.TimeoutError as exc:
raise TimeoutError("Request timed out") from exc
except (TimeoutError, ValueError):
raise
except Exception as exc:
raise ValueError(f"An unexpected error occurred: {exc}") from exc
def _extract_output_url(result: Any, context: str) -> str:
if isinstance(result, str):
return result
if isinstance(result, dict):
url = cast(Any, result.get("url"))
if isinstance(url, str) and url:
return url
if isinstance(result, list) and len(result) > 0:
first: Any = result[0]
if isinstance(first, str) and first:
return first
if isinstance(first, Mapping):
url = cast(Any, first.get("url"))
if isinstance(url, str) and url:
return url
raise ValueError(f"Unexpected response from {context}: {result}")
async def call_replicate_model(
model_path: str, input: dict[str, Any], api_token: str
) -> Any:
return await _run_prediction(
f"{REPLICATE_API_BASE_URL}/models/{model_path}/predictions",
{"input": input},
api_token,
)
async def call_replicate_version(
version: str, input: dict[str, Any], api_token: str
) -> Any:
return await _run_prediction(
f"{REPLICATE_API_BASE_URL}/predictions",
{"version": version, "input": input},
api_token,
)
async def remove_background(image_url: str, api_token: str) -> str:
result = await call_replicate_version(
REMOVE_BACKGROUND_VERSION,
{
"image": image_url,
"format": "png",
"reverse": False,
"threshold": 0,
"background_type": "rgba",
},
api_token,
)
return _extract_output_url(result, "background remover")
async def call_replicate(input: dict[str, str | int], api_token: str) -> str:
result = await call_replicate_model(FLUX_MODEL_PATH, input, api_token)
return _extract_output_url(result, "Flux prediction")
================================================
FILE: backend/llm.py
================================================
from enum import Enum
from typing import TypedDict
# Actual model versions that are passed to the LLMs and stored in our logs
class Llm(Enum):
# GPT
GPT_4_1_2025_04_14 = "gpt-4.1-2025-04-14"
GPT_5_2_CODEX_LOW = "gpt-5.2-codex (low thinking)"
GPT_5_2_CODEX_MEDIUM = "gpt-5.2-codex (medium thinking)"
GPT_5_2_CODEX_HIGH = "gpt-5.2-codex (high thinking)"
GPT_5_2_CODEX_XHIGH = "gpt-5.2-codex (xhigh thinking)"
GPT_5_3_CODEX_LOW = "gpt-5.3-codex (low thinking)"
GPT_5_3_CODEX_MEDIUM = "gpt-5.3-codex (medium thinking)"
GPT_5_3_CODEX_HIGH = "gpt-5.3-codex (high thinking)"
GPT_5_3_CODEX_XHIGH = "gpt-5.3-codex (xhigh thinking)"
GPT_5_4_2026_03_05_NONE = "gpt-5.4-2026-03-05 (no thinking)"
GPT_5_4_2026_03_05_LOW = "gpt-5.4-2026-03-05 (low thinking)"
GPT_5_4_2026_03_05_MEDIUM = "gpt-5.4-2026-03-05 (medium thinking)"
GPT_5_4_2026_03_05_HIGH = "gpt-5.4-2026-03-05 (high thinking)"
GPT_5_4_2026_03_05_XHIGH = "gpt-5.4-2026-03-05 (xhigh thinking)"
# Claude
CLAUDE_SONNET_4_6 = "claude-sonnet-4-6"
CLAUDE_4_5_SONNET_2025_09_29 = "claude-sonnet-4-5-20250929"
CLAUDE_4_5_OPUS_2025_11_01 = "claude-opus-4-5-20251101"
CLAUDE_OPUS_4_6 = "claude-opus-4-6"
# Gemini
GEMINI_3_FLASH_PREVIEW_HIGH = "gemini-3-flash-preview (high thinking)"
GEMINI_3_FLASH_PREVIEW_MINIMAL = "gemini-3-flash-preview (minimal thinking)"
GEMINI_3_1_PRO_PREVIEW_HIGH = "gemini-3.1-pro-preview (high thinking)"
GEMINI_3_1_PRO_PREVIEW_MEDIUM = "gemini-3.1-pro-preview (medium thinking)"
GEMINI_3_1_PRO_PREVIEW_LOW = "gemini-3.1-pro-preview (low thinking)"
class Completion(TypedDict):
duration: float
code: str
# Explicitly map each model to the provider backing it. This keeps provider
# groupings authoritative and avoids relying on name conventions when checking
# models elsewhere in the codebase.
MODEL_PROVIDER: dict[Llm, str] = {
# OpenAI models
Llm.GPT_4_1_2025_04_14: "openai",
Llm.GPT_5_2_CODEX_LOW: "openai",
Llm.GPT_5_2_CODEX_MEDIUM: "openai",
Llm.GPT_5_2_CODEX_HIGH: "openai",
Llm.GPT_5_2_CODEX_XHIGH: "openai",
Llm.GPT_5_3_CODEX_LOW: "openai",
Llm.GPT_5_3_CODEX_MEDIUM: "openai",
Llm.GPT_5_3_CODEX_HIGH: "openai",
Llm.GPT_5_3_CODEX_XHIGH: "openai",
Llm.GPT_5_4_2026_03_05_NONE: "openai",
Llm.GPT_5_4_2026_03_05_LOW: "openai",
Llm.GPT_5_4_2026_03_05_MEDIUM: "openai",
Llm.GPT_5_4_2026_03_05_HIGH: "openai",
Llm.GPT_5_4_2026_03_05_XHIGH: "openai",
# Anthropic models
Llm.CLAUDE_SONNET_4_6: "anthropic",
Llm.CLAUDE_4_5_SONNET_2025_09_29: "anthropic",
Llm.CLAUDE_4_5_OPUS_2025_11_01: "anthropic",
Llm.CLAUDE_OPUS_4_6: "anthropic",
# Gemini models
Llm.GEMINI_3_FLASH_PREVIEW_HIGH: "gemini",
Llm.GEMINI_3_FLASH_PREVIEW_MINIMAL: "gemini",
Llm.GEMINI_3_1_PRO_PREVIEW_HIGH: "gemini",
Llm.GEMINI_3_1_PRO_PREVIEW_MEDIUM: "gemini",
Llm.GEMINI_3_1_PRO_PREVIEW_LOW: "gemini",
}
# Convenience sets for membership checks
OPENAI_MODELS = {m for m, p in MODEL_PROVIDER.items() if p == "openai"}
ANTHROPIC_MODELS = {m for m, p in MODEL_PROVIDER.items() if p == "anthropic"}
GEMINI_MODELS = {m for m, p in MODEL_PROVIDER.items() if p == "gemini"}
OPENAI_MODEL_CONFIG: dict[Llm, dict[str, str]] = {
Llm.GPT_4_1_2025_04_14: {"api_name": "gpt-4.1-2025-04-14"},
Llm.GPT_5_2_CODEX_LOW: {"api_name": "gpt-5.2-codex", "reasoning_effort": "low"},
Llm.GPT_5_2_CODEX_MEDIUM: {"api_name": "gpt-5.2-codex", "reasoning_effort": "medium"},
Llm.GPT_5_2_CODEX_HIGH: {"api_name": "gpt-5.2-codex", "reasoning_effort": "high"},
Llm.GPT_5_2_CODEX_XHIGH: {"api_name": "gpt-5.2-codex", "reasoning_effort": "xhigh"},
Llm.GPT_5_3_CODEX_LOW: {"api_name": "gpt-5.3-codex", "reasoning_effort": "low"},
Llm.GPT_5_3_CODEX_MEDIUM: {"api_name": "gpt-5.3-codex", "reasoning_effort": "medium"},
Llm.GPT_5_3_CODEX_HIGH: {"api_name": "gpt-5.3-codex", "reasoning_effort": "high"},
Llm.GPT_5_3_CODEX_XHIGH: {"api_name": "gpt-5.3-codex", "reasoning_effort": "xhigh"},
Llm.GPT_5_4_2026_03_05_NONE: {
"api_name": "gpt-5.4-2026-03-05",
"reasoning_effort": "none",
},
Llm.GPT_5_4_2026_03_05_LOW: {
"api_name": "gpt-5.4-2026-03-05",
"reasoning_effort": "low",
},
Llm.GPT_5_4_2026_03_05_MEDIUM: {
"api_name": "gpt-5.4-2026-03-05",
"reasoning_effort": "medium",
},
Llm.GPT_5_4_2026_03_05_HIGH: {
"api_name": "gpt-5.4-2026-03-05",
"reasoning_effort": "high",
},
Llm.GPT_5_4_2026_03_05_XHIGH: {
"api_name": "gpt-5.4-2026-03-05",
"reasoning_effort": "xhigh",
},
}
def get_openai_api_name(model: Llm) -> str:
return OPENAI_MODEL_CONFIG[model]["api_name"]
def get_openai_reasoning_effort(model: Llm) -> str | None:
return OPENAI_MODEL_CONFIG.get(model, {}).get("reasoning_effort")
================================================
FILE: backend/main.py
================================================
# Load environment variables first
from dotenv import load_dotenv
load_dotenv()
from fastapi import FastAPI
from fastapi.middleware.cors import CORSMiddleware
from config import IS_DEBUG_ENABLED
from routes import screenshot, generate_code, home, evals
app = FastAPI(openapi_url=None, docs_url=None, redoc_url=None)
@app.on_event("startup")
async def log_debug_mode() -> None:
debug_status = "ENABLED" if IS_DEBUG_ENABLED else "DISABLED"
print(f"Backend startup complete. Debug mode is {debug_status}.")
# Configure CORS settings
app.add_middleware(
CORSMiddleware,
allow_origins=["*"],
allow_credentials=True,
allow_methods=["*"],
allow_headers=["*"],
)
# Add routes
app.include_router(generate_code.router)
app.include_router(screenshot.router)
app.include_router(home.router)
app.include_router(evals.router)
================================================
FILE: backend/prompts/__init__.py
================================================
from prompts.system_prompt import SYSTEM_PROMPT
__all__ = [
"SYSTEM_PROMPT",
]
================================================
FILE: backend/prompts/create/__init__.py
================================================
from custom_types import InputMode
from prompts.create.image import build_image_prompt_messages
from prompts.create.text import build_text_prompt_messages
from prompts.create.video import build_video_prompt_messages
from prompts.prompt_types import Stack, UserTurnInput
from prompts.message_builder import Prompt
def build_create_prompt_from_input(
input_mode: InputMode,
stack: Stack,
prompt: UserTurnInput,
image_generation_enabled: bool,
) -> Prompt:
if input_mode == "image":
image_urls = prompt.get("images", [])
text_prompt = prompt.get("text", "")
return build_image_prompt_messages(
image_data_urls=image_urls,
stack=stack,
text_prompt=text_prompt,
image_generation_enabled=image_generation_enabled,
)
if input_mode == "text":
return build_text_prompt_messages(
text_prompt=prompt["text"],
stack=stack,
image_generation_enabled=image_generation_enabled,
)
if input_mode == "video":
video_urls = prompt.get("videos", [])
if not video_urls:
raise ValueError("Video mode requires a video to be provided")
video_url = video_urls[0]
return build_video_prompt_messages(
video_data_url=video_url,
stack=stack,
text_prompt=prompt.get("text", ""),
image_generation_enabled=image_generation_enabled,
)
raise ValueError(f"Unsupported input mode: {input_mode}")
__all__ = ["build_create_prompt_from_input"]
================================================
FILE: backend/prompts/create/image.py
================================================
from openai.types.chat import ChatCompletionContentPartParam, ChatCompletionMessageParam
from prompts.prompt_types import Stack
from prompts import system_prompt
from prompts.policies import build_selected_stack_policy, build_user_image_policy
def build_image_prompt_messages(
image_data_urls: list[str],
stack: Stack,
text_prompt: str,
image_generation_enabled: bool,
) -> list[ChatCompletionMessageParam]:
image_policy = build_user_image_policy(image_generation_enabled)
selected_stack = build_selected_stack_policy(stack)
user_prompt = f"""
Generate code for a web page that looks exactly like the provided screenshot(s).
{selected_stack}
## Replication instructions
- Make sure the app looks exactly like the screenshot.
- Use the exact text from the screenshot.
- {image_policy}
## Multiple screenshots
If multiple screenshots are provided, organize them meaningfully:
- If they appear to be different pages in a website, make them distinct pages and link them.
- If they look like different tabs or views in an app, connect them with appropriate navigation.
- If they appear unrelated, create a scaffold that separates them into "Screenshot 1", "Screenshot 2", "Screenshot 3", etc. so it is easy to navigate.
- For mobile screenshots, do not include the device frame or browser chrome; focus only on the actual UI mockups.
"""
# Add additional instructions provided by the user
if text_prompt.strip():
user_prompt = f"{user_prompt}\n\nAdditional instructions: {text_prompt}"
user_content: list[ChatCompletionContentPartParam] = []
for image_data_url in image_data_urls:
user_content.append(
{
"type": "image_url",
"image_url": {"url": image_data_url, "detail": "high"},
}
)
user_content.append(
{
"type": "text",
"text": user_prompt,
}
)
return [
{
"role": "system",
"content": system_prompt.SYSTEM_PROMPT,
},
{
"role": "user",
"content": user_content,
},
]
================================================
FILE: backend/prompts/create/text.py
================================================
from openai.types.chat import ChatCompletionMessageParam
from prompts.prompt_types import Stack
from prompts import system_prompt
from prompts.policies import build_selected_stack_policy, build_user_image_policy
def build_text_prompt_messages(
text_prompt: str,
stack: Stack,
image_generation_enabled: bool,
) -> list[ChatCompletionMessageParam]:
image_policy = build_user_image_policy(image_generation_enabled)
selected_stack = build_selected_stack_policy(stack)
USER_PROMPT = f"""
Generate UI for {text_prompt}.
{selected_stack}
# Instructions
- Make sure to make it look modern and sleek.
- Use modern, professional fonts and colors.
- Follow UX best practices.
- {image_policy}"""
return [
{
"role": "system",
"content": system_prompt.SYSTEM_PROMPT,
},
{
"role": "user",
"content": USER_PROMPT,
},
]
================================================
FILE: backend/prompts/create/video.py
================================================
from openai.types.chat import ChatCompletionContentPartParam, ChatCompletionMessageParam
from prompts.prompt_types import Stack
from prompts import system_prompt
from prompts.policies import build_selected_stack_policy, build_user_image_policy
def build_video_prompt_messages(
video_data_url: str,
stack: Stack,
text_prompt: str,
image_generation_enabled: bool,
) -> list[ChatCompletionMessageParam]:
image_policy = build_user_image_policy(image_generation_enabled)
selected_stack = build_selected_stack_policy(stack)
user_text = f"""
You have been given a video of a user interacting with a web app. You need to re-create the same app exactly such that the same user interactions will produce the same results in the app you build.
- Watch the entire video carefully and understand all the user interactions and UI state changes.
- Make sure the app looks exactly like what you see in the video.
- Pay close attention to background color, text color, font size, font family,
padding, margin, border, etc. Match the colors and sizes exactly.
- {image_policy}
- If some functionality requires a backend call, just mock the data instead.
- MAKE THE APP FUNCTIONAL using JavaScript. Allow the user to interact with the app and get the same behavior as shown in the video.
- Use SVGs and interactive 3D elements if needed to match the functionality shown in the video.
Analyze this video and generate the code.
{selected_stack}
"""
if text_prompt.strip():
user_text = user_text + "\n\nAdditional instructions: " + text_prompt
user_content: list[ChatCompletionContentPartParam] = [
{
"type": "image_url",
"image_url": {"url": video_data_url, "detail": "high"},
},
{
"type": "text",
"text": user_text,
},
]
return [
{
"role": "system",
"content": system_prompt.SYSTEM_PROMPT,
},
{
"role": "user",
"content": user_content,
},
]
================================================
FILE: backend/prompts/message_builder.py
================================================
from typing import cast
from openai.types.chat import ChatCompletionContentPartParam, ChatCompletionMessageParam
from prompts.prompt_types import PromptHistoryMessage
Prompt = list[ChatCompletionMessageParam]
def _wrap_assistant_file_content(content: str, path: str = "index.html") -> str:
stripped = content.strip()
if stripped.startswith("<file ") and stripped.endswith("</file>"):
return stripped
return f'<file path="{path}">\n{stripped}\n</file>'
def build_history_message(item: PromptHistoryMessage) -> ChatCompletionMessageParam:
role = item["role"]
image_urls = item.get("images", [])
video_urls = item.get("videos", [])
media_urls = [*image_urls, *video_urls]
if role == "user" and len(media_urls) > 0:
user_content: list[ChatCompletionContentPartParam] = []
for media_url in media_urls:
user_content.append(
{
"type": "image_url",
"image_url": {"url": media_url, "detail": "high"},
}
)
user_content.append(
{
"type": "text",
"text": item.get("text", ""),
}
)
return cast(
ChatCompletionMessageParam,
{
"role": role,
"content": user_content,
},
)
return cast(
ChatCompletionMessageParam,
{
"role": role,
"content": (
_wrap_assistant_file_content(item.get("text", ""))
if role == "assistant"
else item.get("text", "")
),
},
)
================================================
FILE: backend/prompts/pipeline.py
================================================
from custom_types import InputMode
from prompts.create import build_create_prompt_from_input
from prompts.plan import derive_prompt_construction_plan
from prompts.prompt_types import PromptHistoryMessage, Stack, UserTurnInput
from prompts.message_builder import Prompt
from prompts.update import (
build_update_prompt_from_file_snapshot,
build_update_prompt_from_history,
)
async def build_prompt_messages(
stack: Stack,
input_mode: InputMode,
generation_type: str,
prompt: UserTurnInput,
history: list[PromptHistoryMessage],
file_state: dict[str, str] | None = None,
image_generation_enabled: bool = True,
) -> Prompt:
plan = derive_prompt_construction_plan(
stack=stack,
input_mode=input_mode,
generation_type=generation_type,
history=history,
file_state=file_state,
)
strategy = plan["construction_strategy"]
if strategy == "update_from_history":
return build_update_prompt_from_history(
stack=stack,
history=history,
image_generation_enabled=image_generation_enabled,
)
if strategy == "update_from_file_snapshot":
assert file_state is not None
return build_update_prompt_from_file_snapshot(
stack=stack,
prompt=prompt,
file_state=file_state,
image_generation_enabled=image_generation_enabled,
)
return build_create_prompt_from_input(
input_mode,
stack,
prompt,
image_generation_enabled,
)
================================================
FILE: backend/prompts/plan.py
================================================
from custom_types import InputMode
from prompts.prompt_types import (
PromptConstructionPlan,
PromptHistoryMessage,
Stack,
)
def derive_prompt_construction_plan(
stack: Stack,
input_mode: InputMode,
generation_type: str,
history: list[PromptHistoryMessage],
file_state: dict[str, str] | None,
) -> PromptConstructionPlan:
if generation_type == "update":
if len(history) > 0:
strategy = "update_from_history"
elif file_state and file_state.get("content", "").strip():
strategy = "update_from_file_snapshot"
else:
raise ValueError("Update requests require history or fileState.content")
return {
"generation_type": "update",
"input_mode": input_mode,
"stack": stack,
"construction_strategy": strategy,
}
return {
"generation_type": "create",
"input_mode": input_mode,
"stack": stack,
"construction_strategy": "create_from_input",
}
================================================
FILE: backend/prompts/policies.py
================================================
from prompts.prompt_types import Stack
def build_selected_stack_policy(stack: Stack) -> str:
return f"Selected stack: {stack}."
def build_user_image_policy(image_generation_enabled: bool) -> str:
if image_generation_enabled:
return (
"Image generation is enabled for this request. Use generate_images for "
"missing assets when needed."
)
return (
"Image generation is disabled for this request. Do not call generate_images. "
"Use provided media, CSS effects, or placeholder URLs (https://placehold.co)."
)
================================================
FILE: backend/prompts/prompt_types.py
================================================
from typing import List, Literal, TypedDict
class UserTurnInput(TypedDict):
"""Normalized current user turn payload from the request."""
text: str
images: List[str]
videos: List[str]
class PromptHistoryMessage(TypedDict):
"""Explicit role-based message structure for edit history."""
role: Literal["user", "assistant"]
text: str
images: List[str]
videos: List[str]
PromptConstructionStrategy = Literal[
"create_from_input",
"update_from_history",
"update_from_file_snapshot",
]
Stack = Literal[
"html_css",
"html_tailwind",
"react_tailwind",
"bootstrap",
"ionic_tailwind",
"vue_tailwind",
]
class PromptConstructionPlan(TypedDict):
"""Derived plan used by prompt builders to choose a single construction path."""
generation_type: Literal["create", "update"]
input_mode: Literal["image", "video", "text"]
stack: Stack
construction_strategy: PromptConstructionStrategy
================================================
FILE: backend/prompts/request_parsing.py
================================================
from typing import List, cast
from prompts.prompt_types import PromptHistoryMessage, UserTurnInput
def _to_string_list(value: object) -> List[str]:
if not isinstance(value, list):
return []
raw_list = cast(List[object], value)
return [item for item in raw_list if isinstance(item, str)]
def parse_prompt_content(raw_prompt: object) -> UserTurnInput:
if not isinstance(raw_prompt, dict):
return {"text": "", "images": [], "videos": []}
prompt_dict = cast(dict[str, object], raw_prompt)
text = prompt_dict.get("text")
return {
"text": text if isinstance(text, str) else "",
"images": _to_string_list(prompt_dict.get("images")),
"videos": _to_string_list(prompt_dict.get("videos")),
}
def parse_prompt_history(raw_history: object) -> List[PromptHistoryMessage]:
if not isinstance(raw_history, list):
return []
history: List[PromptHistoryMessage] = []
raw_items = cast(List[object], raw_history)
for item in raw_items:
if not isinstance(item, dict):
continue
item_dict = cast(dict[str, object], item)
role_value = item_dict.get("role")
if not isinstance(role_value, str) or role_value not in ("user", "assistant"):
continue
text = item_dict.get("text")
history.append(
{
"role": role_value,
"text": text if isinstance(text, str) else "",
"images": _to_string_list(item_dict.get("images")),
"videos": _to_string_list(item_dict.get("videos")),
}
)
return history
================================================
FILE: backend/prompts/system_prompt.py
================================================
SYSTEM_PROMPT = """
You are a coding agent that's an expert at building front-ends.
# Tone and style
- Be extremely concise in your chat responses.
- Do not include code snippets in your messages. Use the file creation and editing tools for all code.
- At the end of the task, respond with a one or two sentence summary of what was built.
- Always respond to the user in the language that they used. Our system prompts and tooling instructions are in English, but the user may choose to speak in another language and you should respond in that language. But if you're unsure, always pick English.
# Tooling instructions
- You have access to tools for file creation, file editing, image handling, and option retrieval.
- The main file is a single HTML file. Use path "index.html" unless told otherwise.
- For a brand new app, call create_file exactly once with the full HTML.
- For updates, call edit_file using exact string replacements. Do NOT regenerate the entire file.
- Do not output raw HTML in chat. Any code changes must go through tools.
- When available, use generate_images to create image URLs from prompts (you may pass multiple prompts). The image generation AI is not capable of generating images with a transparent background.
- Use remove_background to remove backgrounds from provided image URLs when needed (you may pass multiple image URLs).
- Use retrieve_option to fetch the full HTML for a specific option (1-based option_number) when a user references another option.
# Stack-specific instructions
## Tailwind
- Use this script to include Tailwind: <script src="https://cdn.tailwindcss.com"></script>
## html_css
- Only use HTML, CSS and JS.
- Do not use Tailwind
## Bootstrap
- Use this script to include Bootstrap: <link href="https://cdn.jsdelivr.net/npm/bootstrap@5.3.2/dist/css/bootstrap.min.css" rel="stylesheet" integrity="sha384-T3c6CoIi6uLrA9TneNEoa7RxnatzjcDSCmG1MXxSR1GAsXEV/Dwwykc2MPK8M2HN" crossorigin="ano
gitextract_l7qpd28l/ ├── .claude/ │ └── launch.json ├── .gitattributes ├── .github/ │ ├── FUNDING.yml │ └── ISSUE_TEMPLATE/ │ ├── bug_report.md │ ├── custom.md │ └── feature_request.md ├── .gitignore ├── .vscode/ │ └── settings.json ├── AGENTS.md ├── CLAUDE.md ├── Evaluation.md ├── LICENSE ├── README.md ├── TESTING.md ├── Troubleshooting.md ├── backend/ │ ├── .gitignore │ ├── .pre-commit-config.yaml │ ├── Dockerfile │ ├── README.md │ ├── agent/ │ │ ├── engine.py │ │ ├── providers/ │ │ │ ├── __init__.py │ │ │ ├── anthropic/ │ │ │ │ ├── __init__.py │ │ │ │ ├── image.py │ │ │ │ └── provider.py │ │ │ ├── base.py │ │ │ ├── factory.py │ │ │ ├── gemini.py │ │ │ ├── openai.py │ │ │ ├── pricing.py │ │ │ ├── token_usage.py │ │ │ └── types.py │ │ ├── runner.py │ │ ├── state.py │ │ └── tools/ │ │ ├── __init__.py │ │ ├── definitions.py │ │ ├── parsing.py │ │ ├── runtime.py │ │ ├── summaries.py │ │ └── types.py │ ├── codegen/ │ │ ├── __init__.py │ │ ├── test_utils.py │ │ └── utils.py │ ├── config.py │ ├── custom_types.py │ ├── debug/ │ │ ├── DebugFileWriter.py │ │ └── __init__.py │ ├── evals/ │ │ ├── __init__.py │ │ ├── config.py │ │ ├── core.py │ │ ├── runner.py │ │ └── utils.py │ ├── fs_logging/ │ │ ├── __init__.py │ │ ├── openai_input_compare.py │ │ ├── openai_input_formatting.py │ │ └── openai_turn_inputs.py │ ├── image_generation/ │ │ ├── __init__.py │ │ ├── core.py │ │ ├── generation.py │ │ └── replicate.py │ ├── llm.py │ ├── main.py │ ├── prompts/ │ │ ├── __init__.py │ │ ├── create/ │ │ │ ├── __init__.py │ │ │ ├── image.py │ │ │ ├── text.py │ │ │ └── video.py │ │ ├── message_builder.py │ │ ├── pipeline.py │ │ ├── plan.py │ │ ├── policies.py │ │ ├── prompt_types.py │ │ ├── request_parsing.py │ │ ├── system_prompt.py │ │ └── update/ │ │ ├── __init__.py │ │ ├── from_file_snapshot.py │ │ └── from_history.py │ ├── pyproject.toml │ ├── pyrightconfig.json │ ├── pytest.ini │ ├── routes/ │ │ ├── evals.py │ │ ├── generate_code.py │ │ ├── home.py │ │ ├── model_choice_sets.py │ │ └── screenshot.py │ ├── run_evals.py │ ├── run_image_generation_evals.py │ ├── start.py │ ├── tests/ │ │ ├── __init__.py │ │ ├── test_agent_tool_runtime.py │ │ ├── test_agent_tools.py │ │ ├── test_batching.py │ │ ├── test_codegen_utils.py │ │ ├── test_evals_openai_input_compare.py │ │ ├── test_image_generation_replicate.py │ │ ├── test_model_selection.py │ │ ├── test_openai_input_compare.py │ │ ├── test_openai_provider_session.py │ │ ├── test_openai_reasoning_parser.py │ │ ├── test_openai_turn_input_logging.py │ │ ├── test_parameter_extraction_stage.py │ │ ├── test_prompt_summary.py │ │ ├── test_prompts.py │ │ ├── test_request_parsing.py │ │ ├── test_screenshot.py │ │ ├── test_status_broadcast.py │ │ └── test_token_usage.py │ ├── utils.py │ ├── video/ │ │ ├── __init__.py │ │ ├── cost_estimation.py │ │ └── utils.py │ └── ws/ │ ├── __init__.py │ └── constants.py ├── blog/ │ └── evaluating-claude.md ├── design-docs/ │ ├── agent-tool-calling-flow.md │ ├── agentic-runner-refactor.md │ ├── commits-and-variants.md │ ├── general.md │ ├── images-in-update-history.md │ ├── prompt-history-refactor.md │ └── variant-system.md ├── docker-compose.yml ├── frontend/ │ ├── .eslintrc.cjs │ ├── .gitignore │ ├── Dockerfile │ ├── components.json │ ├── index.html │ ├── jest.config.js │ ├── package.json │ ├── postcss.config.js │ ├── src/ │ │ ├── App.tsx │ │ ├── components/ │ │ │ ├── ImageLightbox.tsx │ │ │ ├── ImageUpload.tsx │ │ │ ├── ImportCodeSection.tsx │ │ │ ├── TermsOfServiceDialog.tsx │ │ │ ├── UpdateImageUpload.tsx │ │ │ ├── agent/ │ │ │ │ └── AgentActivity.tsx │ │ │ ├── commits/ │ │ │ │ ├── types.ts │ │ │ │ └── utils.ts │ │ │ ├── core/ │ │ │ │ ├── KeyboardShortcutBadge.tsx │ │ │ │ ├── Spinner.tsx │ │ │ │ ├── StackLabel.tsx │ │ │ │ └── WorkingPulse.tsx │ │ │ ├── evals/ │ │ │ │ ├── AllEvalsPage.tsx │ │ │ │ ├── BestOfNEvalsPage.tsx │ │ │ │ ├── EvalNavigation.tsx │ │ │ │ ├── EvalsPage.tsx │ │ │ │ ├── InputFileSelector.tsx │ │ │ │ ├── OpenAIInputComparePage.tsx │ │ │ │ ├── PairwiseEvalsPage.tsx │ │ │ │ ├── RatingPicker.tsx │ │ │ │ └── RunEvalsPage.tsx │ │ │ ├── generate-from-text/ │ │ │ │ └── GenerateFromText.tsx │ │ │ ├── history/ │ │ │ │ ├── HistoryDisplay.tsx │ │ │ │ ├── utils.test.ts │ │ │ │ └── utils.ts │ │ │ ├── messages/ │ │ │ │ ├── OnboardingNote.tsx │ │ │ │ ├── PicoBadge.tsx │ │ │ │ └── TipLink.tsx │ │ │ ├── preview/ │ │ │ │ ├── CodeMirror.tsx │ │ │ │ ├── CodePreview.tsx │ │ │ │ ├── CodeTab.tsx │ │ │ │ ├── PreviewComponent.tsx │ │ │ │ ├── PreviewPane.tsx │ │ │ │ ├── download.ts │ │ │ │ ├── extractHtml.ts │ │ │ │ └── simpleHash.ts │ │ │ ├── recording/ │ │ │ │ ├── ScreenRecorder.tsx │ │ │ │ └── utils.ts │ │ │ ├── select-and-edit/ │ │ │ │ └── utils.ts │ │ │ ├── settings/ │ │ │ │ ├── GenerationSettings.tsx │ │ │ │ ├── OutputSettingsSection.tsx │ │ │ │ └── SettingsTab.tsx │ │ │ ├── sidebar/ │ │ │ │ ├── IconStrip.tsx │ │ │ │ └── Sidebar.tsx │ │ │ ├── start-pane/ │ │ │ │ └── StartPane.tsx │ │ │ ├── thinking/ │ │ │ │ └── ThinkingIndicator.tsx │ │ │ ├── ui/ │ │ │ │ ├── accordion.tsx │ │ │ │ ├── alert-dialog.tsx │ │ │ │ ├── badge.tsx │ │ │ │ ├── button.tsx │ │ │ │ ├── checkbox.tsx │ │ │ │ ├── collapsible.tsx │ │ │ │ ├── dialog.tsx │ │ │ │ ├── hover-card.tsx │ │ │ │ ├── input.tsx │ │ │ │ ├── label.tsx │ │ │ │ ├── popover.tsx │ │ │ │ ├── progress.tsx │ │ │ │ ├── scroll-area.tsx │ │ │ │ ├── select.tsx │ │ │ │ ├── separator.tsx │ │ │ │ ├── switch.tsx │ │ │ │ ├── tabs.tsx │ │ │ │ └── textarea.tsx │ │ │ ├── unified-input/ │ │ │ │ ├── UnifiedInputPane.tsx │ │ │ │ └── tabs/ │ │ │ │ ├── ImportTab.tsx │ │ │ │ ├── TextTab.tsx │ │ │ │ ├── UploadTab.tsx │ │ │ │ └── UrlTab.tsx │ │ │ └── variants/ │ │ │ └── Variants.tsx │ │ ├── config.ts │ │ ├── constants.ts │ │ ├── generateCode.ts │ │ ├── hooks/ │ │ │ ├── useBrowserTabIndicator.ts │ │ │ ├── usePersistedState.ts │ │ │ └── useThrottle.ts │ │ ├── index.css │ │ ├── lib/ │ │ │ ├── models.ts │ │ │ ├── prompt-history.test.ts │ │ │ ├── prompt-history.ts │ │ │ ├── stacks.ts │ │ │ ├── takeScreenshot.ts │ │ │ └── utils.ts │ │ ├── main.tsx │ │ ├── setupTests.ts │ │ ├── store/ │ │ │ ├── app-store.ts │ │ │ └── project-store.ts │ │ ├── tests/ │ │ │ ├── fixtures/ │ │ │ │ └── simple_page.html │ │ │ └── qa.test.ts │ │ ├── types.ts │ │ ├── urls.ts │ │ └── vite-env.d.ts │ ├── tailwind.config.js │ ├── tsconfig.json │ ├── tsconfig.node.json │ └── vite.config.ts ├── package.json └── plan.md
SYMBOL INDEX (662 symbols across 138 files)
FILE: backend/agent/engine.py
class AgentEngine (line 22) | class AgentEngine:
method __init__ (line 23) | def __init__(
method _next_event_id (line 60) | def _next_event_id(self, prefix: str) -> str:
method _send (line 63) | async def _send(
method _mark_preview_length (line 72) | def _mark_preview_length(self, tool_event_id: Optional[str], length: i...
method _stream_code_preview (line 79) | async def _stream_code_preview(self, tool_event_id: Optional[str], con...
method _handle_streamed_tool_delta (line 101) | async def _handle_streamed_tool_delta(
method _run_with_session (line 149) | async def _run_with_session(self, session: ProviderSession) -> str:
method run (line 228) | async def run(self, model: Llm, prompt_messages: List[ChatCompletionMe...
method _finalize_response (line 245) | async def _finalize_response(self, assistant_text: str) -> str:
FILE: backend/agent/providers/anthropic/image.py
function process_image (line 46) | def process_image(image_data_url: str) -> tuple[str, str]:
FILE: backend/agent/providers/anthropic/provider.py
function _convert_openai_messages_to_claude (line 34) | def _convert_openai_messages_to_claude(
function serialize_anthropic_tools (line 63) | def serialize_anthropic_tools(
class AnthropicParseState (line 78) | class AnthropicParseState:
function _parse_stream_event (line 84) | async def _parse_stream_event(
function _extract_tool_calls (line 148) | def _extract_tool_calls(final_message: Any) -> List[ToolCall]:
function _extract_anthropic_usage (line 174) | def _extract_anthropic_usage(final_message: Any) -> TokenUsage:
class AnthropicProviderSession (line 196) | class AnthropicProviderSession(ProviderSession):
method __init__ (line 197) | def __init__(
method stream_turn (line 212) | async def stream_turn(self, on_event: EventSink) -> ProviderTurn:
method append_tool_results (line 255) | def append_tool_results(
method close (line 289) | async def close(self) -> None:
FILE: backend/agent/providers/base.py
class StreamEvent (line 15) | class StreamEvent:
class ProviderTurn (line 24) | class ProviderTurn:
class ExecutedToolCall (line 32) | class ExecutedToolCall:
class ProviderSession (line 40) | class ProviderSession(Protocol):
method stream_turn (line 41) | async def stream_turn(self, on_event: EventSink) -> ProviderTurn:
method append_tool_results (line 44) | def append_tool_results(
method close (line 51) | async def close(self) -> None:
FILE: backend/agent/providers/factory.py
function create_provider_session (line 16) | def create_provider_session(
FILE: backend/agent/providers/gemini.py
function serialize_gemini_tools (line 28) | def serialize_gemini_tools(tools: List[CanonicalToolDefinition]) -> List...
function _get_gemini_api_model_name (line 40) | def _get_gemini_api_model_name(model: Llm) -> str:
function _get_thinking_level_for_model (line 52) | def _get_thinking_level_for_model(model: Llm) -> str:
function _extract_text_from_content (line 67) | def _extract_text_from_content(content: str | List[Dict[str, Any]]) -> str:
function _detect_mime_type_from_base64 (line 78) | def _detect_mime_type_from_base64(base64_data: str) -> str | None:
function _extract_images_from_content (line 101) | def _extract_images_from_content(content: str | List[Dict[str, Any]]) ->...
function _convert_message_to_gemini_content (line 131) | def _convert_message_to_gemini_content(
class GeminiParseState (line 176) | class GeminiParseState:
function _extract_usage (line 183) | def _extract_usage(chunk: types.GenerateContentResponse) -> TokenUsage |...
function _parse_chunk (line 209) | async def _parse_chunk(
class GeminiProviderSession (line 260) | class GeminiProviderSession(ProviderSession):
method __init__ (line 261) | def __init__(
method stream_turn (line 278) | async def stream_turn(self, on_event: EventSink) -> ProviderTurn:
method append_tool_results (line 320) | def append_tool_results(
method close (line 344) | async def close(self) -> None:
FILE: backend/agent/providers/openai.py
function _convert_message_to_responses_input (line 27) | def _convert_message_to_responses_input(
function _get_event_attr (line 56) | def _get_event_attr(event: Any, key: str, default: Any = None) -> Any:
function _copy_schema (line 64) | def _copy_schema(schema: Dict[str, Any]) -> Dict[str, Any]:
function _nullable_type (line 68) | def _nullable_type(type_value: Any) -> Any:
function _make_responses_schema_strict (line 78) | def _make_responses_schema_strict(schema: Dict[str, Any]) -> Dict[str, A...
function serialize_openai_tools (line 109) | def serialize_openai_tools(
class OpenAIResponsesParseState (line 126) | class OpenAIResponsesParseState:
function _extract_openai_usage (line 136) | def _extract_openai_usage(response: Any) -> TokenUsage:
function parse_event (line 161) | async def parse_event(
function _build_provider_turn (line 353) | def _build_provider_turn(state: OpenAIResponsesParseState) -> ProviderTurn:
class OpenAIProviderSession (line 407) | class OpenAIProviderSession(ProviderSession):
method __init__ (line 408) | def __init__(
method stream_turn (line 427) | async def stream_turn(self, on_event: EventSink) -> ProviderTurn:
method append_tool_results (line 459) | def append_tool_results(
method close (line 479) | async def close(self) -> None:
FILE: backend/agent/providers/pricing.py
class ModelPricing (line 6) | class ModelPricing:
FILE: backend/agent/providers/token_usage.py
class TokenUsage (line 9) | class TokenUsage:
method accumulate (line 40) | def accumulate(self, other: TokenUsage) -> None:
method cost (line 47) | def cost(self, pricing: ModelPricing) -> float:
method total_input_tokens (line 56) | def total_input_tokens(self) -> int:
method cache_hit_rate_percent (line 60) | def cache_hit_rate_percent(self) -> float:
FILE: backend/agent/runner.py
class Agent (line 4) | class Agent(AgentEngine):
FILE: backend/agent/state.py
class AgentFileState (line 10) | class AgentFileState:
function ensure_str (line 15) | def ensure_str(value: Any) -> str:
function extract_text_content (line 21) | def extract_text_content(message: ChatCompletionMessageParam) -> str:
function seed_file_state_from_messages (line 32) | def seed_file_state_from_messages(
FILE: backend/agent/tools/definitions.py
function _create_schema (line 6) | def _create_schema() -> Dict[str, Any]:
function _edit_schema (line 23) | def _edit_schema() -> Dict[str, Any]:
function _image_schema (line 59) | def _image_schema() -> Dict[str, Any]:
function _remove_background_schema (line 75) | def _remove_background_schema() -> Dict[str, Any]:
function _retrieve_option_schema (line 91) | def _retrieve_option_schema() -> Dict[str, Any]:
function canonical_tool_definitions (line 104) | def canonical_tool_definitions(
FILE: backend/agent/tools/parsing.py
function parse_json_arguments (line 8) | def parse_json_arguments(raw_args: Any) -> Tuple[Dict[str, Any], Optiona...
function _strip_incomplete_escape (line 22) | def _strip_incomplete_escape(value: str) -> str:
function _extract_partial_json_string (line 36) | def _extract_partial_json_string(raw_text: str, key: str) -> Optional[str]:
function extract_content_from_args (line 83) | def extract_content_from_args(raw_args: Any) -> Optional[str]:
function extract_path_from_args (line 93) | def extract_path_from_args(raw_args: Any) -> Optional[str]:
FILE: backend/agent/tools/runtime.py
class AgentToolRuntime (line 16) | class AgentToolRuntime:
method __init__ (line 17) | def __init__(
method execute (line 31) | async def execute(self, tool_call: ToolCall) -> ToolExecutionResult:
method _create_file (line 59) | def _create_file(self, args: Dict[str, Any]) -> ToolExecutionResult:
method _generate_diff (line 93) | def _generate_diff(old_content: str, new_content: str, path: str) -> D...
method _apply_single_edit (line 118) | def _apply_single_edit(
method _edit_file (line 138) | def _edit_file(self, args: Dict[str, Any]) -> ToolExecutionResult:
method _generate_images (line 217) | async def _generate_images(self, args: Dict[str, Any]) -> ToolExecutio...
method _remove_background (line 272) | async def _remove_background(self, args: Dict[str, Any]) -> ToolExecut...
method _retrieve_option (line 332) | def _retrieve_option(self, args: Dict[str, Any]) -> ToolExecutionResult:
FILE: backend/agent/tools/summaries.py
function summarize_text (line 8) | def summarize_text(value: str, limit: int = 240) -> str:
function summarize_tool_input (line 14) | def summarize_tool_input(tool_call: ToolCall, file_state: AgentFileState...
FILE: backend/agent/tools/types.py
class ToolCall (line 6) | class ToolCall:
class ToolExecutionResult (line 13) | class ToolExecutionResult:
class CanonicalToolDefinition (line 21) | class CanonicalToolDefinition:
FILE: backend/codegen/test_utils.py
class TestUtils (line 5) | class TestUtils(unittest.TestCase):
method test_extract_html_content_with_html_tags (line 7) | def test_extract_html_content_with_html_tags(self):
method test_extract_html_content_without_html_tags (line 13) | def test_extract_html_content_without_html_tags(self):
method test_extract_html_content_with_partial_html_tags (line 19) | def test_extract_html_content_with_partial_html_tags(self):
method test_extract_html_content_with_multiple_html_tags (line 25) | def test_extract_html_content_with_multiple_html_tags(self):
method test_extract_html_content_some_explanation_before (line 33) | def test_extract_html_content_some_explanation_before(self):
method test_markdown_tags (line 43) | def test_markdown_tags(self):
method test_doctype_text (line 49) | def test_doctype_text(self):
FILE: backend/codegen/utils.py
function extract_html_content (line 4) | def extract_html_content(text: str) -> str:
FILE: backend/debug/DebugFileWriter.py
class DebugFileWriter (line 8) | class DebugFileWriter:
method __init__ (line 9) | def __init__(self):
method write_to_file (line 22) | def write_to_file(self, filename: str, content: str) -> None:
method extract_html_content (line 29) | def extract_html_content(self, text: str) -> str:
FILE: backend/evals/core.py
function generate_code_for_image (line 15) | async def generate_code_for_image(image_url: str, stack: Stack, model: L...
FILE: backend/evals/runner.py
function _resolve_eval_filenames (line 16) | def _resolve_eval_filenames(input_files: Optional[List[str]]) -> List[str]:
function _output_html_filename (line 23) | def _output_html_filename(original_filename: str, attempt_idx: int) -> str:
function get_eval_output_subfolder (line 27) | def get_eval_output_subfolder(stack: Stack, model: str) -> str:
function count_pending_eval_tasks (line 33) | def count_pending_eval_tasks(
function generate_code_and_time (line 58) | async def generate_code_and_time(
function run_image_evals (line 110) | async def run_image_evals(
FILE: backend/evals/utils.py
function image_to_data_url (line 4) | async def image_to_data_url(filepath: str):
FILE: backend/fs_logging/openai_input_compare.py
class OpenAIInputDifference (line 15) | class OpenAIInputDifference:
class OpenAIInputComparison (line 25) | class OpenAIInputComparison:
function _extract_input_items (line 32) | def _extract_input_items(payload: Any) -> list[JSONValue]:
function _as_json_dict (line 44) | def _as_json_dict(value: JSONValue) -> dict[str, JSONValue]:
function _as_json_list (line 48) | def _as_json_list(value: JSONValue) -> list[JSONValue]:
function _append_dict_path (line 52) | def _append_dict_path(path: str, key: str) -> str:
function _append_list_path (line 58) | def _append_list_path(path: str, index: int) -> str:
function _find_first_value_difference (line 62) | def _find_first_value_difference(
function compare_openai_inputs (line 129) | def compare_openai_inputs(
function format_openai_input_comparison (line 204) | def format_openai_input_comparison(comparison: OpenAIInputComparison) ->...
function compare_openai_input_json_strings (line 234) | def compare_openai_input_json_strings(
FILE: backend/fs_logging/openai_input_formatting.py
function truncate_for_log (line 8) | def truncate_for_log(value: Any, max_len: int = 120) -> str:
function as_dict (line 15) | def as_dict(value: Any) -> dict[str, Any] | None:
function to_serializable (line 46) | def to_serializable(value: Any) -> Any:
function summarize_content_part (line 63) | def summarize_content_part(part: Any) -> str:
function summarize_function_call_output_payload (line 96) | def summarize_function_call_output_payload(output_text: str) -> str:
function summarize_responses_input_item (line 158) | def summarize_responses_input_item(index: int, item: Any) -> str:
FILE: backend/fs_logging/openai_turn_inputs.py
function _render_json_scalar (line 20) | def _render_json_scalar(value: Any) -> str:
function _render_json_node (line 38) | def _render_json_node(value: Any, label: str | None = None) -> str:
function _render_copy_controls (line 80) | def _render_copy_controls(copy_target_id: str, button_label: str) -> str:
function _log_openai_turn_input (line 90) | def _log_openai_turn_input(model: Llm, turn_index: int, input_items: Seq...
function _is_openai_turn_input_console_enabled (line 103) | def _is_openai_turn_input_console_enabled() -> bool:
class OpenAITurnInputItem (line 109) | class OpenAITurnInputItem:
class OpenAITurnUsageSummary (line 116) | class OpenAITurnUsageSummary:
class OpenAITurnInputReport (line 127) | class OpenAITurnInputReport:
class OpenAITurnInputLogger (line 135) | class OpenAITurnInputLogger:
method record_turn_input (line 142) | def record_turn_input(
method record_turn_usage (line 170) | def record_turn_usage(self, usage: TokenUsage) -> None:
method write_html_report (line 186) | def write_html_report(self) -> str | None:
method _render_html_report (line 209) | def _render_html_report(self) -> str:
FILE: backend/image_generation/generation.py
function process_tasks (line 13) | async def process_tasks(
function generate_image_dalle (line 45) | async def generate_image_dalle(
function generate_image_replicate (line 63) | async def generate_image_replicate(prompt: str, api_key: str) -> str:
FILE: backend/image_generation/replicate.py
function _build_headers (line 15) | def _build_headers(api_token: str) -> dict[str, str]:
function _extract_prediction_id (line 22) | def _extract_prediction_id(response_json: Mapping[str, Any]) -> str:
function _poll_prediction (line 29) | async def _poll_prediction(
function _run_prediction (line 55) | async def _run_prediction(
function _extract_output_url (line 83) | def _extract_output_url(result: Any, context: str) -> str:
function call_replicate_model (line 104) | async def call_replicate_model(
function call_replicate_version (line 114) | async def call_replicate_version(
function remove_background (line 124) | async def remove_background(image_url: str, api_token: str) -> str:
function call_replicate (line 139) | async def call_replicate(input: dict[str, str | int], api_token: str) ->...
FILE: backend/llm.py
class Llm (line 6) | class Llm(Enum):
class Completion (line 35) | class Completion(TypedDict):
function get_openai_api_name (line 110) | def get_openai_api_name(model: Llm) -> str:
function get_openai_reasoning_effort (line 114) | def get_openai_reasoning_effort(model: Llm) -> str | None:
FILE: backend/main.py
function log_debug_mode (line 16) | async def log_debug_mode() -> None:
FILE: backend/prompts/create/__init__.py
function build_create_prompt_from_input (line 9) | def build_create_prompt_from_input(
FILE: backend/prompts/create/image.py
function build_image_prompt_messages (line 7) | def build_image_prompt_messages(
FILE: backend/prompts/create/text.py
function build_text_prompt_messages (line 8) | def build_text_prompt_messages(
FILE: backend/prompts/create/video.py
function build_video_prompt_messages (line 7) | def build_video_prompt_messages(
FILE: backend/prompts/message_builder.py
function _wrap_assistant_file_content (line 10) | def _wrap_assistant_file_content(content: str, path: str = "index.html")...
function build_history_message (line 17) | def build_history_message(item: PromptHistoryMessage) -> ChatCompletionM...
FILE: backend/prompts/pipeline.py
function build_prompt_messages (line 12) | async def build_prompt_messages(
FILE: backend/prompts/plan.py
function derive_prompt_construction_plan (line 9) | def derive_prompt_construction_plan(
FILE: backend/prompts/policies.py
function build_selected_stack_policy (line 4) | def build_selected_stack_policy(stack: Stack) -> str:
function build_user_image_policy (line 8) | def build_user_image_policy(image_generation_enabled: bool) -> str:
FILE: backend/prompts/prompt_types.py
class UserTurnInput (line 4) | class UserTurnInput(TypedDict):
class PromptHistoryMessage (line 12) | class PromptHistoryMessage(TypedDict):
class PromptConstructionPlan (line 38) | class PromptConstructionPlan(TypedDict):
FILE: backend/prompts/request_parsing.py
function _to_string_list (line 6) | def _to_string_list(value: object) -> List[str]:
function parse_prompt_content (line 13) | def parse_prompt_content(raw_prompt: object) -> UserTurnInput:
function parse_prompt_history (line 26) | def parse_prompt_history(raw_history: object) -> List[PromptHistoryMessa...
FILE: backend/prompts/update/from_file_snapshot.py
function build_update_prompt_from_file_snapshot (line 11) | def build_update_prompt_from_file_snapshot(
FILE: backend/prompts/update/from_history.py
function build_update_prompt_from_history (line 11) | def build_update_prompt_from_history(
FILE: backend/routes/evals.py
class Eval (line 26) | class Eval(BaseModel):
class InputFile (line 31) | class InputFile(BaseModel):
function get_eval_input_files (line 37) | async def get_eval_input_files():
function get_evals (line 54) | async def get_evals(folder: str):
class PairwiseEvalResponse (line 105) | class PairwiseEvalResponse(BaseModel):
function get_pairwise_evals (line 112) | async def get_pairwise_evals(
class RunEvalsRequest (line 183) | class RunEvalsRequest(BaseModel):
class OpenAIInputCompareRequest (line 190) | class OpenAIInputCompareRequest(BaseModel):
class OpenAIInputCompareDifferenceResponse (line 195) | class OpenAIInputCompareDifferenceResponse(BaseModel):
class OpenAIInputCompareResponse (line 204) | class OpenAIInputCompareResponse(BaseModel):
function _load_openai_input_compare_payload (line 212) | def _load_openai_input_compare_payload(raw_json: str, side: str) -> object:
function compare_openai_inputs_for_evals (line 233) | async def compare_openai_inputs_for_evals(
function run_evals (line 261) | async def run_evals(request: RunEvalsRequest) -> List[str]:
function _count_eval_files (line 277) | def _count_eval_files(selected_files: List[str]) -> int:
function run_evals_stream (line 286) | async def run_evals_stream(request: RunEvalsRequest):
function get_models (line 398) | async def get_models():
class BestOfNEvalsResponse (line 407) | class BestOfNEvalsResponse(BaseModel):
function get_best_of_n_evals (line 413) | async def get_best_of_n_evals(request: Request):
class OutputFolder (line 491) | class OutputFolder(BaseModel):
function get_output_folders (line 498) | async def get_output_folders():
FILE: backend/routes/generate_code.py
class PipelineContext (line 78) | class PipelineContext:
method send_message (line 92) | def send_message(self):
method throw_error (line 97) | def throw_error(self):
class Middleware (line 102) | class Middleware(ABC):
method process (line 106) | async def process(
class Pipeline (line 113) | class Pipeline:
method __init__ (line 116) | def __init__(self):
method use (line 119) | def use(self, middleware: Middleware) -> "Pipeline":
method execute (line 124) | async def execute(self, websocket: WebSocket) -> None:
method _wrap_middleware (line 138) | def _wrap_middleware(
class WebSocketCommunicator (line 151) | class WebSocketCommunicator:
method __init__ (line 154) | def __init__(self, websocket: WebSocket):
method accept (line 158) | async def accept(self) -> None:
method send_message (line 163) | async def send_message(
method throw_error (line 198) | async def throw_error(self, message: str) -> None:
method receive_params (line 209) | async def receive_params(self) -> Dict[str, Any]:
method close (line 215) | async def close(self) -> None:
class ExtractedParams (line 226) | class ExtractedParams:
class ParameterExtractionStage (line 241) | class ParameterExtractionStage:
method __init__ (line 244) | def __init__(self, throw_error: Callable[[str], Coroutine[Any, Any, No...
method extract_and_validate (line 247) | async def extract_and_validate(self, params: Dict[str, Any]) -> Extrac...
method _get_from_settings_dialog_or_env (line 340) | def _get_from_settings_dialog_or_env(
class ModelSelectionStage (line 356) | class ModelSelectionStage:
method __init__ (line 359) | def __init__(self, throw_error: Callable[[str], Coroutine[Any, Any, No...
method select_models (line 362) | async def select_models(
method _get_variant_models (line 396) | def _get_variant_models(
class PromptCreationStage (line 447) | class PromptCreationStage:
method __init__ (line 450) | def __init__(self, throw_error: Callable[[str], Coroutine[Any, Any, No...
method build_prompt_messages (line 453) | async def build_prompt_messages(
class PostProcessingStage (line 478) | class PostProcessingStage:
method __init__ (line 481) | def __init__(self):
method process_completions (line 484) | async def process_completions(
class AgenticGenerationStage (line 493) | class AgenticGenerationStage:
method __init__ (line 496) | def __init__(
method process_variants (line 516) | async def process_variants(
method _run_variant (line 540) | async def _run_variant(
class WebSocketSetupMiddleware (line 634) | class WebSocketSetupMiddleware(Middleware):
method process (line 637) | async def process(
class ParameterExtractionMiddleware (line 651) | class ParameterExtractionMiddleware(Middleware):
method process (line 654) | async def process(
class StatusBroadcastMiddleware (line 675) | class StatusBroadcastMiddleware(Middleware):
method process (line 678) | async def process(
class PromptCreationMiddleware (line 699) | class PromptCreationMiddleware(Middleware):
method process (line 702) | async def process(
class CodeGenerationMiddleware (line 713) | class CodeGenerationMiddleware(Middleware):
method process (line 716) | async def process(
class PostProcessingMiddleware (line 779) | class PostProcessingMiddleware(Middleware):
method process (line 782) | async def process(
function stream_code (line 794) | async def stream_code(websocket: WebSocket):
FILE: backend/routes/home.py
function get_status (line 9) | async def get_status():
FILE: backend/routes/screenshot.py
function normalize_url (line 10) | def normalize_url(url: str) -> str:
function bytes_to_data_url (line 40) | def bytes_to_data_url(image_bytes: bytes, mime_type: str) -> str:
function capture_screenshot (line 45) | async def capture_screenshot(
class ScreenshotRequest (line 76) | class ScreenshotRequest(BaseModel):
class ScreenshotResponse (line 81) | class ScreenshotResponse(BaseModel):
function app_screenshot (line 86) | async def app_screenshot(request: ScreenshotRequest):
FILE: backend/run_evals.py
function main (line 10) | async def main():
FILE: backend/run_image_generation_evals.py
function generate_and_save_images (line 42) | async def generate_and_save_images(
function main (line 79) | async def main() -> None:
FILE: backend/tests/test_agent_tool_runtime.py
function test_edit_file_returns_structured_result_with_diff (line 8) | def test_edit_file_returns_structured_result_with_diff() -> None:
function test_execute_edit_file_uses_updated_result_shape (line 40) | async def test_execute_edit_file_uses_updated_result_shape() -> None:
FILE: backend/tests/test_agent_tools.py
function test_canonical_tool_definitions_include_generate_images_when_enabled (line 4) | def test_canonical_tool_definitions_include_generate_images_when_enabled...
function test_canonical_tool_definitions_exclude_generate_images_when_disabled (line 9) | def test_canonical_tool_definitions_exclude_generate_images_when_disable...
function test_edit_file_tool_description_matches_runtime_output_shape (line 14) | def test_edit_file_tool_description_matches_runtime_output_shape() -> None:
FILE: backend/tests/test_batching.py
function test_process_tasks_batches_replicate_calls (line 12) | async def test_process_tasks_batches_replicate_calls(
function test_remove_background_batches_calls (line 39) | async def test_remove_background_batches_calls(
FILE: backend/tests/test_codegen_utils.py
function test_extract_html_content_from_wrapped_file_tag (line 4) | def test_extract_html_content_from_wrapped_file_tag() -> None:
FILE: backend/tests/test_evals_openai_input_compare.py
function test_compare_openai_inputs_for_evals_returns_first_difference (line 8) | async def test_compare_openai_inputs_for_evals_returns_first_difference(...
function test_compare_openai_inputs_for_evals_rejects_invalid_json (line 32) | async def test_compare_openai_inputs_for_evals_rejects_invalid_json() ->...
FILE: backend/tests/test_image_generation_replicate.py
function test_extract_output_url_from_string (line 6) | def test_extract_output_url_from_string() -> None:
function test_extract_output_url_from_dict (line 13) | def test_extract_output_url_from_dict() -> None:
function test_extract_output_url_from_list (line 20) | def test_extract_output_url_from_list() -> None:
function test_extract_output_url_from_list_item_dict (line 27) | def test_extract_output_url_from_list_item_dict() -> None:
function test_extract_output_url_invalid_raises (line 36) | def test_extract_output_url_invalid_raises() -> None:
function test_call_replicate_uses_flux_model (line 42) | async def test_call_replicate_uses_flux_model(monkeypatch: pytest.Monkey...
function test_remove_background_uses_version_and_normalizes_output (line 63) | async def test_remove_background_uses_version_and_normalizes_output(
FILE: backend/tests/test_model_selection.py
class TestModelSelectionAllKeys (line 7) | class TestModelSelectionAllKeys:
method setup_method (line 10) | def setup_method(self):
method test_gemini_anthropic_create (line 16) | async def test_gemini_anthropic_create(self):
method test_gemini_anthropic_update_text (line 35) | async def test_gemini_anthropic_update_text(self):
method test_gemini_anthropic_update (line 52) | async def test_gemini_anthropic_update(self):
method test_video_create_prefers_gemini_minimal_then_3_1_high (line 69) | async def test_video_create_prefers_gemini_minimal_then_3_1_high(self):
method test_video_update_prefers_gemini_minimal_then_3_1_high (line 86) | async def test_video_update_prefers_gemini_minimal_then_3_1_high(self):
class TestModelSelectionOpenAIAnthropic (line 103) | class TestModelSelectionOpenAIAnthropic:
method setup_method (line 106) | def setup_method(self):
method test_openai_anthropic (line 112) | async def test_openai_anthropic(self):
class TestModelSelectionAnthropicOnly (line 131) | class TestModelSelectionAnthropicOnly:
method setup_method (line 134) | def setup_method(self):
method test_anthropic_only (line 140) | async def test_anthropic_only(self):
class TestModelSelectionOpenAIOnly (line 159) | class TestModelSelectionOpenAIOnly:
method setup_method (line 162) | def setup_method(self):
method test_openai_only (line 168) | async def test_openai_only(self):
class TestModelSelectionNoKeys (line 187) | class TestModelSelectionNoKeys:
method setup_method (line 190) | def setup_method(self):
method test_no_keys_raises_error (line 196) | async def test_no_keys_raises_error(self):
FILE: backend/tests/test_openai_input_compare.py
function test_compare_openai_inputs_returns_none_for_identical_inputs (line 9) | def test_compare_openai_inputs_returns_none_for_identical_inputs() -> None:
function test_compare_openai_inputs_finds_first_different_block_and_field (line 26) | def test_compare_openai_inputs_finds_first_different_block_and_field() -...
function test_compare_openai_inputs_accepts_raw_input_arrays (line 70) | def test_compare_openai_inputs_accepts_raw_input_arrays() -> None:
FILE: backend/tests/test_openai_provider_session.py
class _EmptyAsyncStream (line 12) | class _EmptyAsyncStream:
method __aiter__ (line 13) | def __aiter__(self) -> "_EmptyAsyncStream":
method __anext__ (line 16) | async def __anext__(self) -> object:
class _FakeResponses (line 20) | class _FakeResponses:
method __init__ (line 21) | def __init__(self) -> None:
method create (line 24) | async def create(self, **kwargs: Any) -> _EmptyAsyncStream:
class _FakeOpenAIClient (line 29) | class _FakeOpenAIClient:
method __init__ (line 30) | def __init__(self) -> None:
method close (line 33) | async def close(self) -> None:
function _noop_event_sink (line 37) | async def _noop_event_sink(_: Any) -> None:
function _test_tools (line 41) | def _test_tools() -> list[dict[str, Any]]:
function test_openai_provider_session_omits_prompt_cache_key_across_turns (line 60) | async def test_openai_provider_session_omits_prompt_cache_key_across_tur...
function test_openai_provider_session_omits_prompt_cache_key_for_all_prompts (line 121) | async def test_openai_provider_session_omits_prompt_cache_key_for_all_pr...
function test_openai_provider_session_uses_gpt_5_4_none_reasoning_effort (line 155) | async def test_openai_provider_session_uses_gpt_5_4_none_reasoning_effor...
function test_openai_provider_session_uses_gpt_5_4_high_reasoning_effort (line 174) | async def test_openai_provider_session_uses_gpt_5_4_high_reasoning_effor...
FILE: backend/tests/test_openai_reasoning_parser.py
function test_reasoning_summary_part_skipped_after_summary_delta (line 12) | async def test_reasoning_summary_part_skipped_after_summary_delta() -> N...
function test_reasoning_summary_part_added_and_done_emits_once (line 38) | async def test_reasoning_summary_part_added_and_done_emits_once() -> None:
function test_convert_image_url_defaults_to_high_detail (line 66) | def test_convert_image_url_defaults_to_high_detail() -> None:
function test_convert_image_url_preserves_explicit_detail (line 78) | def test_convert_image_url_preserves_explicit_detail() -> None:
FILE: backend/tests/test_openai_turn_input_logging.py
function test_openai_turn_input_logger_writes_html_report (line 8) | def test_openai_turn_input_logger_writes_html_report(tmp_path, monkeypat...
function test_openai_turn_input_logger_preserves_full_large_payloads (line 55) | def test_openai_turn_input_logger_preserves_full_large_payloads(
function test_openai_turn_input_logger_includes_request_payload (line 82) | def test_openai_turn_input_logger_includes_request_payload(
function test_openai_turn_input_logger_disabled_writes_nothing (line 110) | def test_openai_turn_input_logger_disabled_writes_nothing(tmp_path, monk...
function test_openai_turn_input_logger_summarizes_function_call_output (line 123) | def test_openai_turn_input_logger_summarizes_function_call_output(
FILE: backend/tests/test_parameter_extraction_stage.py
function test_extracts_gemini_api_key_from_settings_dialog (line 9) | async def test_extracts_gemini_api_key_from_settings_dialog() -> None:
function test_extracts_gemini_api_key_from_env_when_not_in_request (line 27) | async def test_extracts_gemini_api_key_from_env_when_not_in_request(monk...
FILE: backend/tests/test_prompt_summary.py
function test_format_prompt_summary (line 15) | def test_format_prompt_summary():
function test_print_prompt_summary (line 39) | def test_print_prompt_summary():
function test_print_prompt_summary_long_content (line 64) | def test_print_prompt_summary_long_content():
function test_format_prompt_summary_no_truncate (line 94) | def test_format_prompt_summary_no_truncate():
function test_print_prompt_summary_no_truncate (line 114) | def test_print_prompt_summary_no_truncate():
function test_format_prompt_preview_collapses_long_content (line 137) | def test_format_prompt_preview_collapses_long_content():
function test_print_prompt_preview (line 153) | def test_print_prompt_preview():
FILE: backend/tests/test_prompts.py
class ExpectedResult (line 16) | class ExpectedResult(TypedDict):
function assert_structure_match (line 20) | def assert_structure_match(actual: object, expected: object, path: str =...
class TestCreatePrompt (line 77) | class TestCreatePrompt:
method wrapped_file (line 91) | def wrapped_file(content: str) -> str:
method test_plan_create_uses_create_from_input (line 94) | def test_plan_create_uses_create_from_input(self) -> None:
method test_plan_update_with_history_uses_history_strategy (line 104) | def test_plan_update_with_history_uses_history_strategy(self) -> None:
method test_plan_update_without_history_uses_file_snapshot_strategy (line 114) | def test_plan_update_without_history_uses_file_snapshot_strategy(self)...
method test_image_mode_create_single_image (line 125) | async def test_image_mode_create_single_image(self) -> None:
method test_image_mode_create_with_image_generation_disabled (line 174) | async def test_image_mode_create_with_image_generation_disabled(self) ...
method test_image_mode_update_with_history (line 211) | async def test_image_mode_update_with_history(self) -> None:
method test_update_history_with_image_generation_disabled (line 270) | async def test_update_history_with_image_generation_disabled(self) -> ...
method test_text_mode_create_generation (line 296) | async def test_text_mode_create_generation(self) -> None:
method test_text_mode_update_with_history (line 339) | async def test_text_mode_update_with_history(self) -> None:
method test_video_mode_basic_prompt_creation (line 406) | async def test_video_mode_basic_prompt_creation(self) -> None:
method test_create_raises_on_unsupported_input_mode (line 460) | async def test_create_raises_on_unsupported_input_mode(self) -> None:
method test_image_mode_update_with_single_image_in_history (line 477) | async def test_image_mode_update_with_single_image_in_history(self) ->...
method test_image_mode_update_with_multiple_images_in_history (line 547) | async def test_image_mode_update_with_multiple_images_in_history(self)...
method test_update_with_empty_images_arrays (line 625) | async def test_update_with_empty_images_arrays(self) -> None:
method test_update_bootstraps_from_file_state_when_history_is_empty (line 682) | async def test_update_bootstraps_from_file_state_when_history_is_empty...
method test_update_requires_history_or_file_state (line 750) | async def test_update_requires_history_or_file_state(self) -> None:
method test_update_history_requires_user_message (line 761) | async def test_update_history_requires_user_message(self) -> None:
FILE: backend/tests/test_request_parsing.py
function test_parse_prompt_content_with_valid_data (line 4) | def test_parse_prompt_content_with_valid_data() -> None:
function test_parse_prompt_content_filters_invalid_media_types (line 20) | def test_parse_prompt_content_filters_invalid_media_types() -> None:
function test_parse_prompt_content_defaults_for_invalid_payload (line 36) | def test_parse_prompt_content_defaults_for_invalid_payload() -> None:
function test_parse_prompt_history_with_valid_entries (line 42) | def test_parse_prompt_history_with_valid_entries() -> None:
function test_parse_prompt_history_filters_invalid_items (line 76) | def test_parse_prompt_history_filters_invalid_items() -> None:
function test_parse_prompt_history_defaults_for_invalid_payload (line 102) | def test_parse_prompt_history_defaults_for_invalid_payload() -> None:
FILE: backend/tests/test_screenshot.py
class TestNormalizeUrl (line 5) | class TestNormalizeUrl:
method test_url_without_protocol (line 8) | def test_url_without_protocol(self):
method test_url_with_http_protocol (line 14) | def test_url_with_http_protocol(self):
method test_url_with_https_protocol (line 19) | def test_url_with_https_protocol(self):
method test_url_with_path_and_params (line 24) | def test_url_with_path_and_params(self):
method test_url_with_whitespace (line 30) | def test_url_with_whitespace(self):
method test_invalid_protocols (line 35) | def test_invalid_protocols(self):
method test_localhost_urls (line 43) | def test_localhost_urls(self):
method test_ip_address_urls (line 49) | def test_ip_address_urls(self):
method test_complex_urls (line 55) | def test_complex_urls(self):
FILE: backend/tests/test_status_broadcast.py
function test_video_update_broadcasts_two_variants (line 15) | async def test_video_update_broadcasts_two_variants() -> None:
function test_image_update_broadcasts_two_variants (line 67) | async def test_image_update_broadcasts_two_variants() -> None:
FILE: backend/tests/test_token_usage.py
class TestAccumulate (line 17) | class TestAccumulate:
method test_sums_all_fields (line 18) | def test_sums_all_fields(self) -> None:
method test_accumulate_zero_is_noop (line 26) | def test_accumulate_zero_is_noop(self) -> None:
method test_multiple_accumulations (line 31) | def test_multiple_accumulations(self) -> None:
class TestCost (line 46) | class TestCost:
method test_basic_cost (line 47) | def test_basic_cost(self) -> None:
method test_zero_tokens_zero_cost (line 53) | def test_zero_tokens_zero_cost(self) -> None:
method test_cache_heavy_scenario (line 58) | def test_cache_heavy_scenario(self) -> None:
method test_anthropic_with_cache_write (line 69) | def test_anthropic_with_cache_write(self) -> None:
class TestCacheHitRate (line 89) | class TestCacheHitRate:
method test_zero_total_input_is_zero_percent (line 90) | def test_zero_total_input_is_zero_percent(self) -> None:
method test_cache_hit_rate_without_cache_write (line 95) | def test_cache_hit_rate_without_cache_write(self) -> None:
method test_cache_hit_rate_includes_cache_write_in_denominator (line 100) | def test_cache_hit_rate_includes_cache_write_in_denominator(self) -> N...
function _gemini_chunk (line 111) | def _gemini_chunk(
class TestGeminiExtract (line 130) | class TestGeminiExtract:
method test_normal_response (line 131) | def test_normal_response(self) -> None:
method test_no_cache (line 143) | def test_no_cache(self) -> None:
method test_no_usage_metadata_returns_none (line 150) | def test_no_usage_metadata_returns_none(self) -> None:
method test_none_subfields_default_to_zero (line 154) | def test_none_subfields_default_to_zero(self) -> None:
function _openai_response (line 173) | def _openai_response(
class TestOpenAIExtract (line 190) | class TestOpenAIExtract:
method test_normal_response (line 191) | def test_normal_response(self) -> None:
method test_no_cache (line 202) | def test_no_cache(self) -> None:
method test_no_usage_returns_empty (line 210) | def test_no_usage_returns_empty(self) -> None:
method test_no_input_tokens_details (line 215) | def test_no_input_tokens_details(self) -> None:
function _anthropic_message (line 233) | def _anthropic_message(
class TestAnthropicExtract (line 250) | class TestAnthropicExtract:
method test_normal_response (line 251) | def test_normal_response(self) -> None:
method test_no_cache (line 262) | def test_no_cache(self) -> None:
method test_no_usage_returns_empty (line 270) | def test_no_usage_returns_empty(self) -> None:
class TestModelPricing (line 281) | class TestModelPricing:
method test_known_models_have_pricing (line 282) | def test_known_models_have_pricing(self) -> None:
method test_unknown_model_returns_none (line 294) | def test_unknown_model_returns_none(self) -> None:
method test_anthropic_has_cache_write_rate (line 297) | def test_anthropic_has_cache_write_rate(self) -> None:
method test_openai_gemini_no_cache_write (line 301) | def test_openai_gemini_no_cache_write(self) -> None:
FILE: backend/utils.py
function pprint_prompt (line 8) | def pprint_prompt(prompt_messages: List[ChatCompletionMessageParam]):
function format_prompt_summary (line 12) | def format_prompt_summary(prompt_messages: List[ChatCompletionMessagePar...
function print_prompt_summary (line 39) | def print_prompt_summary(prompt_messages: List[ChatCompletionMessagePara...
function _collapse_preview_text (line 79) | def _collapse_preview_text(text: str, max_chars: int = 280) -> str:
function format_prompt_preview (line 100) | def format_prompt_preview(
function print_prompt_preview (line 142) | def print_prompt_preview(prompt_messages: List[ChatCompletionMessagePara...
function truncate_data_strings (line 172) | def truncate_data_strings(data: List[ChatCompletionMessageParam]): # ty...
FILE: backend/video/cost_estimation.py
class MediaResolution (line 7) | class MediaResolution(Enum):
class TokenEstimate (line 14) | class TokenEstimate:
class CostEstimate (line 21) | class CostEstimate:
function get_model_api_name (line 56) | def get_model_api_name(model: Llm) -> str:
function estimate_video_input_tokens (line 68) | def estimate_video_input_tokens(
function estimate_output_tokens (line 84) | def estimate_output_tokens(
function calculate_cost (line 106) | def calculate_cost(
function estimate_video_generation_cost (line 127) | def estimate_video_generation_cost(
function format_cost_estimate (line 153) | def format_cost_estimate(cost: CostEstimate) -> str:
function format_detailed_input_estimate (line 162) | def format_detailed_input_estimate(
function get_video_duration_from_bytes (line 187) | def get_video_duration_from_bytes(video_bytes: bytes) -> float | None:
FILE: backend/video/utils.py
function extract_tag_content (line 4) | def extract_tag_content(tag: str, text: str) -> str:
function get_video_bytes_and_mime_type (line 21) | def get_video_bytes_and_mime_type(video_data_url: str) -> tuple[bytes, s...
FILE: frontend/src/App.tsx
function App (line 37) | function App() {
FILE: frontend/src/components/ImageLightbox.tsx
constant MIN_ZOOM (line 5) | const MIN_ZOOM = 0.5;
constant MAX_ZOOM (line 6) | const MAX_ZOOM = 10;
constant DEFAULT_DISPLAY_WIDTH (line 7) | const DEFAULT_DISPLAY_WIDTH = 1000;
type ImageLightboxProps (line 9) | interface ImageLightboxProps {
function ImageLightbox (line 14) | function ImageLightbox({ image, onClose }: ImageLightboxProps) {
FILE: frontend/src/components/ImageUpload.tsx
function fileToDataURL (line 41) | function fileToDataURL(file: File): Promise<string> {
type FileWithPreview (line 64) | type FileWithPreview = {
type Props (line 68) | interface Props {
function ImageUpload (line 79) | function ImageUpload({ setReferenceImages, onUploadStateChange, stack, s...
FILE: frontend/src/components/ImportCodeSection.tsx
type Props (line 17) | interface Props {
function ImportCodeSection (line 21) | function ImportCodeSection({ importFromCode }: Props) {
FILE: frontend/src/components/TermsOfServiceDialog.tsx
constant LOGOS (line 14) | const LOGOS = ["microsoft", "amazon", "mit", "stanford", "bytedance", "b...
FILE: frontend/src/components/UpdateImageUpload.tsx
constant MAX_UPDATE_IMAGES (line 6) | const MAX_UPDATE_IMAGES = 5;
function fileToDataURL (line 9) | function fileToDataURL(file: File): Promise<string> {
type Props (line 18) | interface Props {
function UpdateImagePreview (line 23) | function UpdateImagePreview({ updateImages, setUpdateImages }: Props) {
function UpdateImageUpload (line 56) | function UpdateImageUpload({ updateImages, setUpdateImages }: Props) {
FILE: frontend/src/components/agent/AgentActivity.tsx
function CodePreviewBlock (line 29) | function CodePreviewBlock({ code, isGenerating }: { code: string; isGene...
function isFiniteNumber (line 52) | function isFiniteNumber(value: unknown): value is number {
function formatDurationMs (line 56) | function formatDurationMs(milliseconds: number): string {
function formatDuration (line 67) | function formatDuration(startedAt?: number, endedAt?: number): string {
function formatElapsedSince (line 72) | function formatElapsedSince(timestampMs: number | undefined, nowMs: numb...
function formatVariantWallClockDuration (line 77) | function formatVariantWallClockDuration(
function getEventIcon (line 88) | function getEventIcon(type: AgentEventType, toolName?: string) {
function getEventTitle (line 113) | function getEventTitle(event: AgentEvent): string {
function renderToolDetails (line 158) | function renderToolDetails(event: AgentEvent, variantCode?: string) {
function AgentEventCard (line 372) | function AgentEventCard({
function AgentActivity (line 459) | function AgentActivity() {
FILE: frontend/src/components/commits/types.ts
type CommitHash (line 3) | type CommitHash = string;
type VariantStatus (line 5) | type VariantStatus = "generating" | "complete" | "cancelled" | "error";
type AgentEventStatus (line 7) | type AgentEventStatus = "running" | "complete" | "error";
type AgentEventType (line 8) | type AgentEventType = "thinking" | "assistant" | "tool";
type AgentEvent (line 10) | type AgentEvent = {
type VariantHistoryMessage (line 22) | type VariantHistoryMessage = {
type Variant (line 29) | type Variant = {
type BaseCommit (line 43) | type BaseCommit = {
type CommitType (line 52) | type CommitType = "ai_create" | "ai_edit" | "code_create";
type AiCreateCommit (line 54) | type AiCreateCommit = BaseCommit & {
type AiEditCommit (line 59) | type AiEditCommit = BaseCommit & {
type CodeCreateCommit (line 64) | type CodeCreateCommit = BaseCommit & {
type Commit (line 69) | type Commit = AiCreateCommit | AiEditCommit | CodeCreateCommit;
FILE: frontend/src/components/commits/utils.ts
function createCommit (line 9) | function createCommit(
FILE: frontend/src/components/core/KeyboardShortcutBadge.tsx
type KeyboardShortcutBadgeProps (line 4) | interface KeyboardShortcutBadgeProps {
FILE: frontend/src/components/core/Spinner.tsx
function Spinner (line 1) | function Spinner() {
FILE: frontend/src/components/core/StackLabel.tsx
type StackLabelProps (line 4) | interface StackLabelProps {
FILE: frontend/src/components/core/WorkingPulse.tsx
function WorkingPulse (line 1) | function WorkingPulse() {
FILE: frontend/src/components/evals/AllEvalsPage.tsx
function AllEvalsPage (line 3) | function AllEvalsPage() {
FILE: frontend/src/components/evals/BestOfNEvalsPage.tsx
type Eval (line 6) | interface Eval {
type Outcome (line 12) | type Outcome = number | "tie" | null;
type BestOfNEvalsResponse (line 14) | interface BestOfNEvalsResponse {
type OutputFolder (line 19) | interface OutputFolder {
function BestOfNEvalsPage (line 25) | function BestOfNEvalsPage() {
FILE: frontend/src/components/evals/EvalNavigation.tsx
function EvalNavigation (line 3) | function EvalNavigation() {
FILE: frontend/src/components/evals/EvalsPage.tsx
type Eval (line 6) | interface Eval {
type RatingCriteria (line 11) | interface RatingCriteria {
type OutputDisplay (line 19) | interface OutputDisplay {
function EvalsPage (line 23) | function EvalsPage() {
FILE: frontend/src/components/evals/InputFileSelector.tsx
type InputFile (line 6) | interface InputFile {
type InputFileSelectorProps (line 11) | interface InputFileSelectorProps {
function InputFileSelector (line 15) | function InputFileSelector({ onFilesSelected }: InputFileSelectorProps) {
FILE: frontend/src/components/evals/OpenAIInputComparePage.tsx
type OpenAIInputDifference (line 6) | interface OpenAIInputDifference {
type OpenAIInputCompareResponse (line 15) | interface OpenAIInputCompareResponse {
function formatJson (line 23) | function formatJson(value: unknown): string {
function OpenAIInputComparePage (line 28) | function OpenAIInputComparePage() {
FILE: frontend/src/components/evals/PairwiseEvalsPage.tsx
type Eval (line 6) | interface Eval {
type Outcome (line 11) | type Outcome = "left" | "right" | "tie" | null;
type PairwiseEvalsResponse (line 13) | interface PairwiseEvalsResponse {
function PairwiseEvalsPage (line 19) | function PairwiseEvalsPage() {
FILE: frontend/src/components/evals/RatingPicker.tsx
type RatingPickerProps (line 1) | interface RatingPickerProps {
function RatingPicker (line 7) | function RatingPicker({
FILE: frontend/src/components/evals/RunEvalsPage.tsx
type ModelResponse (line 9) | interface ModelResponse {
type EvalProgressEvent (line 14) | interface EvalProgressEvent {
type FailedTask (line 34) | interface FailedTask {
function RunEvalsPage (line 40) | function RunEvalsPage() {
FILE: frontend/src/components/generate-from-text/GenerateFromText.tsx
type GenerateFromTextProps (line 6) | interface GenerateFromTextProps {
function GenerateFromText (line 10) | function GenerateFromText({ doCreateFromText }: GenerateFromTextProps) {
FILE: frontend/src/components/history/HistoryDisplay.tsx
function MediaThumbnail (line 6) | function MediaThumbnail({
function ExpandedMedia (line 57) | function ExpandedMedia({
function HistoryDisplay (line 110) | function HistoryDisplay() {
FILE: frontend/src/components/history/utils.ts
function displayHistoryItemType (line 3) | function displayHistoryItemType(itemType: CommitType) {
function extractTagName (line 34) | function extractTagName(html: string): string {
function getCommitMedia (line 39) | function getCommitMedia(commit: Commit): { images: string[]; videos: str...
function summarizeHistoryItem (line 49) | function summarizeHistoryItem(commit: Commit): string {
function getSelectedElementTag (line 65) | function getSelectedElementTag(commit: Commit): string | null {
type RenderedHistoryItem (line 72) | type RenderedHistoryItem = Omit<Commit, "type"> & {
FILE: frontend/src/components/messages/OnboardingNote.tsx
function OnboardingNote (line 1) | function OnboardingNote() {
FILE: frontend/src/components/messages/PicoBadge.tsx
function PicoBadge (line 1) | function PicoBadge() {
FILE: frontend/src/components/messages/TipLink.tsx
function TipLink (line 3) | function TipLink() {
FILE: frontend/src/components/preview/CodeMirror.tsx
type Props (line 16) | interface Props {
function CodeMirror (line 22) | function CodeMirror({ code, editorTheme, onCodeChange }: Props) {
FILE: frontend/src/components/preview/CodePreview.tsx
type Props (line 3) | interface Props {
function CodePreview (line 7) | function CodePreview({ code }: Props) {
FILE: frontend/src/components/preview/CodeTab.tsx
type Props (line 9) | interface Props {
function CodeTab (line 15) | function CodeTab({ code, setCode, settings }: Props) {
FILE: frontend/src/components/preview/PreviewComponent.tsx
type Props (line 7) | interface Props {
constant MOBILE_VIEWPORT_WIDTH (line 14) | const MOBILE_VIEWPORT_WIDTH = 375;
constant DESKTOP_VIEWPORT_WIDTH (line 15) | const DESKTOP_VIEWPORT_WIDTH = 1366;
function PreviewComponent (line 17) | function PreviewComponent({
FILE: frontend/src/components/preview/PreviewPane.tsx
function openInNewTab (line 24) | function openInNewTab(code: string) {
type Props (line 33) | interface Props {
function PreviewPane (line 38) | function PreviewPane({ settings, onOpenVersions }: Props) {
FILE: frontend/src/components/preview/extractHtml.ts
function extractHtml (line 2) | function extractHtml(code: string): string {
FILE: frontend/src/components/preview/simpleHash.ts
function simpleHash (line 2) | function simpleHash(str: string, seed = 0) {
FILE: frontend/src/components/recording/ScreenRecorder.tsx
type Props (line 10) | interface Props {
function ScreenRecorder (line 21) | function ScreenRecorder({
FILE: frontend/src/components/recording/utils.ts
function downloadBlob (line 1) | function downloadBlob(blob: Blob) {
function blobToBase64DataUrl (line 17) | function blobToBase64DataUrl(blob: Blob): Promise<string> {
FILE: frontend/src/components/select-and-edit/utils.ts
function removeHighlight (line 1) | function removeHighlight(element: HTMLElement) {
function addHighlight (line 7) | function addHighlight(element: HTMLElement) {
FILE: frontend/src/components/settings/GenerationSettings.tsx
type GenerationSettingsProps (line 8) | interface GenerationSettingsProps {
function setStack (line 20) | function setStack(stack: Stack) {
FILE: frontend/src/components/settings/OutputSettingsSection.tsx
type Props (line 12) | interface Props {
function OutputSettingsSection (line 19) | function OutputSettingsSection({
FILE: frontend/src/components/settings/SettingsTab.tsx
type Props (line 14) | interface Props {
function SettingsTab (line 21) | function SettingsTab({ settings, setSettings, appTheme, setAppTheme }: P...
FILE: frontend/src/components/sidebar/IconStrip.tsx
type IconStripProps (line 3) | interface IconStripProps {
function IconStrip (line 16) | function IconStrip({
FILE: frontend/src/components/sidebar/Sidebar.tsx
type SidebarProps (line 18) | interface SidebarProps {
constant MAX_UPDATE_IMAGES (line 26) | const MAX_UPDATE_IMAGES = 5;
function extractTagName (line 28) | function extractTagName(html: string): string {
function summarizeLatestChange (line 33) | function summarizeLatestChange(commit: Commit | null): string | null {
function getSelectedElementTag (line 53) | function getSelectedElementTag(commit: Commit | null): string | null {
function isSlowGeminiModel (line 60) | function isSlowGeminiModel(model?: string): boolean {
function Sidebar (line 67) | function Sidebar({
FILE: frontend/src/components/start-pane/StartPane.tsx
type Props (line 6) | interface Props {
FILE: frontend/src/components/thinking/ThinkingIndicator.tsx
function getLastSentence (line 6) | function getLastSentence(text: string): string {
function ThinkingIndicator (line 20) | function ThinkingIndicator() {
FILE: frontend/src/components/ui/badge.tsx
type BadgeProps (line 26) | interface BadgeProps
function Badge (line 30) | function Badge({ className, variant, ...props }: BadgeProps) {
FILE: frontend/src/components/ui/button.tsx
type ButtonProps (line 37) | interface ButtonProps
FILE: frontend/src/components/ui/input.tsx
type InputProps (line 5) | interface InputProps
FILE: frontend/src/components/ui/textarea.tsx
type TextareaProps (line 5) | interface TextareaProps
FILE: frontend/src/components/unified-input/UnifiedInputPane.tsx
type Props (line 10) | interface Props {
type InputTab (line 22) | type InputTab = "upload" | "url" | "text" | "import";
function UnifiedInputPane (line 24) | function UnifiedInputPane({
function UploadIcon (line 115) | function UploadIcon() {
function UrlIcon (line 135) | function UrlIcon() {
function TextIcon (line 154) | function TextIcon() {
function ImportIcon (line 174) | function ImportIcon() {
FILE: frontend/src/components/unified-input/tabs/ImportTab.tsx
type Props (line 9) | interface Props {
function ImportTab (line 13) | function ImportTab({ importFromCode }: Props) {
FILE: frontend/src/components/unified-input/tabs/TextTab.tsx
type Props (line 8) | interface Props {
constant EXAMPLE_PROMPTS (line 14) | const EXAMPLE_PROMPTS = [
function TextTab (line 21) | function TextTab({ doCreateFromText, stack, setStack }: Props) {
FILE: frontend/src/components/unified-input/tabs/UploadTab.tsx
function fileToDataURL (line 11) | function fileToDataURL(file: File): Promise<string> {
type FileWithPreview (line 31) | type FileWithPreview = {
constant MAX_FILES (line 35) | const MAX_FILES = 5;
type Props (line 43) | interface Props {
function UploadTab (line 53) | function UploadTab({ doCreate, stack, setStack }: Props) {
FILE: frontend/src/components/unified-input/tabs/UrlTab.tsx
type Props (line 9) | interface Props {
function isFigmaUrl (line 20) | function isFigmaUrl(url: string): boolean {
function UrlTab (line 24) | function UrlTab({ doCreate, screenshotOneApiKey, stack, setStack }: Prop...
FILE: frontend/src/components/variants/Variants.tsx
constant IFRAME_WIDTH (line 10) | const IFRAME_WIDTH = 1280;
constant IFRAME_HEIGHT (line 11) | const IFRAME_HEIGHT = 550;
type VariantThumbnailProps (line 13) | interface VariantThumbnailProps {
function VariantThumbnail (line 18) | function VariantThumbnail({ code, isSelected }: VariantThumbnailProps) {
function Variants (line 71) | function Variants() {
FILE: frontend/src/config.ts
constant IS_RUNNING_ON_CLOUD (line 2) | const IS_RUNNING_ON_CLOUD =
constant WS_BACKEND_URL (line 5) | const WS_BACKEND_URL =
constant HTTP_BACKEND_URL (line 8) | const HTTP_BACKEND_URL =
constant PICO_BACKEND_FORM_SECRET (line 11) | const PICO_BACKEND_FORM_SECRET =
FILE: frontend/src/constants.ts
constant APP_ERROR_WEB_SOCKET_CODE (line 2) | const APP_ERROR_WEB_SOCKET_CODE = 4332;
constant USER_CLOSE_WEB_SOCKET_CODE (line 3) | const USER_CLOSE_WEB_SOCKET_CODE = 4333;
FILE: frontend/src/generateCode.ts
constant ERROR_MESSAGE (line 9) | const ERROR_MESSAGE =
constant CANCEL_MESSAGE (line 12) | const CANCEL_MESSAGE = "Code generation cancelled";
type WebSocketResponse (line 14) | type WebSocketResponse = {
type CodeGenerationCallbacks (line 34) | interface CodeGenerationCallbacks {
function generateCode (line 53) | function generateCode(
FILE: frontend/src/hooks/useBrowserTabIndicator.ts
constant CODING_SETTINGS (line 3) | const CODING_SETTINGS = {
constant DEFAULT_SETTINGS (line 7) | const DEFAULT_SETTINGS = {
constant DEV_FAVICON_COLORS (line 12) | const DEV_FAVICON_COLORS = {
FILE: frontend/src/hooks/usePersistedState.ts
type PersistedState (line 3) | type PersistedState<T> = [T, Dispatch<SetStateAction<T>>];
function usePersistedState (line 5) | function usePersistedState<T>(defaultValue: T, key: string): PersistedSt...
FILE: frontend/src/hooks/useThrottle.ts
function useThrottle (line 6) | function useThrottle(value: string, interval = 500) {
FILE: frontend/src/lib/models.ts
type CodeGenerationModel (line 3) | enum CodeGenerationModel {
constant CODE_GENERATION_MODEL_DESCRIPTIONS (line 24) | const CODE_GENERATION_MODEL_DESCRIPTIONS: {
FILE: frontend/src/lib/prompt-history.ts
type GenerationRequest (line 9) | type GenerationRequest = CodeGenerationParams & {
type AssetsById (line 13) | type AssetsById = Record<string, PromptAsset>;
type GetAssetsById (line 14) | type GetAssetsById = () => AssetsById;
type UpsertPromptAssets (line 15) | type UpsertPromptAssets = (assets: PromptAsset[]) => void;
type CreateId (line 16) | type CreateId = () => string;
function cloneVariantHistory (line 18) | function cloneVariantHistory(
function registerAssetIds (line 28) | function registerAssetIds(
function resolveAssetIdsToDataUrls (line 65) | function resolveAssetIdsToDataUrls(
function toRequestHistory (line 75) | function toRequestHistory(
function buildUserHistoryMessage (line 87) | function buildUserHistoryMessage(
function buildAssistantHistoryMessage (line 100) | function buildAssistantHistoryMessage(
FILE: frontend/src/lib/stacks.ts
type Stack (line 3) | enum Stack {
constant STACK_DESCRIPTIONS (line 12) | const STACK_DESCRIPTIONS: {
FILE: frontend/src/lib/utils.ts
function cn (line 4) | function cn(...inputs: ClassValue[]) {
function capitalize (line 8) | function capitalize(str: string) {
FILE: frontend/src/store/app-store.ts
type AppStore (line 5) | interface AppStore {
FILE: frontend/src/store/project-store.ts
type ProjectStore (line 12) | interface ProjectStore {
FILE: frontend/src/tests/qa.test.ts
type Window (line 8) | interface Window {
constant RUN_E2E (line 15) | const RUN_E2E = process.env.RUN_E2E === "true";
constant TESTS_ROOT_PATH (line 18) | const TESTS_ROOT_PATH =
constant FIXTURES_PATH (line 22) | const FIXTURES_PATH = path.join(TESTS_ROOT_PATH, "fixtures");
constant SIMPLE_SCREENSHOT (line 23) | const SIMPLE_SCREENSHOT = path.join(FIXTURES_PATH, "simple_button.png");
constant SCREENSHOT_WITH_IMAGES (line 24) | const SCREENSHOT_WITH_IMAGES = path.join(
constant SIMPLE_HTML (line 28) | const SIMPLE_HTML = path.join(FIXTURES_PATH, "simple_page.html");
constant RESULTS_DIR (line 31) | const RESULTS_DIR = path.join(TESTS_ROOT_PATH, "results");
class App (line 197) | class App {
method constructor (line 203) | constructor(page: Page, stack: Stack, model: string, testId: string) {
method init (line 210) | async init() {
method setupLocalStorage (line 215) | async setupLocalStorage() {
method resetTestHooks (line 236) | async resetTestHooks() {
method takeScreenshot (line 244) | async takeScreenshot(step: string) {
method waitUntilVersionIsReady (line 250) | async waitUntilVersionIsReady(version: string) {
method switchToTab (line 263) | async switchToTab(testId: string) {
method generateFromUrl (line 267) | async generateFromUrl(url: string) {
method generateFromText (line 276) | async generateFromText(prompt: string) {
method uploadImage (line 285) | async uploadImage(screenshotPath: string) {
method importFromCode (line 302) | async importFromCode(code: string) {
method edit (line 321) | async edit(edit: string, version: string) {
method clickVersion (line 329) | async clickVersion(version: string) {
method regenerate (line 339) | async regenerate() {
method assertCodeActions (line 345) | async assertCodeActions() {
function setupRequestInterception (line 366) | async function setupRequestInterception(
function installDomTestHooks (line 394) | async function installDomTestHooks(page: Page) {
function installMockWebSocket (line 423) | async function installMockWebSocket(page: Page) {
FILE: frontend/src/types.ts
type EditorTheme (line 4) | enum EditorTheme {
type AppTheme (line 9) | enum AppTheme {
type Settings (line 15) | interface Settings {
type AppState (line 29) | enum AppState {
type ScreenRecorderState (line 35) | enum ScreenRecorderState {
type PromptMessageRole (line 41) | type PromptMessageRole = "user" | "assistant";
type PromptAssetType (line 42) | type PromptAssetType = "image" | "video";
type PromptAsset (line 44) | interface PromptAsset {
type PromptContent (line 50) | interface PromptContent {
type PromptHistoryMessage (line 57) | interface PromptHistoryMessage {
type CodeGenerationParams (line 64) | interface CodeGenerationParams {
type FullGenerationSettings (line 76) | type FullGenerationSettings = CodeGenerationParams & Settings;
FILE: frontend/src/urls.ts
constant URLS (line 1) | const URLS = {
Condensed preview — 228 files, each showing path, character count, and a content snippet. Download the .json file or copy for the full structured content (865K chars).
[
{
"path": ".claude/launch.json",
"chars": 199,
"preview": "{\n \"version\": \"0.0.1\",\n \"configurations\": [\n {\n \"name\": \"frontend\",\n \"runtimeExecutable\": \"yarn\",\n \""
},
{
"path": ".gitattributes",
"chars": 66,
"preview": "# Auto detect text files and perform LF normalization\n* text=auto\n"
},
{
"path": ".github/FUNDING.yml",
"chars": 14,
"preview": "github: [abi]\n"
},
{
"path": ".github/ISSUE_TEMPLATE/bug_report.md",
"chars": 416,
"preview": "---\nname: Bug report\nabout: Create a report to help us improve\ntitle: ''\nlabels: ''\nassignees: ''\n\n---\n\n**Describe the b"
},
{
"path": ".github/ISSUE_TEMPLATE/custom.md",
"chars": 126,
"preview": "---\nname: Custom issue template\nabout: Describe this issue template's purpose here.\ntitle: ''\nlabels: ''\nassignees: ''\n\n"
},
{
"path": ".github/ISSUE_TEMPLATE/feature_request.md",
"chars": 595,
"preview": "---\nname: Feature request\nabout: Suggest an idea for this project\ntitle: ''\nlabels: ''\nassignees: ''\n\n---\n\n**Is your fea"
},
{
"path": ".gitignore",
"chars": 208,
"preview": ".aider*\nnode_modules\n\n# Project-related files\n\n# Run logs\nbackend/run_logs/*\n\n# Weird Docker setup related files\nbackend"
},
{
"path": ".vscode/settings.json",
"chars": 149,
"preview": "{\n \"python.analysis.typeCheckingMode\": \"strict\",\n \"python.analysis.extraPaths\": [\"./backend\"],\n \"python.autoComplete."
},
{
"path": "AGENTS.md",
"chars": 1114,
"preview": "# Project Agent Instructions\n\nPython environment:\n\n- Always use the backend Poetry virtualenv (`backend-py3.10`) for Pyt"
},
{
"path": "CLAUDE.md",
"chars": 1114,
"preview": "# Project Agent Instructions\n\nPython environment:\n\n- Always use the backend Poetry virtualenv (`backend-py3.10`) for Pyt"
},
{
"path": "Evaluation.md",
"chars": 1193,
"preview": "## Evaluating models and prompts\n\nEvaluation dataset consists of 16 screenshots. A Python script for running screenshot-"
},
{
"path": "LICENSE",
"chars": 1065,
"preview": "MIT License\n\nCopyright (c) 2023 Abi Raja\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\no"
},
{
"path": "README.md",
"chars": 5242,
"preview": "# screenshot-to-code\n\nA simple tool to convert screenshots, mockups and Figma designs into clean, functional code using "
},
{
"path": "TESTING.md",
"chars": 1793,
"preview": "# Testing Guide\n\nThis guide explains how to run tests for the Screenshot to Code project.\n\n## Backend Tests\n\nThe backend"
},
{
"path": "Troubleshooting.md",
"chars": 1851,
"preview": "### Getting an OpenAI API key with GPT-4 model access\n\nYou don't need a ChatGPT Pro account. Screenshot to code uses API"
},
{
"path": "backend/.gitignore",
"chars": 2860,
"preview": "# Byte-compiled / optimized / DLL files\n__pycache__/\n*.py[cod]\n*$py.class\n\n# C extensions\n*.so\n\n# Distribution / packagi"
},
{
"path": "backend/.pre-commit-config.yaml",
"chars": 805,
"preview": "# See https://pre-commit.com for more information\n# See https://pre-commit.com/hooks.html for more hooks\nrepos:\n - repo"
},
{
"path": "backend/Dockerfile",
"chars": 475,
"preview": "FROM python:3.12.3-slim-bullseye\n\nENV POETRY_VERSION 1.8.0\n\n# Install system dependencies\nRUN pip install \"poetry==$POET"
},
{
"path": "backend/README.md",
"chars": 260,
"preview": "# Run the type checker\n\npoetry run pyright\n\n# Run tests\n\npoetry run pytest\n\n## Prompt Summary\n\nUse `print_prompt_summary"
},
{
"path": "backend/agent/engine.py",
"chars": 9370,
"preview": "import asyncio\nimport uuid\nfrom typing import Any, Awaitable, Callable, Dict, List, Optional\n\nfrom openai.types.chat imp"
},
{
"path": "backend/agent/providers/__init__.py",
"chars": 806,
"preview": "from agent.providers.anthropic import AnthropicProviderSession, serialize_anthropic_tools\nfrom agent.providers.base impo"
},
{
"path": "backend/agent/providers/anthropic/__init__.py",
"chars": 254,
"preview": "from agent.providers.anthropic.provider import (\n AnthropicProviderSession,\n serialize_anthropic_tools,\n _extra"
},
{
"path": "backend/agent/providers/anthropic/image.py",
"chars": 3447,
"preview": "# pyright: reportUnknownVariableType=false\n\"\"\"Claude-specific image processing.\n\nHandles resizing and compressing images"
},
{
"path": "backend/agent/providers/anthropic/provider.py",
"chars": 9917,
"preview": "# pyright: reportUnknownVariableType=false\nimport copy\nimport json\nimport uuid\nfrom dataclasses import dataclass, field\n"
},
{
"path": "backend/agent/providers/base.py",
"chars": 1111,
"preview": "from dataclasses import dataclass\nfrom typing import Any, Awaitable, Callable, Literal, Optional, Protocol\n\nfrom agent.t"
},
{
"path": "backend/agent/providers/factory.py",
"chars": 2258,
"preview": "from typing import Optional\n\nfrom anthropic import AsyncAnthropic\nfrom google import genai\nfrom openai import AsyncOpenA"
},
{
"path": "backend/agent/providers/gemini.py",
"chars": 11574,
"preview": "# pyright: reportUnknownVariableType=false\nimport base64\nimport copy\nimport uuid\nfrom dataclasses import dataclass, fiel"
},
{
"path": "backend/agent/providers/openai.py",
"chars": 17456,
"preview": "# pyright: reportUnknownVariableType=false\nimport copy\nimport json\nimport uuid\nfrom dataclasses import dataclass, field\n"
},
{
"path": "backend/agent/providers/pricing.py",
"chars": 1571,
"preview": "from dataclasses import dataclass\nfrom typing import Dict\n\n\n@dataclass\nclass ModelPricing:\n \"\"\"Per-million-token pric"
},
{
"path": "backend/agent/providers/token_usage.py",
"chars": 2458,
"preview": "from __future__ import annotations\n\nfrom dataclasses import dataclass\n\nfrom agent.providers.pricing import ModelPricing\n"
},
{
"path": "backend/agent/providers/types.py",
"chars": 297,
"preview": "from agent.providers.base import (\n EventSink,\n ExecutedToolCall,\n ProviderTurn,\n StreamEvent,\n)\n\n# Backward"
},
{
"path": "backend/agent/runner.py",
"chars": 74,
"preview": "from agent.engine import AgentEngine\n\n\nclass Agent(AgentEngine):\n pass\n"
},
{
"path": "backend/agent/state.py",
"chars": 1903,
"preview": "from dataclasses import dataclass\nfrom typing import Any, List\n\nfrom openai.types.chat import ChatCompletionMessageParam"
},
{
"path": "backend/agent/tools/__init__.py",
"chars": 728,
"preview": "from agent.tools.definitions import canonical_tool_definitions\nfrom agent.tools.parsing import (\n extract_content_fro"
},
{
"path": "backend/agent/tools/definitions.py",
"chars": 4925,
"preview": "from typing import Any, Dict, List\n\nfrom agent.tools.types import CanonicalToolDefinition\n\n\ndef _create_schema() -> Dict"
},
{
"path": "backend/agent/tools/parsing.py",
"chars": 2826,
"preview": "# pyright: reportUnknownVariableType=false\nimport json\nfrom typing import Any, Dict, Optional, Tuple\n\nfrom agent.state i"
},
{
"path": "backend/agent/tools/runtime.py",
"chars": 14896,
"preview": "# pyright: reportUnknownVariableType=false\nimport asyncio\nimport difflib\nfrom typing import Any, Dict, List, Optional, T"
},
{
"path": "backend/agent/tools/summaries.py",
"chars": 2308,
"preview": "# pyright: reportUnknownVariableType=false\nfrom typing import Any, Dict\n\nfrom agent.state import AgentFileState, ensure_"
},
{
"path": "backend/agent/tools/types.py",
"chars": 444,
"preview": "from dataclasses import dataclass\nfrom typing import Any, Dict, Optional\n\n\n@dataclass(frozen=True)\nclass ToolCall:\n i"
},
{
"path": "backend/codegen/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "backend/codegen/test_utils.py",
"chars": 2328,
"preview": "import unittest\nfrom codegen.utils import extract_html_content\n\n\nclass TestUtils(unittest.TestCase):\n\n def test_extra"
},
{
"path": "backend/codegen/utils.py",
"chars": 1048,
"preview": "import re\n\n\ndef extract_html_content(text: str) -> str:\n file_match = re.search(\n r\"<file\\s+path=\\\"[^\\\"]+\\\">\\s"
},
{
"path": "backend/config.py",
"chars": 695,
"preview": "import os\n\nNUM_VARIANTS = 4\nNUM_VARIANTS_VIDEO = 2\n\n# LLM-related\nOPENAI_API_KEY = os.environ.get(\"OPENAI_API_KEY\", None"
},
{
"path": "backend/custom_types.py",
"chars": 90,
"preview": "from typing import Literal\n\n\nInputMode = Literal[\n \"image\",\n \"video\",\n \"text\",\n]\n"
},
{
"path": "backend/debug/DebugFileWriter.py",
"chars": 987,
"preview": "import os\nimport logging\nimport uuid\n\nfrom config import DEBUG_DIR, IS_DEBUG_ENABLED\n\n\nclass DebugFileWriter:\n def __"
},
{
"path": "backend/debug/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "backend/evals/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "backend/evals/config.py",
"chars": 27,
"preview": "EVALS_DIR = \"./evals_data\"\n"
},
{
"path": "backend/evals/core.py",
"chars": 1713,
"preview": "from config import (\n ANTHROPIC_API_KEY,\n GEMINI_API_KEY,\n OPENAI_API_KEY,\n OPENAI_BASE_URL,\n)\nfrom llm impo"
},
{
"path": "backend/evals/runner.py",
"chars": 12936,
"preview": "from typing import Any, Awaitable, Callable, Coroutine, List, Optional, Tuple\nimport asyncio\nimport os\nfrom datetime imp"
},
{
"path": "backend/evals/utils.py",
"chars": 228,
"preview": "import base64\n\n\nasync def image_to_data_url(filepath: str):\n with open(filepath, \"rb\") as image_file:\n encoded"
},
{
"path": "backend/fs_logging/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "backend/fs_logging/openai_input_compare.py",
"chars": 8126,
"preview": "import json\nfrom dataclasses import dataclass\nfrom typing import Any, TypeAlias, cast\n\nfrom fs_logging.openai_input_form"
},
{
"path": "backend/fs_logging/openai_input_formatting.py",
"chars": 7728,
"preview": "# pyright: reportUnknownVariableType=false\nimport json\nfrom typing import Any\n\nfrom agent.state import ensure_str\n\n\ndef "
},
{
"path": "backend/fs_logging/openai_turn_inputs.py",
"chars": 17004,
"preview": "# pyright: reportUnknownVariableType=false\nimport json\nimport os\nimport uuid\nfrom dataclasses import dataclass, field\nfr"
},
{
"path": "backend/image_generation/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "backend/image_generation/core.py",
"chars": 216,
"preview": "from image_generation.generation import (\n generate_image_dalle,\n generate_image_replicate,\n process_tasks,\n)\n\n"
},
{
"path": "backend/image_generation/generation.py",
"chars": 2086,
"preview": "import asyncio\nimport time\nfrom typing import List, Literal, Union\n\nfrom openai import AsyncOpenAI\n\nfrom image_generatio"
},
{
"path": "backend/image_generation/replicate.py",
"chars": 4885,
"preview": "import asyncio\nimport httpx\nfrom typing import Any, Mapping, cast\n\n\nREPLICATE_API_BASE_URL = \"https://api.replicate.com/"
},
{
"path": "backend/llm.py",
"chars": 4875,
"preview": "from enum import Enum\nfrom typing import TypedDict\n\n\n# Actual model versions that are passed to the LLMs and stored in o"
},
{
"path": "backend/main.py",
"chars": 848,
"preview": "# Load environment variables first\nfrom dotenv import load_dotenv\n\nload_dotenv()\n\n\nfrom fastapi import FastAPI\nfrom fast"
},
{
"path": "backend/prompts/__init__.py",
"chars": 84,
"preview": "from prompts.system_prompt import SYSTEM_PROMPT\n\n__all__ = [\n \"SYSTEM_PROMPT\",\n]\n"
},
{
"path": "backend/prompts/create/__init__.py",
"chars": 1571,
"preview": "from custom_types import InputMode\nfrom prompts.create.image import build_image_prompt_messages\nfrom prompts.create.text"
},
{
"path": "backend/prompts/create/image.py",
"chars": 2134,
"preview": "from openai.types.chat import ChatCompletionContentPartParam, ChatCompletionMessageParam\n\nfrom prompts.prompt_types impo"
},
{
"path": "backend/prompts/create/text.py",
"chars": 923,
"preview": "from openai.types.chat import ChatCompletionMessageParam\n\nfrom prompts.prompt_types import Stack\nfrom prompts import sys"
},
{
"path": "backend/prompts/create/video.py",
"chars": 2096,
"preview": "from openai.types.chat import ChatCompletionContentPartParam, ChatCompletionMessageParam\nfrom prompts.prompt_types impor"
},
{
"path": "backend/prompts/message_builder.py",
"chars": 1670,
"preview": "from typing import cast\n\nfrom openai.types.chat import ChatCompletionContentPartParam, ChatCompletionMessageParam\n\nfrom "
},
{
"path": "backend/prompts/pipeline.py",
"chars": 1556,
"preview": "from custom_types import InputMode\nfrom prompts.create import build_create_prompt_from_input\nfrom prompts.plan import de"
},
{
"path": "backend/prompts/plan.py",
"chars": 1033,
"preview": "from custom_types import InputMode\nfrom prompts.prompt_types import (\n PromptConstructionPlan,\n PromptHistoryMessa"
},
{
"path": "backend/prompts/policies.py",
"chars": 585,
"preview": "from prompts.prompt_types import Stack\n\n\ndef build_selected_stack_policy(stack: Stack) -> str:\n return f\"Selected sta"
},
{
"path": "backend/prompts/prompt_types.py",
"chars": 972,
"preview": "from typing import List, Literal, TypedDict\n\n\nclass UserTurnInput(TypedDict):\n \"\"\"Normalized current user turn payloa"
},
{
"path": "backend/prompts/request_parsing.py",
"chars": 1637,
"preview": "from typing import List, cast\n\nfrom prompts.prompt_types import PromptHistoryMessage, UserTurnInput\n\n\ndef _to_string_lis"
},
{
"path": "backend/prompts/system_prompt.py",
"chars": 4330,
"preview": "SYSTEM_PROMPT = \"\"\"\nYou are a coding agent that's an expert at building front-ends.\n\n# Tone and style\n\n- Be extremely co"
},
{
"path": "backend/prompts/update/__init__.py",
"chars": 259,
"preview": "from prompts.update.from_file_snapshot import build_update_prompt_from_file_snapshot\nfrom prompts.update.from_history im"
},
{
"path": "backend/prompts/update/from_file_snapshot.py",
"chars": 1413,
"preview": "from typing import cast\n\nfrom openai.types.chat import ChatCompletionMessageParam\n\nfrom prompts import system_prompt\nfro"
},
{
"path": "backend/prompts/update/from_history.py",
"chars": 1851,
"preview": "from typing import cast\n\nfrom openai.types.chat import ChatCompletionMessageParam\n\nfrom prompts import system_prompt\nfro"
},
{
"path": "backend/pyproject.toml",
"chars": 732,
"preview": "[tool.poetry]\nname = \"backend\"\nversion = \"0.1.0\"\ndescription = \"\"\nauthors = [\"Abi Raja <abimanyuraja@gmail.com>\"]\nlicens"
},
{
"path": "backend/pyrightconfig.json",
"chars": 150,
"preview": "{\n \"exclude\": [\"image_generation.py\"],\n \"typeCheckingMode\": \"basic\",\n \"reportMissingTypeStubs\": \"none\",\n \"reportUnkn"
},
{
"path": "backend/pytest.ini",
"chars": 145,
"preview": "[pytest]\ntestpaths = tests\npython_files = test_*.py\npython_classes = Test*\npython_functions = test_*\naddopts = -v --tb=s"
},
{
"path": "backend/routes/evals.py",
"chars": 17956,
"preview": "import os\nimport asyncio\nimport json\nfrom fastapi import APIRouter, Query, Request, HTTPException\nfrom fastapi.responses"
},
{
"path": "backend/routes/generate_code.py",
"chars": 29147,
"preview": "import asyncio\nfrom dataclasses import dataclass, field\nfrom abc import ABC, abstractmethod\nimport traceback\nfrom typing"
},
{
"path": "backend/routes/home.py",
"chars": 324,
"preview": "from fastapi import APIRouter\nfrom fastapi.responses import HTMLResponse\n\n\nrouter = APIRouter()\n\n\n@router.get(\"/\")\nasync"
},
{
"path": "backend/routes/model_choice_sets.py",
"chars": 1529,
"preview": "from llm import Llm\n\n# Video variants always use Gemini.\nVIDEO_VARIANT_MODELS = (\n Llm.GEMINI_3_FLASH_PREVIEW_MINIMAL"
},
{
"path": "backend/routes/screenshot.py",
"chars": 3180,
"preview": "import base64\nfrom fastapi import APIRouter, HTTPException\nfrom pydantic import BaseModel\nimport httpx\nfrom urllib.parse"
},
{
"path": "backend/run_evals.py",
"chars": 1755,
"preview": "# Load environment variables first\nfrom dotenv import load_dotenv\n\nload_dotenv()\n\nimport asyncio\nfrom evals.runner impor"
},
{
"path": "backend/run_image_generation_evals.py",
"chars": 3270,
"preview": "import asyncio\nimport os\nfrom typing import List, Optional, Literal\nfrom dotenv import load_dotenv\nimport aiohttp\nfrom i"
},
{
"path": "backend/start.py",
"chars": 245,
"preview": "import argparse\n\nimport uvicorn\n\nif __name__ == \"__main__\":\n parser = argparse.ArgumentParser()\n parser.add_argume"
},
{
"path": "backend/tests/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "backend/tests/test_agent_tool_runtime.py",
"chars": 2169,
"preview": "import pytest\n\nfrom agent.state import AgentFileState\nfrom agent.tools.runtime import AgentToolRuntime\nfrom agent.tools."
},
{
"path": "backend/tests/test_agent_tools.py",
"chars": 761,
"preview": "from agent.tools import canonical_tool_definitions\n\n\ndef test_canonical_tool_definitions_include_generate_images_when_en"
},
{
"path": "backend/tests/test_batching.py",
"chars": 2203,
"preview": "import asyncio\n\nimport pytest\n\nfrom image_generation import generation\nfrom agent.tools.runtime import AgentToolRuntime\n"
},
{
"path": "backend/tests/test_codegen_utils.py",
"chars": 303,
"preview": "from codegen.utils import extract_html_content\n\n\ndef test_extract_html_content_from_wrapped_file_tag() -> None:\n text"
},
{
"path": "backend/tests/test_evals_openai_input_compare.py",
"chars": 1492,
"preview": "import pytest\nfrom fastapi import HTTPException\n\nfrom routes.evals import OpenAIInputCompareRequest, compare_openai_inpu"
},
{
"path": "backend/tests/test_image_generation_replicate.py",
"chars": 2632,
"preview": "import pytest\n\nfrom image_generation import replicate\n\n\ndef test_extract_output_url_from_string() -> None:\n assert (\n"
},
{
"path": "backend/tests/test_model_selection.py",
"chars": 6589,
"preview": "import pytest\nfrom unittest.mock import AsyncMock\nfrom routes.generate_code import ModelSelectionStage\nfrom llm import L"
},
{
"path": "backend/tests/test_openai_input_compare.py",
"chars": 3070,
"preview": "from typing import Any\n\nfrom fs_logging.openai_input_compare import (\n compare_openai_inputs,\n format_openai_input"
},
{
"path": "backend/tests/test_openai_provider_session.py",
"chars": 6233,
"preview": "import copy\nfrom typing import Any\n\nimport pytest\n\nfrom agent.providers.base import ExecutedToolCall, ProviderTurn\nfrom "
},
{
"path": "backend/tests/test_openai_reasoning_parser.py",
"chars": 2563,
"preview": "import pytest\n\nfrom agent.providers.openai import (\n OpenAIResponsesParseState,\n _convert_message_to_responses_inp"
},
{
"path": "backend/tests/test_openai_turn_input_logging.py",
"chars": 4616,
"preview": "from pathlib import Path\n\nfrom agent.providers.token_usage import TokenUsage\nfrom fs_logging.openai_turn_inputs import O"
},
{
"path": "backend/tests/test_parameter_extraction_stage.py",
"chars": 1153,
"preview": "from unittest.mock import AsyncMock\n\nimport pytest\n\nfrom routes.generate_code import ParameterExtractionStage\n\n\n@pytest."
},
{
"path": "backend/tests/test_prompt_summary.py",
"chars": 5118,
"preview": "import io\nimport sys\nfrom typing import cast\n\nfrom openai.types.chat import ChatCompletionMessageParam\n\nfrom utils impor"
},
{
"path": "backend/tests/test_prompts.py",
"chars": 30954,
"preview": "import pytest\nfrom unittest.mock import patch, MagicMock\nimport sys\nfrom typing import Any, Dict, List, TypedDict, cast\n"
},
{
"path": "backend/tests/test_request_parsing.py",
"chars": 2781,
"preview": "from prompts.request_parsing import parse_prompt_content, parse_prompt_history\n\n\ndef test_parse_prompt_content_with_vali"
},
{
"path": "backend/tests/test_screenshot.py",
"chars": 3004,
"preview": "import pytest\nfrom routes.screenshot import normalize_url\n\n\nclass TestNormalizeUrl:\n \"\"\"Test cases for URL normalizat"
},
{
"path": "backend/tests/test_status_broadcast.py",
"chars": 3300,
"preview": "from types import SimpleNamespace\nfrom unittest.mock import AsyncMock, MagicMock\nfrom typing import Any, cast\n\nimport py"
},
{
"path": "backend/tests/test_token_usage.py",
"chars": 10842,
"preview": "\"\"\"Tests for unified token usage tracking and cost computation.\"\"\"\n\nfrom types import SimpleNamespace\n\nfrom agent.provid"
},
{
"path": "backend/utils.py",
"chars": 6912,
"preview": "import copy\nimport json\nimport textwrap\nfrom typing import List\nfrom openai.types.chat import ChatCompletionMessageParam"
},
{
"path": "backend/video/__init__.py",
"chars": 681,
"preview": "from video.cost_estimation import (\n CostEstimate,\n MediaResolution,\n TokenEstimate,\n calculate_cost,\n es"
},
{
"path": "backend/video/cost_estimation.py",
"chars": 6169,
"preview": "from dataclasses import dataclass\nfrom enum import Enum\nfrom typing import Tuple\nfrom llm import Llm\n\n\nclass MediaResolu"
},
{
"path": "backend/video/utils.py",
"chars": 812,
"preview": "import base64\n\n\ndef extract_tag_content(tag: str, text: str) -> str:\n \"\"\"\n Extracts content for a given tag from t"
},
{
"path": "backend/ws/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "backend/ws/constants.py",
"chars": 129,
"preview": "# WebSocket protocol (RFC 6455) allows for the use of custom close codes in the range 4000-4999\nAPP_ERROR_WEB_SOCKET_COD"
},
{
"path": "blog/evaluating-claude.md",
"chars": 5073,
"preview": "# Claude 3 for converting screenshots to code\n\nClaude 3 dropped yesterday, claiming to rival GPT-4 on a wide variety of "
},
{
"path": "design-docs/agent-tool-calling-flow.md",
"chars": 5344,
"preview": "# Agent Tool-Calling Flow (Backend)\n\nThis document explains exactly what happens after prompt messages are built and a v"
},
{
"path": "design-docs/agentic-runner-refactor.md",
"chars": 2606,
"preview": "# Agentic Runner Refactor Spec\n\n## Goals\n- Reduce duplicated streaming logic across OpenAI/Anthropic/Gemini runners.\n- C"
},
{
"path": "design-docs/commits-and-variants.md",
"chars": 6914,
"preview": "# Commits and Non-Blocking Variants\n\nThis document explains how the commit system and non-blocking variant generation wo"
},
{
"path": "design-docs/general.md",
"chars": 262,
"preview": "## Input mode\n\n- Input mode is used for model selection (but it shouldn’t be really? I don’t know)\n - Model selection\n "
},
{
"path": "design-docs/images-in-update-history.md",
"chars": 1120,
"preview": "# Images in Update History\n\n## Status: ✅ IMPLEMENTED\n\nMultiple images in update history are fully supported in the backe"
},
{
"path": "design-docs/prompt-history-refactor.md",
"chars": 5464,
"preview": "# Prompt History Refactor (Frontend -> Backend)\n\n## Goal\nSimplify edit prompt history so we no longer reconstruct conver"
},
{
"path": "design-docs/variant-system.md",
"chars": 2320,
"preview": "# Variant System\n\n## Overview\n\nThe variant system generates multiple code options in parallel, allowing users to compare"
},
{
"path": "docker-compose.yml",
"chars": 609,
"preview": "version: '3.9'\n\nservices:\n backend:\n build: \n context: ./backend\n dockerfile: Dockerfile\n \n env_file"
},
{
"path": "frontend/.eslintrc.cjs",
"chars": 436,
"preview": "module.exports = {\n root: true,\n env: { browser: true, es2020: true },\n extends: [\n 'eslint:recommended',\n 'plu"
},
{
"path": "frontend/.gitignore",
"chars": 305,
"preview": "# Logs\nlogs\n*.log\nnpm-debug.log*\nyarn-debug.log*\nyarn-error.log*\npnpm-debug.log*\nlerna-debug.log*\n\nnode_modules\ndist\ndis"
},
{
"path": "frontend/Dockerfile",
"chars": 496,
"preview": "FROM node:22-bullseye-slim\n\n# Set the working directory in the container\nWORKDIR /app\n\n# Copy package.json and yarn.lock"
},
{
"path": "frontend/components.json",
"chars": 324,
"preview": "{\n \"$schema\": \"https://ui.shadcn.com/schema.json\",\n \"style\": \"new-york\",\n \"rsc\": false,\n \"tsx\": true,\n \"tailwind\": "
},
{
"path": "frontend/index.html",
"chars": 1809,
"preview": "<!DOCTYPE html>\n<html lang=\"en\">\n <head>\n <meta charset=\"UTF-8\" />\n <link rel=\"icon\" type=\"image/png\" href=\"/favi"
},
{
"path": "frontend/jest.config.js",
"chars": 187,
"preview": "export default {\n preset: \"ts-jest\",\n testEnvironment: \"node\",\n setupFiles: [\"<rootDir>/src/setupTests.ts\"],\n transf"
},
{
"path": "frontend/package.json",
"chars": 2701,
"preview": "{\n \"name\": \"screenshot-to-code\",\n \"private\": true,\n \"version\": \"0.0.0\",\n \"type\": \"module\",\n \"scripts\": {\n \"dev\":"
},
{
"path": "frontend/postcss.config.js",
"chars": 80,
"preview": "export default {\n plugins: {\n tailwindcss: {},\n autoprefixer: {},\n },\n}\n"
},
{
"path": "frontend/src/App.tsx",
"chars": 26942,
"preview": "import { useEffect, useRef, useState } from \"react\";\nimport { generateCode } from \"./generateCode\";\nimport { AppState, A"
},
{
"path": "frontend/src/components/ImageLightbox.tsx",
"chars": 8577,
"preview": "import { useEffect, useRef, useState, useCallback } from \"react\";\nimport { LuMinus, LuPlus, LuX } from \"react-icons/lu\";"
},
{
"path": "frontend/src/components/ImageUpload.tsx",
"chars": 10937,
"preview": "import { useState, useEffect, useMemo, useRef, useCallback } from \"react\";\nimport { useDropzone } from \"react-dropzone\";"
},
{
"path": "frontend/src/components/ImportCodeSection.tsx",
"chars": 1950,
"preview": "import { useState } from \"react\";\nimport { Button } from \"./ui/button\";\nimport {\n Dialog,\n DialogContent,\n DialogDesc"
},
{
"path": "frontend/src/components/TermsOfServiceDialog.tsx",
"chars": 3526,
"preview": "import React from \"react\";\nimport {\n AlertDialog,\n AlertDialogAction,\n AlertDialogContent,\n AlertDialogFooter,\n Ale"
},
{
"path": "frontend/src/components/UpdateImageUpload.tsx",
"chars": 4536,
"preview": "import { useRef } from \"react\";\nimport { toast } from \"react-hot-toast\";\nimport { Cross2Icon } from \"@radix-ui/react-ico"
},
{
"path": "frontend/src/components/agent/AgentActivity.tsx",
"chars": 21262,
"preview": "import { useEffect, useRef, useState } from \"react\";\nimport { useProjectStore } from \"../../store/project-store\";\nimport"
},
{
"path": "frontend/src/components/commits/types.ts",
"chars": 1565,
"preview": "import { PromptContent, PromptMessageRole } from \"../../types\";\n\nexport type CommitHash = string;\n\nexport type VariantSt"
},
{
"path": "frontend/src/components/commits/utils.ts",
"chars": 675,
"preview": "import { nanoid } from \"nanoid\";\nimport {\n AiCreateCommit,\n AiEditCommit,\n CodeCreateCommit,\n Commit,\n} from \"./type"
},
{
"path": "frontend/src/components/core/KeyboardShortcutBadge.tsx",
"chars": 586,
"preview": "import React from \"react\";\nimport { BsArrowReturnLeft } from \"react-icons/bs\";\n\ninterface KeyboardShortcutBadgeProps {\n "
},
{
"path": "frontend/src/components/core/Spinner.tsx",
"chars": 1481,
"preview": "function Spinner() {\n return (\n <div role=\"status\">\n <svg\n aria-hidden=\"true\"\n className=\"w-6 h-6"
},
{
"path": "frontend/src/components/core/StackLabel.tsx",
"chars": 613,
"preview": "import React from \"react\";\nimport { Stack, STACK_DESCRIPTIONS } from \"../../lib/stacks\";\n\ninterface StackLabelProps {\n "
},
{
"path": "frontend/src/components/core/WorkingPulse.tsx",
"chars": 605,
"preview": "function WorkingPulse() {\n return (\n <span className=\"inline-flex items-end gap-0.5\" aria-hidden=\"true\">\n <span"
},
{
"path": "frontend/src/components/evals/AllEvalsPage.tsx",
"chars": 2787,
"preview": "import { Link } from \"react-router-dom\";\n\nfunction AllEvalsPage() {\n return (\n <div className=\"flex flex-col items-c"
},
{
"path": "frontend/src/components/evals/BestOfNEvalsPage.tsx",
"chars": 35989,
"preview": "import React, { useState, useEffect, useRef } from \"react\";\nimport { HTTP_BACKEND_URL } from \"../../config\";\nimport { Di"
},
{
"path": "frontend/src/components/evals/EvalNavigation.tsx",
"chars": 1450,
"preview": "import { Link } from \"react-router-dom\";\n\nfunction EvalNavigation() {\n return (\n <div className=\"flex justify-betwee"
},
{
"path": "frontend/src/components/evals/EvalsPage.tsx",
"chars": 9180,
"preview": "import React, { useState } from \"react\";\nimport { HTTP_BACKEND_URL } from \"../../config\";\nimport RatingPicker from \"./Ra"
},
{
"path": "frontend/src/components/evals/InputFileSelector.tsx",
"chars": 4601,
"preview": "import { useState, useEffect } from \"react\";\nimport { HTTP_BACKEND_URL } from \"../../config\";\nimport { BsCheckLg, BsChev"
},
{
"path": "frontend/src/components/evals/OpenAIInputComparePage.tsx",
"chars": 10072,
"preview": "import { useState } from \"react\";\n\nimport { HTTP_BACKEND_URL } from \"../../config\";\nimport EvalNavigation from \"./EvalNa"
},
{
"path": "frontend/src/components/evals/PairwiseEvalsPage.tsx",
"chars": 7633,
"preview": "import React, { useState } from \"react\";\nimport { HTTP_BACKEND_URL } from \"../../config\";\nimport { Dialog, DialogContent"
},
{
"path": "frontend/src/components/evals/RatingPicker.tsx",
"chars": 737,
"preview": "interface RatingPickerProps {\n onSelect: (rating: number) => void;\n maxRating?: number;\n value?: number;\n}\n\nfunction "
},
{
"path": "frontend/src/components/evals/RunEvalsPage.tsx",
"chars": 20796,
"preview": "import { useState, useEffect, useCallback, useRef } from \"react\";\nimport { Button } from \"../ui/button\";\nimport { Progre"
},
{
"path": "frontend/src/components/generate-from-text/GenerateFromText.tsx",
"chars": 2057,
"preview": "import { useState, useRef, useEffect } from \"react\";\nimport { Button } from \"../ui/button\";\nimport { Textarea } from \".."
},
{
"path": "frontend/src/components/history/HistoryDisplay.tsx",
"chars": 8908,
"preview": "import { renderHistory, RenderedHistoryItem } from \"./utils\";\nimport { useProjectStore } from \"../../store/project-store"
},
{
"path": "frontend/src/components/history/utils.test.ts",
"chars": 4388,
"preview": "import { renderHistory } from \"./utils\";\nimport { Commit, CommitHash } from \"../commits/types\";\n\nconst basicLinearHistor"
},
{
"path": "frontend/src/components/history/utils.ts",
"chars": 2801,
"preview": "import { Commit, CommitType } from \"../commits/types\";\n\nfunction displayHistoryItemType(itemType: CommitType) {\n switch"
},
{
"path": "frontend/src/components/messages/OnboardingNote.tsx",
"chars": 936,
"preview": "export function OnboardingNote() {\n return (\n <div className=\"flex flex-col space-y-4 bg-green-700 p-2 rounded text-"
},
{
"path": "frontend/src/components/messages/PicoBadge.tsx",
"chars": 677,
"preview": "export function PicoBadge() {\n return (\n <>\n <a\n href=\"https://screenshot-to-code.canny.io/feature-reque"
},
{
"path": "frontend/src/components/messages/TipLink.tsx",
"chars": 279,
"preview": "import { URLS } from \"../../urls\";\n\nfunction TipLink() {\n return (\n <a\n className=\"text-xs underline text-gray-"
},
{
"path": "frontend/src/components/preview/CodeMirror.tsx",
"chars": 2193,
"preview": "import { useRef, useEffect, useMemo } from \"react\";\nimport { EditorState } from \"@codemirror/state\";\nimport { EditorView"
},
{
"path": "frontend/src/components/preview/CodePreview.tsx",
"chars": 545,
"preview": "import { useRef, useEffect } from \"react\";\n\ninterface Props {\n code: string;\n}\n\nfunction CodePreview({ code }: Props) {"
},
{
"path": "frontend/src/components/preview/CodeTab.tsx",
"chars": 2987,
"preview": "import { FaCopy } from \"react-icons/fa\";\nimport CodeMirror from \"./CodeMirror\";\nimport { Button } from \"../ui/button\";\ni"
},
{
"path": "frontend/src/components/preview/PreviewComponent.tsx",
"chars": 5666,
"preview": "import { useCallback, useEffect, useRef, useState } from \"react\";\nimport classNames from \"classnames\";\nimport useThrottl"
},
{
"path": "frontend/src/components/preview/PreviewPane.tsx",
"chars": 9543,
"preview": "import { Tabs, TabsList, TabsTrigger, TabsContent } from \"../ui/tabs\";\nimport {\n FaDesktop,\n FaMobile,\n FaCode,\n} fro"
},
{
"path": "frontend/src/components/preview/download.ts",
"chars": 626,
"preview": "export const downloadCode = (code: string) => {\n // Create a blob from the generated code\n const blob = new Blob([code"
},
{
"path": "frontend/src/components/preview/extractHtml.ts",
"chars": 836,
"preview": "// Extract HTML content, supporting <html> tags with attributes like <html lang=\"en\">\nexport function extractHtml(code: "
},
{
"path": "frontend/src/components/preview/simpleHash.ts",
"chars": 251,
"preview": "\nexport function simpleHash(str: string, seed = 0) {\n let hash = seed;\n for (let i = 0; i < str.length; i++) {\n con"
},
{
"path": "frontend/src/components/recording/ScreenRecorder.tsx",
"chars": 4850,
"preview": "import { useState } from \"react\";\nimport { Button } from \"../ui/button\";\nimport { ScreenRecorderState } from \"../../type"
},
{
"path": "frontend/src/components/recording/utils.ts",
"chars": 895,
"preview": "export function downloadBlob(blob: Blob) {\n // Create a URL for the blob object\n const videoURL = URL.createObjectURL("
},
{
"path": "frontend/src/components/select-and-edit/utils.ts",
"chars": 311,
"preview": "export function removeHighlight(element: HTMLElement) {\n element.style.outline = \"\";\n element.style.backgroundColor = "
},
{
"path": "frontend/src/components/settings/GenerationSettings.tsx",
"chars": 1214,
"preview": "import React from \"react\";\nimport { useAppStore } from \"../../store/app-store\";\nimport { useProjectStore } from \"../../s"
},
{
"path": "frontend/src/components/settings/OutputSettingsSection.tsx",
"chars": 1509,
"preview": "import {\n Select,\n SelectContent,\n SelectGroup,\n SelectItem,\n SelectTrigger,\n SelectValue,\n} from \"../ui/select\";\n"
},
{
"path": "frontend/src/components/settings/SettingsTab.tsx",
"chars": 10659,
"preview": "import React from \"react\";\nimport { AppTheme, EditorTheme, Settings } from \"../../types\";\nimport { capitalize } from \".."
},
{
"path": "frontend/src/components/sidebar/IconStrip.tsx",
"chars": 3741,
"preview": "import { LuClock, LuCode, LuSettings, LuPlus } from \"react-icons/lu\";\n\ninterface IconStripProps {\n isHistoryOpen: boole"
},
{
"path": "frontend/src/components/sidebar/Sidebar.tsx",
"chars": 25037,
"preview": "import { useAppStore } from \"../../store/app-store\";\nimport { useProjectStore } from \"../../store/project-store\";\nimport"
},
{
"path": "frontend/src/components/start-pane/StartPane.tsx",
"chars": 945,
"preview": "import React from \"react\";\nimport { Settings } from \"../../types\";\nimport { Stack } from \"../../lib/stacks\";\nimport Unif"
},
{
"path": "frontend/src/components/thinking/ThinkingIndicator.tsx",
"chars": 5386,
"preview": "import { useState } from \"react\";\nimport { useProjectStore } from \"../../store/project-store\";\nimport { BsChevronDown, B"
},
{
"path": "frontend/src/components/ui/accordion.tsx",
"chars": 2008,
"preview": "import * as React from \"react\"\nimport * as AccordionPrimitive from \"@radix-ui/react-accordion\"\nimport { ChevronDownIcon "
},
{
"path": "frontend/src/components/ui/alert-dialog.tsx",
"chars": 4441,
"preview": "import * as React from \"react\"\nimport * as AlertDialogPrimitive from \"@radix-ui/react-alert-dialog\"\n\nimport { cn } from "
},
{
"path": "frontend/src/components/ui/badge.tsx",
"chars": 1140,
"preview": "import * as React from \"react\"\nimport { cva, type VariantProps } from \"class-variance-authority\"\n\nimport { cn } from \"@/"
},
{
"path": "frontend/src/components/ui/button.tsx",
"chars": 1924,
"preview": "import * as React from \"react\";\nimport { Slot } from \"@radix-ui/react-slot\";\nimport { cva, type VariantProps } from \"cla"
},
{
"path": "frontend/src/components/ui/checkbox.tsx",
"chars": 1029,
"preview": "import * as React from \"react\"\nimport * as CheckboxPrimitive from \"@radix-ui/react-checkbox\"\nimport { CheckIcon } from \""
},
{
"path": "frontend/src/components/ui/collapsible.tsx",
"chars": 315,
"preview": "import * as CollapsiblePrimitive from \"@radix-ui/react-collapsible\"\n\nconst Collapsible = CollapsiblePrimitive.Root\n\ncons"
},
{
"path": "frontend/src/components/ui/dialog.tsx",
"chars": 3883,
"preview": "import * as React from \"react\"\nimport * as DialogPrimitive from \"@radix-ui/react-dialog\"\nimport { Cross2Icon } from \"@ra"
},
{
"path": "frontend/src/components/ui/hover-card.tsx",
"chars": 1184,
"preview": "import * as React from \"react\"\nimport * as HoverCardPrimitive from \"@radix-ui/react-hover-card\"\n\nimport { cn } from \"@/l"
},
{
"path": "frontend/src/components/ui/input.tsx",
"chars": 801,
"preview": "import * as React from \"react\"\n\nimport { cn } from \"@/lib/utils\"\n\nexport interface InputProps\n extends React.InputHTMLA"
},
{
"path": "frontend/src/components/ui/label.tsx",
"chars": 710,
"preview": "import * as React from \"react\"\nimport * as LabelPrimitive from \"@radix-ui/react-label\"\nimport { cva, type VariantProps }"
},
{
"path": "frontend/src/components/ui/popover.tsx",
"chars": 1230,
"preview": "import * as React from \"react\"\nimport * as PopoverPrimitive from \"@radix-ui/react-popover\"\n\nimport { cn } from \"@/lib/ut"
},
{
"path": "frontend/src/components/ui/progress.tsx",
"chars": 778,
"preview": "import * as React from \"react\"\nimport * as ProgressPrimitive from \"@radix-ui/react-progress\"\n\nimport { cn } from \"@/lib/"
},
{
"path": "frontend/src/components/ui/scroll-area.tsx",
"chars": 1642,
"preview": "import * as React from \"react\"\nimport * as ScrollAreaPrimitive from \"@radix-ui/react-scroll-area\"\n\nimport { cn } from \"@"
},
{
"path": "frontend/src/components/ui/select.tsx",
"chars": 6178,
"preview": "import * as React from \"react\"\nimport {\n CaretSortIcon,\n CheckIcon,\n ChevronDownIcon,\n ChevronUpIcon,\n} from \"@radix"
},
{
"path": "frontend/src/components/ui/separator.tsx",
"chars": 756,
"preview": "import * as React from \"react\"\nimport * as SeparatorPrimitive from \"@radix-ui/react-separator\"\n\nimport { cn } from \"@/li"
},
{
"path": "frontend/src/components/ui/switch.tsx",
"chars": 1154,
"preview": "import * as React from \"react\";\nimport * as SwitchPrimitives from \"@radix-ui/react-switch\";\n\nimport { cn } from \"@/lib/u"
},
{
"path": "frontend/src/components/ui/tabs.tsx",
"chars": 1877,
"preview": "import * as React from \"react\"\nimport * as TabsPrimitive from \"@radix-ui/react-tabs\"\n\nimport { cn } from \"@/lib/utils\"\n\n"
},
{
"path": "frontend/src/components/ui/textarea.tsx",
"chars": 732,
"preview": "import * as React from \"react\"\n\nimport { cn } from \"@/lib/utils\"\n\nexport interface TextareaProps\n extends React.Textare"
},
{
"path": "frontend/src/components/unified-input/UnifiedInputPane.tsx",
"chars": 4953,
"preview": "import React, { useState } from \"react\";\nimport { Tabs, TabsContent, TabsList, TabsTrigger } from \"../ui/tabs\";\nimport {"
},
{
"path": "frontend/src/components/unified-input/tabs/ImportTab.tsx",
"chars": 4324,
"preview": "import { useState, useRef, useEffect } from \"react\";\nimport { useDropzone } from \"react-dropzone\";\nimport { Button } fro"
},
{
"path": "frontend/src/components/unified-input/tabs/TextTab.tsx",
"chars": 4456,
"preview": "import { useState, useRef, useEffect } from \"react\";\nimport { Button } from \"../../ui/button\";\nimport { Textarea } from "
},
{
"path": "frontend/src/components/unified-input/tabs/UploadTab.tsx",
"chars": 19255,
"preview": "import { useState, useEffect, useMemo, useRef, useCallback } from \"react\";\nimport { useDropzone } from \"react-dropzone\";"
},
{
"path": "frontend/src/components/unified-input/tabs/UrlTab.tsx",
"chars": 5877,
"preview": "import { useState } from \"react\";\nimport { HTTP_BACKEND_URL } from \"../../../config\";\nimport { Button } from \"../../ui/b"
},
{
"path": "frontend/src/components/variants/Variants.tsx",
"chars": 5444,
"preview": "import { useProjectStore } from \"../../store/project-store\";\nimport { useEffect, useRef, useState } from \"react\";\nimport"
}
]
// ... and 28 more files (download for full content)
About this extraction
This page contains the full source code of the abi/screenshot-to-code GitHub repository, extracted and formatted as plain text for AI agents and large language models (LLMs). The extraction includes 228 files (793.7 KB), approximately 194.2k tokens, and a symbol index with 662 extracted functions, classes, methods, constants, and types. Use this with OpenClaw, Claude, ChatGPT, Cursor, Windsurf, or any other AI tool that accepts text input. You can copy the full output to your clipboard or download it as a .txt file.
Extracted by GitExtract — free GitHub repo to text converter for AI. Built by Nikandr Surkov.