[ { "path": ".dockerignore", "content": "# Git\n.git\n.gitignore\n\n# Python\n__pycache__\n*.pyc\n*.pyo\n*.pyd\n.venv\nvenv\nENV\nenv\n.pytest_cache\n.mypy_cache\n.ruff_cache\n\n# Frontend\nfrontend/node_modules\nfrontend/.next\nfrontend/dist\nfrontend/out\nfrontend/.env*\nfrontend/*.log\n\n# Project data\n.antigravity\n.gemini\ntmp\ndata\nmydata\nnotebook_data\nsurreal_data\nsurreal-data\nsurreal_single_data\n*.db\n*.log\ndocker.env\n.env\ndocker-compose*\n\n# Documentation & CI (not needed in image)\ndocs\n.github\n\n# IDE and OS files\n.vscode\n.idea\n*.swp\n*.swo\n*~\n.DS_Store" }, { "path": ".github/ISSUE_TEMPLATE/bug_report.yml", "content": "name: 🐛 Bug Report\ndescription: Report a bug or unexpected behavior (app is running but misbehaving)\ntitle: \"[Bug]: \"\nlabels: [\"bug\", \"needs-triage\"]\nbody:\n - type: markdown\n attributes:\n value: |\n Thanks for reporting a bug! Please fill out the information below to help us understand and fix the issue.\n\n **Note**: If you're having installation or setup issues, please use the \"Installation Issue\" template instead.\n\n - type: textarea\n id: what-happened\n attributes:\n label: What did you do when it broke?\n description: Describe the steps you took that led to the bug\n placeholder: |\n 1. I went to the Notebooks page\n 2. I clicked on \"Create New Notebook\"\n 3. I filled in the form and clicked \"Save\"\n 4. Then the error occurred...\n validations:\n required: true\n\n - type: textarea\n id: how-broke\n attributes:\n label: How did it break?\n description: What happened that was unexpected? What did you expect to happen instead?\n placeholder: |\n Expected: The notebook should be created and I should see it in the list\n Actual: I got an error message saying \"Failed to create notebook\"\n validations:\n required: true\n\n - type: textarea\n id: logs-screenshots\n attributes:\n label: Logs or Screenshots\n description: |\n Please provide any error messages, logs, or screenshots that might help us understand the issue.\n\n **How to get logs:**\n - Docker: `docker compose logs -f open_notebook`\n - Check browser console (F12 → Console tab)\n placeholder: |\n Paste logs here or drag and drop screenshots.\n\n Error messages, stack traces, or browser console errors are very helpful!\n validations:\n required: false\n\n - type: dropdown\n id: version\n attributes:\n label: Open Notebook Version\n description: Which version are you using?\n options:\n - v1-latest (Docker)\n - v1-latest-single (Docker)\n - Latest from main branch\n - Other (please specify in additional context)\n validations:\n required: true\n\n - type: textarea\n id: environment\n attributes:\n label: Environment\n description: What environment are you running in?\n placeholder: |\n - OS: Ubuntu 22.04 / Windows 11 / macOS 14\n - Browser: Chrome 120\n validations:\n required: false\n\n - type: textarea\n id: additional-context\n attributes:\n label: Additional Context\n description: Any other information that might be helpful\n placeholder: \"This started happening after I upgraded to v1.5.0...\"\n validations:\n required: false\n\n - type: checkboxes\n id: willing-to-contribute\n attributes:\n label: Contribution\n description: Would you like to work on fixing this bug?\n options:\n - label: I am a developer and would like to work on fixing this issue (pending maintainer approval)\n required: false\n\n - type: markdown\n attributes:\n value: |\n ---\n **Next Steps:**\n 1. A maintainer will review your bug report\n 2. If you checked the box above and want to fix it, please propose your solution approach\n 3. Wait for assignment before starting development\n 4. See our [Contributing Guide](https://github.com/lfnovo/open-notebook/blob/main/CONTRIBUTING.md) for more details\n" }, { "path": ".github/ISSUE_TEMPLATE/config.yml", "content": "blank_issues_enabled: false\ncontact_links:\n - name: 💬 Discord Community\n url: https://discord.gg/37XJPXfz2w\n about: Get help from the community and share ideas\n - name: 🤖 Installation Assistant (ChatGPT)\n url: https://chatgpt.com/g/g-68776e2765b48191bd1bae3f30212631-open-notebook-installation-assistant\n about: CustomGPT that knows all our docs. Really useful. Try it.\n - name: 📚 Documentation\n url: https://github.com/lfnovo/open-notebook/tree/main/docs\n about: Browse our comprehensive documentation\n" }, { "path": ".github/ISSUE_TEMPLATE/feature_request.yml", "content": "name: ✨ Feature Suggestion\ndescription: Suggest a new feature or improvement for Open Notebook\ntitle: \"[Feature]: \"\nlabels: [\"enhancement\", \"needs-triage\"]\nbody:\n - type: markdown\n attributes:\n value: |\n Thanks for taking the time to suggest a feature! Your ideas help make Open Notebook better for everyone.\n\n - type: textarea\n id: feature-description\n attributes:\n label: Feature Description\n description: What feature would you like to see added or improved?\n placeholder: \"I would like to be able to...\"\n validations:\n required: true\n\n - type: textarea\n id: why-helpful\n attributes:\n label: Why would this be helpful?\n description: Explain how this feature would benefit you and other users\n placeholder: \"This would help because...\"\n validations:\n required: true\n\n - type: textarea\n id: proposed-solution\n attributes:\n label: Proposed Solution (Optional)\n description: If you have ideas on how to implement this feature, please share them\n placeholder: \"This could be implemented by...\"\n validations:\n required: false\n\n - type: textarea\n id: additional-context\n attributes:\n label: Additional Context\n description: Any other context, screenshots, or examples that might be helpful\n placeholder: \"For example, other tools do this by...\"\n validations:\n required: false\n\n - type: checkboxes\n id: willing-to-contribute\n attributes:\n label: Contribution\n description: Would you like to work on implementing this feature?\n options:\n - label: I am a developer and would like to work on implementing this feature (pending maintainer approval)\n required: false\n\n - type: markdown\n attributes:\n value: |\n ---\n **Next Steps:**\n 1. A maintainer will review your feature request\n 2. If approved and you checked the box above, the issue will be assigned to you\n 3. Please wait for assignment before starting development\n 4. See our [Contributing Guide](https://github.com/lfnovo/open-notebook/blob/main/CONTRIBUTING.md) for more details\n\n" }, { "path": ".github/ISSUE_TEMPLATE/installation_issue.yml", "content": "name: 🔧 Installation Issue\ndescription: Report problems with installation, setup, or connectivity\ntitle: \"[Install]: \"\nlabels: [\"installation\", \"needs-triage\"]\nbody:\n - type: markdown\n attributes:\n value: |\n ## ⚠️ Before You Continue\n\n **Please try these resources first:**\n\n 1. 🤖 **[Installation Assistant ChatGPT](https://chatgpt.com/g/g-68776e2765b48191bd1bae3f30212631-open-notebook-installation-assistant)** - Our AI assistant can help you troubleshoot most installation issues instantly!\n\n 2. 📚 **[Installation Guide](https://github.com/lfnovo/open-notebook/blob/main/docs/getting-started/installation.md)** - Comprehensive setup instructions\n\n 3. 🐋 **[Docker Deployment Guide](https://github.com/lfnovo/open-notebook/blob/main/docs/deployment/docker.md)** - Detailed Docker setup\n\n 4. 🦙 **Ollama Issues?** Read our [Ollama Guide](https://github.com/lfnovo/open-notebook/blob/main/docs/features/ollama.md) first\n\n 5. 💬 **[Discord Community](https://discord.gg/37XJPXfz2w)** - Get real-time help from the community\n\n ---\n\n If you've tried the above and still need help, please fill out the form below with as much detail as possible.\n\n - type: dropdown\n id: installation-method\n attributes:\n label: Installation Method\n description: How are you trying to install Open Notebook?\n options:\n - Docker (single container - v1-latest-single)\n - Docker (multi-container - docker-compose)\n - Local development (make start-all)\n - Other (please specify below)\n validations:\n required: true\n\n - type: textarea\n id: issue-description\n attributes:\n label: What is the issue?\n description: Describe the installation or setup problem you're experiencing\n placeholder: |\n Example: \"I can't connect to the database\" or \"The container won't start\" or \"Getting 404 errors when accessing the UI\"\n validations:\n required: true\n\n - type: textarea\n id: logs\n attributes:\n label: Logs\n description: |\n Please provide relevant logs. **This is very important for diagnosing issues!**\n\n **How to get logs:**\n - Docker single container: `docker logs open-notebook`\n - Docker Compose: `docker compose logs -f`\n - Specific service: `docker compose logs -f open_notebook`\n placeholder: |\n Paste your logs here. Include the full error message and stack trace if available.\n render: shell\n validations:\n required: false\n\n - type: textarea\n id: docker-compose\n attributes:\n label: Docker Compose Configuration\n description: |\n If using Docker Compose, please paste your `docker-compose.yml` file here.\n\n **⚠️ IMPORTANT: Redact any sensitive information (API keys, passwords, etc.)**\n placeholder: |\n services:\n open_notebook:\n image: lfnovo/open_notebook:v1-latest-single\n ports:\n - \"8502:8502\"\n - \"5055:5055\"\n environment:\n - OPENAI_API_KEY=sk-***REDACTED***\n ...\n render: yaml\n validations:\n required: false\n\n - type: textarea\n id: env-file\n attributes:\n label: Environment File\n description: |\n If using an `.env` or `docker.env` file, please paste it here.\n\n **⚠️ IMPORTANT: REDACT ALL API KEYS AND PASSWORDS!**\n placeholder: |\n SURREAL_URL=ws://surrealdb:8000/rpc\n SURREAL_USER=root\n SURREAL_PASSWORD=***REDACTED***\n OPENAI_API_KEY=sk-***REDACTED***\n ANTHROPIC_API_KEY=sk-ant-***REDACTED***\n render: shell\n validations:\n required: false\n\n - type: textarea\n id: system-info\n attributes:\n label: System Information\n description: Tell us about your setup\n placeholder: |\n - Operating System: Ubuntu 22.04 / Windows 11 / macOS 14\n - Docker version: `docker --version`\n - Docker Compose version: `docker compose version`\n - Architecture: amd64 / arm64 (Apple Silicon)\n - Available disk space: `df -h`\n - Available memory: `free -h` (Linux) or Activity Monitor (Mac)\n validations:\n required: false\n\n - type: textarea\n id: additional-context\n attributes:\n label: Additional Context\n description: Any other information that might be helpful\n placeholder: |\n - Are you behind a corporate proxy or firewall?\n - Are you using a VPN?\n - Have you made any custom modifications?\n - Did this work before and suddenly break?\n validations:\n required: false\n\n - type: checkboxes\n id: checklist\n attributes:\n label: Pre-submission Checklist\n description: Please confirm you've tried these steps\n options:\n - label: I tried the [Installation Assistant ChatGPT](https://chatgpt.com/g/g-68776e2765b48191bd1bae3f30212631-open-notebook-installation-assistant)\n required: false\n - label: I read the relevant documentation ([Installation Guide](https://github.com/lfnovo/open-notebook/blob/main/docs/getting-started/installation.md) or [Ollama Guide](https://github.com/lfnovo/open-notebook/blob/main/docs/features/ollama.md))\n required: false\n - label: I searched existing issues to see if this was already reported\n required: true\n - label: I redacted all sensitive information (API keys, passwords, etc.)\n required: true\n" }, { "path": ".github/pull_request_template.md", "content": "## Description\n\n\n\n## Related Issue\n\n\n\nFixes #\n\n## Type of Change\n\n\n\n- [ ] Bug fix (non-breaking change that fixes an issue)\n- [ ] New feature (non-breaking change that adds functionality)\n- [ ] Breaking change (fix or feature that would cause existing functionality to not work as expected)\n- [ ] Documentation update\n- [ ] Code refactoring (no functional changes)\n- [ ] Performance improvement\n- [ ] Test coverage improvement\n\n## How Has This Been Tested?\n\n\n\n- [ ] Tested locally with Docker\n- [ ] Tested locally with development setup\n- [ ] Added new unit tests\n- [ ] Existing tests pass (`uv run pytest`)\n- [ ] Manual testing performed (describe below)\n\n**Test Details:**\n\n\n## Design Alignment\n\n\n\n**Which design principles does this PR support?** (See [DESIGN_PRINCIPLES.md](../DESIGN_PRINCIPLES.md))\n\n- [ ] Privacy First\n- [ ] Simplicity Over Features\n- [ ] API-First Architecture\n- [ ] Multi-Provider Flexibility\n- [ ] Extensibility Through Standards\n- [ ] Async-First for Performance\n\n**Explanation:**\n\n\n## Checklist\n\n\n\n### Code Quality\n- [ ] My code follows PEP 8 style guidelines (Python)\n- [ ] My code follows TypeScript best practices (Frontend)\n- [ ] I have added type hints to my code (Python)\n- [ ] I have added JSDoc comments where appropriate (TypeScript)\n- [ ] I have performed a self-review of my code\n- [ ] I have commented my code, particularly in hard-to-understand areas\n- [ ] My changes generate no new warnings or errors\n\n### Testing\n- [ ] I have added tests that prove my fix is effective or that my feature works\n- [ ] New and existing unit tests pass locally with my changes\n- [ ] I ran linting: `make ruff` or `ruff check . --fix`\n- [ ] I ran type checking: `make lint` or `uv run python -m mypy .`\n\n### Documentation\n- [ ] I have updated the relevant documentation in `/docs` (if applicable)\n- [ ] I have added/updated docstrings for new/modified functions\n- [ ] I have updated the API documentation (if API changes were made)\n- [ ] I have added comments to complex logic\n\n### Database Changes\n- [ ] I have created migration scripts for any database schema changes (in `/migrations`)\n- [ ] Migration includes both up and down scripts\n- [ ] Migration has been tested locally\n\n### Breaking Changes\n- [ ] This PR includes breaking changes\n- [ ] I have documented the migration path for users\n- [ ] I have updated MIGRATION.md (if applicable)\n\n## Screenshots (if applicable)\n\n\n\n## Additional Context\n\n\n\n## Pre-Submission Verification\n\nBefore submitting, please verify:\n\n- [ ] I have read [CONTRIBUTING.md](../CONTRIBUTING.md)\n- [ ] I have read [DESIGN_PRINCIPLES.md](../DESIGN_PRINCIPLES.md)\n- [ ] This PR addresses an approved issue that was assigned to me\n- [ ] I have not included unrelated changes in this PR\n- [ ] My PR title follows conventional commits format (e.g., \"feat: add user authentication\")\n\n---\n\n**Thank you for contributing to Open Notebook!** 🎉\n" }, { "path": ".github/workflows/build-and-release.yml", "content": "name: Build and Release\n\non:\n workflow_dispatch:\n inputs:\n push_latest:\n description: 'Also push v1-latest tags'\n required: true\n default: false\n type: boolean\n release:\n types: [published]\n\npermissions:\n contents: read\n packages: write\n\nenv:\n GHCR_IMAGE: ghcr.io/lfnovo/open-notebook\n DOCKERHUB_IMAGE: lfnovo/open_notebook\n\njobs:\n extract-version:\n runs-on: ubuntu-latest\n outputs:\n version: ${{ steps.version.outputs.version }}\n has_dockerhub_secrets: ${{ steps.check.outputs.has_dockerhub_secrets }}\n steps:\n - name: Checkout\n uses: actions/checkout@v4\n\n - name: Extract version from pyproject.toml\n id: version\n run: |\n VERSION=$(grep -m1 '^version = ' pyproject.toml | cut -d'\"' -f2)\n echo \"version=$VERSION\" >> $GITHUB_OUTPUT\n echo \"Extracted version: $VERSION\"\n\n - name: Check for Docker Hub credentials\n id: check\n env:\n SECRET_DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}\n SECRET_DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}\n run: |\n if [[ -n \"\"$SECRET_DOCKER_USERNAME\"\" && -n \"\"$SECRET_DOCKER_PASSWORD\"\" ]]; then\n echo \"has_dockerhub_secrets=true\" >> $GITHUB_OUTPUT\n echo \"Docker Hub credentials available\"\n else\n echo \"has_dockerhub_secrets=false\" >> $GITHUB_OUTPUT\n echo \"Docker Hub credentials not available - will only push to GHCR\"\n fi\n\n build-regular:\n needs: extract-version\n runs-on: ubuntu-latest\n steps:\n - name: Checkout\n uses: actions/checkout@v4\n\n - name: Free up disk space\n run: |\n sudo rm -rf /usr/share/dotnet\n sudo rm -rf /usr/local/lib/android\n sudo rm -rf /opt/ghc\n sudo rm -rf /opt/hostedtoolcache/CodeQL\n sudo docker image prune --all --force\n df -h\n\n - name: Set up Docker Buildx\n uses: docker/setup-buildx-action@v3\n\n - name: Login to GitHub Container Registry\n uses: docker/login-action@v3\n with:\n registry: ghcr.io\n username: ${{ github.actor }}\n password: ${{ secrets.GITHUB_TOKEN }}\n\n - name: Login to Docker Hub\n if: needs.extract-version.outputs.has_dockerhub_secrets == 'true'\n uses: docker/login-action@v3\n with:\n username: ${{ secrets.DOCKER_USERNAME }}\n password: ${{ secrets.DOCKER_PASSWORD }}\n\n - name: Cache Docker layers\n uses: actions/cache@v3\n with:\n path: /tmp/.buildx-cache\n key: ${{ runner.os }}-buildx-regular-${{ github.sha }}\n restore-keys: |\n ${{ runner.os }}-buildx-regular-\n\n - name: Prepare Docker tags for regular build\n id: tags-regular\n env:\n ENV_GHCR_IMAGE: ${{ env.GHCR_IMAGE }}\n GITHUB_EVENT_INPUTS_PUSH_LATEST: ${{ github.event.inputs.push_latest }}\n GITHUB_EVENT_NAME: ${{ github.event_name }}\n GITHUB_EVENT_RELEASE_PRERELEASE: ${{ github.event.release.prerelease }}\n ENV_DOCKERHUB_IMAGE: ${{ env.DOCKERHUB_IMAGE }}\n run: |\n TAGS=\"\"$ENV_GHCR_IMAGE\":${{ needs.extract-version.outputs.version }}\"\n\n # Determine if we should push latest tags\n PUSH_LATEST=\"\"$GITHUB_EVENT_INPUTS_PUSH_LATEST\"\"\n if [[ -z \"$PUSH_LATEST\" ]]; then\n PUSH_LATEST=\"false\"\n fi\n\n # Add GHCR latest tag if requested or for non-prerelease releases\n if [[ \"$PUSH_LATEST\" == \"true\" ]] || [[ \"\"$GITHUB_EVENT_NAME\"\" == \"release\" && \"\"$GITHUB_EVENT_RELEASE_PRERELEASE\"\" != \"true\" ]]; then\n TAGS=\"${TAGS},\"$ENV_GHCR_IMAGE\":v1-latest\"\n fi\n\n # Add Docker Hub tags if credentials available\n if [[ \"${{ needs.extract-version.outputs.has_dockerhub_secrets }}\" == \"true\" ]]; then\n TAGS=\"${TAGS},\"$ENV_DOCKERHUB_IMAGE\":${{ needs.extract-version.outputs.version }}\"\n\n if [[ \"$PUSH_LATEST\" == \"true\" ]] || [[ \"\"$GITHUB_EVENT_NAME\"\" == \"release\" && \"\"$GITHUB_EVENT_RELEASE_PRERELEASE\"\" != \"true\" ]]; then\n TAGS=\"${TAGS},\"$ENV_DOCKERHUB_IMAGE\":v1-latest\"\n fi\n fi\n\n echo \"tags=${TAGS}\" >> $GITHUB_OUTPUT\n echo \"Generated tags: ${TAGS}\"\n\n - name: Build and push regular image\n uses: docker/build-push-action@v5\n with:\n context: .\n file: ./Dockerfile\n platforms: linux/amd64,linux/arm64\n push: true\n tags: ${{ steps.tags-regular.outputs.tags }}\n cache-from: type=local,src=/tmp/.buildx-cache\n cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max\n\n - name: Move cache\n run: |\n rm -rf /tmp/.buildx-cache\n mv /tmp/.buildx-cache-new /tmp/.buildx-cache\n\n build-single:\n needs: extract-version\n runs-on: ubuntu-latest\n steps:\n - name: Checkout\n uses: actions/checkout@v4\n\n - name: Free up disk space\n run: |\n sudo rm -rf /usr/share/dotnet\n sudo rm -rf /usr/local/lib/android\n sudo rm -rf /opt/ghc\n sudo rm -rf /opt/hostedtoolcache/CodeQL\n sudo docker image prune --all --force\n df -h\n\n - name: Set up Docker Buildx\n uses: docker/setup-buildx-action@v3\n\n - name: Login to GitHub Container Registry\n uses: docker/login-action@v3\n with:\n registry: ghcr.io\n username: ${{ github.actor }}\n password: ${{ secrets.GITHUB_TOKEN }}\n\n - name: Login to Docker Hub\n if: needs.extract-version.outputs.has_dockerhub_secrets == 'true'\n uses: docker/login-action@v3\n with:\n username: ${{ secrets.DOCKER_USERNAME }}\n password: ${{ secrets.DOCKER_PASSWORD }}\n\n - name: Cache Docker layers\n uses: actions/cache@v3\n with:\n path: /tmp/.buildx-cache-single\n key: ${{ runner.os }}-buildx-single-${{ github.sha }}\n restore-keys: |\n ${{ runner.os }}-buildx-single-\n\n - name: Prepare Docker tags for single build\n id: tags-single\n env:\n ENV_GHCR_IMAGE: ${{ env.GHCR_IMAGE }}\n GITHUB_EVENT_INPUTS_PUSH_LATEST: ${{ github.event.inputs.push_latest }}\n GITHUB_EVENT_NAME: ${{ github.event_name }}\n GITHUB_EVENT_RELEASE_PRERELEASE: ${{ github.event.release.prerelease }}\n ENV_DOCKERHUB_IMAGE: ${{ env.DOCKERHUB_IMAGE }}\n run: |\n TAGS=\"\"$ENV_GHCR_IMAGE\":${{ needs.extract-version.outputs.version }}-single\"\n\n # Determine if we should push latest tags\n PUSH_LATEST=\"\"$GITHUB_EVENT_INPUTS_PUSH_LATEST\"\"\n if [[ -z \"$PUSH_LATEST\" ]]; then\n PUSH_LATEST=\"false\"\n fi\n\n # Add GHCR latest tag if requested or for non-prerelease releases\n if [[ \"$PUSH_LATEST\" == \"true\" ]] || [[ \"\"$GITHUB_EVENT_NAME\"\" == \"release\" && \"\"$GITHUB_EVENT_RELEASE_PRERELEASE\"\" != \"true\" ]]; then\n TAGS=\"${TAGS},\"$ENV_GHCR_IMAGE\":v1-latest-single\"\n fi\n\n # Add Docker Hub tags if credentials available\n if [[ \"${{ needs.extract-version.outputs.has_dockerhub_secrets }}\" == \"true\" ]]; then\n TAGS=\"${TAGS},\"$ENV_DOCKERHUB_IMAGE\":${{ needs.extract-version.outputs.version }}-single\"\n\n if [[ \"$PUSH_LATEST\" == \"true\" ]] || [[ \"\"$GITHUB_EVENT_NAME\"\" == \"release\" && \"\"$GITHUB_EVENT_RELEASE_PRERELEASE\"\" != \"true\" ]]; then\n TAGS=\"${TAGS},\"$ENV_DOCKERHUB_IMAGE\":v1-latest-single\"\n fi\n fi\n\n echo \"tags=${TAGS}\" >> $GITHUB_OUTPUT\n echo \"Generated tags: ${TAGS}\"\n\n - name: Build and push single-container image\n uses: docker/build-push-action@v5\n with:\n context: .\n file: ./Dockerfile.single\n platforms: linux/amd64,linux/arm64\n push: true\n tags: ${{ steps.tags-single.outputs.tags }}\n cache-from: type=local,src=/tmp/.buildx-cache-single\n cache-to: type=local,dest=/tmp/.buildx-cache-single-new,mode=max\n\n - name: Move cache\n run: |\n rm -rf /tmp/.buildx-cache-single\n mv /tmp/.buildx-cache-single-new /tmp/.buildx-cache-single\n\n summary:\n needs: [extract-version, build-regular, build-single]\n runs-on: ubuntu-latest\n if: always()\n steps:\n - name: Build Summary\n env:\n GITHUB_EVENT_INPUTS_PUSH_LATEST_____FALSE_: ${{ github.event.inputs.push_latest || 'false' }}\n ENV_GHCR_IMAGE: ${{ env.GHCR_IMAGE }}\n ENV_DOCKERHUB_IMAGE: ${{ env.DOCKERHUB_IMAGE }}\n GITHUB_EVENT_INPUTS_PUSH_LATEST: ${{ github.event.inputs.push_latest }}\n run: |\n echo \"## Build Summary\" >> $GITHUB_STEP_SUMMARY\n echo \"**Version:** ${{ needs.extract-version.outputs.version }}\" >> $GITHUB_STEP_SUMMARY\n echo \"**Push v1-Latest:** \"$GITHUB_EVENT_INPUTS_PUSH_LATEST_____FALSE_\"\" >> $GITHUB_STEP_SUMMARY\n echo \"\" >> $GITHUB_STEP_SUMMARY\n echo \"### Registries:\" >> $GITHUB_STEP_SUMMARY\n echo \"✅ **GHCR:** \\`\"$ENV_GHCR_IMAGE\"\\`\" >> $GITHUB_STEP_SUMMARY\n if [[ \"${{ needs.extract-version.outputs.has_dockerhub_secrets }}\" == \"true\" ]]; then\n echo \"✅ **Docker Hub:** \\`\"$ENV_DOCKERHUB_IMAGE\"\\`\" >> $GITHUB_STEP_SUMMARY\n else\n echo \"⏭️ **Docker Hub:** Skipped (credentials not configured)\" >> $GITHUB_STEP_SUMMARY\n fi\n echo \"\" >> $GITHUB_STEP_SUMMARY\n echo \"### Images Built:\" >> $GITHUB_STEP_SUMMARY\n\n if [[ \"${{ needs.build-regular.result }}\" == \"success\" ]]; then\n echo \"✅ **Regular (GHCR):** \\`\"$ENV_GHCR_IMAGE\":${{ needs.extract-version.outputs.version }}\\`\" >> $GITHUB_STEP_SUMMARY\n if [[ \"\"$GITHUB_EVENT_INPUTS_PUSH_LATEST\"\" == \"true\" ]]; then\n echo \"✅ **Regular v1-Latest (GHCR):** \\`\"$ENV_GHCR_IMAGE\":v1-latest\\`\" >> $GITHUB_STEP_SUMMARY\n fi\n if [[ \"${{ needs.extract-version.outputs.has_dockerhub_secrets }}\" == \"true\" ]]; then\n echo \"✅ **Regular (Docker Hub):** \\`\"$ENV_DOCKERHUB_IMAGE\":${{ needs.extract-version.outputs.version }}\\`\" >> $GITHUB_STEP_SUMMARY\n if [[ \"\"$GITHUB_EVENT_INPUTS_PUSH_LATEST\"\" == \"true\" ]]; then\n echo \"✅ **Regular v1-Latest (Docker Hub):** \\`\"$ENV_DOCKERHUB_IMAGE\":v1-latest\\`\" >> $GITHUB_STEP_SUMMARY\n fi\n fi\n elif [[ \"${{ needs.build-regular.result }}\" == \"skipped\" ]]; then\n echo \"⏭️ **Regular:** Skipped\" >> $GITHUB_STEP_SUMMARY\n else\n echo \"❌ **Regular:** Failed\" >> $GITHUB_STEP_SUMMARY\n fi\n\n if [[ \"${{ needs.build-single.result }}\" == \"success\" ]]; then\n echo \"✅ **Single (GHCR):** \\`\"$ENV_GHCR_IMAGE\":${{ needs.extract-version.outputs.version }}-single\\`\" >> $GITHUB_STEP_SUMMARY\n if [[ \"\"$GITHUB_EVENT_INPUTS_PUSH_LATEST\"\" == \"true\" ]]; then\n echo \"✅ **Single v1-Latest (GHCR):** \\`\"$ENV_GHCR_IMAGE\":v1-latest-single\\`\" >> $GITHUB_STEP_SUMMARY\n fi\n if [[ \"${{ needs.extract-version.outputs.has_dockerhub_secrets }}\" == \"true\" ]]; then\n echo \"✅ **Single (Docker Hub):** \\`\"$ENV_DOCKERHUB_IMAGE\":${{ needs.extract-version.outputs.version }}-single\\`\" >> $GITHUB_STEP_SUMMARY\n if [[ \"\"$GITHUB_EVENT_INPUTS_PUSH_LATEST\"\" == \"true\" ]]; then\n echo \"✅ **Single v1-Latest (Docker Hub):** \\`\"$ENV_DOCKERHUB_IMAGE\":v1-latest-single\\`\" >> $GITHUB_STEP_SUMMARY\n fi\n fi\n elif [[ \"${{ needs.build-single.result }}\" == \"skipped\" ]]; then\n echo \"⏭️ **Single:** Skipped\" >> $GITHUB_STEP_SUMMARY\n else\n echo \"❌ **Single:** Failed\" >> $GITHUB_STEP_SUMMARY\n fi\n\n echo \"\" >> $GITHUB_STEP_SUMMARY\n echo \"### Platforms:\" >> $GITHUB_STEP_SUMMARY\n echo \"- linux/amd64\" >> $GITHUB_STEP_SUMMARY\n echo \"- linux/arm64\" >> $GITHUB_STEP_SUMMARY" }, { "path": ".github/workflows/build-dev.yml", "content": "name: Development Build\n\non:\n pull_request:\n branches: [ main ]\n push:\n branches: [ main ]\n paths-ignore:\n - '**.md'\n - 'docs/**'\n - 'notebooks/**'\n - '.github/workflows/claude*.yml'\n workflow_dispatch:\n inputs:\n platform:\n description: 'Platform to build'\n required: true\n default: 'linux/amd64'\n type: choice\n options:\n - linux/amd64\n - linux/arm64\n - linux/amd64,linux/arm64\n\npermissions:\n contents: read\n packages: write\n\nenv:\n GHCR_IMAGE: ghcr.io/lfnovo/open-notebook\n DOCKERHUB_IMAGE: lfnovo/open_notebook\n\njobs:\n extract-version:\n runs-on: ubuntu-latest\n outputs:\n version: ${{ steps.version.outputs.version }}\n has_dockerhub_secrets: ${{ steps.check.outputs.has_dockerhub_secrets }}\n is_push_to_main: ${{ steps.check.outputs.is_push_to_main }}\n steps:\n - name: Checkout\n uses: actions/checkout@v4\n\n - name: Extract version from pyproject.toml\n id: version\n run: |\n VERSION=$(grep -m1 '^version = ' pyproject.toml | cut -d'\"' -f2)\n echo \"version=$VERSION\" >> $GITHUB_OUTPUT\n echo \"Extracted version: $VERSION\"\n\n - name: Check environment\n id: check\n env:\n SECRET_DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}\n SECRET_DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}\n run: |\n # Check for Docker Hub credentials\n if [[ -n \"$SECRET_DOCKER_USERNAME\" && -n \"$SECRET_DOCKER_PASSWORD\" ]]; then\n echo \"has_dockerhub_secrets=true\" >> $GITHUB_OUTPUT\n echo \"Docker Hub credentials available\"\n else\n echo \"has_dockerhub_secrets=false\" >> $GITHUB_OUTPUT\n echo \"Docker Hub credentials not available\"\n fi\n\n # Check if this is a push to main (not a PR)\n if [[ \"${{ github.event_name }}\" == \"push\" && \"${{ github.ref }}\" == \"refs/heads/main\" ]]; then\n echo \"is_push_to_main=true\" >> $GITHUB_OUTPUT\n echo \"This is a push to main - will publish v1-dev tags\"\n else\n echo \"is_push_to_main=false\" >> $GITHUB_OUTPUT\n echo \"This is a PR or manual run - test build only\"\n fi\n\n build-regular:\n needs: extract-version\n runs-on: ubuntu-latest\n steps:\n - name: Checkout\n uses: actions/checkout@v4\n\n - name: Free up disk space\n if: needs.extract-version.outputs.is_push_to_main == 'true'\n run: |\n sudo rm -rf /usr/share/dotnet\n sudo rm -rf /usr/local/lib/android\n sudo rm -rf /opt/ghc\n sudo rm -rf /opt/hostedtoolcache/CodeQL\n sudo docker image prune --all --force\n df -h\n\n - name: Set up Docker Buildx\n uses: docker/setup-buildx-action@v3\n\n - name: Login to GitHub Container Registry\n if: needs.extract-version.outputs.is_push_to_main == 'true'\n uses: docker/login-action@v3\n with:\n registry: ghcr.io\n username: ${{ github.actor }}\n password: ${{ secrets.GITHUB_TOKEN }}\n\n - name: Login to Docker Hub\n if: needs.extract-version.outputs.is_push_to_main == 'true' && needs.extract-version.outputs.has_dockerhub_secrets == 'true'\n uses: docker/login-action@v3\n with:\n username: ${{ secrets.DOCKER_USERNAME }}\n password: ${{ secrets.DOCKER_PASSWORD }}\n\n - name: Cache Docker layers\n uses: actions/cache@v3\n with:\n path: /tmp/.buildx-cache-dev\n key: ${{ runner.os }}-buildx-dev-${{ github.sha }}\n restore-keys: |\n ${{ runner.os }}-buildx-dev-\n\n - name: Prepare Docker tags\n id: tags\n run: |\n if [[ \"${{ needs.extract-version.outputs.is_push_to_main }}\" == \"true\" ]]; then\n # Push to main: build and push v1-dev tags\n TAGS=\"${{ env.GHCR_IMAGE }}:v1-dev\"\n if [[ \"${{ needs.extract-version.outputs.has_dockerhub_secrets }}\" == \"true\" ]]; then\n TAGS=\"${TAGS},${{ env.DOCKERHUB_IMAGE }}:v1-dev\"\n fi\n echo \"tags=${TAGS}\" >> $GITHUB_OUTPUT\n echo \"push=true\" >> $GITHUB_OUTPUT\n echo \"platforms=linux/amd64,linux/arm64\" >> $GITHUB_OUTPUT\n else\n # PR or manual: test build only\n echo \"tags=${{ env.DOCKERHUB_IMAGE }}:${{ needs.extract-version.outputs.version }}-dev\" >> $GITHUB_OUTPUT\n echo \"push=false\" >> $GITHUB_OUTPUT\n echo \"platforms=${{ github.event.inputs.platform || 'linux/amd64' }}\" >> $GITHUB_OUTPUT\n fi\n\n - name: Build and push regular image\n uses: docker/build-push-action@v5\n with:\n context: .\n file: ./Dockerfile\n platforms: ${{ steps.tags.outputs.platforms }}\n push: ${{ steps.tags.outputs.push }}\n tags: ${{ steps.tags.outputs.tags }}\n cache-from: type=local,src=/tmp/.buildx-cache-dev\n cache-to: type=local,dest=/tmp/.buildx-cache-dev-new,mode=max\n\n - name: Move cache\n run: |\n rm -rf /tmp/.buildx-cache-dev\n mv /tmp/.buildx-cache-dev-new /tmp/.buildx-cache-dev\n\n build-single:\n needs: extract-version\n # Only build single image on push to main\n if: needs.extract-version.outputs.is_push_to_main == 'true'\n runs-on: ubuntu-latest\n steps:\n - name: Checkout\n uses: actions/checkout@v4\n\n - name: Free up disk space\n run: |\n sudo rm -rf /usr/share/dotnet\n sudo rm -rf /usr/local/lib/android\n sudo rm -rf /opt/ghc\n sudo rm -rf /opt/hostedtoolcache/CodeQL\n sudo docker image prune --all --force\n df -h\n\n - name: Set up Docker Buildx\n uses: docker/setup-buildx-action@v3\n\n - name: Login to GitHub Container Registry\n uses: docker/login-action@v3\n with:\n registry: ghcr.io\n username: ${{ github.actor }}\n password: ${{ secrets.GITHUB_TOKEN }}\n\n - name: Login to Docker Hub\n if: needs.extract-version.outputs.has_dockerhub_secrets == 'true'\n uses: docker/login-action@v3\n with:\n username: ${{ secrets.DOCKER_USERNAME }}\n password: ${{ secrets.DOCKER_PASSWORD }}\n\n - name: Cache Docker layers\n uses: actions/cache@v3\n with:\n path: /tmp/.buildx-cache-dev-single\n key: ${{ runner.os }}-buildx-dev-single-${{ github.sha }}\n restore-keys: |\n ${{ runner.os }}-buildx-dev-single-\n\n - name: Prepare Docker tags\n id: tags\n run: |\n TAGS=\"${{ env.GHCR_IMAGE }}:v1-dev-single\"\n if [[ \"${{ needs.extract-version.outputs.has_dockerhub_secrets }}\" == \"true\" ]]; then\n TAGS=\"${TAGS},${{ env.DOCKERHUB_IMAGE }}:v1-dev-single\"\n fi\n echo \"tags=${TAGS}\" >> $GITHUB_OUTPUT\n\n - name: Build and push single-container image\n uses: docker/build-push-action@v5\n with:\n context: .\n file: ./Dockerfile.single\n platforms: linux/amd64,linux/arm64\n push: true\n tags: ${{ steps.tags.outputs.tags }}\n cache-from: type=local,src=/tmp/.buildx-cache-dev-single\n cache-to: type=local,dest=/tmp/.buildx-cache-dev-single-new,mode=max\n\n - name: Move cache\n run: |\n rm -rf /tmp/.buildx-cache-dev-single\n mv /tmp/.buildx-cache-dev-single-new /tmp/.buildx-cache-dev-single\n\n summary:\n needs: [extract-version, build-regular, build-single]\n runs-on: ubuntu-latest\n if: always()\n steps:\n - name: Development Build Summary\n run: |\n echo \"## Development Build Summary\" >> $GITHUB_STEP_SUMMARY\n echo \"**Version:** ${{ needs.extract-version.outputs.version }}\" >> $GITHUB_STEP_SUMMARY\n echo \"**Event:** ${{ github.event_name }}\" >> $GITHUB_STEP_SUMMARY\n echo \"**Push to Main:** ${{ needs.extract-version.outputs.is_push_to_main }}\" >> $GITHUB_STEP_SUMMARY\n echo \"\" >> $GITHUB_STEP_SUMMARY\n\n if [[ \"${{ needs.extract-version.outputs.is_push_to_main }}\" == \"true\" ]]; then\n echo \"### Published Tags:\" >> $GITHUB_STEP_SUMMARY\n\n if [[ \"${{ needs.build-regular.result }}\" == \"success\" ]]; then\n echo \"✅ **Regular:** \\`${{ env.GHCR_IMAGE }}:v1-dev\\`\" >> $GITHUB_STEP_SUMMARY\n if [[ \"${{ needs.extract-version.outputs.has_dockerhub_secrets }}\" == \"true\" ]]; then\n echo \"✅ **Regular (Docker Hub):** \\`${{ env.DOCKERHUB_IMAGE }}:v1-dev\\`\" >> $GITHUB_STEP_SUMMARY\n fi\n else\n echo \"❌ **Regular:** Build failed\" >> $GITHUB_STEP_SUMMARY\n fi\n\n if [[ \"${{ needs.build-single.result }}\" == \"success\" ]]; then\n echo \"✅ **Single:** \\`${{ env.GHCR_IMAGE }}:v1-dev-single\\`\" >> $GITHUB_STEP_SUMMARY\n if [[ \"${{ needs.extract-version.outputs.has_dockerhub_secrets }}\" == \"true\" ]]; then\n echo \"✅ **Single (Docker Hub):** \\`${{ env.DOCKERHUB_IMAGE }}:v1-dev-single\\`\" >> $GITHUB_STEP_SUMMARY\n fi\n elif [[ \"${{ needs.build-single.result }}\" == \"skipped\" ]]; then\n echo \"⏭️ **Single:** Skipped\" >> $GITHUB_STEP_SUMMARY\n else\n echo \"❌ **Single:** Build failed\" >> $GITHUB_STEP_SUMMARY\n fi\n\n echo \"\" >> $GITHUB_STEP_SUMMARY\n echo \"### Platforms:\" >> $GITHUB_STEP_SUMMARY\n echo \"- linux/amd64\" >> $GITHUB_STEP_SUMMARY\n echo \"- linux/arm64\" >> $GITHUB_STEP_SUMMARY\n else\n echo \"### Test Build Results:\" >> $GITHUB_STEP_SUMMARY\n if [[ \"${{ needs.build-regular.result }}\" == \"success\" ]]; then\n echo \"✅ **Dockerfile:** Build successful\" >> $GITHUB_STEP_SUMMARY\n else\n echo \"❌ **Dockerfile:** Build failed\" >> $GITHUB_STEP_SUMMARY\n fi\n echo \"\" >> $GITHUB_STEP_SUMMARY\n echo \"### Notes:\" >> $GITHUB_STEP_SUMMARY\n echo \"- This is a test build (no images pushed to registry)\" >> $GITHUB_STEP_SUMMARY\n echo \"- Merge to main to publish \\`v1-dev\\` tags\" >> $GITHUB_STEP_SUMMARY\n echo \"- For stable releases, use the 'Build and Release' workflow\" >> $GITHUB_STEP_SUMMARY\n fi\n" }, { "path": ".github/workflows/claude-code-review.yml", "content": "name: Claude Code Review\n\non:\n pull_request:\n types: [opened, synchronize, ready_for_review, reopened]\n pull_request_target:\n types: [opened, synchronize, ready_for_review, reopened]\n # Optional: Only run on specific file changes\n # paths:\n # - \"src/**/*.ts\"\n # - \"src/**/*.tsx\"\n # - \"src/**/*.js\"\n # - \"src/**/*.jsx\"\n\njobs:\n claude-review:\n # Run for fork PRs (via pull_request_target) OR same-repo PRs (via pull_request), but not both\n if: |\n (github.event_name == 'pull_request_target' && github.event.pull_request.head.repo.full_name != github.repository) ||\n (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository)\n\n runs-on: ubuntu-latest\n permissions:\n contents: read\n pull-requests: write\n issues: write\n id-token: write\n\n steps:\n - name: Checkout repository\n uses: actions/checkout@v4\n with:\n ref: ${{ github.event.pull_request.head.sha }}\n fetch-depth: 1\n persist-credentials: false\n\n - name: Run Claude Code Review\n id: claude-review\n uses: anthropics/claude-code-action@v1\n with:\n github_token: ${{ secrets.GITHUB_TOKEN }}\n claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }}\n plugin_marketplaces: 'https://github.com/anthropics/claude-code.git'\n plugins: 'code-review@claude-code-plugins'\n prompt: '/code-review:code-review ${{ github.repository }}/pull/${{ github.event.pull_request.number }}'\n # See https://github.com/anthropics/claude-code-action/blob/main/docs/usage.md\n # or https://code.claude.com/docs/en/cli-reference for available options\n\n" }, { "path": ".github/workflows/claude.yml", "content": "name: Claude Code\n\non:\n issue_comment:\n types: [created]\n pull_request_review_comment:\n types: [created]\n issues:\n types: [opened, assigned]\n pull_request_review:\n types: [submitted]\n\njobs:\n claude:\n if: |\n (github.event_name == 'issue_comment' && contains(github.event.comment.body, '@claude')) ||\n (github.event_name == 'pull_request_review_comment' && contains(github.event.comment.body, '@claude')) ||\n (github.event_name == 'pull_request_review' && contains(github.event.review.body, '@claude')) ||\n (github.event_name == 'issues' && (contains(github.event.issue.body, '@claude') || contains(github.event.issue.title, '@claude')))\n runs-on: ubuntu-latest\n permissions:\n contents: read\n pull-requests: write\n issues: write\n id-token: write\n actions: read # Required for Claude to read CI results on PRs\n steps:\n - name: Checkout repository\n uses: actions/checkout@v4\n with:\n fetch-depth: 1\n\n - name: Run Claude Code\n id: claude\n uses: anthropics/claude-code-action@v1\n with:\n github_token: ${{ secrets.GITHUB_TOKEN }}\n claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }}\n\n # This is an optional setting that allows Claude to read CI results on PRs\n additional_permissions: |\n actions: read\n\n # Optional: Give a custom prompt to Claude. If this is not specified, Claude will perform the instructions specified in the comment that tagged it.\n # prompt: 'Update the pull request description to include a summary of changes.'\n\n # Optional: Add claude_args to customize behavior and configuration\n # See https://github.com/anthropics/claude-code-action/blob/main/docs/usage.md\n # or https://code.claude.com/docs/en/cli-reference for available options\n # claude_args: '--allowed-tools Bash(gh pr:*)'\n\n" }, { "path": ".github/workflows/test.yml", "content": "name: Tests\n\non:\n pull_request:\n branches: [main]\n push:\n branches: [main]\n paths-ignore:\n - '**.md'\n - 'docs/**'\n - '.github/workflows/claude*.yml'\n\npermissions:\n contents: read\n\njobs:\n backend:\n name: Backend Tests\n runs-on: ubuntu-latest\n steps:\n - name: Checkout\n uses: actions/checkout@v4\n\n - name: Set up uv\n uses: astral-sh/setup-uv@v4\n with:\n enable-cache: true\n\n - name: Set up Python\n run: uv python install\n\n - name: Install dependencies\n run: uv sync\n\n - name: Run tests\n run: uv run pytest tests/ -v\n\n frontend:\n name: Frontend Tests\n runs-on: ubuntu-latest\n defaults:\n run:\n working-directory: frontend\n steps:\n - name: Checkout\n uses: actions/checkout@v4\n\n - name: Set up Node.js\n uses: actions/setup-node@v4\n with:\n node-version: 22\n cache: npm\n cache-dependency-path: frontend/package-lock.json\n\n - name: Install dependencies\n run: npm ci\n\n - name: Run tests\n run: npm test\n" }, { "path": ".gitignore", "content": ".env\nprompts/patterns/user/\n/notebooks/\ndata/\n.uploads/\nsqlite-db/\nsurreal-data/\ndocker.env\nnotebook_data/\n# Python-specific\n*.py[cod]\n__pycache__/\n*.so\ntodo.md\ntemp/\ngoogle-credentials.json\n# Distribution / packaging\n.Python\nbuild/\ndevelop-eggs/\ndist/\ndownloads/\neggs/\n.eggs/\n/lib/\n/lib64/\nparts/\nsdist/\nvar/\nwheels/\nshare/python-wheels/\n*.egg-info/\n.installed.cfg\n*.egg\n\n# PyInstaller\n*.manifest\n*.spec\n\n# Installer logs\npip-log.txt\npip-delete-this-directory.txt\n\n# Unit test / coverage reports\nhtmlcov/\n.tox/\n.nox/\n.coverage\n.coverage.*\n.cache\nnosetests.xml\ncoverage.xml\n*.cover\n*.py,cover\n.hypothesis/\n.pytest_cache/\n\n# Jupyter Notebook\n.ipynb_checkpoints\n\n# IPython\nprofile_default/\nipython_config.py\n\n# Environments\n.env\n.venv\nenv/\nvenv/\nENV/\nenv.bak/\nvenv.bak/\n\n# PyCharm\n.idea/\n\n# VS Code\n.vscode/\n\n# Spyder project settings\n.spyderproject\n.spyproject\n\n# Rope project settings\n.ropeproject\n\n# mkdocs documentation\n/site\n\n# mypy\n.mypy_cache/\n.dmypy.json\ndmypy.json\n\n# Pyre type checker\n.pyre/\n\n# pytype static type analyzer\n.pytype/\n\n# Cython debug symbols\ncython_debug/\n\n# macOS\n.DS_Store\n\n# Windows\nThumbs.db\nehthumbs.db\ndesktop.ini\n\n# Linux\n*~\n\n# Log files\n*.log\n\n# Database files\n*.db\n*.sqlite3\n\n.quarentena\n\nclaude-logs/\n.claude/sessions\n**/claude-logs\n\n\ndocs/custom_gpt\ndoc_exports/\n\nspecs/\n.claude\n.sisyphus\n\n.playwright-mcp/\n\n\n\n*.local.yml\n**/*.local.md" }, { "path": ".python-version", "content": "3.12\n" }, { "path": ".worktreeinclude", "content": ".env\n.env.local\n.env.*\n**/.claude/settings.local.json\nCLAUDE.local.md\n" }, { "path": "CHANGELOG.md", "content": "# Changelog\n\nAll notable changes to this project will be documented in this file.\n\nThe format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),\nand this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).\n\n## [Unreleased]\n\n## [1.8.1] - 2026-03-10\n\n### Added\n- i18n support for Bengali (bn-IN) (#643)\n- Podcast language support via podcast-creator 0.12.0 (#645)\n- Upgrade default Azure API version for model testing and fetching (#638)\n\n### Fixed\n- Tiktoken network errors in offline/air-gapped Docker deployments — pre-downloads encoding at build time (#264, #622)\n- SurrealDB getting stuck (#656)\n\n### Dependencies\n- Bump esperanto to 2.19.5 (#657)\n- Bump langgraph from 1.0.6 to 1.0.10rc1 (#658)\n- Bump authlib from 1.6.6 to 1.6.7 (#649)\n- Bump lxml-html-clean from 0.4.3 to 0.4.4 (#646)\n- Bump rollup from 4.55.1 to 4.59.0 (#635)\n- Bump minimatch in frontend (#634)\n- Bump tar from 7.5.9 to 7.5.11 (#650, #659)\n\n## [1.7.4] - 2026-02-18\n\n### Fixed\n- Embedding large documents (3MB+) fails with 413 Payload Too Large (#594)\n- `generate_embeddings()` now batches texts in groups of 50 with per-batch retry, preventing provider payload limits from being exceeded\n- 413 errors now classified with user-friendly message in error classifier\n- Misleading \"Created 0 embedded chunks\" log in `process_source_command` — embedding is fire-and-forget, so the count was always 0; now logs \"embedding submitted\" instead\n\n## [1.7.3] - 2026-02-17\n\n### Added\n- Retry button for failed podcast episodes in the UI (#211, #218)\n- Error details displayed on failed podcast episodes (#185, #355)\n- `POST /podcasts/episodes/{id}/retry` API endpoint for re-submitting failed episodes\n- `error_message` field in podcast episode API responses\n\n### Fixed\n- Podcast generation failures now correctly marked as \"failed\" instead of \"completed\" (#300, #335)\n- Disabled automatic retries for podcast generation to prevent duplicate episode records (#302)\n\n### Dependencies\n- Bump podcast-creator to >= 0.11.2\n- Bump esperanto to >= 2.19.4\n\n## [1.7.2] - 2026-02-16\n\n### Added\n- Error classification utility that maps LLM provider errors to user-friendly messages (#506)\n- Global exception handlers in FastAPI for all custom exception types with proper HTTP status codes\n- `getApiErrorMessage()` frontend helper that falls back to backend messages when no i18n mapping exists\n\n### Fixed\n- LLM errors (invalid API key, wrong model, rate limits) now show descriptive messages instead of \"An unexpected error occurred\" (#590)\n- SSE streaming error events in source chat and ask hooks were swallowed by inner JSON parse catch blocks\n- Transformation execution errors were caught and re-wrapped as generic 500s instead of using proper status codes\n- Fail fast when source content extraction returns empty instead of retrying (#589)\n- Chat input and message overflow with long unbroken strings (#588)\n- Word-wrap overflow in source cards, note editor, inline edit, note titles, and dialog content (#588)\n- Translation proxy shadowing `name` keys (#588)\n- OpenAI-compatible provider name handling via Esperanto update (#583)\n\n### Changed\n- `ValueError` replaced with `ConfigurationError` in model provisioning for proper error classification\n- `ConfigurationError` added to command retry `stop_on` lists to avoid retrying permanent config failures\n\n### Dependencies\n- Bump esperanto to 2.19.3 (#583)\n- Bump podcast-creator to 0.9.1\n\n## [1.7.1] - 2026-02-14\n\n### Added\n- French (fr-FR) language support (#581)\n- CI test workflow and improved i18n validation (#580)\n- Expose embed `command_id` in note API responses (#545)\n\n### Fixed\n- ElevenLabs TTS credential passthrough via Esperanto update (#578)\n- Handle empty/whitespace source content without retry loop (#576)\n- Increase transformation `max_tokens` and update Esperanto dep (#568)\n- Turn the embedding field into optional (#557)\n\n### Docs\n- Fix docker container names in local setup guides (#577)\n\n### Dependencies\n- Bump langchain-core from 1.2.7 to 1.2.11 (#564)\n- Bump cryptography from 46.0.3 to 46.0.5 (#563)\n\n## [1.7.0] - 2026-02-10\n\n### Added\n- **Credential-Based Provider Management** (#477)\n - New Settings → API Keys page for managing AI provider credentials via the UI\n - Support for 14 providers: OpenAI, Anthropic, Google, Groq, Mistral, DeepSeek, xAI, OpenRouter, Voyage AI, ElevenLabs, Ollama, Azure OpenAI, OpenAI-Compatible, and Vertex AI\n - Secure storage of API keys in SurrealDB with field-level encryption (Fernet AES-128-CBC + HMAC-SHA256)\n - One-click connection testing, model discovery, and model registration per credential\n - Migration tool to import existing environment variable keys into the credential system\n - Azure OpenAI support with service-specific endpoints (LLM, Embedding, STT, TTS)\n - OpenAI-Compatible support with per-service URL configurations\n - Vertex AI support with project, location, and credentials path\n - Environment variable API keys deprecated in favor of Settings UI\n\n- **Security Enhancements**\n - Docker secrets support via `_FILE` suffix pattern (e.g., `OPEN_NOTEBOOK_PASSWORD_FILE`)\n - Default encryption key derived from \"0p3n-N0t3b0ok\" for easy setup (change in production!)\n - Default password \"open-notebook-change-me\" for out-of-box experience (change in production!)\n - URL validation for SSRF protection - blocks private IPs and localhost (except for Ollama which runs locally)\n - Security warnings logged when using default credentials\n\n- HTML clipboard detection for text sources (#426)\n - When pasting content, automatically detects HTML format (e.g., from Word, web pages)\n - Shows info message when HTML is detected, informing user it will be converted to Markdown\n - Preserves formatting that would be lost with plain text paste\n - Bump content-core to 0.11.0 for HTML to Markdown conversion support\n\n- **Improved Getting Started Experience**\n - Simplified docker-compose.yml in repository root (single official file)\n - Added examples/ folder with ready-made configurations:\n - `docker-compose-ollama.yml` - Local AI with Ollama\n - `docker-compose-speaches.yml` - Local TTS/STT with Speaches\n - `docker-compose-full-local.yml` - 100% local setup (Ollama + Speaches)\n - Inline quick start in README (no need to navigate to docs)\n - Cross-references between docker-compose examples and documentation\n - .env.example template with all configuration options\n\n### Fixed\n- Azure form race condition: all configuration now saved in single atomic request\n- Migration API \"error error\" display: added proper MigrationResult model with message field\n- Connection tester for Ollama providers: improved error handling and URL validation\n- SqliteSaver async compatibility issues in chat system (#509, #525, #538)\n- Re-embedding failures with empty content (#513, #515)\n- Deletion cascade for notes and sources (#77)\n- YouTube content availability issues (#494)\n- Large document embedding errors (#489)\n\n### Security\n- API keys are encrypted at rest using Fernet symmetric encryption\n- Keys are never returned to the frontend, only configuration status\n- SSRF protection prevents internal network access via URL validation\n\n### Docs\n- Complete documentation update for credential-based system across 25 files\n- All quick-start, installation, and configuration guides now use Settings UI workflow\n- Environment variable API key instructions moved to deprecated/legacy sections\n- Fixed broken links in installation docs\n- Added comprehensive examples/ folder with documented docker-compose configurations\n- Updated local-tts.md and local-stt.md with links to ready-made examples\n\n### Internationalization\n- Added Russian (ru-RU) language support (#524)\n- Added Italian (it-IT) language support (#508)\n\n## [1.6.2] - 2026-01-24\n\n### Fixed\n- Connection error with llama.cpp and OpenAI-compatible providers (#465)\n - Bump Esperanto to 2.17.2 which fixes LangChain connection errors caused by garbage collection\n\n## [1.6.1] - 2026-01-22\n\n### Fixed\n- \"Failed to send message\" error with unhelpful logs when chat model is not configured (#358)\n - Added detailed error logging with model selection context and full traceback\n - Improved error messages to guide users to Settings → Models\n - Added warnings when default models are not configured\n\n### Docs\n- Ollama troubleshooting: Added \"Model Name Configuration\" section emphasizing exact model names from `ollama list`\n- Added troubleshooting entry for \"Failed to send message\" error with step-by-step solutions\n- Updated AI Chat Issues documentation with model configuration guidance\n\n\n## [1.6.0] - 2026-01-21\n\n### Added\n- Content-type aware text chunking with automatic HTML, Markdown, and plain text detection (#350, #142)\n- Unified embedding generation with mean pooling for large content that exceeds model context limits\n- Dedicated embedding commands: `embed_note`, `embed_insight`, `embed_source`\n- New utility modules: `chunking.py` and `embedding.py` in `open_notebook/utils/`\n- Japanese (ja-JP) language support (#450)\n\n### Changed\n- Embedding is now fire-and-forget: domain models submit embedding commands asynchronously after save\n- `rebuild_embeddings_command` now delegates to individual embed_* commands instead of inline processing\n- Chunk size reduced to 1500 characters for better compatibility with Ollama embedding models\n- Bump Esperanto to 2.16 for increased Ollama context window support\n\n### Removed\n- Legacy embedding commands: `embed_single_item_command`, `embed_chunk_command`, `vectorize_source_command`\n- `needs_embedding()` and `get_embedding_content()` methods from domain models\n- `split_text()` function from text_utils (replaced by `chunk_text()` in chunking module)\n\n### Fixed\n- Embedding failures when content exceeds model context limits (#350, #142)\n- Empty note titles when saving from chat (clean thinking tags from prompt graph output)\n- Orphaned embedding/insight records when deleting sources (cascade delete)\n- Search results crash with null parent_id (defensive frontend check)\n- Database migration 10 cleans up existing orphaned records\n\n## [1.5.2] - 2026-01-15\n\n### Performance\n- Improved source listing speed by 20-30x (#436, closes #351)\n - Added database indexes on `source` field for `source_insight` and `source_embedding` tables\n - Use SurrealDB `FETCH` clause for command status instead of N async calls\n\n## [1.5.1] - 2026-01-15\n\n### Fixed\n- Podcast dialog infinite loop error caused by excessive translation Proxy accesses in loops\n- Podcast dialog UI freezing when typing episode name or additional instructions\n- Removed incorrect translation keys for user-defined episode profiles (user content should not be translated)\n\n## [1.5.0] - 2026-01-15\n\n### Added\n- Internationalization (i18n) support with Chinese (Simplified and Traditional) translations (#371, closes #344, #349, #360)\n- Frontend test infrastructure with Vitest (#371)\n- Language toggle component for switching UI language (#371)\n- Date localization using date-fns locales (#371)\n- Error message translation system (#371)\n\n### Fixed\n- Accessibility improvements: added missing `id`, `name`, and `autoComplete` attributes to form inputs (#371)\n- Added `DialogDescription` to dialogs for Radix UI accessibility compliance (#371)\n- Fixed \"Collapsible is changing from uncontrolled to controlled\" warning in SettingsForm (#371)\n- Fixed lint command for Next.js 16 compatibility (`eslint` instead of `next lint`)\n\n### Changed\n- Dockerfile optimizations: better layer caching, `--no-install-recommends` for smaller images (#371)\n- Dockerfile.single refactored into 3 separate build stages for better caching (#371)\n\n## [1.4.0] - 2026-01-14\n\n### Added\n- CTA button to empty state notebook list for better onboarding (#408)\n- Offline deployment support for Docker containers (#414)\n\n### Fixed\n- Large file uploads (>10MB) by upgrading to Next.js 16 (#423)\n- Orphaned uploaded files when sources are removed (#421)\n- Broken documentation links to ai-providers.md (#419)\n- ZIP support indication removed from UI (#418)\n- Duplicate Claude Code workflow runs on PRs (#417)\n- Claude Code review workflow now runs on PRs from forks (#416)\n\n### Changed\n- Upgraded Next.js from 15.4.10 to 16.1.1 (#423)\n- Upgraded React from 19.1.0 to 19.2.3 (#423)\n- Renamed `middleware.ts` to `proxy.ts` for Next.js 16 compatibility (#423)\n\n### Dependencies\n- next: 15.4.10 → 16.1.1\n- react: 19.1.0 → 19.2.3\n- react-dom: 19.1.0 → 19.2.3\n\n## [1.2.4] - 2025-12-14\n\n### Added\n- Infinite scroll for notebook sources - no more 50 source limit (#325)\n- Markdown table rendering in chat responses, search results, and insights (#325)\n\n### Fixed\n- Timeout errors with Ollama and local LLMs - increased to 10 minutes (#325)\n- \"Unable to Connect to API Server\" on Docker startup - frontend now waits for API health check (#325, #315)\n- SSL issues with langchain (#274)\n- Query key consistency for source mutations to properly refresh infinite scroll (#325)\n- Docker compose start-all flow (#323)\n\n### Changed\n- Timeout configuration now uses granular httpx.Timeout (short connect, long read) (#325)\n\n### Dependencies\n- Updated next.js to 15.4.10\n- Updated httpx to >=0.27.0 for SSL fix\n" }, { "path": "CLAUDE.md", "content": "# Open Notebook - Root CLAUDE.md\n\nThis file provides architectural guidance for contributors working on Open Notebook at the project level.\n\n## Project Overview\n\n**Open Notebook** is an open-source, privacy-focused alternative to Google's Notebook LM. It's an AI-powered research assistant enabling users to upload multi-modal content (PDFs, audio, video, web pages), generate intelligent notes, search semantically, chat with AI models, and produce professional podcasts—all with complete control over data and choice of AI providers.\n\n**Key Values**: Privacy-first, multi-provider AI support, fully self-hosted option, open-source transparency.\n\n---\n\n## Three-Tier Architecture\n\n```\n┌─────────────────────────────────────────────────────────┐\n│ Frontend (React/Next.js) │\n│ frontend/ @ port 3000 │\n├─────────────────────────────────────────────────────────┤\n│ - Notebooks, sources, notes, chat, podcasts, search UI │\n│ - Zustand state management, TanStack Query (React Query)│\n│ - Shadcn/ui component library with Tailwind CSS │\n└────────────────────────┬────────────────────────────────┘\n │ HTTP REST\n┌────────────────────────▼────────────────────────────────┐\n│ API (FastAPI) │\n│ api/ @ port 5055 │\n├─────────────────────────────────────────────────────────┤\n│ - REST endpoints for notebooks, sources, notes, chat │\n│ - LangGraph workflow orchestration │\n│ - Job queue for async operations (podcasts) │\n│ - Multi-provider AI provisioning via Esperanto │\n└────────────────────────┬────────────────────────────────┘\n │ SurrealQL\n┌────────────────────────▼────────────────────────────────┐\n│ Database (SurrealDB) │\n│ Graph database @ port 8000 │\n├─────────────────────────────────────────────────────────┤\n│ - Records: Notebook, Source, Note, ChatSession, Credential│\n│ - Relationships: source-to-notebook, note-to-source │\n│ - Vector embeddings for semantic search │\n└─────────────────────────────────────────────────────────┘\n```\n\n---\n\n## Useful sources\n\nUser documentation is at @docs/\n\n## Tech Stack\n\n### Frontend (`frontend/`)\n- **Framework**: Next.js 16 (React 19)\n- **Language**: TypeScript\n- **State Management**: Zustand\n- **Data Fetching**: TanStack Query (React Query)\n- **Styling**: Tailwind CSS + Shadcn/ui\n- **Build Tool**: Webpack (via Next.js)\n- **i18n compatible**: All front-end changes must also consider the translation keys\n\n### API Backend (`api/` + `open_notebook/`)\n- **Framework**: FastAPI 0.104+\n- **Language**: Python 3.11+\n- **Workflows**: LangGraph state machines\n- **Database**: SurrealDB async driver\n- **AI Providers**: Esperanto library (8+ providers: OpenAI, Anthropic, Google, Groq, Ollama, Mistral, DeepSeek, xAI)\n- **Job Queue**: Surreal-Commands for async jobs (podcasts)\n- **Logging**: Loguru\n- **Validation**: Pydantic v2\n- **Testing**: Pytest\n\n### Database\n- **SurrealDB**: Graph database with built-in embedding storage and vector search\n- **Schema Migrations**: Automatic on API startup via AsyncMigrationManager\n\n### Additional Services\n- **Content Processing**: content-core library (file/URL extraction)\n- **Prompts**: AI-Prompter with Jinja2 templating\n- **Podcast Generation**: podcast-creator library\n- **Embeddings**: Multi-provider via Esperanto\n\n---\n\n## Architecture Highlights\n\n### 1. Async-First Design\n- All database queries, graph invocations, and API calls are async (await)\n- SurrealDB async driver with connection pooling\n- FastAPI handles concurrent requests efficiently\n\n### 2. LangGraph Workflows\n- **source.py**: Content ingestion (extract → embed → save)\n- **chat.py**: Conversational agent with message history\n- **ask.py**: Search + synthesis (retrieve relevant sources → LLM)\n- **transformation.py**: Custom transformations on sources\n- All use `provision_langchain_model()` for smart model selection\n\n### 3. Multi-Provider AI\n- **Esperanto library**: Unified interface to 8+ AI providers\n- **Credential system**: Individual encrypted credential records per provider; models link to credentials for direct config\n- **ModelManager**: Factory pattern with fallback logic; uses credential config when available, env vars as fallback\n- **Smart selection**: Detects large contexts, prefers long-context models\n- **Override support**: Per-request model configuration\n\n### 4. Database Schema\n- **Automatic migrations**: AsyncMigrationManager runs on API startup\n- **SurrealDB graph model**: Records with relationships and embeddings\n- **Vector search**: Built-in semantic search across all content\n- **Transactions**: Repo functions handle ACID operations\n\n### 5. Authentication\n- **Current**: Simple password middleware (insecure, dev-only)\n- **Production**: Replace with OAuth/JWT (see CONFIGURATION.md)\n\n---\n\n## Important Quirks & Gotchas\n\n### API Startup\n- **Migrations run automatically** on startup; check logs for errors\n- **Must start API before UI**: UI depends on API for all data\n- **SurrealDB must be running**: API fails without database connection\n\n### Frontend-Backend Communication\n- **Base API URL**: Configured in `.env.local` (default: http://localhost:5055)\n- **CORS enabled**: Configured in `api/main.py` (allow all origins in dev)\n- **Rate limiting**: Not built-in; add at proxy layer for production\n\n### LangGraph Workflows\n- **Blocking operations**: Chat/podcast workflows may take minutes; no timeout\n- **State persistence**: Uses SQLite checkpoint storage in `/data/sqlite-db/`\n- **Model fallback**: If primary model fails, falls back to cheaper/smaller model\n\n### Podcast Generation\n- **Async job queue**: `podcast_service.py` submits jobs but doesn't wait\n- **Track status**: Use `/commands/{command_id}` endpoint to poll status\n- **TTS failures**: Fall back to silent audio if speech synthesis fails\n\n### Content Processing\n- **File extraction**: Uses content-core library; supports 50+ file types\n- **URL handling**: Extracts text + metadata from web pages\n- **Large files**: Content processing is sync; may block API briefly\n\n---\n\n## Component References\n\nSee dedicated CLAUDE.md files for detailed guidance:\n\n- **[frontend/CLAUDE.md](frontend/CLAUDE.md)**: React/Next.js architecture, state management, API integration\n- **[api/CLAUDE.md](api/CLAUDE.md)**: FastAPI structure, service pattern, endpoint development\n- **[open_notebook/CLAUDE.md](open_notebook/CLAUDE.md)**: Backend core, domain models, LangGraph workflows, AI provisioning\n- **[open_notebook/domain/CLAUDE.md](open_notebook/domain/CLAUDE.md)**: Data models, repository pattern, search functions\n- **[open_notebook/ai/CLAUDE.md](open_notebook/ai/CLAUDE.md)**: ModelManager, AI provider integration, Esperanto usage\n- **[open_notebook/graphs/CLAUDE.md](open_notebook/graphs/CLAUDE.md)**: LangGraph workflow design, state machines\n- **[open_notebook/database/CLAUDE.md](open_notebook/database/CLAUDE.md)**: SurrealDB operations, migrations, async patterns\n\n---\n\n## Documentation Map\n\n- **[README.md](README.md)**: Project overview, features, quick start\n- **[docs/index.md](docs/index.md)**: Complete user & deployment documentation\n- **[CONFIGURATION.md](CONFIGURATION.md)**: Environment variables, model configuration\n- **[CONTRIBUTING.md](CONTRIBUTING.md)**: Contribution guidelines\n- **[MAINTAINER_GUIDE.md](MAINTAINER_GUIDE.md)**: Release & maintenance procedures\n\n---\n\n## Testing Strategy\n\n- **Unit tests**: `tests/test_domain.py`, `test_models_api.py`\n- **Graph tests**: `tests/test_graphs.py` (workflow integration)\n- **Utils tests**: `tests/test_utils.py`, `tests/test_chunking.py`, `tests/test_embedding.py`\n- **Run all**: `uv run pytest tests/`\n- **Coverage**: Check with `pytest --cov`\n\n---\n\n## Common Tasks\n\n### Add a New API Endpoint\n1. Create router in `api/routers/feature.py`\n2. Create service in `api/feature_service.py`\n3. Define schemas in `api/models.py`\n4. Register router in `api/main.py`\n5. Test via http://localhost:5055/docs\n\n### Add a New LangGraph Workflow\n1. Create `open_notebook/graphs/workflow_name.py`\n2. Define StateDict and node functions\n3. Build graph with `.add_node()` / `.add_edge()`\n4. Invoke in service: `graph.ainvoke({\"input\": ...}, config={\"...\"})`\n5. Test with sample data in `tests/`\n\n### Add Database Migration\n1. Create `migrations/XXX_description.surql`\n2. Write SurrealQL schema changes\n3. Create `migrations/XXX_description_down.surql` (optional rollback)\n4. API auto-detects on startup; migration runs if newer than recorded version\n\n### Deploy to Production\n1. Review [CONFIGURATION.md](CONFIGURATION.md) for security settings\n2. Use `make docker-release` for multi-platform image\n3. Push to Docker Hub / GitHub Container Registry\n4. Deploy `docker compose --profile multi up`\n5. Verify migrations via API logs\n\n---\n\n## Support & Community\n\n- **Documentation**: https://open-notebook.ai\n- **Discord**: https://discord.gg/37XJPXfz2w\n- **Issues**: https://github.com/lfnovo/open-notebook/issues\n- **License**: MIT (see LICENSE)\n\n" }, { "path": "CONFIGURATION.md", "content": "# Configuration Guide\n\n**📍 This file has moved!**\n\nAll configuration documentation has been consolidated into the new documentation structure.\n\n👉 **[Read the Configuration Guide](docs/5-CONFIGURATION/index.md)**\n\n---\n\n## Quick Links\n\n- **AI Provider Setup** → [AI Providers](docs/5-CONFIGURATION/ai-providers.md)\n- **Environment Variables Reference** → [Environment Reference](docs/5-CONFIGURATION/environment-reference.md)\n- **Database Configuration** → [Database Setup](docs/5-CONFIGURATION/database.md)\n- **Server Configuration** → [Server Settings](docs/5-CONFIGURATION/server.md)\n- **Security Setup** → [Security Configuration](docs/5-CONFIGURATION/security.md)\n- **Reverse Proxy** → [Reverse Proxy Setup](docs/5-CONFIGURATION/reverse-proxy.md)\n- **Advanced Tuning** → [Advanced Configuration](docs/5-CONFIGURATION/advanced.md)\n\n---\n\n## What You'll Find\n\nThe new configuration documentation includes:\n\n- **Complete environment variable reference** with examples\n- **Provider-specific setup guides** for OpenAI, Anthropic, Google, Groq, Ollama, and more\n- **Production deployment configurations** with security best practices\n- **Reverse proxy examples** for Nginx, Caddy, Traefik\n- **Database tuning** for performance optimization\n- **Troubleshooting guides** for common configuration issues\n\n---\n\nFor all configuration details, see **[docs/5-CONFIGURATION/](docs/5-CONFIGURATION/index.md)**.\n" }, { "path": "CONTRIBUTING.md", "content": "# Contributing to Open Notebook\n\n**📍 This file has moved!**\n\nAll contribution guidelines have been consolidated into the new development documentation structure.\n\n👉 **[Read the Contributing Guide](docs/7-DEVELOPMENT/contributing.md)**\n\n---\n\n## Quick Links\n\n- **Want to contribute code?** → [Contributing Guide](docs/7-DEVELOPMENT/contributing.md)\n- **Want to understand the architecture?** → [Architecture Overview](docs/7-DEVELOPMENT/architecture.md)\n- **Want to understand our design philosophy?** → [Design Principles](docs/7-DEVELOPMENT/design-principles.md)\n- **Are you a maintainer?** → [Maintainer Guide](docs/7-DEVELOPMENT/maintainer-guide.md)\n- **New developer?** → [Quick Start](docs/7-DEVELOPMENT/quick-start.md)\n\n---\n\n## The Issue-First Workflow\n\n**TL;DR**: Create an issue first, get it assigned, THEN code.\n\nThis prevents wasted effort and ensures your work aligns with the project. [See details →](docs/7-DEVELOPMENT/contributing.md)\n\n---\n\nFor all contribution details, see **[docs/7-DEVELOPMENT/contributing.md](docs/7-DEVELOPMENT/contributing.md)**.\n" }, { "path": "Dockerfile", "content": "# Build stage\nFROM python:3.12-slim-bookworm AS builder\n\n# Install uv using the official method\nCOPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/\n\n# Install system dependencies required for building certain Python packages\n# Add Node.js 20.x LTS for building frontend\n# NOTE: gcc/g++/make removed - uv should download pre-built wheels. Add back if build fails.\n# NOTE: gcc/g++/make required for some python dependencies\nRUN apt-get update && apt-get install -y --no-install-recommends \\\n curl \\\n build-essential \\\n && curl -fsSL https://deb.nodesource.com/setup_20.x | bash - \\\n && apt-get install -y nodejs \\\n && rm -rf /var/lib/apt/lists/*\n\n# Set build optimization environment variables\nENV MAKEFLAGS=\"-j$(nproc)\"\nENV PYTHONDONTWRITEBYTECODE=1\nENV PYTHONUNBUFFERED=1\nENV UV_COMPILE_BYTECODE=1\nENV UV_LINK_MODE=copy\n\n# Set the working directory in the container to /app\nWORKDIR /app\n\n# Copy dependency files and minimal package structure first for better layer caching\nCOPY pyproject.toml uv.lock ./\nCOPY open_notebook/__init__.py ./open_notebook/__init__.py\n\n# Install dependencies with optimizations (this layer will be cached unless dependencies change)\nRUN uv sync --frozen --no-dev\n\n# Pre-download tiktoken encoding so the app works offline (issue #264).\n# /app/tiktoken-cache is intentionally outside /app/data/ so that volume mounts\n# of /app/data (for user data persistence) do not hide the pre-baked encoding.\n# config.py reads TIKTOKEN_CACHE_DIR from the environment to pick up this path.\nENV TIKTOKEN_CACHE_DIR=/app/tiktoken-cache\nRUN mkdir -p /app/tiktoken-cache && \\\n .venv/bin/python -c \"import tiktoken; tiktoken.get_encoding('o200k_base')\"\n\n# Copy the rest of the application code\nCOPY . /app\n\n# Install frontend dependencies and build\nWORKDIR /app/frontend\nARG NPM_REGISTRY=https://registry.npmjs.org/\nCOPY frontend/package.json frontend/package-lock.json ./\nRUN npm config set registry ${NPM_REGISTRY}\nRUN npm ci\nCOPY frontend/ ./\nRUN npm run build\n\n# Return to app root\nWORKDIR /app\n\n# Runtime stage\nFROM python:3.12-slim-bookworm AS runtime\n\n# Install only runtime system dependencies (no build tools)\n# Add Node.js 20.x LTS for running frontend\nRUN apt-get update && apt-get upgrade -y && apt-get install -y --no-install-recommends \\\n ffmpeg \\\n supervisor \\\n curl \\\n && curl -fsSL https://deb.nodesource.com/setup_20.x | bash - \\\n && apt-get install -y nodejs \\\n && rm -rf /var/lib/apt/lists/*\n\n# Install uv using the official method\nCOPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/\n\n# Set the working directory in the container to /app\nWORKDIR /app\n\n# Copy the virtual environment from builder stage\nCOPY --from=builder /app/.venv /app/.venv\n\n# Copy the source code (the rest)\nCOPY . /app\n\n# Copy pre-downloaded tiktoken encoding from builder (outside /data/ — volume-mount safe)\nCOPY --from=builder /app/tiktoken-cache /app/tiktoken-cache\n\n# Ensure uv uses the existing venv without attempting network operations\nENV UV_NO_SYNC=1\nENV VIRTUAL_ENV=/app/.venv\n# Point the app at the pre-baked tiktoken encoding (see open_notebook/config.py)\nENV TIKTOKEN_CACHE_DIR=/app/tiktoken-cache\n\n# Bind Next.js to all interfaces (required for Docker networking and reverse proxies)\nENV HOSTNAME=0.0.0.0\n\n# Copy built frontend from builder stage\nCOPY --from=builder /app/frontend/.next/standalone /app/frontend/\nCOPY --from=builder /app/frontend/.next/static /app/frontend/.next/static\nCOPY --from=builder /app/frontend/public /app/frontend/public\nCOPY --from=builder /app/frontend/start-server.js /app/frontend/start-server.js\n\n# Expose ports for Frontend and API\nEXPOSE 8502 5055\n\nRUN mkdir -p /app/data\n\n# Copy and make executable the wait-for-api script\nCOPY scripts/wait-for-api.sh /app/scripts/wait-for-api.sh\nRUN chmod +x /app/scripts/wait-for-api.sh\n\n# Copy supervisord configuration\nCOPY supervisord.conf /etc/supervisor/conf.d/supervisord.conf\n\n# Create log directories\nRUN mkdir -p /var/log/supervisor\n\n# Runtime API URL Configuration\n# The API_URL environment variable can be set at container runtime to configure\n# where the frontend should connect to the API. This allows the same Docker image\n# to work in different deployment scenarios without rebuilding.\n#\n# If not set, the system will auto-detect based on incoming requests.\n# Set API_URL when using reverse proxies or custom domains.\n#\n# Example: docker run -e API_URL=https://your-domain.com/api ...\n\nCMD [\"/usr/bin/supervisord\", \"-c\", \"/etc/supervisor/conf.d/supervisord.conf\"]\n" }, { "path": "Dockerfile.single", "content": "# Stage 1: Frontend Builder\nFROM node:20-slim AS frontend-builder\nWORKDIR /app/frontend\n\n# Copy dependency files first to leverage cache\nCOPY frontend/package.json frontend/package-lock.json ./\nARG NPM_REGISTRY=https://registry.npmjs.org/\nRUN npm config set registry ${NPM_REGISTRY}\nRUN npm ci\n\n# Copy the rest of the frontend source\nCOPY frontend/ ./\n# Build the frontend\nRUN npm run build\n\n# Stage 2: SurrealDB binary (pinned to v2 to match docker-compose.yml)\nFROM surrealdb/surrealdb:v2 AS surreal-binary\n\n# Stage 4: Backend Builder\nFROM python:3.12-slim-bookworm AS backend-builder\n# Install build dependencies\nRUN apt-get update && apt-get upgrade -y && apt-get install -y --no-install-recommends build-essential && rm -rf /var/lib/apt/lists/*\n# Install uv\nCOPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/\nWORKDIR /app\n\n# Set build optimization environment variables\nENV UV_HTTP_TIMEOUT=120\n\n# Copy dependency files first\nCOPY pyproject.toml uv.lock ./\nCOPY open_notebook/__init__.py ./open_notebook/__init__.py\n# Install dependencies\nRUN uv sync --frozen --no-dev\n\n# Pre-download tiktoken encoding so the app works offline (issue #264).\n# /app/tiktoken-cache is intentionally outside /app/data/ so that volume mounts\n# of /app/data (for user data persistence) do not hide the pre-baked encoding.\n# config.py reads TIKTOKEN_CACHE_DIR from the environment to pick up this path.\nENV TIKTOKEN_CACHE_DIR=/app/tiktoken-cache\nRUN mkdir -p /app/tiktoken-cache && \\\n .venv/bin/python -c \"import tiktoken; tiktoken.get_encoding('o200k_base')\"\n\n# Stage 5: Runtime\nFROM python:3.12-slim-bookworm AS runtime\n\n# Install runtime dependencies\nRUN apt-get update && apt-get upgrade -y && apt-get install -y \\\n ffmpeg \\\n supervisor \\\n curl \\\n && curl -fsSL https://deb.nodesource.com/setup_20.x | bash - \\\n && apt-get install -y nodejs \\\n && rm -rf /var/lib/apt/lists/*\n\n# Install SurrealDB (copied from pinned v2 image to match docker-compose.yml)\nCOPY --from=surreal-binary /surreal /usr/local/bin/surreal\n\n# Install uv (optional but helpful for some scripts)\nCOPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/\n\nWORKDIR /app\n\n# Copy backend virtualenv and source code\nCOPY --from=backend-builder /app/.venv /app/.venv\nCOPY . /app/\n\n# Copy pre-downloaded tiktoken encoding from builder (outside /data/ — volume-mount safe)\nCOPY --from=backend-builder /app/tiktoken-cache /app/tiktoken-cache\n\n# Copy built frontend from standalone output\nCOPY --from=frontend-builder /app/frontend/.next/standalone /app/frontend/\nCOPY --from=frontend-builder /app/frontend/.next/static /app/frontend/.next/static\nCOPY --from=frontend-builder /app/frontend/public /app/frontend/public\n\n# Bind Next.js to all interfaces (required for Docker networking and reverse proxies)\nENV HOSTNAME=0.0.0.0\n# Point the app at the pre-baked tiktoken encoding (see open_notebook/config.py)\nENV TIKTOKEN_CACHE_DIR=/app/tiktoken-cache\n\n# Setup directories and permissions\nRUN mkdir -p /app/data /mydata\n\n# Ensure wait-for-api script is executable\nRUN chmod +x /app/scripts/wait-for-api.sh\n\n# Copy supervisord configuration\nCOPY supervisord.single.conf /etc/supervisor/conf.d/supervisord.conf\n\n# Create log directories\nRUN mkdir -p /var/log/supervisor\n\n# Expose ports\nEXPOSE 8502 5055\n\n# Set startup command\nCMD [\"/usr/bin/supervisord\", \"-c\", \"/etc/supervisor/conf.d/supervisord.conf\"]" }, { "path": "LICENSE", "content": "MIT License\nCopyright (c) 2024 Luis Novo\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the \"Software\"), to deal\nin the Software without restriction, including without limitation the rights\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\ncopies of the Software, and to permit persons to whom the Software is\nfurnished to do so, subject to the following conditions:\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\nSOFTWARE." }, { "path": "MAINTAINER_GUIDE.md", "content": "# Maintainer Guide\n\n**📍 This file has moved!**\n\nAll maintainer guidelines have been consolidated into the new development documentation structure.\n\n👉 **[Read the Maintainer Guide](docs/7-DEVELOPMENT/maintainer-guide.md)**\n\n---\n\n## Quick Links\n\n- **Maintainer Guide** → [docs/7-DEVELOPMENT/maintainer-guide.md](docs/7-DEVELOPMENT/maintainer-guide.md)\n- **Contributing Guide** → [docs/7-DEVELOPMENT/contributing.md](docs/7-DEVELOPMENT/contributing.md)\n- **Design Principles** → [docs/7-DEVELOPMENT/design-principles.md](docs/7-DEVELOPMENT/design-principles.md)\n\n---\n\nFor all maintainer details, see **[docs/7-DEVELOPMENT/maintainer-guide.md](docs/7-DEVELOPMENT/maintainer-guide.md)**.\n" }, { "path": "Makefile", "content": ".PHONY: run frontend check ruff database lint api start-all stop-all status clean-cache worker worker-start worker-stop worker-restart\n.PHONY: docker-buildx-prepare docker-buildx-clean docker-buildx-reset\n.PHONY: docker-push docker-push-latest docker-release docker-build-local tag export-docs\n\n# Get version from pyproject.toml\nVERSION := $(shell grep -m1 version pyproject.toml | cut -d'\"' -f2)\n\n# Image names for both registries\nDOCKERHUB_IMAGE := lfnovo/open_notebook\nGHCR_IMAGE := ghcr.io/lfnovo/open-notebook\n\n# Build platforms\nPLATFORMS := linux/amd64,linux/arm64\n\ndatabase:\n\tdocker compose up -d surrealdb\n\nrun:\n\t@echo \"⚠️ Warning: Starting frontend only. For full functionality, use 'make start-all'\"\n\tcd frontend && npm run dev\n\nfrontend:\n\tcd frontend && npm run dev\n\nlint:\n\tuv run python -m mypy .\n\nruff:\n\truff check . --fix\n\n# === Docker Build Setup ===\ndocker-buildx-prepare:\n\t@docker buildx inspect multi-platform-builder >/dev/null 2>&1 || \\\n\t\tdocker buildx create --use --name multi-platform-builder --driver docker-container\n\t@docker buildx use multi-platform-builder\n\ndocker-buildx-clean:\n\t@echo \"🧹 Cleaning up buildx builders...\"\n\t@docker buildx rm multi-platform-builder 2>/dev/null || true\n\t@docker ps -a | grep buildx_buildkit | awk '{print $$1}' | xargs -r docker rm -f 2>/dev/null || true\n\t@echo \"✅ Buildx cleanup complete!\"\n\ndocker-buildx-reset: docker-buildx-clean docker-buildx-prepare\n\t@echo \"✅ Buildx reset complete!\"\n\n# === Docker Build Targets ===\n\n# Build production image for local platform only (no push)\ndocker-build-local:\n\t@echo \"🔨 Building production image locally ($(shell uname -m))...\"\n\tdocker build \\\n\t\t-t $(DOCKERHUB_IMAGE):$(VERSION) \\\n\t\t-t $(DOCKERHUB_IMAGE):local \\\n\t\t.\n\t@echo \"✅ Built $(DOCKERHUB_IMAGE):$(VERSION) and $(DOCKERHUB_IMAGE):local\"\n\t@echo \"Run with: docker run -p 5055:5055 -p 3000:3000 $(DOCKERHUB_IMAGE):local\"\n\n# Build and push version tags ONLY (no latest) for both regular and single images\ndocker-push: docker-buildx-prepare\n\t@echo \"📤 Building and pushing version $(VERSION) to both registries...\"\n\t@echo \"🔨 Building regular image...\"\n\tdocker buildx build --pull \\\n\t\t--platform $(PLATFORMS) \\\n\t\t--progress=plain \\\n\t\t-t $(DOCKERHUB_IMAGE):$(VERSION) \\\n\t\t-t $(GHCR_IMAGE):$(VERSION) \\\n\t\t--push \\\n\t\t.\n\t@echo \"🔨 Building single-container image...\"\n\tdocker buildx build --pull \\\n\t\t--platform $(PLATFORMS) \\\n\t\t--progress=plain \\\n\t\t-f Dockerfile.single \\\n\t\t-t $(DOCKERHUB_IMAGE):$(VERSION)-single \\\n\t\t-t $(GHCR_IMAGE):$(VERSION)-single \\\n\t\t--push \\\n\t\t.\n\t@echo \"✅ Pushed version $(VERSION) to both registries (latest NOT updated)\"\n\t@echo \" 📦 Docker Hub:\"\n\t@echo \" - $(DOCKERHUB_IMAGE):$(VERSION)\"\n\t@echo \" - $(DOCKERHUB_IMAGE):$(VERSION)-single\"\n\t@echo \" 📦 GHCR:\"\n\t@echo \" - $(GHCR_IMAGE):$(VERSION)\"\n\t@echo \" - $(GHCR_IMAGE):$(VERSION)-single\"\n\n# Update v1-latest tags to current version (both regular and single images)\ndocker-push-latest: docker-buildx-prepare\n\t@echo \"📤 Updating v1-latest tags to version $(VERSION)...\"\n\t@echo \"🔨 Building regular image with latest tag...\"\n\tdocker buildx build --pull \\\n\t\t--platform $(PLATFORMS) \\\n\t\t--progress=plain \\\n\t\t-t $(DOCKERHUB_IMAGE):$(VERSION) \\\n\t\t-t $(DOCKERHUB_IMAGE):v1-latest \\\n\t\t-t $(GHCR_IMAGE):$(VERSION) \\\n\t\t-t $(GHCR_IMAGE):v1-latest \\\n\t\t--push \\\n\t\t.\n\t@echo \"🔨 Building single-container image with latest tag...\"\n\tdocker buildx build --pull \\\n\t\t--platform $(PLATFORMS) \\\n\t\t--progress=plain \\\n\t\t-f Dockerfile.single \\\n\t\t-t $(DOCKERHUB_IMAGE):$(VERSION)-single \\\n\t\t-t $(DOCKERHUB_IMAGE):v1-latest-single \\\n\t\t-t $(GHCR_IMAGE):$(VERSION)-single \\\n\t\t-t $(GHCR_IMAGE):v1-latest-single \\\n\t\t--push \\\n\t\t.\n\t@echo \"✅ Updated v1-latest to version $(VERSION)\"\n\t@echo \" 📦 Docker Hub:\"\n\t@echo \" - $(DOCKERHUB_IMAGE):$(VERSION) → v1-latest\"\n\t@echo \" - $(DOCKERHUB_IMAGE):$(VERSION)-single → v1-latest-single\"\n\t@echo \" 📦 GHCR:\"\n\t@echo \" - $(GHCR_IMAGE):$(VERSION) → v1-latest\"\n\t@echo \" - $(GHCR_IMAGE):$(VERSION)-single → v1-latest-single\"\n\n# Full release: push version AND update latest tags\ndocker-release: docker-push-latest\n\t@echo \"✅ Full release complete for version $(VERSION)\"\n\ntag:\n\t@version=$$(grep '^version = ' pyproject.toml | sed 's/version = \"\\(.*\\)\"/\\1/'); \\\n\techo \"Creating tag v$$version\"; \\\n\tgit tag \"v$$version\"; \\\n\tgit push origin \"v$$version\"\n\n\ndev:\n\tdocker compose -f docker-compose.dev.yml up --build \n\nfull:\n\tdocker compose -f docker-compose.full.yml up --build \n\n\napi:\n\tuv run --env-file .env run_api.py\n\n.PHONY: worker worker-start worker-stop worker-restart\n\nworker: worker-start\n\nworker-start:\n\t@echo \"Starting surreal-commands worker...\"\n\tuv run --env-file .env surreal-commands-worker --import-modules commands\n\nworker-stop:\n\t@echo \"Stopping surreal-commands worker...\"\n\tpkill -f \"surreal-commands-worker\" || true\n\nworker-restart: worker-stop\n\t@sleep 2\n\t@$(MAKE) worker-start\n\n# === Service Management ===\nstart-all:\n\t@echo \"🚀 Starting Open Notebook (Database + API + Worker + Frontend)...\"\n\t@echo \"📊 Starting SurrealDB...\"\n\t@docker compose -f docker-compose.dev.yml up -d surrealdb\n\t@sleep 3\n\t@echo \"🔧 Starting API backend...\"\n\t@uv run run_api.py &\n\t@sleep 3\n\t@echo \"⚙️ Starting background worker...\"\n\t@uv run --env-file .env surreal-commands-worker --import-modules commands &\n\t@sleep 2\n\t@echo \"🌐 Starting Next.js frontend...\"\n\t@echo \"✅ All services started!\"\n\t@echo \"📱 Frontend: http://localhost:3000\"\n\t@echo \"🔗 API: http://localhost:5055\"\n\t@echo \"📚 API Docs: http://localhost:5055/docs\"\n\tcd frontend && npm run dev\n\nstop-all:\n\t@echo \"🛑 Stopping all Open Notebook services...\"\n\t@pkill -f \"next dev\" || true\n\t@pkill -f \"surreal-commands-worker\" || true\n\t@pkill -f \"run_api.py\" || true\n\t@pkill -f \"uvicorn api.main:app\" || true\n\t@docker compose down\n\t@echo \"✅ All services stopped!\"\n\nstatus:\n\t@echo \"📊 Open Notebook Service Status:\"\n\t@echo \"Database (SurrealDB):\"\n\t@docker compose ps surrealdb 2>/dev/null || echo \" ❌ Not running\"\n\t@echo \"API Backend:\"\n\t@pgrep -f \"run_api.py\\|uvicorn api.main:app\" >/dev/null && echo \" ✅ Running\" || echo \" ❌ Not running\"\n\t@echo \"Background Worker:\"\n\t@pgrep -f \"surreal-commands-worker\" >/dev/null && echo \" ✅ Running\" || echo \" ❌ Not running\"\n\t@echo \"Next.js Frontend:\"\n\t@pgrep -f \"next dev\" >/dev/null && echo \" ✅ Running\" || echo \" ❌ Not running\"\n\n# === Documentation Export ===\nexport-docs:\n\t@echo \"📚 Exporting documentation...\"\n\t@uv run python scripts/export_docs.py\n\t@echo \"✅ Documentation export complete!\"\n\n# === Cleanup ===\nclean-cache:\n\t@echo \"🧹 Cleaning cache directories...\"\n\t@find . -name \"__pycache__\" -type d -exec rm -rf {} + 2>/dev/null || true\n\t@find . -name \".mypy_cache\" -type d -exec rm -rf {} + 2>/dev/null || true\n\t@find . -name \".ruff_cache\" -type d -exec rm -rf {} + 2>/dev/null || true\n\t@find . -name \".pytest_cache\" -type d -exec rm -rf {} + 2>/dev/null || true\n\t@find . -name \"*.pyc\" -type f -delete 2>/dev/null || true\n\t@find . -name \"*.pyo\" -type f -delete 2>/dev/null || true\n\t@find . -name \"*.pyd\" -type f -delete 2>/dev/null || true\n\t@echo \"✅ Cache directories cleaned!\"" }, { "path": "README.dev.md", "content": "# Developer Guide\n\nThis guide is for developers working on Open Notebook. For end-user documentation, see [README.md](README.md) and [docs/](docs/).\n\n## Quick Start for Development\n\n```bash\n# 1. Clone and setup\ngit clone https://github.com/lfnovo/open-notebook.git\ncd open-notebook\n\n# 2. Copy environment files\ncp .env.example .env\ncp .env.example docker.env\n\n# 3. Install dependencies\nuv sync\n\n# 4. Start all services (recommended for development)\nmake start-all\n```\n\n## Development Workflows\n\n### When to Use What?\n\n| Workflow | Use Case | Speed | Production Parity |\n|----------|----------|-------|-------------------|\n| **Local Services** (`make start-all`) | Day-to-day development, fastest iteration | ⚡⚡⚡ Fast | Medium |\n| **Docker Compose** (`make dev`) | Testing containerized setup | ⚡⚡ Medium | High |\n| **Local Docker Build** (`make docker-build-local`) | Testing Dockerfile changes | ⚡ Slow | Very High |\n| **Multi-platform Build** (`make docker-push`) | Publishing releases | 🐌 Very Slow | Exact |\n\n---\n\n## 1. Local Development (Recommended)\n\n**Best for:** Daily development, hot reload, debugging\n\n### Setup\n\n```bash\n# Start database\nmake database\n\n# Start all services (DB + API + Worker + Frontend)\nmake start-all\n```\n\n### What This Does\n\n1. Starts SurrealDB in Docker (port 8000)\n2. Starts FastAPI backend (port 5055)\n3. Starts background worker (surreal-commands)\n4. Starts Next.js frontend (port 3000)\n\n### Individual Services\n\n```bash\n# Just the database\nmake database\n\n# Just the API\nmake api\n\n# Just the frontend\nmake frontend\n\n# Just the worker\nmake worker\n```\n\n### Checking Status\n\n```bash\n# See what's running\nmake status\n\n# Stop everything\nmake stop-all\n```\n\n### Advantages\n- ✅ Fastest iteration (hot reload)\n- ✅ Easy debugging (direct process access)\n- ✅ Low resource usage\n- ✅ Direct log access\n\n### Disadvantages\n- ❌ Doesn't test Docker build\n- ❌ Environment may differ from production\n- ❌ Requires local Python/Node setup\n\n---\n\n## 2. Docker Compose Development\n\n**Best for:** Testing containerized setup, CI/CD verification\n\n```bash\n# Start with dev profile\nmake dev\n\n# Or full stack\nmake full\n```\n\n### Configuration Files\n\n- `docker-compose.dev.yml` - Development setup\n- `docker-compose.full.yml` - Full stack setup\n- `docker-compose.yml` - Base configuration\n\n### Advantages\n- ✅ Closer to production environment\n- ✅ Isolated dependencies\n- ✅ Easy to share exact environment\n\n### Disadvantages\n- ❌ Slower rebuilds\n- ❌ More complex debugging\n- ❌ Higher resource usage\n\n---\n\n## 3. Testing Production Docker Images\n\n**Best for:** Verifying Dockerfile changes before publishing\n\n### Build Locally\n\n```bash\n# Build production image for your platform only\nmake docker-build-local\n```\n\nThis creates two tags:\n- `lfnovo/open_notebook:` (from pyproject.toml)\n- `lfnovo/open_notebook:local`\n\n### Run Locally\n\n```bash\ndocker run -p 5055:5055 -p 3000:3000 lfnovo/open_notebook:local\n```\n\n### When to Use\n- ✅ Before pushing to registry\n- ✅ Testing Dockerfile changes\n- ✅ Debugging production-specific issues\n- ✅ Verifying build process\n\n---\n\n## 4. Publishing Docker Images\n\n### Workflow\n\n```bash\n# 1. Test locally first\nmake docker-build-local\n\n# 2. If successful, push version tag (no latest update)\nmake docker-push\n\n# 3. Test the pushed version in staging/production\n\n# 4. When ready, promote to latest\nmake docker-push-latest\n```\n\n### Available Commands\n\n| Command | What It Does | Updates Latest? |\n|---------|--------------|-----------------|\n| `make docker-build-local` | Build for current platform only | No registry push |\n| `make docker-push` | Push version tags to registries | ❌ No |\n| `make docker-push-latest` | Push version + update v1-latest | ✅ Yes |\n| `make docker-release` | Full release (same as docker-push-latest) | ✅ Yes |\n\n### Publishing Details\n\n- **Platforms:** `linux/amd64`, `linux/arm64`\n- **Registries:** Docker Hub + GitHub Container Registry\n- **Image Variants:** Regular + Single-container (`-single`)\n- **Version Source:** `pyproject.toml`\n\n### Creating Git Tags\n\n```bash\n# Create and push git tag matching pyproject.toml version\nmake tag\n```\n\n---\n\n## Code Quality\n\n```bash\n# Run linter with auto-fix\nmake ruff\n\n# Run type checking\nmake lint\n\n# Run tests\nuv run pytest tests/\n\n# Clean cache directories\nmake clean-cache\n```\n\n---\n\n## Common Development Tasks\n\n### Adding a New Feature\n\n1. Create feature branch\n2. Develop using `make start-all`\n3. Write tests\n4. Run `make ruff` and `make lint`\n5. Test with `make docker-build-local`\n6. Create PR\n\n### Fixing a Bug\n\n1. Reproduce locally with `make start-all`\n2. Add test case demonstrating bug\n3. Fix the bug\n4. Verify test passes\n5. Check with `make docker-build-local`\n\n### Updating Dependencies\n\n```bash\n# Add Python dependency\nuv add package-name\n\n# Update dependencies\nuv sync\n\n# Frontend dependencies\ncd frontend && npm install package-name\n```\n\n### Adding a New Language (i18n)\n\nOpen Notebook supports internationalization. To add a new language:\n\n1. **Create locale file**: Copy an existing locale as template\n ```bash\n cp frontend/src/lib/locales/en-US/index.ts frontend/src/lib/locales/pt-BR/index.ts\n ```\n\n2. **Translate all strings** in the new file. The structure includes:\n - `common`: Shared UI elements (buttons, labels)\n - `notebooks`, `sources`, `notes`: Feature-specific strings\n - `chat`, `search`, `podcasts`: Module-specific strings\n - `apiErrors`: Error message translations\n\n3. **Register the locale** in `frontend/src/lib/locales/index.ts`:\n ```typescript\n import { ptBR } from './pt-BR'\n\n export const locales = {\n 'en-US': enUS,\n 'zh-CN': zhCN,\n 'zh-TW': zhTW,\n 'pt-BR': ptBR, // Add your locale\n }\n ```\n\n4. **Add date-fns locale** in `frontend/src/lib/utils/date-locale.ts`:\n ```typescript\n import { zhCN, enUS, zhTW, ptBR } from 'date-fns/locale'\n\n const LOCALE_MAP: Record = {\n 'zh-CN': zhCN,\n 'zh-TW': zhTW,\n 'en-US': enUS,\n 'pt-BR': ptBR, // Add your locale\n }\n ```\n\n5. **Test**: Switch languages using the language toggle in the UI header.\n\n### Database Migrations\n\nDatabase migrations run **automatically** when the API starts.\n\n1. Create migration file: `migrations/XXX_description.surql`\n2. Write SurrealQL schema changes\n3. (Optional) Create rollback: `migrations/XXX_description_down.surql`\n4. Restart API - migration runs on startup\n\n---\n\n## Troubleshooting\n\n### Services Won't Start\n\n```bash\n# Check status\nmake status\n\n# Check database\ndocker compose ps surrealdb\n\n# View logs\ndocker compose logs surrealdb\n\n# Restart everything\nmake stop-all\nmake start-all\n```\n\n### Port Already in Use\n\n```bash\n# Find process using port\nlsof -i :5055\nlsof -i :3000\nlsof -i :8000\n\n# Kill stuck processes\nmake stop-all\n```\n\n### Database Connection Issues\n\n```bash\n# Verify SurrealDB is running\ndocker compose ps surrealdb\n\n# Check connection settings in .env\ncat .env | grep SURREAL\n```\n\n### Docker Build Fails\n\n```bash\n# Clean Docker cache\ndocker builder prune\n\n# Reset buildx\nmake docker-buildx-reset\n\n# Try local build first\nmake docker-build-local\n```\n\n---\n\n## Project Structure\n\n```\nopen-notebook/\n├── api/ # FastAPI backend\n├── frontend/ # Next.js React frontend\n├── open_notebook/ # Python core library\n│ ├── domain/ # Domain models\n│ ├── graphs/ # LangGraph workflows\n│ ├── ai/ # AI provider integration\n│ └── database/ # SurrealDB operations\n├── migrations/ # Database migrations\n├── tests/ # Test suite\n├── docs/ # User documentation\n└── Makefile # Development commands\n```\n\nSee component-specific CLAUDE.md files for detailed architecture:\n- [frontend/CLAUDE.md](frontend/CLAUDE.md)\n- [api/CLAUDE.md](api/CLAUDE.md)\n- [open_notebook/CLAUDE.md](open_notebook/CLAUDE.md)\n\n---\n\n## Environment Variables\n\n### Required for Local Development\n\n```bash\n# .env file\nSURREAL_URL=ws://localhost:8000\nSURREAL_USER=root\nSURREAL_PASS=root\nSURREAL_DB=open_notebook\nSURREAL_NS=production\n\n# AI Provider (at least one required)\nOPENAI_API_KEY=sk-...\n# OR\nANTHROPIC_API_KEY=sk-ant-...\n# OR configure other providers (see docs/5-CONFIGURATION/)\n```\n\nSee [docs/5-CONFIGURATION/](docs/5-CONFIGURATION/) for complete configuration guide.\n\n---\n\n## Performance Tips\n\n### Speed Up Local Development\n\n1. **Use `make start-all`** instead of Docker for daily work\n2. **Keep SurrealDB running** between sessions (`make database`)\n3. **Use `make docker-build-local`** only when testing Dockerfile changes\n4. **Skip multi-platform builds** until ready to publish\n\n### Reduce Resource Usage\n\n```bash\n# Stop unused services\nmake stop-all\n\n# Clean up Docker\ndocker system prune -a\n\n# Clean Python cache\nmake clean-cache\n```\n\n---\n\n## TODO: Sections to Add\n\n- [ ] Frontend development guide (hot reload, component structure)\n- [ ] API development guide (adding endpoints, services)\n- [ ] LangGraph workflow development\n- [ ] Testing strategy and coverage\n- [ ] Debugging tips (VSCode/PyCharm setup)\n- [ ] CI/CD pipeline overview\n- [ ] Release process checklist\n- [ ] Common error messages and solutions\n\n---\n\n## Resources\n\n- **Documentation:** https://open-notebook.ai\n- **Discord:** https://discord.gg/37XJPXfz2w\n- **Issues:** https://github.com/lfnovo/open-notebook/issues\n- **Contributing:** [CONTRIBUTING.md](CONTRIBUTING.md)\n- **Maintainer Guide:** [MAINTAINER_GUIDE.md](MAINTAINER_GUIDE.md)\n\n---\n\n**Last Updated:** January 2025\n" }, { "path": "README.md", "content": "\n\n\n[![Forks][forks-shield]][forks-url]\n[![Stargazers][stars-shield]][stars-url]\n[![Issues][issues-shield]][issues-url]\n[![MIT License][license-shield]][license-url]\n\n\n\n\n
\n
\n \n \"Logo\"\n \n\n

Open Notebook

\n\n

\n An open source, privacy-focused alternative to Google's Notebook LM!\n
Join our Discord server for help, to share workflow ideas, and suggest features!\n
\n Checkout our website »\n
\n
\n 📚 Get Started\n ·\n 📖 User Guide\n ·\n ✨ Features\n ·\n 🚀 Deploy\n

\n
\n\n

\n\"lfnovo%2Fopen-notebook\n

\n\n
\n \n Deutsch | \n Español | \n français | \n 日本語 | \n 한국어 | \n Português | \n Русский | \n 中文\n
\n\n## A private, multi-model, 100% local, full-featured alternative to Notebook LM\n\n![New Notebook](docs/assets/asset_list.png)\n\nIn a world dominated by Artificial Intelligence, having the ability to think 🧠 and acquire new knowledge 💡, is a skill that should not be a privilege for a few, nor restricted to a single provider.\n\n**Open Notebook empowers you to:**\n- 🔒 **Control your data** - Keep your research private and secure\n- 🤖 **Choose your AI models** - Support for 16+ providers including OpenAI, Anthropic, Ollama, LM Studio, and more\n- 📚 **Organize multi-modal content** - PDFs, videos, audio, web pages, and more\n- 🎙️ **Generate professional podcasts** - Advanced multi-speaker podcast generation\n- 🔍 **Search intelligently** - Full-text and vector search across all your content\n- 💬 **Chat with context** - AI conversations powered by your research\n- 🌐 **Multi-language UI** - English, Portuguese, Chinese (Simplified & Traditional), Japanese, Russian, and Bengali support\n\nLearn more about our project at [https://www.open-notebook.ai](https://www.open-notebook.ai)\n\n---\n\n## 🆚 Open Notebook vs Google Notebook LM\n\n| Feature | Open Notebook | Google Notebook LM | Advantage |\n|---------|---------------|--------------------|-----------|\n| **Privacy & Control** | Self-hosted, your data | Google cloud only | Complete data sovereignty |\n| **AI Provider Choice** | 16+ providers (OpenAI, Anthropic, Ollama, LM Studio, etc.) | Google models only | Flexibility and cost optimization |\n| **Podcast Speakers** | 1-4 speakers with custom profiles | 2 speakers only | Extreme flexibility |\n| **Content Transformations** | Custom and built-in | Limited options | Unlimited processing power |\n| **API Access** | Full REST API | No API | Complete automation |\n| **Deployment** | Docker, cloud, or local | Google hosted only | Deploy anywhere |\n| **Citations** | Basic references (will improve) | Comprehensive with sources | Research integrity |\n| **Customization** | Open source, fully customizable | Closed system | Unlimited extensibility |\n| **Cost** | Pay only for AI usage | Free tier + Monthly subscription | Transparent and controllable |\n\n**Why Choose Open Notebook?**\n- 🔒 **Privacy First**: Your sensitive research stays completely private\n- 💰 **Cost Control**: Choose cheaper AI providers or run locally with Ollama\n- 🎙️ **Better Podcasts**: Full script control and multi-speaker flexibility vs limited 2-speaker deep-dive format\n- 🔧 **Unlimited Customization**: Modify, extend, and integrate as needed\n- 🌐 **No Vendor Lock-in**: Switch providers, deploy anywhere, own your data\n\n### Built With\n\n[![Python][Python]][Python-url] [![Next.js][Next.js]][Next-url] [![React][React]][React-url] [![SurrealDB][SurrealDB]][SurrealDB-url] [![LangChain][LangChain]][LangChain-url]\n\n## 🚀 Quick Start (2 Minutes)\n\n### Prerequisites\n- [Docker Desktop](https://www.docker.com/products/docker-desktop/) installed\n- That's it! (API keys configured later in the UI)\n\n### Step 1: Get docker-compose.yml\n\n**Option A:** Download directly\n```bash\ncurl -o docker-compose.yml https://raw.githubusercontent.com/lfnovo/open-notebook/main/docker-compose.yml\n```\n\n**Option B:** Create the file manually\nCopy this into a new file called `docker-compose.yml`:\n\n```yaml\nservices:\n surrealdb:\n image: surrealdb/surrealdb:v2\n command: start --log info --user root --pass root rocksdb:/mydata/mydatabase.db\n user: root\n ports:\n - \"8000:8000\"\n volumes:\n - ./surreal_data:/mydata\n restart: always\n\n open_notebook:\n image: lfnovo/open_notebook:v1-latest\n ports:\n - \"8502:8502\"\n - \"5055:5055\"\n environment:\n - OPEN_NOTEBOOK_ENCRYPTION_KEY=change-me-to-a-secret-string\n - SURREAL_URL=ws://surrealdb:8000/rpc\n - SURREAL_USER=root\n - SURREAL_PASSWORD=root\n - SURREAL_NAMESPACE=open_notebook\n - SURREAL_DATABASE=open_notebook\n volumes:\n - ./notebook_data:/app/data\n depends_on:\n - surrealdb\n restart: always\n```\n\n### Step 2: Set Your Encryption Key\nEdit `docker-compose.yml` and change this line:\n```yaml\n- OPEN_NOTEBOOK_ENCRYPTION_KEY=change-me-to-a-secret-string\n```\nto any secret value (e.g., `my-super-secret-key-123`)\n\n### Step 3: Start Services\n```bash\ndocker compose up -d\n```\n\nWait 15-20 seconds, then open: **http://localhost:8502**\n\n### Step 4: Configure AI Provider\n1. Go to **Settings** → **API Keys**\n2. Click **Add Credential**\n3. Choose your provider (OpenAI, Anthropic, Google, etc.)\n4. Paste your API key and click **Save**\n5. Click **Test Connection** → **Discover Models** → **Register Models**\n\nDone! You're ready to create your first notebook.\n\n> **Need an API key?** Get one from:\n> [OpenAI](https://platform.openai.com/api-keys) · [Anthropic](https://console.anthropic.com/) · [Google](https://aistudio.google.com/) · [Groq](https://console.groq.com/) (free tier)\n\n> **Want free local AI?** See [examples/docker-compose-ollama.yml](examples/) for Ollama setup\n\n---\n\n### 📚 More Installation Options\n\n- **[With Ollama (Free Local AI)](examples/docker-compose-ollama.yml)** - Run models locally without API costs\n- **[From Source (Developers)](docs/1-INSTALLATION/from-source.md)** - For development and contributions\n- **[Complete Installation Guide](docs/1-INSTALLATION/index.md)** - All deployment scenarios\n\n---\n\n### 📖 Need Help?\n\n- **🤖 AI Installation Assistant**: [CustomGPT to help you install](https://chatgpt.com/g/g-68776e2765b48191bd1bae3f30212631-open-notebook-installation-assistant)\n- **🆘 Troubleshooting**: [5-minute troubleshooting guide](docs/6-TROUBLESHOOTING/quick-fixes.md)\n- **💬 Community Support**: [Discord Server](https://discord.gg/37XJPXfz2w)\n- **🐛 Report Issues**: [GitHub Issues](https://github.com/lfnovo/open-notebook/issues)\n\n---\n\n## Star History\n\n[![Star History Chart](https://api.star-history.com/svg?repos=lfnovo/open-notebook&type=date&legend=top-left)](https://www.star-history.com/#lfnovo/open-notebook&type=date&legend=top-left)\n\n\n## Provider Support Matrix\n\nThanks to the [Esperanto](https://github.com/lfnovo/esperanto) library, we support this providers out of the box!\n\n| Provider | LLM Support | Embedding Support | Speech-to-Text | Text-to-Speech |\n|--------------|-------------|------------------|----------------|----------------|\n| OpenAI | ✅ | ✅ | ✅ | ✅ |\n| Anthropic | ✅ | ❌ | ❌ | ❌ |\n| Groq | ✅ | ❌ | ✅ | ❌ |\n| Google (GenAI) | ✅ | ✅ | ❌ | ✅ |\n| Vertex AI | ✅ | ✅ | ❌ | ✅ |\n| Ollama | ✅ | ✅ | ❌ | ❌ |\n| Perplexity | ✅ | ❌ | ❌ | ❌ |\n| ElevenLabs | ❌ | ❌ | ✅ | ✅ |\n| Azure OpenAI | ✅ | ✅ | ❌ | ❌ |\n| Mistral | ✅ | ✅ | ❌ | ❌ |\n| DeepSeek | ✅ | ❌ | ❌ | ❌ |\n| Voyage | ❌ | ✅ | ❌ | ❌ |\n| xAI | ✅ | ❌ | ❌ | ❌ |\n| OpenRouter | ✅ | ❌ | ❌ | ❌ |\n| OpenAI Compatible* | ✅ | ❌ | ❌ | ❌ |\n\n*Supports LM Studio and any OpenAI-compatible endpoint\n\n## ✨ Key Features\n\n### Core Capabilities\n- **🔒 Privacy-First**: Your data stays under your control - no cloud dependencies\n- **🎯 Multi-Notebook Organization**: Manage multiple research projects seamlessly\n- **📚 Universal Content Support**: PDFs, videos, audio, web pages, Office docs, and more\n- **🤖 Multi-Model AI Support**: 16+ providers including OpenAI, Anthropic, Ollama, Google, LM Studio, and more\n- **🎙️ Professional Podcast Generation**: Advanced multi-speaker podcasts with Episode Profiles\n- **🔍 Intelligent Search**: Full-text and vector search across all your content\n- **💬 Context-Aware Chat**: AI conversations powered by your research materials\n- **📝 AI-Assisted Notes**: Generate insights or write notes manually\n\n### Advanced Features\n- **⚡ Reasoning Model Support**: Full support for thinking models like DeepSeek-R1 and Qwen3\n- **🔧 Content Transformations**: Powerful customizable actions to summarize and extract insights\n- **🌐 Comprehensive REST API**: Full programmatic access for custom integrations [![API Docs](https://img.shields.io/badge/API-Documentation-blue?style=flat-square)](http://localhost:5055/docs)\n- **🔐 Optional Password Protection**: Secure public deployments with authentication\n- **📊 Fine-Grained Context Control**: Choose exactly what to share with AI models\n- **📎 Citations**: Get answers with proper source citations\n\n\n## Podcast Feature\n\n[![Check out our podcast sample](https://img.youtube.com/vi/D-760MlGwaI/0.jpg)](https://www.youtube.com/watch?v=D-760MlGwaI)\n\n## 📚 Documentation\n\n### Getting Started\n- **[📖 Introduction](docs/0-START-HERE/index.md)** - Learn what Open Notebook offers\n- **[⚡ Quick Start](docs/0-START-HERE/quick-start.md)** - Get up and running in 5 minutes\n- **[🔧 Installation](docs/1-INSTALLATION/index.md)** - Comprehensive setup guide\n- **[🎯 Your First Notebook](docs/0-START-HERE/first-notebook.md)** - Step-by-step tutorial\n\n### User Guide\n- **[📱 Interface Overview](docs/3-USER-GUIDE/interface-overview.md)** - Understanding the layout\n- **[📚 Notebooks](docs/3-USER-GUIDE/notebooks.md)** - Organizing your research\n- **[📄 Sources](docs/3-USER-GUIDE/sources.md)** - Managing content types\n- **[📝 Notes](docs/3-USER-GUIDE/notes.md)** - Creating and managing notes\n- **[💬 Chat](docs/3-USER-GUIDE/chat.md)** - AI conversations\n- **[🔍 Search](docs/3-USER-GUIDE/search.md)** - Finding information\n\n### Advanced Topics\n- **[🎙️ Podcast Generation](docs/2-CORE-CONCEPTS/podcasts.md)** - Create professional podcasts\n- **[🔧 Content Transformations](docs/2-CORE-CONCEPTS/transformations.md)** - Customize content processing\n- **[🤖 AI Models](docs/4-AI-PROVIDERS/index.md)** - AI model configuration\n- **[🔌 MCP Integration](docs/5-CONFIGURATION/mcp-integration.md)** - Connect with Claude Desktop, VS Code and other MCP clients\n- **[🔧 REST API Reference](docs/7-DEVELOPMENT/api-reference.md)** - Complete API documentation\n- **[🔐 Security](docs/5-CONFIGURATION/security.md)** - Password protection and privacy\n- **[🚀 Deployment](docs/1-INSTALLATION/index.md)** - Complete deployment guides for all scenarios\n\n

(back to top)

\n\n## 🗺️ Roadmap\n\n### Upcoming Features\n- **Live Front-End Updates**: Real-time UI updates for smoother experience\n- **Async Processing**: Faster UI through asynchronous content processing\n- **Cross-Notebook Sources**: Reuse research materials across projects\n- **Bookmark Integration**: Connect with your favorite bookmarking apps\n\n### Recently Completed ✅\n- **Next.js Frontend**: Modern React-based frontend with improved performance\n- **Comprehensive REST API**: Full programmatic access to all functionality\n- **Multi-Model Support**: 16+ AI providers including OpenAI, Anthropic, Ollama, LM Studio\n- **Advanced Podcast Generator**: Professional multi-speaker podcasts with Episode Profiles\n- **Content Transformations**: Powerful customizable actions for content processing\n- **Enhanced Citations**: Improved layout and finer control for source citations\n- **Multiple Chat Sessions**: Manage different conversations within notebooks\n\nSee the [open issues](https://github.com/lfnovo/open-notebook/issues) for a full list of proposed features and known issues.\n\n

(back to top)

\n\n\n## 📖 Need Help?\n- **🤖 AI Installation Assistant**: We have a [CustomGPT built to help you install Open Notebook](https://chatgpt.com/g/g-68776e2765b48191bd1bae3f30212631-open-notebook-installation-assistant) - it will guide you through each step!\n- **New to Open Notebook?** Start with our [Getting Started Guide](docs/0-START-HERE/index.md)\n- **Need installation help?** Check our [Installation Guide](docs/1-INSTALLATION/index.md)\n- **Want to see it in action?** Try our [Quick Start Tutorial](docs/0-START-HERE/quick-start.md)\n\n## 🤝 Community & Contributing\n\n### Join the Community\n- 💬 **[Discord Server](https://discord.gg/37XJPXfz2w)** - Get help, share ideas, and connect with other users\n- 🐛 **[GitHub Issues](https://github.com/lfnovo/open-notebook/issues)** - Report bugs and request features\n- ⭐ **Star this repo** - Show your support and help others discover Open Notebook\n\n### Contributing\nWe welcome contributions! We're especially looking for help with:\n- **Frontend Development**: Help improve our modern Next.js/React UI\n- **Testing & Bug Fixes**: Make Open Notebook more robust\n- **Feature Development**: Build the coolest research tool together\n- **Documentation**: Improve guides and tutorials\n\n**Current Tech Stack**: Python, FastAPI, Next.js, React, SurrealDB\n**Future Roadmap**: Real-time updates, enhanced async processing\n\nSee our [Contributing Guide](CONTRIBUTING.md) for detailed information on how to get started.\n\n

(back to top)

\n\n\n## 📄 License\n\nOpen Notebook is MIT licensed. See the [LICENSE](LICENSE) file for details.\n\n\n**Community Support**:\n- 💬 [Discord Server](https://discord.gg/37XJPXfz2w) - Get help, share ideas, and connect with users\n- 🐛 [GitHub Issues](https://github.com/lfnovo/open-notebook/issues) - Report bugs and request features\n- 🌐 [Website](https://www.open-notebook.ai) - Learn more about the project\n\n

(back to top)

\n\n\n\n\n[contributors-shield]: https://img.shields.io/github/contributors/lfnovo/open-notebook.svg?style=for-the-badge\n[contributors-url]: https://github.com/lfnovo/open-notebook/graphs/contributors\n[forks-shield]: https://img.shields.io/github/forks/lfnovo/open-notebook.svg?style=for-the-badge\n[forks-url]: https://github.com/lfnovo/open-notebook/network/members\n[stars-shield]: https://img.shields.io/github/stars/lfnovo/open-notebook.svg?style=for-the-badge\n[stars-url]: https://github.com/lfnovo/open-notebook/stargazers\n[issues-shield]: https://img.shields.io/github/issues/lfnovo/open-notebook.svg?style=for-the-badge\n[issues-url]: https://github.com/lfnovo/open-notebook/issues\n[license-shield]: https://img.shields.io/github/license/lfnovo/open-notebook.svg?style=for-the-badge\n[license-url]: https://github.com/lfnovo/open-notebook/blob/master/LICENSE.txt\n[linkedin-shield]: https://img.shields.io/badge/-LinkedIn-black.svg?style=for-the-badge&logo=linkedin&colorB=555\n[linkedin-url]: https://linkedin.com/in/lfnovo\n[product-screenshot]: images/screenshot.png\n[Next.js]: https://img.shields.io/badge/Next.js-000000?style=for-the-badge&logo=next.js&logoColor=white\n[Next-url]: https://nextjs.org/\n[React]: https://img.shields.io/badge/React-61DAFB?style=for-the-badge&logo=react&logoColor=black\n[React-url]: https://reactjs.org/\n[Python]: https://img.shields.io/badge/Python-3776AB?style=for-the-badge&logo=python&logoColor=white\n[Python-url]: https://www.python.org/\n[LangChain]: https://img.shields.io/badge/LangChain-3A3A3A?style=for-the-badge&logo=chainlink&logoColor=white\n[LangChain-url]: https://www.langchain.com/\n[SurrealDB]: https://img.shields.io/badge/SurrealDB-FF5E00?style=for-the-badge&logo=databricks&logoColor=white\n[SurrealDB-url]: https://surrealdb.com/\n" }, { "path": "api/CLAUDE.md", "content": "# API Module\n\nFastAPI-based REST backend exposing services for notebooks, sources, notes, chat, podcasts, and AI model management.\n\n## Purpose\n\nFastAPI application serving three architectural layers: routes (HTTP endpoints), services (business logic), and models (request/response schemas). Integrates LangGraph workflows (chat, ask, source_chat), SurrealDB persistence, and AI providers via Esperanto.\n\n## Architecture Overview\n\n**Three layers**:\n1. **Routes** (`routers/*`): HTTP endpoints mapping to services\n2. **Services** (`*_service.py`): Business logic orchestrating domain models, database, graphs, AI providers\n3. **Models** (`models.py`): Pydantic request/response schemas with validation\n\n**Startup flow**:\n- Load .env environment variables\n- Initialize CORS middleware + password auth middleware\n- Run database migrations via AsyncMigrationManager on lifespan startup\n- Run podcast profile data migration (legacy string to model registry conversion)\n- Register all routers\n\n**Key services**:\n- `chat_service.py`: Invokes chat graph with messages, context\n- `podcast_service.py`: Orchestrates outline + transcript generation\n- `sources_service.py`: Content ingestion, vectorization, metadata\n- `notes_service.py`: Note creation, linking to sources/insights\n- `transformations_service.py`: Applies transformations to content\n- `models_service.py`: Manages AI provider/model configuration\n- `episode_profiles_service.py`: Manages podcast speaker/episode profiles\n\n## Component Catalog\n\n### Main Application\n- **main.py**: FastAPI app initialization, CORS setup, auth middleware, lifespan event, router registration\n- **Lifespan handler**: Runs AsyncMigrationManager on startup (database schema migration)\n- **Auth middleware**: PasswordAuthMiddleware protects endpoints (password-based access control)\n\n### Services (Business Logic)\n- **chat_service.py**: Invokes chat.py graph; handles message history via SqliteSaver\n- **podcast_service.py**: Generates outline (outline.jinja), then transcript (transcript.jinja) for episodes\n- **sources_service.py**: Ingests files/URLs (content_core), extracts text, vectorizes, saves to SurrealDB\n- **transformations_service.py**: Applies transformations via transformation.py graph\n- **models_service.py**: Manages ModelManager config (AI provider overrides)\n- **episode_profiles_service.py**: CRUD for EpisodeProfile and SpeakerProfile models\n- **insights_service.py**: Generates and retrieves source insights\n- **notes_service.py**: Creates notes linked to sources/insights\n\n### Models (Schemas)\n- **models.py**: Pydantic schemas for request/response validation\n- Request bodies: ChatRequest, CreateNoteRequest, PodcastGenerationRequest, etc.\n- Response bodies: ChatResponse, NoteResponse, PodcastResponse, etc.\n- Custom validators for enum fields, file paths, model references\n\n### Routers\n- **routers/chat.py**: POST /chat\n- **routers/source_chat.py**: POST /source/{source_id}/chat\n- **routers/podcasts.py**: POST /podcasts, GET /podcasts/{id}, POST /podcasts/episodes/{id}/retry, etc.\n- **routers/notes.py**: POST /notes, GET /notes/{id}\n- **routers/sources.py**: POST /sources, GET /sources/{id}, DELETE /sources/{id}\n- **routers/models.py**: GET /models, POST /models/config\n- **routers/credentials.py**: CRUD + test + discover + migrate for credential management\n- **routers/transformations.py**: POST /transformations\n- **routers/insights.py**: GET /sources/{source_id}/insights\n- **routers/auth.py**: POST /auth/password (password-based auth)\n- **routers/languages.py**: GET /languages (available podcast languages via pycountry+babel)\n- **routers/commands.py**: GET /commands/{command_id} (job status tracking)\n\n## Common Patterns\n\n- **Service injection via FastAPI**: Routers import services directly; no DI framework\n- **Async/await throughout**: All DB queries, graph invocations, AI calls are async\n- **SurrealDB transactions**: Services use repo_query, repo_create, repo_upsert from database layer\n- **Config override pattern**: Models/config override via models_service passed to graph.ainvoke(config=...)\n- **Error handling**: Custom exception hierarchy (`open_notebook.exceptions`) with global FastAPI exception handlers mapping to HTTP status codes (see Error Handling section below). LangGraph nodes use `classify_error()` to convert raw LLM provider errors into typed exceptions with user-friendly messages.\n- **Logging**: loguru logger in main.py; services expected to log key operations\n- **Response normalization**: All responses follow standard schema (data + metadata structure)\n\n## Key Dependencies\n\n- `fastapi`: FastAPI app, routers, HTTPException\n- `pydantic`: Validation models with Field, field_validator\n- `open_notebook.graphs`: chat, ask, source_chat, source, transformation graphs\n- `open_notebook.database`: SurrealDB repository functions (repo_query, repo_create, repo_upsert)\n- `open_notebook.domain`: Notebook, Source, Note, SourceInsight models\n- `open_notebook.ai.provision`: provision_langchain_model() factory\n- `ai_prompter`: Prompter for template rendering\n- `content_core`: extract_content() for file/URL processing\n- `esperanto`: AI provider client library (LLM, embeddings, TTS)\n- `surreal_commands`: Job queue for async operations (podcast generation)\n- `loguru`: Structured logging\n\n## Important Quirks & Gotchas\n\n- **Migration auto-run**: Database schema migrations run on every API startup (via lifespan); no manual migration steps\n- **PasswordAuthMiddleware is basic**: Uses simple password check; production deployments should replace with OAuth/JWT\n- **No request rate limiting**: No built-in rate limiting; deployment must add via proxy/middleware\n- **Service state is stateless**: Services don't cache results; each request re-queries database/AI models\n- **Graph invocation is blocking**: chat/podcast workflows may take minutes; no timeout handling in services\n- **Command job fire-and-forget**: podcast_service.py submits jobs but doesn't wait (async job queue pattern)\n- **Model override scoping**: Model config override via RunnableConfig is per-request only (not persistent)\n- **CORS open by default**: main.py CORS settings allow all origins (restrict before production)\n- **No OpenAPI security scheme**: API docs available without auth (disable before production)\n- **Services don't validate user permission**: All endpoints trust authentication layer; no per-notebook permission checks\n\n## Error Handling\n\n### Global Exception Handlers (`main.py`)\n\nFastAPI exception handlers map custom exception types from `open_notebook.exceptions` to HTTP status codes. All error responses include CORS headers.\n\n| Exception Class | HTTP Status | Use Case |\n|----------------|-------------|----------|\n| `NotFoundError` | 404 | Resource not found |\n| `InvalidInputError` | 400 | Bad request data |\n| `AuthenticationError` | 401 | Invalid/missing API key |\n| `RateLimitError` | 429 | Provider rate limit exceeded |\n| `ConfigurationError` | 422 | Wrong model name, missing config |\n| `NetworkError` | 502 | Cannot reach AI provider |\n| `ExternalServiceError` | 502 | Provider returned error (500/503, context length) |\n| `OpenNotebookError` (base) | 500 | Any other application error |\n\n### Error Classification (`open_notebook.utils.error_classifier`)\n\nThe `classify_error()` function maps raw exceptions from LLM providers/Esperanto/LangChain into the typed exceptions above with user-friendly messages. Used in all LangGraph graph nodes and SSE streaming handlers.\n\n**Flow**: Raw exception → keyword matching → `(ExceptionClass, user_message)` → raised → caught by global handler → HTTP response with descriptive message.\n\n### Frontend Integration\n\nThe frontend `getApiErrorMessage()` helper (`lib/utils/error-handler.ts`) tries i18n mapping first, then falls back to displaying the backend's descriptive error message directly.\n\n---\n\n## How to Add New Endpoint\n\n1. Create router file in `routers/` (e.g., `routers/new_feature.py`)\n2. Import router into `main.py` and register: `app.include_router(new_feature.router, tags=[\"new_feature\"])`\n3. Create service in `new_feature_service.py` with business logic\n4. Define request/response schemas in `models.py` (or create `new_feature_models.py`)\n5. Implement router functions calling service methods\n6. Test with `uv run uvicorn api.main:app --host 0.0.0.0 --port 5055`\n\n## Testing Patterns\n\n- **Interactive docs**: http://localhost:5055/docs (Swagger UI)\n- **Direct service tests**: Import service, call methods directly with test data\n- **Mock graphs**: Replace graph.ainvoke() with mock for testing service logic\n- **Database: Use test database** (separate SurrealDB instance or mock repo_query)\n\n---\n\n## Credential Management (API Configuration UI)\n\nThe Credential Management system enables users to configure AI provider credentials through the UI instead of environment variables. Keys are stored securely in SurrealDB (encrypted via Fernet) with database-first fallback to environment variables.\n\n### Router: `routers/credentials.py`\n\n**Endpoints**:\n\n| Method | Endpoint | Description |\n|--------|----------|-------------|\n| GET | `/credentials` | List all credentials (optional `?provider=` filter) |\n| GET | `/credentials/by-provider/{provider}` | List credentials for a provider |\n| POST | `/credentials` | Create a new credential |\n| GET | `/credentials/{credential_id}` | Get a specific credential |\n| PUT | `/credentials/{credential_id}` | Update a credential |\n| DELETE | `/credentials/{credential_id}` | Delete a credential |\n| POST | `/credentials/{credential_id}/test` | Test connection using credential |\n| POST | `/credentials/{credential_id}/discover` | Discover available models |\n| POST | `/credentials/{credential_id}/register-models` | Register discovered models |\n| POST | `/credentials/migrate-from-provider-config` | Migrate from legacy ProviderConfig |\n\n**Supported Providers** (13 total):\n- Simple API key: `openai`, `anthropic`, `google`, `groq`, `mistral`, `deepseek`, `xai`, `openrouter`, `voyage`, `elevenlabs`\n- URL-based: `ollama`\n- Multi-field: `azure`, `vertex`, `openai_compatible`\n\n**Security Features**:\n- NEVER returns actual API key values (only metadata)\n- URL validation (SSRF protection) on all URL fields via `_validate_url()`\n- Allows private IPs and localhost for self-hosted services (Ollama, LM Studio)\n- Requires `OPEN_NOTEBOOK_ENCRYPTION_KEY` to be set for storing credentials\n\n### Domain Model: `Credential` (`open_notebook/domain/credential.py`)\n\nIndividual credential records replacing the old `ProviderConfig` singleton. Each credential stores:\n- Provider name, display name, modalities\n- Encrypted API key (via Fernet)\n- Provider-specific config (base_url, endpoint, api_version, etc.)\n\n### Integration with Key Provider (`open_notebook/ai/key_provider.py`)\n\nThe `key_provider` module provisions DB-stored credentials into environment variables for Esperanto compatibility:\n\n**Database-first Pattern**:\n1. API endpoint saves keys to `Credential` records (encrypted in SurrealDB)\n2. Before model provisioning, `provision_provider_keys(provider)` checks DB, then env vars\n3. Keys from DB are set as environment variables for Esperanto compatibility\n4. Existing env vars remain unchanged if no DB config exists\n\n**Key Functions**:\n- `get_api_key(provider)`: Get API key (DB first, env fallback)\n- `provision_provider_keys(provider)`: Set env vars from DB for a provider\n- `provision_all_keys()`: Load all provider keys from DB into env vars\n\n### Authentication\n\nNo changes to authentication. The `credentials` router uses the same `PasswordAuthMiddleware` as all other endpoints. Keys are protected by the same password-based auth.\n\n**Auth Flow** (unchanged from `api/auth.py`):\n- `PasswordAuthMiddleware`: Global middleware checking `Authorization: Bearer {password}` header\n- Default password: `open-notebook-change-me` (set `OPEN_NOTEBOOK_PASSWORD` in production)\n- Docker secrets support via `OPEN_NOTEBOOK_PASSWORD_FILE`\n\n### Connection Testing (`open_notebook/ai/connection_tester.py`)\n\nThe `/credentials/{credential_id}/test` endpoint uses minimal API calls to verify credentials:\n- Loads Credential via `Credential.get(config_id)`, uses `credential.to_esperanto_config()`\n- Uses cheapest/smallest models per provider (TEST_MODELS map)\n- Returns success status and descriptive message\n- Special handlers for ollama, openai_compatible, and azure providers\n\n### Migration Workflows\n\nTwo migration endpoints help users transition to the credential system:\n\n**From environment variables** (`POST /credentials/migrate-from-env`):\n1. Checks each provider for env var presence\n2. Creates Credential records from env var values\n3. Returns summary: migrated, skipped, errors\n\n**From legacy ProviderConfig** (`POST /credentials/migrate-from-provider-config`):\n1. Reads old ProviderConfig records from database\n2. Converts each to individual Credential records\n3. Returns summary: migrated, skipped, errors\n\n### Example Usage\n\n```python\n# Check status\nGET /credentials/status\n# Response: {\"configured\": {\"openai\": true, \"anthropic\": false}, \"source\": {\"openai\": \"database\", \"anthropic\": \"none\"}, \"encryption_configured\": true}\n\n# Create credential\nPOST /credentials\n{\"name\": \"My OpenAI Key\", \"provider\": \"openai\", \"modalities\": [\"language\", \"embedding\"], \"api_key\": \"sk-proj-...\"}\n\n# Test connection\nPOST /credentials/{credential_id}/test\n# Response: {\"provider\": \"openai\", \"success\": true, \"message\": \"Connection successful\"}\n\n# Discover models\nPOST /credentials/{credential_id}/discover\n# Response: {\"provider\": \"openai\", \"models\": [{\"model_id\": \"gpt-4\", \"name\": \"gpt-4\", ...}], \"credential_id\": \"...\"}\n\n# Migrate from env\nPOST /credentials/migrate-from-env\n# Response: {\"message\": \"Migration complete. Migrated 3 providers.\", \"migrated\": [\"openai\", \"anthropic\", \"groq\"], \"skipped\": [], \"errors\": []}\n```\n" }, { "path": "api/__init__.py", "content": "" }, { "path": "api/auth.py", "content": "from typing import Optional\n\nfrom fastapi import Depends, HTTPException, Request\nfrom fastapi.security import HTTPAuthorizationCredentials, HTTPBearer\nfrom loguru import logger\nfrom starlette.middleware.base import BaseHTTPMiddleware\nfrom starlette.responses import JSONResponse\n\nfrom open_notebook.utils.encryption import get_secret_from_env\n\n\nclass PasswordAuthMiddleware(BaseHTTPMiddleware):\n \"\"\"\n Middleware to check password authentication for all API requests.\n Always active with default password if OPEN_NOTEBOOK_PASSWORD is not set.\n Supports Docker secrets via OPEN_NOTEBOOK_PASSWORD_FILE.\n \"\"\"\n\n def __init__(self, app, excluded_paths: Optional[list] = None):\n super().__init__(app)\n self.password = get_secret_from_env(\"OPEN_NOTEBOOK_PASSWORD\")\n self.excluded_paths = excluded_paths or [\n \"/\",\n \"/health\",\n \"/docs\",\n \"/openapi.json\",\n \"/redoc\",\n ]\n\n async def dispatch(self, request: Request, call_next):\n # Skip authentication if no password is set\n if not self.password:\n return await call_next(request)\n\n # Skip authentication for excluded paths\n if request.url.path in self.excluded_paths:\n return await call_next(request)\n\n # Skip authentication for CORS preflight requests (OPTIONS)\n if request.method == \"OPTIONS\":\n return await call_next(request)\n\n # Check authorization header\n auth_header = request.headers.get(\"Authorization\")\n\n if not auth_header:\n return JSONResponse(\n status_code=401,\n content={\"detail\": \"Missing authorization header\"},\n headers={\"WWW-Authenticate\": \"Bearer\"},\n )\n\n # Expected format: \"Bearer {password}\"\n try:\n scheme, credentials = auth_header.split(\" \", 1)\n if scheme.lower() != \"bearer\":\n raise ValueError(\"Invalid authentication scheme\")\n except ValueError:\n return JSONResponse(\n status_code=401,\n content={\"detail\": \"Invalid authorization header format\"},\n headers={\"WWW-Authenticate\": \"Bearer\"},\n )\n\n # Check password\n if credentials != self.password:\n return JSONResponse(\n status_code=401,\n content={\"detail\": \"Invalid password\"},\n headers={\"WWW-Authenticate\": \"Bearer\"},\n )\n\n # Password is correct, proceed with the request\n response = await call_next(request)\n return response\n\n\n# Optional: HTTPBearer security scheme for OpenAPI documentation\nsecurity = HTTPBearer(auto_error=False)\n\n\ndef check_api_password(\n credentials: Optional[HTTPAuthorizationCredentials] = Depends(security),\n) -> bool:\n \"\"\"\n Utility function to check API password.\n Can be used as a dependency in individual routes if needed.\n Supports Docker secrets via OPEN_NOTEBOOK_PASSWORD_FILE.\n Returns True without checking credentials if OPEN_NOTEBOOK_PASSWORD is not configured.\n Raises 401 if credentials are missing or don't match the configured password.\n \"\"\"\n password = get_secret_from_env(\"OPEN_NOTEBOOK_PASSWORD\")\n\n # No password configured - skip authentication\n if not password:\n return True\n\n # No credentials provided\n if not credentials:\n raise HTTPException(\n status_code=401,\n detail=\"Missing authorization\",\n headers={\"WWW-Authenticate\": \"Bearer\"},\n )\n\n # Check password\n if credentials.credentials != password:\n raise HTTPException(\n status_code=401,\n detail=\"Invalid password\",\n headers={\"WWW-Authenticate\": \"Bearer\"},\n )\n\n return True\n" }, { "path": "api/chat_service.py", "content": "\"\"\"\nChat service for API operations.\nProvides async interface for chat functionality.\n\"\"\"\n\nimport os\nfrom typing import Any, Dict, List, Optional\n\nimport httpx\nfrom loguru import logger\n\n\nclass ChatService:\n \"\"\"Service for chat-related API operations\"\"\"\n\n def __init__(self):\n self.base_url = os.getenv(\"API_BASE_URL\", \"http://127.0.0.1:5055\")\n # Add authentication header if password is set\n self.headers = {}\n password = os.getenv(\"OPEN_NOTEBOOK_PASSWORD\")\n if password:\n self.headers[\"Authorization\"] = f\"Bearer {password}\"\n\n async def get_sessions(self, notebook_id: str) -> List[Dict[str, Any]]:\n \"\"\"Get all chat sessions for a notebook\"\"\"\n try:\n async with httpx.AsyncClient() as client:\n response = await client.get(\n f\"{self.base_url}/api/chat/sessions\",\n params={\"notebook_id\": notebook_id},\n headers=self.headers,\n )\n response.raise_for_status()\n return response.json()\n except Exception as e:\n logger.error(f\"Error fetching chat sessions: {str(e)}\")\n raise\n\n async def create_session(\n self,\n notebook_id: str,\n title: Optional[str] = None,\n model_override: Optional[str] = None,\n ) -> Dict[str, Any]:\n \"\"\"Create a new chat session\"\"\"\n try:\n data: Dict[str, Any] = {\"notebook_id\": notebook_id}\n if title is not None:\n data[\"title\"] = title\n if model_override is not None:\n data[\"model_override\"] = model_override\n\n async with httpx.AsyncClient() as client:\n response = await client.post(\n f\"{self.base_url}/api/chat/sessions\",\n json=data,\n headers=self.headers,\n )\n response.raise_for_status()\n return response.json()\n except Exception as e:\n logger.error(f\"Error creating chat session: {str(e)}\")\n raise\n\n async def get_session(self, session_id: str) -> Dict[str, Any]:\n \"\"\"Get a specific session with messages\"\"\"\n try:\n async with httpx.AsyncClient() as client:\n response = await client.get(\n f\"{self.base_url}/api/chat/sessions/{session_id}\",\n headers=self.headers,\n )\n response.raise_for_status()\n return response.json()\n except Exception as e:\n logger.error(f\"Error fetching session: {str(e)}\")\n raise\n\n async def update_session(\n self,\n session_id: str,\n title: Optional[str] = None,\n model_override: Optional[str] = None,\n ) -> Dict[str, Any]:\n \"\"\"Update session properties\"\"\"\n try:\n data: Dict[str, Any] = {}\n if title is not None:\n data[\"title\"] = title\n if model_override is not None:\n data[\"model_override\"] = model_override\n\n if not data:\n raise ValueError(\n \"At least one field must be provided to update a session\"\n )\n\n async with httpx.AsyncClient() as client:\n response = await client.put(\n f\"{self.base_url}/api/chat/sessions/{session_id}\",\n json=data,\n headers=self.headers,\n )\n response.raise_for_status()\n return response.json()\n except Exception as e:\n logger.error(f\"Error updating session: {str(e)}\")\n raise\n\n async def delete_session(self, session_id: str) -> Dict[str, Any]:\n \"\"\"Delete a chat session\"\"\"\n try:\n async with httpx.AsyncClient() as client:\n response = await client.delete(\n f\"{self.base_url}/api/chat/sessions/{session_id}\",\n headers=self.headers,\n )\n response.raise_for_status()\n return response.json()\n except Exception as e:\n logger.error(f\"Error deleting session: {str(e)}\")\n raise\n\n async def execute_chat(\n self,\n session_id: str,\n message: str,\n context: Dict[str, Any],\n model_override: Optional[str] = None,\n ) -> Dict[str, Any]:\n \"\"\"Execute a chat request\"\"\"\n try:\n data = {\"session_id\": session_id, \"message\": message, \"context\": context}\n if model_override is not None:\n data[\"model_override\"] = model_override\n\n # Short connect timeout (10s), long read timeout (10 min) for Ollama/local LLMs\n timeout = httpx.Timeout(connect=10.0, read=600.0, write=30.0, pool=10.0)\n async with httpx.AsyncClient(timeout=timeout) as client:\n response = await client.post(\n f\"{self.base_url}/api/chat/execute\", json=data, headers=self.headers\n )\n response.raise_for_status()\n return response.json()\n except Exception as e:\n logger.error(f\"Error executing chat: {str(e)}\")\n raise\n\n async def build_context(\n self, notebook_id: str, context_config: Dict[str, Any]\n ) -> Dict[str, Any]:\n \"\"\"Build context for a notebook\"\"\"\n try:\n data = {\"notebook_id\": notebook_id, \"context_config\": context_config}\n\n async with httpx.AsyncClient() as client:\n response = await client.post(\n f\"{self.base_url}/api/chat/context\", json=data, headers=self.headers\n )\n response.raise_for_status()\n return response.json()\n except Exception as e:\n logger.error(f\"Error building context: {str(e)}\")\n raise\n\n\n# Global instance\nchat_service = ChatService()\n" }, { "path": "api/client.py", "content": "\"\"\"\nAPI client for Open Notebook API.\nThis module provides a client interface to interact with the Open Notebook API.\n\"\"\"\n\nimport os\nfrom typing import Any, Dict, List, Optional, Union\n\nimport httpx\nfrom loguru import logger\n\n\nclass APIClient:\n \"\"\"Client for Open Notebook API.\"\"\"\n\n def __init__(self, base_url: Optional[str] = None):\n self.base_url = base_url or os.getenv(\"API_BASE_URL\", \"http://127.0.0.1:5055\")\n # Timeout increased to 5 minutes (300s) to accommodate slow LLM operations\n # (transformations, insights) on slower hardware (Ollama, LM Studio, remote APIs)\n # Configurable via API_CLIENT_TIMEOUT environment variable (in seconds)\n timeout_str = os.getenv(\"API_CLIENT_TIMEOUT\", \"300.0\")\n try:\n timeout_value = float(timeout_str)\n # Validate timeout is within reasonable bounds (30s - 3600s / 1 hour)\n if timeout_value < 30:\n logger.warning(\n f\"API_CLIENT_TIMEOUT={timeout_value}s is too low, using minimum of 30s\"\n )\n timeout_value = 30.0\n elif timeout_value > 3600:\n logger.warning(\n f\"API_CLIENT_TIMEOUT={timeout_value}s is too high, using maximum of 3600s\"\n )\n timeout_value = 3600.0\n self.timeout = timeout_value\n except ValueError:\n logger.error(\n f\"Invalid API_CLIENT_TIMEOUT value '{timeout_str}', using default 300s\"\n )\n self.timeout = 300.0\n\n # Add authentication header if password is set\n self.headers = {}\n password = os.getenv(\"OPEN_NOTEBOOK_PASSWORD\")\n if password:\n self.headers[\"Authorization\"] = f\"Bearer {password}\"\n\n def _make_request(\n self, method: str, endpoint: str, timeout: Optional[float] = None, **kwargs\n ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:\n \"\"\"Make HTTP request to the API.\"\"\"\n url = f\"{self.base_url}{endpoint}\"\n request_timeout = timeout if timeout is not None else self.timeout\n\n # Merge headers\n headers = kwargs.get(\"headers\", {})\n headers.update(self.headers)\n kwargs[\"headers\"] = headers\n\n try:\n with httpx.Client(timeout=request_timeout) as client:\n response = client.request(method, url, **kwargs)\n response.raise_for_status()\n return response.json()\n except httpx.RequestError as e:\n logger.error(f\"Request error for {method} {url}: {str(e)}\")\n raise ConnectionError(f\"Failed to connect to API: {str(e)}\")\n except httpx.HTTPStatusError as e:\n logger.error(\n f\"HTTP error {e.response.status_code} for {method} {url}: {e.response.text}\"\n )\n raise RuntimeError(\n f\"API request failed: {e.response.status_code} - {e.response.text}\"\n )\n except Exception as e:\n logger.error(f\"Unexpected error for {method} {url}: {str(e)}\")\n raise\n\n # Notebooks API methods\n def get_notebooks(\n self, archived: Optional[bool] = None, order_by: str = \"updated desc\"\n ) -> List[Dict[Any, Any]]:\n \"\"\"Get all notebooks.\"\"\"\n params: Dict[str, Any] = {\"order_by\": order_by}\n if archived is not None:\n params[\"archived\"] = str(archived).lower()\n\n result = self._make_request(\"GET\", \"/api/notebooks\", params=params)\n return result if isinstance(result, list) else [result]\n\n def create_notebook(\n self, name: str, description: str = \"\"\n ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:\n \"\"\"Create a new notebook.\"\"\"\n data = {\"name\": name, \"description\": description}\n return self._make_request(\"POST\", \"/api/notebooks\", json=data)\n\n def get_notebook(\n self, notebook_id: str\n ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:\n \"\"\"Get a specific notebook.\"\"\"\n return self._make_request(\"GET\", f\"/api/notebooks/{notebook_id}\")\n\n def update_notebook(\n self, notebook_id: str, **updates\n ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:\n \"\"\"Update a notebook.\"\"\"\n return self._make_request(\"PUT\", f\"/api/notebooks/{notebook_id}\", json=updates)\n\n def delete_notebook(\n self, notebook_id: str\n ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:\n \"\"\"Delete a notebook.\"\"\"\n return self._make_request(\"DELETE\", f\"/api/notebooks/{notebook_id}\")\n\n # Search API methods\n def search(\n self,\n query: str,\n search_type: str = \"text\",\n limit: int = 100,\n search_sources: bool = True,\n search_notes: bool = True,\n minimum_score: float = 0.2,\n ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:\n \"\"\"Search the knowledge base.\"\"\"\n data = {\n \"query\": query,\n \"type\": search_type,\n \"limit\": limit,\n \"search_sources\": search_sources,\n \"search_notes\": search_notes,\n \"minimum_score\": minimum_score,\n }\n return self._make_request(\"POST\", \"/api/search\", json=data)\n\n def ask_simple(\n self,\n question: str,\n strategy_model: str,\n answer_model: str,\n final_answer_model: str,\n ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:\n \"\"\"Ask the knowledge base a question (simple, non-streaming).\"\"\"\n data = {\n \"question\": question,\n \"strategy_model\": strategy_model,\n \"answer_model\": answer_model,\n \"final_answer_model\": final_answer_model,\n }\n # Use configured timeout for long-running ask operations\n return self._make_request(\n \"POST\", \"/api/search/ask/simple\", json=data, timeout=self.timeout\n )\n\n # Models API methods\n def get_models(self, model_type: Optional[str] = None) -> List[Dict[Any, Any]]:\n \"\"\"Get all models with optional type filtering.\"\"\"\n params = {}\n if model_type:\n params[\"type\"] = model_type\n result = self._make_request(\"GET\", \"/api/models\", params=params)\n return result if isinstance(result, list) else [result]\n\n def create_model(\n self, name: str, provider: str, model_type: str\n ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:\n \"\"\"Create a new model.\"\"\"\n data = {\n \"name\": name,\n \"provider\": provider,\n \"type\": model_type,\n }\n return self._make_request(\"POST\", \"/api/models\", json=data)\n\n def delete_model(\n self, model_id: str\n ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:\n \"\"\"Delete a model.\"\"\"\n return self._make_request(\"DELETE\", f\"/api/models/{model_id}\")\n\n def get_default_models(self) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:\n \"\"\"Get default model assignments.\"\"\"\n return self._make_request(\"GET\", \"/api/models/defaults\")\n\n def update_default_models(\n self, **defaults\n ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:\n \"\"\"Update default model assignments.\"\"\"\n return self._make_request(\"PUT\", \"/api/models/defaults\", json=defaults)\n\n # Transformations API methods\n def get_transformations(self) -> List[Dict[Any, Any]]:\n \"\"\"Get all transformations.\"\"\"\n result = self._make_request(\"GET\", \"/api/transformations\")\n return result if isinstance(result, list) else [result]\n\n def create_transformation(\n self,\n name: str,\n title: str,\n description: str,\n prompt: str,\n apply_default: bool = False,\n ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:\n \"\"\"Create a new transformation.\"\"\"\n data = {\n \"name\": name,\n \"title\": title,\n \"description\": description,\n \"prompt\": prompt,\n \"apply_default\": apply_default,\n }\n return self._make_request(\"POST\", \"/api/transformations\", json=data)\n\n def get_transformation(\n self, transformation_id: str\n ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:\n \"\"\"Get a specific transformation.\"\"\"\n return self._make_request(\"GET\", f\"/api/transformations/{transformation_id}\")\n\n def update_transformation(\n self, transformation_id: str, **updates\n ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:\n \"\"\"Update a transformation.\"\"\"\n return self._make_request(\n \"PUT\", f\"/api/transformations/{transformation_id}\", json=updates\n )\n\n def delete_transformation(\n self, transformation_id: str\n ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:\n \"\"\"Delete a transformation.\"\"\"\n return self._make_request(\"DELETE\", f\"/api/transformations/{transformation_id}\")\n\n def execute_transformation(\n self, transformation_id: str, input_text: str, model_id: str\n ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:\n \"\"\"Execute a transformation on input text.\"\"\"\n data = {\n \"transformation_id\": transformation_id,\n \"input_text\": input_text,\n \"model_id\": model_id,\n }\n # Use configured timeout for transformation operations\n return self._make_request(\n \"POST\", \"/api/transformations/execute\", json=data, timeout=self.timeout\n )\n\n # Notes API methods\n def get_notes(self, notebook_id: Optional[str] = None) -> List[Dict[Any, Any]]:\n \"\"\"Get all notes with optional notebook filtering.\"\"\"\n params = {}\n if notebook_id:\n params[\"notebook_id\"] = notebook_id\n result = self._make_request(\"GET\", \"/api/notes\", params=params)\n return result if isinstance(result, list) else [result]\n\n def create_note(\n self,\n content: str,\n title: Optional[str] = None,\n note_type: str = \"human\",\n notebook_id: Optional[str] = None,\n ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:\n \"\"\"Create a new note.\"\"\"\n data = {\n \"content\": content,\n \"note_type\": note_type,\n }\n if title:\n data[\"title\"] = title\n if notebook_id:\n data[\"notebook_id\"] = notebook_id\n return self._make_request(\"POST\", \"/api/notes\", json=data)\n\n def get_note(self, note_id: str) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:\n \"\"\"Get a specific note.\"\"\"\n return self._make_request(\"GET\", f\"/api/notes/{note_id}\")\n\n def update_note(\n self, note_id: str, **updates\n ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:\n \"\"\"Update a note.\"\"\"\n return self._make_request(\"PUT\", f\"/api/notes/{note_id}\", json=updates)\n\n def delete_note(self, note_id: str) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:\n \"\"\"Delete a note.\"\"\"\n return self._make_request(\"DELETE\", f\"/api/notes/{note_id}\")\n\n # Embedding API methods\n def embed_content(\n self, item_id: str, item_type: str, async_processing: bool = False\n ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:\n \"\"\"Embed content for vector search.\"\"\"\n data = {\n \"item_id\": item_id,\n \"item_type\": item_type,\n \"async_processing\": async_processing,\n }\n # Use configured timeout for embedding operations\n return self._make_request(\"POST\", \"/api/embed\", json=data, timeout=self.timeout)\n\n def rebuild_embeddings(\n self,\n mode: str = \"existing\",\n include_sources: bool = True,\n include_notes: bool = True,\n include_insights: bool = True,\n ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:\n \"\"\"Rebuild embeddings in bulk.\n\n Note: This operation can take a long time for large databases.\n Consider increasing API_CLIENT_TIMEOUT to 600-900s for bulk rebuilds.\n \"\"\"\n data = {\n \"mode\": mode,\n \"include_sources\": include_sources,\n \"include_notes\": include_notes,\n \"include_insights\": include_insights,\n }\n # Use double the configured timeout for bulk rebuild operations (or configured value if already high)\n rebuild_timeout = max(self.timeout, min(self.timeout * 2, 3600.0))\n return self._make_request(\n \"POST\", \"/api/embeddings/rebuild\", json=data, timeout=rebuild_timeout\n )\n\n def get_rebuild_status(\n self, command_id: str\n ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:\n \"\"\"Get status of a rebuild operation.\"\"\"\n return self._make_request(\"GET\", f\"/api/embeddings/rebuild/{command_id}/status\")\n\n # Settings API methods\n def get_settings(self) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:\n \"\"\"Get all application settings.\"\"\"\n return self._make_request(\"GET\", \"/api/settings\")\n\n def update_settings(\n self, **settings\n ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:\n \"\"\"Update application settings.\"\"\"\n return self._make_request(\"PUT\", \"/api/settings\", json=settings)\n\n # Context API methods\n def get_notebook_context(\n self, notebook_id: str, context_config: Optional[Dict] = None\n ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:\n \"\"\"Get context for a notebook.\"\"\"\n data: Dict[str, Any] = {\"notebook_id\": notebook_id}\n if context_config:\n data[\"context_config\"] = context_config\n result = self._make_request(\n \"POST\", f\"/api/notebooks/{notebook_id}/context\", json=data\n )\n return result if isinstance(result, dict) else {}\n\n # Sources API methods\n def get_sources(self, notebook_id: Optional[str] = None) -> List[Dict[Any, Any]]:\n \"\"\"Get all sources with optional notebook filtering.\"\"\"\n params = {}\n if notebook_id:\n params[\"notebook_id\"] = notebook_id\n result = self._make_request(\"GET\", \"/api/sources\", params=params)\n return result if isinstance(result, list) else [result]\n\n def create_source(\n self,\n notebook_id: Optional[str] = None,\n notebooks: Optional[List[str]] = None,\n source_type: str = \"text\",\n url: Optional[str] = None,\n file_path: Optional[str] = None,\n content: Optional[str] = None,\n title: Optional[str] = None,\n transformations: Optional[List[str]] = None,\n embed: bool = False,\n delete_source: bool = False,\n async_processing: bool = False,\n ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:\n \"\"\"Create a new source.\"\"\"\n data = {\n \"type\": source_type,\n \"embed\": embed,\n \"delete_source\": delete_source,\n \"async_processing\": async_processing,\n }\n\n # Handle backward compatibility for notebook_id vs notebooks\n if notebooks:\n data[\"notebooks\"] = notebooks\n elif notebook_id:\n data[\"notebook_id\"] = notebook_id\n else:\n raise ValueError(\"Either notebook_id or notebooks must be provided\")\n\n if url:\n data[\"url\"] = url\n if file_path:\n data[\"file_path\"] = file_path\n if content:\n data[\"content\"] = content\n if title:\n data[\"title\"] = title\n if transformations:\n data[\"transformations\"] = transformations\n\n # Use configured timeout for source creation (especially PDF processing with OCR)\n return self._make_request(\n \"POST\", \"/api/sources/json\", json=data, timeout=self.timeout\n )\n\n def get_source(self, source_id: str) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:\n \"\"\"Get a specific source.\"\"\"\n return self._make_request(\"GET\", f\"/api/sources/{source_id}\")\n\n def get_source_status(\n self, source_id: str\n ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:\n \"\"\"Get processing status for a source.\"\"\"\n return self._make_request(\"GET\", f\"/api/sources/{source_id}/status\")\n\n def update_source(\n self, source_id: str, **updates\n ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:\n \"\"\"Update a source.\"\"\"\n return self._make_request(\"PUT\", f\"/api/sources/{source_id}\", json=updates)\n\n def delete_source(\n self, source_id: str\n ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:\n \"\"\"Delete a source.\"\"\"\n return self._make_request(\"DELETE\", f\"/api/sources/{source_id}\")\n\n # Insights API methods\n def get_source_insights(self, source_id: str) -> List[Dict[Any, Any]]:\n \"\"\"Get all insights for a specific source.\"\"\"\n result = self._make_request(\"GET\", f\"/api/sources/{source_id}/insights\")\n return result if isinstance(result, list) else [result]\n\n def get_insight(\n self, insight_id: str\n ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:\n \"\"\"Get a specific insight.\"\"\"\n return self._make_request(\"GET\", f\"/api/insights/{insight_id}\")\n\n def delete_insight(\n self, insight_id: str\n ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:\n \"\"\"Delete a specific insight.\"\"\"\n return self._make_request(\"DELETE\", f\"/api/insights/{insight_id}\")\n\n def save_insight_as_note(\n self, insight_id: str, notebook_id: Optional[str] = None\n ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:\n \"\"\"Convert an insight to a note.\"\"\"\n data = {}\n if notebook_id:\n data[\"notebook_id\"] = notebook_id\n return self._make_request(\n \"POST\", f\"/api/insights/{insight_id}/save-as-note\", json=data\n )\n\n def create_source_insight(\n self, source_id: str, transformation_id: str, model_id: Optional[str] = None\n ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:\n \"\"\"Create a new insight for a source by running a transformation.\"\"\"\n data = {\"transformation_id\": transformation_id}\n if model_id:\n data[\"model_id\"] = model_id\n return self._make_request(\n \"POST\", f\"/api/sources/{source_id}/insights\", json=data\n )\n\n # Episode Profiles API methods\n def get_episode_profiles(self) -> List[Dict[Any, Any]]:\n \"\"\"Get all episode profiles.\"\"\"\n result = self._make_request(\"GET\", \"/api/episode-profiles\")\n return result if isinstance(result, list) else [result]\n\n def get_episode_profile(\n self, profile_name: str\n ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:\n \"\"\"Get a specific episode profile by name.\"\"\"\n return self._make_request(\"GET\", f\"/api/episode-profiles/{profile_name}\")\n\n def create_episode_profile(\n self,\n name: str,\n description: str = \"\",\n speaker_config: str = \"\",\n outline_provider: str = \"\",\n outline_model: str = \"\",\n transcript_provider: str = \"\",\n transcript_model: str = \"\",\n default_briefing: str = \"\",\n num_segments: int = 5,\n ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:\n \"\"\"Create a new episode profile.\"\"\"\n data = {\n \"name\": name,\n \"description\": description,\n \"speaker_config\": speaker_config,\n \"outline_provider\": outline_provider,\n \"outline_model\": outline_model,\n \"transcript_provider\": transcript_provider,\n \"transcript_model\": transcript_model,\n \"default_briefing\": default_briefing,\n \"num_segments\": num_segments,\n }\n return self._make_request(\"POST\", \"/api/episode-profiles\", json=data)\n\n def update_episode_profile(\n self, profile_id: str, **updates\n ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:\n \"\"\"Update an episode profile.\"\"\"\n return self._make_request(\n \"PUT\", f\"/api/episode-profiles/{profile_id}\", json=updates\n )\n\n def delete_episode_profile(\n self, profile_id: str\n ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:\n \"\"\"Delete an episode profile.\"\"\"\n return self._make_request(\"DELETE\", f\"/api/episode-profiles/{profile_id}\")\n\n\n# Global client instance\napi_client = APIClient()\n" }, { "path": "api/command_service.py", "content": "from typing import Any, Dict, List, Optional\n\nfrom loguru import logger\nfrom surreal_commands import get_command_status, submit_command\n\n\nclass CommandService:\n \"\"\"Generic service layer for command operations\"\"\"\n\n @staticmethod\n async def submit_command_job(\n module_name: str, # Actually app_name for surreal-commands\n command_name: str,\n command_args: Dict[str, Any],\n context: Optional[Dict[str, Any]] = None,\n ) -> str:\n \"\"\"Submit a generic command job for background processing\"\"\"\n try:\n # Ensure command modules are imported before submitting\n # This is needed because submit_command validates against local registry\n try:\n import commands.podcast_commands # noqa: F401\n except ImportError as import_err:\n logger.error(f\"Failed to import command modules: {import_err}\")\n raise ValueError(\"Command modules not available\")\n\n # surreal-commands expects: submit_command(app_name, command_name, args)\n cmd_id = submit_command(\n module_name, # This is actually the app name (e.g., \"open_notebook\")\n command_name, # Command name (e.g., \"process_text\")\n command_args, # Input data\n )\n # Convert RecordID to string if needed\n if not cmd_id:\n raise ValueError(\"Failed to get cmd_id from submit_command\")\n cmd_id_str = str(cmd_id)\n logger.info(\n f\"Submitted command job: {cmd_id_str} for {module_name}.{command_name}\"\n )\n return cmd_id_str\n\n except Exception as e:\n logger.error(f\"Failed to submit command job: {e}\")\n raise\n\n @staticmethod\n async def get_command_status(job_id: str) -> Dict[str, Any]:\n \"\"\"Get status of any command job\"\"\"\n try:\n status = await get_command_status(job_id)\n return {\n \"job_id\": job_id,\n \"status\": status.status if status else \"unknown\",\n \"result\": status.result if status else None,\n \"error_message\": getattr(status, \"error_message\", None)\n if status\n else None,\n \"created\": str(status.created)\n if status and hasattr(status, \"created\") and status.created\n else None,\n \"updated\": str(status.updated)\n if status and hasattr(status, \"updated\") and status.updated\n else None,\n \"progress\": getattr(status, \"progress\", None) if status else None,\n }\n except Exception as e:\n logger.error(f\"Failed to get command status: {e}\")\n raise\n\n @staticmethod\n async def list_command_jobs(\n module_filter: Optional[str] = None,\n command_filter: Optional[str] = None,\n status_filter: Optional[str] = None,\n limit: int = 50,\n ) -> List[Dict[str, Any]]:\n \"\"\"List command jobs with optional filtering\"\"\"\n # This will be implemented with proper SurrealDB queries\n # For now, return empty list as this is foundation phase\n return []\n\n @staticmethod\n async def cancel_command_job(job_id: str) -> bool:\n \"\"\"Cancel a running command job\"\"\"\n try:\n # Implementation depends on surreal-commands cancellation support\n # For now, just log the attempt\n logger.info(f\"Attempting to cancel job: {job_id}\")\n return True\n except Exception as e:\n logger.error(f\"Failed to cancel command job: {e}\")\n raise\n" }, { "path": "api/context_service.py", "content": "\"\"\"\nContext service layer using API.\n\"\"\"\n\nfrom typing import Any, Dict, List, Optional, Union\n\nfrom loguru import logger\n\nfrom api.client import api_client\n\n\nclass ContextService:\n \"\"\"Service layer for context operations using API.\"\"\"\n\n def __init__(self):\n logger.info(\"Using API for context operations\")\n\n def get_notebook_context(\n self, notebook_id: str, context_config: Optional[Dict] = None\n ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:\n \"\"\"Get context for a notebook.\"\"\"\n result = api_client.get_notebook_context(\n notebook_id=notebook_id, context_config=context_config\n )\n return result\n\n\n# Global service instance\ncontext_service = ContextService()\n" }, { "path": "api/credentials_service.py", "content": "\"\"\"\nCredentials Service\n\nBusiness logic for managing AI provider credentials.\nExtracted from the credentials router to follow the service layer pattern.\n\nAll functions raise ValueError for business errors (router converts to HTTPException).\n\"\"\"\n\nimport ipaddress\nimport os\nimport socket\nfrom typing import Dict, List, Optional\nfrom urllib.parse import urlparse\n\nimport httpx\nfrom loguru import logger\nfrom pydantic import SecretStr\n\nfrom api.models import CredentialResponse\nfrom open_notebook.domain.credential import Credential\nfrom open_notebook.utils.encryption import get_secret_from_env\n\n# =============================================================================\n# Constants\n# =============================================================================\n\n# Provider environment variable configuration.\n# - \"required\": ALL listed env vars must be set for the provider to be considered configured.\n# - \"required_any\": at least ONE of the listed env vars must be set.\n# - \"optional\": additional env vars used during migration but not required.\nPROVIDER_ENV_CONFIG: Dict[str, dict] = {\n \"openai\": {\"required\": [\"OPENAI_API_KEY\"]},\n \"anthropic\": {\"required\": [\"ANTHROPIC_API_KEY\"]},\n \"google\": {\"required_any\": [\"GOOGLE_API_KEY\", \"GEMINI_API_KEY\"]},\n \"groq\": {\"required\": [\"GROQ_API_KEY\"]},\n \"mistral\": {\"required\": [\"MISTRAL_API_KEY\"]},\n \"deepseek\": {\"required\": [\"DEEPSEEK_API_KEY\"]},\n \"xai\": {\"required\": [\"XAI_API_KEY\"]},\n \"openrouter\": {\"required\": [\"OPENROUTER_API_KEY\"]},\n \"voyage\": {\"required\": [\"VOYAGE_API_KEY\"]},\n \"elevenlabs\": {\"required\": [\"ELEVENLABS_API_KEY\"]},\n \"ollama\": {\"required\": [\"OLLAMA_API_BASE\"]},\n \"vertex\": {\n \"required\": [\"VERTEX_PROJECT\", \"VERTEX_LOCATION\"],\n \"optional\": [\"GOOGLE_APPLICATION_CREDENTIALS\"],\n },\n \"azure\": {\n \"required\": [\"AZURE_OPENAI_API_KEY\", \"AZURE_OPENAI_ENDPOINT\", \"AZURE_OPENAI_API_VERSION\"],\n \"optional\": [\n \"AZURE_OPENAI_ENDPOINT_LLM\",\n \"AZURE_OPENAI_ENDPOINT_EMBEDDING\",\n \"AZURE_OPENAI_ENDPOINT_STT\",\n \"AZURE_OPENAI_ENDPOINT_TTS\",\n ],\n },\n \"openai_compatible\": {\n \"required_any\": [\"OPENAI_COMPATIBLE_BASE_URL\", \"OPENAI_COMPATIBLE_API_KEY\"],\n },\n}\n\nPROVIDER_MODALITIES: Dict[str, List[str]] = {\n \"openai\": [\"language\", \"embedding\", \"speech_to_text\", \"text_to_speech\"],\n \"anthropic\": [\"language\"],\n \"google\": [\"language\", \"embedding\"],\n \"groq\": [\"language\", \"speech_to_text\"],\n \"mistral\": [\"language\", \"embedding\"],\n \"deepseek\": [\"language\"],\n \"xai\": [\"language\"],\n \"openrouter\": [\"language\"],\n \"voyage\": [\"embedding\"],\n \"elevenlabs\": [\"text_to_speech\"],\n \"ollama\": [\"language\", \"embedding\"],\n \"vertex\": [\"language\", \"embedding\"],\n \"azure\": [\"language\", \"embedding\", \"speech_to_text\", \"text_to_speech\"],\n \"openai_compatible\": [\"language\", \"embedding\", \"speech_to_text\", \"text_to_speech\"],\n}\n\n\n# =============================================================================\n# URL Validation (SSRF protection)\n# =============================================================================\n\n\ndef validate_url(url: str, provider: str) -> None:\n \"\"\"\n Validate URL format for API endpoints.\n\n This is a self-hosted application, so we allow:\n - Private IPs (10.x, 172.16-31.x, 192.168.x) for self-hosted services\n - Localhost for local services (Ollama, LM Studio, etc.)\n\n We only block:\n - Invalid schemes (must be http or https)\n - Malformed URLs\n - Link-local addresses (169.254.x.x) - used for cloud metadata endpoints\n - Hostnames that resolve to link-local addresses\n\n Args:\n url: The URL to validate\n provider: The provider name (for logging/context)\n\n Raises:\n ValueError: If the URL is invalid\n \"\"\"\n if not url or not url.strip():\n return # Empty URLs handled elsewhere\n\n try:\n parsed = urlparse(url.strip())\n\n # Validate scheme - only http/https allowed\n if parsed.scheme not in (\"http\", \"https\"):\n raise ValueError(\n f\"Invalid URL scheme: '{parsed.scheme}'. Only http and https are allowed.\"\n )\n\n # Extract hostname\n hostname = parsed.hostname\n if not hostname:\n raise ValueError(\"Invalid URL: hostname could not be determined.\")\n\n # Try to parse as IP address to check for dangerous addresses\n try:\n ip = ipaddress.ip_address(hostname)\n\n # Block link-local addresses (169.254.x.x) - used for cloud metadata\n # These are dangerous as they can expose cloud instance credentials\n if ip.is_link_local:\n raise ValueError(\n \"Link-local addresses (169.254.x.x) are not allowed for security reasons. \"\n \"These addresses are used for cloud metadata endpoints.\"\n )\n\n # Block IPv4-mapped IPv6 addresses pointing to link-local\n # e.g. ::ffff:169.254.169.254 bypasses IPv6 is_link_local check\n if hasattr(ip, \"ipv4_mapped\") and ip.ipv4_mapped and ip.ipv4_mapped.is_link_local:\n raise ValueError(\n \"Link-local addresses (169.254.x.x) are not allowed for security reasons. \"\n \"These addresses are used for cloud metadata endpoints.\"\n )\n\n except ValueError as ve:\n # Re-raise our own ValueErrors\n if \"Link-local\" in str(ve) or \"Invalid URL\" in str(ve):\n raise\n # Not an IP address, it's a hostname - need to resolve and check\n try:\n # Resolve hostname to IP address\n resolved_ips = socket.getaddrinfo(hostname, None)\n for family, _, _, _, sockaddr in resolved_ips:\n ip_addr = sockaddr[0]\n try:\n parsed_ip = ipaddress.ip_address(ip_addr)\n if parsed_ip.is_link_local:\n raise ValueError(\n f\"Hostname '{hostname}' resolves to a link-local address (169.254.x.x) which is not allowed for security reasons. \"\n \"These addresses are used for cloud metadata endpoints.\"\n )\n # Block IPv4-mapped IPv6 addresses pointing to link-local\n if (\n hasattr(parsed_ip, \"ipv4_mapped\")\n and parsed_ip.ipv4_mapped\n and parsed_ip.ipv4_mapped.is_link_local\n ):\n raise ValueError(\n f\"Hostname '{hostname}' resolves to a link-local address (169.254.x.x) which is not allowed for security reasons. \"\n \"These addresses are used for cloud metadata endpoints.\"\n )\n except ValueError as inner_ve:\n if \"link-local\" in str(inner_ve).lower() or \"Link-local\" in str(inner_ve):\n raise\n # Skip non-IP addresses (e.g., IPv6 zones)\n continue\n except socket.gaierror:\n # Could not resolve hostname - allow it since the URL may be\n # valid in the deployment environment (e.g., Azure endpoints,\n # internal DNS names). We only block link-local addresses.\n pass\n\n except ValueError:\n raise\n except Exception:\n raise ValueError(\"Invalid URL format. Check server logs for details.\")\n\n\n# =============================================================================\n# Helpers\n# =============================================================================\n\n\ndef require_encryption_key() -> None:\n \"\"\"Raise ValueError if encryption key is not configured.\"\"\"\n if not get_secret_from_env(\"OPEN_NOTEBOOK_ENCRYPTION_KEY\"):\n raise ValueError(\n \"Encryption key not configured. \"\n \"Set OPEN_NOTEBOOK_ENCRYPTION_KEY to enable storing API keys.\"\n )\n\n\ndef credential_to_response(cred: Credential, model_count: int = 0) -> CredentialResponse:\n \"\"\"Convert a Credential domain object to API response.\"\"\"\n return CredentialResponse(\n id=cred.id or \"\",\n name=cred.name,\n provider=cred.provider,\n modalities=cred.modalities,\n base_url=cred.base_url,\n endpoint=cred.endpoint,\n api_version=cred.api_version,\n endpoint_llm=cred.endpoint_llm,\n endpoint_embedding=cred.endpoint_embedding,\n endpoint_stt=cred.endpoint_stt,\n endpoint_tts=cred.endpoint_tts,\n project=cred.project,\n location=cred.location,\n credentials_path=cred.credentials_path,\n has_api_key=cred.api_key is not None,\n created=str(cred.created) if cred.created else \"\",\n updated=str(cred.updated) if cred.updated else \"\",\n model_count=model_count,\n )\n\n\ndef check_env_configured(provider: str) -> bool:\n \"\"\"Check if a provider has sufficient env vars configured for migration.\"\"\"\n config = PROVIDER_ENV_CONFIG.get(provider)\n if not config:\n return False\n\n if \"required_any\" in config:\n return any(bool(os.environ.get(v, \"\").strip()) for v in config[\"required_any\"])\n elif \"required\" in config:\n return all(bool(os.environ.get(v, \"\").strip()) for v in config[\"required\"])\n return False\n\n\ndef get_default_modalities(provider: str) -> List[str]:\n \"\"\"Get default modalities for a provider.\"\"\"\n return PROVIDER_MODALITIES.get(provider.lower(), [\"language\"])\n\n\ndef create_credential_from_env(provider: str) -> Credential:\n \"\"\"Create a Credential from environment variables for a given provider.\"\"\"\n modalities = get_default_modalities(provider)\n name = \"Default (Migrated from env)\"\n\n if provider == \"ollama\":\n return Credential(\n name=name,\n provider=provider,\n modalities=modalities,\n base_url=os.environ.get(\"OLLAMA_API_BASE\"),\n )\n elif provider == \"vertex\":\n return Credential(\n name=name,\n provider=provider,\n modalities=modalities,\n project=os.environ.get(\"VERTEX_PROJECT\"),\n location=os.environ.get(\"VERTEX_LOCATION\"),\n credentials_path=os.environ.get(\"GOOGLE_APPLICATION_CREDENTIALS\"),\n )\n elif provider == \"azure\":\n return Credential(\n name=name,\n provider=provider,\n modalities=modalities,\n api_key=SecretStr(os.environ[\"AZURE_OPENAI_API_KEY\"]),\n endpoint=os.environ.get(\"AZURE_OPENAI_ENDPOINT\"),\n api_version=os.environ.get(\"AZURE_OPENAI_API_VERSION\"),\n endpoint_llm=os.environ.get(\"AZURE_OPENAI_ENDPOINT_LLM\"),\n endpoint_embedding=os.environ.get(\"AZURE_OPENAI_ENDPOINT_EMBEDDING\"),\n endpoint_stt=os.environ.get(\"AZURE_OPENAI_ENDPOINT_STT\"),\n endpoint_tts=os.environ.get(\"AZURE_OPENAI_ENDPOINT_TTS\"),\n )\n elif provider == \"openai_compatible\":\n api_key = os.environ.get(\"OPENAI_COMPATIBLE_API_KEY\")\n return Credential(\n name=name,\n provider=provider,\n modalities=modalities,\n api_key=SecretStr(api_key) if api_key else None,\n base_url=os.environ.get(\"OPENAI_COMPATIBLE_BASE_URL\"),\n )\n elif provider == \"google\":\n # Support both GOOGLE_API_KEY and GEMINI_API_KEY (fallback)\n api_key = os.environ.get(\"GOOGLE_API_KEY\") or os.environ.get(\"GEMINI_API_KEY\")\n return Credential(\n name=name,\n provider=provider,\n modalities=modalities,\n api_key=SecretStr(api_key) if api_key else None,\n )\n else:\n # Simple API key providers\n config = PROVIDER_ENV_CONFIG.get(provider, {})\n required = config.get(\"required\", [])\n env_var = required[0] if required else None\n api_key = os.environ.get(env_var) if env_var else None\n return Credential(\n name=name,\n provider=provider,\n modalities=modalities,\n api_key=SecretStr(api_key) if api_key else None,\n )\n\n\n# =============================================================================\n# Service Functions\n# =============================================================================\n\n\nasync def get_provider_status() -> dict:\n \"\"\"\n Get configuration status: encryption key status, and per-provider\n configured/source information.\n \"\"\"\n encryption_configured = bool(get_secret_from_env(\"OPEN_NOTEBOOK_ENCRYPTION_KEY\"))\n\n configured: Dict[str, bool] = {}\n source: Dict[str, str] = {}\n\n for provider in PROVIDER_ENV_CONFIG:\n env_configured = check_env_configured(provider)\n try:\n db_credentials = await Credential.get_by_provider(provider)\n db_configured = len(db_credentials) > 0\n except Exception:\n db_configured = False\n\n configured[provider] = db_configured or env_configured\n\n if db_configured:\n source[provider] = \"database\"\n elif env_configured:\n source[provider] = \"environment\"\n else:\n source[provider] = \"none\"\n\n return {\n \"configured\": configured,\n \"source\": source,\n \"encryption_configured\": encryption_configured,\n }\n\n\nasync def get_env_status() -> Dict[str, bool]:\n \"\"\"Check what's configured via environment variables.\"\"\"\n env_status: Dict[str, bool] = {}\n for provider in PROVIDER_ENV_CONFIG:\n env_status[provider] = check_env_configured(provider)\n return env_status\n\n\nasync def test_credential(credential_id: str) -> dict:\n \"\"\"\n Test connection using a credential's configuration.\n\n Returns dict with provider, success, message keys.\n \"\"\"\n provider = \"unknown\"\n try:\n cred = await Credential.get(credential_id)\n config = cred.to_esperanto_config()\n\n from open_notebook.ai.connection_tester import (\n _test_azure_connection,\n _test_ollama_connection,\n _test_openai_compatible_connection,\n )\n\n provider = cred.provider.lower()\n\n # Handle special providers\n if provider == \"ollama\":\n base_url = config.get(\"base_url\", \"http://localhost:11434\")\n success, message = await _test_ollama_connection(base_url)\n return {\"provider\": provider, \"success\": success, \"message\": message}\n\n if provider == \"openai_compatible\":\n base_url = config.get(\"base_url\")\n api_key = config.get(\"api_key\")\n if not base_url:\n return {\n \"provider\": provider,\n \"success\": False,\n \"message\": \"No base URL configured\",\n }\n success, message = await _test_openai_compatible_connection(\n base_url, api_key\n )\n return {\"provider\": provider, \"success\": success, \"message\": message}\n\n if provider == \"azure\":\n success, message = await _test_azure_connection(\n endpoint=config.get(\"endpoint\"),\n api_key=config.get(\"api_key\"),\n api_version=config.get(\"api_version\"),\n )\n return {\"provider\": provider, \"success\": success, \"message\": message}\n\n # Standard provider: use Esperanto to create and test\n from esperanto.factory import AIFactory\n\n from open_notebook.ai.connection_tester import TEST_MODELS\n\n if provider not in TEST_MODELS:\n return {\n \"provider\": provider,\n \"success\": False,\n \"message\": f\"Unknown provider: {provider}\",\n }\n\n test_model, test_type = TEST_MODELS[provider]\n if not test_model:\n return {\n \"provider\": provider,\n \"success\": False,\n \"message\": f\"No test model configured for {provider}\",\n }\n\n if test_type == \"language\":\n model = AIFactory.create_language(\n model_name=test_model, provider=provider, config=config\n )\n lc_model = model.to_langchain()\n await lc_model.ainvoke(\"Hi\")\n return {\"provider\": provider, \"success\": True, \"message\": \"Connection successful\"}\n\n elif test_type == \"embedding\":\n model = AIFactory.create_embedding(\n model_name=test_model, provider=provider, config=config\n )\n await model.aembed([\"test\"])\n return {\"provider\": provider, \"success\": True, \"message\": \"Connection successful\"}\n\n elif test_type == \"text_to_speech\":\n AIFactory.create_text_to_speech(model_name=test_model, provider=provider, config=config)\n return {\n \"provider\": provider,\n \"success\": True,\n \"message\": \"Connection successful (key format valid)\",\n }\n\n return {\n \"provider\": provider,\n \"success\": False,\n \"message\": f\"Unsupported test type: {test_type}\",\n }\n\n except Exception as e:\n error_msg = str(e)\n if \"401\" in error_msg or \"unauthorized\" in error_msg.lower():\n return {\"provider\": provider, \"success\": False, \"message\": \"Invalid API key\"}\n elif \"403\" in error_msg or \"forbidden\" in error_msg.lower():\n return {\"provider\": provider, \"success\": False, \"message\": \"API key lacks required permissions\"}\n elif \"rate\" in error_msg.lower() and \"limit\" in error_msg.lower():\n return {\"provider\": provider, \"success\": True, \"message\": \"Rate limited - but connection works\"}\n elif \"not found\" in error_msg.lower() and \"model\" in error_msg.lower():\n return {\"provider\": provider, \"success\": True, \"message\": \"API key valid (test model not available)\"}\n else:\n logger.debug(f\"Test connection error for credential {credential_id}: {e}\")\n truncated = error_msg[:100] + \"...\" if len(error_msg) > 100 else error_msg\n return {\"provider\": provider, \"success\": False, \"message\": f\"Error: {truncated}\"}\n\n\nasync def discover_with_config(provider: str, config: dict) -> List[dict]:\n \"\"\"\n Discover models using explicit config instead of env vars.\n\n Returns model names only — no type classification.\n The user chooses the model type when registering.\n \"\"\"\n api_key = config.get(\"api_key\")\n base_url = config.get(\"base_url\")\n\n # Static model lists for providers without a listing API\n STATIC_MODELS: Dict[str, List[str]] = {\n \"anthropic\": [\n \"claude-opus-4-20250514\",\n \"claude-sonnet-4-20250514\",\n \"claude-3-5-sonnet-20241022\",\n \"claude-3-5-haiku-20241022\",\n \"claude-3-opus-20240229\",\n \"claude-3-sonnet-20240229\",\n \"claude-3-haiku-20240307\",\n ],\n \"voyage\": [\n \"voyage-3\", \"voyage-3-lite\", \"voyage-code-3\",\n \"voyage-finance-2\", \"voyage-law-2\", \"voyage-multilingual-2\",\n ],\n \"elevenlabs\": [\n \"eleven_multilingual_v2\", \"eleven_turbo_v2_5\",\n \"eleven_turbo_v2\", \"eleven_monolingual_v1\",\n ],\n }\n\n if provider in STATIC_MODELS:\n if not api_key and provider != \"ollama\":\n return []\n return [\n {\"name\": m, \"provider\": provider}\n for m in STATIC_MODELS[provider]\n ]\n\n # API-based discovery URLs (OpenAI-style /models endpoints)\n url_map = {\n \"openai\": \"https://api.openai.com/v1/models\",\n \"groq\": \"https://api.groq.com/openai/v1/models\",\n \"mistral\": \"https://api.mistral.ai/v1/models\",\n \"deepseek\": \"https://api.deepseek.com/models\",\n \"xai\": \"https://api.x.ai/v1/models\",\n \"openrouter\": \"https://openrouter.ai/api/v1/models\",\n }\n\n if provider == \"ollama\":\n ollama_url = base_url or \"http://localhost:11434\"\n try:\n async with httpx.AsyncClient() as client:\n response = await client.get(f\"{ollama_url}/api/tags\", timeout=10.0)\n response.raise_for_status()\n data = response.json()\n return [\n {\"name\": m.get(\"name\", \"\"), \"provider\": \"ollama\"}\n for m in data.get(\"models\", [])\n if m.get(\"name\")\n ]\n except Exception as e:\n logger.warning(f\"Failed to discover Ollama models: {e}\")\n return []\n\n if provider == \"openai_compatible\":\n if not base_url:\n return []\n try:\n headers = {}\n if api_key:\n headers[\"Authorization\"] = f\"Bearer {api_key}\"\n async with httpx.AsyncClient() as client:\n response = await client.get(\n f\"{base_url.rstrip('/')}/models\", headers=headers, timeout=30.0,\n )\n response.raise_for_status()\n data = response.json()\n return [\n {\"name\": m.get(\"id\", \"\"), \"provider\": \"openai_compatible\"}\n for m in data.get(\"data\", [])\n if m.get(\"id\")\n ]\n except Exception as e:\n logger.warning(f\"Failed to discover openai_compatible models: {e}\")\n return []\n\n if provider == \"azure\":\n endpoint = config.get(\"endpoint\")\n api_version = config.get(\"api_version\", \"2024-10-21\")\n if not endpoint or not api_key:\n return []\n try:\n url = f\"{endpoint.rstrip('/')}/openai/models?api-version={api_version}\"\n headers = {\"api-key\": api_key}\n async with httpx.AsyncClient() as client:\n response = await client.get(url, headers=headers, timeout=30.0)\n response.raise_for_status()\n data = response.json()\n return [\n {\"name\": m.get(\"id\", \"\"), \"provider\": \"azure\"}\n for m in data.get(\"data\", [])\n if m.get(\"id\")\n ]\n except Exception as e:\n logger.warning(f\"Failed to discover Azure models: {e}\")\n return []\n\n if provider == \"vertex\":\n # Vertex AI requires service-account OAuth2 for model listing.\n # Return a curated static list of well-known Vertex models instead.\n VERTEX_MODELS = [\n \"gemini-2.0-flash\",\n \"gemini-2.0-flash-lite\",\n \"gemini-1.5-pro\",\n \"gemini-1.5-flash\",\n \"text-embedding-005\",\n ]\n return [{\"name\": m, \"provider\": \"vertex\"} for m in VERTEX_MODELS]\n\n if provider == \"google\":\n try:\n headers = {\"X-Goog-Api-Key\": api_key} if api_key else {}\n async with httpx.AsyncClient() as client:\n response = await client.get(\n \"https://generativelanguage.googleapis.com/v1/models\",\n headers=headers,\n timeout=30.0,\n )\n response.raise_for_status()\n data = response.json()\n return [\n {\n \"name\": model.get(\"name\", \"\").replace(\"models/\", \"\"),\n \"provider\": \"google\",\n \"description\": model.get(\"displayName\"),\n }\n for model in data.get(\"models\", [])\n if model.get(\"name\")\n ]\n except Exception as e:\n logger.warning(f\"Failed to discover Google models: {e}\")\n return []\n\n # Standard OpenAI-style API discovery\n discovery_url = url_map.get(provider)\n if not discovery_url or not api_key:\n return []\n\n try:\n async with httpx.AsyncClient() as client:\n response = await client.get(\n discovery_url,\n headers={\"Authorization\": f\"Bearer {api_key}\"},\n timeout=30.0,\n )\n response.raise_for_status()\n data = response.json()\n\n return [\n {\n \"name\": m.get(\"id\", \"\"),\n \"provider\": provider,\n \"description\": m.get(\"name\"),\n }\n for m in data.get(\"data\", [])\n if m.get(\"id\")\n ]\n except Exception as e:\n logger.warning(f\"Failed to discover {provider} models: {e}\")\n return []\n\n\nasync def register_models(credential_id: str, models_data: list) -> dict:\n \"\"\"\n Register discovered models and link them to a credential.\n\n Args:\n credential_id: The credential ID to link models to\n models_data: List of dicts with name, provider, model_type\n\n Returns:\n dict with created and existing counts\n \"\"\"\n cred = await Credential.get(credential_id)\n\n from open_notebook.ai.models import Model\n from open_notebook.database.repository import repo_query\n\n # Batch fetch existing models for this provider\n existing_models = await repo_query(\n \"SELECT string::lowercase(name) as name, string::lowercase(type) as type FROM model \"\n \"WHERE string::lowercase(provider) = $provider\",\n {\"provider\": cred.provider.lower()},\n )\n existing_keys = {(m[\"name\"], m[\"type\"]) for m in existing_models}\n\n created = 0\n existing = 0\n\n for model_data in models_data:\n key = (model_data.name.lower(), model_data.model_type.lower())\n if key in existing_keys:\n existing += 1\n continue\n\n new_model = Model(\n name=model_data.name,\n provider=model_data.provider or cred.provider,\n type=model_data.model_type,\n credential=cred.id,\n )\n await new_model.save()\n created += 1\n\n return {\"created\": created, \"existing\": existing}\n\n\nasync def migrate_from_provider_config() -> dict:\n \"\"\"\n Migrate existing ProviderConfig data to individual credential records.\n\n Returns dict with message, migrated, skipped, errors.\n \"\"\"\n logger.info(\"=== Starting ProviderConfig migration ===\")\n\n require_encryption_key()\n logger.info(\"Encryption key verified\")\n\n from open_notebook.domain.provider_config import ProviderConfig\n\n config = await ProviderConfig.get_instance()\n logger.info(\n f\"Found ProviderConfig with {len(config.credentials)} provider(s): \"\n f\"{', '.join(config.credentials.keys())}\"\n )\n\n migrated = []\n skipped = []\n errors = []\n\n for provider, credentials_list in config.credentials.items():\n for old_cred in credentials_list:\n try:\n # Check if a credential already exists for this provider with same name\n existing = await Credential.get_by_provider(provider)\n names = [c.name for c in existing]\n if old_cred.name in names:\n logger.info(\n f\"[{provider}/{old_cred.name}] Already exists in DB, skipping\"\n )\n skipped.append(f\"{provider}/{old_cred.name}\")\n continue\n\n # Determine modalities from the provider type\n modalities = get_default_modalities(provider)\n\n logger.info(f\"[{provider}/{old_cred.name}] Creating credential\")\n new_cred = Credential(\n name=old_cred.name,\n provider=provider,\n modalities=modalities,\n api_key=old_cred.api_key,\n base_url=old_cred.base_url,\n endpoint=old_cred.endpoint,\n api_version=old_cred.api_version,\n endpoint_llm=old_cred.endpoint_llm,\n endpoint_embedding=old_cred.endpoint_embedding,\n endpoint_stt=old_cred.endpoint_stt,\n endpoint_tts=old_cred.endpoint_tts,\n project=old_cred.project,\n location=old_cred.location,\n credentials_path=old_cred.credentials_path,\n )\n await new_cred.save()\n logger.info(\n f\"[{provider}/{old_cred.name}] Credential saved (id={new_cred.id})\"\n )\n\n # Link existing models for this provider to the new credential\n from open_notebook.ai.models import Model\n from open_notebook.database.repository import repo_query\n\n provider_models = await repo_query(\n \"SELECT * FROM model WHERE string::lowercase(provider) = $provider AND credential IS NONE\",\n {\"provider\": provider.lower()},\n )\n if provider_models:\n logger.info(\n f\"[{provider}/{old_cred.name}] Linking {len(provider_models)} \"\n f\"unassigned model(s)\"\n )\n for model_data in provider_models:\n model = Model(**model_data)\n model.credential = new_cred.id\n await model.save()\n\n migrated.append(f\"{provider}/{old_cred.name}\")\n\n except Exception as e:\n logger.error(\n f\"[{provider}/{old_cred.name}] Migration FAILED: \"\n f\"{type(e).__name__}: {e}\",\n exc_info=True,\n )\n errors.append(f\"{provider}/{old_cred.name}: {e}\")\n\n logger.info(\n f\"=== ProviderConfig migration complete === \"\n f\"migrated={len(migrated)} skipped={len(skipped)} errors={len(errors)}\"\n )\n if migrated:\n logger.info(f\" Migrated: {', '.join(migrated)}\")\n if skipped:\n logger.info(f\" Skipped: {', '.join(skipped)}\")\n if errors:\n logger.error(f\" Errors: {'; '.join(errors)}\")\n\n return {\n \"message\": f\"Migration complete. Migrated {len(migrated)} credentials.\",\n \"migrated\": migrated,\n \"skipped\": skipped,\n \"errors\": errors,\n }\n\n\nasync def migrate_from_env() -> dict:\n \"\"\"\n Migrate API keys from environment variables to credential records.\n\n Returns dict with message, migrated, skipped, not_configured, errors.\n \"\"\"\n logger.info(\"=== Starting environment variable migration ===\")\n logger.info(\n f\"Checking {len(PROVIDER_ENV_CONFIG)} providers: \"\n f\"{', '.join(PROVIDER_ENV_CONFIG.keys())}\"\n )\n\n require_encryption_key()\n logger.info(\"Encryption key verified\")\n\n from open_notebook.ai.models import Model\n from open_notebook.database.repository import repo_query\n\n migrated = []\n skipped = []\n not_configured = []\n errors = []\n\n for provider in PROVIDER_ENV_CONFIG:\n try:\n if not check_env_configured(provider):\n logger.debug(f\"[{provider}] No env vars configured, skipping\")\n not_configured.append(provider)\n continue\n\n logger.info(f\"[{provider}] Env vars detected, checking for existing credentials\")\n\n existing = await Credential.get_by_provider(provider)\n if existing:\n logger.info(\n f\"[{provider}] Already has {len(existing)} credential(s) in DB, skipping\"\n )\n skipped.append(provider)\n continue\n\n logger.info(f\"[{provider}] Creating credential from env vars\")\n cred = create_credential_from_env(provider)\n await cred.save()\n logger.info(f\"[{provider}] Credential saved successfully (id={cred.id})\")\n\n # Link unassigned models to this credential\n provider_models = await repo_query(\n \"SELECT * FROM model WHERE string::lowercase(provider) = $provider AND credential IS NONE\",\n {\"provider\": provider.lower()},\n )\n if provider_models:\n logger.info(\n f\"[{provider}] Linking {len(provider_models)} unassigned model(s) \"\n f\"to credential {cred.id}\"\n )\n for model_data in provider_models:\n model = Model(**model_data)\n model.credential = cred.id\n await model.save()\n else:\n logger.info(f\"[{provider}] No unassigned models to link\")\n\n migrated.append(provider)\n\n except Exception as e:\n logger.error(\n f\"[{provider}] Migration FAILED: {type(e).__name__}: {e}\",\n exc_info=True,\n )\n errors.append(f\"{provider}: {e}\")\n\n logger.info(\n f\"=== Environment variable migration complete === \"\n f\"migrated={len(migrated)} skipped={len(skipped)} \"\n f\"not_configured={len(not_configured)} errors={len(errors)}\"\n )\n if migrated:\n logger.info(f\" Migrated: {', '.join(migrated)}\")\n if skipped:\n logger.info(f\" Skipped (already in DB): {', '.join(skipped)}\")\n if errors:\n logger.error(f\" Errors: {'; '.join(errors)}\")\n\n return {\n \"message\": f\"Migration complete. Migrated {len(migrated)} providers.\",\n \"migrated\": migrated,\n \"skipped\": skipped,\n \"not_configured\": not_configured,\n \"errors\": errors,\n }\n" }, { "path": "api/embedding_service.py", "content": "\"\"\"\nEmbedding service layer using API.\n\"\"\"\n\nfrom typing import Any, Dict, List, Union\n\nfrom loguru import logger\n\nfrom api.client import api_client\n\n\nclass EmbeddingService:\n \"\"\"Service layer for embedding operations using API.\"\"\"\n\n def __init__(self):\n logger.info(\"Using API for embedding operations\")\n\n def embed_content(\n self, item_id: str, item_type: str\n ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:\n \"\"\"Embed content for vector search.\"\"\"\n result = api_client.embed_content(item_id=item_id, item_type=item_type)\n return result\n\n\n# Global service instance\nembedding_service = EmbeddingService()\n" }, { "path": "api/episode_profiles_service.py", "content": "\"\"\"\nEpisode profiles service layer using API.\n\"\"\"\n\nfrom typing import List\n\nfrom loguru import logger\n\nfrom api.client import api_client\nfrom open_notebook.podcasts.models import EpisodeProfile\n\n\nclass EpisodeProfilesService:\n \"\"\"Service layer for episode profiles operations using API.\"\"\"\n\n def __init__(self):\n logger.info(\"Using API for episode profiles operations\")\n\n def get_all_episode_profiles(self) -> List[EpisodeProfile]:\n \"\"\"Get all episode profiles.\"\"\"\n profiles_data = api_client.get_episode_profiles()\n # Convert API response to EpisodeProfile objects\n profiles = []\n for profile_data in profiles_data:\n profile = EpisodeProfile(\n name=profile_data[\"name\"],\n description=profile_data.get(\"description\", \"\"),\n speaker_config=profile_data[\"speaker_config\"],\n outline_provider=profile_data[\"outline_provider\"],\n outline_model=profile_data[\"outline_model\"],\n transcript_provider=profile_data[\"transcript_provider\"],\n transcript_model=profile_data[\"transcript_model\"],\n default_briefing=profile_data[\"default_briefing\"],\n num_segments=profile_data[\"num_segments\"],\n )\n profile.id = profile_data[\"id\"]\n profiles.append(profile)\n return profiles\n\n def get_episode_profile(self, profile_name: str) -> EpisodeProfile:\n \"\"\"Get a specific episode profile by name.\"\"\"\n profile_response = api_client.get_episode_profile(profile_name)\n profile_data = (\n profile_response\n if isinstance(profile_response, dict)\n else profile_response[0]\n )\n profile = EpisodeProfile(\n name=profile_data[\"name\"],\n description=profile_data.get(\"description\", \"\"),\n speaker_config=profile_data[\"speaker_config\"],\n outline_provider=profile_data[\"outline_provider\"],\n outline_model=profile_data[\"outline_model\"],\n transcript_provider=profile_data[\"transcript_provider\"],\n transcript_model=profile_data[\"transcript_model\"],\n default_briefing=profile_data[\"default_briefing\"],\n num_segments=profile_data[\"num_segments\"],\n )\n profile.id = profile_data[\"id\"]\n return profile\n\n def create_episode_profile(\n self,\n name: str,\n description: str = \"\",\n speaker_config: str = \"\",\n outline_provider: str = \"\",\n outline_model: str = \"\",\n transcript_provider: str = \"\",\n transcript_model: str = \"\",\n default_briefing: str = \"\",\n num_segments: int = 5,\n ) -> EpisodeProfile:\n \"\"\"Create a new episode profile.\"\"\"\n profile_response = api_client.create_episode_profile(\n name=name,\n description=description,\n speaker_config=speaker_config,\n outline_provider=outline_provider,\n outline_model=outline_model,\n transcript_provider=transcript_provider,\n transcript_model=transcript_model,\n default_briefing=default_briefing,\n num_segments=num_segments,\n )\n profile_data = (\n profile_response\n if isinstance(profile_response, dict)\n else profile_response[0]\n )\n profile = EpisodeProfile(\n name=profile_data[\"name\"],\n description=profile_data.get(\"description\", \"\"),\n speaker_config=profile_data[\"speaker_config\"],\n outline_provider=profile_data[\"outline_provider\"],\n outline_model=profile_data[\"outline_model\"],\n transcript_provider=profile_data[\"transcript_provider\"],\n transcript_model=profile_data[\"transcript_model\"],\n default_briefing=profile_data[\"default_briefing\"],\n num_segments=profile_data[\"num_segments\"],\n )\n profile.id = profile_data[\"id\"]\n return profile\n\n def delete_episode_profile(self, profile_id: str) -> bool:\n \"\"\"Delete an episode profile.\"\"\"\n api_client.delete_episode_profile(profile_id)\n return True\n\n\n# Global service instance\nepisode_profiles_service = EpisodeProfilesService()\n" }, { "path": "api/insights_service.py", "content": "\"\"\"\nInsights service layer using API.\n\"\"\"\n\nfrom typing import List, Optional\n\nfrom loguru import logger\n\nfrom api.client import api_client\nfrom open_notebook.domain.notebook import Note, SourceInsight\n\n\nclass InsightsService:\n \"\"\"Service layer for insights operations using API.\"\"\"\n\n def __init__(self):\n logger.info(\"Using API for insights operations\")\n\n def get_source_insights(self, source_id: str) -> List[SourceInsight]:\n \"\"\"Get all insights for a specific source.\"\"\"\n insights_data = api_client.get_source_insights(source_id)\n # Convert API response to SourceInsight objects\n insights = []\n for insight_data in insights_data:\n insight = SourceInsight(\n insight_type=insight_data[\"insight_type\"],\n content=insight_data[\"content\"],\n )\n insight.id = insight_data[\"id\"]\n insight.created = insight_data[\"created\"]\n insight.updated = insight_data[\"updated\"]\n insights.append(insight)\n return insights\n\n def get_insight(self, insight_id: str) -> SourceInsight:\n \"\"\"Get a specific insight.\"\"\"\n insight_response = api_client.get_insight(insight_id)\n insight_data = (\n insight_response\n if isinstance(insight_response, dict)\n else insight_response[0]\n )\n insight = SourceInsight(\n insight_type=insight_data[\"insight_type\"],\n content=insight_data[\"content\"],\n )\n insight.id = insight_data[\"id\"]\n insight.created = insight_data[\"created\"]\n insight.updated = insight_data[\"updated\"]\n # Note: source_id from API response is not stored; use await insight.get_source() if needed\n return insight\n\n def delete_insight(self, insight_id: str) -> bool:\n \"\"\"Delete a specific insight.\"\"\"\n api_client.delete_insight(insight_id)\n return True\n\n def save_insight_as_note(\n self, insight_id: str, notebook_id: Optional[str] = None\n ) -> Note:\n \"\"\"Convert an insight to a note.\"\"\"\n note_response = api_client.save_insight_as_note(insight_id, notebook_id)\n note_data = (\n note_response if isinstance(note_response, dict) else note_response[0]\n )\n note = Note(\n title=note_data[\"title\"],\n content=note_data[\"content\"],\n note_type=note_data[\"note_type\"],\n )\n note.id = note_data[\"id\"]\n note.created = note_data[\"created\"]\n note.updated = note_data[\"updated\"]\n return note\n\n def create_source_insight(\n self, source_id: str, transformation_id: str, model_id: Optional[str] = None\n ) -> SourceInsight:\n \"\"\"Create a new insight for a source by running a transformation.\"\"\"\n insight_response = api_client.create_source_insight(\n source_id, transformation_id, model_id\n )\n insight_data = (\n insight_response\n if isinstance(insight_response, dict)\n else insight_response[0]\n )\n insight = SourceInsight(\n insight_type=insight_data[\"insight_type\"],\n content=insight_data[\"content\"],\n )\n insight.id = insight_data[\"id\"]\n insight.created = insight_data[\"created\"]\n insight.updated = insight_data[\"updated\"]\n # Note: source_id from API response is not stored; use await insight.get_source() if needed\n return insight\n\n\n# Global service instance\ninsights_service = InsightsService()\n" }, { "path": "api/main.py", "content": "# Load environment variables\nfrom dotenv import load_dotenv\n\nload_dotenv()\n\nfrom contextlib import asynccontextmanager\n\nfrom fastapi import FastAPI, Request\nfrom fastapi.middleware.cors import CORSMiddleware\nfrom fastapi.responses import JSONResponse\nfrom loguru import logger\nfrom starlette.exceptions import HTTPException as StarletteHTTPException\n\nfrom api.auth import PasswordAuthMiddleware\nfrom open_notebook.exceptions import (\n AuthenticationError,\n ConfigurationError,\n ExternalServiceError,\n InvalidInputError,\n NetworkError,\n NotFoundError,\n OpenNotebookError,\n RateLimitError,\n)\nfrom api.routers import (\n auth,\n chat,\n config,\n context,\n credentials,\n embedding,\n embedding_rebuild,\n episode_profiles,\n insights,\n languages,\n models,\n notebooks,\n notes,\n podcasts,\n search,\n settings,\n source_chat,\n sources,\n speaker_profiles,\n transformations,\n)\nfrom api.routers import commands as commands_router\nfrom open_notebook.database.async_migrate import AsyncMigrationManager\nfrom open_notebook.utils.encryption import get_secret_from_env\n\n# Import commands to register them in the API process\ntry:\n logger.info(\"Commands imported in API process\")\nexcept Exception as e:\n logger.error(f\"Failed to import commands in API process: {e}\")\n\n\n@asynccontextmanager\nasync def lifespan(app: FastAPI):\n \"\"\"\n Lifespan event handler for the FastAPI application.\n Runs database migrations automatically on startup.\n \"\"\"\n import os\n\n # Startup: Security checks\n logger.info(\"Starting API initialization...\")\n\n # Security check: Encryption key\n if not get_secret_from_env(\"OPEN_NOTEBOOK_ENCRYPTION_KEY\"):\n logger.warning(\n \"OPEN_NOTEBOOK_ENCRYPTION_KEY not set. \"\n \"API key encryption will fail until this is configured. \"\n \"Set OPEN_NOTEBOOK_ENCRYPTION_KEY to any secret string.\"\n )\n\n # Run database migrations\n\n try:\n migration_manager = AsyncMigrationManager()\n current_version = await migration_manager.get_current_version()\n logger.info(f\"Current database version: {current_version}\")\n\n if await migration_manager.needs_migration():\n logger.warning(\"Database migrations are pending. Running migrations...\")\n await migration_manager.run_migration_up()\n new_version = await migration_manager.get_current_version()\n logger.success(\n f\"Migrations completed successfully. Database is now at version {new_version}\"\n )\n else:\n logger.info(\n \"Database is already at the latest version. No migrations needed.\"\n )\n except Exception as e:\n logger.error(f\"CRITICAL: Database migration failed: {str(e)}\")\n logger.exception(e)\n # Fail fast - don't start the API with an outdated database schema\n raise RuntimeError(f\"Failed to run database migrations: {str(e)}\") from e\n\n # Run podcast profile data migration (legacy strings -> Model registry)\n try:\n from open_notebook.podcasts.migration import migrate_podcast_profiles\n\n await migrate_podcast_profiles()\n except Exception as e:\n logger.warning(f\"Podcast profile migration encountered errors: {e}\")\n # Non-fatal: profiles can be migrated manually via UI\n\n logger.success(\"API initialization completed successfully\")\n\n # Yield control to the application\n yield\n\n # Shutdown: cleanup if needed\n logger.info(\"API shutdown complete\")\n\n\napp = FastAPI(\n title=\"Open Notebook API\",\n description=\"API for Open Notebook - Research Assistant\",\n lifespan=lifespan,\n)\n\n# Add password authentication middleware first\n# Exclude /api/auth/status and /api/config from authentication\napp.add_middleware(\n PasswordAuthMiddleware,\n excluded_paths=[\n \"/\",\n \"/health\",\n \"/docs\",\n \"/openapi.json\",\n \"/redoc\",\n \"/api/auth/status\",\n \"/api/config\",\n ],\n)\n\n# Add CORS middleware last (so it processes first)\napp.add_middleware(\n CORSMiddleware,\n allow_origins=[\"*\"], # In production, replace with specific origins\n allow_credentials=True,\n allow_methods=[\"*\"],\n allow_headers=[\"*\"],\n)\n\n\n# Custom exception handler to ensure CORS headers are included in error responses\n# This helps when errors occur before the CORS middleware can process them\n@app.exception_handler(StarletteHTTPException)\nasync def custom_http_exception_handler(request: Request, exc: StarletteHTTPException):\n \"\"\"\n Custom exception handler that ensures CORS headers are included in error responses.\n This is particularly important for 413 (Payload Too Large) errors during file uploads.\n\n Note: If a reverse proxy (nginx, traefik) returns 413 before the request reaches\n FastAPI, this handler won't be called. In that case, configure your reverse proxy\n to add CORS headers to error responses.\n \"\"\"\n # Get the origin from the request\n origin = request.headers.get(\"origin\", \"*\")\n\n return JSONResponse(\n status_code=exc.status_code,\n content={\"detail\": exc.detail},\n headers={\n **(exc.headers or {}), \"Access-Control-Allow-Origin\": origin,\n \"Access-Control-Allow-Credentials\": \"true\",\n \"Access-Control-Allow-Methods\": \"*\",\n \"Access-Control-Allow-Headers\": \"*\",\n },\n )\n\n\ndef _cors_headers(request: Request) -> dict[str, str]:\n origin = request.headers.get(\"origin\", \"*\")\n return {\n \"Access-Control-Allow-Origin\": origin,\n \"Access-Control-Allow-Credentials\": \"true\",\n \"Access-Control-Allow-Methods\": \"*\",\n \"Access-Control-Allow-Headers\": \"*\",\n }\n\n\n@app.exception_handler(NotFoundError)\nasync def not_found_error_handler(request: Request, exc: NotFoundError):\n return JSONResponse(\n status_code=404,\n content={\"detail\": str(exc)},\n headers=_cors_headers(request),\n )\n\n\n@app.exception_handler(InvalidInputError)\nasync def invalid_input_error_handler(request: Request, exc: InvalidInputError):\n return JSONResponse(\n status_code=400,\n content={\"detail\": str(exc)},\n headers=_cors_headers(request),\n )\n\n\n@app.exception_handler(AuthenticationError)\nasync def authentication_error_handler(request: Request, exc: AuthenticationError):\n return JSONResponse(\n status_code=401,\n content={\"detail\": str(exc)},\n headers=_cors_headers(request),\n )\n\n\n@app.exception_handler(RateLimitError)\nasync def rate_limit_error_handler(request: Request, exc: RateLimitError):\n return JSONResponse(\n status_code=429,\n content={\"detail\": str(exc)},\n headers=_cors_headers(request),\n )\n\n\n@app.exception_handler(ConfigurationError)\nasync def configuration_error_handler(request: Request, exc: ConfigurationError):\n return JSONResponse(\n status_code=422,\n content={\"detail\": str(exc)},\n headers=_cors_headers(request),\n )\n\n\n@app.exception_handler(NetworkError)\nasync def network_error_handler(request: Request, exc: NetworkError):\n return JSONResponse(\n status_code=502,\n content={\"detail\": str(exc)},\n headers=_cors_headers(request),\n )\n\n\n@app.exception_handler(ExternalServiceError)\nasync def external_service_error_handler(request: Request, exc: ExternalServiceError):\n return JSONResponse(\n status_code=502,\n content={\"detail\": str(exc)},\n headers=_cors_headers(request),\n )\n\n\n@app.exception_handler(OpenNotebookError)\nasync def open_notebook_error_handler(request: Request, exc: OpenNotebookError):\n return JSONResponse(\n status_code=500,\n content={\"detail\": str(exc)},\n headers=_cors_headers(request),\n )\n\n\n# Include routers\napp.include_router(auth.router, prefix=\"/api\", tags=[\"auth\"])\napp.include_router(config.router, prefix=\"/api\", tags=[\"config\"])\napp.include_router(notebooks.router, prefix=\"/api\", tags=[\"notebooks\"])\napp.include_router(search.router, prefix=\"/api\", tags=[\"search\"])\napp.include_router(models.router, prefix=\"/api\", tags=[\"models\"])\napp.include_router(transformations.router, prefix=\"/api\", tags=[\"transformations\"])\napp.include_router(notes.router, prefix=\"/api\", tags=[\"notes\"])\napp.include_router(embedding.router, prefix=\"/api\", tags=[\"embedding\"])\napp.include_router(\n embedding_rebuild.router, prefix=\"/api/embeddings\", tags=[\"embeddings\"]\n)\napp.include_router(settings.router, prefix=\"/api\", tags=[\"settings\"])\napp.include_router(context.router, prefix=\"/api\", tags=[\"context\"])\napp.include_router(sources.router, prefix=\"/api\", tags=[\"sources\"])\napp.include_router(insights.router, prefix=\"/api\", tags=[\"insights\"])\napp.include_router(commands_router.router, prefix=\"/api\", tags=[\"commands\"])\napp.include_router(podcasts.router, prefix=\"/api\", tags=[\"podcasts\"])\napp.include_router(episode_profiles.router, prefix=\"/api\", tags=[\"episode-profiles\"])\napp.include_router(speaker_profiles.router, prefix=\"/api\", tags=[\"speaker-profiles\"])\napp.include_router(chat.router, prefix=\"/api\", tags=[\"chat\"])\napp.include_router(source_chat.router, prefix=\"/api\", tags=[\"source-chat\"])\napp.include_router(credentials.router, prefix=\"/api\", tags=[\"credentials\"])\napp.include_router(languages.router, prefix=\"/api\", tags=[\"languages\"])\n\n\n@app.get(\"/\")\nasync def root():\n return {\"message\": \"Open Notebook API is running\"}\n\n\n@app.get(\"/health\")\nasync def health():\n return {\"status\": \"healthy\"}\n" }, { "path": "api/models.py", "content": "from typing import Any, Dict, List, Literal, Optional\n\nfrom pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator\n\n\n# Notebook models\nclass NotebookCreate(BaseModel):\n name: str = Field(..., description=\"Name of the notebook\")\n description: str = Field(default=\"\", description=\"Description of the notebook\")\n\n\nclass NotebookUpdate(BaseModel):\n name: Optional[str] = Field(None, description=\"Name of the notebook\")\n description: Optional[str] = Field(None, description=\"Description of the notebook\")\n archived: Optional[bool] = Field(\n None, description=\"Whether the notebook is archived\"\n )\n\n\nclass NotebookResponse(BaseModel):\n id: str\n name: str\n description: str\n archived: bool\n created: str\n updated: str\n source_count: int\n note_count: int\n\n\n# Search models\nclass SearchRequest(BaseModel):\n query: str = Field(..., description=\"Search query\")\n type: Literal[\"text\", \"vector\"] = Field(\"text\", description=\"Search type\")\n limit: int = Field(100, description=\"Maximum number of results\", le=1000)\n search_sources: bool = Field(True, description=\"Include sources in search\")\n search_notes: bool = Field(True, description=\"Include notes in search\")\n minimum_score: float = Field(\n 0.2, description=\"Minimum score for vector search\", ge=0, le=1\n )\n\n\nclass SearchResponse(BaseModel):\n results: List[Dict[str, Any]] = Field(..., description=\"Search results\")\n total_count: int = Field(..., description=\"Total number of results\")\n search_type: str = Field(..., description=\"Type of search performed\")\n\n\nclass AskRequest(BaseModel):\n question: str = Field(..., description=\"Question to ask the knowledge base\")\n strategy_model: str = Field(..., description=\"Model ID for query strategy\")\n answer_model: str = Field(..., description=\"Model ID for individual answers\")\n final_answer_model: str = Field(..., description=\"Model ID for final answer\")\n\n\nclass AskResponse(BaseModel):\n answer: str = Field(..., description=\"Final answer from the knowledge base\")\n question: str = Field(..., description=\"Original question\")\n\n\n# Models API models\nclass ModelCreate(BaseModel):\n name: str = Field(..., description=\"Model name (e.g., gpt-5-mini, claude, gemini)\")\n provider: str = Field(\n ..., description=\"Provider name (e.g., openai, anthropic, gemini)\"\n )\n type: str = Field(\n ...,\n description=\"Model type (language, embedding, text_to_speech, speech_to_text)\",\n )\n credential: Optional[str] = Field(\n None, description=\"Credential ID to link this model to\"\n )\n\n\nclass ModelResponse(BaseModel):\n id: str\n name: str\n provider: str\n type: str\n credential: Optional[str] = None\n created: str\n updated: str\n\n\nclass DefaultModelsResponse(BaseModel):\n default_chat_model: Optional[str] = None\n default_transformation_model: Optional[str] = None\n large_context_model: Optional[str] = None\n default_text_to_speech_model: Optional[str] = None\n default_speech_to_text_model: Optional[str] = None\n default_embedding_model: Optional[str] = None\n default_tools_model: Optional[str] = None\n\n\nclass ProviderAvailabilityResponse(BaseModel):\n available: List[str] = Field(..., description=\"List of available providers\")\n unavailable: List[str] = Field(..., description=\"List of unavailable providers\")\n supported_types: Dict[str, List[str]] = Field(\n ..., description=\"Provider to supported model types mapping\"\n )\n\n\n# Transformations API models\nclass TransformationCreate(BaseModel):\n name: str = Field(..., description=\"Transformation name\")\n title: str = Field(..., description=\"Display title for the transformation\")\n description: str = Field(\n ..., description=\"Description of what this transformation does\"\n )\n prompt: str = Field(..., description=\"The transformation prompt\")\n apply_default: bool = Field(\n False, description=\"Whether to apply this transformation by default\"\n )\n\n\nclass TransformationUpdate(BaseModel):\n name: Optional[str] = Field(None, description=\"Transformation name\")\n title: Optional[str] = Field(\n None, description=\"Display title for the transformation\"\n )\n description: Optional[str] = Field(\n None, description=\"Description of what this transformation does\"\n )\n prompt: Optional[str] = Field(None, description=\"The transformation prompt\")\n apply_default: Optional[bool] = Field(\n None, description=\"Whether to apply this transformation by default\"\n )\n\n\nclass TransformationResponse(BaseModel):\n id: str\n name: str\n title: str\n description: str\n prompt: str\n apply_default: bool\n created: str\n updated: str\n\n\nclass TransformationExecuteRequest(BaseModel):\n model_config = ConfigDict(protected_namespaces=())\n\n transformation_id: str = Field(\n ..., description=\"ID of the transformation to execute\"\n )\n input_text: str = Field(..., description=\"Text to transform\")\n model_id: str = Field(..., description=\"Model ID to use for the transformation\")\n\n\nclass TransformationExecuteResponse(BaseModel):\n model_config = ConfigDict(protected_namespaces=())\n\n output: str = Field(..., description=\"Transformed text\")\n transformation_id: str = Field(..., description=\"ID of the transformation used\")\n model_id: str = Field(..., description=\"Model ID used\")\n\n\n# Default Prompt API models\nclass DefaultPromptResponse(BaseModel):\n transformation_instructions: str = Field(\n ..., description=\"Default transformation instructions\"\n )\n\n\nclass DefaultPromptUpdate(BaseModel):\n transformation_instructions: str = Field(\n ..., description=\"Default transformation instructions\"\n )\n\n\n# Notes API models\nclass NoteCreate(BaseModel):\n title: Optional[str] = Field(None, description=\"Note title\")\n content: str = Field(..., description=\"Note content\")\n note_type: Optional[str] = Field(\"human\", description=\"Type of note (human, ai)\")\n notebook_id: Optional[str] = Field(\n None, description=\"Notebook ID to add the note to\"\n )\n\n\nclass NoteUpdate(BaseModel):\n title: Optional[str] = Field(None, description=\"Note title\")\n content: Optional[str] = Field(None, description=\"Note content\")\n note_type: Optional[str] = Field(None, description=\"Type of note (human, ai)\")\n\n\nclass NoteResponse(BaseModel):\n id: str\n title: Optional[str]\n content: Optional[str]\n note_type: Optional[str]\n created: str\n updated: str\n command_id: Optional[str] = None\n\n\n# Embedding API models\nclass EmbedRequest(BaseModel):\n item_id: str = Field(..., description=\"ID of the item to embed\")\n item_type: str = Field(..., description=\"Type of item (source, note)\")\n async_processing: bool = Field(\n False, description=\"Process asynchronously in background\"\n )\n\n\nclass EmbedResponse(BaseModel):\n success: bool = Field(..., description=\"Whether embedding was successful\")\n message: str = Field(..., description=\"Result message\")\n item_id: str = Field(..., description=\"ID of the item that was embedded\")\n item_type: str = Field(..., description=\"Type of item that was embedded\")\n command_id: Optional[str] = Field(\n None, description=\"Command ID for async processing\"\n )\n\n\n# Rebuild request/response models\nclass RebuildRequest(BaseModel):\n mode: Literal[\"existing\", \"all\"] = Field(\n ...,\n description=\"Rebuild mode: 'existing' only re-embeds items with embeddings, 'all' embeds everything\",\n )\n include_sources: bool = Field(True, description=\"Include sources in rebuild\")\n include_notes: bool = Field(True, description=\"Include notes in rebuild\")\n include_insights: bool = Field(True, description=\"Include insights in rebuild\")\n\n\nclass RebuildResponse(BaseModel):\n command_id: str = Field(..., description=\"Command ID to track progress\")\n total_items: int = Field(..., description=\"Estimated number of items to process\")\n message: str = Field(..., description=\"Status message\")\n\n\nclass RebuildProgress(BaseModel):\n processed: int = Field(..., description=\"Number of items processed\")\n total: int = Field(..., description=\"Total items to process\")\n percentage: float = Field(..., description=\"Progress percentage\")\n\n\nclass RebuildStats(BaseModel):\n sources: int = Field(0, description=\"Sources processed\")\n notes: int = Field(0, description=\"Notes processed\")\n insights: int = Field(0, description=\"Insights processed\")\n failed: int = Field(0, description=\"Failed items\")\n\n\nclass RebuildStatusResponse(BaseModel):\n command_id: str = Field(..., description=\"Command ID\")\n status: str = Field(..., description=\"Status: queued, running, completed, failed\")\n progress: Optional[RebuildProgress] = None\n stats: Optional[RebuildStats] = None\n started_at: Optional[str] = None\n completed_at: Optional[str] = None\n error_message: Optional[str] = None\n\n\n# Settings API models\nclass SettingsResponse(BaseModel):\n default_content_processing_engine_doc: Optional[str] = None\n default_content_processing_engine_url: Optional[str] = None\n default_embedding_option: Optional[str] = None\n auto_delete_files: Optional[str] = None\n youtube_preferred_languages: Optional[List[str]] = None\n\n\nclass SettingsUpdate(BaseModel):\n default_content_processing_engine_doc: Optional[str] = None\n default_content_processing_engine_url: Optional[str] = None\n default_embedding_option: Optional[str] = None\n auto_delete_files: Optional[str] = None\n youtube_preferred_languages: Optional[List[str]] = None\n\n\n# Sources API models\nclass AssetModel(BaseModel):\n file_path: Optional[str] = None\n url: Optional[str] = None\n\n\nclass SourceCreate(BaseModel):\n # Backward compatibility: support old single notebook_id\n notebook_id: Optional[str] = Field(\n None, description=\"Notebook ID to add the source to (deprecated, use notebooks)\"\n )\n # New multi-notebook support\n notebooks: Optional[List[str]] = Field(\n None, description=\"List of notebook IDs to add the source to\"\n )\n # Required fields\n type: str = Field(..., description=\"Source type: link, upload, or text\")\n url: Optional[str] = Field(None, description=\"URL for link type\")\n file_path: Optional[str] = Field(None, description=\"File path for upload type\")\n content: Optional[str] = Field(None, description=\"Text content for text type\")\n title: Optional[str] = Field(None, description=\"Source title\")\n transformations: Optional[List[str]] = Field(\n default_factory=list, description=\"Transformation IDs to apply\"\n )\n embed: bool = Field(False, description=\"Whether to embed content for vector search\")\n delete_source: bool = Field(\n False, description=\"Whether to delete uploaded file after processing\"\n )\n # New async processing support\n async_processing: bool = Field(\n False, description=\"Whether to process source asynchronously\"\n )\n\n @model_validator(mode=\"after\")\n def validate_notebook_fields(self):\n # Ensure only one of notebook_id or notebooks is provided\n if self.notebook_id is not None and self.notebooks is not None:\n raise ValueError(\n \"Cannot specify both 'notebook_id' and 'notebooks'. Use 'notebooks' for multi-notebook support.\"\n )\n\n # Convert single notebook_id to notebooks array for internal processing\n if self.notebook_id is not None:\n self.notebooks = [self.notebook_id]\n # Keep notebook_id for backward compatibility in response\n\n # Set empty array if no notebooks specified (allow sources without notebooks)\n if self.notebooks is None:\n self.notebooks = []\n\n return self\n\n\nclass SourceUpdate(BaseModel):\n title: Optional[str] = Field(None, description=\"Source title\")\n topics: Optional[List[str]] = Field(None, description=\"Source topics\")\n\n\nclass SourceResponse(BaseModel):\n id: str\n title: Optional[str]\n topics: Optional[List[str]]\n asset: Optional[AssetModel]\n full_text: Optional[str]\n embedded: bool\n embedded_chunks: int\n file_available: Optional[bool] = None\n created: str\n updated: str\n # New fields for async processing\n command_id: Optional[str] = None\n status: Optional[str] = None\n processing_info: Optional[Dict] = None\n # Notebook associations\n notebooks: Optional[List[str]] = None\n\n\nclass SourceListResponse(BaseModel):\n id: str\n title: Optional[str]\n topics: Optional[List[str]]\n asset: Optional[AssetModel]\n embedded: bool # Boolean flag indicating if source has embeddings\n embedded_chunks: int # Number of embedded chunks\n insights_count: int\n created: str\n updated: str\n file_available: Optional[bool] = None\n # Status fields for async processing\n command_id: Optional[str] = None\n status: Optional[str] = None\n processing_info: Optional[Dict[str, Any]] = None\n\n\n# Context API models\nclass ContextConfig(BaseModel):\n sources: Dict[str, str] = Field(\n default_factory=dict, description=\"Source inclusion config {source_id: level}\"\n )\n notes: Dict[str, str] = Field(\n default_factory=dict, description=\"Note inclusion config {note_id: level}\"\n )\n\n\nclass ContextRequest(BaseModel):\n notebook_id: str = Field(..., description=\"Notebook ID to get context for\")\n context_config: Optional[ContextConfig] = Field(\n None, description=\"Context configuration\"\n )\n\n\nclass ContextResponse(BaseModel):\n notebook_id: str\n sources: List[Dict[str, Any]] = Field(..., description=\"Source context data\")\n notes: List[Dict[str, Any]] = Field(..., description=\"Note context data\")\n total_tokens: Optional[int] = Field(None, description=\"Estimated token count\")\n\n\n# Insights API models\nclass SourceInsightResponse(BaseModel):\n id: str\n source_id: str\n insight_type: str\n content: str\n created: str\n updated: str\n\n\nclass InsightCreationResponse(BaseModel):\n \"\"\"Response for async insight creation.\"\"\"\n\n status: Literal[\"pending\"] = \"pending\"\n message: str = \"Insight generation started\"\n source_id: str\n transformation_id: str\n command_id: Optional[str] = None\n\n\nclass SaveAsNoteRequest(BaseModel):\n notebook_id: Optional[str] = Field(None, description=\"Notebook ID to add note to\")\n\n\nclass CreateSourceInsightRequest(BaseModel):\n model_config = ConfigDict(protected_namespaces=())\n\n transformation_id: str = Field(..., description=\"ID of transformation to apply\")\n model_id: Optional[str] = Field(\n None, description=\"Model ID (uses default if not provided)\"\n )\n\n\n# Source status response\nclass SourceStatusResponse(BaseModel):\n status: Optional[str] = Field(None, description=\"Processing status\")\n message: str = Field(..., description=\"Descriptive message about the status\")\n processing_info: Optional[Dict[str, Any]] = Field(\n None, description=\"Detailed processing information\"\n )\n command_id: Optional[str] = Field(None, description=\"Command ID if available\")\n\n\n# Error response\nclass ErrorResponse(BaseModel):\n error: str\n message: str\n\n\n# API Key Configuration models\nclass SetApiKeyRequest(BaseModel):\n \"\"\"Request to set an API key for a provider.\"\"\"\n\n api_key: Optional[str] = Field(None, description=\"API key for the provider\")\n base_url: Optional[str] = Field(\n None, description=\"Base URL for URL-based providers (Ollama, OpenAI-compatible)\"\n )\n endpoint: Optional[str] = Field(\n None, description=\"Endpoint URL for Azure OpenAI\"\n )\n api_version: Optional[str] = Field(\n None, description=\"API version for Azure OpenAI\"\n )\n endpoint_llm: Optional[str] = Field(\n None, description=\"Service-specific endpoint for LLM (Azure)\"\n )\n endpoint_embedding: Optional[str] = Field(\n None, description=\"Service-specific endpoint for embedding (Azure)\"\n )\n endpoint_stt: Optional[str] = Field(\n None, description=\"Service-specific endpoint for STT (Azure)\"\n )\n endpoint_tts: Optional[str] = Field(\n None, description=\"Service-specific endpoint for TTS (Azure)\"\n )\n service_type: Optional[Literal[\"llm\", \"embedding\", \"stt\", \"tts\"]] = Field(\n None,\n description=\"Service type for OpenAI-compatible providers (llm, embedding, stt, tts)\",\n )\n # Vertex AI specific fields\n vertex_project: Optional[str] = Field(\n None, description=\"Google Cloud Project ID for Vertex AI\"\n )\n vertex_location: Optional[str] = Field(\n None, description=\"Google Cloud Region for Vertex AI (e.g., us-central1)\"\n )\n vertex_credentials_path: Optional[str] = Field(\n None, description=\"Path to Google Cloud service account JSON file\"\n )\n\n @field_validator(\n \"api_key\",\n \"base_url\",\n \"endpoint\",\n \"api_version\",\n \"endpoint_llm\",\n \"endpoint_embedding\",\n \"endpoint_stt\",\n \"endpoint_tts\",\n \"vertex_project\",\n \"vertex_location\",\n \"vertex_credentials_path\",\n mode=\"before\",\n )\n @classmethod\n def validate_not_empty_string(cls, v: Optional[str]) -> Optional[str]:\n \"\"\"Reject empty strings - convert to None or raise error.\"\"\"\n if v is not None:\n stripped = v.strip()\n if not stripped:\n return None # Treat empty/whitespace-only as None\n return stripped\n return v\n\n\nclass ApiKeyStatusResponse(BaseModel):\n \"\"\"Response showing which providers are configured and their source.\"\"\"\n\n configured: Dict[str, bool] = Field(\n ..., description=\"Map of provider name to whether it is configured\"\n )\n source: Dict[str, Literal[\"database\", \"environment\", \"none\"]] = Field(\n ...,\n description=\"Map of provider name to configuration source (database, environment, or none)\",\n )\n encryption_configured: bool = Field(\n ...,\n description=\"Whether OPEN_NOTEBOOK_ENCRYPTION_KEY is set (required to store keys in database)\",\n )\n\n\nclass TestConnectionResponse(BaseModel):\n \"\"\"Response from testing a provider connection.\"\"\"\n\n provider: str = Field(..., description=\"Provider name that was tested\")\n success: bool = Field(..., description=\"Whether connection test succeeded\")\n message: str = Field(..., description=\"Result message with details\")\n\n\nclass MigrateFromEnvRequest(BaseModel):\n \"\"\"Request to migrate API keys from environment variables to database.\"\"\"\n\n force: bool = Field(\n False, description=\"Force overwrite existing database configurations\"\n )\n\n\nclass MigrationResult(BaseModel):\n \"\"\"Response from migrating API keys from environment to database.\"\"\"\n\n message: str = Field(..., description=\"Summary message\")\n migrated: List[str] = Field(\n default_factory=list, description=\"Providers successfully migrated\"\n )\n skipped: List[str] = Field(\n default_factory=list, description=\"Providers skipped (already in DB)\"\n )\n errors: List[str] = Field(\n default_factory=list, description=\"Migration errors by provider\"\n )\n\n\n# Notebook delete cascade models\n# Credential models\nclass CreateCredentialRequest(BaseModel):\n \"\"\"Request to create a new credential.\"\"\"\n\n name: str = Field(..., description=\"Credential name\")\n provider: str = Field(..., description=\"Provider name (openai, anthropic, etc.)\")\n modalities: List[str] = Field(\n default_factory=list,\n description=\"Supported modalities (language, embedding, text_to_speech, speech_to_text)\",\n )\n api_key: Optional[str] = Field(None, description=\"API key (stored encrypted)\")\n base_url: Optional[str] = Field(None, description=\"Base URL\")\n endpoint: Optional[str] = Field(None, description=\"Endpoint URL (Azure)\")\n api_version: Optional[str] = Field(None, description=\"API version (Azure)\")\n endpoint_llm: Optional[str] = Field(None, description=\"LLM endpoint\")\n endpoint_embedding: Optional[str] = Field(None, description=\"Embedding endpoint\")\n endpoint_stt: Optional[str] = Field(None, description=\"STT endpoint\")\n endpoint_tts: Optional[str] = Field(None, description=\"TTS endpoint\")\n project: Optional[str] = Field(None, description=\"Project ID (Vertex)\")\n location: Optional[str] = Field(None, description=\"Location (Vertex)\")\n credentials_path: Optional[str] = Field(\n None, description=\"Credentials file path (Vertex)\"\n )\n\n\nclass UpdateCredentialRequest(BaseModel):\n \"\"\"Request to update an existing credential.\"\"\"\n\n name: Optional[str] = Field(None, description=\"Credential name\")\n modalities: Optional[List[str]] = Field(None, description=\"Supported modalities\")\n api_key: Optional[str] = Field(None, description=\"API key (stored encrypted)\")\n base_url: Optional[str] = Field(None, description=\"Base URL\")\n endpoint: Optional[str] = Field(None, description=\"Endpoint URL\")\n api_version: Optional[str] = Field(None, description=\"API version\")\n endpoint_llm: Optional[str] = Field(None, description=\"LLM endpoint\")\n endpoint_embedding: Optional[str] = Field(None, description=\"Embedding endpoint\")\n endpoint_stt: Optional[str] = Field(None, description=\"STT endpoint\")\n endpoint_tts: Optional[str] = Field(None, description=\"TTS endpoint\")\n project: Optional[str] = Field(None, description=\"Project ID\")\n location: Optional[str] = Field(None, description=\"Location\")\n credentials_path: Optional[str] = Field(None, description=\"Credentials path\")\n\n\nclass CredentialResponse(BaseModel):\n \"\"\"Response for a credential (never includes api_key).\"\"\"\n\n id: str\n name: str\n provider: str\n modalities: List[str]\n base_url: Optional[str] = None\n endpoint: Optional[str] = None\n api_version: Optional[str] = None\n endpoint_llm: Optional[str] = None\n endpoint_embedding: Optional[str] = None\n endpoint_stt: Optional[str] = None\n endpoint_tts: Optional[str] = None\n project: Optional[str] = None\n location: Optional[str] = None\n credentials_path: Optional[str] = None\n has_api_key: bool = False\n created: str\n updated: str\n model_count: int = 0\n\n\nclass CredentialDeleteResponse(BaseModel):\n \"\"\"Response for credential deletion.\"\"\"\n\n message: str\n deleted_models: int = 0\n\n\nclass DiscoveredModelResponse(BaseModel):\n \"\"\"A model discovered from a provider.\"\"\"\n\n name: str\n provider: str\n model_type: Optional[str] = None\n description: Optional[str] = None\n\n\nclass DiscoverModelsResponse(BaseModel):\n \"\"\"Response from model discovery.\"\"\"\n\n credential_id: str\n provider: str\n discovered: List[DiscoveredModelResponse]\n\n\nclass RegisterModelData(BaseModel):\n \"\"\"A model to register with user-specified type.\"\"\"\n\n name: str\n provider: str\n model_type: str # Required: user specifies the type\n\n\nclass RegisterModelsRequest(BaseModel):\n \"\"\"Request to register discovered models.\"\"\"\n\n models: List[RegisterModelData]\n\n\nclass RegisterModelsResponse(BaseModel):\n \"\"\"Response from model registration.\"\"\"\n\n created: int\n existing: int\n\n\nclass NotebookDeletePreview(BaseModel):\n notebook_id: str = Field(..., description=\"ID of the notebook\")\n notebook_name: str = Field(..., description=\"Name of the notebook\")\n note_count: int = Field(..., description=\"Number of notes that will be deleted\")\n exclusive_source_count: int = Field(\n ..., description=\"Number of sources only in this notebook\"\n )\n shared_source_count: int = Field(\n ..., description=\"Number of sources shared with other notebooks\"\n )\n\n\nclass NotebookDeleteResponse(BaseModel):\n message: str = Field(..., description=\"Success message\")\n deleted_notes: int = Field(..., description=\"Number of notes deleted\")\n deleted_sources: int = Field(..., description=\"Number of exclusive sources deleted\")\n unlinked_sources: int = Field(\n ..., description=\"Number of sources unlinked from notebook\"\n )\n" }, { "path": "api/models_service.py", "content": "\"\"\"\nModels service layer using API.\n\"\"\"\n\nfrom typing import List, Optional\n\nfrom loguru import logger\n\nfrom api.client import api_client\nfrom open_notebook.ai.models import DefaultModels, Model\n\n\nclass ModelsService:\n \"\"\"Service layer for models operations using API.\"\"\"\n\n def __init__(self):\n logger.info(\"Using API for models operations\")\n\n def get_all_models(self, model_type: Optional[str] = None) -> List[Model]:\n \"\"\"Get all models with optional type filtering.\"\"\"\n models_data = api_client.get_models(model_type=model_type)\n # Convert API response to Model objects\n models = []\n for model_data in models_data:\n model = Model(\n name=model_data[\"name\"],\n provider=model_data[\"provider\"],\n type=model_data[\"type\"],\n )\n model.id = model_data[\"id\"]\n model.created = model_data[\"created\"]\n model.updated = model_data[\"updated\"]\n models.append(model)\n return models\n\n def create_model(self, name: str, provider: str, model_type: str) -> Model:\n \"\"\"Create a new model.\"\"\"\n response = api_client.create_model(name, provider, model_type)\n model_data = response if isinstance(response, dict) else response[0]\n model = Model(\n name=model_data[\"name\"],\n provider=model_data[\"provider\"],\n type=model_data[\"type\"],\n )\n model.id = model_data[\"id\"]\n model.created = model_data[\"created\"]\n model.updated = model_data[\"updated\"]\n return model\n\n def delete_model(self, model_id: str) -> bool:\n \"\"\"Delete a model.\"\"\"\n api_client.delete_model(model_id)\n return True\n\n def get_default_models(self) -> DefaultModels:\n \"\"\"Get default model assignments.\"\"\"\n response = api_client.get_default_models()\n defaults_data = response if isinstance(response, dict) else response[0]\n defaults = DefaultModels()\n\n # Set the values from API response\n defaults.default_chat_model = defaults_data.get(\"default_chat_model\")\n defaults.default_transformation_model = defaults_data.get(\n \"default_transformation_model\"\n )\n defaults.large_context_model = defaults_data.get(\"large_context_model\")\n defaults.default_text_to_speech_model = defaults_data.get(\n \"default_text_to_speech_model\"\n )\n defaults.default_speech_to_text_model = defaults_data.get(\n \"default_speech_to_text_model\"\n )\n defaults.default_embedding_model = defaults_data.get(\"default_embedding_model\")\n defaults.default_tools_model = defaults_data.get(\"default_tools_model\")\n\n return defaults\n\n def update_default_models(self, defaults: DefaultModels) -> DefaultModels:\n \"\"\"Update default model assignments.\"\"\"\n updates = {\n \"default_chat_model\": defaults.default_chat_model,\n \"default_transformation_model\": defaults.default_transformation_model,\n \"large_context_model\": defaults.large_context_model,\n \"default_text_to_speech_model\": defaults.default_text_to_speech_model,\n \"default_speech_to_text_model\": defaults.default_speech_to_text_model,\n \"default_embedding_model\": defaults.default_embedding_model,\n \"default_tools_model\": defaults.default_tools_model,\n }\n\n response = api_client.update_default_models(**updates)\n defaults_data = response if isinstance(response, dict) else response[0]\n\n # Update the defaults object with the response\n defaults.default_chat_model = defaults_data.get(\"default_chat_model\")\n defaults.default_transformation_model = defaults_data.get(\n \"default_transformation_model\"\n )\n defaults.large_context_model = defaults_data.get(\"large_context_model\")\n defaults.default_text_to_speech_model = defaults_data.get(\n \"default_text_to_speech_model\"\n )\n defaults.default_speech_to_text_model = defaults_data.get(\n \"default_speech_to_text_model\"\n )\n defaults.default_embedding_model = defaults_data.get(\"default_embedding_model\")\n defaults.default_tools_model = defaults_data.get(\"default_tools_model\")\n\n return defaults\n\n\n# Global service instance\nmodels_service = ModelsService()\n" }, { "path": "api/notebook_service.py", "content": "\"\"\"\nNotebook service layer using API.\n\"\"\"\n\nfrom typing import List, Optional\n\nfrom loguru import logger\n\nfrom api.client import api_client\nfrom open_notebook.domain.notebook import Notebook\n\n\nclass NotebookService:\n \"\"\"Service layer for notebook operations using API.\"\"\"\n\n def __init__(self):\n logger.info(\"Using API for notebook operations\")\n\n def get_all_notebooks(self, order_by: str = \"updated desc\") -> List[Notebook]:\n \"\"\"Get all notebooks.\"\"\"\n notebooks_data = api_client.get_notebooks(order_by=order_by)\n # Convert API response to Notebook objects\n notebooks = []\n for nb_data in notebooks_data:\n nb = Notebook(\n name=nb_data[\"name\"],\n description=nb_data[\"description\"],\n archived=nb_data[\"archived\"],\n )\n nb.id = nb_data[\"id\"]\n nb.created = nb_data[\"created\"]\n nb.updated = nb_data[\"updated\"]\n notebooks.append(nb)\n return notebooks\n\n def get_notebook(self, notebook_id: str) -> Optional[Notebook]:\n \"\"\"Get a specific notebook.\"\"\"\n response = api_client.get_notebook(notebook_id)\n nb_data = response if isinstance(response, dict) else response[0]\n nb = Notebook(\n name=nb_data[\"name\"],\n description=nb_data[\"description\"],\n archived=nb_data[\"archived\"],\n )\n nb.id = nb_data[\"id\"]\n nb.created = nb_data[\"created\"]\n nb.updated = nb_data[\"updated\"]\n return nb\n\n def create_notebook(self, name: str, description: str = \"\") -> Notebook:\n \"\"\"Create a new notebook.\"\"\"\n response = api_client.create_notebook(name, description)\n nb_data = response if isinstance(response, dict) else response[0]\n nb = Notebook(\n name=nb_data[\"name\"],\n description=nb_data[\"description\"],\n archived=nb_data[\"archived\"],\n )\n nb.id = nb_data[\"id\"]\n nb.created = nb_data[\"created\"]\n nb.updated = nb_data[\"updated\"]\n return nb\n\n def update_notebook(self, notebook: Notebook) -> Notebook:\n \"\"\"Update a notebook.\"\"\"\n updates = {\n \"name\": notebook.name,\n \"description\": notebook.description,\n \"archived\": notebook.archived,\n }\n response = api_client.update_notebook(notebook.id or \"\", **updates)\n nb_data = response if isinstance(response, dict) else response[0]\n # Update the notebook object with the response\n notebook.name = nb_data[\"name\"]\n notebook.description = nb_data[\"description\"]\n notebook.archived = nb_data[\"archived\"]\n notebook.updated = nb_data[\"updated\"]\n return notebook\n\n def delete_notebook(self, notebook: Notebook) -> bool:\n \"\"\"Delete a notebook.\"\"\"\n api_client.delete_notebook(notebook.id or \"\")\n return True\n\n\n# Global service instance\nnotebook_service = NotebookService()\n" }, { "path": "api/notes_service.py", "content": "\"\"\"\nNotes service layer using API.\n\"\"\"\n\nfrom typing import List, Optional\n\nfrom loguru import logger\n\nfrom api.client import api_client\nfrom open_notebook.domain.notebook import Note\n\n\nclass NotesService:\n \"\"\"Service layer for notes operations using API.\"\"\"\n\n def __init__(self):\n logger.info(\"Using API for notes operations\")\n\n def get_all_notes(self, notebook_id: Optional[str] = None) -> List[Note]:\n \"\"\"Get all notes with optional notebook filtering.\"\"\"\n notes_data = api_client.get_notes(notebook_id=notebook_id)\n # Convert API response to Note objects\n notes = []\n for note_data in notes_data:\n note = Note(\n title=note_data[\"title\"],\n content=note_data[\"content\"],\n note_type=note_data[\"note_type\"],\n )\n note.id = note_data[\"id\"]\n note.created = note_data[\"created\"]\n note.updated = note_data[\"updated\"]\n notes.append(note)\n return notes\n\n def get_note(self, note_id: str) -> Note:\n \"\"\"Get a specific note.\"\"\"\n note_response = api_client.get_note(note_id)\n note_data = (\n note_response if isinstance(note_response, dict) else note_response[0]\n )\n note = Note(\n title=note_data[\"title\"],\n content=note_data[\"content\"],\n note_type=note_data[\"note_type\"],\n )\n note.id = note_data[\"id\"]\n note.created = note_data[\"created\"]\n note.updated = note_data[\"updated\"]\n return note\n\n def create_note(\n self,\n content: str,\n title: Optional[str] = None,\n note_type: str = \"human\",\n notebook_id: Optional[str] = None,\n ) -> Note:\n \"\"\"Create a new note.\"\"\"\n note_response = api_client.create_note(\n content=content, title=title, note_type=note_type, notebook_id=notebook_id\n )\n note_data = (\n note_response if isinstance(note_response, dict) else note_response[0]\n )\n note = Note(\n title=note_data[\"title\"],\n content=note_data[\"content\"],\n note_type=note_data[\"note_type\"],\n )\n note.id = note_data[\"id\"]\n note.created = note_data[\"created\"]\n note.updated = note_data[\"updated\"]\n return note\n\n def update_note(self, note: Note) -> Note:\n \"\"\"Update a note.\"\"\"\n updates = {\n \"title\": note.title,\n \"content\": note.content,\n \"note_type\": note.note_type,\n }\n note_response = api_client.update_note(note.id or \"\", **updates)\n note_data = (\n note_response if isinstance(note_response, dict) else note_response[0]\n )\n\n # Update the note object with the response\n note.title = note_data[\"title\"]\n note.content = note_data[\"content\"]\n note.note_type = note_data[\"note_type\"]\n note.updated = note_data[\"updated\"]\n\n return note\n\n def delete_note(self, note_id: str) -> bool:\n \"\"\"Delete a note.\"\"\"\n api_client.delete_note(note_id)\n return True\n\n\n# Global service instance\nnotes_service = NotesService()\n" }, { "path": "api/podcast_api_service.py", "content": "\"\"\"\nPodcast service layer using API client.\nThis replaces direct httpx calls in the Streamlit pages.\n\"\"\"\n\nfrom typing import Any, Dict, List\n\nfrom loguru import logger\n\nfrom api.client import api_client\n\n\nclass PodcastAPIService:\n \"\"\"Service layer for podcast operations using API client.\"\"\"\n\n def __init__(self):\n logger.info(\"Using API client for podcast operations\")\n\n # Episode methods\n def get_episodes(self) -> List[Dict[Any, Any]]:\n \"\"\"Get all podcast episodes.\"\"\"\n result = api_client._make_request(\"GET\", \"/api/podcasts/episodes\")\n return result if isinstance(result, list) else [result]\n\n def delete_episode(self, episode_id: str) -> bool:\n \"\"\"Delete a podcast episode.\"\"\"\n try:\n api_client._make_request(\"DELETE\", f\"/api/podcasts/episodes/{episode_id}\")\n return True\n except Exception as e:\n logger.error(f\"Failed to delete episode: {e}\")\n return False\n\n # Episode Profile methods\n def get_episode_profiles(self) -> List[Dict]:\n \"\"\"Get all episode profiles.\"\"\"\n return api_client.get_episode_profiles()\n\n def create_episode_profile(self, profile_data: Dict) -> bool:\n \"\"\"Create a new episode profile.\"\"\"\n try:\n api_client.create_episode_profile(**profile_data)\n return True\n except Exception as e:\n logger.error(f\"Failed to create episode profile: {e}\")\n return False\n\n def update_episode_profile(self, profile_id: str, profile_data: Dict) -> bool:\n \"\"\"Update an episode profile.\"\"\"\n try:\n api_client.update_episode_profile(profile_id, **profile_data)\n return True\n except Exception as e:\n logger.error(f\"Failed to update episode profile: {e}\")\n return False\n\n def delete_episode_profile(self, profile_id: str) -> bool:\n \"\"\"Delete an episode profile.\"\"\"\n try:\n api_client.delete_episode_profile(profile_id)\n return True\n except Exception as e:\n logger.error(f\"Failed to delete episode profile: {e}\")\n return False\n\n def duplicate_episode_profile(self, profile_id: str) -> bool:\n \"\"\"Duplicate an episode profile.\"\"\"\n try:\n api_client._make_request(\n \"POST\", f\"/api/episode-profiles/{profile_id}/duplicate\"\n )\n return True\n except Exception as e:\n logger.error(f\"Failed to duplicate episode profile: {e}\")\n return False\n\n # Speaker Profile methods\n def get_speaker_profiles(self) -> List[Dict[Any, Any]]:\n \"\"\"Get all speaker profiles.\"\"\"\n result = api_client._make_request(\"GET\", \"/api/speaker-profiles\")\n return result if isinstance(result, list) else [result]\n\n def create_speaker_profile(self, profile_data: Dict) -> bool:\n \"\"\"Create a new speaker profile.\"\"\"\n try:\n api_client._make_request(\"POST\", \"/api/speaker-profiles\", json=profile_data)\n return True\n except Exception as e:\n logger.error(f\"Failed to create speaker profile: {e}\")\n return False\n\n def update_speaker_profile(self, profile_id: str, profile_data: Dict) -> bool:\n \"\"\"Update a speaker profile.\"\"\"\n try:\n api_client._make_request(\n \"PUT\", f\"/api/speaker-profiles/{profile_id}\", json=profile_data\n )\n return True\n except Exception as e:\n logger.error(f\"Failed to update speaker profile: {e}\")\n return False\n\n def delete_speaker_profile(self, profile_id: str) -> bool:\n \"\"\"Delete a speaker profile.\"\"\"\n try:\n api_client._make_request(\"DELETE\", f\"/api/speaker-profiles/{profile_id}\")\n return True\n except Exception as e:\n logger.error(f\"Failed to delete speaker profile: {e}\")\n return False\n\n def duplicate_speaker_profile(self, profile_id: str) -> bool:\n \"\"\"Duplicate a speaker profile.\"\"\"\n try:\n api_client._make_request(\n \"POST\", f\"/api/speaker-profiles/{profile_id}/duplicate\"\n )\n return True\n except Exception as e:\n logger.error(f\"Failed to duplicate speaker profile: {e}\")\n return False\n\n\n# Global service instance\npodcast_api_service = PodcastAPIService()\n" }, { "path": "api/podcast_service.py", "content": "from typing import Any, Dict, Optional\n\nfrom fastapi import HTTPException\nfrom loguru import logger\nfrom pydantic import BaseModel\nfrom surreal_commands import get_command_status, submit_command\n\nfrom open_notebook.domain.notebook import Notebook\nfrom open_notebook.podcasts.models import EpisodeProfile, PodcastEpisode, SpeakerProfile\n\n\nclass PodcastGenerationRequest(BaseModel):\n \"\"\"Request model for podcast generation\"\"\"\n\n episode_profile: str\n speaker_profile: str\n episode_name: str\n content: Optional[str] = None\n notebook_id: Optional[str] = None\n briefing_suffix: Optional[str] = None\n\n\nclass PodcastGenerationResponse(BaseModel):\n \"\"\"Response model for podcast generation\"\"\"\n\n job_id: str\n status: str\n message: str\n episode_profile: str\n episode_name: str\n\n\nclass PodcastService:\n \"\"\"Service layer for podcast operations\"\"\"\n\n @staticmethod\n async def submit_generation_job(\n episode_profile_name: str,\n speaker_profile_name: str,\n episode_name: str,\n notebook_id: Optional[str] = None,\n content: Optional[str] = None,\n briefing_suffix: Optional[str] = None,\n ) -> str:\n \"\"\"Submit a podcast generation job for background processing\"\"\"\n try:\n # Validate episode profile exists\n episode_profile = await EpisodeProfile.get_by_name(episode_profile_name)\n if not episode_profile:\n raise ValueError(f\"Episode profile '{episode_profile_name}' not found\")\n\n # Validate speaker profile exists\n speaker_profile = await SpeakerProfile.get_by_name(speaker_profile_name)\n if not speaker_profile:\n raise ValueError(f\"Speaker profile '{speaker_profile_name}' not found\")\n\n # Get content from notebook if not provided directly\n if not content and notebook_id:\n try:\n notebook = await Notebook.get(notebook_id)\n # Get notebook context (this may need to be adjusted based on actual Notebook implementation)\n content = (\n await notebook.get_context()\n if hasattr(notebook, \"get_context\")\n else str(notebook)\n )\n except Exception as e:\n logger.warning(\n f\"Failed to get notebook content, using notebook_id as content: {e}\"\n )\n content = f\"Notebook ID: {notebook_id}\"\n\n if not content:\n raise ValueError(\n \"Content is required - provide either content or notebook_id\"\n )\n\n # Prepare command arguments\n command_args = {\n \"episode_profile\": episode_profile_name,\n \"speaker_profile\": speaker_profile_name,\n \"episode_name\": episode_name,\n \"content\": str(content),\n \"briefing_suffix\": briefing_suffix,\n }\n\n # Ensure command modules are imported before submitting\n # This is needed because submit_command validates against local registry\n try:\n import commands.podcast_commands # noqa: F401\n except ImportError as import_err:\n logger.error(f\"Failed to import podcast commands: {import_err}\")\n raise ValueError(\"Podcast commands not available\")\n\n # Submit command to surreal-commands\n job_id = submit_command(\"open_notebook\", \"generate_podcast\", command_args)\n\n # Convert RecordID to string if needed\n if not job_id:\n raise ValueError(\"Failed to get job_id from submit_command\")\n job_id_str = str(job_id)\n logger.info(\n f\"Submitted podcast generation job: {job_id_str} for episode '{episode_name}'\"\n )\n return job_id_str\n\n except Exception as e:\n logger.error(f\"Failed to submit podcast generation job: {e}\")\n raise HTTPException(\n status_code=500,\n detail=f\"Failed to submit podcast generation job: {str(e)}\",\n )\n\n @staticmethod\n async def get_job_status(job_id: str) -> Dict[str, Any]:\n \"\"\"Get status of a podcast generation job\"\"\"\n try:\n status = await get_command_status(job_id)\n return {\n \"job_id\": job_id,\n \"status\": status.status if status else \"unknown\",\n \"result\": status.result if status else None,\n \"error_message\": getattr(status, \"error_message\", None)\n if status\n else None,\n \"created\": str(status.created)\n if status and hasattr(status, \"created\") and status.created\n else None,\n \"updated\": str(status.updated)\n if status and hasattr(status, \"updated\") and status.updated\n else None,\n \"progress\": getattr(status, \"progress\", None) if status else None,\n }\n except Exception as e:\n logger.error(f\"Failed to get podcast job status: {e}\")\n raise HTTPException(\n status_code=500, detail=f\"Failed to get job status: {str(e)}\"\n )\n\n @staticmethod\n async def list_episodes() -> list:\n \"\"\"List all podcast episodes\"\"\"\n try:\n episodes = await PodcastEpisode.get_all(order_by=\"created desc\")\n return episodes\n except Exception as e:\n logger.error(f\"Failed to list podcast episodes: {e}\")\n raise HTTPException(\n status_code=500, detail=f\"Failed to list episodes: {str(e)}\"\n )\n\n @staticmethod\n async def get_episode(episode_id: str) -> PodcastEpisode:\n \"\"\"Get a specific podcast episode\"\"\"\n try:\n episode = await PodcastEpisode.get(episode_id)\n return episode\n except Exception as e:\n logger.error(f\"Failed to get podcast episode {episode_id}: {e}\")\n raise HTTPException(status_code=404, detail=f\"Episode not found: {str(e)}\")\n\n\nclass DefaultProfiles:\n \"\"\"Utility class for creating default profiles (if needed beyond migration data)\"\"\"\n\n @staticmethod\n async def create_default_episode_profiles():\n \"\"\"Create default episode profiles if they don't exist\"\"\"\n try:\n # Check if profiles already exist\n existing = await EpisodeProfile.get_all()\n if existing:\n logger.info(f\"Episode profiles already exist: {len(existing)} found\")\n return existing\n\n # This would create profiles, but since we have migration data,\n # this is mainly for future extensibility\n logger.info(\n \"Default episode profiles should be created via database migration\"\n )\n return []\n\n except Exception as e:\n logger.error(f\"Failed to create default episode profiles: {e}\")\n raise\n\n @staticmethod\n async def create_default_speaker_profiles():\n \"\"\"Create default speaker profiles if they don't exist\"\"\"\n try:\n # Check if profiles already exist\n existing = await SpeakerProfile.get_all()\n if existing:\n logger.info(f\"Speaker profiles already exist: {len(existing)} found\")\n return existing\n\n # This would create profiles, but since we have migration data,\n # this is mainly for future extensibility\n logger.info(\n \"Default speaker profiles should be created via database migration\"\n )\n return []\n\n except Exception as e:\n logger.error(f\"Failed to create default speaker profiles: {e}\")\n raise\n" }, { "path": "api/routers/__init__.py", "content": "" }, { "path": "api/routers/auth.py", "content": "\"\"\"\nAuthentication router for Open Notebook API.\nProvides endpoints to check authentication status.\n\"\"\"\n\nfrom fastapi import APIRouter\n\nfrom open_notebook.utils.encryption import get_secret_from_env\n\nrouter = APIRouter(prefix=\"/auth\", tags=[\"auth\"])\n\n\n@router.get(\"/status\")\nasync def get_auth_status():\n \"\"\"\n Check if authentication is enabled.\n Returns whether a password is required to access the API.\n Supports Docker secrets via OPEN_NOTEBOOK_PASSWORD_FILE.\n \"\"\"\n auth_enabled = bool(get_secret_from_env(\"OPEN_NOTEBOOK_PASSWORD\"))\n\n return {\n \"auth_enabled\": auth_enabled,\n \"message\": \"Authentication is required\"\n if auth_enabled\n else \"Authentication is disabled\",\n }" }, { "path": "api/routers/chat.py", "content": "import asyncio\nimport traceback\nfrom typing import Any, Dict, List, Optional\n\nfrom fastapi import APIRouter, HTTPException, Query\nfrom langchain_core.runnables import RunnableConfig\nfrom loguru import logger\nfrom pydantic import BaseModel, Field\n\nfrom open_notebook.database.repository import ensure_record_id, repo_query\nfrom open_notebook.domain.notebook import ChatSession, Note, Notebook, Source\nfrom open_notebook.exceptions import (\n NotFoundError,\n)\nfrom open_notebook.graphs.chat import graph as chat_graph\nfrom open_notebook.utils.graph_utils import get_session_message_count\n\nrouter = APIRouter()\n\n\n# Request/Response models\nclass CreateSessionRequest(BaseModel):\n notebook_id: str = Field(..., description=\"Notebook ID to create session for\")\n title: Optional[str] = Field(None, description=\"Optional session title\")\n model_override: Optional[str] = Field(\n None, description=\"Optional model override for this session\"\n )\n\n\nclass UpdateSessionRequest(BaseModel):\n title: Optional[str] = Field(None, description=\"New session title\")\n model_override: Optional[str] = Field(\n None, description=\"Model override for this session\"\n )\n\n\nclass ChatMessage(BaseModel):\n id: str = Field(..., description=\"Message ID\")\n type: str = Field(..., description=\"Message type (human|ai)\")\n content: str = Field(..., description=\"Message content\")\n timestamp: Optional[str] = Field(None, description=\"Message timestamp\")\n\n\nclass ChatSessionResponse(BaseModel):\n id: str = Field(..., description=\"Session ID\")\n title: str = Field(..., description=\"Session title\")\n notebook_id: Optional[str] = Field(None, description=\"Notebook ID\")\n created: str = Field(..., description=\"Creation timestamp\")\n updated: str = Field(..., description=\"Last update timestamp\")\n message_count: Optional[int] = Field(\n None, description=\"Number of messages in session\"\n )\n model_override: Optional[str] = Field(\n None, description=\"Model override for this session\"\n )\n\n\nclass ChatSessionWithMessagesResponse(ChatSessionResponse):\n messages: List[ChatMessage] = Field(\n default_factory=list, description=\"Session messages\"\n )\n\n\nclass ExecuteChatRequest(BaseModel):\n session_id: str = Field(..., description=\"Chat session ID\")\n message: str = Field(..., description=\"User message content\")\n context: Dict[str, Any] = Field(\n ..., description=\"Chat context with sources and notes\"\n )\n model_override: Optional[str] = Field(\n None, description=\"Optional model override for this message\"\n )\n\n\nclass ExecuteChatResponse(BaseModel):\n session_id: str = Field(..., description=\"Session ID\")\n messages: List[ChatMessage] = Field(..., description=\"Updated message list\")\n\n\nclass BuildContextRequest(BaseModel):\n notebook_id: str = Field(..., description=\"Notebook ID\")\n context_config: Dict[str, Any] = Field(..., description=\"Context configuration\")\n\n\nclass BuildContextResponse(BaseModel):\n context: Dict[str, Any] = Field(..., description=\"Built context data\")\n token_count: int = Field(..., description=\"Estimated token count\")\n char_count: int = Field(..., description=\"Character count\")\n\n\nclass SuccessResponse(BaseModel):\n success: bool = Field(True, description=\"Operation success status\")\n message: str = Field(..., description=\"Success message\")\n\n\n@router.get(\"/chat/sessions\", response_model=List[ChatSessionResponse])\nasync def get_sessions(notebook_id: str = Query(..., description=\"Notebook ID\")):\n \"\"\"Get all chat sessions for a notebook.\"\"\"\n try:\n # Get notebook to verify it exists\n notebook = await Notebook.get(notebook_id)\n if not notebook:\n raise HTTPException(status_code=404, detail=\"Notebook not found\")\n\n # Get sessions for this notebook\n sessions_list = await notebook.get_chat_sessions()\n\n results = []\n for session in sessions_list:\n session_id = str(session.id)\n\n # Get message count from LangGraph state\n msg_count = await get_session_message_count(chat_graph, session_id)\n\n results.append(\n ChatSessionResponse(\n id=session.id or \"\",\n title=session.title or \"Untitled Session\",\n notebook_id=notebook_id,\n created=str(session.created),\n updated=str(session.updated),\n message_count=msg_count,\n model_override=getattr(session, \"model_override\", None),\n )\n )\n\n return results\n except NotFoundError:\n raise HTTPException(status_code=404, detail=\"Notebook not found\")\n except Exception as e:\n logger.error(f\"Error fetching chat sessions: {str(e)}\")\n raise HTTPException(\n status_code=500, detail=f\"Error fetching chat sessions: {str(e)}\"\n )\n\n\n@router.post(\"/chat/sessions\", response_model=ChatSessionResponse)\nasync def create_session(request: CreateSessionRequest):\n \"\"\"Create a new chat session.\"\"\"\n try:\n # Verify notebook exists\n notebook = await Notebook.get(request.notebook_id)\n if not notebook:\n raise HTTPException(status_code=404, detail=\"Notebook not found\")\n\n # Create new session\n session = ChatSession(\n title=request.title\n or f\"Chat Session {asyncio.get_event_loop().time():.0f}\",\n model_override=request.model_override,\n )\n await session.save()\n\n # Relate session to notebook\n await session.relate_to_notebook(request.notebook_id)\n\n return ChatSessionResponse(\n id=session.id or \"\",\n title=session.title or \"\",\n notebook_id=request.notebook_id,\n created=str(session.created),\n updated=str(session.updated),\n message_count=0,\n model_override=session.model_override,\n )\n except NotFoundError:\n raise HTTPException(status_code=404, detail=\"Notebook not found\")\n except Exception as e:\n logger.error(f\"Error creating chat session: {str(e)}\")\n raise HTTPException(\n status_code=500, detail=f\"Error creating chat session: {str(e)}\"\n )\n\n\n@router.get(\n \"/chat/sessions/{session_id}\", response_model=ChatSessionWithMessagesResponse\n)\nasync def get_session(session_id: str):\n \"\"\"Get a specific session with its messages.\"\"\"\n try:\n # Get session\n # Ensure session_id has proper table prefix\n full_session_id = (\n session_id\n if session_id.startswith(\"chat_session:\")\n else f\"chat_session:{session_id}\"\n )\n session = await ChatSession.get(full_session_id)\n if not session:\n raise HTTPException(status_code=404, detail=\"Session not found\")\n\n # Get session state from LangGraph to retrieve messages\n # Use sync get_state() in a thread since SqliteSaver doesn't support async\n thread_state = await asyncio.to_thread(\n chat_graph.get_state,\n config=RunnableConfig(configurable={\"thread_id\": full_session_id}),\n )\n\n # Extract messages from state\n messages: list[ChatMessage] = []\n if thread_state and thread_state.values and \"messages\" in thread_state.values:\n for msg in thread_state.values[\"messages\"]:\n messages.append(\n ChatMessage(\n id=getattr(msg, \"id\", f\"msg_{len(messages)}\"),\n type=msg.type if hasattr(msg, \"type\") else \"unknown\",\n content=msg.content if hasattr(msg, \"content\") else str(msg),\n timestamp=None, # LangChain messages don't have timestamps by default\n )\n )\n\n # Find notebook_id (we need to query the relationship)\n # Ensure session_id has proper table prefix\n full_session_id = (\n session_id\n if session_id.startswith(\"chat_session:\")\n else f\"chat_session:{session_id}\"\n )\n\n notebook_query = await repo_query(\n \"SELECT out FROM refers_to WHERE in = $session_id\",\n {\"session_id\": ensure_record_id(full_session_id)},\n )\n\n notebook_id = notebook_query[0][\"out\"] if notebook_query else None\n\n if not notebook_id:\n # This might be an old session created before API migration\n logger.warning(\n f\"No notebook relationship found for session {session_id} - may be an orphaned session\"\n )\n\n return ChatSessionWithMessagesResponse(\n id=session.id or \"\",\n title=session.title or \"Untitled Session\",\n notebook_id=notebook_id,\n created=str(session.created),\n updated=str(session.updated),\n message_count=len(messages),\n messages=messages,\n model_override=getattr(session, \"model_override\", None),\n )\n except NotFoundError:\n raise HTTPException(status_code=404, detail=\"Session not found\")\n except Exception as e:\n logger.error(f\"Error fetching session: {str(e)}\")\n raise HTTPException(status_code=500, detail=f\"Error fetching session: {str(e)}\")\n\n\n@router.put(\"/chat/sessions/{session_id}\", response_model=ChatSessionResponse)\nasync def update_session(session_id: str, request: UpdateSessionRequest):\n \"\"\"Update session title.\"\"\"\n try:\n # Ensure session_id has proper table prefix\n full_session_id = (\n session_id\n if session_id.startswith(\"chat_session:\")\n else f\"chat_session:{session_id}\"\n )\n session = await ChatSession.get(full_session_id)\n if not session:\n raise HTTPException(status_code=404, detail=\"Session not found\")\n\n update_data = request.model_dump(exclude_unset=True)\n\n if \"title\" in update_data:\n session.title = update_data[\"title\"]\n\n if \"model_override\" in update_data:\n session.model_override = update_data[\"model_override\"]\n\n await session.save()\n\n # Find notebook_id\n # Ensure session_id has proper table prefix\n full_session_id = (\n session_id\n if session_id.startswith(\"chat_session:\")\n else f\"chat_session:{session_id}\"\n )\n notebook_query = await repo_query(\n \"SELECT out FROM refers_to WHERE in = $session_id\",\n {\"session_id\": ensure_record_id(full_session_id)},\n )\n notebook_id = notebook_query[0][\"out\"] if notebook_query else None\n\n # Get message count from LangGraph state\n msg_count = await get_session_message_count(chat_graph, full_session_id)\n\n return ChatSessionResponse(\n id=session.id or \"\",\n title=session.title or \"\",\n notebook_id=notebook_id,\n created=str(session.created),\n updated=str(session.updated),\n message_count=msg_count,\n model_override=session.model_override,\n )\n except NotFoundError:\n raise HTTPException(status_code=404, detail=\"Session not found\")\n except Exception as e:\n logger.error(f\"Error updating session: {str(e)}\")\n raise HTTPException(status_code=500, detail=f\"Error updating session: {str(e)}\")\n\n\n@router.delete(\"/chat/sessions/{session_id}\", response_model=SuccessResponse)\nasync def delete_session(session_id: str):\n \"\"\"Delete a chat session.\"\"\"\n try:\n # Ensure session_id has proper table prefix\n full_session_id = (\n session_id\n if session_id.startswith(\"chat_session:\")\n else f\"chat_session:{session_id}\"\n )\n session = await ChatSession.get(full_session_id)\n if not session:\n raise HTTPException(status_code=404, detail=\"Session not found\")\n\n await session.delete()\n\n return SuccessResponse(success=True, message=\"Session deleted successfully\")\n except NotFoundError:\n raise HTTPException(status_code=404, detail=\"Session not found\")\n except Exception as e:\n logger.error(f\"Error deleting session: {str(e)}\")\n raise HTTPException(status_code=500, detail=f\"Error deleting session: {str(e)}\")\n\n\n@router.post(\"/chat/execute\", response_model=ExecuteChatResponse)\nasync def execute_chat(request: ExecuteChatRequest):\n \"\"\"Execute a chat request and get AI response.\"\"\"\n try:\n # Verify session exists\n # Ensure session_id has proper table prefix\n full_session_id = (\n request.session_id\n if request.session_id.startswith(\"chat_session:\")\n else f\"chat_session:{request.session_id}\"\n )\n session = await ChatSession.get(full_session_id)\n if not session:\n raise HTTPException(status_code=404, detail=\"Session not found\")\n\n # Determine model override (per-request override takes precedence over session-level)\n model_override = (\n request.model_override\n if request.model_override is not None\n else getattr(session, \"model_override\", None)\n )\n\n # Get current state\n # Use sync get_state() in a thread since SqliteSaver doesn't support async\n current_state = await asyncio.to_thread(\n chat_graph.get_state,\n config=RunnableConfig(configurable={\"thread_id\": full_session_id}),\n )\n\n # Prepare state for execution\n state_values = current_state.values if current_state else {}\n state_values[\"messages\"] = state_values.get(\"messages\", [])\n state_values[\"context\"] = request.context\n state_values[\"model_override\"] = model_override\n\n # Add user message to state\n from langchain_core.messages import HumanMessage\n\n user_message = HumanMessage(content=request.message)\n state_values[\"messages\"].append(user_message)\n\n # Execute chat graph\n result = chat_graph.invoke(\n input=state_values, # type: ignore[arg-type]\n config=RunnableConfig(\n configurable={\n \"thread_id\": full_session_id,\n \"model_id\": model_override,\n }\n ),\n )\n\n # Update session timestamp\n await session.save()\n\n # Convert messages to response format\n messages: list[ChatMessage] = []\n for msg in result.get(\"messages\", []):\n messages.append(\n ChatMessage(\n id=getattr(msg, \"id\", f\"msg_{len(messages)}\"),\n type=msg.type if hasattr(msg, \"type\") else \"unknown\",\n content=msg.content if hasattr(msg, \"content\") else str(msg),\n timestamp=None,\n )\n )\n\n return ExecuteChatResponse(session_id=request.session_id, messages=messages)\n except NotFoundError:\n raise HTTPException(status_code=404, detail=\"Session not found\")\n except Exception as e:\n # Log detailed error with context for debugging\n logger.error(\n f\"Error executing chat: {str(e)}\\n\"\n f\" Session ID: {request.session_id}\\n\"\n f\" Model override: {request.model_override}\\n\"\n f\" Traceback:\\n{traceback.format_exc()}\"\n )\n raise HTTPException(status_code=500, detail=f\"Error executing chat: {str(e)}\")\n\n\n@router.post(\"/chat/context\", response_model=BuildContextResponse)\nasync def build_context(request: BuildContextRequest):\n \"\"\"Build context for a notebook based on context configuration.\"\"\"\n try:\n # Verify notebook exists\n notebook = await Notebook.get(request.notebook_id)\n if not notebook:\n raise HTTPException(status_code=404, detail=\"Notebook not found\")\n\n context_data: dict[str, list[dict[str, str]]] = {\"sources\": [], \"notes\": []}\n total_content = \"\"\n\n # Process context configuration if provided\n if request.context_config:\n # Process sources\n for source_id, status in request.context_config.get(\"sources\", {}).items():\n if \"not in\" in status:\n continue\n\n try:\n # Add table prefix if not present\n full_source_id = (\n source_id\n if source_id.startswith(\"source:\")\n else f\"source:{source_id}\"\n )\n\n try:\n source = await Source.get(full_source_id)\n except Exception:\n continue\n\n if \"insights\" in status:\n source_context = await source.get_context(context_size=\"short\")\n context_data[\"sources\"].append(source_context)\n total_content += str(source_context)\n elif \"full content\" in status:\n source_context = await source.get_context(context_size=\"long\")\n context_data[\"sources\"].append(source_context)\n total_content += str(source_context)\n except Exception as e:\n logger.warning(f\"Error processing source {source_id}: {str(e)}\")\n continue\n\n # Process notes\n for note_id, status in request.context_config.get(\"notes\", {}).items():\n if \"not in\" in status:\n continue\n\n try:\n # Add table prefix if not present\n full_note_id = (\n note_id if note_id.startswith(\"note:\") else f\"note:{note_id}\"\n )\n note = await Note.get(full_note_id)\n if not note:\n continue\n\n if \"full content\" in status:\n note_context = note.get_context(context_size=\"long\")\n context_data[\"notes\"].append(note_context)\n total_content += str(note_context)\n except Exception as e:\n logger.warning(f\"Error processing note {note_id}: {str(e)}\")\n continue\n else:\n # Default behavior - include all sources and notes with short context\n sources = await notebook.get_sources()\n for source in sources:\n try:\n source_context = await source.get_context(context_size=\"short\")\n context_data[\"sources\"].append(source_context)\n total_content += str(source_context)\n except Exception as e:\n logger.warning(f\"Error processing source {source.id}: {str(e)}\")\n continue\n\n notes = await notebook.get_notes()\n for note in notes:\n try:\n note_context = note.get_context(context_size=\"short\")\n context_data[\"notes\"].append(note_context)\n total_content += str(note_context)\n except Exception as e:\n logger.warning(f\"Error processing note {note.id}: {str(e)}\")\n continue\n\n # Calculate character and token counts\n char_count = len(total_content)\n # Use token count utility if available\n try:\n from open_notebook.utils import token_count\n\n estimated_tokens = token_count(total_content) if total_content else 0\n except ImportError:\n # Fallback to simple estimation\n estimated_tokens = char_count // 4\n\n return BuildContextResponse(\n context=context_data, token_count=estimated_tokens, char_count=char_count\n )\n except HTTPException:\n raise\n except Exception as e:\n logger.error(f\"Error building context: {str(e)}\")\n raise HTTPException(status_code=500, detail=f\"Error building context: {str(e)}\")\n" }, { "path": "api/routers/commands.py", "content": "from typing import Any, Dict, List, Optional\n\nfrom fastapi import APIRouter, HTTPException, Query\nfrom loguru import logger\nfrom pydantic import BaseModel, Field\nfrom surreal_commands import registry\n\nfrom api.command_service import CommandService\n\nrouter = APIRouter()\n\n\nclass CommandExecutionRequest(BaseModel):\n command: str = Field(\n ..., description=\"Command function name (e.g., 'process_text')\"\n )\n app: str = Field(..., description=\"Application name (e.g., 'open_notebook')\")\n input: Dict[str, Any] = Field(..., description=\"Arguments to pass to the command\")\n\n\nclass CommandJobResponse(BaseModel):\n job_id: str\n status: str\n message: str\n\n\nclass CommandJobStatusResponse(BaseModel):\n job_id: str\n status: str\n result: Optional[Dict[str, Any]] = None\n error_message: Optional[str] = None\n created: Optional[str] = None\n updated: Optional[str] = None\n progress: Optional[Dict[str, Any]] = None\n\n\n@router.post(\"/commands/jobs\", response_model=CommandJobResponse)\nasync def execute_command(request: CommandExecutionRequest):\n \"\"\"\n Submit a command for background processing.\n Returns immediately with job ID for status tracking.\n\n Example request:\n {\n \"command\": \"process_text\",\n \"app\": \"open_notebook\",\n \"input\": {\n \"text\": \"Hello world\",\n \"operation\": \"uppercase\"\n }\n }\n \"\"\"\n try:\n # Submit command using app name (not module name)\n job_id = await CommandService.submit_command_job(\n module_name=request.app, # This should be \"open_notebook\"\n command_name=request.command,\n command_args=request.input,\n )\n\n return CommandJobResponse(\n job_id=job_id,\n status=\"submitted\",\n message=f\"Command '{request.command}' submitted successfully\",\n )\n\n except Exception as e:\n logger.error(f\"Error submitting command: {str(e)}\")\n raise HTTPException(\n status_code=500, detail=\"Failed to submit command\"\n )\n\n\n@router.get(\"/commands/jobs/{job_id}\", response_model=CommandJobStatusResponse)\nasync def get_command_job_status(job_id: str):\n \"\"\"Get the status of a specific command job\"\"\"\n try:\n status_data = await CommandService.get_command_status(job_id)\n return CommandJobStatusResponse(**status_data)\n\n except Exception as e:\n logger.error(f\"Error fetching job status: {str(e)}\")\n raise HTTPException(\n status_code=500, detail=\"Failed to fetch job status\"\n )\n\n\n@router.get(\"/commands/jobs\", response_model=List[Dict[str, Any]])\nasync def list_command_jobs(\n command_filter: Optional[str] = Query(None, description=\"Filter by command name\"),\n status_filter: Optional[str] = Query(None, description=\"Filter by status\"),\n limit: int = Query(50, description=\"Maximum number of jobs to return\"),\n):\n \"\"\"List command jobs with optional filtering\"\"\"\n try:\n jobs = await CommandService.list_command_jobs(\n command_filter=command_filter, status_filter=status_filter, limit=limit\n )\n return jobs\n\n except Exception as e:\n logger.error(f\"Error listing command jobs: {str(e)}\")\n raise HTTPException(\n status_code=500, detail=\"Failed to list command jobs\"\n )\n\n\n@router.delete(\"/commands/jobs/{job_id}\")\nasync def cancel_command_job(job_id: str):\n \"\"\"Cancel a running command job\"\"\"\n try:\n success = await CommandService.cancel_command_job(job_id)\n return {\"job_id\": job_id, \"cancelled\": success}\n\n except Exception as e:\n logger.error(f\"Error cancelling command job: {str(e)}\")\n raise HTTPException(\n status_code=500, detail=\"Failed to cancel command job\"\n )\n\n\n@router.get(\"/commands/registry/debug\")\nasync def debug_registry():\n \"\"\"Debug endpoint to see what commands are registered\"\"\"\n try:\n # Get all registered commands\n all_items = registry.get_all_commands()\n\n # Create JSON-serializable data\n command_items = []\n for item in all_items:\n try:\n command_items.append(\n {\n \"app_id\": item.app_id,\n \"name\": item.name,\n \"full_id\": f\"{item.app_id}.{item.name}\",\n }\n )\n except Exception as item_error:\n logger.error(f\"Error processing item: {item_error}\")\n\n # Get the basic command structure\n try:\n commands_dict: dict[str, list[str]] = {}\n for item in all_items:\n if item.app_id not in commands_dict:\n commands_dict[item.app_id] = []\n commands_dict[item.app_id].append(item.name)\n except Exception:\n commands_dict = {}\n\n return {\n \"total_commands\": len(all_items),\n \"commands_by_app\": commands_dict,\n \"command_items\": command_items,\n }\n\n except Exception as e:\n logger.error(f\"Error debugging registry: {str(e)}\")\n return {\n \"error\": str(e),\n \"total_commands\": 0,\n \"commands_by_app\": {},\n \"command_items\": [],\n }\n" }, { "path": "api/routers/config.py", "content": "import asyncio\nimport os\nimport time\nimport tomllib\nfrom pathlib import Path\nfrom typing import Optional\n\nfrom fastapi import APIRouter, Request\nfrom loguru import logger\n\nfrom open_notebook.database.repository import repo_query\nfrom open_notebook.utils.version_utils import (\n compare_versions,\n get_version_from_github_async,\n)\n\nrouter = APIRouter()\n\n# In-memory cache for version check results\n_version_cache: dict = {\n \"latest_version\": None,\n \"has_update\": False,\n \"timestamp\": 0,\n \"check_failed\": False,\n}\n\n# Cache TTL in seconds (24 hours)\nVERSION_CACHE_TTL = 24 * 60 * 60\n\n\ndef get_version() -> str:\n \"\"\"Read version from pyproject.toml\"\"\"\n try:\n pyproject_path = Path(__file__).parent.parent.parent / \"pyproject.toml\"\n with open(pyproject_path, \"rb\") as f:\n pyproject = tomllib.load(f)\n return pyproject.get(\"project\", {}).get(\"version\", \"unknown\")\n except Exception as e:\n logger.warning(f\"Could not read version from pyproject.toml: {e}\")\n return \"unknown\"\n\n\nasync def get_latest_version_cached(current_version: str) -> tuple[Optional[str], bool]:\n \"\"\"\n Check for the latest version from GitHub with caching.\n\n Returns:\n tuple: (latest_version, has_update)\n - latest_version: str or None if check failed\n - has_update: bool indicating if update is available\n \"\"\"\n global _version_cache\n\n # Check if cache is still valid (within TTL)\n cache_age = time.time() - _version_cache[\"timestamp\"]\n if _version_cache[\"timestamp\"] > 0 and cache_age < VERSION_CACHE_TTL:\n logger.debug(f\"Using cached version check result (age: {cache_age:.0f}s)\")\n return _version_cache[\"latest_version\"], _version_cache[\"has_update\"]\n\n # Cache expired or not yet set\n if _version_cache[\"timestamp\"] > 0:\n logger.info(f\"Version cache expired (age: {cache_age:.0f}s), refreshing...\")\n\n # Perform version check with strict error handling\n try:\n logger.info(\"Checking for latest version from GitHub...\")\n\n # Fetch latest version from GitHub with 10-second timeout\n latest_version = await get_version_from_github_async(\n \"https://github.com/lfnovo/open-notebook\", \"main\"\n )\n\n logger.info(\n f\"Latest version from GitHub: {latest_version}, Current version: {current_version}\"\n )\n\n # Compare versions\n has_update = compare_versions(current_version, latest_version) < 0\n\n # Cache the result\n _version_cache[\"latest_version\"] = latest_version\n _version_cache[\"has_update\"] = has_update\n _version_cache[\"timestamp\"] = time.time()\n _version_cache[\"check_failed\"] = False\n\n logger.info(f\"Version check complete. Update available: {has_update}\")\n\n return latest_version, has_update\n\n except Exception as e:\n logger.warning(f\"Version check failed: {e}\")\n\n # Cache the failure to avoid repeated attempts\n _version_cache[\"latest_version\"] = None\n _version_cache[\"has_update\"] = False\n _version_cache[\"timestamp\"] = time.time()\n _version_cache[\"check_failed\"] = True\n\n return None, False\n\n\nasync def check_database_health() -> dict:\n \"\"\"\n Check if database is reachable using a lightweight query.\n\n Returns:\n dict with 'status' (\"online\" | \"offline\") and optional 'error'\n \"\"\"\n try:\n # 2-second timeout for database health check\n result = await asyncio.wait_for(repo_query(\"RETURN 1\"), timeout=2.0)\n if result:\n return {\"status\": \"online\"}\n return {\"status\": \"offline\", \"error\": \"Empty result\"}\n except asyncio.TimeoutError:\n logger.warning(\"Database health check timed out after 2 seconds\")\n return {\"status\": \"offline\", \"error\": \"Health check timeout\"}\n except Exception as e:\n logger.warning(f\"Database health check failed: {e}\")\n return {\"status\": \"offline\", \"error\": str(e)}\n\n\n@router.get(\"/config\")\nasync def get_config(request: Request):\n \"\"\"\n Get frontend configuration.\n\n Returns version information and health status.\n Note: The frontend determines the API URL via its own runtime-config endpoint,\n so this endpoint no longer returns apiUrl.\n\n Also checks for version updates from GitHub (with caching and error handling).\n \"\"\"\n # Get current version\n current_version = get_version()\n\n # Check for updates (with caching and error handling)\n # This MUST NOT break the endpoint - wrapped in try-except as extra safety\n latest_version = None\n has_update = False\n\n try:\n latest_version, has_update = await get_latest_version_cached(current_version)\n except Exception as e:\n # Extra safety: ensure version check never breaks the config endpoint\n logger.error(f\"Unexpected error during version check: {e}\")\n\n # Check database health\n db_health = await check_database_health()\n db_status = db_health[\"status\"]\n\n if db_status == \"offline\":\n logger.warning(f\"Database offline: {db_health.get('error', 'Unknown error')}\")\n\n return {\n \"version\": current_version,\n \"latestVersion\": latest_version,\n \"hasUpdate\": has_update,\n \"dbStatus\": db_status,\n }\n" }, { "path": "api/routers/context.py", "content": "from fastapi import APIRouter, HTTPException\nfrom loguru import logger\n\nfrom api.models import ContextRequest, ContextResponse\nfrom open_notebook.domain.notebook import Note, Notebook, Source\nfrom open_notebook.exceptions import InvalidInputError\nfrom open_notebook.utils import token_count\n\nrouter = APIRouter()\n\n\n@router.post(\"/notebooks/{notebook_id}/context\", response_model=ContextResponse)\nasync def get_notebook_context(notebook_id: str, context_request: ContextRequest):\n \"\"\"Get context for a notebook based on configuration.\"\"\"\n try:\n # Verify notebook exists\n notebook = await Notebook.get(notebook_id)\n if not notebook:\n raise HTTPException(status_code=404, detail=\"Notebook not found\")\n\n context_data: dict[str, list[dict[str, str]]] = {\"note\": [], \"source\": []}\n total_content = \"\"\n\n # Process context configuration if provided\n if context_request.context_config:\n # Process sources\n for source_id, status in context_request.context_config.sources.items():\n if \"not in\" in status:\n continue\n\n try:\n # Add table prefix if not present\n full_source_id = (\n source_id\n if source_id.startswith(\"source:\")\n else f\"source:{source_id}\"\n )\n\n try:\n source = await Source.get(full_source_id)\n except Exception:\n continue\n\n if \"insights\" in status:\n source_context = await source.get_context(context_size=\"short\")\n context_data[\"source\"].append(source_context)\n total_content += str(source_context)\n elif \"full content\" in status:\n source_context = await source.get_context(context_size=\"long\")\n context_data[\"source\"].append(source_context)\n total_content += str(source_context)\n except Exception as e:\n logger.warning(f\"Error processing source {source_id}: {str(e)}\")\n continue\n\n # Process notes\n for note_id, status in context_request.context_config.notes.items():\n if \"not in\" in status:\n continue\n\n try:\n # Add table prefix if not present\n full_note_id = (\n note_id if note_id.startswith(\"note:\") else f\"note:{note_id}\"\n )\n note = await Note.get(full_note_id)\n if not note:\n continue\n\n if \"full content\" in status:\n note_context = note.get_context(context_size=\"long\")\n context_data[\"note\"].append(note_context)\n total_content += str(note_context)\n except Exception as e:\n logger.warning(f\"Error processing note {note_id}: {str(e)}\")\n continue\n else:\n # Default behavior - include all sources and notes with short context\n sources = await notebook.get_sources()\n for source in sources:\n try:\n source_context = await source.get_context(context_size=\"short\")\n context_data[\"source\"].append(source_context)\n total_content += str(source_context)\n except Exception as e:\n logger.warning(f\"Error processing source {source.id}: {str(e)}\")\n continue\n\n notes = await notebook.get_notes()\n for note in notes:\n try:\n note_context = note.get_context(context_size=\"short\")\n context_data[\"note\"].append(note_context)\n total_content += str(note_context)\n except Exception as e:\n logger.warning(f\"Error processing note {note.id}: {str(e)}\")\n continue\n\n # Calculate estimated token count\n estimated_tokens = token_count(total_content) if total_content else 0\n\n return ContextResponse(\n notebook_id=notebook_id,\n sources=context_data[\"source\"],\n notes=context_data[\"note\"],\n total_tokens=estimated_tokens,\n )\n\n except HTTPException:\n raise\n except InvalidInputError as e:\n raise HTTPException(status_code=400, detail=str(e))\n except Exception as e:\n logger.error(f\"Error getting context for notebook {notebook_id}: {str(e)}\")\n raise HTTPException(status_code=500, detail=f\"Error getting context: {str(e)}\")\n" }, { "path": "api/routers/credentials.py", "content": "\"\"\"\nCredentials Router\n\nThin HTTP layer for managing individual AI provider credentials.\nBusiness logic lives in api.credentials_service.\n\nEndpoints:\n- GET /credentials - List all credentials\n- GET /credentials/by-provider/{provider} - List credentials for a provider\n- POST /credentials - Create a new credential\n- GET /credentials/{credential_id} - Get a specific credential\n- PUT /credentials/{credential_id} - Update a credential\n- DELETE /credentials/{credential_id} - Delete a credential\n- POST /credentials/{credential_id}/test - Test connection\n- POST /credentials/{credential_id}/discover - Discover models\n- POST /credentials/{credential_id}/register-models - Register models\n\nNEVER returns actual API key values - only metadata.\n\"\"\"\n\nfrom typing import List, Optional\n\nfrom fastapi import APIRouter, HTTPException, Query\nfrom loguru import logger\nfrom pydantic import SecretStr\n\nfrom api.credentials_service import (\n credential_to_response,\n discover_with_config,\n migrate_from_env as svc_migrate_from_env,\n migrate_from_provider_config as svc_migrate_from_provider_config,\n register_models,\n require_encryption_key,\n test_credential as svc_test_credential,\n validate_url,\n)\nfrom api.credentials_service import (\n get_env_status as svc_get_env_status,\n get_provider_status,\n)\nfrom api.models import (\n CreateCredentialRequest,\n CredentialDeleteResponse,\n CredentialResponse,\n DiscoveredModelResponse,\n DiscoverModelsResponse,\n RegisterModelsRequest,\n RegisterModelsResponse,\n UpdateCredentialRequest,\n)\nfrom open_notebook.domain.credential import Credential\n\nrouter = APIRouter(prefix=\"/credentials\", tags=[\"credentials\"])\n\n\ndef _handle_value_error(e: ValueError, status_code: int = 400) -> HTTPException:\n \"\"\"Convert a ValueError from the service layer to an HTTPException.\"\"\"\n return HTTPException(status_code=status_code, detail=str(e))\n\n\n# =============================================================================\n# Status endpoints\n# =============================================================================\n\n\n@router.get(\"/status\")\nasync def get_status():\n \"\"\"\n Get configuration status: encryption key status, and per-provider\n configured/source information.\n \"\"\"\n try:\n return await get_provider_status()\n except Exception as e:\n logger.error(f\"Error fetching status: {e}\")\n raise HTTPException(status_code=500, detail=\"Failed to fetch credential status\")\n\n\n@router.get(\"/env-status\")\nasync def get_env_status():\n \"\"\"Check what's configured via environment variables.\"\"\"\n try:\n return await svc_get_env_status()\n except Exception as e:\n logger.error(f\"Error checking env status: {e}\")\n raise HTTPException(status_code=500, detail=\"Failed to check environment status\")\n\n\n# =============================================================================\n# CRUD endpoints\n# =============================================================================\n\n\n@router.get(\"\", response_model=List[CredentialResponse])\nasync def list_credentials(\n provider: Optional[str] = Query(None, description=\"Filter by provider\"),\n):\n \"\"\"List all credentials, optionally filtered by provider.\"\"\"\n try:\n if provider:\n credentials = await Credential.get_by_provider(provider)\n else:\n credentials = await Credential.get_all(order_by=\"provider, created\")\n\n result = []\n for cred in credentials:\n models = await cred.get_linked_models()\n result.append(credential_to_response(cred, len(models)))\n\n return result\n\n except Exception as e:\n logger.error(f\"Error listing credentials: {e}\")\n raise HTTPException(status_code=500, detail=\"Failed to list credentials\")\n\n\n@router.get(\"/by-provider/{provider}\", response_model=List[CredentialResponse])\nasync def list_credentials_by_provider(provider: str):\n \"\"\"List all credentials for a specific provider.\"\"\"\n try:\n credentials = await Credential.get_by_provider(provider.lower())\n result = []\n for cred in credentials:\n models = await cred.get_linked_models()\n result.append(credential_to_response(cred, len(models)))\n return result\n except Exception as e:\n logger.error(f\"Error listing credentials for {provider}: {e}\")\n raise HTTPException(status_code=500, detail=\"Failed to list credentials for provider\")\n\n\n@router.post(\"\", response_model=CredentialResponse, status_code=201)\nasync def create_credential(request: CreateCredentialRequest):\n \"\"\"Create a new credential.\"\"\"\n try:\n require_encryption_key()\n except ValueError as e:\n raise _handle_value_error(e)\n\n # Validate all URL fields\n for url_field in [\n request.base_url, request.endpoint, request.endpoint_llm,\n request.endpoint_embedding, request.endpoint_stt, request.endpoint_tts,\n ]:\n if url_field:\n try:\n validate_url(url_field, request.provider)\n except ValueError as e:\n raise _handle_value_error(e)\n\n try:\n cred = Credential(\n name=request.name,\n provider=request.provider.lower(),\n modalities=request.modalities,\n api_key=SecretStr(request.api_key) if request.api_key else None,\n base_url=request.base_url,\n endpoint=request.endpoint,\n api_version=request.api_version,\n endpoint_llm=request.endpoint_llm,\n endpoint_embedding=request.endpoint_embedding,\n endpoint_stt=request.endpoint_stt,\n endpoint_tts=request.endpoint_tts,\n project=request.project,\n location=request.location,\n credentials_path=request.credentials_path,\n )\n await cred.save()\n return credential_to_response(cred, 0)\n\n except Exception as e:\n logger.error(f\"Error creating credential: {e}\")\n raise HTTPException(status_code=500, detail=\"Failed to create credential\")\n\n\n@router.get(\"/{credential_id}\", response_model=CredentialResponse)\nasync def get_credential(credential_id: str):\n \"\"\"Get a specific credential by ID. Never returns api_key.\"\"\"\n try:\n cred = await Credential.get(credential_id)\n models = await cred.get_linked_models()\n return credential_to_response(cred, len(models))\n except Exception as e:\n logger.error(f\"Error fetching credential {credential_id}: {e}\")\n raise HTTPException(status_code=404, detail=\"Credential not found\")\n\n\n@router.put(\"/{credential_id}\", response_model=CredentialResponse)\nasync def update_credential(credential_id: str, request: UpdateCredentialRequest):\n \"\"\"Update an existing credential.\"\"\"\n try:\n require_encryption_key()\n except ValueError as e:\n raise _handle_value_error(e)\n\n # Validate all URL fields being updated\n for url_field in [\n request.base_url, request.endpoint, request.endpoint_llm,\n request.endpoint_embedding, request.endpoint_stt, request.endpoint_tts,\n ]:\n if url_field:\n try:\n validate_url(url_field, \"update\")\n except ValueError as e:\n raise _handle_value_error(e)\n\n try:\n cred = await Credential.get(credential_id)\n\n if request.name is not None:\n cred.name = request.name\n if request.modalities is not None:\n cred.modalities = request.modalities\n if request.api_key is not None:\n cred.api_key = SecretStr(request.api_key)\n if request.base_url is not None:\n cred.base_url = request.base_url or None\n if request.endpoint is not None:\n cred.endpoint = request.endpoint or None\n if request.api_version is not None:\n cred.api_version = request.api_version or None\n if request.endpoint_llm is not None:\n cred.endpoint_llm = request.endpoint_llm or None\n if request.endpoint_embedding is not None:\n cred.endpoint_embedding = request.endpoint_embedding or None\n if request.endpoint_stt is not None:\n cred.endpoint_stt = request.endpoint_stt or None\n if request.endpoint_tts is not None:\n cred.endpoint_tts = request.endpoint_tts or None\n if request.project is not None:\n cred.project = request.project or None\n if request.location is not None:\n cred.location = request.location or None\n if request.credentials_path is not None:\n cred.credentials_path = request.credentials_path or None\n\n await cred.save()\n models = await cred.get_linked_models()\n return credential_to_response(cred, len(models))\n\n except HTTPException:\n raise\n except Exception as e:\n logger.error(f\"Error updating credential {credential_id}: {e}\")\n raise HTTPException(status_code=500, detail=\"Failed to update credential\")\n\n\n@router.delete(\"/{credential_id}\", response_model=CredentialDeleteResponse)\nasync def delete_credential(\n credential_id: str,\n delete_models: bool = Query(False, description=\"Also delete linked models\"),\n migrate_to: Optional[str] = Query(\n None, description=\"Migrate linked models to this credential ID\"\n ),\n):\n \"\"\"\n Delete a credential.\n\n If the credential has linked models:\n - Pass delete_models=true to delete them\n - Pass migrate_to= to reassign them\n - Without either, returns 409 with linked model info\n \"\"\"\n try:\n cred = await Credential.get(credential_id)\n linked_models = await cred.get_linked_models()\n\n if linked_models and not delete_models and not migrate_to:\n raise HTTPException(\n status_code=409,\n detail={\n \"message\": f\"Credential has {len(linked_models)} linked model(s)\",\n \"model_ids\": [m.id for m in linked_models],\n \"model_names\": [f\"{m.provider}/{m.name}\" for m in linked_models],\n },\n )\n\n deleted_models = 0\n\n if linked_models and migrate_to:\n # Migrate models to another credential\n target_cred = await Credential.get(migrate_to)\n for model in linked_models:\n model.credential = target_cred.id\n await model.save()\n\n elif linked_models and delete_models:\n # Delete linked models\n for model in linked_models:\n await model.delete()\n deleted_models += 1\n\n # Delete the credential\n await cred.delete()\n\n return CredentialDeleteResponse(\n message=\"Credential deleted successfully\",\n deleted_models=deleted_models,\n )\n\n except HTTPException:\n raise\n except Exception as e:\n logger.error(f\"Error deleting credential {credential_id}: {e}\")\n raise HTTPException(status_code=500, detail=\"Failed to delete credential\")\n\n\n# =============================================================================\n# Test / Discover / Register endpoints\n# =============================================================================\n\n\n@router.post(\"/{credential_id}/test\")\nasync def test_credential(credential_id: str):\n \"\"\"Test connection using this credential's configuration.\"\"\"\n return await svc_test_credential(credential_id)\n\n\n@router.post(\"/{credential_id}/discover\", response_model=DiscoverModelsResponse)\nasync def discover_models_for_credential(credential_id: str):\n \"\"\"Discover available models using this credential's API key.\"\"\"\n try:\n cred = await Credential.get(credential_id)\n config = cred.to_esperanto_config()\n provider = cred.provider.lower()\n\n discovered = await discover_with_config(provider, config)\n\n return DiscoverModelsResponse(\n credential_id=cred.id or \"\",\n provider=provider,\n discovered=[\n DiscoveredModelResponse(\n name=d[\"name\"],\n provider=d[\"provider\"],\n description=d.get(\"description\"),\n )\n for d in discovered\n ],\n )\n\n except Exception as e:\n logger.error(f\"Error discovering models for credential {credential_id}: {e}\")\n raise HTTPException(status_code=500, detail=\"Failed to discover models\")\n\n\n@router.post(\"/{credential_id}/register-models\", response_model=RegisterModelsResponse)\nasync def register_models_for_credential(\n credential_id: str, request: RegisterModelsRequest\n):\n \"\"\"Register discovered models and link them to this credential.\"\"\"\n try:\n result = await register_models(credential_id, request.models)\n return RegisterModelsResponse(**result)\n except Exception as e:\n logger.error(f\"Error registering models for credential {credential_id}: {e}\")\n raise HTTPException(status_code=500, detail=\"Failed to register models\")\n\n\n# =============================================================================\n# Migration endpoints\n# =============================================================================\n\n\n@router.post(\"/migrate-from-provider-config\")\nasync def migrate_from_provider_config():\n \"\"\"Migrate existing ProviderConfig data to individual credential records.\"\"\"\n try:\n return await svc_migrate_from_provider_config()\n except ValueError as e:\n raise _handle_value_error(e)\n except Exception as e:\n logger.error(f\"ProviderConfig migration FAILED: {type(e).__name__}: {e}\", exc_info=True)\n raise HTTPException(status_code=500, detail=\"Migration from provider config failed\")\n\n\n@router.post(\"/migrate-from-env\")\nasync def migrate_from_env():\n \"\"\"Migrate API keys from environment variables to credential records.\"\"\"\n try:\n return await svc_migrate_from_env()\n except ValueError as e:\n raise _handle_value_error(e)\n except Exception as e:\n logger.error(f\"Env migration FAILED: {type(e).__name__}: {e}\", exc_info=True)\n raise HTTPException(status_code=500, detail=\"Migration from environment variables failed\")\n" }, { "path": "api/routers/embedding.py", "content": "from fastapi import APIRouter, HTTPException\nfrom loguru import logger\n\nfrom api.command_service import CommandService\nfrom api.models import EmbedRequest, EmbedResponse\nfrom open_notebook.ai.models import model_manager\nfrom open_notebook.domain.notebook import Note, Source\n\nrouter = APIRouter()\n\n\n@router.post(\"/embed\", response_model=EmbedResponse)\nasync def embed_content(embed_request: EmbedRequest):\n \"\"\"Embed content for vector search.\"\"\"\n try:\n # Check if embedding model is available\n if not await model_manager.get_embedding_model():\n raise HTTPException(\n status_code=400,\n detail=\"No embedding model configured. Please configure one in the Models section.\",\n )\n\n item_id = embed_request.item_id\n item_type = embed_request.item_type.lower()\n\n # Validate item type\n if item_type not in [\"source\", \"note\"]:\n raise HTTPException(\n status_code=400, detail=\"Item type must be either 'source' or 'note'\"\n )\n\n # Branch based on processing mode\n if embed_request.async_processing:\n # ASYNC PATH: Submit command for background processing\n logger.info(f\"Using async processing for {item_type} {item_id}\")\n\n try:\n # Import commands to ensure they're registered\n import commands.embedding_commands # noqa: F401\n\n # Submit type-specific command\n if item_type == \"source\":\n command_name = \"embed_source\"\n command_input = {\"source_id\": item_id}\n else: # note\n command_name = \"embed_note\"\n command_input = {\"note_id\": item_id}\n\n command_id = await CommandService.submit_command_job(\n \"open_notebook\",\n command_name,\n command_input,\n )\n\n logger.info(f\"Submitted async {command_name} command: {command_id}\")\n\n return EmbedResponse(\n success=True,\n message=\"Embedding queued for background processing\",\n item_id=item_id,\n item_type=item_type,\n command_id=command_id,\n )\n\n except Exception as e:\n logger.error(f\"Failed to submit async embedding command: {e}\")\n raise HTTPException(\n status_code=500, detail=f\"Failed to queue embedding: {str(e)}\"\n )\n\n else:\n # DOMAIN MODEL PATH: Submit job via domain model convenience methods\n # These methods internally call submit_command() - still fire-and-forget\n logger.info(f\"Using domain model path for {item_type} {item_id}\")\n\n command_id = None\n\n # Get the item and submit embedding job\n if item_type == \"source\":\n source_item = await Source.get(item_id)\n if not source_item:\n raise HTTPException(status_code=404, detail=\"Source not found\")\n\n # Submit embed_source job (returns command_id for tracking)\n command_id = await source_item.vectorize()\n message = \"Source embedding job submitted\"\n\n elif item_type == \"note\":\n note_item = await Note.get(item_id)\n if not note_item:\n raise HTTPException(status_code=404, detail=\"Note not found\")\n\n # Note.save() internally submits embed_note command and returns command_id\n command_id = await note_item.save()\n message = \"Note embedding job submitted\"\n\n return EmbedResponse(\n success=True,\n message=message,\n item_id=item_id,\n item_type=item_type,\n command_id=command_id,\n )\n\n except HTTPException:\n raise\n except Exception as e:\n logger.error(\n f\"Error embedding {embed_request.item_type} {embed_request.item_id}: {str(e)}\"\n )\n raise HTTPException(\n status_code=500, detail=f\"Error embedding content: {str(e)}\"\n )\n" }, { "path": "api/routers/embedding_rebuild.py", "content": "from fastapi import APIRouter, HTTPException\nfrom loguru import logger\nfrom surreal_commands import get_command_status\n\nfrom api.command_service import CommandService\nfrom api.models import (\n RebuildProgress,\n RebuildRequest,\n RebuildResponse,\n RebuildStats,\n RebuildStatusResponse,\n)\nfrom open_notebook.database.repository import repo_query\n\nrouter = APIRouter()\n\n\n@router.post(\"/rebuild\", response_model=RebuildResponse)\nasync def start_rebuild(request: RebuildRequest):\n \"\"\"\n Start a background job to rebuild embeddings.\n\n - **mode**: \"existing\" (re-embed items with embeddings) or \"all\" (embed everything)\n - **include_sources**: Include sources in rebuild (default: true)\n - **include_notes**: Include notes in rebuild (default: true)\n - **include_insights**: Include insights in rebuild (default: true)\n\n Returns command ID to track progress and estimated item count.\n \"\"\"\n try:\n logger.info(f\"Starting rebuild request: mode={request.mode}\")\n\n # Import commands to ensure they're registered\n import commands.embedding_commands # noqa: F401\n\n # Estimate total items (quick count query)\n # This is a rough estimate before the command runs\n total_estimate = 0\n\n if request.include_sources:\n if request.mode == \"existing\":\n # Count sources with embeddings\n result = await repo_query(\n \"\"\"\n SELECT VALUE count(array::distinct(\n SELECT VALUE source.id\n FROM source_embedding\n WHERE embedding != none AND array::len(embedding) > 0\n )) as count FROM {}\n \"\"\"\n )\n else:\n # Count all sources with content\n result = await repo_query(\n \"SELECT VALUE count() as count FROM source WHERE full_text != none GROUP ALL\"\n )\n\n if result and isinstance(result[0], dict):\n total_estimate += result[0].get(\"count\", 0)\n elif result:\n total_estimate += result[0] if isinstance(result[0], int) else 0\n\n if request.include_notes:\n if request.mode == \"existing\":\n result = await repo_query(\n \"SELECT VALUE count() as count FROM note WHERE embedding != none AND array::len(embedding) > 0 GROUP ALL\"\n )\n else:\n result = await repo_query(\n \"SELECT VALUE count() as count FROM note WHERE content != none GROUP ALL\"\n )\n\n if result and isinstance(result[0], dict):\n total_estimate += result[0].get(\"count\", 0)\n elif result:\n total_estimate += result[0] if isinstance(result[0], int) else 0\n\n if request.include_insights:\n if request.mode == \"existing\":\n result = await repo_query(\n \"SELECT VALUE count() as count FROM source_insight WHERE embedding != none AND array::len(embedding) > 0 GROUP ALL\"\n )\n else:\n result = await repo_query(\n \"SELECT VALUE count() as count FROM source_insight GROUP ALL\"\n )\n\n if result and isinstance(result[0], dict):\n total_estimate += result[0].get(\"count\", 0)\n elif result:\n total_estimate += result[0] if isinstance(result[0], int) else 0\n\n logger.info(f\"Estimated {total_estimate} items to process\")\n\n # Submit command\n command_id = await CommandService.submit_command_job(\n \"open_notebook\",\n \"rebuild_embeddings\",\n {\n \"mode\": request.mode,\n \"include_sources\": request.include_sources,\n \"include_notes\": request.include_notes,\n \"include_insights\": request.include_insights,\n },\n )\n\n logger.info(f\"Submitted rebuild command: {command_id}\")\n\n return RebuildResponse(\n command_id=command_id,\n total_items=total_estimate,\n message=f\"Rebuild operation started. Estimated {total_estimate} items to process.\",\n )\n\n except Exception as e:\n logger.error(f\"Failed to start rebuild: {e}\")\n logger.exception(e)\n raise HTTPException(\n status_code=500, detail=f\"Failed to start rebuild operation: {str(e)}\"\n )\n\n\n@router.get(\"/rebuild/{command_id}/status\", response_model=RebuildStatusResponse)\nasync def get_rebuild_status(command_id: str):\n \"\"\"\n Get the status of a rebuild operation.\n\n Returns:\n - **status**: queued, running, completed, failed\n - **progress**: processed count, total count, percentage\n - **stats**: breakdown by type (sources, notes, insights, failed)\n - **timestamps**: started_at, completed_at\n \"\"\"\n try:\n # Get command status from surreal_commands\n status = await get_command_status(command_id)\n\n if not status:\n raise HTTPException(status_code=404, detail=\"Rebuild command not found\")\n\n # Build response based on status\n response = RebuildStatusResponse(\n command_id=command_id,\n status=status.status,\n )\n\n # Extract metadata from command result\n if status.result and isinstance(status.result, dict):\n result = status.result\n\n # Build progress info\n if \"total_items\" in result and \"jobs_submitted\" in result:\n total = result[\"total_items\"]\n submitted = result[\"jobs_submitted\"]\n response.progress = RebuildProgress(\n processed=submitted,\n total=total,\n percentage=round((submitted / total * 100) if total > 0 else 0, 2),\n )\n\n # Build stats\n response.stats = RebuildStats(\n sources=result.get(\"sources_submitted\", 0),\n notes=result.get(\"notes_submitted\", 0),\n insights=result.get(\"insights_submitted\", 0),\n failed=result.get(\"failed_submissions\", 0),\n )\n\n # Add timestamps\n if hasattr(status, \"created\") and status.created:\n response.started_at = str(status.created)\n if hasattr(status, \"updated\") and status.updated:\n response.completed_at = str(status.updated)\n\n # Add error message if failed\n if (\n status.status == \"failed\"\n and status.result\n and isinstance(status.result, dict)\n ):\n response.error_message = status.result.get(\"error_message\", \"Unknown error\")\n\n return response\n\n except HTTPException:\n raise\n except Exception as e:\n logger.error(f\"Failed to get rebuild status: {e}\")\n logger.exception(e)\n raise HTTPException(\n status_code=500, detail=f\"Failed to get rebuild status: {str(e)}\"\n )\n" }, { "path": "api/routers/episode_profiles.py", "content": "from typing import List, Optional\n\nfrom fastapi import APIRouter, HTTPException\nfrom loguru import logger\nfrom pydantic import BaseModel, Field\n\nfrom open_notebook.podcasts.models import EpisodeProfile\n\nrouter = APIRouter()\n\n\nclass EpisodeProfileResponse(BaseModel):\n id: str\n name: str\n description: str\n speaker_config: str\n outline_llm: Optional[str] = None\n transcript_llm: Optional[str] = None\n language: Optional[str] = None\n default_briefing: str\n num_segments: int\n # Legacy fields (for display/migration awareness)\n outline_provider: Optional[str] = None\n outline_model: Optional[str] = None\n transcript_provider: Optional[str] = None\n transcript_model: Optional[str] = None\n\n\ndef _profile_to_response(profile: EpisodeProfile) -> EpisodeProfileResponse:\n return EpisodeProfileResponse(\n id=str(profile.id),\n name=profile.name,\n description=profile.description or \"\",\n speaker_config=profile.speaker_config,\n outline_llm=profile.outline_llm,\n transcript_llm=profile.transcript_llm,\n language=profile.language,\n default_briefing=profile.default_briefing,\n num_segments=profile.num_segments,\n outline_provider=profile.outline_provider,\n outline_model=profile.outline_model,\n transcript_provider=profile.transcript_provider,\n transcript_model=profile.transcript_model,\n )\n\n\n@router.get(\"/episode-profiles\", response_model=List[EpisodeProfileResponse])\nasync def list_episode_profiles():\n \"\"\"List all available episode profiles\"\"\"\n try:\n profiles = await EpisodeProfile.get_all(order_by=\"name asc\")\n return [_profile_to_response(p) for p in profiles]\n except Exception as e:\n logger.error(f\"Failed to fetch episode profiles: {e}\")\n raise HTTPException(\n status_code=500, detail=\"Failed to fetch episode profiles\"\n )\n\n\n@router.get(\"/episode-profiles/{profile_name}\", response_model=EpisodeProfileResponse)\nasync def get_episode_profile(profile_name: str):\n \"\"\"Get a specific episode profile by name\"\"\"\n try:\n profile = await EpisodeProfile.get_by_name(profile_name)\n\n if not profile:\n raise HTTPException(\n status_code=404, detail=f\"Episode profile '{profile_name}' not found\"\n )\n\n return _profile_to_response(profile)\n\n except HTTPException:\n raise\n except Exception as e:\n logger.error(f\"Failed to fetch episode profile '{profile_name}': {e}\")\n raise HTTPException(\n status_code=500, detail=\"Failed to fetch episode profile\"\n )\n\n\nclass EpisodeProfileCreate(BaseModel):\n name: str = Field(..., description=\"Unique profile name\")\n description: str = Field(\"\", description=\"Profile description\")\n speaker_config: str = Field(..., description=\"Reference to speaker profile name\")\n outline_llm: Optional[str] = Field(None, description=\"Model record ID for outline\")\n transcript_llm: Optional[str] = Field(\n None, description=\"Model record ID for transcript\"\n )\n language: Optional[str] = Field(None, description=\"Podcast language code\")\n default_briefing: str = Field(..., description=\"Default briefing template\")\n num_segments: int = Field(default=5, description=\"Number of podcast segments\")\n # Legacy fields (accepted but not required)\n outline_provider: Optional[str] = None\n outline_model: Optional[str] = None\n transcript_provider: Optional[str] = None\n transcript_model: Optional[str] = None\n\n\n@router.post(\"/episode-profiles\", response_model=EpisodeProfileResponse)\nasync def create_episode_profile(profile_data: EpisodeProfileCreate):\n \"\"\"Create a new episode profile\"\"\"\n try:\n profile = EpisodeProfile(\n name=profile_data.name,\n description=profile_data.description,\n speaker_config=profile_data.speaker_config,\n outline_llm=profile_data.outline_llm,\n transcript_llm=profile_data.transcript_llm,\n language=profile_data.language,\n default_briefing=profile_data.default_briefing,\n num_segments=profile_data.num_segments,\n outline_provider=profile_data.outline_provider,\n outline_model=profile_data.outline_model,\n transcript_provider=profile_data.transcript_provider,\n transcript_model=profile_data.transcript_model,\n )\n\n await profile.save()\n return _profile_to_response(profile)\n\n except Exception as e:\n logger.error(f\"Failed to create episode profile: {e}\")\n raise HTTPException(\n status_code=500, detail=\"Failed to create episode profile\"\n )\n\n\n@router.put(\"/episode-profiles/{profile_id}\", response_model=EpisodeProfileResponse)\nasync def update_episode_profile(profile_id: str, profile_data: EpisodeProfileCreate):\n \"\"\"Update an existing episode profile\"\"\"\n try:\n profile = await EpisodeProfile.get(profile_id)\n\n if not profile:\n raise HTTPException(\n status_code=404, detail=f\"Episode profile '{profile_id}' not found\"\n )\n\n profile.name = profile_data.name\n profile.description = profile_data.description\n profile.speaker_config = profile_data.speaker_config\n profile.outline_llm = profile_data.outline_llm\n profile.transcript_llm = profile_data.transcript_llm\n profile.language = profile_data.language\n profile.default_briefing = profile_data.default_briefing\n profile.num_segments = profile_data.num_segments\n profile.outline_provider = profile_data.outline_provider\n profile.outline_model = profile_data.outline_model\n profile.transcript_provider = profile_data.transcript_provider\n profile.transcript_model = profile_data.transcript_model\n\n await profile.save()\n return _profile_to_response(profile)\n\n except HTTPException:\n raise\n except Exception as e:\n logger.error(f\"Failed to update episode profile: {e}\")\n raise HTTPException(\n status_code=500, detail=\"Failed to update episode profile\"\n )\n\n\n@router.delete(\"/episode-profiles/{profile_id}\")\nasync def delete_episode_profile(profile_id: str):\n \"\"\"Delete an episode profile\"\"\"\n try:\n profile = await EpisodeProfile.get(profile_id)\n\n if not profile:\n raise HTTPException(\n status_code=404, detail=f\"Episode profile '{profile_id}' not found\"\n )\n\n await profile.delete()\n\n return {\"message\": \"Episode profile deleted successfully\"}\n\n except HTTPException:\n raise\n except Exception as e:\n logger.error(f\"Failed to delete episode profile: {e}\")\n raise HTTPException(\n status_code=500, detail=\"Failed to delete episode profile\"\n )\n\n\n@router.post(\n \"/episode-profiles/{profile_id}/duplicate\", response_model=EpisodeProfileResponse\n)\nasync def duplicate_episode_profile(profile_id: str):\n \"\"\"Duplicate an episode profile\"\"\"\n try:\n original = await EpisodeProfile.get(profile_id)\n\n if not original:\n raise HTTPException(\n status_code=404, detail=f\"Episode profile '{profile_id}' not found\"\n )\n\n duplicate = EpisodeProfile(\n name=f\"{original.name} - Copy\",\n description=original.description,\n speaker_config=original.speaker_config,\n outline_llm=original.outline_llm,\n transcript_llm=original.transcript_llm,\n language=original.language,\n default_briefing=original.default_briefing,\n num_segments=original.num_segments,\n outline_provider=original.outline_provider,\n outline_model=original.outline_model,\n transcript_provider=original.transcript_provider,\n transcript_model=original.transcript_model,\n )\n\n await duplicate.save()\n return _profile_to_response(duplicate)\n\n except HTTPException:\n raise\n except Exception as e:\n logger.error(f\"Failed to duplicate episode profile: {e}\")\n raise HTTPException(\n status_code=500, detail=\"Failed to duplicate episode profile\"\n )\n" }, { "path": "api/routers/insights.py", "content": "from fastapi import APIRouter, HTTPException\nfrom loguru import logger\n\nfrom api.models import NoteResponse, SaveAsNoteRequest, SourceInsightResponse\nfrom open_notebook.domain.notebook import SourceInsight\nfrom open_notebook.exceptions import InvalidInputError\n\nrouter = APIRouter()\n\n\n@router.get(\"/insights/{insight_id}\", response_model=SourceInsightResponse)\nasync def get_insight(insight_id: str):\n \"\"\"Get a specific insight by ID.\"\"\"\n try:\n insight = await SourceInsight.get(insight_id)\n if not insight:\n raise HTTPException(status_code=404, detail=\"Insight not found\")\n\n # Get source ID from the insight relationship\n source = await insight.get_source()\n\n return SourceInsightResponse(\n id=insight.id or \"\",\n source_id=source.id or \"\",\n insight_type=insight.insight_type,\n content=insight.content,\n created=str(insight.created),\n updated=str(insight.updated),\n )\n except HTTPException:\n raise\n except Exception as e:\n logger.error(f\"Error fetching insight {insight_id}: {str(e)}\")\n raise HTTPException(status_code=500, detail=\"Error fetching insight\")\n\n\n@router.delete(\"/insights/{insight_id}\")\nasync def delete_insight(insight_id: str):\n \"\"\"Delete a specific insight.\"\"\"\n try:\n insight = await SourceInsight.get(insight_id)\n if not insight:\n raise HTTPException(status_code=404, detail=\"Insight not found\")\n\n await insight.delete()\n\n return {\"message\": \"Insight deleted successfully\"}\n except HTTPException:\n raise\n except Exception as e:\n logger.error(f\"Error deleting insight {insight_id}: {str(e)}\")\n raise HTTPException(status_code=500, detail=\"Error deleting insight\")\n\n\n@router.post(\"/insights/{insight_id}/save-as-note\", response_model=NoteResponse)\nasync def save_insight_as_note(insight_id: str, request: SaveAsNoteRequest):\n \"\"\"Convert an insight to a note.\"\"\"\n try:\n insight = await SourceInsight.get(insight_id)\n if not insight:\n raise HTTPException(status_code=404, detail=\"Insight not found\")\n\n # Use the existing save_as_note method from the domain model\n note = await insight.save_as_note(request.notebook_id)\n\n return NoteResponse(\n id=note.id or \"\",\n title=note.title,\n content=note.content,\n note_type=note.note_type,\n created=str(note.created),\n updated=str(note.updated),\n )\n except HTTPException:\n raise\n except InvalidInputError as e:\n raise HTTPException(status_code=400, detail=str(e))\n except Exception as e:\n logger.error(f\"Error saving insight {insight_id} as note: {str(e)}\")\n raise HTTPException(\n status_code=500, detail=\"Error saving insight as note\"\n )\n" }, { "path": "api/routers/languages.py", "content": "from typing import List\n\nimport pycountry\nfrom babel import Locale\nfrom babel.core import get_global\nfrom fastapi import APIRouter\nfrom pydantic import BaseModel\n\nrouter = APIRouter()\n\n# Additional regional variants for languages where the distinction matters\n# (TTS accent, vocabulary, spelling differences)\n_EXTRA_VARIANTS = [\n \"pt_PT\",\n \"en_GB\",\n \"en_AU\",\n \"en_IN\",\n \"es_MX\",\n \"es_AR\",\n \"es_CO\",\n \"fr_CA\",\n \"fr_CH\",\n \"zh_TW\",\n \"zh_HK\",\n \"de_AT\",\n \"de_CH\",\n \"ar_SA\",\n \"nl_BE\",\n]\n\n\nclass LanguageResponse(BaseModel):\n code: str\n name: str\n\n\n@router.get(\"/languages\", response_model=List[LanguageResponse])\nasync def list_languages():\n \"\"\"List available languages as BCP 47 locale codes (e.g. pt-BR, en-US).\"\"\"\n likely_subtags = get_global(\"likely_subtags\")\n languages = []\n seen = set()\n\n # 1. For each language, resolve its default locale via CLDR likely subtags\n for lang in pycountry.languages:\n if not hasattr(lang, \"alpha_2\"):\n continue\n\n code = lang.alpha_2\n likely = likely_subtags.get(code)\n\n if likely:\n try:\n loc = Locale.parse(likely)\n if loc.territory:\n bcp47 = f\"{loc.language}-{loc.territory}\"\n display = loc.get_display_name(\"en\")\n if bcp47 not in seen:\n seen.add(bcp47)\n languages.append(LanguageResponse(code=bcp47, name=display))\n continue\n except Exception:\n pass\n\n # Fallback: bare language code\n if code not in seen:\n seen.add(code)\n languages.append(LanguageResponse(code=code, name=lang.name))\n\n # 2. Add important regional variants\n for locale_str in _EXTRA_VARIANTS:\n try:\n loc = Locale.parse(locale_str)\n bcp47 = f\"{loc.language}-{loc.territory}\"\n if bcp47 not in seen:\n seen.add(bcp47)\n display = loc.get_display_name(\"en\")\n languages.append(LanguageResponse(code=bcp47, name=display))\n except Exception:\n pass\n\n languages.sort(key=lambda x: x.name)\n return languages\n" }, { "path": "api/routers/models.py", "content": "import os\nimport traceback\nfrom typing import Dict, List, Optional\n\nfrom esperanto import AIFactory\nfrom fastapi import APIRouter, HTTPException, Query\nfrom loguru import logger\nfrom pydantic import BaseModel\n\nfrom api.models import (\n DefaultModelsResponse,\n ModelCreate,\n ModelResponse,\n ProviderAvailabilityResponse,\n)\nfrom open_notebook.domain.credential import Credential\nfrom open_notebook.ai.connection_tester import test_individual_model\nfrom open_notebook.ai.key_provider import provision_provider_keys\nfrom open_notebook.ai.model_discovery import (\n discover_provider_models,\n get_provider_model_count,\n sync_all_providers,\n sync_provider_models,\n)\nfrom open_notebook.ai.models import DefaultModels, Model\nfrom open_notebook.exceptions import InvalidInputError\n\nrouter = APIRouter()\n\n\n# =============================================================================\n# Model Discovery Response Models\n# =============================================================================\n\n\nclass DiscoveredModelResponse(BaseModel):\n \"\"\"Response model for a discovered model.\"\"\"\n\n name: str\n provider: str\n model_type: str\n description: Optional[str] = None\n\n\nclass ProviderSyncResponse(BaseModel):\n \"\"\"Response model for provider sync operation.\"\"\"\n\n provider: str\n discovered: int\n new: int\n existing: int\n\n\nclass AllProvidersSyncResponse(BaseModel):\n \"\"\"Response model for syncing all providers.\"\"\"\n\n results: Dict[str, ProviderSyncResponse]\n total_discovered: int\n total_new: int\n\n\nclass ProviderModelCountResponse(BaseModel):\n \"\"\"Response model for provider model counts.\"\"\"\n\n provider: str\n counts: Dict[str, int]\n total: int\n\n\nclass AutoAssignResult(BaseModel):\n \"\"\"Response model for auto-assign operation.\"\"\"\n\n assigned: Dict[str, str] # slot_name -> model_id\n skipped: List[str] # slots already assigned\n missing: List[str] # slots with no available models\n\n\nclass ModelTestResponse(BaseModel):\n \"\"\"Response model for individual model test.\"\"\"\n\n success: bool\n message: str\n details: Optional[str] = None\n\n\n# Provider priority for auto-assignment (higher priority first)\nPROVIDER_PRIORITY = [\n \"openai\",\n \"anthropic\",\n \"google\",\n \"mistral\",\n \"groq\",\n \"deepseek\",\n \"xai\",\n \"openrouter\",\n \"ollama\",\n \"azure\",\n \"openai_compatible\",\n]\n\n# Model preference patterns (preferred models within each provider)\nMODEL_PREFERENCES = {\n \"openai\": [\"gpt-4o\", \"gpt-4\", \"gpt-3.5-turbo\"],\n \"anthropic\": [\"claude-3-5-sonnet\", \"claude-3-opus\", \"claude-3-sonnet\"],\n \"google\": [\"gemini-2.0\", \"gemini-1.5-pro\", \"gemini-pro\"],\n \"mistral\": [\"mistral-large\", \"mixtral\"],\n \"groq\": [\"llama-3.3\", \"llama-3.1\", \"mixtral\"],\n}\n\n\nasync def _check_provider_has_credential(provider: str) -> bool:\n \"\"\"Check if a provider has any credentials configured in the database.\"\"\"\n try:\n credentials = await Credential.get_by_provider(provider)\n return len(credentials) > 0\n except Exception:\n pass\n return False\n\n\ndef _check_azure_support(mode: str) -> bool:\n \"\"\"\n Check if Azure OpenAI provider is available for a specific mode.\n\n Args:\n mode: One of 'LLM', 'EMBEDDING', 'STT', 'TTS'\n\n Returns:\n bool: True if either generic or mode-specific env vars are set\n \"\"\"\n # Check generic configuration (applies to all modes)\n generic = (\n os.environ.get(\"AZURE_OPENAI_API_KEY\") is not None\n and os.environ.get(\"AZURE_OPENAI_ENDPOINT\") is not None\n and os.environ.get(\"AZURE_OPENAI_API_VERSION\") is not None\n )\n\n # Check mode-specific configuration (takes precedence)\n specific = (\n os.environ.get(f\"AZURE_OPENAI_API_KEY_{mode}\") is not None\n and os.environ.get(f\"AZURE_OPENAI_ENDPOINT_{mode}\") is not None\n and os.environ.get(f\"AZURE_OPENAI_API_VERSION_{mode}\") is not None\n )\n\n return generic or specific\n\n\ndef _check_openai_compatible_support(mode: str) -> bool:\n \"\"\"\n Check if OpenAI-compatible provider is available for a specific mode.\n\n Args:\n mode: One of 'LLM', 'EMBEDDING', 'STT', 'TTS'\n\n Returns:\n bool: True if either generic or mode-specific env var is set\n \"\"\"\n generic = os.environ.get(\"OPENAI_COMPATIBLE_BASE_URL\") is not None\n specific = os.environ.get(f\"OPENAI_COMPATIBLE_BASE_URL_{mode}\") is not None\n generic_key = os.environ.get(\"OPENAI_COMPATIBLE_API_KEY\") is not None\n specific_key = os.environ.get(f\"OPENAI_COMPATIBLE_API_KEY_{mode}\") is not None\n return generic or specific or generic_key or specific_key\n\n\n@router.get(\"/models\", response_model=List[ModelResponse])\nasync def get_models(\n type: Optional[str] = Query(None, description=\"Filter by model type\"),\n):\n \"\"\"Get all configured models with optional type filtering.\"\"\"\n try:\n if type:\n models = await Model.get_models_by_type(type)\n else:\n models = await Model.get_all()\n\n return [\n ModelResponse(\n id=model.id,\n name=model.name,\n provider=model.provider,\n type=model.type,\n credential=model.credential,\n created=str(model.created),\n updated=str(model.updated),\n )\n for model in models\n ]\n except Exception as e:\n logger.error(f\"Error fetching models: {str(e)}\")\n raise HTTPException(status_code=500, detail=f\"Error fetching models: {str(e)}\")\n\n\n@router.post(\"/models\", response_model=ModelResponse)\nasync def create_model(model_data: ModelCreate):\n \"\"\"Create a new model configuration.\"\"\"\n try:\n # Validate model type\n valid_types = [\"language\", \"embedding\", \"text_to_speech\", \"speech_to_text\"]\n if model_data.type not in valid_types:\n raise HTTPException(\n status_code=400,\n detail=f\"Invalid model type. Must be one of: {valid_types}\",\n )\n\n # Check for duplicate model name under the same provider and type (case-insensitive)\n from open_notebook.database.repository import repo_query\n\n existing = await repo_query(\n \"SELECT * FROM model WHERE string::lowercase(provider) = $provider AND string::lowercase(name) = $name AND string::lowercase(type) = $type LIMIT 1\",\n {\n \"provider\": model_data.provider.lower(),\n \"name\": model_data.name.lower(),\n \"type\": model_data.type.lower(),\n },\n )\n if existing:\n raise HTTPException(\n status_code=400,\n detail=f\"Model '{model_data.name}' already exists for provider '{model_data.provider}' with type '{model_data.type}'\",\n )\n\n new_model = Model(\n name=model_data.name,\n provider=model_data.provider,\n type=model_data.type,\n credential=model_data.credential,\n )\n await new_model.save()\n\n return ModelResponse(\n id=new_model.id or \"\",\n name=new_model.name,\n provider=new_model.provider,\n type=new_model.type,\n credential=new_model.credential,\n created=str(new_model.created),\n updated=str(new_model.updated),\n )\n except HTTPException:\n raise\n except InvalidInputError as e:\n raise HTTPException(status_code=400, detail=str(e))\n except Exception as e:\n logger.error(f\"Error creating model: {str(e)}\")\n raise HTTPException(status_code=500, detail=f\"Error creating model: {str(e)}\")\n\n\n@router.delete(\"/models/{model_id}\")\nasync def delete_model(model_id: str):\n \"\"\"Delete a model configuration.\"\"\"\n try:\n model = await Model.get(model_id)\n if not model:\n raise HTTPException(status_code=404, detail=\"Model not found\")\n\n await model.delete()\n\n return {\"message\": \"Model deleted successfully\"}\n except HTTPException:\n raise\n except Exception as e:\n logger.error(f\"Error deleting model {model_id}: {str(e)}\")\n raise HTTPException(status_code=500, detail=f\"Error deleting model: {str(e)}\")\n\n\n@router.post(\"/models/{model_id}/test\", response_model=ModelTestResponse)\nasync def test_model(model_id: str):\n \"\"\"Test if a specific model is correctly configured and functional.\"\"\"\n try:\n model = await Model.get(model_id)\n if not model:\n raise HTTPException(status_code=404, detail=\"Model not found\")\n except HTTPException:\n raise\n except Exception:\n raise HTTPException(status_code=404, detail=\"Model not found\")\n\n try:\n success, message = await test_individual_model(model)\n return ModelTestResponse(success=success, message=message)\n except Exception as e:\n logger.error(f\"Error testing model {model_id}: {traceback.format_exc()}\")\n return ModelTestResponse(\n success=False,\n message=str(e)[:200],\n )\n\n\n@router.get(\"/models/defaults\", response_model=DefaultModelsResponse)\nasync def get_default_models():\n \"\"\"Get default model assignments.\"\"\"\n try:\n defaults = await DefaultModels.get_instance()\n\n return DefaultModelsResponse(\n default_chat_model=defaults.default_chat_model, # type: ignore[attr-defined]\n default_transformation_model=defaults.default_transformation_model, # type: ignore[attr-defined]\n large_context_model=defaults.large_context_model, # type: ignore[attr-defined]\n default_text_to_speech_model=defaults.default_text_to_speech_model, # type: ignore[attr-defined]\n default_speech_to_text_model=defaults.default_speech_to_text_model, # type: ignore[attr-defined]\n default_embedding_model=defaults.default_embedding_model, # type: ignore[attr-defined]\n default_tools_model=defaults.default_tools_model, # type: ignore[attr-defined]\n )\n except Exception as e:\n logger.error(f\"Error fetching default models: {str(e)}\")\n raise HTTPException(\n status_code=500, detail=f\"Error fetching default models: {str(e)}\"\n )\n\n\n@router.put(\"/models/defaults\", response_model=DefaultModelsResponse)\nasync def update_default_models(defaults_data: DefaultModelsResponse):\n \"\"\"Update default model assignments.\"\"\"\n try:\n defaults = await DefaultModels.get_instance()\n\n # Update only provided fields\n if defaults_data.default_chat_model is not None:\n defaults.default_chat_model = defaults_data.default_chat_model # type: ignore[attr-defined]\n if defaults_data.default_transformation_model is not None:\n defaults.default_transformation_model = (\n defaults_data.default_transformation_model\n ) # type: ignore[attr-defined]\n if defaults_data.large_context_model is not None:\n defaults.large_context_model = defaults_data.large_context_model # type: ignore[attr-defined]\n if defaults_data.default_text_to_speech_model is not None:\n defaults.default_text_to_speech_model = (\n defaults_data.default_text_to_speech_model\n ) # type: ignore[attr-defined]\n if defaults_data.default_speech_to_text_model is not None:\n defaults.default_speech_to_text_model = (\n defaults_data.default_speech_to_text_model\n ) # type: ignore[attr-defined]\n if defaults_data.default_embedding_model is not None:\n defaults.default_embedding_model = defaults_data.default_embedding_model # type: ignore[attr-defined]\n if defaults_data.default_tools_model is not None:\n defaults.default_tools_model = defaults_data.default_tools_model # type: ignore[attr-defined]\n\n await defaults.update()\n\n # No cache refresh needed - next access will fetch fresh data from DB\n\n return DefaultModelsResponse(\n default_chat_model=defaults.default_chat_model, # type: ignore[attr-defined]\n default_transformation_model=defaults.default_transformation_model, # type: ignore[attr-defined]\n large_context_model=defaults.large_context_model, # type: ignore[attr-defined]\n default_text_to_speech_model=defaults.default_text_to_speech_model, # type: ignore[attr-defined]\n default_speech_to_text_model=defaults.default_speech_to_text_model, # type: ignore[attr-defined]\n default_embedding_model=defaults.default_embedding_model, # type: ignore[attr-defined]\n default_tools_model=defaults.default_tools_model, # type: ignore[attr-defined]\n )\n except HTTPException:\n raise\n except Exception as e:\n logger.error(f\"Error updating default models: {str(e)}\")\n raise HTTPException(\n status_code=500, detail=f\"Error updating default models: {str(e)}\"\n )\n\n\n@router.get(\"/models/providers\", response_model=ProviderAvailabilityResponse)\nasync def get_provider_availability():\n \"\"\"Get provider availability based on database config and environment variables.\"\"\"\n try:\n # Check which providers have credentials in the database or env vars\n # For each provider, check DB credentials first, then env vars as fallback\n\n # Simple env var mapping for backward compatibility\n env_var_map = {\n \"openai\": \"OPENAI_API_KEY\",\n \"anthropic\": \"ANTHROPIC_API_KEY\",\n \"google\": \"GOOGLE_API_KEY\",\n \"groq\": \"GROQ_API_KEY\",\n \"mistral\": \"MISTRAL_API_KEY\",\n \"deepseek\": \"DEEPSEEK_API_KEY\",\n \"xai\": \"XAI_API_KEY\",\n \"openrouter\": \"OPENROUTER_API_KEY\",\n \"voyage\": \"VOYAGE_API_KEY\",\n \"elevenlabs\": \"ELEVENLABS_API_KEY\",\n \"ollama\": \"OLLAMA_API_BASE\",\n }\n\n provider_status = {}\n\n # Check simple providers: credential in DB or env var\n for provider, env_var in env_var_map.items():\n has_cred = await _check_provider_has_credential(provider)\n has_env = os.environ.get(env_var) is not None\n provider_status[provider] = has_cred or has_env\n\n # Google also supports GEMINI_API_KEY\n if not provider_status.get(\"google\"):\n provider_status[\"google\"] = os.environ.get(\"GEMINI_API_KEY\") is not None\n\n # Vertex: DB credential or env vars\n provider_status[\"vertex\"] = (\n await _check_provider_has_credential(\"vertex\")\n or os.environ.get(\"VERTEX_PROJECT\") is not None\n )\n\n # Azure: DB credential or env vars\n provider_status[\"azure\"] = (\n await _check_provider_has_credential(\"azure\")\n or _check_azure_support(\"LLM\")\n or _check_azure_support(\"EMBEDDING\")\n or _check_azure_support(\"STT\")\n or _check_azure_support(\"TTS\")\n )\n\n # OpenAI-compatible: DB credential or env vars\n provider_status[\"openai-compatible\"] = (\n await _check_provider_has_credential(\"openai_compatible\")\n or _check_openai_compatible_support(\"LLM\")\n or _check_openai_compatible_support(\"EMBEDDING\")\n or _check_openai_compatible_support(\"STT\")\n or _check_openai_compatible_support(\"TTS\")\n )\n\n available_providers = [k for k, v in provider_status.items() if v]\n unavailable_providers = [k for k, v in provider_status.items() if not v]\n\n # Get supported model types from Esperanto\n esperanto_available = AIFactory.get_available_providers()\n\n # Build supported types mapping only for available providers\n supported_types: dict[str, list[str]] = {}\n for provider in available_providers:\n supported_types[provider] = []\n\n # Map Esperanto model types to our environment variable modes\n mode_mapping = {\n \"language\": \"LLM\",\n \"embedding\": \"EMBEDDING\",\n \"speech_to_text\": \"STT\",\n \"text_to_speech\": \"TTS\",\n }\n\n # Special handling for openai-compatible to check mode-specific availability\n if provider == \"openai-compatible\":\n has_db_cred = await _check_provider_has_credential(\"openai_compatible\")\n for model_type, mode in mode_mapping.items():\n if (\n model_type in esperanto_available\n and provider in esperanto_available[model_type]\n ):\n if has_db_cred or _check_openai_compatible_support(mode):\n supported_types[provider].append(model_type)\n # Special handling for azure to check mode-specific availability\n elif provider == \"azure\":\n has_db_cred = await _check_provider_has_credential(\"azure\")\n for model_type, mode in mode_mapping.items():\n if (\n model_type in esperanto_available\n and provider in esperanto_available[model_type]\n ):\n if has_db_cred or _check_azure_support(mode):\n supported_types[provider].append(model_type)\n else:\n # Standard provider detection\n for model_type, providers in esperanto_available.items():\n if provider in providers:\n supported_types[provider].append(model_type)\n\n return ProviderAvailabilityResponse(\n available=available_providers,\n unavailable=unavailable_providers,\n supported_types=supported_types,\n )\n except Exception as e:\n logger.error(f\"Error checking provider availability: {str(e)}\")\n raise HTTPException(\n status_code=500, detail=f\"Error checking provider availability: {str(e)}\"\n )\n\n\n# =============================================================================\n# Model Discovery Endpoints\n# =============================================================================\n\n\n@router.get(\n \"/models/discover/{provider}\", response_model=List[DiscoveredModelResponse]\n)\nasync def discover_models(provider: str):\n \"\"\"\n Discover available models from a provider without registering them.\n\n This endpoint queries the provider's API to list available models\n but does not save them to the database. Use the sync endpoint\n to both discover and register models.\n \"\"\"\n try:\n # Provision DB-stored credentials into env vars before discovery\n await provision_provider_keys(provider)\n discovered = await discover_provider_models(provider)\n return [\n DiscoveredModelResponse(\n name=m.name,\n provider=m.provider,\n model_type=m.model_type,\n description=m.description,\n )\n for m in discovered\n ]\n except Exception as e:\n logger.error(f\"Error discovering models for {provider}: {str(e)}\")\n raise HTTPException(\n status_code=500, detail=\"Error discovering models. Check server logs for details.\"\n )\n\n\n@router.post(\"/models/sync/{provider}\", response_model=ProviderSyncResponse)\nasync def sync_models(provider: str):\n \"\"\"\n Sync models for a specific provider.\n\n Discovers available models from the provider's API and registers\n any new models in the database. Existing models are skipped.\n\n Returns counts of discovered, new, and existing models.\n \"\"\"\n try:\n # Provision DB-stored credentials into env vars before discovery\n await provision_provider_keys(provider)\n discovered, new, existing = await sync_provider_models(\n provider, auto_register=True\n )\n return ProviderSyncResponse(\n provider=provider,\n discovered=discovered,\n new=new,\n existing=existing,\n )\n except Exception as e:\n logger.error(f\"Error syncing models for {provider}: {str(e)}\")\n raise HTTPException(status_code=500, detail=\"Error syncing models. Check server logs for details.\")\n\n\n@router.post(\"/models/sync\", response_model=AllProvidersSyncResponse)\nasync def sync_all_models():\n \"\"\"\n Sync models for all configured providers.\n\n Discovers and registers models from all providers that have\n valid API keys configured. This is useful for initial setup\n or periodic refresh of available models.\n \"\"\"\n try:\n results = await sync_all_providers()\n\n response_results = {}\n total_discovered = 0\n total_new = 0\n\n for provider, (discovered, new, existing) in results.items():\n response_results[provider] = ProviderSyncResponse(\n provider=provider,\n discovered=discovered,\n new=new,\n existing=existing,\n )\n total_discovered += discovered\n total_new += new\n\n return AllProvidersSyncResponse(\n results=response_results,\n total_discovered=total_discovered,\n total_new=total_new,\n )\n except Exception as e:\n logger.error(f\"Error syncing all models: {str(e)}\")\n raise HTTPException(\n status_code=500, detail=f\"Error syncing all models: {str(e)}\"\n )\n\n\n@router.get(\"/models/count/{provider}\", response_model=ProviderModelCountResponse)\nasync def get_model_count(provider: str):\n \"\"\"\n Get count of registered models for a provider, grouped by type.\n\n Returns counts for each model type (language, embedding,\n speech_to_text, text_to_speech) as well as total count.\n \"\"\"\n try:\n counts = await get_provider_model_count(provider)\n total = sum(counts.values())\n return ProviderModelCountResponse(\n provider=provider,\n counts=counts,\n total=total,\n )\n except Exception as e:\n logger.error(f\"Error getting model count for {provider}: {str(e)}\")\n raise HTTPException(\n status_code=500, detail=f\"Error getting model count: {str(e)}\"\n )\n\n\n@router.get(\"/models/by-provider/{provider}\", response_model=List[ModelResponse])\nasync def get_models_by_provider(provider: str):\n \"\"\"\n Get all registered models for a specific provider.\n\n Returns models from the database that belong to the specified provider.\n \"\"\"\n try:\n from open_notebook.database.repository import repo_query\n\n models = await repo_query(\n \"SELECT * FROM model WHERE provider = $provider ORDER BY type, name\",\n {\"provider\": provider},\n )\n\n return [\n ModelResponse(\n id=model.get(\"id\", \"\"),\n name=model.get(\"name\", \"\"),\n provider=model.get(\"provider\", \"\"),\n type=model.get(\"type\", \"\"),\n credential=model.get(\"credential\"),\n created=str(model.get(\"created\", \"\")),\n updated=str(model.get(\"updated\", \"\")),\n )\n for model in models\n ]\n except Exception as e:\n logger.error(f\"Error fetching models for {provider}: {str(e)}\")\n raise HTTPException(\n status_code=500, detail=f\"Error fetching models: {str(e)}\"\n )\n\n\ndef _get_preferred_model(\n models: List[Dict], provider_priority: List[str], model_preferences: Dict\n) -> Optional[Dict]:\n \"\"\"\n Select the best model from a list based on provider priority and model preferences.\n\n Args:\n models: List of model dictionaries with 'provider', 'name', 'id' keys\n provider_priority: List of providers in preference order\n model_preferences: Dict mapping provider to list of preferred model name patterns\n\n Returns:\n The best model dict, or None if no models available\n \"\"\"\n if not models:\n return None\n\n # Group models by provider\n by_provider: Dict[str, List[Dict]] = {}\n for model in models:\n provider = model.get(\"provider\", \"\")\n if provider not in by_provider:\n by_provider[provider] = []\n by_provider[provider].append(model)\n\n # Find first provider with models (in priority order)\n for provider in provider_priority:\n if provider in by_provider:\n provider_models = by_provider[provider]\n\n # Check for preferred models within this provider\n if provider in model_preferences:\n for preference in model_preferences[provider]:\n for model in provider_models:\n if preference.lower() in model.get(\"name\", \"\").lower():\n return model\n\n # Fall back to first model from this provider\n return provider_models[0]\n\n # Fall back to first model from any provider\n return models[0] if models else None\n\n\n@router.post(\"/models/auto-assign\", response_model=AutoAssignResult)\nasync def auto_assign_defaults():\n \"\"\"\n Auto-assign default models based on available models.\n\n This endpoint intelligently assigns the first available model of each\n required type to the corresponding default slot. It uses provider\n priority (preferring premium providers like OpenAI, Anthropic) and\n model preferences within each provider.\n\n Returns:\n - assigned: Dict of slot names to assigned model IDs\n - skipped: List of slots that already have models assigned\n - missing: List of slots with no available models\n \"\"\"\n try:\n from open_notebook.database.repository import repo_query\n\n # Get current defaults\n defaults = await DefaultModels.get_instance()\n\n # Get all models grouped by type\n all_models = await repo_query(\n \"SELECT * FROM model ORDER BY provider, name\",\n {},\n )\n\n # Group models by type\n models_by_type: Dict[str, List[Dict]] = {\n \"language\": [],\n \"embedding\": [],\n \"text_to_speech\": [],\n \"speech_to_text\": [],\n }\n\n for model in all_models:\n model_type = model.get(\"type\", \"\")\n if model_type in models_by_type:\n models_by_type[model_type].append(model)\n\n # Define slot configuration: (slot_name, model_type, current_value)\n slot_configs = [\n (\"default_chat_model\", \"language\", defaults.default_chat_model), # type: ignore[attr-defined]\n (\"default_transformation_model\", \"language\", defaults.default_transformation_model), # type: ignore[attr-defined]\n (\"default_tools_model\", \"language\", defaults.default_tools_model), # type: ignore[attr-defined]\n (\"large_context_model\", \"language\", defaults.large_context_model), # type: ignore[attr-defined]\n (\"default_embedding_model\", \"embedding\", defaults.default_embedding_model), # type: ignore[attr-defined]\n (\"default_text_to_speech_model\", \"text_to_speech\", defaults.default_text_to_speech_model), # type: ignore[attr-defined]\n (\"default_speech_to_text_model\", \"speech_to_text\", defaults.default_speech_to_text_model), # type: ignore[attr-defined]\n ]\n\n assigned: Dict[str, str] = {}\n skipped: List[str] = []\n missing: List[str] = []\n\n for slot_name, model_type, current_value in slot_configs:\n if current_value:\n # Slot already has a value\n skipped.append(slot_name)\n continue\n\n available_models = models_by_type.get(model_type, [])\n if not available_models:\n # No models of this type available\n missing.append(slot_name)\n continue\n\n # Select best model for this slot\n best_model = _get_preferred_model(\n available_models, PROVIDER_PRIORITY, MODEL_PREFERENCES\n )\n\n if best_model:\n model_id = best_model.get(\"id\", \"\")\n assigned[slot_name] = model_id\n # Update the defaults object\n setattr(defaults, slot_name, model_id)\n\n # Save updated defaults if any assignments were made\n if assigned:\n await defaults.update()\n\n return AutoAssignResult(\n assigned=assigned,\n skipped=skipped,\n missing=missing,\n )\n\n except Exception as e:\n logger.error(f\"Error auto-assigning defaults: {str(e)}\")\n raise HTTPException(\n status_code=500, detail=f\"Error auto-assigning defaults: {str(e)}\"\n )\n" }, { "path": "api/routers/notebooks.py", "content": "from typing import List, Optional\n\nfrom fastapi import APIRouter, HTTPException, Query\nfrom loguru import logger\n\nfrom api.models import (\n NotebookCreate,\n NotebookDeletePreview,\n NotebookDeleteResponse,\n NotebookResponse,\n NotebookUpdate,\n)\nfrom open_notebook.database.repository import ensure_record_id, repo_query\nfrom open_notebook.domain.notebook import Notebook, Source\nfrom open_notebook.exceptions import InvalidInputError\n\nrouter = APIRouter()\n\n\n@router.get(\"/notebooks\", response_model=List[NotebookResponse])\nasync def get_notebooks(\n archived: Optional[bool] = Query(None, description=\"Filter by archived status\"),\n order_by: str = Query(\"updated desc\", description=\"Order by field and direction\"),\n):\n \"\"\"Get all notebooks with optional filtering and ordering.\"\"\"\n try:\n # Build the query with counts\n query = f\"\"\"\n SELECT *,\n count(<-reference.in) as source_count,\n count(<-artifact.in) as note_count\n FROM notebook\n ORDER BY {order_by}\n \"\"\"\n\n result = await repo_query(query)\n\n # Filter by archived status if specified\n if archived is not None:\n result = [nb for nb in result if nb.get(\"archived\") == archived]\n\n return [\n NotebookResponse(\n id=str(nb.get(\"id\", \"\")),\n name=nb.get(\"name\", \"\"),\n description=nb.get(\"description\", \"\"),\n archived=nb.get(\"archived\", False),\n created=str(nb.get(\"created\", \"\")),\n updated=str(nb.get(\"updated\", \"\")),\n source_count=nb.get(\"source_count\", 0),\n note_count=nb.get(\"note_count\", 0),\n )\n for nb in result\n ]\n except Exception as e:\n logger.error(f\"Error fetching notebooks: {str(e)}\")\n raise HTTPException(\n status_code=500, detail=f\"Error fetching notebooks: {str(e)}\"\n )\n\n\n@router.post(\"/notebooks\", response_model=NotebookResponse)\nasync def create_notebook(notebook: NotebookCreate):\n \"\"\"Create a new notebook.\"\"\"\n try:\n new_notebook = Notebook(\n name=notebook.name,\n description=notebook.description,\n )\n await new_notebook.save()\n\n return NotebookResponse(\n id=new_notebook.id or \"\",\n name=new_notebook.name,\n description=new_notebook.description,\n archived=new_notebook.archived or False,\n created=str(new_notebook.created),\n updated=str(new_notebook.updated),\n source_count=0, # New notebook has no sources\n note_count=0, # New notebook has no notes\n )\n except InvalidInputError as e:\n raise HTTPException(status_code=400, detail=str(e))\n except Exception as e:\n logger.error(f\"Error creating notebook: {str(e)}\")\n raise HTTPException(\n status_code=500, detail=f\"Error creating notebook: {str(e)}\"\n )\n\n\n@router.get(\n \"/notebooks/{notebook_id}/delete-preview\", response_model=NotebookDeletePreview\n)\nasync def get_notebook_delete_preview(notebook_id: str):\n \"\"\"Get a preview of what will be deleted when this notebook is deleted.\"\"\"\n try:\n notebook = await Notebook.get(notebook_id)\n if not notebook:\n raise HTTPException(status_code=404, detail=\"Notebook not found\")\n\n preview = await notebook.get_delete_preview()\n\n return NotebookDeletePreview(\n notebook_id=str(notebook.id),\n notebook_name=notebook.name,\n note_count=preview[\"note_count\"],\n exclusive_source_count=preview[\"exclusive_source_count\"],\n shared_source_count=preview[\"shared_source_count\"],\n )\n except HTTPException:\n raise\n except Exception as e:\n logger.error(f\"Error getting delete preview for notebook {notebook_id}: {e}\")\n raise HTTPException(\n status_code=500,\n detail=f\"Error fetching notebook deletion preview: {str(e)}\",\n )\n\n\n@router.get(\"/notebooks/{notebook_id}\", response_model=NotebookResponse)\nasync def get_notebook(notebook_id: str):\n \"\"\"Get a specific notebook by ID.\"\"\"\n try:\n # Query with counts for single notebook\n query = \"\"\"\n SELECT *,\n count(<-reference.in) as source_count,\n count(<-artifact.in) as note_count\n FROM $notebook_id\n \"\"\"\n result = await repo_query(query, {\"notebook_id\": ensure_record_id(notebook_id)})\n\n if not result:\n raise HTTPException(status_code=404, detail=\"Notebook not found\")\n\n nb = result[0]\n return NotebookResponse(\n id=str(nb.get(\"id\", \"\")),\n name=nb.get(\"name\", \"\"),\n description=nb.get(\"description\", \"\"),\n archived=nb.get(\"archived\", False),\n created=str(nb.get(\"created\", \"\")),\n updated=str(nb.get(\"updated\", \"\")),\n source_count=nb.get(\"source_count\", 0),\n note_count=nb.get(\"note_count\", 0),\n )\n except HTTPException:\n raise\n except Exception as e:\n logger.error(f\"Error fetching notebook {notebook_id}: {str(e)}\")\n raise HTTPException(\n status_code=500, detail=f\"Error fetching notebook: {str(e)}\"\n )\n\n\n@router.put(\"/notebooks/{notebook_id}\", response_model=NotebookResponse)\nasync def update_notebook(notebook_id: str, notebook_update: NotebookUpdate):\n \"\"\"Update a notebook.\"\"\"\n try:\n notebook = await Notebook.get(notebook_id)\n if not notebook:\n raise HTTPException(status_code=404, detail=\"Notebook not found\")\n\n # Update only provided fields\n if notebook_update.name is not None:\n notebook.name = notebook_update.name\n if notebook_update.description is not None:\n notebook.description = notebook_update.description\n if notebook_update.archived is not None:\n notebook.archived = notebook_update.archived\n\n await notebook.save()\n\n # Query with counts after update\n query = \"\"\"\n SELECT *,\n count(<-reference.in) as source_count,\n count(<-artifact.in) as note_count\n FROM $notebook_id\n \"\"\"\n result = await repo_query(query, {\"notebook_id\": ensure_record_id(notebook_id)})\n\n if result:\n nb = result[0]\n return NotebookResponse(\n id=str(nb.get(\"id\", \"\")),\n name=nb.get(\"name\", \"\"),\n description=nb.get(\"description\", \"\"),\n archived=nb.get(\"archived\", False),\n created=str(nb.get(\"created\", \"\")),\n updated=str(nb.get(\"updated\", \"\")),\n source_count=nb.get(\"source_count\", 0),\n note_count=nb.get(\"note_count\", 0),\n )\n\n # Fallback if query fails\n return NotebookResponse(\n id=notebook.id or \"\",\n name=notebook.name,\n description=notebook.description,\n archived=notebook.archived or False,\n created=str(notebook.created),\n updated=str(notebook.updated),\n source_count=0,\n note_count=0,\n )\n except HTTPException:\n raise\n except InvalidInputError as e:\n raise HTTPException(status_code=400, detail=str(e))\n except Exception as e:\n logger.error(f\"Error updating notebook {notebook_id}: {str(e)}\")\n raise HTTPException(\n status_code=500, detail=f\"Error updating notebook: {str(e)}\"\n )\n\n\n@router.post(\"/notebooks/{notebook_id}/sources/{source_id}\")\nasync def add_source_to_notebook(notebook_id: str, source_id: str):\n \"\"\"Add an existing source to a notebook (create the reference).\"\"\"\n try:\n # Check if notebook exists\n notebook = await Notebook.get(notebook_id)\n if not notebook:\n raise HTTPException(status_code=404, detail=\"Notebook not found\")\n\n # Check if source exists\n source = await Source.get(source_id)\n if not source:\n raise HTTPException(status_code=404, detail=\"Source not found\")\n\n # Check if reference already exists (idempotency)\n existing_ref = await repo_query(\n \"SELECT * FROM reference WHERE out = $source_id AND in = $notebook_id\",\n {\n \"notebook_id\": ensure_record_id(notebook_id),\n \"source_id\": ensure_record_id(source_id),\n },\n )\n\n # If reference doesn't exist, create it\n if not existing_ref:\n await repo_query(\n \"RELATE $source_id->reference->$notebook_id\",\n {\n \"notebook_id\": ensure_record_id(notebook_id),\n \"source_id\": ensure_record_id(source_id),\n },\n )\n\n return {\"message\": \"Source linked to notebook successfully\"}\n except HTTPException:\n raise\n except Exception as e:\n logger.error(\n f\"Error linking source {source_id} to notebook {notebook_id}: {str(e)}\"\n )\n raise HTTPException(\n status_code=500, detail=f\"Error linking source to notebook: {str(e)}\"\n )\n\n\n@router.delete(\"/notebooks/{notebook_id}/sources/{source_id}\")\nasync def remove_source_from_notebook(notebook_id: str, source_id: str):\n \"\"\"Remove a source from a notebook (delete the reference).\"\"\"\n try:\n # Check if notebook exists\n notebook = await Notebook.get(notebook_id)\n if not notebook:\n raise HTTPException(status_code=404, detail=\"Notebook not found\")\n\n # Delete the reference record linking source to notebook\n await repo_query(\n \"DELETE FROM reference WHERE out = $notebook_id AND in = $source_id\",\n {\n \"notebook_id\": ensure_record_id(notebook_id),\n \"source_id\": ensure_record_id(source_id),\n },\n )\n\n return {\"message\": \"Source removed from notebook successfully\"}\n except HTTPException:\n raise\n except Exception as e:\n logger.error(\n f\"Error removing source {source_id} from notebook {notebook_id}: {str(e)}\"\n )\n raise HTTPException(\n status_code=500, detail=f\"Error removing source from notebook: {str(e)}\"\n )\n\n\n@router.delete(\"/notebooks/{notebook_id}\", response_model=NotebookDeleteResponse)\nasync def delete_notebook(\n notebook_id: str,\n delete_exclusive_sources: bool = Query(\n False,\n description=\"Whether to delete sources that belong only to this notebook\",\n ),\n):\n \"\"\"\n Delete a notebook with cascade deletion.\n\n Always deletes all notes associated with the notebook.\n If delete_exclusive_sources is True, also deletes sources that belong only\n to this notebook (not linked to any other notebooks).\n \"\"\"\n try:\n notebook = await Notebook.get(notebook_id)\n if not notebook:\n raise HTTPException(status_code=404, detail=\"Notebook not found\")\n\n result = await notebook.delete(delete_exclusive_sources=delete_exclusive_sources)\n\n return NotebookDeleteResponse(\n message=\"Notebook deleted successfully\",\n deleted_notes=result[\"deleted_notes\"],\n deleted_sources=result[\"deleted_sources\"],\n unlinked_sources=result[\"unlinked_sources\"],\n )\n except HTTPException:\n raise\n except Exception as e:\n logger.error(f\"Error deleting notebook {notebook_id}: {str(e)}\")\n raise HTTPException(\n status_code=500, detail=f\"Error deleting notebook: {str(e)}\"\n )\n" }, { "path": "api/routers/notes.py", "content": "from typing import List, Literal, Optional\n\nfrom fastapi import APIRouter, HTTPException, Query\nfrom loguru import logger\n\nfrom api.models import NoteCreate, NoteResponse, NoteUpdate\nfrom open_notebook.domain.notebook import Note\nfrom open_notebook.exceptions import InvalidInputError\n\nrouter = APIRouter()\n\n\n@router.get(\"/notes\", response_model=List[NoteResponse])\nasync def get_notes(\n notebook_id: Optional[str] = Query(None, description=\"Filter by notebook ID\"),\n):\n \"\"\"Get all notes with optional notebook filtering.\"\"\"\n try:\n if notebook_id:\n # Get notes for a specific notebook\n from open_notebook.domain.notebook import Notebook\n\n notebook = await Notebook.get(notebook_id)\n if not notebook:\n raise HTTPException(status_code=404, detail=\"Notebook not found\")\n notes = await notebook.get_notes()\n else:\n # Get all notes\n notes = await Note.get_all(order_by=\"updated desc\")\n\n return [\n NoteResponse(\n id=note.id or \"\",\n title=note.title,\n content=note.content,\n note_type=note.note_type,\n created=str(note.created),\n updated=str(note.updated),\n )\n for note in notes\n ]\n except HTTPException:\n raise\n except Exception as e:\n logger.error(f\"Error fetching notes: {str(e)}\")\n raise HTTPException(status_code=500, detail=f\"Error fetching notes: {str(e)}\")\n\n\n@router.post(\"/notes\", response_model=NoteResponse)\nasync def create_note(note_data: NoteCreate):\n \"\"\"Create a new note.\"\"\"\n try:\n # Auto-generate title if not provided and it's an AI note\n title = note_data.title\n if not title and note_data.note_type == \"ai\" and note_data.content:\n from open_notebook.graphs.prompt import graph as prompt_graph\n\n prompt = \"Based on the Note below, please provide a Title for this content, with max 15 words\"\n result = await prompt_graph.ainvoke(\n { # type: ignore[arg-type]\n \"input_text\": note_data.content,\n \"prompt\": prompt,\n }\n )\n title = result.get(\"output\", \"Untitled Note\")\n\n # Validate note_type\n note_type: Optional[Literal[\"human\", \"ai\"]] = None\n if note_data.note_type in (\"human\", \"ai\"):\n note_type = note_data.note_type # type: ignore[assignment]\n elif note_data.note_type is not None:\n raise HTTPException(\n status_code=400, detail=\"note_type must be 'human' or 'ai'\"\n )\n\n new_note = Note(\n title=title,\n content=note_data.content,\n note_type=note_type,\n )\n command_id = await new_note.save()\n\n # Add to notebook if specified\n if note_data.notebook_id:\n from open_notebook.domain.notebook import Notebook\n\n notebook = await Notebook.get(note_data.notebook_id)\n if not notebook:\n raise HTTPException(status_code=404, detail=\"Notebook not found\")\n await new_note.add_to_notebook(note_data.notebook_id)\n\n return NoteResponse(\n id=new_note.id or \"\",\n title=new_note.title,\n content=new_note.content,\n note_type=new_note.note_type,\n created=str(new_note.created),\n updated=str(new_note.updated),\n command_id=str(command_id) if command_id else None,\n )\n except HTTPException:\n raise\n except InvalidInputError as e:\n raise HTTPException(status_code=400, detail=str(e))\n except Exception as e:\n logger.error(f\"Error creating note: {str(e)}\")\n raise HTTPException(status_code=500, detail=f\"Error creating note: {str(e)}\")\n\n\n@router.get(\"/notes/{note_id}\", response_model=NoteResponse)\nasync def get_note(note_id: str):\n \"\"\"Get a specific note by ID.\"\"\"\n try:\n note = await Note.get(note_id)\n if not note:\n raise HTTPException(status_code=404, detail=\"Note not found\")\n\n return NoteResponse(\n id=note.id or \"\",\n title=note.title,\n content=note.content,\n note_type=note.note_type,\n created=str(note.created),\n updated=str(note.updated),\n )\n except HTTPException:\n raise\n except Exception as e:\n logger.error(f\"Error fetching note {note_id}: {str(e)}\")\n raise HTTPException(status_code=500, detail=f\"Error fetching note: {str(e)}\")\n\n\n@router.put(\"/notes/{note_id}\", response_model=NoteResponse)\nasync def update_note(note_id: str, note_update: NoteUpdate):\n \"\"\"Update a note.\"\"\"\n try:\n note = await Note.get(note_id)\n if not note:\n raise HTTPException(status_code=404, detail=\"Note not found\")\n\n # Update only provided fields\n if note_update.title is not None:\n note.title = note_update.title\n if note_update.content is not None:\n note.content = note_update.content\n if note_update.note_type is not None:\n if note_update.note_type in (\"human\", \"ai\"):\n note.note_type = note_update.note_type # type: ignore[assignment]\n else:\n raise HTTPException(\n status_code=400, detail=\"note_type must be 'human' or 'ai'\"\n )\n\n command_id = await note.save()\n\n return NoteResponse(\n id=note.id or \"\",\n title=note.title,\n content=note.content,\n note_type=note.note_type,\n created=str(note.created),\n updated=str(note.updated),\n command_id=str(command_id) if command_id else None,\n )\n except HTTPException:\n raise\n except InvalidInputError as e:\n raise HTTPException(status_code=400, detail=str(e))\n except Exception as e:\n logger.error(f\"Error updating note {note_id}: {str(e)}\")\n raise HTTPException(status_code=500, detail=f\"Error updating note: {str(e)}\")\n\n\n@router.delete(\"/notes/{note_id}\")\nasync def delete_note(note_id: str):\n \"\"\"Delete a note.\"\"\"\n try:\n note = await Note.get(note_id)\n if not note:\n raise HTTPException(status_code=404, detail=\"Note not found\")\n\n await note.delete()\n\n return {\"message\": \"Note deleted successfully\"}\n except HTTPException:\n raise\n except Exception as e:\n logger.error(f\"Error deleting note {note_id}: {str(e)}\")\n raise HTTPException(status_code=500, detail=f\"Error deleting note: {str(e)}\")\n" }, { "path": "api/routers/podcasts.py", "content": "from pathlib import Path\nfrom typing import List, Optional\nfrom urllib.parse import unquote, urlparse\n\nfrom fastapi import APIRouter, HTTPException\nfrom fastapi.responses import FileResponse\nfrom loguru import logger\nfrom pydantic import BaseModel\n\nfrom api.podcast_service import (\n PodcastGenerationRequest,\n PodcastGenerationResponse,\n PodcastService,\n)\n\nrouter = APIRouter()\n\n\nclass PodcastEpisodeResponse(BaseModel):\n id: str\n name: str\n episode_profile: dict\n speaker_profile: dict\n briefing: str\n audio_file: Optional[str] = None\n audio_url: Optional[str] = None\n transcript: Optional[dict] = None\n outline: Optional[dict] = None\n created: Optional[str] = None\n job_status: Optional[str] = None\n error_message: Optional[str] = None\n\n\ndef _resolve_audio_path(audio_file: str) -> Path:\n if audio_file.startswith(\"file://\"):\n parsed = urlparse(audio_file)\n return Path(unquote(parsed.path))\n return Path(audio_file)\n\n\n@router.post(\"/podcasts/generate\", response_model=PodcastGenerationResponse)\nasync def generate_podcast(request: PodcastGenerationRequest):\n \"\"\"\n Generate a podcast episode using Episode Profiles.\n Returns immediately with job ID for status tracking.\n \"\"\"\n try:\n job_id = await PodcastService.submit_generation_job(\n episode_profile_name=request.episode_profile,\n speaker_profile_name=request.speaker_profile,\n episode_name=request.episode_name,\n notebook_id=request.notebook_id,\n content=request.content,\n briefing_suffix=request.briefing_suffix,\n )\n\n return PodcastGenerationResponse(\n job_id=job_id,\n status=\"submitted\",\n message=f\"Podcast generation started for episode '{request.episode_name}'\",\n episode_profile=request.episode_profile,\n episode_name=request.episode_name,\n )\n\n except Exception as e:\n logger.error(f\"Error generating podcast: {str(e)}\")\n raise HTTPException(\n status_code=500, detail=\"Failed to generate podcast\"\n )\n\n\n@router.get(\"/podcasts/jobs/{job_id}\")\nasync def get_podcast_job_status(job_id: str):\n \"\"\"Get the status of a podcast generation job\"\"\"\n try:\n status_data = await PodcastService.get_job_status(job_id)\n return status_data\n\n except Exception as e:\n logger.error(f\"Error fetching podcast job status: {str(e)}\")\n raise HTTPException(\n status_code=500, detail=\"Failed to fetch job status\"\n )\n\n\n@router.get(\"/podcasts/episodes\", response_model=List[PodcastEpisodeResponse])\nasync def list_podcast_episodes():\n \"\"\"List all podcast episodes\"\"\"\n try:\n episodes = await PodcastService.list_episodes()\n\n response_episodes = []\n for episode in episodes:\n # Skip incomplete episodes without command or audio\n if not episode.command and not episode.audio_file:\n continue\n\n # Get job status and error message if available\n job_status = None\n error_message = None\n if episode.command:\n try:\n detail = await episode.get_job_detail()\n job_status = detail[\"status\"]\n error_message = detail[\"error_message\"]\n except Exception:\n job_status = \"unknown\"\n else:\n # No command but has audio file = completed import\n job_status = \"completed\"\n\n audio_url = None\n if episode.audio_file:\n audio_path = _resolve_audio_path(episode.audio_file)\n if audio_path.exists():\n audio_url = f\"/api/podcasts/episodes/{episode.id}/audio\"\n\n response_episodes.append(\n PodcastEpisodeResponse(\n id=str(episode.id),\n name=episode.name,\n episode_profile=episode.episode_profile,\n speaker_profile=episode.speaker_profile,\n briefing=episode.briefing,\n audio_file=episode.audio_file,\n audio_url=audio_url,\n transcript=episode.transcript,\n outline=episode.outline,\n created=str(episode.created) if episode.created else None,\n job_status=job_status,\n error_message=error_message,\n )\n )\n\n return response_episodes\n\n except Exception as e:\n logger.error(f\"Error listing podcast episodes: {str(e)}\")\n raise HTTPException(\n status_code=500, detail=\"Failed to list podcast episodes\"\n )\n\n\n@router.get(\"/podcasts/episodes/{episode_id}\", response_model=PodcastEpisodeResponse)\nasync def get_podcast_episode(episode_id: str):\n \"\"\"Get a specific podcast episode\"\"\"\n try:\n episode = await PodcastService.get_episode(episode_id)\n\n # Get job status and error message if available\n job_status = None\n error_message = None\n if episode.command:\n try:\n detail = await episode.get_job_detail()\n job_status = detail[\"status\"]\n error_message = detail[\"error_message\"]\n except Exception:\n job_status = \"unknown\"\n else:\n # No command but has audio file = completed import\n job_status = \"completed\" if episode.audio_file else \"unknown\"\n\n audio_url = None\n if episode.audio_file:\n audio_path = _resolve_audio_path(episode.audio_file)\n if audio_path.exists():\n audio_url = f\"/api/podcasts/episodes/{episode.id}/audio\"\n\n return PodcastEpisodeResponse(\n id=str(episode.id),\n name=episode.name,\n episode_profile=episode.episode_profile,\n speaker_profile=episode.speaker_profile,\n briefing=episode.briefing,\n audio_file=episode.audio_file,\n audio_url=audio_url,\n transcript=episode.transcript,\n outline=episode.outline,\n created=str(episode.created) if episode.created else None,\n job_status=job_status,\n error_message=error_message,\n )\n\n except Exception as e:\n logger.error(f\"Error fetching podcast episode: {str(e)}\")\n raise HTTPException(status_code=404, detail=\"Episode not found\")\n\n\n@router.get(\"/podcasts/episodes/{episode_id}/audio\")\nasync def stream_podcast_episode_audio(episode_id: str):\n \"\"\"Stream the audio file associated with a podcast episode\"\"\"\n try:\n episode = await PodcastService.get_episode(episode_id)\n except HTTPException:\n raise\n except Exception as e:\n logger.error(f\"Error fetching podcast episode for audio: {str(e)}\")\n raise HTTPException(status_code=404, detail=\"Episode not found\")\n\n if not episode.audio_file:\n raise HTTPException(status_code=404, detail=\"Episode has no audio file\")\n\n audio_path = _resolve_audio_path(episode.audio_file)\n if not audio_path.exists():\n raise HTTPException(status_code=404, detail=\"Audio file not found on disk\")\n\n return FileResponse(\n audio_path,\n media_type=\"audio/mpeg\",\n filename=audio_path.name,\n )\n\n\n@router.post(\"/podcasts/episodes/{episode_id}/retry\")\nasync def retry_podcast_episode(episode_id: str):\n \"\"\"Retry a failed podcast episode by deleting it and submitting a new job\"\"\"\n try:\n episode = await PodcastService.get_episode(episode_id)\n\n # Validate episode is in a failed state\n detail = await episode.get_job_detail()\n if detail[\"status\"] not in (\"failed\", \"error\"):\n raise HTTPException(\n status_code=400,\n detail=f\"Episode is not in a failed state (current: {detail['status']})\",\n )\n\n # Extract params for re-submission\n ep_profile_name = episode.episode_profile.get(\"name\")\n sp_profile_name = episode.speaker_profile.get(\"name\")\n episode_name = episode.name\n content = episode.content\n\n if not ep_profile_name or not sp_profile_name:\n raise HTTPException(\n status_code=400,\n detail=\"Cannot retry: episode or speaker profile name missing from stored data\",\n )\n\n # Delete audio file if any\n if episode.audio_file:\n audio_path = _resolve_audio_path(episode.audio_file)\n if audio_path.exists():\n try:\n audio_path.unlink()\n except Exception as e:\n logger.warning(f\"Failed to delete audio file {audio_path}: {e}\")\n\n # Delete the failed episode\n await episode.delete()\n\n # Submit a new job\n job_id = await PodcastService.submit_generation_job(\n episode_profile_name=ep_profile_name,\n speaker_profile_name=sp_profile_name,\n episode_name=episode_name,\n content=content,\n )\n\n return {\"job_id\": job_id, \"message\": \"Retry submitted successfully\"}\n\n except HTTPException:\n raise\n except Exception as e:\n logger.error(f\"Error retrying podcast episode: {str(e)}\")\n raise HTTPException(\n status_code=500, detail=\"Failed to retry episode\"\n )\n\n\n@router.delete(\"/podcasts/episodes/{episode_id}\")\nasync def delete_podcast_episode(episode_id: str):\n \"\"\"Delete a podcast episode and its associated audio file\"\"\"\n try:\n # Get the episode first to check if it exists and get the audio file path\n episode = await PodcastService.get_episode(episode_id)\n\n # Delete the physical audio file if it exists\n if episode.audio_file:\n audio_path = _resolve_audio_path(episode.audio_file)\n if audio_path.exists():\n try:\n audio_path.unlink()\n logger.info(f\"Deleted audio file: {audio_path}\")\n except Exception as e:\n logger.warning(f\"Failed to delete audio file {audio_path}: {e}\")\n\n # Delete the episode from the database\n await episode.delete()\n\n logger.info(f\"Deleted podcast episode: {episode_id}\")\n return {\"message\": \"Episode deleted successfully\", \"episode_id\": episode_id}\n\n except Exception as e:\n logger.error(f\"Error deleting podcast episode: {str(e)}\")\n raise HTTPException(\n status_code=500, detail=\"Failed to delete episode\"\n )\n" }, { "path": "api/routers/search.py", "content": "import json\nfrom typing import AsyncGenerator\n\nfrom fastapi import APIRouter, HTTPException\nfrom fastapi.responses import StreamingResponse\nfrom loguru import logger\n\nfrom api.models import AskRequest, AskResponse, SearchRequest, SearchResponse\nfrom open_notebook.ai.models import Model, model_manager\nfrom open_notebook.domain.notebook import text_search, vector_search\nfrom open_notebook.exceptions import DatabaseOperationError, InvalidInputError\nfrom open_notebook.graphs.ask import graph as ask_graph\n\nrouter = APIRouter()\n\n\n@router.post(\"/search\", response_model=SearchResponse)\nasync def search_knowledge_base(search_request: SearchRequest):\n \"\"\"Search the knowledge base using text or vector search.\"\"\"\n try:\n if search_request.type == \"vector\":\n # Check if embedding model is available for vector search\n if not await model_manager.get_embedding_model():\n raise HTTPException(\n status_code=400,\n detail=\"Vector search requires an embedding model. Please configure one in the Models section.\",\n )\n\n results = await vector_search(\n keyword=search_request.query,\n results=search_request.limit,\n source=search_request.search_sources,\n note=search_request.search_notes,\n minimum_score=search_request.minimum_score,\n )\n else:\n # Text search\n results = await text_search(\n keyword=search_request.query,\n results=search_request.limit,\n source=search_request.search_sources,\n note=search_request.search_notes,\n )\n\n return SearchResponse(\n results=results or [],\n total_count=len(results) if results else 0,\n search_type=search_request.type,\n )\n\n except InvalidInputError as e:\n raise HTTPException(status_code=400, detail=str(e))\n except DatabaseOperationError as e:\n logger.error(f\"Database error during search: {str(e)}\")\n raise HTTPException(status_code=500, detail=f\"Search failed: {str(e)}\")\n except Exception as e:\n logger.error(f\"Unexpected error during search: {str(e)}\")\n raise HTTPException(status_code=500, detail=f\"Search failed: {str(e)}\")\n\n\nasync def stream_ask_response(\n question: str, strategy_model: Model, answer_model: Model, final_answer_model: Model\n) -> AsyncGenerator[str, None]:\n \"\"\"Stream the ask response as Server-Sent Events.\"\"\"\n try:\n final_answer = None\n\n async for chunk in ask_graph.astream(\n input=dict(question=question), # type: ignore[arg-type]\n config=dict(\n configurable=dict(\n strategy_model=strategy_model.id,\n answer_model=answer_model.id,\n final_answer_model=final_answer_model.id,\n )\n ),\n stream_mode=\"updates\",\n ):\n if \"agent\" in chunk:\n strategy_data = {\n \"type\": \"strategy\",\n \"reasoning\": chunk[\"agent\"][\"strategy\"].reasoning,\n \"searches\": [\n {\"term\": search.term, \"instructions\": search.instructions}\n for search in chunk[\"agent\"][\"strategy\"].searches\n ],\n }\n yield f\"data: {json.dumps(strategy_data)}\\n\\n\"\n\n elif \"provide_answer\" in chunk:\n for answer in chunk[\"provide_answer\"][\"answers\"]:\n answer_data = {\"type\": \"answer\", \"content\": answer}\n yield f\"data: {json.dumps(answer_data)}\\n\\n\"\n\n elif \"write_final_answer\" in chunk:\n final_answer = chunk[\"write_final_answer\"][\"final_answer\"]\n final_data = {\"type\": \"final_answer\", \"content\": final_answer}\n yield f\"data: {json.dumps(final_data)}\\n\\n\"\n\n # Send completion signal\n completion_data = {\"type\": \"complete\", \"final_answer\": final_answer}\n yield f\"data: {json.dumps(completion_data)}\\n\\n\"\n\n except Exception as e:\n from open_notebook.utils.error_classifier import classify_error\n\n _, user_message = classify_error(e)\n logger.error(f\"Error in ask streaming: {str(e)}\")\n error_data = {\"type\": \"error\", \"message\": user_message}\n yield f\"data: {json.dumps(error_data)}\\n\\n\"\n\n\n@router.post(\"/search/ask\")\nasync def ask_knowledge_base(ask_request: AskRequest):\n \"\"\"Ask the knowledge base a question using AI models.\"\"\"\n try:\n # Validate models exist\n strategy_model = await Model.get(ask_request.strategy_model)\n answer_model = await Model.get(ask_request.answer_model)\n final_answer_model = await Model.get(ask_request.final_answer_model)\n\n if not strategy_model:\n raise HTTPException(\n status_code=400,\n detail=f\"Strategy model {ask_request.strategy_model} not found\",\n )\n if not answer_model:\n raise HTTPException(\n status_code=400,\n detail=f\"Answer model {ask_request.answer_model} not found\",\n )\n if not final_answer_model:\n raise HTTPException(\n status_code=400,\n detail=f\"Final answer model {ask_request.final_answer_model} not found\",\n )\n\n # Check if embedding model is available\n if not await model_manager.get_embedding_model():\n raise HTTPException(\n status_code=400,\n detail=\"Ask feature requires an embedding model. Please configure one in the Models section.\",\n )\n\n # For streaming response\n return StreamingResponse(\n stream_ask_response(\n ask_request.question, strategy_model, answer_model, final_answer_model\n ),\n media_type=\"text/plain\",\n )\n\n except HTTPException:\n raise\n except Exception as e:\n logger.error(f\"Error in ask endpoint: {str(e)}\")\n raise HTTPException(status_code=500, detail=f\"Ask operation failed: {str(e)}\")\n\n\n@router.post(\"/search/ask/simple\", response_model=AskResponse)\nasync def ask_knowledge_base_simple(ask_request: AskRequest):\n \"\"\"Ask the knowledge base a question and return a simple response (non-streaming).\"\"\"\n try:\n # Validate models exist\n strategy_model = await Model.get(ask_request.strategy_model)\n answer_model = await Model.get(ask_request.answer_model)\n final_answer_model = await Model.get(ask_request.final_answer_model)\n\n if not strategy_model:\n raise HTTPException(\n status_code=400,\n detail=f\"Strategy model {ask_request.strategy_model} not found\",\n )\n if not answer_model:\n raise HTTPException(\n status_code=400,\n detail=f\"Answer model {ask_request.answer_model} not found\",\n )\n if not final_answer_model:\n raise HTTPException(\n status_code=400,\n detail=f\"Final answer model {ask_request.final_answer_model} not found\",\n )\n\n # Check if embedding model is available\n if not await model_manager.get_embedding_model():\n raise HTTPException(\n status_code=400,\n detail=\"Ask feature requires an embedding model. Please configure one in the Models section.\",\n )\n\n # Run the ask graph and get final result\n final_answer = None\n async for chunk in ask_graph.astream(\n input=dict(question=ask_request.question), # type: ignore[arg-type]\n config=dict(\n configurable=dict(\n strategy_model=strategy_model.id,\n answer_model=answer_model.id,\n final_answer_model=final_answer_model.id,\n )\n ),\n stream_mode=\"updates\",\n ):\n if \"write_final_answer\" in chunk:\n final_answer = chunk[\"write_final_answer\"][\"final_answer\"]\n\n if not final_answer:\n raise HTTPException(status_code=500, detail=\"No answer generated\")\n\n return AskResponse(answer=final_answer, question=ask_request.question)\n\n except HTTPException:\n raise\n except Exception as e:\n logger.error(f\"Error in ask simple endpoint: {str(e)}\")\n raise HTTPException(status_code=500, detail=f\"Ask operation failed: {str(e)}\")\n" }, { "path": "api/routers/settings.py", "content": "from fastapi import APIRouter, HTTPException\nfrom loguru import logger\n\nfrom api.models import SettingsResponse, SettingsUpdate\nfrom open_notebook.domain.content_settings import ContentSettings\nfrom open_notebook.exceptions import InvalidInputError\n\nrouter = APIRouter()\n\n\n@router.get(\"/settings\", response_model=SettingsResponse)\nasync def get_settings():\n \"\"\"Get all application settings.\"\"\"\n try:\n settings: ContentSettings = await ContentSettings.get_instance() # type: ignore[assignment]\n\n return SettingsResponse(\n default_content_processing_engine_doc=settings.default_content_processing_engine_doc,\n default_content_processing_engine_url=settings.default_content_processing_engine_url,\n default_embedding_option=settings.default_embedding_option,\n auto_delete_files=settings.auto_delete_files,\n youtube_preferred_languages=settings.youtube_preferred_languages,\n )\n except Exception as e:\n logger.error(f\"Error fetching settings: {str(e)}\")\n raise HTTPException(\n status_code=500, detail=\"Error fetching settings\"\n )\n\n\n@router.put(\"/settings\", response_model=SettingsResponse)\nasync def update_settings(settings_update: SettingsUpdate):\n \"\"\"Update application settings.\"\"\"\n try:\n settings: ContentSettings = await ContentSettings.get_instance() # type: ignore[assignment]\n\n # Update only provided fields\n if settings_update.default_content_processing_engine_doc is not None:\n # Cast to proper literal type\n from typing import Literal, cast\n\n settings.default_content_processing_engine_doc = cast(\n Literal[\"auto\", \"docling\", \"simple\"],\n settings_update.default_content_processing_engine_doc,\n )\n if settings_update.default_content_processing_engine_url is not None:\n from typing import Literal, cast\n\n settings.default_content_processing_engine_url = cast(\n Literal[\"auto\", \"firecrawl\", \"jina\", \"simple\"],\n settings_update.default_content_processing_engine_url,\n )\n if settings_update.default_embedding_option is not None:\n from typing import Literal, cast\n\n settings.default_embedding_option = cast(\n Literal[\"ask\", \"always\", \"never\"],\n settings_update.default_embedding_option,\n )\n if settings_update.auto_delete_files is not None:\n from typing import Literal, cast\n\n settings.auto_delete_files = cast(\n Literal[\"yes\", \"no\"], settings_update.auto_delete_files\n )\n if settings_update.youtube_preferred_languages is not None:\n settings.youtube_preferred_languages = (\n settings_update.youtube_preferred_languages\n )\n\n await settings.update()\n\n return SettingsResponse(\n default_content_processing_engine_doc=settings.default_content_processing_engine_doc,\n default_content_processing_engine_url=settings.default_content_processing_engine_url,\n default_embedding_option=settings.default_embedding_option,\n auto_delete_files=settings.auto_delete_files,\n youtube_preferred_languages=settings.youtube_preferred_languages,\n )\n except HTTPException:\n raise\n except InvalidInputError as e:\n raise HTTPException(status_code=400, detail=str(e))\n except Exception as e:\n logger.error(f\"Error updating settings: {str(e)}\")\n raise HTTPException(\n status_code=500, detail=\"Error updating settings\"\n )\n" }, { "path": "api/routers/source_chat.py", "content": "import asyncio\nimport json\nfrom typing import AsyncGenerator, List, Optional\n\nfrom fastapi import APIRouter, HTTPException, Path\nfrom fastapi.responses import StreamingResponse\nfrom langchain_core.messages import HumanMessage\nfrom langchain_core.runnables import RunnableConfig\nfrom loguru import logger\nfrom pydantic import BaseModel, Field\n\nfrom open_notebook.database.repository import ensure_record_id, repo_query\nfrom open_notebook.domain.notebook import ChatSession, Source\nfrom open_notebook.exceptions import (\n NotFoundError,\n)\nfrom open_notebook.graphs.source_chat import source_chat_graph as source_chat_graph\nfrom open_notebook.utils.graph_utils import get_session_message_count\n\nrouter = APIRouter()\n\n\n# Request/Response models\nclass CreateSourceChatSessionRequest(BaseModel):\n source_id: str = Field(..., description=\"Source ID to create chat session for\")\n title: Optional[str] = Field(None, description=\"Optional session title\")\n model_override: Optional[str] = Field(\n None, description=\"Optional model override for this session\"\n )\n\nclass UpdateSourceChatSessionRequest(BaseModel):\n title: Optional[str] = Field(None, description=\"New session title\")\n model_override: Optional[str] = Field(\n None, description=\"Model override for this session\"\n )\n\nclass ChatMessage(BaseModel):\n id: str = Field(..., description=\"Message ID\")\n type: str = Field(..., description=\"Message type (human|ai)\")\n content: str = Field(..., description=\"Message content\")\n timestamp: Optional[str] = Field(None, description=\"Message timestamp\")\n\n\nclass ContextIndicator(BaseModel):\n sources: List[str] = Field(\n default_factory=list, description=\"Source IDs used in context\"\n )\n insights: List[str] = Field(\n default_factory=list, description=\"Insight IDs used in context\"\n )\n notes: List[str] = Field(\n default_factory=list, description=\"Note IDs used in context\"\n )\n\nclass SourceChatSessionResponse(BaseModel):\n id: str = Field(..., description=\"Session ID\")\n title: str = Field(..., description=\"Session title\")\n source_id: str = Field(..., description=\"Source ID\")\n model_override: Optional[str] = Field(\n None, description=\"Model override for this session\"\n )\n created: str = Field(..., description=\"Creation timestamp\")\n updated: str = Field(..., description=\"Last update timestamp\")\n message_count: Optional[int] = Field(\n None, description=\"Number of messages in session\"\n )\n\nclass SourceChatSessionWithMessagesResponse(SourceChatSessionResponse):\n messages: List[ChatMessage] = Field(\n default_factory=list, description=\"Session messages\"\n )\n context_indicators: Optional[ContextIndicator] = Field(\n None, description=\"Context indicators from last response\"\n )\n\nclass SendMessageRequest(BaseModel):\n message: str = Field(..., description=\"User message content\")\n model_override: Optional[str] = Field(\n None, description=\"Optional model override for this message\"\n )\n\nclass SuccessResponse(BaseModel):\n success: bool = Field(True, description=\"Operation success status\")\n message: str = Field(..., description=\"Success message\")\n\n\n@router.post(\n \"/sources/{source_id}/chat/sessions\", response_model=SourceChatSessionResponse\n)\nasync def create_source_chat_session(\n request: CreateSourceChatSessionRequest,\n source_id: str = Path(..., description=\"Source ID\"),\n):\n \"\"\"Create a new chat session for a source.\"\"\"\n try:\n # Verify source exists\n full_source_id = (\n source_id if source_id.startswith(\"source:\") else f\"source:{source_id}\"\n )\n source = await Source.get(full_source_id)\n if not source:\n raise HTTPException(status_code=404, detail=\"Source not found\")\n\n # Create new session with model_override support\n session = ChatSession(\n title=request.title or f\"Source Chat {asyncio.get_event_loop().time():.0f}\",\n model_override=request.model_override,\n )\n await session.save()\n\n # Relate session to source using \"refers_to\" relation\n await session.relate(\"refers_to\", full_source_id)\n\n return SourceChatSessionResponse(\n id=session.id or \"\",\n title=session.title or \"Untitled Session\",\n source_id=source_id,\n model_override=session.model_override,\n created=str(session.created),\n updated=str(session.updated),\n message_count=0,\n )\n except NotFoundError:\n raise HTTPException(status_code=404, detail=\"Source not found\")\n except Exception as e:\n logger.error(f\"Error creating source chat session: {str(e)}\")\n raise HTTPException(\n status_code=500, detail=f\"Error creating source chat session: {str(e)}\"\n )\n\n\n@router.get(\n \"/sources/{source_id}/chat/sessions\", response_model=List[SourceChatSessionResponse]\n)\nasync def get_source_chat_sessions(source_id: str = Path(..., description=\"Source ID\")):\n \"\"\"Get all chat sessions for a source.\"\"\"\n try:\n # Verify source exists\n full_source_id = (\n source_id if source_id.startswith(\"source:\") else f\"source:{source_id}\"\n )\n source = await Source.get(full_source_id)\n if not source:\n raise HTTPException(status_code=404, detail=\"Source not found\")\n\n # Get sessions that refer to this source - first get relations, then sessions\n relations = await repo_query(\n \"SELECT in FROM refers_to WHERE out = $source_id\",\n {\"source_id\": ensure_record_id(full_source_id)},\n )\n\n sessions = []\n for relation in relations:\n session_id_raw = relation.get(\"in\")\n if session_id_raw:\n session_id = str(session_id_raw)\n\n session_result = await repo_query(f\"SELECT * FROM {session_id_raw}\")\n if session_result and len(session_result) > 0:\n session_data = session_result[0]\n\n # Get message count from LangGraph state\n msg_count = await get_session_message_count(\n source_chat_graph, session_id\n )\n\n sessions.append(\n SourceChatSessionResponse(\n id=session_data.get(\"id\") or \"\",\n title=session_data.get(\"title\") or \"Untitled Session\",\n source_id=source_id,\n model_override=session_data.get(\"model_override\"),\n created=str(session_data.get(\"created\")),\n updated=str(session_data.get(\"updated\")),\n message_count=msg_count,\n )\n )\n\n # Sort sessions by created date (newest first)\n sessions.sort(key=lambda x: x.created, reverse=True)\n return sessions\n except NotFoundError:\n raise HTTPException(status_code=404, detail=\"Source not found\")\n except Exception as e:\n logger.error(f\"Error fetching source chat sessions: {str(e)}\")\n raise HTTPException(\n status_code=500, detail=f\"Error fetching source chat sessions: {str(e)}\"\n )\n\n\n@router.get(\n \"/sources/{source_id}/chat/sessions/{session_id}\",\n response_model=SourceChatSessionWithMessagesResponse,\n)\nasync def get_source_chat_session(\n source_id: str = Path(..., description=\"Source ID\"),\n session_id: str = Path(..., description=\"Session ID\"),\n):\n \"\"\"Get a specific source chat session with its messages.\"\"\"\n try:\n # Verify source exists\n full_source_id = (\n source_id if source_id.startswith(\"source:\") else f\"source:{source_id}\"\n )\n source = await Source.get(full_source_id)\n if not source:\n raise HTTPException(status_code=404, detail=\"Source not found\")\n\n # Get session\n full_session_id = (\n session_id\n if session_id.startswith(\"chat_session:\")\n else f\"chat_session:{session_id}\"\n )\n session = await ChatSession.get(full_session_id)\n if not session:\n raise HTTPException(status_code=404, detail=\"Session not found\")\n\n # Verify session is related to this source\n relation_query = await repo_query(\n \"SELECT * FROM refers_to WHERE in = $session_id AND out = $source_id\",\n {\n \"session_id\": ensure_record_id(full_session_id),\n \"source_id\": ensure_record_id(full_source_id),\n },\n )\n\n if not relation_query:\n raise HTTPException(\n status_code=404, detail=\"Session not found for this source\"\n )\n\n # Get session state from LangGraph to retrieve messages\n # Use sync get_state() in a thread since SqliteSaver doesn't support async\n thread_state = await asyncio.to_thread(\n source_chat_graph.get_state,\n config=RunnableConfig(configurable={\"thread_id\": full_session_id}),\n )\n\n # Extract messages from state\n messages: list[ChatMessage] = []\n context_indicators = None\n\n if thread_state and thread_state.values:\n # Extract messages\n if \"messages\" in thread_state.values:\n for msg in thread_state.values[\"messages\"]:\n messages.append(\n ChatMessage(\n id=getattr(msg, \"id\", f\"msg_{len(messages)}\"),\n type=msg.type if hasattr(msg, \"type\") else \"unknown\",\n content=msg.content\n if hasattr(msg, \"content\")\n else str(msg),\n timestamp=None, # LangChain messages don't have timestamps by default\n )\n )\n\n # Extract context indicators from the last state\n if \"context_indicators\" in thread_state.values:\n context_data = thread_state.values[\"context_indicators\"]\n context_indicators = ContextIndicator(\n sources=context_data.get(\"sources\", []),\n insights=context_data.get(\"insights\", []),\n notes=context_data.get(\"notes\", []),\n )\n\n return SourceChatSessionWithMessagesResponse(\n id=session.id or \"\",\n title=session.title or \"Untitled Session\",\n source_id=source_id,\n model_override=getattr(session, \"model_override\", None),\n created=str(session.created),\n updated=str(session.updated),\n message_count=len(messages),\n messages=messages,\n context_indicators=context_indicators,\n )\n except NotFoundError:\n raise HTTPException(status_code=404, detail=\"Source or session not found\")\n except Exception as e:\n logger.error(f\"Error fetching source chat session: {str(e)}\")\n raise HTTPException(\n status_code=500, detail=f\"Error fetching source chat session: {str(e)}\"\n )\n\n\n@router.put(\n \"/sources/{source_id}/chat/sessions/{session_id}\",\n response_model=SourceChatSessionResponse,\n)\nasync def update_source_chat_session(\n request: UpdateSourceChatSessionRequest,\n source_id: str = Path(..., description=\"Source ID\"),\n session_id: str = Path(..., description=\"Session ID\"),\n):\n \"\"\"Update source chat session title and/or model override.\"\"\"\n try:\n # Verify source exists\n full_source_id = (\n source_id if source_id.startswith(\"source:\") else f\"source:{source_id}\"\n )\n source = await Source.get(full_source_id)\n if not source:\n raise HTTPException(status_code=404, detail=\"Source not found\")\n\n # Get session\n full_session_id = (\n session_id\n if session_id.startswith(\"chat_session:\")\n else f\"chat_session:{session_id}\"\n )\n session = await ChatSession.get(full_session_id)\n if not session:\n raise HTTPException(status_code=404, detail=\"Session not found\")\n\n # Verify session is related to this source\n relation_query = await repo_query(\n \"SELECT * FROM refers_to WHERE in = $session_id AND out = $source_id\",\n {\n \"session_id\": ensure_record_id(full_session_id),\n \"source_id\": ensure_record_id(full_source_id),\n },\n )\n\n if not relation_query:\n raise HTTPException(\n status_code=404, detail=\"Session not found for this source\"\n )\n\n # Update session fields\n if request.title is not None:\n session.title = request.title\n if request.model_override is not None:\n session.model_override = request.model_override\n\n await session.save()\n\n # Get message count from LangGraph state\n msg_count = await get_session_message_count(source_chat_graph, full_session_id)\n\n return SourceChatSessionResponse(\n id=session.id or \"\",\n title=session.title or \"Untitled Session\",\n source_id=source_id,\n model_override=getattr(session, \"model_override\", None),\n created=str(session.created),\n updated=str(session.updated),\n message_count=msg_count,\n )\n except NotFoundError:\n raise HTTPException(status_code=404, detail=\"Source or session not found\")\n except Exception as e:\n logger.error(f\"Error updating source chat session: {str(e)}\")\n raise HTTPException(\n status_code=500, detail=f\"Error updating source chat session: {str(e)}\"\n )\n\n\n@router.delete(\n \"/sources/{source_id}/chat/sessions/{session_id}\", response_model=SuccessResponse\n)\nasync def delete_source_chat_session(\n source_id: str = Path(..., description=\"Source ID\"),\n session_id: str = Path(..., description=\"Session ID\"),\n):\n \"\"\"Delete a source chat session.\"\"\"\n try:\n # Verify source exists\n full_source_id = (\n source_id if source_id.startswith(\"source:\") else f\"source:{source_id}\"\n )\n source = await Source.get(full_source_id)\n if not source:\n raise HTTPException(status_code=404, detail=\"Source not found\")\n\n # Get session\n full_session_id = (\n session_id\n if session_id.startswith(\"chat_session:\")\n else f\"chat_session:{session_id}\"\n )\n session = await ChatSession.get(full_session_id)\n if not session:\n raise HTTPException(status_code=404, detail=\"Session not found\")\n\n # Verify session is related to this source\n relation_query = await repo_query(\n \"SELECT * FROM refers_to WHERE in = $session_id AND out = $source_id\",\n {\n \"session_id\": ensure_record_id(full_session_id),\n \"source_id\": ensure_record_id(full_source_id),\n },\n )\n\n if not relation_query:\n raise HTTPException(\n status_code=404, detail=\"Session not found for this source\"\n )\n\n await session.delete()\n\n return SuccessResponse(\n success=True, message=\"Source chat session deleted successfully\"\n )\n except NotFoundError:\n raise HTTPException(status_code=404, detail=\"Source or session not found\")\n except Exception as e:\n logger.error(f\"Error deleting source chat session: {str(e)}\")\n raise HTTPException(\n status_code=500, detail=f\"Error deleting source chat session: {str(e)}\"\n )\n\n\nasync def stream_source_chat_response(\n session_id: str, source_id: str, message: str, model_override: Optional[str] = None\n) -> AsyncGenerator[str, None]:\n \"\"\"Stream the source chat response as Server-Sent Events.\"\"\"\n try:\n # Get current state\n # Use sync get_state() in a thread since SqliteSaver doesn't support async\n current_state = await asyncio.to_thread(\n source_chat_graph.get_state,\n config=RunnableConfig(configurable={\"thread_id\": session_id}),\n )\n\n # Prepare state for execution\n state_values = current_state.values if current_state else {}\n state_values[\"messages\"] = state_values.get(\"messages\", [])\n state_values[\"source_id\"] = source_id\n state_values[\"model_override\"] = model_override\n\n # Add user message to state\n user_message = HumanMessage(content=message)\n state_values[\"messages\"].append(user_message)\n\n # Send user message event\n user_event = {\"type\": \"user_message\", \"content\": message, \"timestamp\": None}\n yield f\"data: {json.dumps(user_event)}\\n\\n\"\n\n # Execute source chat graph synchronously (like notebook chat does)\n result = source_chat_graph.invoke(\n input=state_values, # type: ignore[arg-type]\n config=RunnableConfig(\n configurable={\"thread_id\": session_id, \"model_id\": model_override}\n ),\n )\n\n # Stream the complete AI response\n if \"messages\" in result:\n for msg in result[\"messages\"]:\n if hasattr(msg, \"type\") and msg.type == \"ai\":\n ai_event = {\n \"type\": \"ai_message\",\n \"content\": msg.content if hasattr(msg, \"content\") else str(msg),\n \"timestamp\": None,\n }\n yield f\"data: {json.dumps(ai_event)}\\n\\n\"\n\n # Stream context indicators\n if \"context_indicators\" in result:\n context_event = {\n \"type\": \"context_indicators\",\n \"data\": result[\"context_indicators\"],\n }\n yield f\"data: {json.dumps(context_event)}\\n\\n\"\n\n # Send completion signal\n completion_event = {\"type\": \"complete\"}\n yield f\"data: {json.dumps(completion_event)}\\n\\n\"\n\n except Exception as e:\n from open_notebook.utils.error_classifier import classify_error\n\n _, user_message = classify_error(e)\n logger.error(f\"Error in source chat streaming: {str(e)}\")\n error_event = {\"type\": \"error\", \"message\": user_message}\n yield f\"data: {json.dumps(error_event)}\\n\\n\"\n\n\n@router.post(\"/sources/{source_id}/chat/sessions/{session_id}/messages\")\nasync def send_message_to_source_chat(\n request: SendMessageRequest,\n source_id: str = Path(..., description=\"Source ID\"),\n session_id: str = Path(..., description=\"Session ID\"),\n):\n \"\"\"Send a message to source chat session with SSE streaming response.\"\"\"\n try:\n # Verify source exists\n full_source_id = (\n source_id if source_id.startswith(\"source:\") else f\"source:{source_id}\"\n )\n source = await Source.get(full_source_id)\n if not source:\n raise HTTPException(status_code=404, detail=\"Source not found\")\n\n # Verify session exists and is related to source\n full_session_id = (\n session_id\n if session_id.startswith(\"chat_session:\")\n else f\"chat_session:{session_id}\"\n )\n session = await ChatSession.get(full_session_id)\n if not session:\n raise HTTPException(status_code=404, detail=\"Session not found\")\n\n # Verify session is related to this source\n relation_query = await repo_query(\n \"SELECT * FROM refers_to WHERE in = $session_id AND out = $source_id\",\n {\n \"session_id\": ensure_record_id(full_session_id),\n \"source_id\": ensure_record_id(full_source_id),\n },\n )\n\n if not relation_query:\n raise HTTPException(\n status_code=404, detail=\"Session not found for this source\"\n )\n\n if not request.message:\n raise HTTPException(status_code=400, detail=\"Message content is required\")\n\n # Determine model override (request override takes precedence over session override)\n model_override = request.model_override or getattr(\n session, \"model_override\", None\n )\n\n # Update session timestamp\n await session.save()\n\n # Return streaming response\n return StreamingResponse(\n stream_source_chat_response(\n session_id=full_session_id,\n source_id=full_source_id,\n message=request.message,\n model_override=model_override,\n ),\n media_type=\"text/plain\",\n headers={\n \"Cache-Control\": \"no-cache\",\n \"Connection\": \"keep-alive\",\n \"Content-Type\": \"text/plain; charset=utf-8\",\n },\n )\n\n except HTTPException:\n raise\n except Exception as e:\n logger.error(f\"Error sending message to source chat: {str(e)}\")\n raise HTTPException(status_code=500, detail=f\"Error sending message: {str(e)}\")\n" }, { "path": "api/routers/sources.py", "content": "import asyncio\nimport os\nfrom pathlib import Path\nfrom typing import Any, List, Optional\n\nfrom fastapi import (\n APIRouter,\n Depends,\n File,\n Form,\n HTTPException,\n Query,\n UploadFile,\n)\nfrom fastapi.responses import FileResponse, Response\nfrom loguru import logger\nfrom surreal_commands import execute_command_sync, submit_command\n\nfrom api.command_service import CommandService\nfrom api.models import (\n AssetModel,\n CreateSourceInsightRequest,\n InsightCreationResponse,\n SourceCreate,\n SourceInsightResponse,\n SourceListResponse,\n SourceResponse,\n SourceStatusResponse,\n SourceUpdate,\n)\nfrom commands.source_commands import SourceProcessingInput\nfrom open_notebook.config import UPLOADS_FOLDER\nfrom open_notebook.database.repository import ensure_record_id, repo_query\nfrom open_notebook.domain.notebook import Notebook, Source\nfrom open_notebook.domain.transformation import Transformation\nfrom open_notebook.exceptions import InvalidInputError\n\nrouter = APIRouter()\n\n\ndef generate_unique_filename(original_filename: str, upload_folder: str) -> str:\n \"\"\"Generate unique filename like Streamlit app (append counter if file exists).\"\"\"\n file_path = Path(upload_folder)\n file_path.mkdir(parents=True, exist_ok=True)\n\n # Split filename and extension\n stem = Path(original_filename).stem\n suffix = Path(original_filename).suffix\n\n # Check if file exists and generate unique name\n counter = 0\n while True:\n if counter == 0:\n new_filename = original_filename\n else:\n new_filename = f\"{stem} ({counter}){suffix}\"\n\n full_path = file_path / new_filename\n if not full_path.exists():\n return str(full_path)\n counter += 1\n\n\nasync def save_uploaded_file(upload_file: UploadFile) -> str:\n \"\"\"Save uploaded file to uploads folder and return file path.\"\"\"\n if not upload_file.filename:\n raise ValueError(\"No filename provided\")\n\n # Generate unique filename\n file_path = generate_unique_filename(upload_file.filename, UPLOADS_FOLDER)\n\n try:\n # Save file\n with open(file_path, \"wb\") as f:\n content = await upload_file.read()\n f.write(content)\n\n logger.info(f\"Saved uploaded file to: {file_path}\")\n return file_path\n except Exception as e:\n logger.error(f\"Failed to save uploaded file: {e}\")\n # Clean up partial file if it exists\n if os.path.exists(file_path):\n os.unlink(file_path)\n raise\n\n\ndef parse_source_form_data(\n type: str = Form(...),\n notebook_id: Optional[str] = Form(None),\n notebooks: Optional[str] = Form(None), # JSON string of notebook IDs\n url: Optional[str] = Form(None),\n content: Optional[str] = Form(None),\n title: Optional[str] = Form(None),\n transformations: Optional[str] = Form(None), # JSON string of transformation IDs\n embed: str = Form(\"false\"), # Accept as string, convert to bool\n delete_source: str = Form(\"false\"), # Accept as string, convert to bool\n async_processing: str = Form(\"false\"), # Accept as string, convert to bool\n file: Optional[UploadFile] = File(None),\n) -> tuple[SourceCreate, Optional[UploadFile]]:\n \"\"\"Parse form data into SourceCreate model and return upload file separately.\"\"\"\n import json\n\n # Convert string booleans to actual booleans\n def str_to_bool(value: str) -> bool:\n return value.lower() in (\"true\", \"1\", \"yes\", \"on\")\n\n embed_bool = str_to_bool(embed)\n delete_source_bool = str_to_bool(delete_source)\n async_processing_bool = str_to_bool(async_processing)\n\n # Parse JSON strings\n notebooks_list = None\n if notebooks:\n try:\n notebooks_list = json.loads(notebooks)\n except json.JSONDecodeError:\n logger.error(f\"Invalid JSON in notebooks field: {notebooks}\")\n raise ValueError(\"Invalid JSON in notebooks field\")\n\n transformations_list = []\n if transformations:\n try:\n transformations_list = json.loads(transformations)\n except json.JSONDecodeError:\n logger.error(f\"Invalid JSON in transformations field: {transformations}\")\n raise ValueError(\"Invalid JSON in transformations field\")\n\n # Create SourceCreate instance\n try:\n source_data = SourceCreate(\n type=type,\n notebook_id=notebook_id,\n notebooks=notebooks_list,\n url=url,\n content=content,\n title=title,\n file_path=None, # Will be set later if file is uploaded\n transformations=transformations_list,\n embed=embed_bool,\n delete_source=delete_source_bool,\n async_processing=async_processing_bool,\n )\n pass # SourceCreate instance created successfully\n except Exception as e:\n logger.error(f\"Failed to create SourceCreate instance: {e}\")\n raise\n\n return source_data, file\n\n\n@router.get(\"/sources\", response_model=List[SourceListResponse])\nasync def get_sources(\n notebook_id: Optional[str] = Query(None, description=\"Filter by notebook ID\"),\n limit: int = Query(\n 50, ge=1, le=100, description=\"Number of sources to return (1-100)\"\n ),\n offset: int = Query(0, ge=0, description=\"Number of sources to skip\"),\n sort_by: str = Query(\n \"updated\", description=\"Field to sort by (created or updated)\"\n ),\n sort_order: str = Query(\"desc\", description=\"Sort order (asc or desc)\"),\n):\n \"\"\"Get sources with pagination and sorting support.\"\"\"\n try:\n # Validate sort parameters\n if sort_by not in [\"created\", \"updated\"]:\n raise HTTPException(\n status_code=400, detail=\"sort_by must be 'created' or 'updated'\"\n )\n if sort_order.lower() not in [\"asc\", \"desc\"]:\n raise HTTPException(\n status_code=400, detail=\"sort_order must be 'asc' or 'desc'\"\n )\n\n # Build ORDER BY clause\n order_clause = f\"ORDER BY {sort_by} {sort_order.upper()}\"\n\n # Build the query\n if notebook_id:\n # Verify notebook exists first\n notebook = await Notebook.get(notebook_id)\n if not notebook:\n raise HTTPException(status_code=404, detail=\"Notebook not found\")\n\n # Query sources for specific notebook - include command field with FETCH\n query = f\"\"\"\n SELECT id, asset, created, title, updated, topics, command,\n (SELECT VALUE count() FROM source_insight WHERE source = $parent.id GROUP ALL)[0].count OR 0 AS insights_count,\n (SELECT VALUE id FROM source_embedding WHERE source = $parent.id LIMIT 1) != [] AS embedded\n FROM (select value in from reference where out=$notebook_id)\n {order_clause}\n LIMIT $limit START $offset\n FETCH command\n \"\"\"\n result = await repo_query(\n query,\n {\n \"notebook_id\": ensure_record_id(notebook_id),\n \"limit\": limit,\n \"offset\": offset,\n },\n )\n else:\n # Query all sources - include command field with FETCH\n query = f\"\"\"\n SELECT id, asset, created, title, updated, topics, command,\n (SELECT VALUE count() FROM source_insight WHERE source = $parent.id GROUP ALL)[0].count OR 0 AS insights_count,\n (SELECT VALUE id FROM source_embedding WHERE source = $parent.id LIMIT 1) != [] AS embedded\n FROM source\n {order_clause}\n LIMIT $limit START $offset\n FETCH command\n \"\"\"\n result = await repo_query(query, {\"limit\": limit, \"offset\": offset})\n\n # Convert result to response model\n # Command data is already fetched via FETCH command clause\n response_list = []\n for row in result:\n command = row.get(\"command\")\n command_id = None\n status = None\n processing_info = None\n\n # Extract status from fetched command object (already resolved by FETCH)\n if command and isinstance(command, dict):\n command_id = str(command.get(\"id\")) if command.get(\"id\") else None\n status = command.get(\"status\")\n # Extract execution metadata from nested result structure\n result_data = command.get(\"result\")\n execution_metadata = (\n result_data.get(\"execution_metadata\", {})\n if isinstance(result_data, dict)\n else {}\n )\n processing_info = {\n \"started_at\": execution_metadata.get(\"started_at\"),\n \"completed_at\": execution_metadata.get(\"completed_at\"),\n \"error\": command.get(\"error_message\"),\n }\n elif command:\n # Command exists but FETCH failed to resolve it (broken reference)\n command_id = str(command)\n status = \"unknown\"\n\n response_list.append(\n SourceListResponse(\n id=row[\"id\"],\n title=row.get(\"title\"),\n topics=row.get(\"topics\") or [],\n asset=AssetModel(\n file_path=row[\"asset\"].get(\"file_path\")\n if row.get(\"asset\")\n else None,\n url=row[\"asset\"].get(\"url\") if row.get(\"asset\") else None,\n )\n if row.get(\"asset\")\n else None,\n embedded=row.get(\"embedded\", False),\n embedded_chunks=0, # Not needed in list view\n insights_count=row.get(\"insights_count\", 0),\n created=str(row[\"created\"]),\n updated=str(row[\"updated\"]),\n # Status fields from fetched command\n command_id=command_id,\n status=status,\n processing_info=processing_info,\n )\n )\n\n return response_list\n except HTTPException:\n raise\n except Exception as e:\n logger.error(f\"Error fetching sources: {str(e)}\")\n raise HTTPException(status_code=500, detail=f\"Error fetching sources: {str(e)}\")\n\n\n@router.post(\"/sources\", response_model=SourceResponse)\nasync def create_source(\n form_data: tuple[SourceCreate, Optional[UploadFile]] = Depends(\n parse_source_form_data\n ),\n):\n \"\"\"Create a new source with support for both JSON and multipart form data.\"\"\"\n source_data, upload_file = form_data\n\n # Initialize file_path before try block so exception handlers can reference it\n file_path = None\n\n try:\n # Verify all specified notebooks exist (backward compatibility support)\n for notebook_id in source_data.notebooks or []:\n notebook = await Notebook.get(notebook_id)\n if not notebook:\n raise HTTPException(\n status_code=404, detail=f\"Notebook {notebook_id} not found\"\n )\n\n # Handle file upload if provided\n if upload_file and source_data.type == \"upload\":\n try:\n file_path = await save_uploaded_file(upload_file)\n except Exception as e:\n logger.error(f\"File upload failed: {e}\")\n raise HTTPException(\n status_code=400, detail=f\"File upload failed: {str(e)}\"\n )\n\n # Prepare content_state for processing\n content_state: dict[str, Any] = {}\n\n if source_data.type == \"link\":\n if not source_data.url:\n raise HTTPException(\n status_code=400, detail=\"URL is required for link type\"\n )\n content_state[\"url\"] = source_data.url\n elif source_data.type == \"upload\":\n # Use uploaded file path or provided file_path (backward compatibility)\n final_file_path = file_path or source_data.file_path\n if not final_file_path:\n raise HTTPException(\n status_code=400,\n detail=\"File upload or file_path is required for upload type\",\n )\n content_state[\"file_path\"] = final_file_path\n content_state[\"delete_source\"] = source_data.delete_source\n elif source_data.type == \"text\":\n if not source_data.content:\n raise HTTPException(\n status_code=400, detail=\"Content is required for text type\"\n )\n content_state[\"content\"] = source_data.content\n else:\n raise HTTPException(\n status_code=400,\n detail=\"Invalid source type. Must be link, upload, or text\",\n )\n\n # Validate transformations exist\n transformation_ids = source_data.transformations or []\n for trans_id in transformation_ids:\n transformation = await Transformation.get(trans_id)\n if not transformation:\n raise HTTPException(\n status_code=404, detail=f\"Transformation {trans_id} not found\"\n )\n\n # Branch based on processing mode\n if source_data.async_processing:\n # ASYNC PATH: Create source record first, then queue command\n logger.info(\"Using async processing path\")\n\n # Create minimal source record - let SurrealDB generate the ID\n source = Source(\n title=source_data.title or \"Processing...\",\n topics=[],\n )\n await source.save()\n\n # Add source to notebooks immediately so it appears in the UI\n # The source_graph will skip adding duplicates\n for notebook_id in source_data.notebooks or []:\n await source.add_to_notebook(notebook_id)\n\n try:\n # Import command modules to ensure they're registered\n import commands.source_commands # noqa: F401\n\n # Submit command for background processing\n command_input = SourceProcessingInput(\n source_id=str(source.id),\n content_state=content_state,\n notebook_ids=source_data.notebooks,\n transformations=transformation_ids,\n embed=source_data.embed,\n )\n\n command_id = await CommandService.submit_command_job(\n \"open_notebook\", # app name\n \"process_source\", # command name\n command_input.model_dump(),\n )\n\n logger.info(f\"Submitted async processing command: {command_id}\")\n\n # Update source with command reference immediately\n # command_id already includes 'command:' prefix\n source.command = ensure_record_id(command_id)\n await source.save()\n\n # Return source with command info\n return SourceResponse(\n id=source.id or \"\",\n title=source.title,\n topics=source.topics or [],\n asset=None, # Will be populated after processing\n full_text=None, # Will be populated after processing\n embedded=False, # Will be updated after processing\n embedded_chunks=0,\n created=str(source.created),\n updated=str(source.updated),\n command_id=command_id,\n status=\"new\",\n processing_info={\"async\": True, \"queued\": True},\n )\n\n except Exception as e:\n logger.error(f\"Failed to submit async processing command: {e}\")\n # Clean up source record on command submission failure\n try:\n await source.delete()\n except Exception:\n pass\n # Clean up uploaded file if we created it\n if file_path and upload_file:\n try:\n os.unlink(file_path)\n except Exception:\n pass\n raise HTTPException(\n status_code=500, detail=f\"Failed to queue processing: {str(e)}\"\n )\n\n else:\n # SYNC PATH: Execute synchronously using execute_command_sync\n logger.info(\"Using sync processing path\")\n\n try:\n # Import command modules to ensure they're registered\n import commands.source_commands # noqa: F401\n\n # Create source record - let SurrealDB generate the ID\n source = Source(\n title=source_data.title or \"Processing...\",\n topics=[],\n )\n await source.save()\n\n # Add source to notebooks immediately so it appears in the UI\n # The source_graph will skip adding duplicates\n for notebook_id in source_data.notebooks or []:\n await source.add_to_notebook(notebook_id)\n\n # Execute command synchronously\n command_input = SourceProcessingInput(\n source_id=str(source.id),\n content_state=content_state,\n notebook_ids=source_data.notebooks,\n transformations=transformation_ids,\n embed=source_data.embed,\n )\n\n # Run in thread pool to avoid blocking the event loop\n # execute_command_sync uses asyncio.run() internally which can't\n # be called from an already-running event loop (FastAPI)\n result = await asyncio.to_thread(\n execute_command_sync,\n \"open_notebook\", # app name\n \"process_source\", # command name\n command_input.model_dump(),\n timeout=300, # 5 minute timeout for sync processing\n )\n\n if not result.is_success():\n logger.error(f\"Sync processing failed: {result.error_message}\")\n # Clean up source record\n try:\n await source.delete()\n except Exception:\n pass\n # Clean up uploaded file if we created it\n if file_path and upload_file:\n try:\n os.unlink(file_path)\n except Exception:\n pass\n raise HTTPException(\n status_code=500,\n detail=f\"Processing failed: {result.error_message}\",\n )\n\n # Get the processed source\n if not source.id:\n raise HTTPException(status_code=500, detail=\"Source ID is missing\")\n processed_source = await Source.get(source.id)\n if not processed_source:\n raise HTTPException(\n status_code=500, detail=\"Processed source not found\"\n )\n\n embedded_chunks = await processed_source.get_embedded_chunks()\n return SourceResponse(\n id=processed_source.id or \"\",\n title=processed_source.title,\n topics=processed_source.topics or [],\n asset=AssetModel(\n file_path=processed_source.asset.file_path\n if processed_source.asset\n else None,\n url=processed_source.asset.url\n if processed_source.asset\n else None,\n )\n if processed_source.asset\n else None,\n full_text=processed_source.full_text,\n embedded=embedded_chunks > 0,\n embedded_chunks=embedded_chunks,\n created=str(processed_source.created),\n updated=str(processed_source.updated),\n # No command_id or status for sync processing (legacy behavior)\n )\n\n except Exception as e:\n logger.error(f\"Sync processing failed: {e}\")\n # Clean up uploaded file if we created it\n if file_path and upload_file:\n try:\n os.unlink(file_path)\n except Exception:\n pass\n raise\n\n except HTTPException:\n # Clean up uploaded file on HTTP exceptions if we created it\n if file_path and upload_file:\n try:\n os.unlink(file_path)\n except Exception:\n pass\n raise\n except InvalidInputError as e:\n # Clean up uploaded file on validation errors if we created it\n if file_path and upload_file:\n try:\n os.unlink(file_path)\n except Exception:\n pass\n raise HTTPException(status_code=400, detail=str(e))\n except Exception as e:\n logger.error(f\"Error creating source: {str(e)}\")\n # Clean up uploaded file on unexpected errors if we created it\n if file_path and upload_file:\n try:\n os.unlink(file_path)\n except Exception:\n pass\n raise HTTPException(status_code=500, detail=f\"Error creating source: {str(e)}\")\n\n\n@router.post(\"/sources/json\", response_model=SourceResponse)\nasync def create_source_json(source_data: SourceCreate):\n \"\"\"Create a new source using JSON payload (legacy endpoint for backward compatibility).\"\"\"\n # Convert to form data format and call main endpoint\n form_data = (source_data, None)\n return await create_source(form_data)\n\n\nasync def _resolve_source_file(source_id: str) -> tuple[str, str]:\n source = await Source.get(source_id)\n if not source:\n raise HTTPException(status_code=404, detail=\"Source not found\")\n\n file_path = source.asset.file_path if source.asset else None\n if not file_path:\n raise HTTPException(status_code=404, detail=\"Source has no file to download\")\n\n safe_root = os.path.realpath(UPLOADS_FOLDER)\n resolved_path = os.path.realpath(file_path)\n\n if not resolved_path.startswith(safe_root):\n logger.warning(\n f\"Blocked download outside uploads directory for source {source_id}: {resolved_path}\"\n )\n raise HTTPException(status_code=403, detail=\"Access to file denied\")\n\n if not os.path.exists(resolved_path):\n raise HTTPException(status_code=404, detail=\"File not found on server\")\n\n filename = os.path.basename(resolved_path)\n return resolved_path, filename\n\n\ndef _is_source_file_available(source: Source) -> Optional[bool]:\n if not source or not source.asset or not source.asset.file_path:\n return None\n\n file_path = source.asset.file_path\n safe_root = os.path.realpath(UPLOADS_FOLDER)\n resolved_path = os.path.realpath(file_path)\n\n if not resolved_path.startswith(safe_root):\n return False\n\n return os.path.exists(resolved_path)\n\n\n@router.get(\"/sources/{source_id}\", response_model=SourceResponse)\nasync def get_source(source_id: str):\n \"\"\"Get a specific source by ID.\"\"\"\n try:\n source = await Source.get(source_id)\n if not source:\n raise HTTPException(status_code=404, detail=\"Source not found\")\n\n # Get status information if command exists\n status = None\n processing_info = None\n if source.command:\n try:\n status = await source.get_status()\n processing_info = await source.get_processing_progress()\n except Exception as e:\n logger.warning(f\"Failed to get status for source {source_id}: {e}\")\n status = \"unknown\"\n\n embedded_chunks = await source.get_embedded_chunks()\n\n # Get associated notebooks\n notebooks_query = await repo_query(\n \"SELECT VALUE out FROM reference WHERE in = $source_id\",\n {\"source_id\": ensure_record_id(source.id or source_id)},\n )\n notebook_ids = (\n [str(nb_id) for nb_id in notebooks_query] if notebooks_query else []\n )\n\n return SourceResponse(\n id=source.id or \"\",\n title=source.title,\n topics=source.topics or [],\n asset=AssetModel(\n file_path=source.asset.file_path if source.asset else None,\n url=source.asset.url if source.asset else None,\n )\n if source.asset\n else None,\n full_text=source.full_text,\n embedded=embedded_chunks > 0,\n embedded_chunks=embedded_chunks,\n file_available=_is_source_file_available(source),\n created=str(source.created),\n updated=str(source.updated),\n # Status fields\n command_id=str(source.command) if source.command else None,\n status=status,\n processing_info=processing_info,\n # Notebook associations\n notebooks=notebook_ids,\n )\n except HTTPException:\n raise\n except Exception as e:\n logger.error(f\"Error fetching source {source_id}: {str(e)}\")\n raise HTTPException(status_code=500, detail=f\"Error fetching source: {str(e)}\")\n\n\n@router.head(\"/sources/{source_id}/download\")\nasync def check_source_file(source_id: str):\n \"\"\"Check if a source has a downloadable file.\"\"\"\n try:\n await _resolve_source_file(source_id)\n return Response(status_code=200)\n except HTTPException:\n raise\n except Exception as e:\n logger.error(f\"Error checking file for source {source_id}: {str(e)}\")\n raise HTTPException(status_code=500, detail=\"Failed to verify file\")\n\n\n@router.get(\"/sources/{source_id}/download\")\nasync def download_source_file(source_id: str):\n \"\"\"Download the original file associated with an uploaded source.\"\"\"\n try:\n resolved_path, filename = await _resolve_source_file(source_id)\n return FileResponse(\n path=resolved_path,\n filename=filename,\n media_type=\"application/octet-stream\",\n )\n except HTTPException:\n raise\n except Exception as e:\n logger.error(f\"Error downloading file for source {source_id}: {str(e)}\")\n raise HTTPException(status_code=500, detail=\"Failed to download source file\")\n\n\n@router.get(\"/sources/{source_id}/status\", response_model=SourceStatusResponse)\nasync def get_source_status(source_id: str):\n \"\"\"Get processing status for a source.\"\"\"\n try:\n # First, verify source exists\n source = await Source.get(source_id)\n if not source:\n raise HTTPException(status_code=404, detail=\"Source not found\")\n\n # Check if this is a legacy source (no command)\n if not source.command:\n return SourceStatusResponse(\n status=None,\n message=\"Legacy source (completed before async processing)\",\n processing_info=None,\n command_id=None,\n )\n\n # Get command status and processing info\n try:\n status = await source.get_status()\n processing_info = await source.get_processing_progress()\n\n # Generate descriptive message based on status\n if status == \"completed\":\n message = \"Source processing completed successfully\"\n elif status == \"failed\":\n message = \"Source processing failed\"\n elif status == \"running\":\n message = \"Source processing in progress\"\n elif status == \"queued\":\n message = \"Source processing queued\"\n elif status == \"unknown\":\n message = \"Source processing status unknown\"\n else:\n message = f\"Source processing status: {status}\"\n\n return SourceStatusResponse(\n status=status,\n message=message,\n processing_info=processing_info,\n command_id=str(source.command) if source.command else None,\n )\n\n except Exception as e:\n logger.warning(f\"Failed to get status for source {source_id}: {e}\")\n return SourceStatusResponse(\n status=\"unknown\",\n message=\"Failed to retrieve processing status\",\n processing_info=None,\n command_id=str(source.command) if source.command else None,\n )\n\n except HTTPException:\n raise\n except Exception as e:\n logger.error(f\"Error fetching status for source {source_id}: {str(e)}\")\n raise HTTPException(\n status_code=500, detail=f\"Error fetching source status: {str(e)}\"\n )\n\n\n@router.put(\"/sources/{source_id}\", response_model=SourceResponse)\nasync def update_source(source_id: str, source_update: SourceUpdate):\n \"\"\"Update a source.\"\"\"\n try:\n source = await Source.get(source_id)\n if not source:\n raise HTTPException(status_code=404, detail=\"Source not found\")\n\n # Update only provided fields\n if source_update.title is not None:\n source.title = source_update.title\n if source_update.topics is not None:\n source.topics = source_update.topics\n\n await source.save()\n\n embedded_chunks = await source.get_embedded_chunks()\n return SourceResponse(\n id=source.id or \"\",\n title=source.title,\n topics=source.topics or [],\n asset=AssetModel(\n file_path=source.asset.file_path if source.asset else None,\n url=source.asset.url if source.asset else None,\n )\n if source.asset\n else None,\n full_text=source.full_text,\n embedded=embedded_chunks > 0,\n embedded_chunks=embedded_chunks,\n created=str(source.created),\n updated=str(source.updated),\n )\n except HTTPException:\n raise\n except InvalidInputError as e:\n raise HTTPException(status_code=400, detail=str(e))\n except Exception as e:\n logger.error(f\"Error updating source {source_id}: {str(e)}\")\n raise HTTPException(status_code=500, detail=f\"Error updating source: {str(e)}\")\n\n\n@router.post(\"/sources/{source_id}/retry\", response_model=SourceResponse)\nasync def retry_source_processing(source_id: str):\n \"\"\"Retry processing for a failed or stuck source.\"\"\"\n try:\n # First, verify source exists\n source = await Source.get(source_id)\n if not source:\n raise HTTPException(status_code=404, detail=\"Source not found\")\n\n # Check if source already has a running command\n if source.command:\n try:\n status = await source.get_status()\n if status in [\"running\", \"queued\"]:\n raise HTTPException(\n status_code=400,\n detail=\"Source is already processing. Cannot retry while processing is active.\",\n )\n except Exception as e:\n logger.warning(\n f\"Failed to check current status for source {source_id}: {e}\"\n )\n # Continue with retry if we can't check status\n\n # Get notebooks that this source belongs to\n query = \"SELECT notebook FROM reference WHERE source = $source_id\"\n references = await repo_query(query, {\"source_id\": source_id})\n notebook_ids = [str(ref[\"notebook\"]) for ref in references]\n\n if not notebook_ids:\n raise HTTPException(\n status_code=400, detail=\"Source is not associated with any notebooks\"\n )\n\n # Prepare content_state based on source asset\n content_state = {}\n if source.asset:\n if source.asset.file_path:\n content_state = {\n \"file_path\": source.asset.file_path,\n \"delete_source\": False, # Don't delete on retry\n }\n elif source.asset.url:\n content_state = {\"url\": source.asset.url}\n else:\n raise HTTPException(\n status_code=400, detail=\"Source asset has no file_path or url\"\n )\n else:\n # Check if it's a text source by trying to get full_text\n if source.full_text:\n content_state = {\"content\": source.full_text}\n else:\n raise HTTPException(\n status_code=400, detail=\"Cannot determine source content for retry\"\n )\n\n try:\n # Import command modules to ensure they're registered\n import commands.source_commands # noqa: F401\n\n # Submit new command for background processing\n command_input = SourceProcessingInput(\n source_id=str(source.id),\n content_state=content_state,\n notebook_ids=notebook_ids,\n transformations=[], # Use default transformations on retry\n embed=True, # Always embed on retry\n )\n\n command_id = await CommandService.submit_command_job(\n \"open_notebook\", # app name\n \"process_source\", # command name\n command_input.model_dump(),\n )\n\n logger.info(\n f\"Submitted retry processing command: {command_id} for source {source_id}\"\n )\n\n # Update source with new command ID\n source.command = ensure_record_id(f\"command:{command_id}\")\n await source.save()\n\n # Get current embedded chunks count\n embedded_chunks = await source.get_embedded_chunks()\n\n # Return updated source response\n return SourceResponse(\n id=source.id or \"\",\n title=source.title,\n topics=source.topics or [],\n asset=AssetModel(\n file_path=source.asset.file_path if source.asset else None,\n url=source.asset.url if source.asset else None,\n )\n if source.asset\n else None,\n full_text=source.full_text,\n embedded=embedded_chunks > 0,\n embedded_chunks=embedded_chunks,\n created=str(source.created),\n updated=str(source.updated),\n command_id=command_id,\n status=\"queued\",\n processing_info={\"retry\": True, \"queued\": True},\n )\n\n except Exception as e:\n logger.error(\n f\"Failed to submit retry processing command for source {source_id}: {e}\"\n )\n raise HTTPException(\n status_code=500, detail=f\"Failed to queue retry processing: {str(e)}\"\n )\n\n except HTTPException:\n raise\n except Exception as e:\n logger.error(f\"Error retrying source processing for {source_id}: {str(e)}\")\n raise HTTPException(\n status_code=500, detail=f\"Error retrying source processing: {str(e)}\"\n )\n\n\n@router.delete(\"/sources/{source_id}\")\nasync def delete_source(source_id: str):\n \"\"\"Delete a source.\"\"\"\n try:\n source = await Source.get(source_id)\n if not source:\n raise HTTPException(status_code=404, detail=\"Source not found\")\n\n await source.delete()\n\n return {\"message\": \"Source deleted successfully\"}\n except HTTPException:\n raise\n except Exception as e:\n logger.error(f\"Error deleting source {source_id}: {str(e)}\")\n raise HTTPException(status_code=500, detail=f\"Error deleting source: {str(e)}\")\n\n\n@router.get(\"/sources/{source_id}/insights\", response_model=List[SourceInsightResponse])\nasync def get_source_insights(source_id: str):\n \"\"\"Get all insights for a specific source.\"\"\"\n try:\n source = await Source.get(source_id)\n if not source:\n raise HTTPException(status_code=404, detail=\"Source not found\")\n\n insights = await source.get_insights()\n return [\n SourceInsightResponse(\n id=insight.id or \"\",\n source_id=source_id,\n insight_type=insight.insight_type,\n content=insight.content,\n created=str(insight.created),\n updated=str(insight.updated),\n )\n for insight in insights\n ]\n except HTTPException:\n raise\n except Exception as e:\n logger.error(f\"Error fetching insights for source {source_id}: {str(e)}\")\n raise HTTPException(\n status_code=500, detail=f\"Error fetching insights: {str(e)}\"\n )\n\n\n@router.post(\n \"/sources/{source_id}/insights\",\n response_model=InsightCreationResponse,\n status_code=202,\n)\nasync def create_source_insight(source_id: str, request: CreateSourceInsightRequest):\n \"\"\"\n Start insight generation for a source by running a transformation.\n\n This endpoint returns immediately with a 202 Accepted status.\n The transformation runs asynchronously in the background via the job queue.\n Poll GET /sources/{source_id}/insights to see when the insight is ready.\n \"\"\"\n try:\n # Validate source exists\n source = await Source.get(source_id)\n if not source:\n raise HTTPException(status_code=404, detail=\"Source not found\")\n\n # Validate transformation exists\n transformation = await Transformation.get(request.transformation_id)\n if not transformation:\n raise HTTPException(status_code=404, detail=\"Transformation not found\")\n\n # Submit transformation as background job (fire-and-forget)\n command_id = submit_command(\n \"open_notebook\",\n \"run_transformation\",\n {\n \"source_id\": source_id,\n \"transformation_id\": request.transformation_id,\n },\n )\n logger.info(\n f\"Submitted run_transformation command {command_id} for source {source_id}\"\n )\n\n # Return immediately with command_id for status tracking\n return InsightCreationResponse(\n status=\"pending\",\n message=\"Insight generation started\",\n source_id=source_id,\n transformation_id=request.transformation_id,\n command_id=str(command_id),\n )\n\n except HTTPException:\n raise\n except Exception as e:\n logger.error(f\"Error starting insight generation for source {source_id}: {e}\")\n raise HTTPException(\n status_code=500, detail=f\"Error starting insight generation: {str(e)}\"\n )\n" }, { "path": "api/routers/speaker_profiles.py", "content": "from typing import Any, Dict, List, Optional\n\nfrom fastapi import APIRouter, HTTPException\nfrom loguru import logger\nfrom pydantic import BaseModel, Field\n\nfrom open_notebook.podcasts.models import SpeakerProfile\n\nrouter = APIRouter()\n\n\nclass SpeakerProfileResponse(BaseModel):\n id: str\n name: str\n description: str\n voice_model: Optional[str] = None\n speakers: List[Dict[str, Any]]\n # Legacy fields (for display/migration awareness)\n tts_provider: Optional[str] = None\n tts_model: Optional[str] = None\n\n\ndef _profile_to_response(profile: SpeakerProfile) -> SpeakerProfileResponse:\n return SpeakerProfileResponse(\n id=str(profile.id),\n name=profile.name,\n description=profile.description or \"\",\n voice_model=profile.voice_model,\n speakers=profile.speakers,\n tts_provider=profile.tts_provider,\n tts_model=profile.tts_model,\n )\n\n\n@router.get(\"/speaker-profiles\", response_model=List[SpeakerProfileResponse])\nasync def list_speaker_profiles():\n \"\"\"List all available speaker profiles\"\"\"\n try:\n profiles = await SpeakerProfile.get_all(order_by=\"name asc\")\n return [_profile_to_response(p) for p in profiles]\n except Exception as e:\n logger.error(f\"Failed to fetch speaker profiles: {e}\")\n raise HTTPException(\n status_code=500, detail=\"Failed to fetch speaker profiles\"\n )\n\n\n@router.get(\"/speaker-profiles/{profile_name}\", response_model=SpeakerProfileResponse)\nasync def get_speaker_profile(profile_name: str):\n \"\"\"Get a specific speaker profile by name\"\"\"\n try:\n profile = await SpeakerProfile.get_by_name(profile_name)\n\n if not profile:\n raise HTTPException(\n status_code=404, detail=f\"Speaker profile '{profile_name}' not found\"\n )\n\n return _profile_to_response(profile)\n\n except HTTPException:\n raise\n except Exception as e:\n logger.error(f\"Failed to fetch speaker profile '{profile_name}': {e}\")\n raise HTTPException(\n status_code=500, detail=\"Failed to fetch speaker profile\"\n )\n\n\nclass SpeakerProfileCreate(BaseModel):\n name: str = Field(..., description=\"Unique profile name\")\n description: str = Field(\"\", description=\"Profile description\")\n voice_model: Optional[str] = Field(None, description=\"Model record ID for TTS\")\n speakers: List[Dict[str, Any]] = Field(\n ..., description=\"Array of speaker configurations\"\n )\n # Legacy fields (accepted but not required)\n tts_provider: Optional[str] = None\n tts_model: Optional[str] = None\n\n\n@router.post(\"/speaker-profiles\", response_model=SpeakerProfileResponse)\nasync def create_speaker_profile(profile_data: SpeakerProfileCreate):\n \"\"\"Create a new speaker profile\"\"\"\n try:\n profile = SpeakerProfile(\n name=profile_data.name,\n description=profile_data.description,\n voice_model=profile_data.voice_model,\n speakers=profile_data.speakers,\n tts_provider=profile_data.tts_provider,\n tts_model=profile_data.tts_model,\n )\n\n await profile.save()\n return _profile_to_response(profile)\n\n except Exception as e:\n logger.error(f\"Failed to create speaker profile: {e}\")\n raise HTTPException(\n status_code=500, detail=\"Failed to create speaker profile\"\n )\n\n\n@router.put(\"/speaker-profiles/{profile_id}\", response_model=SpeakerProfileResponse)\nasync def update_speaker_profile(profile_id: str, profile_data: SpeakerProfileCreate):\n \"\"\"Update an existing speaker profile\"\"\"\n try:\n profile = await SpeakerProfile.get(profile_id)\n\n if not profile:\n raise HTTPException(\n status_code=404, detail=f\"Speaker profile '{profile_id}' not found\"\n )\n\n profile.name = profile_data.name\n profile.description = profile_data.description\n profile.voice_model = profile_data.voice_model\n profile.speakers = profile_data.speakers\n profile.tts_provider = profile_data.tts_provider\n profile.tts_model = profile_data.tts_model\n\n await profile.save()\n return _profile_to_response(profile)\n\n except HTTPException:\n raise\n except Exception as e:\n logger.error(f\"Failed to update speaker profile: {e}\")\n raise HTTPException(\n status_code=500, detail=\"Failed to update speaker profile\"\n )\n\n\n@router.delete(\"/speaker-profiles/{profile_id}\")\nasync def delete_speaker_profile(profile_id: str):\n \"\"\"Delete a speaker profile\"\"\"\n try:\n profile = await SpeakerProfile.get(profile_id)\n\n if not profile:\n raise HTTPException(\n status_code=404, detail=f\"Speaker profile '{profile_id}' not found\"\n )\n\n await profile.delete()\n\n return {\"message\": \"Speaker profile deleted successfully\"}\n\n except HTTPException:\n raise\n except Exception as e:\n logger.error(f\"Failed to delete speaker profile: {e}\")\n raise HTTPException(\n status_code=500, detail=\"Failed to delete speaker profile\"\n )\n\n\n@router.post(\n \"/speaker-profiles/{profile_id}/duplicate\", response_model=SpeakerProfileResponse\n)\nasync def duplicate_speaker_profile(profile_id: str):\n \"\"\"Duplicate a speaker profile\"\"\"\n try:\n original = await SpeakerProfile.get(profile_id)\n\n if not original:\n raise HTTPException(\n status_code=404, detail=f\"Speaker profile '{profile_id}' not found\"\n )\n\n duplicate = SpeakerProfile(\n name=f\"{original.name} - Copy\",\n description=original.description,\n voice_model=original.voice_model,\n speakers=original.speakers,\n tts_provider=original.tts_provider,\n tts_model=original.tts_model,\n )\n\n await duplicate.save()\n return _profile_to_response(duplicate)\n\n except HTTPException:\n raise\n except Exception as e:\n logger.error(f\"Failed to duplicate speaker profile: {e}\")\n raise HTTPException(\n status_code=500, detail=\"Failed to duplicate speaker profile\"\n )\n" }, { "path": "api/routers/transformations.py", "content": "from typing import List\n\nfrom fastapi import APIRouter, HTTPException\nfrom loguru import logger\n\nfrom api.models import (\n DefaultPromptResponse,\n DefaultPromptUpdate,\n TransformationCreate,\n TransformationExecuteRequest,\n TransformationExecuteResponse,\n TransformationResponse,\n TransformationUpdate,\n)\nfrom open_notebook.ai.models import Model\nfrom open_notebook.domain.transformation import DefaultPrompts, Transformation\nfrom open_notebook.exceptions import InvalidInputError, OpenNotebookError\nfrom open_notebook.graphs.transformation import graph as transformation_graph\n\nrouter = APIRouter()\n\n\n@router.get(\"/transformations\", response_model=List[TransformationResponse])\nasync def get_transformations():\n \"\"\"Get all transformations.\"\"\"\n try:\n transformations = await Transformation.get_all(order_by=\"name asc\")\n\n return [\n TransformationResponse(\n id=transformation.id or \"\",\n name=transformation.name,\n title=transformation.title,\n description=transformation.description,\n prompt=transformation.prompt,\n apply_default=transformation.apply_default,\n created=str(transformation.created),\n updated=str(transformation.updated),\n )\n for transformation in transformations\n ]\n except Exception as e:\n logger.error(f\"Error fetching transformations: {str(e)}\")\n raise HTTPException(\n status_code=500, detail=f\"Error fetching transformations: {str(e)}\"\n )\n\n\n@router.post(\"/transformations\", response_model=TransformationResponse)\nasync def create_transformation(transformation_data: TransformationCreate):\n \"\"\"Create a new transformation.\"\"\"\n try:\n new_transformation = Transformation(\n name=transformation_data.name,\n title=transformation_data.title,\n description=transformation_data.description,\n prompt=transformation_data.prompt,\n apply_default=transformation_data.apply_default,\n )\n await new_transformation.save()\n\n return TransformationResponse(\n id=new_transformation.id or \"\",\n name=new_transformation.name,\n title=new_transformation.title,\n description=new_transformation.description,\n prompt=new_transformation.prompt,\n apply_default=new_transformation.apply_default,\n created=str(new_transformation.created),\n updated=str(new_transformation.updated),\n )\n except InvalidInputError as e:\n raise HTTPException(status_code=400, detail=str(e))\n except Exception as e:\n logger.error(f\"Error creating transformation: {str(e)}\")\n raise HTTPException(\n status_code=500, detail=f\"Error creating transformation: {str(e)}\"\n )\n\n\n@router.post(\"/transformations/execute\", response_model=TransformationExecuteResponse)\nasync def execute_transformation(execute_request: TransformationExecuteRequest):\n \"\"\"Execute a transformation on input text.\"\"\"\n try:\n # Validate transformation exists\n transformation = await Transformation.get(execute_request.transformation_id)\n if not transformation:\n raise HTTPException(status_code=404, detail=\"Transformation not found\")\n\n # Validate model exists\n model = await Model.get(execute_request.model_id)\n if not model:\n raise HTTPException(status_code=404, detail=\"Model not found\")\n\n # Execute the transformation\n result = await transformation_graph.ainvoke(\n dict( # type: ignore[arg-type]\n input_text=execute_request.input_text,\n transformation=transformation,\n ),\n config=dict(configurable={\"model_id\": execute_request.model_id}),\n )\n\n return TransformationExecuteResponse(\n output=result[\"output\"],\n transformation_id=execute_request.transformation_id,\n model_id=execute_request.model_id,\n )\n\n except HTTPException:\n raise\n except OpenNotebookError:\n raise # Let global exception handlers return proper status codes\n except Exception as e:\n logger.error(f\"Error executing transformation: {str(e)}\")\n raise HTTPException(\n status_code=500, detail=f\"Error executing transformation: {str(e)}\"\n )\n\n\n@router.get(\"/transformations/default-prompt\", response_model=DefaultPromptResponse)\nasync def get_default_prompt():\n \"\"\"Get the default transformation prompt.\"\"\"\n try:\n default_prompts: DefaultPrompts = await DefaultPrompts.get_instance() # type: ignore[assignment]\n\n return DefaultPromptResponse(\n transformation_instructions=default_prompts.transformation_instructions\n or \"\"\n )\n except Exception as e:\n logger.error(f\"Error fetching default prompt: {str(e)}\")\n raise HTTPException(\n status_code=500, detail=f\"Error fetching default prompt: {str(e)}\"\n )\n\n\n@router.put(\"/transformations/default-prompt\", response_model=DefaultPromptResponse)\nasync def update_default_prompt(prompt_update: DefaultPromptUpdate):\n \"\"\"Update the default transformation prompt.\"\"\"\n try:\n default_prompts: DefaultPrompts = await DefaultPrompts.get_instance() # type: ignore[assignment]\n\n default_prompts.transformation_instructions = (\n prompt_update.transformation_instructions\n )\n await default_prompts.update()\n\n return DefaultPromptResponse(\n transformation_instructions=default_prompts.transformation_instructions\n )\n except Exception as e:\n logger.error(f\"Error updating default prompt: {str(e)}\")\n raise HTTPException(\n status_code=500, detail=f\"Error updating default prompt: {str(e)}\"\n )\n\n\n@router.get(\n \"/transformations/{transformation_id}\", response_model=TransformationResponse\n)\nasync def get_transformation(transformation_id: str):\n \"\"\"Get a specific transformation by ID.\"\"\"\n try:\n transformation = await Transformation.get(transformation_id)\n if not transformation:\n raise HTTPException(status_code=404, detail=\"Transformation not found\")\n\n return TransformationResponse(\n id=transformation.id or \"\",\n name=transformation.name,\n title=transformation.title,\n description=transformation.description,\n prompt=transformation.prompt,\n apply_default=transformation.apply_default,\n created=str(transformation.created),\n updated=str(transformation.updated),\n )\n except HTTPException:\n raise\n except Exception as e:\n logger.error(f\"Error fetching transformation {transformation_id}: {str(e)}\")\n raise HTTPException(\n status_code=500, detail=f\"Error fetching transformation: {str(e)}\"\n )\n\n\n@router.put(\n \"/transformations/{transformation_id}\", response_model=TransformationResponse\n)\nasync def update_transformation(\n transformation_id: str, transformation_update: TransformationUpdate\n):\n \"\"\"Update a transformation.\"\"\"\n try:\n transformation = await Transformation.get(transformation_id)\n if not transformation:\n raise HTTPException(status_code=404, detail=\"Transformation not found\")\n\n # Update only provided fields\n if transformation_update.name is not None:\n transformation.name = transformation_update.name\n if transformation_update.title is not None:\n transformation.title = transformation_update.title\n if transformation_update.description is not None:\n transformation.description = transformation_update.description\n if transformation_update.prompt is not None:\n transformation.prompt = transformation_update.prompt\n if transformation_update.apply_default is not None:\n transformation.apply_default = transformation_update.apply_default\n\n await transformation.save()\n\n return TransformationResponse(\n id=transformation.id or \"\",\n name=transformation.name,\n title=transformation.title,\n description=transformation.description,\n prompt=transformation.prompt,\n apply_default=transformation.apply_default,\n created=str(transformation.created),\n updated=str(transformation.updated),\n )\n except HTTPException:\n raise\n except InvalidInputError as e:\n raise HTTPException(status_code=400, detail=str(e))\n except Exception as e:\n logger.error(f\"Error updating transformation {transformation_id}: {str(e)}\")\n raise HTTPException(\n status_code=500, detail=f\"Error updating transformation: {str(e)}\"\n )\n\n\n@router.delete(\"/transformations/{transformation_id}\")\nasync def delete_transformation(transformation_id: str):\n \"\"\"Delete a transformation.\"\"\"\n try:\n transformation = await Transformation.get(transformation_id)\n if not transformation:\n raise HTTPException(status_code=404, detail=\"Transformation not found\")\n\n await transformation.delete()\n\n return {\"message\": \"Transformation deleted successfully\"}\n except HTTPException:\n raise\n except Exception as e:\n logger.error(f\"Error deleting transformation {transformation_id}: {str(e)}\")\n raise HTTPException(\n status_code=500, detail=f\"Error deleting transformation: {str(e)}\"\n )\n" }, { "path": "api/search_service.py", "content": "\"\"\"\nSearch service layer using API.\n\"\"\"\n\nfrom typing import Any, Dict, List, Union\n\nfrom loguru import logger\n\nfrom api.client import api_client\n\n\nclass SearchService:\n \"\"\"Service layer for search operations using API.\"\"\"\n\n def __init__(self):\n logger.info(\"Using API for search operations\")\n\n def search(\n self,\n query: str,\n search_type: str = \"text\",\n limit: int = 100,\n search_sources: bool = True,\n search_notes: bool = True,\n minimum_score: float = 0.2,\n ) -> List[Dict[str, Any]]:\n \"\"\"Search the knowledge base.\"\"\"\n response = api_client.search(\n query=query,\n search_type=search_type,\n limit=limit,\n search_sources=search_sources,\n search_notes=search_notes,\n minimum_score=minimum_score,\n )\n if isinstance(response, dict):\n return response.get(\"results\", [])\n return []\n\n def ask_knowledge_base(\n self,\n question: str,\n strategy_model: str,\n answer_model: str,\n final_answer_model: str,\n ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:\n \"\"\"Ask the knowledge base a question.\"\"\"\n response = api_client.ask_simple(\n question=question,\n strategy_model=strategy_model,\n answer_model=answer_model,\n final_answer_model=final_answer_model,\n )\n return response\n\n\n# Global service instance\nsearch_service = SearchService()\n" }, { "path": "api/settings_service.py", "content": "\"\"\"\nSettings service layer using API.\n\"\"\"\n\nfrom loguru import logger\n\nfrom api.client import api_client\nfrom open_notebook.domain.content_settings import ContentSettings\n\n\nclass SettingsService:\n \"\"\"Service layer for settings operations using API.\"\"\"\n\n def __init__(self):\n logger.info(\"Using API for settings operations\")\n\n def get_settings(self) -> ContentSettings:\n \"\"\"Get application settings.\"\"\"\n settings_response = api_client.get_settings()\n settings_data = (\n settings_response\n if isinstance(settings_response, dict)\n else settings_response[0]\n )\n\n # Create ContentSettings object from API response\n settings = ContentSettings(\n default_content_processing_engine_doc=settings_data.get(\n \"default_content_processing_engine_doc\"\n ),\n default_content_processing_engine_url=settings_data.get(\n \"default_content_processing_engine_url\"\n ),\n default_embedding_option=settings_data.get(\"default_embedding_option\"),\n auto_delete_files=settings_data.get(\"auto_delete_files\"),\n youtube_preferred_languages=settings_data.get(\n \"youtube_preferred_languages\"\n ),\n )\n\n return settings\n\n def update_settings(self, settings: ContentSettings) -> ContentSettings:\n \"\"\"Update application settings.\"\"\"\n updates = {\n \"default_content_processing_engine_doc\": settings.default_content_processing_engine_doc,\n \"default_content_processing_engine_url\": settings.default_content_processing_engine_url,\n \"default_embedding_option\": settings.default_embedding_option,\n \"auto_delete_files\": settings.auto_delete_files,\n \"youtube_preferred_languages\": settings.youtube_preferred_languages,\n }\n\n settings_response = api_client.update_settings(**updates)\n settings_data = (\n settings_response\n if isinstance(settings_response, dict)\n else settings_response[0]\n )\n\n # Update the settings object with the response\n settings.default_content_processing_engine_doc = settings_data.get(\n \"default_content_processing_engine_doc\"\n )\n settings.default_content_processing_engine_url = settings_data.get(\n \"default_content_processing_engine_url\"\n )\n settings.default_embedding_option = settings_data.get(\n \"default_embedding_option\"\n )\n settings.auto_delete_files = settings_data.get(\"auto_delete_files\")\n settings.youtube_preferred_languages = settings_data.get(\n \"youtube_preferred_languages\"\n )\n\n return settings\n\n\n# Global service instance\nsettings_service = SettingsService()\n" }, { "path": "api/sources_service.py", "content": "\"\"\"\nSources service layer using API.\n\"\"\"\n\nfrom dataclasses import dataclass\nfrom typing import Dict, List, Optional, Union\n\nfrom loguru import logger\n\nfrom api.client import api_client\nfrom open_notebook.domain.notebook import Asset, Source\n\n\n@dataclass\nclass SourceProcessingResult:\n \"\"\"Result of source creation with optional async processing info.\"\"\"\n\n source: Source\n is_async: bool = False\n command_id: Optional[str] = None\n status: Optional[str] = None\n processing_info: Optional[Dict] = None\n\n\n@dataclass\nclass SourceWithMetadata:\n \"\"\"Source object with additional metadata from API.\"\"\"\n\n source: Source\n embedded_chunks: int\n\n # Expose common source properties for easy access\n @property\n def id(self):\n return self.source.id\n\n @property\n def title(self):\n return self.source.title\n\n @title.setter\n def title(self, value):\n self.source.title = value\n\n @property\n def topics(self):\n return self.source.topics\n\n @property\n def asset(self):\n return self.source.asset\n\n @property\n def full_text(self):\n return self.source.full_text\n\n @property\n def created(self):\n return self.source.created\n\n @property\n def updated(self):\n return self.source.updated\n\n\nclass SourcesService:\n \"\"\"Service layer for sources operations using API.\"\"\"\n\n def __init__(self):\n logger.info(\"Using API for sources operations\")\n\n def get_all_sources(\n self, notebook_id: Optional[str] = None\n ) -> List[SourceWithMetadata]:\n \"\"\"Get all sources with optional notebook filtering.\"\"\"\n sources_data = api_client.get_sources(notebook_id=notebook_id)\n # Convert API response to SourceWithMetadata objects\n sources = []\n for source_data in sources_data:\n source = Source(\n title=source_data[\"title\"],\n topics=source_data[\"topics\"],\n asset=Asset(\n file_path=source_data[\"asset\"][\"file_path\"]\n if source_data[\"asset\"]\n else None,\n url=source_data[\"asset\"][\"url\"] if source_data[\"asset\"] else None,\n )\n if source_data[\"asset\"]\n else None,\n )\n source.id = source_data[\"id\"]\n source.created = source_data[\"created\"]\n source.updated = source_data[\"updated\"]\n\n # Wrap in SourceWithMetadata\n source_with_metadata = SourceWithMetadata(\n source=source, embedded_chunks=source_data.get(\"embedded_chunks\", 0)\n )\n sources.append(source_with_metadata)\n return sources\n\n def get_source(self, source_id: str) -> SourceWithMetadata:\n \"\"\"Get a specific source.\"\"\"\n response = api_client.get_source(source_id)\n source_data = response if isinstance(response, dict) else response[0]\n source = Source(\n title=source_data[\"title\"],\n topics=source_data[\"topics\"],\n full_text=source_data[\"full_text\"],\n asset=Asset(\n file_path=source_data[\"asset\"][\"file_path\"]\n if source_data[\"asset\"]\n else None,\n url=source_data[\"asset\"][\"url\"] if source_data[\"asset\"] else None,\n )\n if source_data[\"asset\"]\n else None,\n )\n source.id = source_data[\"id\"]\n source.created = source_data[\"created\"]\n source.updated = source_data[\"updated\"]\n\n return SourceWithMetadata(\n source=source, embedded_chunks=source_data.get(\"embedded_chunks\", 0)\n )\n\n def create_source(\n self,\n notebook_id: Optional[str] = None,\n source_type: str = \"text\",\n url: Optional[str] = None,\n file_path: Optional[str] = None,\n content: Optional[str] = None,\n title: Optional[str] = None,\n transformations: Optional[List[str]] = None,\n embed: bool = False,\n delete_source: bool = False,\n notebooks: Optional[List[str]] = None,\n async_processing: bool = False,\n ) -> Union[Source, SourceProcessingResult]:\n \"\"\"\n Create a new source with support for async processing.\n\n Args:\n notebook_id: Single notebook ID (deprecated, use notebooks parameter)\n source_type: Type of source (link, upload, text)\n url: URL for link sources\n file_path: File path for upload sources\n content: Text content for text sources\n title: Optional source title\n transformations: List of transformation IDs to apply\n embed: Whether to embed content for vector search\n delete_source: Whether to delete uploaded file after processing\n notebooks: List of notebook IDs to add source to (preferred over notebook_id)\n async_processing: Whether to process source asynchronously\n\n Returns:\n Source object for sync processing (backward compatibility)\n SourceProcessingResult for async processing (contains additional metadata)\n \"\"\"\n source_data = api_client.create_source(\n notebook_id=notebook_id,\n notebooks=notebooks,\n source_type=source_type,\n url=url,\n file_path=file_path,\n content=content,\n title=title,\n transformations=transformations,\n embed=embed,\n delete_source=delete_source,\n async_processing=async_processing,\n )\n\n # Create Source object from response\n response_data = source_data if isinstance(source_data, dict) else source_data[0]\n source = Source(\n title=response_data[\"title\"],\n topics=response_data.get(\"topics\") or [],\n full_text=response_data.get(\"full_text\"),\n asset=Asset(\n file_path=response_data[\"asset\"][\"file_path\"]\n if response_data.get(\"asset\")\n else None,\n url=response_data[\"asset\"][\"url\"]\n if response_data.get(\"asset\")\n else None,\n )\n if response_data.get(\"asset\")\n else None,\n )\n source.id = response_data[\"id\"]\n source.created = response_data[\"created\"]\n source.updated = response_data[\"updated\"]\n\n # Check if this is an async processing response\n if (\n response_data.get(\"command_id\")\n or response_data.get(\"status\")\n or response_data.get(\"processing_info\")\n ):\n # Ensure source_data is a dict for accessing attributes\n source_data_dict = (\n source_data if isinstance(source_data, dict) else source_data[0]\n )\n # Return enhanced result for async processing\n return SourceProcessingResult(\n source=source,\n is_async=True,\n command_id=source_data_dict.get(\"command_id\"),\n status=source_data_dict.get(\"status\"),\n processing_info=source_data_dict.get(\"processing_info\"),\n )\n else:\n # Return simple Source for backward compatibility\n return source\n\n def get_source_status(self, source_id: str) -> Dict:\n \"\"\"Get processing status for a source.\"\"\"\n response = api_client.get_source_status(source_id)\n return response if isinstance(response, dict) else response[0]\n\n def create_source_async(\n self,\n notebook_id: Optional[str] = None,\n source_type: str = \"text\",\n url: Optional[str] = None,\n file_path: Optional[str] = None,\n content: Optional[str] = None,\n title: Optional[str] = None,\n transformations: Optional[List[str]] = None,\n embed: bool = False,\n delete_source: bool = False,\n notebooks: Optional[List[str]] = None,\n ) -> SourceProcessingResult:\n \"\"\"\n Create a new source with async processing enabled.\n\n This is a convenience method that always uses async processing.\n Returns a SourceProcessingResult with processing status information.\n \"\"\"\n result = self.create_source(\n notebook_id=notebook_id,\n notebooks=notebooks,\n source_type=source_type,\n url=url,\n file_path=file_path,\n content=content,\n title=title,\n transformations=transformations,\n embed=embed,\n delete_source=delete_source,\n async_processing=True,\n )\n\n # Since we forced async_processing=True, this should always be a SourceProcessingResult\n if isinstance(result, SourceProcessingResult):\n return result\n else:\n # Fallback: wrap Source in SourceProcessingResult\n return SourceProcessingResult(\n source=result,\n is_async=False, # This shouldn't happen, but handle it gracefully\n )\n\n def is_source_processing_complete(self, source_id: str) -> bool:\n \"\"\"\n Check if a source's async processing is complete.\n\n Returns True if processing is complete (success or failure),\n False if still processing or queued.\n \"\"\"\n try:\n status_data = self.get_source_status(source_id)\n status = status_data.get(\"status\")\n return status in [\n \"completed\",\n \"failed\",\n None,\n ] # None indicates legacy/sync source\n except Exception as e:\n logger.error(f\"Error checking source processing status: {e}\")\n return True # Assume complete on error\n\n def update_source(self, source: Source) -> Source:\n \"\"\"Update a source.\"\"\"\n if not source.id:\n raise ValueError(\"Source ID is required for update\")\n\n updates = {\n \"title\": source.title,\n \"topics\": source.topics,\n }\n source_data = api_client.update_source(source.id, **updates)\n\n # Ensure source_data is a dict\n source_data_dict = (\n source_data if isinstance(source_data, dict) else source_data[0]\n )\n\n # Update the source object with the response\n source.title = source_data_dict[\"title\"]\n source.topics = source_data_dict[\"topics\"]\n source.updated = source_data_dict[\"updated\"]\n\n return source\n\n def delete_source(self, source_id: str) -> bool:\n \"\"\"Delete a source.\"\"\"\n api_client.delete_source(source_id)\n return True\n\n\n# Global service instance\nsources_service = SourcesService()\n\n# Export important classes for easy importing\n__all__ = [\n \"SourcesService\",\n \"SourceWithMetadata\",\n \"SourceProcessingResult\",\n \"sources_service\",\n]\n" }, { "path": "api/transformations_service.py", "content": "\"\"\"\nTransformations service layer using API.\n\"\"\"\n\nfrom datetime import datetime\nfrom typing import Any, Dict, List, Union\n\nfrom loguru import logger\n\nfrom api.client import api_client\nfrom open_notebook.domain.transformation import Transformation\n\n\nclass TransformationsService:\n \"\"\"Service layer for transformations operations using API.\"\"\"\n\n def __init__(self):\n logger.info(\"Using API for transformations operations\")\n\n def get_all_transformations(self) -> List[Transformation]:\n \"\"\"Get all transformations.\"\"\"\n transformations_data = api_client.get_transformations()\n # Convert API response to Transformation objects\n transformations = []\n for trans_data in transformations_data:\n transformation = Transformation(\n name=trans_data[\"name\"],\n title=trans_data[\"title\"],\n description=trans_data[\"description\"],\n prompt=trans_data[\"prompt\"],\n apply_default=trans_data[\"apply_default\"],\n )\n transformation.id = trans_data[\"id\"]\n transformation.created = datetime.fromisoformat(\n trans_data[\"created\"].replace(\"Z\", \"+00:00\")\n )\n transformation.updated = datetime.fromisoformat(\n trans_data[\"updated\"].replace(\"Z\", \"+00:00\")\n )\n transformations.append(transformation)\n return transformations\n\n def get_transformation(self, transformation_id: str) -> Transformation:\n \"\"\"Get a specific transformation.\"\"\"\n response = api_client.get_transformation(transformation_id)\n trans_data = response if isinstance(response, dict) else response[0]\n transformation = Transformation(\n name=trans_data[\"name\"],\n title=trans_data[\"title\"],\n description=trans_data[\"description\"],\n prompt=trans_data[\"prompt\"],\n apply_default=trans_data[\"apply_default\"],\n )\n transformation.id = trans_data[\"id\"]\n transformation.created = datetime.fromisoformat(\n trans_data[\"created\"].replace(\"Z\", \"+00:00\")\n )\n transformation.updated = datetime.fromisoformat(\n trans_data[\"updated\"].replace(\"Z\", \"+00:00\")\n )\n return transformation\n\n def create_transformation(\n self,\n name: str,\n title: str,\n description: str,\n prompt: str,\n apply_default: bool = False,\n ) -> Transformation:\n \"\"\"Create a new transformation.\"\"\"\n response = api_client.create_transformation(\n name=name,\n title=title,\n description=description,\n prompt=prompt,\n apply_default=apply_default,\n )\n trans_data = response if isinstance(response, dict) else response[0]\n transformation = Transformation(\n name=trans_data[\"name\"],\n title=trans_data[\"title\"],\n description=trans_data[\"description\"],\n prompt=trans_data[\"prompt\"],\n apply_default=trans_data[\"apply_default\"],\n )\n transformation.id = trans_data[\"id\"]\n transformation.created = datetime.fromisoformat(\n trans_data[\"created\"].replace(\"Z\", \"+00:00\")\n )\n transformation.updated = datetime.fromisoformat(\n trans_data[\"updated\"].replace(\"Z\", \"+00:00\")\n )\n return transformation\n\n def update_transformation(self, transformation: Transformation) -> Transformation:\n \"\"\"Update a transformation.\"\"\"\n if not transformation.id:\n raise ValueError(\"Transformation ID is required for update\")\n\n updates = {\n \"name\": transformation.name,\n \"title\": transformation.title,\n \"description\": transformation.description,\n \"prompt\": transformation.prompt,\n \"apply_default\": transformation.apply_default,\n }\n response = api_client.update_transformation(transformation.id, **updates)\n trans_data = response if isinstance(response, dict) else response[0]\n\n # Update the transformation object with the response\n transformation.name = trans_data[\"name\"]\n transformation.title = trans_data[\"title\"]\n transformation.description = trans_data[\"description\"]\n transformation.prompt = trans_data[\"prompt\"]\n transformation.apply_default = trans_data[\"apply_default\"]\n transformation.updated = datetime.fromisoformat(\n trans_data[\"updated\"].replace(\"Z\", \"+00:00\")\n )\n\n return transformation\n\n def delete_transformation(self, transformation_id: str) -> bool:\n \"\"\"Delete a transformation.\"\"\"\n api_client.delete_transformation(transformation_id)\n return True\n\n def execute_transformation(\n self, transformation_id: str, input_text: str, model_id: str\n ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:\n \"\"\"Execute a transformation on input text.\"\"\"\n result = api_client.execute_transformation(\n transformation_id=transformation_id,\n input_text=input_text,\n model_id=model_id,\n )\n return result\n\n\n# Global service instance\ntransformations_service = TransformationsService()\n" }, { "path": "commands/CLAUDE.md", "content": "# Commands Module\n\n**Purpose**: Defines async command handlers for long-running operations via `surreal-commands` job queue system.\n\n## Key Components\n\n### Embedding Commands\n\n- **`embed_note_command`**: Embeds a single note using unified embedding pipeline with content-type aware processing. Uses MARKDOWN content type detection. Retry: 5 attempts, exponential jitter 1-60s.\n- **`embed_insight_command`**: Embeds a single source insight. Uses MARKDOWN content type. Retry: 5 attempts, exponential jitter 1-60s.\n- **`embed_source_command`**: Embeds a source by chunking full_text with content-type aware splitters (HTML, Markdown, plain), then batch embedding all chunks (batches of 50 with per-batch retry). Retry: 5 attempts, exponential jitter 1-60s.\n- **`create_insight_command`**: Creates a source insight with automatic retry on transaction conflicts. Creates the DB record, then submits `embed_insight` command (fire-and-forget). Retry: 5 attempts, exponential jitter 1-60s. Used by `Source.add_insight()`.\n- **`rebuild_embeddings_command`**: Submits individual embed_* commands for all sources/notes/insights. Returns immediately; actual embedding happens async. No retry (coordinator only).\n\n### Other Commands\n\n- **`process_source_command`**: Ingests content through `source_graph`, creates embeddings (optional), and generates insights. Retries on transaction conflicts (exp. jitter, max 15×, 1-120s).\n- **`run_transformation_command`**: Runs a transformation on an existing source to generate an insight. Executes the transformation graph (LLM call) then creates insight via `create_insight_command`. Used by `POST /sources/{id}/insights` API endpoint. Retry: 5 attempts, exponential jitter 1-60s.\n- **`generate_podcast_command`**: Creates podcasts via podcast-creator library. Resolves model registry references and credentials for all profiles before invoking podcast-creator. Validates that outline_llm, transcript_llm, and voice_model are configured.\n- **`process_text_command`** (example): Test fixture for text operations (uppercase, lowercase, reverse, word_count).\n- **`analyze_data_command`** (example): Test fixture for numeric aggregations.\n\n## Important Patterns\n\n- **Pydantic I/O**: All commands use `CommandInput`/`CommandOutput` subclasses for type safety and serialization.\n- **Error handling**: Permanent errors (ValueError) return failure output; all other exceptions auto-retry via surreal-commands.\n- **Retry configuration**: Uses `stop_on: [ValueError]` (blocklist approach) - retries all exceptions EXCEPT ValueError. This is more resilient than allowlist as new exception types auto-retry.\n- **Fire-and-forget embedding**: Domain models submit embed_* commands via `submit_command()` without waiting. Commands process asynchronously.\n- **Content-type aware chunking**: `embed_source_command` uses `chunk_text()` with automatic content type detection (HTML, Markdown, plain text) for optimal text splitting. Default: 1500 char chunks with 225 char overlap.\n- **Batch embedding**: `embed_source_command` uses `generate_embeddings()` which automatically batches texts (default 50) with per-batch retry to avoid exceeding provider payload limits.\n- **Mean pooling for large content**: `embed_note_command` and `embed_insight_command` use `generate_embedding()` which handles content larger than chunk size via mean pooling.\n- **Model dumping**: Recursive `full_model_dump()` utility converts Pydantic models → dicts for DB/API responses.\n- **Logging**: Uses `loguru.logger` throughout; logs execution start/end and key metrics (processing time, counts).\n- **Time tracking**: All commands measure `start_time` → `processing_time` for monitoring.\n\n## Dependencies\n\n**External**: `surreal_commands` (command decorator, job queue, submit_command), `loguru`, `pydantic`, `podcast_creator`\n**Internal**: `open_notebook.domain.notebook` (Source, Note, SourceInsight), `open_notebook.utils.chunking` (chunk_text, detect_content_type), `open_notebook.utils.embedding` (generate_embedding, generate_embeddings), `open_notebook.database.repository` (repo_query, repo_insert)\n\n## Quirks & Edge Cases\n\n- **source_commands**: `ensure_record_id()` wraps command IDs for DB storage; transaction conflicts trigger exponential backoff retry. ValueError exceptions are permanent (not retried).\n- **embedding_commands**: Content type detection uses file extension as primary source, heuristics as fallback. Chunks >1800 chars trigger secondary splitting. Empty/whitespace-only content returns ValueError (not retried).\n- **rebuild_embeddings_command**: Returns \"jobs_submitted\" not \"processed_items\" - embedding is async. Individual commands handle failures with their own retries.\n- **podcast_commands**: Profiles loaded from SurrealDB by name; model configs (credentials) resolved for ALL profiles before podcast-creator validation. Validates outline_llm/transcript_llm/voice_model are set. Episode records created mid-execution.\n- **Example commands**: Accept optional `delay_seconds` for testing async behavior; not for production.\n\n## Code Example\n\n```python\n@command(\"process_source\", app=\"open_notebook\", retry={\n \"max_attempts\": 5,\n \"wait_strategy\": \"exponential_jitter\",\n \"stop_on\": [ValueError], # Don't retry validation errors\n})\nasync def process_source_command(input_data: SourceProcessingInput) -> SourceProcessingOutput:\n start_time = time.time()\n try:\n transformations = [await Transformation.get(id) for id in input_data.transformations]\n source = await Source.get(input_data.source_id)\n result = await source_graph.ainvoke({...})\n return SourceProcessingOutput(success=True, ...)\n except ValueError as e:\n return SourceProcessingOutput(success=False, error_message=str(e)) # No retry\n except Exception as e:\n raise # Retry all other exceptions\n```\n" }, { "path": "commands/__init__.py", "content": "\"\"\"Surreal-commands integration for Open Notebook\"\"\"\n\nfrom .embedding_commands import (\n embed_insight_command,\n embed_note_command,\n embed_source_command,\n rebuild_embeddings_command,\n)\nfrom .example_commands import analyze_data_command, process_text_command\nfrom .podcast_commands import generate_podcast_command\nfrom .source_commands import process_source_command\n\n__all__ = [\n # Embedding commands\n \"embed_note_command\",\n \"embed_insight_command\",\n \"embed_source_command\",\n \"rebuild_embeddings_command\",\n # Other commands\n \"generate_podcast_command\",\n \"process_source_command\",\n \"process_text_command\",\n \"analyze_data_command\",\n]\n" }, { "path": "commands/embedding_commands.py", "content": "import time\nfrom typing import Dict, List, Literal, Optional\n\nfrom loguru import logger\nfrom pydantic import BaseModel\nfrom surreal_commands import CommandInput, CommandOutput, command, submit_command\n\nfrom open_notebook.ai.models import model_manager\nfrom open_notebook.database.repository import ensure_record_id, repo_insert, repo_query\nfrom open_notebook.exceptions import ConfigurationError\nfrom open_notebook.domain.notebook import Note, Source, SourceInsight\nfrom open_notebook.utils.chunking import ContentType, chunk_text, detect_content_type\nfrom open_notebook.utils.embedding import generate_embedding, generate_embeddings\n\n\ndef full_model_dump(model):\n if isinstance(model, BaseModel):\n return model.model_dump()\n elif isinstance(model, dict):\n return {k: full_model_dump(v) for k, v in model.items()}\n elif isinstance(model, list):\n return [full_model_dump(item) for item in model]\n else:\n return model\n\n\ndef get_command_id(input_data: CommandInput) -> str:\n \"\"\"Extract command_id from input_data's execution context, or return 'unknown'.\"\"\"\n if input_data.execution_context:\n return str(input_data.execution_context.command_id)\n return \"unknown\"\n\n\nclass RebuildEmbeddingsInput(CommandInput):\n mode: Literal[\"existing\", \"all\"]\n include_sources: bool = True\n include_notes: bool = True\n include_insights: bool = True\n\n\nclass RebuildEmbeddingsOutput(CommandOutput):\n success: bool\n total_items: int\n jobs_submitted: int # Count of embedding commands submitted\n failed_submissions: int # Count of items that failed to submit\n sources_submitted: int = 0\n notes_submitted: int = 0\n insights_submitted: int = 0\n processing_time: float\n error_message: Optional[str] = None\n\n\n# =============================================================================\n# NEW EMBEDDING COMMANDS (Phase 3)\n# =============================================================================\n\n\nclass CreateInsightInput(CommandInput):\n \"\"\"Input for creating a source insight with automatic retry on conflicts.\"\"\"\n\n source_id: str\n insight_type: str\n content: str\n\n\nclass CreateInsightOutput(CommandOutput):\n \"\"\"Output from insight creation command.\"\"\"\n\n success: bool\n insight_id: Optional[str] = None\n processing_time: float\n error_message: Optional[str] = None\n\n\nclass EmbedNoteInput(CommandInput):\n \"\"\"Input for embedding a single note.\"\"\"\n\n note_id: str\n\n\nclass EmbedNoteOutput(CommandOutput):\n \"\"\"Output from note embedding command.\"\"\"\n\n success: bool\n note_id: str\n processing_time: float\n error_message: Optional[str] = None\n\n\nclass EmbedInsightInput(CommandInput):\n \"\"\"Input for embedding a single source insight.\"\"\"\n\n insight_id: str\n\n\nclass EmbedInsightOutput(CommandOutput):\n \"\"\"Output from insight embedding command.\"\"\"\n\n success: bool\n insight_id: str\n processing_time: float\n error_message: Optional[str] = None\n\n\nclass EmbedSourceInput(CommandInput):\n \"\"\"Input for embedding a source (creates multiple chunk embeddings).\"\"\"\n\n source_id: str\n\n\nclass EmbedSourceOutput(CommandOutput):\n \"\"\"Output from source embedding command.\"\"\"\n\n success: bool\n source_id: str\n chunks_created: int\n processing_time: float\n error_message: Optional[str] = None\n\n\n@command(\n \"embed_note\",\n app=\"open_notebook\",\n retry={\n \"max_attempts\": 5,\n \"wait_strategy\": \"exponential_jitter\",\n \"wait_min\": 1,\n \"wait_max\": 60,\n \"stop_on\": [ValueError, ConfigurationError], # Don't retry validation/config errors\n \"retry_log_level\": \"debug\",\n },\n)\nasync def embed_note_command(input_data: EmbedNoteInput) -> EmbedNoteOutput:\n \"\"\"\n Generate and store embedding for a single note.\n\n Uses the unified embedding pipeline with automatic chunking and mean pooling\n for notes that exceed the chunk size limit.\n\n Flow:\n 1. Load Note by ID\n 2. Generate embedding via generate_embedding() (auto-chunks + mean pools if needed)\n 3. UPSERT note embedding in database\n\n Retry Strategy:\n - Retries up to 5 times for transient failures (network, timeout, etc.)\n - Uses exponential-jitter backoff (1-60s)\n - Does NOT retry permanent failures (ValueError for validation errors)\n \"\"\"\n start_time = time.time()\n\n try:\n logger.info(f\"Starting embedding for note: {input_data.note_id}\")\n\n # 1. Load note\n note = await Note.get(input_data.note_id)\n if not note:\n raise ValueError(f\"Note '{input_data.note_id}' not found\")\n\n if not note.content or not note.content.strip():\n raise ValueError(f\"Note '{input_data.note_id}' has no content to embed\")\n\n # 2. Generate embedding (auto-chunks + mean pools if needed)\n # Notes are typically markdown content\n cmd_id = get_command_id(input_data)\n embedding = await generate_embedding(\n note.content, content_type=ContentType.MARKDOWN, command_id=cmd_id\n )\n\n # 3. UPSERT embedding into note record\n await repo_query(\n \"UPDATE $note_id SET embedding = $embedding\",\n {\n \"note_id\": ensure_record_id(input_data.note_id),\n \"embedding\": embedding,\n },\n )\n\n processing_time = time.time() - start_time\n logger.info(\n f\"Successfully embedded note {input_data.note_id} in {processing_time:.2f}s\"\n )\n\n return EmbedNoteOutput(\n success=True,\n note_id=input_data.note_id,\n processing_time=processing_time,\n )\n\n except ValueError as e:\n # Permanent failure - don't retry\n processing_time = time.time() - start_time\n cmd_id = get_command_id(input_data)\n logger.error(\n f\"Failed to embed note {input_data.note_id} (command: {cmd_id}): {e}\"\n )\n return EmbedNoteOutput(\n success=False,\n note_id=input_data.note_id,\n processing_time=processing_time,\n error_message=str(e),\n )\n except Exception as e:\n # Transient failure - will be retried (surreal-commands logs final failure)\n cmd_id = get_command_id(input_data)\n logger.debug(\n f\"Transient error embedding note {input_data.note_id} \"\n f\"(command: {cmd_id}): {e}\"\n )\n raise\n\n\n@command(\n \"embed_insight\",\n app=\"open_notebook\",\n retry={\n \"max_attempts\": 5,\n \"wait_strategy\": \"exponential_jitter\",\n \"wait_min\": 1,\n \"wait_max\": 60,\n \"stop_on\": [ValueError, ConfigurationError], # Don't retry validation/config errors\n \"retry_log_level\": \"debug\",\n },\n)\nasync def embed_insight_command(input_data: EmbedInsightInput) -> EmbedInsightOutput:\n \"\"\"\n Generate and store embedding for a single source insight.\n\n Uses the unified embedding pipeline with automatic chunking and mean pooling\n for insights that exceed the chunk size limit.\n\n Flow:\n 1. Load SourceInsight by ID\n 2. Generate embedding via generate_embedding() (auto-chunks + mean pools if needed)\n 3. UPSERT insight embedding in database\n\n Retry Strategy:\n - Retries up to 5 times for transient failures (network, timeout, etc.)\n - Uses exponential-jitter backoff (1-60s)\n - Does NOT retry permanent failures (ValueError for validation errors)\n \"\"\"\n start_time = time.time()\n\n try:\n logger.info(f\"Starting embedding for insight: {input_data.insight_id}\")\n\n # 1. Load insight\n insight = await SourceInsight.get(input_data.insight_id)\n if not insight:\n raise ValueError(f\"Insight '{input_data.insight_id}' not found\")\n\n if not insight.content or not insight.content.strip():\n raise ValueError(\n f\"Insight '{input_data.insight_id}' has no content to embed\"\n )\n\n # 2. Generate embedding (auto-chunks + mean pools if needed)\n # Insights are typically markdown content (generated by LLM)\n cmd_id = get_command_id(input_data)\n embedding = await generate_embedding(\n insight.content, content_type=ContentType.MARKDOWN, command_id=cmd_id\n )\n\n # 3. UPSERT embedding into insight record\n await repo_query(\n \"UPDATE $insight_id SET embedding = $embedding\",\n {\n \"insight_id\": ensure_record_id(input_data.insight_id),\n \"embedding\": embedding,\n },\n )\n\n processing_time = time.time() - start_time\n logger.info(\n f\"Successfully embedded insight {input_data.insight_id} in {processing_time:.2f}s\"\n )\n\n return EmbedInsightOutput(\n success=True,\n insight_id=input_data.insight_id,\n processing_time=processing_time,\n )\n\n except ValueError as e:\n # Permanent failure - don't retry\n processing_time = time.time() - start_time\n cmd_id = get_command_id(input_data)\n logger.error(\n f\"Failed to embed insight {input_data.insight_id} (command: {cmd_id}): {e}\"\n )\n return EmbedInsightOutput(\n success=False,\n insight_id=input_data.insight_id,\n processing_time=processing_time,\n error_message=str(e),\n )\n except Exception as e:\n # Transient failure - will be retried (surreal-commands logs final failure)\n cmd_id = get_command_id(input_data)\n logger.debug(\n f\"Transient error embedding insight {input_data.insight_id} \"\n f\"(command: {cmd_id}): {e}\"\n )\n raise\n\n\n@command(\n \"embed_source\",\n app=\"open_notebook\",\n retry={\n \"max_attempts\": 5,\n \"wait_strategy\": \"exponential_jitter\",\n \"wait_min\": 1,\n \"wait_max\": 60,\n \"stop_on\": [ValueError, ConfigurationError], # Don't retry validation/config errors\n \"retry_log_level\": \"debug\",\n },\n)\nasync def embed_source_command(input_data: EmbedSourceInput) -> EmbedSourceOutput:\n \"\"\"\n Generate and store embeddings for a source document.\n\n Creates multiple chunk embeddings stored in the source_embedding table.\n Uses content-type aware chunking based on file extension or content heuristics.\n\n Flow:\n 1. Load Source by ID\n 2. DELETE existing source_embedding records for this source\n 3. Detect content type from file path or content\n 4. Chunk text using appropriate splitter\n 5. Generate embeddings for all chunks in batches\n 6. Bulk INSERT source_embedding records\n\n Retry Strategy:\n - Retries up to 5 times for transient failures (network, timeout, etc.)\n - Uses exponential-jitter backoff (1-60s)\n - Does NOT retry permanent failures (ValueError for validation errors)\n \"\"\"\n start_time = time.time()\n\n try:\n logger.info(f\"Starting embedding for source: {input_data.source_id}\")\n\n # 1. Load source\n source = await Source.get(input_data.source_id)\n if not source:\n raise ValueError(f\"Source '{input_data.source_id}' not found\")\n\n if not source.full_text or not source.full_text.strip():\n raise ValueError(f\"Source '{input_data.source_id}' has no text to embed\")\n\n # 2. DELETE existing embeddings (idempotency)\n logger.debug(f\"Deleting existing embeddings for source {input_data.source_id}\")\n await repo_query(\n \"DELETE source_embedding WHERE source = $source_id\",\n {\"source_id\": ensure_record_id(input_data.source_id)},\n )\n\n # 3. Detect content type from file path if available\n file_path = source.asset.file_path if source.asset else None\n content_type = detect_content_type(source.full_text, file_path)\n logger.debug(f\"Detected content type: {content_type.value}\")\n\n # 4. Chunk text using appropriate splitter\n chunks = chunk_text(source.full_text, content_type=content_type)\n total_chunks = len(chunks)\n\n # Log chunk statistics for debugging\n chunk_sizes = [len(c) for c in chunks]\n logger.info(\n f\"Created {total_chunks} chunks for source {input_data.source_id} \"\n f\"(sizes: min={min(chunk_sizes) if chunk_sizes else 0}, \"\n f\"max={max(chunk_sizes) if chunk_sizes else 0}, \"\n f\"avg={sum(chunk_sizes)//len(chunk_sizes) if chunk_sizes else 0} chars)\"\n )\n\n if total_chunks == 0:\n raise ValueError(\"No chunks created after splitting text\")\n\n # 5. Generate embeddings for all chunks in batches\n cmd_id = get_command_id(input_data)\n logger.debug(f\"Generating embeddings for {total_chunks} chunks\")\n embeddings = await generate_embeddings(chunks, command_id=cmd_id)\n\n # Verify we got embeddings for all chunks\n if len(embeddings) != len(chunks):\n raise ValueError(\n f\"Embedding count mismatch: got {len(embeddings)} embeddings \"\n f\"for {len(chunks)} chunks\"\n )\n\n # 6. Bulk INSERT source_embedding records\n records = [\n {\n \"source\": ensure_record_id(input_data.source_id),\n \"order\": idx,\n \"content\": chunk,\n \"embedding\": embedding,\n }\n for idx, (chunk, embedding) in enumerate(zip(chunks, embeddings))\n ]\n\n logger.debug(f\"Inserting {len(records)} source_embedding records\")\n await repo_insert(\"source_embedding\", records)\n\n processing_time = time.time() - start_time\n logger.info(\n f\"Successfully embedded source {input_data.source_id}: \"\n f\"{total_chunks} chunks in {processing_time:.2f}s\"\n )\n\n return EmbedSourceOutput(\n success=True,\n source_id=input_data.source_id,\n chunks_created=total_chunks,\n processing_time=processing_time,\n )\n\n except ValueError as e:\n # Permanent failure - don't retry\n processing_time = time.time() - start_time\n cmd_id = get_command_id(input_data)\n logger.error(\n f\"Failed to embed source {input_data.source_id} (command: {cmd_id}): {e}\"\n )\n return EmbedSourceOutput(\n success=False,\n source_id=input_data.source_id,\n chunks_created=0,\n processing_time=processing_time,\n error_message=str(e),\n )\n except Exception as e:\n # Transient failure - will be retried (surreal-commands logs final failure)\n cmd_id = get_command_id(input_data)\n logger.debug(\n f\"Transient error embedding source {input_data.source_id} \"\n f\"(command: {cmd_id}): {e}\"\n )\n raise\n\n\n@command(\n \"create_insight\",\n app=\"open_notebook\",\n retry={\n \"max_attempts\": 5,\n \"wait_strategy\": \"exponential_jitter\",\n \"wait_min\": 1,\n \"wait_max\": 60,\n \"stop_on\": [ValueError, ConfigurationError], # Don't retry validation/config errors\n \"retry_log_level\": \"debug\",\n },\n)\nasync def create_insight_command(\n input_data: CreateInsightInput,\n) -> CreateInsightOutput:\n \"\"\"\n Create a source insight with automatic retry on transaction conflicts.\n\n This command wraps the CREATE source_insight operation with retry logic\n to handle SurrealDB transaction conflicts that occur during batch imports\n when multiple parallel transformations try to create insights concurrently.\n\n Flow:\n 1. CREATE source_insight record in database\n 2. Submit embed_insight command (fire-and-forget) for async embedding\n 3. Return the insight_id\n\n Retry Strategy:\n - Retries up to 5 times for transient failures (network, timeout, etc.)\n - Uses exponential-jitter backoff (1-60s)\n - Does NOT retry permanent failures (ValueError for validation errors)\n \"\"\"\n start_time = time.time()\n\n try:\n logger.info(\n f\"Creating insight for source {input_data.source_id}: \"\n f\"type={input_data.insight_type}\"\n )\n\n # 1. Create insight record in database\n result = await repo_query(\n \"\"\"\n CREATE source_insight CONTENT {\n \"source\": $source_id,\n \"insight_type\": $insight_type,\n \"content\": $content\n };\n \"\"\",\n {\n \"source_id\": ensure_record_id(input_data.source_id),\n \"insight_type\": input_data.insight_type,\n \"content\": input_data.content,\n },\n )\n\n if not result or len(result) == 0:\n raise ValueError(\"Failed to create insight - no result returned\")\n\n insight_id = str(result[0].get(\"id\", \"\"))\n if not insight_id:\n raise ValueError(\"Failed to create insight - no ID in result\")\n\n # 2. Submit embedding command (fire-and-forget)\n submit_command(\n \"open_notebook\",\n \"embed_insight\",\n {\"insight_id\": insight_id},\n )\n logger.debug(f\"Submitted embed_insight command for {insight_id}\")\n\n processing_time = time.time() - start_time\n logger.info(\n f\"Successfully created insight {insight_id} for source \"\n f\"{input_data.source_id} in {processing_time:.2f}s\"\n )\n\n return CreateInsightOutput(\n success=True,\n insight_id=insight_id,\n processing_time=processing_time,\n )\n\n except ValueError as e:\n # Permanent failure - don't retry\n processing_time = time.time() - start_time\n cmd_id = get_command_id(input_data)\n logger.error(\n f\"Failed to create insight for source {input_data.source_id} \"\n f\"(command: {cmd_id}): {e}\"\n )\n return CreateInsightOutput(\n success=False,\n processing_time=processing_time,\n error_message=str(e),\n )\n except Exception as e:\n # Transient failure - will be retried (surreal-commands logs final failure)\n cmd_id = get_command_id(input_data)\n logger.debug(\n f\"Transient error creating insight for source {input_data.source_id} \"\n f\"(command: {cmd_id}): {e}\"\n )\n raise\n\n\nasync def collect_items_for_rebuild(\n mode: str,\n include_sources: bool,\n include_notes: bool,\n include_insights: bool,\n) -> Dict[str, List[str]]:\n \"\"\"\n Collect items to rebuild based on mode and include flags.\n\n Returns:\n Dict with keys: 'sources', 'notes', 'insights' containing lists of item IDs\n \"\"\"\n items: Dict[str, List[str]] = {\"sources\": [], \"notes\": [], \"insights\": []}\n\n if include_sources:\n if mode == \"existing\":\n # Query sources with embeddings (via source_embedding table)\n result = await repo_query(\n \"\"\"\n RETURN array::distinct(\n SELECT VALUE source.id\n FROM source_embedding\n WHERE embedding != none AND array::len(embedding) > 0\n )\n \"\"\"\n )\n # RETURN returns the array directly as the result (not nested)\n if result:\n items[\"sources\"] = [str(item) for item in result]\n else:\n items[\"sources\"] = []\n else: # mode == \"all\"\n # Query all sources with non-empty content\n result = await repo_query(\n \"SELECT id FROM source WHERE full_text != none AND string::trim(full_text) != ''\"\n )\n items[\"sources\"] = [str(item[\"id\"]) for item in result] if result else []\n\n logger.info(f\"Collected {len(items['sources'])} sources for rebuild\")\n\n if include_notes:\n if mode == \"existing\":\n # Query notes with embeddings\n result = await repo_query(\n \"SELECT id FROM note WHERE embedding != none AND array::len(embedding) > 0\"\n )\n else: # mode == \"all\"\n # Query all notes with non-empty content\n result = await repo_query(\n \"SELECT id FROM note WHERE content != none AND string::trim(content) != ''\"\n )\n\n items[\"notes\"] = [str(item[\"id\"]) for item in result] if result else []\n logger.info(f\"Collected {len(items['notes'])} notes for rebuild\")\n\n if include_insights:\n if mode == \"existing\":\n # Query insights with embeddings\n result = await repo_query(\n \"SELECT id FROM source_insight WHERE embedding != none AND array::len(embedding) > 0\"\n )\n else: # mode == \"all\"\n # Query all insights with non-empty content\n result = await repo_query(\n \"SELECT id FROM source_insight WHERE content != none AND string::trim(content) != ''\"\n )\n\n items[\"insights\"] = [str(item[\"id\"]) for item in result] if result else []\n logger.info(f\"Collected {len(items['insights'])} insights for rebuild\")\n\n return items\n\n\n@command(\"rebuild_embeddings\", app=\"open_notebook\", retry=None)\nasync def rebuild_embeddings_command(\n input_data: RebuildEmbeddingsInput,\n) -> RebuildEmbeddingsOutput:\n \"\"\"\n Rebuild embeddings for sources, notes, and/or insights.\n\n This command submits individual embedding jobs for each item:\n - embed_source for sources\n - embed_note for notes\n - embed_insight for insights\n\n The command returns after submitting all jobs. Actual embedding\n happens asynchronously via the individual commands (which have\n their own retry strategies).\n\n Retry Strategy:\n - Retries disabled (retry=None) for this coordinator command\n - Individual embed_* commands handle their own retries\n \"\"\"\n start_time = time.time()\n\n try:\n logger.info(\"=\" * 60)\n logger.info(f\"Starting embedding rebuild with mode={input_data.mode}\")\n logger.info(\n f\"Include: sources={input_data.include_sources}, notes={input_data.include_notes}, insights={input_data.include_insights}\"\n )\n logger.info(\"=\" * 60)\n\n # Check embedding model availability (fail fast)\n EMBEDDING_MODEL = await model_manager.get_embedding_model()\n if not EMBEDDING_MODEL:\n raise ValueError(\n \"No embedding model configured. Please configure one in the Models section.\"\n )\n\n logger.info(f\"Embedding model configured: {EMBEDDING_MODEL}\")\n\n # Collect items to process (returns IDs only)\n items = await collect_items_for_rebuild(\n input_data.mode,\n input_data.include_sources,\n input_data.include_notes,\n input_data.include_insights,\n )\n\n total_items = (\n len(items[\"sources\"]) + len(items[\"notes\"]) + len(items[\"insights\"])\n )\n logger.info(f\"Total items to rebuild: {total_items}\")\n\n if total_items == 0:\n logger.warning(\"No items found to rebuild\")\n return RebuildEmbeddingsOutput(\n success=True,\n total_items=0,\n jobs_submitted=0,\n failed_submissions=0,\n processing_time=time.time() - start_time,\n )\n\n # Initialize counters\n sources_submitted = 0\n notes_submitted = 0\n insights_submitted = 0\n failed_submissions = 0\n\n # Submit embed_source commands for sources\n logger.info(f\"\\nSubmitting {len(items['sources'])} source embedding jobs...\")\n for idx, source_id in enumerate(items[\"sources\"], 1):\n try:\n submit_command(\n \"open_notebook\",\n \"embed_source\",\n {\"source_id\": source_id},\n )\n sources_submitted += 1\n\n if idx % 50 == 0 or idx == len(items[\"sources\"]):\n logger.info(\n f\" Progress: {idx}/{len(items['sources'])} source jobs submitted\"\n )\n\n except Exception as e:\n logger.error(f\"Failed to submit embed_source for {source_id}: {e}\")\n failed_submissions += 1\n\n # Submit embed_note commands for notes\n logger.info(f\"\\nSubmitting {len(items['notes'])} note embedding jobs...\")\n for idx, note_id in enumerate(items[\"notes\"], 1):\n try:\n submit_command(\n \"open_notebook\",\n \"embed_note\",\n {\"note_id\": note_id},\n )\n notes_submitted += 1\n\n if idx % 50 == 0 or idx == len(items[\"notes\"]):\n logger.info(\n f\" Progress: {idx}/{len(items['notes'])} note jobs submitted\"\n )\n\n except Exception as e:\n logger.error(f\"Failed to submit embed_note for {note_id}: {e}\")\n failed_submissions += 1\n\n # Submit embed_insight commands for insights\n logger.info(f\"\\nSubmitting {len(items['insights'])} insight embedding jobs...\")\n for idx, insight_id in enumerate(items[\"insights\"], 1):\n try:\n submit_command(\n \"open_notebook\",\n \"embed_insight\",\n {\"insight_id\": insight_id},\n )\n insights_submitted += 1\n\n if idx % 50 == 0 or idx == len(items[\"insights\"]):\n logger.info(\n f\" Progress: {idx}/{len(items['insights'])} insight jobs submitted\"\n )\n\n except Exception as e:\n logger.error(f\"Failed to submit embed_insight for {insight_id}: {e}\")\n failed_submissions += 1\n\n processing_time = time.time() - start_time\n jobs_submitted = sources_submitted + notes_submitted + insights_submitted\n\n logger.info(\"=\" * 60)\n logger.info(\"REBUILD JOBS SUBMITTED\")\n logger.info(f\" Total jobs submitted: {jobs_submitted}/{total_items}\")\n logger.info(f\" Sources: {sources_submitted}\")\n logger.info(f\" Notes: {notes_submitted}\")\n logger.info(f\" Insights: {insights_submitted}\")\n logger.info(f\" Failed submissions: {failed_submissions}\")\n logger.info(f\" Submission time: {processing_time:.2f}s\")\n logger.info(\" Note: Actual embedding happens asynchronously\")\n logger.info(\"=\" * 60)\n\n return RebuildEmbeddingsOutput(\n success=True,\n total_items=total_items,\n jobs_submitted=jobs_submitted,\n failed_submissions=failed_submissions,\n sources_submitted=sources_submitted,\n notes_submitted=notes_submitted,\n insights_submitted=insights_submitted,\n processing_time=processing_time,\n )\n\n except Exception as e:\n processing_time = time.time() - start_time\n logger.error(f\"Rebuild embeddings failed: {e}\")\n logger.exception(e)\n\n return RebuildEmbeddingsOutput(\n success=False,\n total_items=0,\n jobs_submitted=0,\n failed_submissions=0,\n processing_time=processing_time,\n error_message=str(e),\n )\n" }, { "path": "commands/example_commands.py", "content": "import asyncio\nimport time\nfrom typing import List, Optional\n\nfrom loguru import logger\nfrom pydantic import BaseModel\nfrom surreal_commands import command\n\n\nclass TextProcessingInput(BaseModel):\n text: str\n operation: str = \"uppercase\" # uppercase, lowercase, word_count, reverse\n delay_seconds: Optional[int] = None # For testing async behavior\n\n\nclass TextProcessingOutput(BaseModel):\n success: bool\n original_text: str\n processed_text: Optional[str] = None\n word_count: Optional[int] = None\n processing_time: float\n error_message: Optional[str] = None\n\n\nclass DataAnalysisInput(BaseModel):\n numbers: List[float]\n analysis_type: str = \"basic\" # basic, detailed\n delay_seconds: Optional[int] = None\n\n\nclass DataAnalysisOutput(BaseModel):\n success: bool\n analysis_type: str\n count: int\n sum: Optional[float] = None\n average: Optional[float] = None\n min_value: Optional[float] = None\n max_value: Optional[float] = None\n processing_time: float\n error_message: Optional[str] = None\n\n\n@command(\"process_text\", app=\"open_notebook\")\nasync def process_text_command(input_data: TextProcessingInput) -> TextProcessingOutput:\n \"\"\"\n Example command for text processing. Tests basic command functionality\n and demonstrates different processing types.\n \"\"\"\n start_time = time.time()\n\n try:\n logger.info(f\"Processing text with operation: {input_data.operation}\")\n\n # Simulate processing delay if specified\n if input_data.delay_seconds:\n await asyncio.sleep(input_data.delay_seconds)\n\n processed_text = None\n word_count = None\n\n if input_data.operation == \"uppercase\":\n processed_text = input_data.text.upper()\n elif input_data.operation == \"lowercase\":\n processed_text = input_data.text.lower()\n elif input_data.operation == \"reverse\":\n processed_text = input_data.text[::-1]\n elif input_data.operation == \"word_count\":\n word_count = len(input_data.text.split())\n processed_text = f\"Word count: {word_count}\"\n else:\n raise ValueError(f\"Unknown operation: {input_data.operation}\")\n\n processing_time = time.time() - start_time\n\n return TextProcessingOutput(\n success=True,\n original_text=input_data.text,\n processed_text=processed_text,\n word_count=word_count,\n processing_time=processing_time,\n )\n\n except Exception as e:\n processing_time = time.time() - start_time\n logger.error(f\"Text processing failed: {e}\")\n return TextProcessingOutput(\n success=False,\n original_text=input_data.text,\n processing_time=processing_time,\n error_message=str(e),\n )\n\n\n@command(\"analyze_data\", app=\"open_notebook\")\nasync def analyze_data_command(input_data: DataAnalysisInput) -> DataAnalysisOutput:\n \"\"\"\n Example command for data analysis. Tests command with complex input/output\n and demonstrates error handling.\n \"\"\"\n start_time = time.time()\n\n try:\n logger.info(\n f\"Analyzing {len(input_data.numbers)} numbers with {input_data.analysis_type} analysis\"\n )\n\n # Simulate processing delay if specified\n if input_data.delay_seconds:\n await asyncio.sleep(input_data.delay_seconds)\n\n if not input_data.numbers:\n raise ValueError(\"No numbers provided for analysis\")\n\n count = len(input_data.numbers)\n sum_value = sum(input_data.numbers)\n average = sum_value / count\n min_value = min(input_data.numbers)\n max_value = max(input_data.numbers)\n\n processing_time = time.time() - start_time\n\n return DataAnalysisOutput(\n success=True,\n analysis_type=input_data.analysis_type,\n count=count,\n sum=sum_value,\n average=average,\n min_value=min_value,\n max_value=max_value,\n processing_time=processing_time,\n )\n\n except Exception as e:\n processing_time = time.time() - start_time\n logger.error(f\"Data analysis failed: {e}\")\n return DataAnalysisOutput(\n success=False,\n analysis_type=input_data.analysis_type,\n count=0,\n processing_time=processing_time,\n error_message=str(e),\n )\n" }, { "path": "commands/podcast_commands.py", "content": "import time\nimport uuid\nfrom pathlib import Path\nfrom typing import Optional\n\nfrom loguru import logger\nfrom pydantic import BaseModel\nfrom surreal_commands import CommandInput, CommandOutput, command\n\nfrom open_notebook.config import DATA_FOLDER\nfrom open_notebook.database.repository import ensure_record_id, repo_query\nfrom open_notebook.podcasts.models import (\n EpisodeProfile,\n PodcastEpisode,\n SpeakerProfile,\n _resolve_model_config,\n)\n\ntry:\n from podcast_creator import configure, create_podcast\nexcept ImportError as e:\n logger.error(f\"Failed to import podcast_creator: {e}\")\n raise ValueError(\"podcast_creator library not available\")\n\n\ndef build_episode_output_dir(data_folder: str) -> tuple[str, Path]:\n \"\"\"Build a filesystem-safe output directory path for a podcast episode.\n\n Uses a UUID as the directory name so the path is safe regardless of\n what the user typed as episode name (spaces, special chars, etc.).\n\n Returns:\n A tuple of (episode_dir_name, output_dir_path).\n \"\"\"\n episode_dir_name = str(uuid.uuid4())\n output_dir = Path(f\"{data_folder}/podcasts/episodes/{episode_dir_name}\")\n return episode_dir_name, output_dir\n\n\ndef full_model_dump(model):\n if isinstance(model, BaseModel):\n return model.model_dump()\n elif isinstance(model, dict):\n return {k: full_model_dump(v) for k, v in model.items()}\n elif isinstance(model, list):\n return [full_model_dump(item) for item in model]\n else:\n return model\n\n\nclass PodcastGenerationInput(CommandInput):\n episode_profile: str\n speaker_profile: str\n episode_name: str\n content: str\n briefing_suffix: Optional[str] = None\n\n\nclass PodcastGenerationOutput(CommandOutput):\n success: bool\n episode_id: Optional[str] = None\n audio_file_path: Optional[str] = None\n transcript: Optional[dict] = None\n outline: Optional[dict] = None\n processing_time: float\n error_message: Optional[str] = None\n\n\n@command(\"generate_podcast\", app=\"open_notebook\", retry={\"max_attempts\": 1})\nasync def generate_podcast_command(\n input_data: PodcastGenerationInput,\n) -> PodcastGenerationOutput:\n \"\"\"\n Real podcast generation using podcast-creator library with Episode Profiles\n \"\"\"\n start_time = time.time()\n\n try:\n logger.info(\n f\"Starting podcast generation for episode: {input_data.episode_name}\"\n )\n logger.info(f\"Using episode profile: {input_data.episode_profile}\")\n\n # 1. Load Episode and Speaker profiles from SurrealDB\n episode_profile = await EpisodeProfile.get_by_name(input_data.episode_profile)\n if not episode_profile:\n raise ValueError(\n f\"Episode profile '{input_data.episode_profile}' not found\"\n )\n\n speaker_profile = await SpeakerProfile.get_by_name(\n episode_profile.speaker_config\n )\n if not speaker_profile:\n raise ValueError(\n f\"Speaker profile '{episode_profile.speaker_config}' not found\"\n )\n\n logger.info(f\"Loaded episode profile: {episode_profile.name}\")\n logger.info(f\"Loaded speaker profile: {speaker_profile.name}\")\n\n # 2. Validate that model registry fields are populated\n if not episode_profile.outline_llm:\n raise ValueError(\n f\"Episode profile '{episode_profile.name}' has no outline model configured. \"\n \"Please update the profile to select an outline model.\"\n )\n if not episode_profile.transcript_llm:\n raise ValueError(\n f\"Episode profile '{episode_profile.name}' has no transcript model configured. \"\n \"Please update the profile to select a transcript model.\"\n )\n if not speaker_profile.voice_model:\n raise ValueError(\n f\"Speaker profile '{speaker_profile.name}' has no voice model configured. \"\n \"Please update the profile to select a voice model.\"\n )\n\n # 3. Resolve model configs with credentials\n outline_provider, outline_model_name, outline_config = (\n await episode_profile.resolve_outline_config()\n )\n transcript_provider, transcript_model_name, transcript_config = (\n await episode_profile.resolve_transcript_config()\n )\n tts_provider, tts_model_name, tts_config = (\n await speaker_profile.resolve_tts_config()\n )\n\n logger.info(\n f\"Resolved models - outline: {outline_provider}/{outline_model_name}, \"\n f\"transcript: {transcript_provider}/{transcript_model_name}, \"\n f\"tts: {tts_provider}/{tts_model_name}\"\n )\n\n # 4. Load all profiles and configure podcast-creator\n episode_profiles = await repo_query(\"SELECT * FROM episode_profile\")\n speaker_profiles = await repo_query(\"SELECT * FROM speaker_profile\")\n\n # Transform the surrealdb array into a dictionary for podcast-creator\n episode_profiles_dict = {\n profile[\"name\"]: profile for profile in episode_profiles\n }\n speaker_profiles_dict = {\n profile[\"name\"]: profile for profile in speaker_profiles\n }\n\n # 5. Inject resolved model configs into profile dicts\n # Resolve ALL episode profiles (podcast-creator validates all).\n # Remove profiles that fail resolution to prevent validation errors.\n for ep_name in list(episode_profiles_dict.keys()):\n ep_dict = episode_profiles_dict[ep_name]\n try:\n if ep_dict.get(\"outline_llm\"):\n prov, model, conf = await _resolve_model_config(\n str(ep_dict[\"outline_llm\"])\n )\n ep_dict[\"outline_provider\"] = prov\n ep_dict[\"outline_model\"] = model\n ep_dict[\"outline_config\"] = conf\n if ep_dict.get(\"transcript_llm\"):\n prov, model, conf = await _resolve_model_config(\n str(ep_dict[\"transcript_llm\"])\n )\n ep_dict[\"transcript_provider\"] = prov\n ep_dict[\"transcript_model\"] = model\n ep_dict[\"transcript_config\"] = conf\n except Exception as e:\n logger.warning(\n f\"Failed to resolve models for episode profile '{ep_name}', \"\n f\"removing from config to prevent validation errors: {e}\"\n )\n del episode_profiles_dict[ep_name]\n\n # Resolve TTS for ALL speaker profiles (podcast-creator validates all).\n # Remove profiles that fail resolution to prevent validation errors.\n for sp_name in list(speaker_profiles_dict.keys()):\n sp_dict = speaker_profiles_dict[sp_name]\n if sp_dict.get(\"voice_model\"):\n try:\n prov, model, conf = await _resolve_model_config(\n str(sp_dict[\"voice_model\"])\n )\n sp_dict[\"tts_provider\"] = prov\n sp_dict[\"tts_model\"] = model\n sp_dict[\"tts_config\"] = conf\n except Exception as e:\n logger.warning(\n f\"Failed to resolve TTS for speaker profile '{sp_name}', \"\n f\"removing from config to prevent validation errors: {e}\"\n )\n del speaker_profiles_dict[sp_name]\n continue\n\n # Per-speaker TTS overrides\n for speaker in sp_dict.get(\"speakers\", []):\n if speaker.get(\"voice_model\"):\n try:\n prov, model, conf = await _resolve_model_config(\n str(speaker[\"voice_model\"])\n )\n speaker[\"tts_provider\"] = prov\n speaker[\"tts_model\"] = model\n speaker[\"tts_config\"] = conf\n except Exception as e:\n logger.warning(\n f\"Failed to resolve per-speaker TTS for '{speaker.get('name')}': {e}\"\n )\n\n # 6. Generate briefing\n briefing = episode_profile.default_briefing\n if input_data.briefing_suffix:\n briefing += f\"\\n\\nAdditional instructions: {input_data.briefing_suffix}\"\n\n # Create the record for the episode and associate with the ongoing command\n episode = PodcastEpisode(\n name=input_data.episode_name,\n episode_profile=full_model_dump(episode_profile.model_dump()),\n speaker_profile=full_model_dump(speaker_profile.model_dump()),\n command=ensure_record_id(input_data.execution_context.command_id)\n if input_data.execution_context\n else None,\n briefing=briefing,\n content=input_data.content,\n audio_file=None,\n transcript=None,\n outline=None,\n )\n await episode.save()\n\n configure(\"speakers_config\", {\"profiles\": speaker_profiles_dict})\n configure(\"episode_config\", {\"profiles\": episode_profiles_dict})\n\n logger.info(\"Configured podcast-creator with episode and speaker profiles\")\n\n logger.info(f\"Generated briefing (length: {len(briefing)} chars)\")\n\n # 7. Create output directory using UUID for filesystem-safe paths\n episode_dir_name, output_dir = build_episode_output_dir(DATA_FOLDER)\n output_dir.mkdir(parents=True, exist_ok=True)\n\n logger.info(f\"Created output directory: {output_dir}\")\n\n # 8. Generate podcast using podcast-creator\n logger.info(\"Starting podcast generation with podcast-creator...\")\n\n result = await create_podcast(\n content=input_data.content,\n briefing=briefing,\n episode_name=episode_dir_name,\n output_dir=str(output_dir),\n speaker_config=speaker_profile.name,\n episode_profile=episode_profile.name,\n )\n\n episode.audio_file = (\n str(result.get(\"final_output_file_path\")) if result else None\n )\n episode.transcript = {\n \"transcript\": full_model_dump(result[\"transcript\"]) if result else None\n }\n episode.outline = full_model_dump(result[\"outline\"]) if result else None\n await episode.save()\n\n processing_time = time.time() - start_time\n logger.info(\n f\"Successfully generated podcast episode: {episode.id} in {processing_time:.2f}s\"\n )\n\n return PodcastGenerationOutput(\n success=True,\n episode_id=str(episode.id),\n audio_file_path=str(result.get(\"final_output_file_path\"))\n if result\n else None,\n transcript={\"transcript\": full_model_dump(result[\"transcript\"])}\n if result.get(\"transcript\")\n else None,\n outline=full_model_dump(result[\"outline\"])\n if result.get(\"outline\")\n else None,\n processing_time=processing_time,\n )\n\n except ValueError:\n raise\n\n except Exception as e:\n logger.error(f\"Podcast generation failed: {e}\")\n logger.exception(e)\n\n error_msg = str(e)\n if \"Invalid json output\" in error_msg or \"Expecting value\" in error_msg:\n error_msg += (\n \"\\n\\nNOTE: This error commonly occurs with GPT-5 models that use extended thinking. \"\n \"The model may be putting all output inside tags, leaving nothing to parse. \"\n \"Try using gpt-4o, gpt-4o-mini, or gpt-4-turbo instead in your episode profile.\"\n )\n\n raise RuntimeError(error_msg) from e\n" }, { "path": "commands/source_commands.py", "content": "import time\nfrom typing import Any, Dict, List, Optional\n\nfrom loguru import logger\nfrom pydantic import BaseModel\nfrom surreal_commands import CommandInput, CommandOutput, command\n\nfrom open_notebook.database.repository import ensure_record_id\nfrom open_notebook.domain.notebook import Source\nfrom open_notebook.domain.transformation import Transformation\nfrom open_notebook.exceptions import ConfigurationError\n\ntry:\n from open_notebook.graphs.source import source_graph\n from open_notebook.graphs.transformation import graph as transform_graph\nexcept ImportError as e:\n logger.error(f\"Failed to import graphs: {e}\")\n raise ValueError(\"graphs not available\")\n\n\ndef full_model_dump(model):\n if isinstance(model, BaseModel):\n return model.model_dump()\n elif isinstance(model, dict):\n return {k: full_model_dump(v) for k, v in model.items()}\n elif isinstance(model, list):\n return [full_model_dump(item) for item in model]\n else:\n return model\n\n\nclass SourceProcessingInput(CommandInput):\n source_id: str\n content_state: Dict[str, Any]\n notebook_ids: List[str]\n transformations: List[str]\n embed: bool\n\n\nclass SourceProcessingOutput(CommandOutput):\n success: bool\n source_id: str\n embedded_chunks: int = 0\n insights_created: int = 0\n processing_time: float\n error_message: Optional[str] = None\n\n\n@command(\n \"process_source\",\n app=\"open_notebook\",\n retry={\n \"max_attempts\": 15, # Handle deep queues (workaround for SurrealDB v2 transaction conflicts)\n \"wait_strategy\": \"exponential_jitter\",\n \"wait_min\": 1,\n \"wait_max\": 120, # Allow queue to drain\n \"stop_on\": [ValueError, ConfigurationError], # Don't retry validation/config errors\n \"retry_log_level\": \"debug\", # Avoid log noise during transaction conflicts\n },\n)\nasync def process_source_command(\n input_data: SourceProcessingInput,\n) -> SourceProcessingOutput:\n \"\"\"\n Process source content using the source_graph workflow\n \"\"\"\n start_time = time.time()\n\n try:\n logger.info(f\"Starting source processing for source: {input_data.source_id}\")\n logger.info(f\"Notebook IDs: {input_data.notebook_ids}\")\n logger.info(f\"Transformations: {input_data.transformations}\")\n logger.info(f\"Embed: {input_data.embed}\")\n\n # 1. Load transformation objects from IDs\n transformations = []\n for trans_id in input_data.transformations:\n logger.info(f\"Loading transformation: {trans_id}\")\n transformation = await Transformation.get(trans_id)\n if not transformation:\n raise ValueError(f\"Transformation '{trans_id}' not found\")\n transformations.append(transformation)\n\n logger.info(f\"Loaded {len(transformations)} transformations\")\n\n # 2. Get existing source record to update its command field\n source = await Source.get(input_data.source_id)\n if not source:\n raise ValueError(f\"Source '{input_data.source_id}' not found\")\n\n # Update source with command reference\n source.command = (\n ensure_record_id(input_data.execution_context.command_id)\n if input_data.execution_context\n else None\n )\n await source.save()\n\n logger.info(f\"Updated source {source.id} with command reference\")\n\n # 3. Process source with all notebooks\n logger.info(f\"Processing source with {len(input_data.notebook_ids)} notebooks\")\n\n # Execute source_graph with all notebooks\n result = await source_graph.ainvoke(\n { # type: ignore[arg-type]\n \"content_state\": input_data.content_state,\n \"notebook_ids\": input_data.notebook_ids, # Use notebook_ids (plural) as expected by SourceState\n \"apply_transformations\": transformations,\n \"embed\": input_data.embed,\n \"source_id\": input_data.source_id, # Add the source_id to the state\n }\n )\n\n processed_source = result[\"source\"]\n\n # 4. Gather processing results (notebook associations handled by source_graph)\n # Note: embedding is fire-and-forget (async job), so we can't query the\n # count here — it hasn't completed yet. The embed_source_command logs\n # the actual count when it finishes.\n insights_list = await processed_source.get_insights()\n insights_created = len(insights_list)\n\n processing_time = time.time() - start_time\n embed_status = \"submitted\" if input_data.embed else \"skipped\"\n logger.info(\n f\"Successfully processed source: {processed_source.id} in {processing_time:.2f}s\"\n )\n logger.info(\n f\"Created {insights_created} insights, embedding {embed_status}\"\n )\n\n return SourceProcessingOutput(\n success=True,\n source_id=str(processed_source.id),\n embedded_chunks=0,\n insights_created=insights_created,\n processing_time=processing_time,\n )\n\n except ValueError as e:\n # Validation errors are permanent failures - don't retry\n processing_time = time.time() - start_time\n logger.error(f\"Source processing failed: {e}\")\n return SourceProcessingOutput(\n success=False,\n source_id=input_data.source_id,\n processing_time=processing_time,\n error_message=str(e),\n )\n except Exception as e:\n # Transient failure - will be retried (surreal-commands logs final failure)\n logger.debug(\n f\"Transient error processing source {input_data.source_id}: {e}\"\n )\n raise\n\n\n# =============================================================================\n# RUN TRANSFORMATION COMMAND\n# =============================================================================\n\n\nclass RunTransformationInput(CommandInput):\n \"\"\"Input for running a transformation on an existing source.\"\"\"\n\n source_id: str\n transformation_id: str\n\n\nclass RunTransformationOutput(CommandOutput):\n \"\"\"Output from transformation command.\"\"\"\n\n success: bool\n source_id: str\n transformation_id: str\n processing_time: float\n error_message: Optional[str] = None\n\n\n@command(\n \"run_transformation\",\n app=\"open_notebook\",\n retry={\n \"max_attempts\": 5,\n \"wait_strategy\": \"exponential_jitter\",\n \"wait_min\": 1,\n \"wait_max\": 60,\n \"stop_on\": [ValueError, ConfigurationError], # Don't retry validation/config errors\n \"retry_log_level\": \"debug\",\n },\n)\nasync def run_transformation_command(\n input_data: RunTransformationInput,\n) -> RunTransformationOutput:\n \"\"\"\n Run a transformation on an existing source to generate an insight.\n\n This command runs the transformation graph which:\n 1. Loads the source and transformation\n 2. Calls the LLM to generate insight content\n 3. Creates the insight via create_insight command (fire-and-forget)\n\n Use this command for UI-triggered insight generation to avoid blocking\n the HTTP request while the LLM processes.\n\n Retry Strategy:\n - Retries up to 5 times for transient failures (network, timeout, etc.)\n - Uses exponential-jitter backoff (1-60s)\n - Does NOT retry permanent failures (ValueError for validation errors)\n \"\"\"\n start_time = time.time()\n\n try:\n logger.info(\n f\"Running transformation {input_data.transformation_id} \"\n f\"on source {input_data.source_id}\"\n )\n\n # Load source\n source = await Source.get(input_data.source_id)\n if not source:\n raise ValueError(f\"Source '{input_data.source_id}' not found\")\n\n # Load transformation\n transformation = await Transformation.get(input_data.transformation_id)\n if not transformation:\n raise ValueError(\n f\"Transformation '{input_data.transformation_id}' not found\"\n )\n\n # Run transformation graph (includes LLM call + insight creation)\n await transform_graph.ainvoke(\n input=dict(source=source, transformation=transformation)\n )\n\n processing_time = time.time() - start_time\n logger.info(\n f\"Successfully ran transformation {input_data.transformation_id} \"\n f\"on source {input_data.source_id} in {processing_time:.2f}s\"\n )\n\n return RunTransformationOutput(\n success=True,\n source_id=input_data.source_id,\n transformation_id=input_data.transformation_id,\n processing_time=processing_time,\n )\n\n except ValueError as e:\n # Validation errors are permanent failures - don't retry\n processing_time = time.time() - start_time\n logger.error(\n f\"Failed to run transformation {input_data.transformation_id} \"\n f\"on source {input_data.source_id}: {e}\"\n )\n return RunTransformationOutput(\n success=False,\n source_id=input_data.source_id,\n transformation_id=input_data.transformation_id,\n processing_time=processing_time,\n error_message=str(e),\n )\n except Exception as e:\n # Transient failure - will be retried (surreal-commands logs final failure)\n logger.debug(\n f\"Transient error running transformation {input_data.transformation_id} \"\n f\"on source {input_data.source_id}: {e}\"\n )\n raise\n" }, { "path": "docker-compose.yml", "content": "services:\n surrealdb:\n image: surrealdb/surrealdb:v2\n command: start --log info --user root --pass root rocksdb:/mydata/mydatabase.db\n user: root # Required for bind mounts on Linux\n ports:\n - \"8000:8000\"\n volumes:\n - ./surreal_data:/mydata\n environment:\n - SURREAL_EXPERIMENTAL_GRAPHQL=true\n restart: always\n pull_policy: always\n\n open_notebook:\n image: lfnovo/open_notebook:v1-latest\n ports:\n - \"8502:8502\" # Web UI\n - \"5055:5055\" # REST API\n environment:\n # REQUIRED: Change this to your own secret string\n # This encrypts your API keys in the database\n - OPEN_NOTEBOOK_ENCRYPTION_KEY=change-me-to-a-secret-string\n\n # Database connection (default values - no need to change)\n - SURREAL_URL=ws://surrealdb:8000/rpc\n - SURREAL_USER=root\n - SURREAL_PASSWORD=root\n - SURREAL_NAMESPACE=open_notebook\n - SURREAL_DATABASE=open_notebook\n volumes:\n - ./notebook_data:/app/data\n depends_on:\n - surrealdb\n restart: always\n pull_policy: always\n" }, { "path": "docs/0-START-HERE/index.md", "content": "# Open Notebook - Start Here\n\n**Open Notebook** is a privacy-focused AI research assistant. Upload documents, chat with AI, generate notes, and create podcasts—all with complete control over your data.\n\n## Choose Your Path\n\n### 🚀 I want to use OpenAI (Fastest)\n**5 minutes to running.** GPT, simple setup, powerful results.\n\n→ [OpenAI Quick Start](quick-start-openai.md)\n\n---\n\n### ☁️ I want to use other cloud AI (Anthropic, Google, OpenRouter, etc.)\n**5 minutes to running.** Choose from 15+ AI providers.\n\n→ [Cloud Providers Quick Start](quick-start-cloud.md)\n\n---\n\n### 🏠 I want to run locally (Ollama or LMStudio, completely private)\n**5 minutes to running.** Keep everything private, on your machine. No costs.\n\n→ [Local Quick Start](quick-start-local.md)\n\n---\n\n## What Can You Do?\n\n- 📄 **Upload Content**: PDFs, web links, audio, video, text\n- 🤖 **Chat with AI**: Ask questions about your documents with citations\n- 📝 **Generate Notes**: AI creates summaries and insights\n- 🎙️ **Create Podcasts**: Turn research into professional audio content\n- 🔍 **Search**: Full-text and semantic search across all content\n- ⚙️ **Transform**: Extract insights, analyze themes, create summaries\n\n## Why Open Notebook?\n\n| Feature | Open Notebook | Notebook LM |\n|---------|---|---|\n| **Privacy** | Self-hosted, your control | Cloud, Google's servers |\n| **AI Choice** | 15+ providers | Google's models only |\n| **Podcast Speakers** | 1-4 customizable | 2 only |\n| **Cost** | Completely free | Free (but your data) |\n| **Offline** | Yes | No |\n\n## Prerequisites\n\n- **Docker**: All paths use Docker (free)\n- **AI Provider**: Either a cloud API key OR use free local models (Ollama)\n\n---\n\n## Next Steps\n\n1. Pick your path above ⬆️\n2. Follow the 5-minute quick start\n3. Create your first notebook\n4. Start uploading documents!\n\n---\n\n**Need Help?** Join our [Discord community](https://discord.gg/37XJPXfz2w) or see [Full Documentation](../index.md).\n" }, { "path": "docs/0-START-HERE/quick-start-cloud.md", "content": "# Quick Start - Cloud AI Providers (5 minutes)\n\nGet Open Notebook running with **Anthropic, Google, Groq, or other cloud providers**. Same simplicity as OpenAI, with more choices.\n\n## Prerequisites\n\n1. **Docker Desktop** installed\n - [Download here](https://www.docker.com/products/docker-desktop/)\n - Already have it? Skip to step 2\n\n2. **API Key** from your chosen provider:\n - **OpenRouter** (100+ models, one key): https://openrouter.ai/keys\n - **Anthropic (Claude)**: https://console.anthropic.com/\n - **Google (Gemini)**: https://aistudio.google.com/\n - **Groq** (fast, free tier): https://console.groq.com/\n - **Mistral**: https://console.mistral.ai/\n - **DeepSeek**: https://platform.deepseek.com/\n - **xAI (Grok)**: https://console.x.ai/\n\n## Step 1: Create Configuration (1 min)\n\nCreate a new folder `open-notebook` and add this file:\n\n**docker-compose.yml**:\n```yaml\nservices:\n surrealdb:\n image: surrealdb/surrealdb:v2\n command: start --user root --pass password --bind 0.0.0.0:8000 rocksdb:/mydata/mydatabase.db\n ports:\n - \"8000:8000\"\n volumes:\n - ./surreal_data:/mydata\n # Removed the healthcheck because the v2 image is too minimal to run wget/curl\n restart: always\n\n open_notebook:\n image: lfnovo/open_notebook:v1-latest\n pull_policy: always\n ports:\n - \"8502:8502\" # Web UI\n - \"5055:5055\" # API\n environment:\n - OPEN_NOTEBOOK_ENCRYPTION_KEY=change-me-to-a-secret-string\n - SURREAL_URL=ws://surrealdb:8000/rpc\n - SURREAL_USER=root\n - SURREAL_PASSWORD=password\n - SURREAL_NAMESPACE=open_notebook\n - SURREAL_DATABASE=open_notebook\n volumes:\n - ./notebook_data:/app/data\n depends_on:\n - surrealdb\n restart: always\n\n```\n\n**Edit the file:**\n- Replace `change-me-to-a-secret-string` with your own secret (any string works)\n\n---\n\n## Step 2: Start Services (1 min)\n\nOpen terminal in your `open-notebook` folder:\n\n```bash\ndocker compose up -d\n```\n\nWait 15-20 seconds for services to start.\n\n---\n\n## Step 3: Access Open Notebook (instant)\n\nOpen your browser:\n```\nhttp://localhost:8502\n```\n\nYou should see the Open Notebook interface!\n\n---\n\n## Step 4: Configure Your AI Provider (1 min)\n\n1. Go to **Settings** → **API Keys**\n2. Click **Add Credential**\n3. Select your provider (e.g., Anthropic, Google, Groq, OpenRouter)\n4. Give it a name, paste your API key\n5. Click **Save**\n6. Click **Test Connection** — should show success\n7. Click **Discover Models** → **Register Models**\n\nYour provider's models are now available!\n\n> **Multiple providers**: You can add credentials for as many providers as you want. Just repeat this step for each provider.\n\n---\n\n## Step 5: Configure Your Model (1 min)\n\n1. Go to **Settings** (gear icon)\n2. Navigate to **Models**\n3. Select your provider's model:\n\n| Provider | Recommended Model | Notes |\n|----------|-------------------|-------|\n| **OpenRouter** | `anthropic/claude-3.5-sonnet` | Access 100+ models |\n| **Anthropic** | `claude-3-5-sonnet-latest` | Best reasoning |\n| **Google** | `gemini-2.0-flash` | Large context, fast |\n| **Groq** | `llama-3.3-70b-versatile` | Ultra-fast |\n| **Mistral** | `mistral-large-latest` | Strong European option |\n\n4. Click **Save**\n\n---\n\n## Step 6: Create Your First Notebook (1 min)\n\n1. Click **New Notebook**\n2. Name: \"My Research\"\n3. Click **Create**\n\n---\n\n## Step 7: Add Content & Chat (2 min)\n\n1. Click **Add Source**\n2. Choose **Web Link**\n3. Paste any article URL\n4. Wait for processing\n5. Go to **Chat** and ask questions!\n\n---\n\n## Verification Checklist\n\n- [ ] Docker is running\n- [ ] You can access `http://localhost:8502`\n- [ ] Provider credential is configured and tested\n- [ ] Models are registered\n- [ ] You created a notebook\n- [ ] Chat works\n\n**All checked?** You're ready to research!\n\n---\n\n## Provider Comparison\n\n| Provider | Speed | Quality | Context | Cost |\n|----------|-------|---------|---------|------|\n| **OpenRouter** | Varies | Varies | Varies | Varies (100+ models) |\n| **Anthropic** | Medium | Excellent | 200K | $$$ |\n| **Google** | Fast | Very Good | 1M+ | $$ |\n| **Groq** | Ultra-fast | Good | 128K | $ (free tier) |\n| **Mistral** | Fast | Good | 128K | $$ |\n| **DeepSeek** | Medium | Very Good | 64K | $ |\n\n---\n\n## Troubleshooting\n\n### \"Model not found\" Error\n\n1. Go to **Settings** → **API Keys**\n2. Click **Test Connection** on your credential\n3. If valid, click **Discover Models** → **Register Models**\n4. Check you have credits/access for the model\n\n### \"Cannot connect to server\"\n\n```bash\ndocker ps # Check all services running\ndocker compose logs # View logs\ndocker compose restart # Restart everything\n```\n\n### Provider-Specific Issues\n\n**Anthropic**: Ensure key starts with `sk-ant-`\n**Google**: Use AI Studio key, not Cloud Console\n**Groq**: Free tier has rate limits; upgrade if needed\n\n---\n\n## Cost Estimates\n\nApproximate costs per 1K tokens:\n\n| Provider | Input | Output |\n|----------|-------|--------|\n| Anthropic (Sonnet) | $0.003 | $0.015 |\n| Google (Flash) | $0.0001 | $0.0004 |\n| Groq (Llama 70B) | Free tier available | - |\n| Mistral (Large) | $0.002 | $0.006 |\n\nCheck provider websites for current pricing.\n\n---\n\n## Next Steps\n\n1. **Add Your Content**: PDFs, web links, documents\n2. **Explore Features**: Podcasts, transformations, search\n3. **Full Documentation**: [See all features](../3-USER-GUIDE/index.md)\n\n---\n\n**Need help?** Join our [Discord community](https://discord.gg/37XJPXfz2w)!\n" }, { "path": "docs/0-START-HERE/quick-start-local.md", "content": "# Quick Start - Local & Private (5 minutes)\n\nGet Open Notebook running with **100% local AI** using Ollama. No cloud API keys needed, completely private.\n\n## Prerequisites\n\n1. **Docker Desktop** installed\n - [Download here](https://www.docker.com/products/docker-desktop/)\n - Already have it? Skip to step 2\n\n2. **Local LLM** - Choose one:\n - **Ollama** (recommended): [Download here](https://ollama.ai/)\n - **LM Studio** (GUI alternative): [Download here](https://lmstudio.ai)\n\n## Step 1: Choose Your Setup (1 min)\n\n### Local Machine (Same Computer)\nEverything runs on your machine. Recommended for testing/learning.\n\n### Remote Server (Raspberry Pi, NAS, Cloud VM)\nRun on a different computer, access from another. Needs network configuration.\n\n---\n\n## Step 2: Create Configuration (1 min)\n\nCreate a new folder `open-notebook-local` and add this file:\n\n**docker-compose.yml**:\n```yaml\nservices:\n surrealdb:\n image: surrealdb/surrealdb:v2\n command: start --user root --pass password --bind 0.0.0.0:8000 rocksdb:/mydata/mydatabase.db\n ports:\n - \"8000:8000\"\n volumes:\n - ./surreal_data:/mydata\n\n open_notebook:\n image: lfnovo/open_notebook:v1-latest-single\n pull_policy: always\n ports:\n - \"8502:8502\" # Web UI (React frontend)\n - \"5055:5055\" # API (required!)\n environment:\n # Encryption key for credential storage (required)\n - OPEN_NOTEBOOK_ENCRYPTION_KEY=change-me-to-a-secret-string\n\n # Database (required)\n - SURREAL_URL=ws://surrealdb:8000/rpc\n - SURREAL_USER=root\n - SURREAL_PASSWORD=password\n - SURREAL_NAMESPACE=open_notebook\n - SURREAL_DATABASE=open_notebook\n volumes:\n - ./notebook_data:/app/data\n - ./surreal_data:/mydata\n depends_on:\n - surrealdb\n restart: always\n\n ollama:\n image: ollama/ollama:latest\n ports:\n - \"11434:11434\"\n volumes:\n - ./ollama_models:/root/.ollama\n environment:\n # Optional: set GPU support if available\n - OLLAMA_NUM_GPU=0\n restart: always\n\n```\n\n**Edit the file:**\n- Replace `change-me-to-a-secret-string` with your own secret (any string works)\n\n---\n\n## Step 3: Start Services (1 min)\n\nOpen terminal in your `open-notebook-local` folder:\n\n```bash\ndocker compose up -d\n```\n\nWait 10-15 seconds for all services to start.\n\n---\n\n## Step 4: Download a Model (2-3 min)\n\nOllama needs at least one language model. Pick one:\n\n```bash\n# Fastest & smallest (recommended for testing)\ndocker exec open-notebook-local-ollama-1 ollama pull mistral\n\n# OR: Better quality but slower\ndocker exec open-notebook-local-ollama-1 ollama pull neural-chat\n\n# OR: Even better quality, more VRAM needed\ndocker exec open-notebook-local-ollama-1 ollama pull llama2\n```\n\nThis downloads the model (will take 1-5 minutes depending on your internet).\n\n---\n\n## Step 5: Access Open Notebook (instant)\n\nOpen your browser:\n```\nhttp://localhost:8502\n```\n\nYou should see the Open Notebook interface.\n\n---\n\n## Step 6: Configure Ollama Provider (1 min)\n\n1. Go to **Settings** → **API Keys**\n2. Click **Add Credential**\n3. Select provider: **Ollama**\n4. Give it a name (e.g., \"Local Ollama\")\n5. Enter the base URL: `http://ollama:11434`\n6. Click **Save**\n7. Click **Test Connection** — should show success\n8. Click **Discover Models** → **Register Models**\n\n---\n\n## Step 7: Configure Local Model (1 min)\n\n1. Go to **Settings** → **Models**\n2. Set:\n - **Language Model**: `ollama/mistral` (or whichever model you downloaded)\n - **Embedding Model**: `ollama/nomic-embed-text` (auto-downloads if missing)\n3. Click **Save**\n\n---\n\n## Step 8: Create Your First Notebook (1 min)\n\n1. Click **New Notebook**\n2. Name: \"My Private Research\"\n3. Click **Create**\n\n---\n\n## Step 9: Add Local Content (1 min)\n\n1. Click **Add Source**\n2. Choose **Text**\n3. Paste some text or a local document\n4. Click **Add**\n\n---\n\n## Step 10: Chat With Your Content (1 min)\n\n1. Go to **Chat**\n2. Type: \"What did you learn from this?\"\n3. Click **Send**\n4. Watch as the local Ollama model responds!\n\n---\n\n## Verification Checklist\n\n- [ ] Docker is running\n- [ ] You can access `http://localhost:8502`\n- [ ] Ollama credential is configured and tested\n- [ ] Models are registered\n- [ ] You created a notebook\n- [ ] Chat works with local model\n\n**All checked?** You have a completely **private, offline** research assistant!\n\n---\n\n## Advantages of Local Setup\n\n- **No API costs** - Free forever\n- **No internet required** - True offline capability\n- **Privacy first** - Your data never leaves your machine\n- **No subscriptions** - No monthly bills\n\n**Trade-off:** Slower than cloud models (depends on your CPU/GPU)\n\n---\n\n## Troubleshooting\n\n### \"ollama: command not found\"\n\nDocker image name might be different:\n```bash\ndocker ps # Find the Ollama container name\ndocker exec ollama pull mistral\n```\n\n### Model Download Stuck\n\nCheck internet connection and restart:\n```bash\ndocker compose restart ollama\n```\n\nThen retry the model pull command.\n\n### \"Address already in use\" Error\n\n```bash\ndocker compose down\ndocker compose up -d\n```\n\n### Low Performance\n\nCheck if GPU is available:\n```bash\n# Show available GPUs\ndocker exec open-notebook-local-ollama-1 ollama ps\n\n# Enable GPU in docker-compose.yml:\n# - OLLAMA_NUM_GPU=1\n```\n\nThen restart: `docker compose restart ollama`\n\n### Adding More Models\n\n```bash\n# List available models\ndocker exec open-notebook-local-ollama-1 ollama list\n\n# Pull additional model\ndocker exec open-notebook-local-ollama-1 ollama pull neural-chat\n```\n\n---\n\n## Next Steps\n\n**Now that it's running:**\n\n1. **Add Your Own Content**: PDFs, documents, articles (see 3-USER-GUIDE)\n2. **Explore Features**: Podcasts, transformations, search\n3. **Full Documentation**: [See all features](../3-USER-GUIDE/index.md)\n4. **Scale Up**: Deploy to a server with better hardware for faster responses\n5. **Benchmark Models**: Try different models to find the speed/quality tradeoff you prefer\n\n---\n\n## Alternative: Using LM Studio Instead of Ollama\n\n**Prefer a GUI?** LM Studio is easier for non-technical users:\n\n1. Download LM Studio: https://lmstudio.ai\n2. Open the app, download a model from the library\n3. Go to \"Local Server\" tab, start server (port 1234)\n4. In Open Notebook, go to **Settings** → **API Keys**\n5. Click **Add Credential** → Select **OpenAI-Compatible**\n6. Enter base URL: `http://host.docker.internal:1234/v1`\n7. Enter API key: `lm-studio` (placeholder)\n8. Click **Save**, then **Test Connection**\n9. Configure in Settings → Models → Select your LM Studio model\n\n**Note**: LM Studio runs outside Docker, use `host.docker.internal` to connect.\n\n---\n\n## Going Further\n\n- **Switch models**: Change in Settings → Models anytime\n- **Add more models**:\n - Ollama: Run `ollama pull `, then re-discover models from the credential\n - LM Studio: Download from the app library\n- **Deploy to server**: Same docker-compose.yml works anywhere\n- **Use cloud hybrid**: Keep some local models, add cloud provider credentials for complex tasks\n\n---\n\n## Common Model Choices\n\n| Model | Speed | Quality | VRAM | Best For |\n|-------|-------|---------|------|----------|\n| **mistral** | Fast | Good | 4GB | Testing, general use |\n| **neural-chat** | Medium | Better | 6GB | Balanced, recommended |\n| **llama2** | Slow | Best | 8GB+ | Complex reasoning |\n| **phi** | Very Fast | Fair | 2GB | Minimal hardware |\n\n---\n\n**Need Help?** Join our [Discord community](https://discord.gg/37XJPXfz2w) - many users run local setups!\n" }, { "path": "docs/0-START-HERE/quick-start-openai.md", "content": "# Quick Start - OpenAI (5 minutes)\n\nGet Open Notebook running with OpenAI's GPT models. Fast, powerful, and simple.\n\n## Prerequisites\n\n1. **Docker Desktop** installed\n - [Download here](https://www.docker.com/products/docker-desktop/)\n - Already have it? Skip to step 2\n\n2. **OpenAI API Key** (required)\n - Go to https://platform.openai.com/api-keys\n - Create account → Create new secret key\n - Add at least $5 in credits to your account\n - Copy the key (starts with `sk-`)\n\n## Step 1: Create Configuration (1 min)\n\nCreate a new folder `open-notebook` and add this file:\n\n**docker-compose.yml**:\n```yaml\nservices:\n surrealdb:\n image: surrealdb/surrealdb:v2\n command: start --user root --pass password --bind 0.0.0.0:8000 rocksdb:/mydata/mydatabase.db\n ports:\n - \"8000:8000\"\n volumes:\n - ./surreal_data:/mydata\n\n open_notebook:\n image: lfnovo/open_notebook:v1-latest\n pull_policy: always\n ports:\n - \"8502:8502\" # Web UI\n - \"5055:5055\" # API\n environment:\n # Encryption key for credential storage (required)\n - OPEN_NOTEBOOK_ENCRYPTION_KEY=change-me-to-a-secret-string\n\n # Database (required)\n - SURREAL_URL=ws://surrealdb:8000/rpc\n - SURREAL_USER=root\n - SURREAL_PASSWORD=password\n - SURREAL_NAMESPACE=open_notebook\n - SURREAL_DATABASE=open_notebook\n volumes:\n - ./notebook_data:/app/data\n depends_on:\n - surrealdb\n restart: always\n\n```\n\n**Edit the file:**\n- Replace `change-me-to-a-secret-string` with your own secret (any string works)\n\n---\n\n## Step 2: Start Services (1 min)\n\nOpen terminal in your `open-notebook` folder:\n\n```bash\ndocker compose up -d\n```\n\nWait 15-20 seconds for services to start.\n\n---\n\n## Step 3: Access Open Notebook (instant)\n\nOpen your browser:\n```\nhttp://localhost:8502\n```\n\nYou should see the Open Notebook interface!\n\n---\n\n## Step 4: Configure Your OpenAI Provider (1 min)\n\n1. Go to **Settings** → **API Keys**\n2. Click **Add Credential**\n3. Select provider: **OpenAI**\n4. Give it a name (e.g., \"My OpenAI Key\")\n5. Paste your OpenAI API key\n6. Click **Save**\n7. Click **Test Connection** — should show success\n8. Click **Discover Models** → **Register Models**\n\nYour OpenAI models are now available!\n\n---\n\n## Step 5: Create Your First Notebook (1 min)\n\n1. Click **New Notebook**\n2. Name: \"My Research\"\n3. Click **Create**\n\n---\n\n## Step 6: Add a Source (1 min)\n\n1. Click **Add Source**\n2. Choose **Web Link**\n3. Paste: `https://en.wikipedia.org/wiki/Artificial_intelligence`\n4. Click **Add**\n5. Wait for processing (30-60 seconds)\n\n---\n\n## Step 7: Chat With Your Content (1 min)\n\n1. Go to **Chat**\n2. Type: \"What is artificial intelligence?\"\n3. Click **Send**\n4. Watch as GPT responds with information from your source!\n\n---\n\n## Verification Checklist\n\n- [ ] Docker is running\n- [ ] You can access `http://localhost:8502`\n- [ ] OpenAI credential is configured and tested\n- [ ] You created a notebook\n- [ ] You added a source\n- [ ] Chat works\n\n**All checked?** You have a fully working AI research assistant!\n\n---\n\n## Using Different Models\n\nIn your notebook, go to **Settings** → **Models** to choose:\n- `gpt-4o` - Best quality (recommended)\n- `gpt-4o-mini` - Fast and cheap (good for testing)\n\n---\n\n## Troubleshooting\n\n### \"Port 8502 already in use\"\n\nChange the port in docker-compose.yml:\n```yaml\nports:\n - \"8503:8502\" # Use 8503 instead\n```\n\nThen access at `http://localhost:8503`\n\n### \"API key not working\"\n\n1. Go to **Settings** → **API Keys**\n2. Click **Test Connection** on your OpenAI credential\n3. If it fails, verify your key at https://platform.openai.com\n4. Delete the credential and create a new one with the correct key\n\n### \"Cannot connect to server\"\n\n```bash\ndocker ps # Check all services running\ndocker compose logs # View logs\ndocker compose restart # Restart everything\n```\n\n---\n\n## Next Steps\n\n1. **Add Your Own Content**: PDFs, web links, documents\n2. **Explore Features**: Podcasts, transformations, search\n3. **Full Documentation**: [See all features](../3-USER-GUIDE/index.md)\n\n---\n\n## Cost Estimate\n\nOpenAI pricing (approximate):\n- **Conversation**: $0.01-0.10 per 1K tokens\n- **Embeddings**: $0.02 per 1M tokens\n- **Typical usage**: $1-5/month for light use, $20-50/month for heavy use\n\nCheck https://openai.com/pricing for current rates.\n\n---\n\n**Need help?** Join our [Discord community](https://discord.gg/37XJPXfz2w)!\n" }, { "path": "docs/1-INSTALLATION/docker-compose.md", "content": "# Docker Compose Installation (Recommended)\n\nMulti-container setup with separate services. **Best for most users.**\n\n> **Alternative Registry:** All images are available on both Docker Hub (`lfnovo/open_notebook`) and GitHub Container Registry (`ghcr.io/lfnovo/open-notebook`). Use GHCR if Docker Hub is blocked or you prefer GitHub-native workflows.\n\n## Prerequisites\n\n- **Docker Desktop** installed ([Download](https://www.docker.com/products/docker-desktop/))\n- **5-10 minutes** of your time\n- **API key** for at least one AI provider (OpenAI recommended for beginners)\n\n## Step 1: Get docker-compose.yml (1 min)\n\n**Option A: Download from repository**\n```bash\ncurl -o docker-compose.yml https://raw.githubusercontent.com/lfnovo/open-notebook/main/docker-compose.yml\n```\n\n**Option B: Use the official file from the repo**\n\nThe official `docker-compose.yml` is in the root of our repository: [View on GitHub](https://github.com/lfnovo/open-notebook/blob/main/docker-compose.yml)\n\nCopy that file to your project folder.\n\n**Option C: Create manually**\n\nCreate a file called `docker-compose.yml` with this content:\n\n```yaml\nservices:\n surrealdb:\n image: surrealdb/surrealdb:v2\n command: start --log info --user root --pass root rocksdb:/mydata/mydatabase.db\n user: root # Required for bind mounts on Linux\n ports:\n - \"8000:8000\"\n volumes:\n - ./surreal_data:/mydata\n environment:\n - SURREAL_EXPERIMENTAL_GRAPHQL=true\n restart: always\n pull_policy: always\n\n open_notebook:\n image: lfnovo/open_notebook:v1-latest\n ports:\n - \"8502:8502\" # Web UI\n - \"5055:5055\" # REST API\n environment:\n # REQUIRED: Change this to your own secret string\n - OPEN_NOTEBOOK_ENCRYPTION_KEY=change-me-to-a-secret-string\n\n # Database connection (default values - no need to change)\n - SURREAL_URL=ws://surrealdb:8000/rpc\n - SURREAL_USER=root\n - SURREAL_PASSWORD=root\n - SURREAL_NAMESPACE=open_notebook\n - SURREAL_DATABASE=open_notebook\n volumes:\n - ./notebook_data:/app/data\n depends_on:\n - surrealdb\n restart: always\n pull_policy: always\n```\n\n**Edit the file:**\n- Replace `change-me-to-a-secret-string` with your own secret (any string works, e.g., `my-super-secret-key-123`)\n\n---\n\n## Step 2: Start Services (2 min)\n\nOpen terminal in the `open-notebook` folder:\n\n```bash\ndocker compose up -d\n```\n\nWait 15-20 seconds for all services to start:\n```\n✅ surrealdb running on :8000\n✅ open_notebook running on :8502 (UI) and :5055 (API)\n```\n\nCheck status:\n```bash\ndocker compose ps\n```\n\n---\n\n## Step 3: Verify Installation (1 min)\n\n**API Health:**\n```bash\ncurl http://localhost:5055/health\n# Should return: {\"status\": \"healthy\"}\n```\n\n**Frontend Access:**\nOpen browser to:\n```\nhttp://localhost:8502\n```\n\nYou should see the Open Notebook interface!\n\n---\n\n## Step 4: Configure AI Provider (2 min)\n\n1. Go to **Settings** → **API Keys**\n2. Click **Add Credential**\n3. Select your provider (e.g., OpenAI, Anthropic, Google)\n4. Give it a name, paste your API key\n5. Click **Save**\n6. Click **Test Connection** — should show success\n7. Click **Discover Models** → **Register Models**\n\nYour models are now available!\n\n> **Need an API key?** Get one from your chosen provider:\n> - **OpenAI**: https://platform.openai.com/api-keys\n> - **Anthropic**: https://console.anthropic.com/\n> - **Google**: https://aistudio.google.com/\n> - **Groq**: https://console.groq.com/\n\n---\n\n## Step 5: First Notebook (2 min)\n\n1. Click **New Notebook**\n2. Name: \"My Research\"\n3. Description: \"Getting started\"\n4. Click **Create**\n\nDone! You now have a fully working Open Notebook instance.\n\n---\n\n## Configuration\n\n### Adding Ollama (Free Local Models)\n\nInstead of manually editing, use our ready-made example:\n\n```bash\n# Download the Ollama example\ncurl -o docker-compose.yml https://raw.githubusercontent.com/lfnovo/open-notebook/main/examples/docker-compose-ollama.yml\n\n# Or copy from repo\ncp examples/docker-compose-ollama.yml docker-compose.yml\n```\n\nSee [examples/docker-compose-ollama.yml](../../examples/docker-compose-ollama.yml) for the complete setup.\n\n**Manual setup:** Add this to your existing `docker-compose.yml`:\n\n```yaml\n ollama:\n image: ollama/ollama:latest\n ports:\n - \"11434:11434\"\n volumes:\n - ollama_models:/root/.ollama\n restart: always\n\nvolumes:\n ollama_models:\n```\n\nThen restart and pull a model:\n```bash\ndocker compose restart\ndocker exec open-notebook-local-ollama-1 ollama pull mistral\n```\n\nConfigure Ollama in the Settings UI:\n1. Go to **Settings** → **API Keys**\n2. Click **Add Credential** → Select **Ollama**\n3. Enter base URL: `http://ollama:11434`\n4. Click **Save**, then **Test Connection**\n5. Click **Discover Models** → **Register Models**\n\n---\n\n## Environment Variables Reference\n\n| Variable | Purpose | Example |\n|----------|---------|---------|\n| `OPEN_NOTEBOOK_ENCRYPTION_KEY` | Encryption key for credentials | `my-secret-key` |\n| `SURREAL_URL` | Database connection | `ws://surrealdb:8000/rpc` |\n| `SURREAL_USER` | Database user | `root` |\n| `SURREAL_PASSWORD` | Database password | `root` |\n| `SURREAL_NAMESPACE` | Database namespace | `open_notebook` |\n| `SURREAL_DATABASE` | Database name | `open_notebook` |\n| `API_URL` | API external URL | `http://localhost:5055` |\n\nSee [Environment Reference](../5-CONFIGURATION/environment-reference.md) for complete list.\n\n---\n\n## Common Tasks\n\n### Stop Services\n```bash\ndocker compose down\n```\n\n### View Logs\n```bash\n# All services\ndocker compose logs -f\n\n# Specific service\ndocker compose logs -f api\n```\n\n### Restart Services\n```bash\ndocker compose restart\n```\n\n### Update to Latest Version\n```bash\ndocker compose down\ndocker compose pull\ndocker compose up -d\n```\n\n### Remove All Data\n```bash\ndocker compose down -v\n```\n\n---\n\n## Troubleshooting\n\n### \"Cannot connect to API\" Error\n\n1. Check if Docker is running:\n```bash\ndocker ps\n```\n\n2. Check if services are running:\n```bash\ndocker compose ps\n```\n\n3. Check API logs:\n```bash\ndocker compose logs api\n```\n\n4. Wait longer - services can take 20-30 seconds to start on first run\n\n---\n\n### Port Already in Use\n\nIf you get \"Port 8502 already in use\", change the port:\n\n```yaml\nports:\n - \"8503:8502\" # Use 8503 instead\n - \"5055:5055\" # Keep API port same\n```\n\nThen access at `http://localhost:8503`\n\n---\n\n### Credential Issues\n\n1. Go to **Settings** → **API Keys**\n2. Click **Test Connection** on the credential\n3. If it fails, verify key at provider's website\n4. Check you have credits in your account\n5. Delete and re-create the credential if needed\n\n---\n\n### Database Connection Issues\n\nCheck SurrealDB is running:\n```bash\ndocker compose logs surrealdb\n```\n\nReset database:\n```bash\ndocker compose down -v\ndocker compose up -d\n```\n\n### Database Permission Denied (Linux)\n\nIf you see `Permission denied` or `Failed to create RocksDB directory` in SurrealDB logs:\n\n```bash\ndocker compose logs surrealdb | grep -i permission\n```\n\nThis happens because SurrealDB runs as a non-root user but Docker creates bind mount directories as root. Add `user: root` to the surrealdb service:\n\n```yaml\nsurrealdb:\n image: surrealdb/surrealdb:v2\n user: root # Fix for Linux bind mount permissions\n # ... rest of config\n```\n\nThen restart:\n```bash\ndocker compose down -v\ndocker compose up -d\n```\n\n---\n\n## Alternative Setups\n\nLooking for different configurations? Check out our [examples/](../../examples/) folder:\n\n- **[Ollama Setup](../../examples/docker-compose-ollama.yml)** - Run local AI models (free, private)\n- **[Single Container](../../examples/docker-compose-single.yml)** - All-in-one container (deprecated, not recommended)\n- **[Development](../../examples/docker-compose-dev.yml)** - For contributors and developers\n\nEach example includes detailed comments and usage instructions.\n\n---\n\n## Next Steps\n\n1. **Add Content**: Sources, notebooks, documents\n2. **Configure Models**: Settings → Models (choose your preferences)\n3. **Explore Features**: Chat, search, transformations\n4. **Read Guide**: [User Guide](../3-USER-GUIDE/index.md)\n\n---\n\n## Production Deployment\n\nFor production use, see:\n- [Security Hardening](../5-CONFIGURATION/security.md)\n- [Reverse Proxy](../5-CONFIGURATION/reverse-proxy.md)\n\n---\n\n## Getting Help\n\n- **Discord**: [Community support](https://discord.gg/37XJPXfz2w)\n- **Issues**: [GitHub Issues](https://github.com/lfnovo/open-notebook/issues)\n- **Docs**: [Full documentation](../index.md)\n" }, { "path": "docs/1-INSTALLATION/from-source.md", "content": "# From Source Installation\n\nClone the repository and run locally. **For developers and contributors.**\n\n## Prerequisites\n\n- **Python 3.11+** - [Download](https://www.python.org/)\n- **Node.js 18+** - [Download](https://nodejs.org/)\n- **Git** - [Download](https://git-scm.com/)\n- **Docker** (for SurrealDB) - [Download](https://docker.com/)\n- **uv** (Python package manager) - `curl -LsSf https://astral.sh/uv/install.sh | sh`\n- API key from OpenAI or similar (or use Ollama for free)\n\n## Quick Setup (10 minutes)\n\n### 1. Clone Repository\n\n```bash\ngit clone https://github.com/lfnovo/open-notebook.git\ncd open-notebook\n\n# If you forked it:\ngit clone https://github.com/YOUR_USERNAME/open-notebook.git\ncd open-notebook\ngit remote add upstream https://github.com/lfnovo/open-notebook.git\n```\n\n### 2. Install Python Dependencies\n\n```bash\nuv sync\nuv pip install python-magic\n```\n\n#### 2.1 Alternative: Conda Setup (Optional)\n\nIf you prefer using **Conda** to manage your environments, follow these steps instead of the standard `uv sync`:\n\n```bash\n# Create and activate the environment\nconda create -n open-notebook python=3.11 -y\nconda activate open-notebook\n\n# Install uv inside conda to maintain compatibility with the Makefile\nconda install -c conda-forge uv nodejs -y\n\n# Sync dependencies\nuv sync\n```\n\n> **Note**: Installing `uv` inside your Conda environment ensures that commands like `make start-all` and `make api` continue to work seamlessly.\n\n### 3. Start SurrealDB\n\n```bash\n# Terminal 1\nmake database\n# or: docker compose up surrealdb\n```\n\n### 4. Set Environment Variables\n\n```bash\ncp .env.example .env\n# Edit .env and set:\n# OPEN_NOTEBOOK_ENCRYPTION_KEY=my-secret-key\n```\n\nAfter starting the app, configure AI providers via the **Settings → API Keys** UI in the browser.\n\n### 5. Start API\n\n```bash\n# Terminal 2\nmake api\n# or: uv run --env-file .env uvicorn api.main:app --host 0.0.0.0 --port 5055\n```\n\n### 6. Start Frontend\n\n```bash\n# Terminal 3\ncd frontend && npm install && npm run dev\n```\n\n### 7. Access\n\n- **Frontend**: http://localhost:3000\n- **API Docs**: http://localhost:5055/docs\n- **Database**: http://localhost:8000\n\n### 8. Configure AI Provider\n\n1. Open http://localhost:3000\n2. Go to **Settings** → **API Keys**\n3. Click **Add Credential** → Select your provider → Paste API key\n4. Click **Save**, then **Test Connection**\n5. Click **Discover Models** → **Register Models**\n\n---\n\n## Development Workflow\n\n### Code Quality\n\n```bash\n# Format and lint Python\nmake ruff\n# or: ruff check . --fix\n\n# Type checking\nmake lint\n# or: uv run python -m mypy .\n```\n\n### Run Tests\n\n```bash\nuv run pytest tests/\n```\n\n### Common Commands\n\n```bash\n# Start everything\nmake start-all\n\n# View API docs\nopen http://localhost:5055/docs\n\n# Check database migrations\n# (Auto-run on API startup)\n\n# Clean up\nmake clean\n```\n\n---\n\n## Troubleshooting\n\n### Python version too old\n\n```bash\npython --version # Check version\nuv sync --python 3.11 # Use specific version\n```\n\n### npm: command not found\n\nInstall Node.js from https://nodejs.org/\n\n### Database connection errors\n\n```bash\ndocker ps # Check SurrealDB running\ndocker logs surrealdb # View logs\n```\n\n### Port 5055 already in use\n\n```bash\n# Use different port\nuv run uvicorn api.main:app --port 5056\n```\n\n---\n\n## Next Steps\n\n1. Read [Development Guide](../7-DEVELOPMENT/quick-start.md)\n2. See [Architecture Overview](../7-DEVELOPMENT/architecture.md)\n3. Check [Contributing Guide](../7-DEVELOPMENT/contributing.md)\n\n---\n\n## Getting Help\n\n- **Discord**: [Community](https://discord.gg/37XJPXfz2w)\n- **Issues**: [GitHub Issues](https://github.com/lfnovo/open-notebook/issues)\n" }, { "path": "docs/1-INSTALLATION/index.md", "content": "# Installation Guide\n\nChoose your installation route based on your setup and use case.\n\n## Quick Decision: Which Route?\n\n### 🚀 I want the easiest setup (Recommended for most)\n**→ [Docker Compose](docker-compose.md)** - Multi-container setup, production-ready\n- ✅ All features working\n- ✅ Clear separation of services\n- ✅ Easy to scale\n- ✅ Works on Mac, Windows, Linux\n- ⏱️ 5 minutes to running\n\n---\n\n### 🏠 I want everything in one container (Simplified)\n**→ [Single Container](single-container.md)** - All-in-one for simple deployments\n- ✅ Minimal configuration\n- ✅ Lower resource usage\n- ✅ Good for shared hosting\n- ✅ Works on PikaPods, Railway, etc.\n- ⏱️ 3 minutes to running\n\n---\n\n### 👨‍💻 I want to develop/contribute (Developers only)\n**→ [From Source](from-source.md)** - Clone repo, set up locally\n- ✅ Full control over code\n- ✅ Easy to debug\n- ✅ Can modify and test\n- ⚠️ Requires Python 3.11+, Node.js\n- ⏱️ 10 minutes to running\n\n---\n\n\n## System Requirements\n\n### Minimum\n- **RAM**: 4GB\n- **Storage**: 2GB for app + space for documents\n- **CPU**: Any modern processor\n- **Network**: Internet (optional for offline setup)\n\n### Recommended\n- **RAM**: 8GB+\n- **Storage**: 10GB+ for documents and models\n- **CPU**: Multi-core processor\n- **GPU**: Optional (speeds up local AI models)\n\n---\n\n## AI Provider Options\n\n### Cloud-Based (Pay-as-you-go)\n- **OpenAI** - GPT-4, GPT-4o, fast and capable\n- **Anthropic (Claude)** - Claude 3.5 Sonnet, excellent reasoning\n- **Google Gemini** - Multimodal, cost-effective\n- **Groq** - Ultra-fast inference\n- **Others**: Mistral, DeepSeek, xAI, OpenRouter\n\n**Cost**: Usually $0.01-$0.10 per 1K tokens\n**Speed**: Fast (sub-second)\n**Privacy**: Your data sent to cloud\n\n### Local (Free, Private)\n- **Ollama** - Run open-source models locally\n- **LM Studio** - Desktop app for local models\n- **Hugging Face models** - Download and run\n\n**Cost**: $0 (just electricity)\n**Speed**: Depends on your hardware (slow to medium)\n**Privacy**: 100% offline\n\n---\n\n## Choose a Route\n\n**Already know which way to go?** Pick your installation path:\n\n- [Docker Compose](docker-compose.md) - **Most users**\n- [Single Container](single-container.md) - **Shared hosting**\n- [From Source](from-source.md) - **Developers**\n\n> **Privacy-first?** Any installation method works with Ollama for 100% local AI. See [Local Quick Start](../0-START-HERE/quick-start-local.md).\n\n---\n\n## Pre-Installation Checklist\n\nBefore installing, you'll need:\n\n- [ ] **Docker** (for Docker routes) or **Node.js 18+** (for source)\n- [ ] **AI Provider API key** (OpenAI, Anthropic, etc.) OR willingness to use free local models\n- [ ] **At least 4GB RAM** available\n- [ ] **Stable internet** (or offline setup with Ollama)\n\n---\n\n## Detailed Installation Instructions\n\n### For Docker Users\n1. Install [Docker Desktop](https://docker.com/products/docker-desktop)\n2. Choose: [Docker Compose](docker-compose.md) or [Single Container](single-container.md)\n3. Follow the step-by-step guide\n4. Access at `http://localhost:8502`\n\n### For Source Installation (Developers)\n1. Have Python 3.11+, Node.js 18+, Git installed\n2. Follow [From Source](from-source.md)\n3. Run `make start-all`\n4. Access at `http://localhost:8502` (frontend) or `http://localhost:5055` (API)\n\n---\n\n## After Installation\n\nOnce you're up and running:\n\n1. **Configure Models** - Choose your AI provider in Settings\n2. **Create First Notebook** - Start organizing research\n3. **Add Sources** - PDFs, web links, documents\n4. **Explore Features** - Chat, search, transformations\n5. **Read Full Guide** - [User Guide](../3-USER-GUIDE/index.md)\n\n---\n\n## Troubleshooting During Installation\n\n**Having issues?** Check the troubleshooting section in your chosen installation guide, or see [Quick Fixes](../6-TROUBLESHOOTING/quick-fixes.md).\n\n---\n\n## Need Help?\n\n- **Discord**: [Join community](https://discord.gg/37XJPXfz2w)\n- **GitHub Issues**: [Report problems](https://github.com/lfnovo/open-notebook/issues)\n- **Docs**: See [Full Documentation](../index.md)\n\n---\n\n## Production Deployment\n\nInstalling for production use? See additional resources:\n\n- [Security Hardening](../5-CONFIGURATION/security.md)\n- [Reverse Proxy Setup](../5-CONFIGURATION/reverse-proxy.md)\n- [Performance Tuning](../5-CONFIGURATION/advanced.md)\n\n---\n\n**Ready to install?** Pick a route above! ⬆️\n" }, { "path": "docs/1-INSTALLATION/single-container.md", "content": "# Single Container Installation\n\nAll-in-one container setup. **Simpler than Docker Compose, but less flexible.**\n\n**Best for:** PikaPods, Railway, shared hosting, minimal setups\n\n> **Alternative Registry:** Images available on both Docker Hub (`lfnovo/open_notebook:v1-latest-single`) and GitHub Container Registry (`ghcr.io/lfnovo/open-notebook:v1-latest-single`).\n\n> **Note**: While this is a simple way to get started, we recommend [Docker Compose](docker-compose.md) for most users. Docker Compose is more flexible and will make it easier if we add more services to the setup in the future. This single-container option is best for platforms that specifically require it (PikaPods, Railway, etc.).\n\n## Prerequisites\n\n- Docker installed (for local testing)\n- API key from OpenAI, Anthropic, or another provider\n- 5 minutes\n\n## Quick Setup\n\n### For Local Testing (Docker)\n\n```yaml\n# docker-compose.yml\nservices:\n open_notebook:\n image: lfnovo/open_notebook:v1-latest-single\n pull_policy: always\n ports:\n - \"8502:8502\" # Web UI (React frontend)\n - \"5055:5055\" # API\n environment:\n - OPEN_NOTEBOOK_ENCRYPTION_KEY=change-me-to-a-secret-string\n - SURREAL_URL=ws://localhost:8000/rpc\n - SURREAL_USER=root\n - SURREAL_PASSWORD=root\n - SURREAL_NAMESPACE=open_notebook\n - SURREAL_DATABASE=open_notebook\n volumes:\n - ./data:/app/data\n restart: always\n```\n\nRun:\n```bash\ndocker compose up -d\n```\n\nAccess: `http://localhost:8502`\n\nThen configure your AI provider:\n1. Go to **Settings** → **API Keys**\n2. Click **Add Credential** → Select your provider → Paste API key\n3. Click **Save**, then **Test Connection**\n4. Click **Discover Models** → **Register Models**\n\n### For Cloud Platforms\n\n**PikaPods:**\n1. Click \"New App\"\n2. Search \"Open Notebook\"\n3. Set environment variables (at minimum: `OPEN_NOTEBOOK_ENCRYPTION_KEY`)\n4. Click \"Deploy\"\n5. Open the app → Go to **Settings → API Keys** to configure your AI provider\n\n**Railway:**\n1. Create new project\n2. Add `lfnovo/open_notebook:v1-latest-single`\n3. Set environment variables (at minimum: `OPEN_NOTEBOOK_ENCRYPTION_KEY`)\n4. Deploy\n5. Open the app → Go to **Settings → API Keys** to configure your AI provider\n\n**Render:**\n1. Create new Web Service\n2. Use Docker image: `lfnovo/open_notebook:v1-latest-single`\n3. Set environment variables in dashboard (at minimum: `OPEN_NOTEBOOK_ENCRYPTION_KEY`)\n4. Configure persistent disk for `/app/data` and `/mydata`\n\n**DigitalOcean App Platform:**\n1. Create new app from Docker Hub\n2. Use image: `lfnovo/open_notebook:v1-latest-single`\n3. Set port to 8502\n4. Add environment variables (at minimum: `OPEN_NOTEBOOK_ENCRYPTION_KEY`)\n5. Configure persistent storage\n\n**Heroku:**\n```bash\n# Using heroku.yml\nheroku container:push web\nheroku container:release web\nheroku config:set OPEN_NOTEBOOK_ENCRYPTION_KEY=your-secret-key\n```\n\n**Coolify:**\n1. Add new service → Docker Image\n2. Image: `lfnovo/open_notebook:v1-latest-single`\n3. Port: 8502\n4. Add environment variables (at minimum: `OPEN_NOTEBOOK_ENCRYPTION_KEY`)\n5. Enable persistent volumes\n6. Coolify handles HTTPS automatically\n\n---\n\n## Environment Variables\n\n| Variable | Purpose | Example |\n|----------|---------|---------|\n| `OPEN_NOTEBOOK_ENCRYPTION_KEY` | Encryption key for credentials (required) | `my-secret-key` |\n| `SURREAL_URL` | Database | `ws://localhost:8000/rpc` |\n| `SURREAL_USER` | DB user | `root` |\n| `SURREAL_PASSWORD` | DB password | `root` |\n| `SURREAL_NAMESPACE` | DB namespace | `open_notebook` |\n| `SURREAL_DATABASE` | DB name | `open_notebook` |\n| `API_URL` | External URL (for remote access) | `https://myapp.example.com` |\n\nAI provider API keys are configured via the **Settings → API Keys** UI after deployment.\n\n---\n\n## Limitations vs Docker Compose\n\n| Feature | Single Container | Docker Compose |\n|---------|------------------|-----------------|\n| Setup time | 2 minutes | 5 minutes |\n| Complexity | Minimal | Moderate |\n| Services | All bundled | Separated |\n| Scalability | Limited | Excellent |\n| Memory usage | ~800MB | ~1.2GB |\n\n---\n\n## Next Steps\n\nSame as Docker Compose setup - just access via `http://localhost:8502` (local) or your platform's URL (cloud).\n\n1. Go to **Settings → API Keys** to add your AI provider credential\n2. **Test Connection** and **Discover Models**\n\nSee [Docker Compose](docker-compose.md) for full post-install guide.\n" }, { "path": "docs/2-CORE-CONCEPTS/ai-context-rag.md", "content": "# AI Context & RAG - How Open Notebook Uses Your Research\n\nOpen Notebook uses different approaches to make AI models aware of your research depending on the feature. This section explains **RAG** (used in Ask) and **full-content context** (used in Chat).\n\n---\n\n## The Problem: Making AI Aware of Your Data\n\n### Traditional Approaches (and their problems)\n\n**Option 1: Fine-Tuning**\n- Train the model on your data\n- Pro: Model becomes specialized\n- Con: Expensive, slow, permanent (can't unlearn)\n\n**Option 2: Send Everything to Cloud**\n- Upload all your data to ChatGPT/Claude API\n- Pro: Works well, fast\n- Con: Privacy nightmare, data leaves your control, expensive\n\n**Option 3: Ignore Your Data**\n- Just use the base model without your research\n- Pro: Private, free\n- Con: AI doesn't know anything about your specific topic\n\n### Open Notebook's Dual Approach\n\n**For Chat**: Sends the entire selected content to the LLM\n- Simple and transparent: You select sources, they're sent in full\n- Maximum context: AI sees everything you choose\n- You control which sources are included\n\n**For Ask (RAG)**: Retrieval-Augmented Generation\n- RAG = Retrieval-Augmented Generation\n- The insight: *Search your content, find relevant pieces, send only those*\n- Automatic: AI decides what's relevant based on your question\n\n---\n\n## How RAG Works: Three Stages\n\n### Stage 1: Content Preparation\n\nWhen you upload a source, Open Notebook prepares it for retrieval:\n\n```\n1. EXTRACT TEXT\n PDF → text\n URL → webpage text\n Audio → transcribed text\n Video → subtitles + transcription\n\n2. CHUNK INTO PIECES\n Long documents → break into ~500-word chunks\n Why? AI context has limits; smaller pieces are more precise\n\n3. CREATE EMBEDDINGS\n Each chunk → semantic vector (numbers representing meaning)\n Why? Allows finding chunks by similarity, not just keywords\n\n4. STORE IN DATABASE\n Chunks + embeddings + metadata → searchable storage\n```\n\n**Example:**\n```\nSource: \"AI Safety Research 2026\" (50-page PDF)\n↓\nExtracted: 50 pages of text\n↓\nChunked: 150 chunks (~500 words each)\n↓\nEmbedded: Each chunk gets a vector (1536 numbers for OpenAI)\n↓\nStored: Ready for search\n```\n\n---\n\n### Stage 2: Query Time (What You Search For)\n\nWhen you ask a question, the system finds relevant content:\n\n```\n1. YOU ASK A QUESTION\n \"What does the paper say about alignment?\"\n\n2. SYSTEM CONVERTS QUESTION TO EMBEDDING\n Your question → vector (same way chunks are vectorized)\n\n3. SIMILARITY SEARCH\n Find chunks most similar to your question\n (using vector math, not keyword matching)\n\n4. RETURN TOP RESULTS\n Usually top 5-10 most similar chunks\n\n5. YOU GET BACK\n ✓ The relevant chunks\n ✓ Where they came from (sources + page numbers)\n ✓ Relevance scores\n```\n\n**Example:**\n```\nQ: \"What does the paper say about alignment?\"\n↓\nQ vector: [0.23, -0.51, 0.88, ..., 0.12]\n↓\nSearch: Compare to all chunk vectors\n↓\nResults:\n - Chunk 47 (alignment section): similarity 0.94\n - Chunk 63 (safety approaches): similarity 0.88\n - Chunk 12 (related work): similarity 0.71\n```\n\n---\n\n### Stage 3: Augmentation (How AI Uses It)\n\nNow you have the relevant pieces. The AI uses them:\n\n```\nSYSTEM BUILDS A PROMPT:\n \"You are an AI research assistant.\n\n The user has the following research materials:\n [CHUNK 47 CONTENT]\n [CHUNK 63 CONTENT]\n\n User question: 'What does the paper say about alignment?'\n\n Answer based on the above materials.\"\n\nAI RESPONDS:\n \"Based on the research materials, the paper approaches\n alignment through [pulls from chunks] and emphasizes\n [pulls from chunks]...\"\n\nSYSTEM ADDS CITATIONS:\n \"- See research materials page 15 for approach details\n - See research materials page 23 for emphasis on X\"\n```\n\n---\n\n## Two Search Modes: Exact vs. Semantic\n\nOpen Notebook provides two different search strategies for different goals.\n\n### 1. Text Search (Keyword Matching)\n\n**How it works:**\n- Uses BM25 ranking (the same algorithm Google uses)\n- Finds chunks containing your keywords\n- Ranks by relevance (how often keywords appear, position, etc.)\n\n**When to use:**\n- \"I remember the exact phrase 'X' and want to find it\"\n- \"I'm looking for a specific name or number\"\n- \"I need the exact quote\"\n\n**Example:**\n```\nSearch: \"transformer architecture\"\nResults:\n 1. Chunk with \"transformer architecture\" 3 times\n 2. Chunk with \"transformer\" and \"architecture\" separately\n 3. Chunk with \"transformer-based models\"\n```\n\n### 2. Vector Search (Semantic Similarity)\n\n**How it works:**\n- Converts your question to a vector (number embedding)\n- Finds chunks with similar vectors\n- No keywords needed—finds conceptually similar content\n\n**When to use:**\n- \"Find content about X (without saying exact words)\"\n- \"I'm exploring a concept\"\n- \"Find similar ideas even if worded differently\"\n\n**Example:**\n```\nSearch: \"what's the mechanism for model understanding?\"\nResults (no \"understanding\" in any chunk):\n 1. Chunk about interpretability and mechanistic analysis\n 2. Chunk about feature analysis\n 3. Chunk about attention mechanisms\n\nWhy? The vectors are semantically similar to your concept.\n```\n\n---\n\n## Context Management: Your Control Panel\n\nHere's where Open Notebook is different: **You decide what the AI sees.**\n\n### The Three Levels\n\n| Level | What's Shared | Example Cost | Privacy | Use Case |\n|-------|---------------|--------------|---------|----------|\n| **Full Content** | Complete source text | 10,000 tokens | Low | Detailed analysis, close reading |\n| **Summary Only** | AI-generated summary | 2,000 tokens | High | Background material, references |\n| **Not in Context** | Nothing | 0 tokens | Max | Confidential, irrelevant, or archived |\n\n### How It Works\n\n**Full Content:**\n```\nYou: \"What's the methodology in paper A?\"\nSystem:\n - Searches paper A\n - Retrieves full paper content (or large chunks)\n - Sends to AI: \"Here's paper A. Answer about methodology.\"\n - AI analyzes complete content\n - Result: Detailed, precise answer\n```\n\n**Summary Only:**\n```\nYou: \"I want to chat using paper A and B\"\nSystem:\n - For Paper A: Sends AI-generated summary (not full text)\n - For Paper B: Sends full content (detailed analysis)\n - AI sees 2 sources but in different detail levels\n - Result: Uses summaries for context, details for focused content\n```\n\n**Not in Context:**\n```\nYou: \"I have 10 sources but only want 5 in context\"\nSystem:\n - Paper A-E: In context (sent to AI)\n - Paper F-J: Not in context (AI can't see them, doesn't search them)\n - AI never knows these 5 sources exist\n - Result: Tight, focused context\n```\n\n### Why This Matters\n\n**Privacy**: You control what leaves your system\n```\nScenario: Confidential company docs + public research\nControl: Public research in context → Confidential docs excluded\nResult: AI never sees confidential content\n```\n\n**Cost**: You control token usage\n```\nScenario: 100 sources for background + 5 for detailed analysis\nControl: Full content for 5 detailed, summaries for 95 background\nResult: 80% lower token cost than sending everything\n```\n\n**Quality**: You control what the AI focuses on\n```\nScenario: 20 sources, question requires deep analysis\nControl: Full content for relevant source, exclude others\nResult: AI doesn't get distracted; gives better answer\n```\n\n---\n\n## The Difference: Chat vs. Ask\n\n**IMPORTANT**: These use completely different approaches!\n\n### Chat: Full-Content Context (NO RAG)\n\n**How it works:**\n```\nYOU:\n 1. Select which sources to include in context\n 2. Set context level (full/summary/excluded)\n 3. Ask question\n\nSYSTEM:\n - Takes ALL selected sources (respecting context levels)\n - Sends the ENTIRE content to the LLM at once\n - NO search, NO retrieval, NO chunking\n - AI sees everything you selected\n\nAI:\n - Responds based on the full content you provided\n - Can reference any part of selected sources\n - Conversational: context stays for follow-ups\n```\n\n**Use this when**:\n- You know which sources are relevant\n- You want conversational back-and-forth\n- You want AI to see the complete context\n- You're doing close reading or analysis\n\n**Advantages:**\n- Simple and transparent\n- AI sees everything (no missed content)\n- Conversational flow\n\n**Limitations:**\n- Limited by LLM context window\n- You must manually select relevant sources\n- Sends more tokens (higher cost with many sources)\n\n---\n\n### Ask: RAG - Automatic Retrieval\n\n**How it works:**\n```\nYOU:\n Ask one complex question\n\nSYSTEM:\n 1. Analyzes your question\n 2. Searches across ALL your sources automatically\n 3. Finds relevant chunks using vector similarity\n 4. Retrieves only the most relevant pieces\n 5. Sends ONLY those chunks to the LLM\n 6. Synthesizes into comprehensive answer\n\nAI:\n - Sees ONLY the retrieved chunks (not full sources)\n - Answers based on what was found to be relevant\n - One-shot answer (not conversational)\n```\n\n**Use this when**:\n- You have many sources and don't know which are relevant\n- You want the AI to search automatically\n- You need a comprehensive answer to a complex question\n- You want to minimize tokens sent to LLM\n\n**Advantages:**\n- Automatic search (you don't pick sources)\n- Works across many sources at once\n- Cost-effective (sends only relevant chunks)\n\n**Limitations:**\n- Not conversational (single question/answer)\n- AI only sees retrieved chunks (might miss context)\n- Search quality depends on how well question matches content\n\n---\n\n## What This Means: Privacy by Design\n\nOpen Notebook's RAG approach gives you something you don't get with ChatGPT or Claude directly:\n\n**You control the boundary between:**\n- What stays private (on your system)\n- What goes to AI (explicitly chosen)\n- What the AI can see (context levels)\n\n### The Audit Trail\n\nBecause everything is retrieved explicitly, you can ask:\n- \"Which sources did the AI use for this answer?\" → See citations\n- \"What exactly did the AI see?\" → See chunks in context level\n- \"Is the AI's claim actually in my sources?\" → Verify citation\n\nThis prevents hallucinations or misrepresentation better than most systems.\n\n---\n\n## How Embeddings Work (Simplified)\n\nThe magic of semantic search comes from embeddings. Here's the intuition:\n\n### The Idea\nInstead of storing text, store it as a list of numbers (vectors) that represent \"meaning.\"\n\n```\nChunk: \"The transformer uses attention mechanisms\"\nVector: [0.23, -0.51, 0.88, 0.12, ..., 0.34]\n (1536 numbers for OpenAI)\n\nAnother chunk: \"Attention allows models to focus on relevant parts\"\nVector: [0.24, -0.48, 0.87, 0.15, ..., 0.35]\n (similar numbers = similar meaning!)\n```\n\n### Why This Works\nWords that are semantically similar produce similar vectors. So:\n- \"alignment\" and \"interpretability\" have similar vectors\n- \"transformer\" and \"attention\" have related vectors\n- \"cat\" and \"dog\" are more similar than \"cat\" and \"radiator\"\n\n### How Search Works\n```\nYour question: \"How do models understand their decisions?\"\nQuestion vector: [0.25, -0.50, 0.86, 0.14, ..., 0.33]\n\nCompare to all stored vectors. Find the most similar:\n- Chunk about interpretability: similarity 0.94\n- Chunk about explainability: similarity 0.91\n- Chunk about feature attribution: similarity 0.88\n\nReturn the top matches.\n```\n\nThis is why semantic search finds conceptually similar content even when words are different.\n\n---\n\n## Key Design Decisions\n\n### 1. Search, Don't Train\n**Why?** Fine-tuning is slow and permanent. Search is flexible and reversible.\n\n### 2. Explicit Retrieval, Not Implicit Knowledge\n**Why?** You can verify what the AI saw. You have audit trails. You control what leaves your system.\n\n### 3. Multiple Search Types\n**Why?** Different questions need different search (keyword vs. semantic). Giving you both is more powerful.\n\n### 4. Context as a Permission System\n**Why?** Not everything you save needs to reach AI. You control granularly.\n\n---\n\n## Summary\n\nOpen Notebook gives you **two ways** to work with AI:\n\n### Chat (Full-Content)\n- Sends entire selected sources to LLM\n- Manual control: you pick sources\n- Conversational: back-and-forth dialog\n- Transparent: you know exactly what AI sees\n- Best for: focused analysis, close reading\n\n### Ask (RAG)\n- Searches and retrieves relevant chunks automatically\n- Automatic: AI finds what's relevant\n- One-shot: single comprehensive answer\n- Efficient: sends only relevant pieces\n- Best for: broad questions across many sources\n\n**Both approaches:**\n1. Keep your data private (doesn't leave your system by default)\n2. Give you control (you choose which features to use)\n3. Create audit trails (citations show what was used)\n4. Support multiple AI providers\n\n**Coming Soon**: The community is working on adding RAG capabilities to Chat as well, giving you the best of both worlds.\n" }, { "path": "docs/2-CORE-CONCEPTS/chat-vs-transformations.md", "content": "# Chat vs. Ask vs. Transformations - Which Tool for Which Job?\n\nOpen Notebook offers different ways to work with your research. Understanding when to use each is key to using the system effectively.\n\n---\n\n## The Three Interaction Modes\n\n### 1. CHAT - Conversational Exploration with Manual Context\n\n**What it is:** Have a conversation with AI about selected sources.\n\n**The flow:**\n```\n1. You select which sources to include (\"in context\")\n2. You ask a question\n3. AI responds using ONLY those sources\n4. You ask follow-up questions (context stays same)\n5. You change sources or context level, then continue\n```\n\n**Context management:** You explicitly choose which sources the AI can see.\n\n**Conversational:** Multiple questions with shared history.\n\n**Example:**\n```\nYou: [Select sources: \"paper1.pdf\", \"research_notes.txt\"]\n [Set context: Full content for paper1, Summary for notes]\n\nYou: \"What's the main argument in these sources?\"\nAI: \"Paper 1 argues X [citation]. Your notes emphasize Y [citation].\"\n\nYou: \"How do they differ?\"\nAI: \"Paper 1 focuses on X [citation], while your notes highlight Y [citation]...\"\n\nYou: [Now select different sources]\n\nYou: \"Compare to this other perspective\"\nAI: \"This new source takes a different approach...\"\n```\n\n**Best for:**\n- Exploring a focused topic with specific sources\n- Having a dialogue (multiple back-and-forth questions)\n- When you know which sources matter\n- When you want tight control over what goes to AI\n\n---\n\n### 2. ASK - Automated Comprehensive Search\n\n**What it is:** Ask one complex question, system automatically finds relevant content.\n\n**The flow:**\n```\n1. You ask a comprehensive question\n2. System analyzes the question\n3. System automatically searches your sources\n4. System retrieves relevant chunks\n5. System synthesizes answer from all results\n6. You get one detailed answer (not conversational)\n```\n\n**Context management:** Automatic. System figures out what's relevant.\n\n**Non-conversational:** One question → one answer. No follow-ups.\n\n**Example:**\n```\nYou: \"How do these papers compare their approaches to alignment?\n What does each one recommend?\"\n\nSystem:\n - Breaks down the question into search strategies\n - Searches all sources for alignment approaches\n - Searches all sources for recommendations\n - Retrieves top 10 relevant chunks\n - Synthesizes: \"Paper A recommends X [citation].\n Paper B recommends Y [citation].\n They differ in Z.\"\n\nYou: [Get back one comprehensive answer]\n [If you want to follow up, use Chat instead]\n```\n\n**Best for:**\n- Comprehensive, one-time questions\n- Comparing multiple sources at once\n- When you want the system to decide what's relevant\n- Complex questions that need multiple search angles\n- When you don't need a back-and-forth conversation\n\n---\n\n### 3. TRANSFORMATIONS - Template-Based Processing\n\n**What it is:** Apply a reusable template to a source and get structured output.\n\n**The flow:**\n```\n1. You define a transformation (or choose a preset)\n \"Extract: main argument, methodology, limitations\"\n\n2. You apply it to ONE source at a time\n (You can repeat for other sources)\n\n3. For the source:\n - Source content + transformation prompt → AI\n - Result stored as new insight/note\n\n4. You get back\n - Structured output (main argument, methodology, limitations)\n - Saved as a note in your notebook\n```\n\n**Context management:** Works on one source at a time.\n\n**Reusable:** Apply the same template to different sources (one by one).\n\n**Note**: Currently processes one source at a time. Batch processing (multiple sources at once) is planned for a future release.\n\n**Example:**\n```\nYou: Define transformation\n \"For each academic paper, extract:\n - Main research question\n - Methodology used\n - Key findings\n - Limitations and gaps\n - Recommended next research\"\n\nYou: Apply to paper 1\n\nSystem:\n - Runs the transformation on paper 1\n - Result stored as new note\n\nYou: Apply same transformation to paper 2, 3, etc.\n\nAfter 10 papers:\n - You have 10 structured notes with consistent format\n - Perfect for writing a literature review or comparison\n```\n\n**Best for:**\n- Extracting the same information from each source (run repeatedly)\n- Creating structured summaries with consistent format\n- Building a knowledge base of categorized insights\n- When you want reusable templates you can apply to each source\n\n---\n\n## Decision Tree: Which Tool to Use?\n\n```\nWhat are you trying to do?\n\n│\n├─→ \"I want to have a conversation about this topic\"\n│ └─→ Is the conversation exploratory or fixed?\n│ ├─→ Exploratory (I'll ask follow-ups)\n│ │ └─→ USE: CHAT\n│ │\n│ └─→ Fixed (One question → done)\n│ └─→ Go to next question\n│\n├─→ \"I need to compare these sources or get a comprehensive answer\"\n│ └─→ USE: ASK\n│\n├─→ \"I want to extract the same info from each source (one at a time)\"\n│ └─→ USE: TRANSFORMATIONS (apply to each source)\n│\n└─→ \"I just want to read and search\"\n └─→ USE: Search (text or vector)\n OR read your notes\n```\n\n---\n\n## Side-by-Side Comparison\n\n| Aspect | CHAT | ASK | TRANSFORMATIONS |\n|--------|------|-----|-----------------|\n| **What's it for?** | Conversational exploration | Comprehensive Q&A | Template-based extraction |\n| **# of questions** | Multiple (conversational) | One | One template per source |\n| **Context control** | Manual (you choose) | Automatic (system searches) | One source at a time |\n| **Conversational?** | Yes (follow-ups work) | No (one question only) | No (single operation) |\n| **Output** | Natural conversation | Natural answer | Structured note |\n| **Time** | Quick (back-and-forth) | Longer (comprehensive) | Per source |\n| **Best when** | Exploring & uncertain | Need full picture | Want consistent format |\n| **Model speed** | Any | Fast preferred | Any |\n\n---\n\n## Workflow Examples\n\n### Example 1: Academic Research\n\n```\nGoal: Write literature review from 15 papers\n\nStep 1: TRANSFORMATIONS\n - Define: \"Extract abstract, methodology, findings, relevance\"\n - Apply to paper 1 → get structured note\n - Apply to paper 2 → get structured note\n - ... repeat for all 15 papers\n - Result: 15 structured notes with consistent format\n\nStep 2: Read the notes\n - Now you have consistent summaries\n\nStep 3: CHAT or ASK\n - Chat: \"Help me organize these by theme\"\n - Ask: \"What are the common methodologies across these papers?\"\n\nStep 4: Write your review\n - Use the transformations as foundation\n - Use chat/ask insights for structure\n```\n\n### Example 2: Product Research\n\n```\nGoal: Understand customer feedback from interviews\n\nStep 1: Add sources (interview transcripts)\n\nStep 2: ASK\n - \"What are the top 10 pain points mentioned?\"\n - Get comprehensive answer with citations\n\nStep 3: CHAT\n - \"Can you help me group these by severity?\"\n - Continue conversation to prioritize\n\nStep 4: TRANSFORMATIONS (optional)\n - Define: \"Extract: pain point, frequency, who mentioned it\"\n - Apply to each interview (one by one)\n - Get structured data for analysis\n```\n\n### Example 3: Policy Analysis\n\n```\nGoal: Compare policy documents\n\nStep 1: Add all policy documents as sources\n\nStep 2: ASK\n - \"How do these policies differ on climate measures?\"\n - System searches all docs, gives comprehensive comparison\n\nStep 3: CHAT (if needed)\n - \"Which policy is most aligned with X goals?\"\n - Have discussion about trade-offs\n\nStep 4: Export notes\n - Save AI responses as notes for reports\n```\n\n---\n\n## Context Management: The Control Panel\n\nAll three modes let you control what the AI sees.\n\n### In CHAT and TRANSFORMATIONS\n```\nYou choose:\n - Which sources to include\n - Context level for each:\n ✓ Full Content (send complete text)\n ✓ Summary Only (send AI summary, not full text)\n ✓ Not in Context (exclude entirely)\n\nExample:\n Paper A: Full Content (analyzing closely)\n Paper B: Summary Only (background)\n Paper C: Not in Context (confidential)\n```\n\n### In ASK\n```\nContext is automatic:\n - System searches ALL your sources\n - Retrieves most relevant chunks\n - Sends those to AI\n\nBut you can:\n - Search in specific notebook\n - Filter by source type\n - Use the results to decide context for follow-up Chat\n```\n\n---\n\n## Model Selection\n\nEach mode works with different models:\n\n### CHAT\n- **Any model** works fine\n- Fast models (GPT-4o mini, Claude Haiku): Quick responses, good for conversation\n- Powerful models (GPT-4o, Claude Sonnet): Better reasoning, better for complex topics\n\n### ASK\n- **Fast models preferred** (because it processes multiple searches)\n- Can use powerful models if you want deep synthesis\n- Example: GPT-4 for strategy planning, GPT-4o-mini for quick facts\n\n### TRANSFORMATIONS\n- **Any model** works\n- Fast models (cost-effective for batch processing)\n- Powerful models (better quality extractions)\n\n---\n\n## Advanced: Chaining Modes Together\n\nYou can combine these modes:\n\n```\nTRANSFORMATIONS → CHAT\n 1. Use transformations to extract structured data\n 2. Use chat to discuss the results\n\nASK → TRANSFORMATIONS\n 1. Use Ask to understand what matters\n 2. Use Transformations to extract it from remaining sources\n\nCHAT → Save as Note → TRANSFORMATIONS\n 1. Have conversation (Chat)\n 2. Save good responses as notes\n 3. Use those notes as context for transformations\n```\n\n---\n\n## Summary: When to Use Each\n\n| Situation | Use | Why |\n|-----------|-----|-----|\n| \"I want to explore a topic with follow-up questions\" | **CHAT** | Conversational, you control context |\n| \"I need a comprehensive answer to one complex question\" | **ASK** | Automatic search, synthesized answer |\n| \"I want consistent summaries from each source\" | **TRANSFORMATIONS** | Template reuse, apply to each source |\n| \"I'm comparing two specific sources\" | **CHAT** | Select just those 2, have discussion |\n| \"I need to categorize each source by X criteria\" | **TRANSFORMATIONS** | Extract category from each source |\n| \"I want to understand the big picture across all sources\" | **ASK** | Automatic comprehensive search |\n| \"I want to build a knowledge base\" | **TRANSFORMATIONS** | Create structured note from each source |\n| \"I want to iterate on understanding\" | **CHAT** | Multiple questions, refine thinking |\n\nThe key insight: **Different questions need different tools.** Open Notebook gives you all three because research rarely fits one mode.\n" }, { "path": "docs/2-CORE-CONCEPTS/index.md", "content": "# Core Concepts - Understand the Mental Model\n\nBefore diving into how to use Open Notebook, it's important to understand **how it thinks**. These core concepts explain the \"why\" behind the design.\n\n## The Five Mental Models\n\n### 1. [Notebooks, Sources, and Notes](notebooks-sources-notes.md)\nHow Open Notebook organizes your research. Understand the three-tier container structure and how information flows from raw materials to finished insights.\n\n**Key idea**: A notebook is a scoped research container. Sources are inputs (PDFs, URLs, etc.). Notes are outputs (your insights, AI-generated summaries, captured responses).\n\n---\n\n### 2. [AI Context & RAG](ai-context-rag.md)\nHow Open Notebook makes AI aware of your research - two different approaches.\n\n**Key idea**: **Chat** sends entire selected sources to the LLM (full context, conversational). **Ask** uses RAG (retrieval-augmented generation) to automatically search and retrieve only relevant chunks. Different tools for different needs.\n\n---\n\n### 3. [Chat vs. Transformations](chat-vs-transformations.md)\nWhy Open Notebook has different interaction modes and when to use each one.\n\n**Key idea**: Chat is conversational exploration (you control context). Transformations are insight extractions. They reduced content to smaller bits of concentrated/dense information, which is much more suitable for an AI to use. \n\n---\n\n### 4. [Context Management](chat-vs-transformations.md#context-management-the-control-panel)\nYour control panel for privacy and cost. Decide what data actually reaches AI.\n\n**Key idea**: You choose three levels—not in context (private), summary only (condensed), or full content (complete access). This gives you fine-grained control.\n\n---\n\n### 5. [Podcasts Explained](podcasts-explained.md)\nWhy Open Notebook can turn research into audio and why this matters.\n\n**Key idea**: Podcasts transform your research into a different consumption format. Instead of reading, someone can listen and absorb your insights passively.\n\n---\n\n## Read This Section If:\n\n- **You're new to Open Notebook** — Start here to understand how the system works conceptually before learning the features\n- **You're confused about Chat vs Ask** — Section 2 explains the difference (full-content vs RAG)\n- **You're wondering when to use Chat vs Transformations** — Section 3 clarifies the differences\n- **You want to understand privacy controls** — Section 4 shows you what you can control\n- **You're curious about podcasts** — Section 5 explains the architecture and why it's different from competitors\n\n---\n\n## The Big Picture\n\nOpen Notebook is built on a simple insight: **Your research deserves to stay yours**.\n\nThat means:\n- **Privacy by default** — Your data doesn't leave your infrastructure unless you explicitly choose\n- **AI as a tool, not a gatekeeper** — You decide which sources the AI sees, not the AI deciding for you\n- **Flexible consumption** — Read, listen, search, chat, or transform your research however makes sense\n\nThese core concepts explain how that works.\n\n---\n\n## Next Steps\n\n1. **Just want to use it?** → Go to [User Guide](../3-USER-GUIDE/index.md)\n2. **Want to understand it first?** → Read the 5 sections above (15 min)\n3. **Setting up for the first time?** → Go to [Installation](../1-INSTALLATION/index.md)\n\n" }, { "path": "docs/2-CORE-CONCEPTS/notebooks-sources-notes.md", "content": "# Notebooks, Sources, and Notes - The Container Model\n\nOpen Notebook organizes research in three connected layers. Understanding this hierarchy is key to using the system effectively.\n\n## The Three-Layer Structure\n\n```\n┌─────────────────────────────────────┐\n│ NOTEBOOK (The Container) │\n│ \"My AI Safety Research 2026\" │\n├─────────────────────────────────────┤\n│ │\n│ SOURCES (The Raw Materials) │\n│ ├─ safety_paper.pdf │\n│ ├─ alignment_video.mp4 │\n│ └─ prompt_injection_article.html │\n│ │\n│ NOTES (The Processed Insights) │\n│ ├─ AI Summary (auto-generated) │\n│ ├─ Key Concepts (transformation) │\n│ ├─ My Research Notes (manual) │\n│ └─ Chat Insights (from conversation)\n│ │\n└─────────────────────────────────────┘\n```\n\n---\n\n## 1. NOTEBOOKS - The Research Container\n\n### What Is a Notebook?\n\nA **notebook** is a *scoped container* for a research project or topic. It's your research workspace.\n\nThink of it like a physical notebook: everything inside is about the same topic, shares the same context, and builds toward the same goals.\n\n### What Goes In?\n\n- **A description** — \"This notebook collects research on X topic\"\n- **Sources** — The raw materials you add\n- **Notes** — Your insights and outputs\n- **Conversation history** — Your chats and questions\n\n### Why This Matters\n\n**Isolation**: Each notebook is completely separate. Sources in Notebook A never appear in Notebook B. This lets you:\n- Keep different research topics completely isolated\n- Reuse source names across notebooks without conflicts\n- Control which AI context applies to which research\n\n**Shared Context**: All sources and notes in a notebook inherit the notebook's context. If your notebook is titled \"AI Safety 2026\" with description \"Focusing on alignment and interpretability,\" that context applies to all AI interactions within that notebook.\n\n**Parallel Projects**: You can have 10 notebooks running simultaneously. Each one is its own isolated research environment.\n\n### Example\n\n```\nNotebook: \"Customer Research - Product Launch\"\nDescription: \"User interviews and feedback for Q1 2026 launch\"\n\n→ All sources added to this notebook are about customer feedback\n→ All notes generated are in that context\n→ When you chat, the AI knows you're analyzing product launch feedback\n→ Different from your \"Market Analysis - Competitors\" notebook\n```\n\n---\n\n## 2. SOURCES - The Raw Materials\n\n### What Is a Source?\n\nA **source** is a *single piece of input material* — the raw content you bring in. Sources never change; they're just processed and indexed.\n\n### What Can Be a Source?\n\n- **PDFs** — Research papers, reports, documents\n- **Web links** — Articles, blog posts, web pages\n- **Audio files** — Podcasts, interviews, lectures\n- **Video files** — Tutorials, presentations, recordings\n- **Plain text** — Notes, transcripts, passages\n- **Uploaded text** — Paste content directly\n\n### What Happens When You Add a Source?\n\n```\n1. EXTRACTION\n File/URL → Extract text and metadata\n (OCR for PDFs, web scraping for URLs, speech-to-text for audio)\n\n2. CHUNKING\n Long text → Break into searchable chunks\n (Prevents \"too much context\" in single query)\n\n3. EMBEDDING\n Each chunk → Generate semantic vector\n (Allows AI to find conceptually similar content)\n\n4. STORAGE\n Chunks + vectors → Store in database\n (Ready for search and retrieval)\n```\n\n### Key Properties\n\n**Immutable**: Once added, the source doesn't change. If you need a new version, add it as a new source.\n\n**Indexed**: Sources are automatically indexed for search (both text and semantic).\n\n**Scoped**: A source belongs to exactly one notebook.\n\n**Referenceable**: Other sources and notes can reference this source by citation.\n\n### Example\n\n```\nSource: \"openai_charter.pdf\"\nType: PDF document\n\nWhat happens:\n→ PDF is uploaded\n→ Text is extracted (including images)\n→ Text is split into 50 chunks (paragraphs, sections)\n→ Each chunk gets an embedding vector\n→ Now searchable by: \"OpenAI's approach to safety\"\n```\n\n---\n\n## 3. NOTES - The Processed Insights\n\n### What Is a Note?\n\nA **note** is a *processed output* — something you created or AI created based on your sources. Notes are the \"results\" of your research work.\n\n### Types of Notes\n\n#### Manual Notes\nYou write them yourself. They're your original thinking, capturing:\n- What you learned from sources\n- Your analysis and interpretations\n- Your next steps and questions\n\n#### AI-Generated Notes\nCreated by applying AI processing to sources:\n- **Transformations** — Structured extraction (main points, key concepts, methodology)\n- **Chat Responses** — Answers you saved from conversations\n- **Ask Results** — Comprehensive answers saved to your notebook\n\n#### Captured Insights\nNotes you explicitly saved from interactions:\n- \"Save this response as a note\"\n- \"Save this transformation result\"\n- Convert any AI output into a permanent note\n\n### What Can Notes Contain?\n\n- **Text** — Your writing or AI-generated content\n- **Citations** — References to specific sources\n- **Metadata** — When created, how created (manual/AI), which sources influenced it\n- **Tags** — Your categorization (optional but useful)\n\n### Why Notes Matter\n\n**Knowledge Accumulation**: Notes become your actual knowledge base. They're what you take away from the research.\n\n**Searchable**: Notes are searchable along with sources. \"Find everything about X\" includes your notes, not just sources.\n\n**Citable**: Notes can cite sources, creating an audit trail of where insights came from.\n\n**Shareable**: Notes are your outputs. You can share them, publish them, or build on them in other projects.\n\n---\n\n## How They Connect: The Data Flow\n\n```\nYOU\n │\n ├─→ Create Notebook (\"AI Research\")\n │\n ├─→ Add Sources (papers, articles, videos)\n │ └─→ System: Extract, embed, index\n │\n ├─→ Search Sources (text or semantic)\n │ └─→ System: Find relevant chunks\n │\n ├─→ Apply Transformations (extract insights)\n │ └─→ Creates Notes\n │\n ├─→ Chat with Sources (explore with context control)\n │ ├─→ Can save responses as Notes\n │ └─→ Notes include citations\n │\n ├─→ Ask Questions (automated comprehensive search)\n │ ├─→ Can save results as Notes\n │ └─→ Notes include citations\n │\n └─→ Generate Podcast (transform notebook into audio)\n └─→ Uses all sources + notes for content\n```\n\n---\n\n## Key Design Decisions\n\n### 1. One Notebook Per Source\n\nEach source belongs to exactly one notebook. This creates clear boundaries:\n- No ambiguity about which research project a source is in\n- Easy to isolate or export a complete project\n- Clean permissions model (if someone gets access to notebook, they get access to all its sources)\n\n### 2. Immutable Sources, Mutable Notes\n\nSources never change (once added, always the same). But notes can be edited or deleted. Why?\n- Sources are evidence → evidence shouldn't be altered\n- Notes are your thinking → thinking evolves as you learn\n\n### 3. Explicit Context Control\n\nSources don't automatically go to AI. You decide which sources are \"in context\" for each interaction:\n- Chat: You manually select which sources to include\n- Ask: System automatically figures out which sources to search\n- Transformations: You choose which sources to transform\n\nThis is different from systems that always send everything to AI.\n\n---\n\n## Mental Models Explained\n\n### Notebook as Boundaries\nThink of a notebook like a Git repository:\n- Everything in it is about the same topic\n- You can clone/fork it (copy to new project)\n- It has clear entry/exit points\n- You know exactly what's included\n\n### Sources as Evidence\nThink of sources like exhibits in a legal case:\n- Once filed, they don't change\n- They can be cited and referenced\n- They're the ground truth for what you're basing claims on\n- Multiple sources can be cross-referenced\n\n### Notes as Synthesis\nThink of notes like your case brief:\n- You write them based on evidence\n- They're your interpretation\n- You can cite which evidence supports each claim\n- They're what you actually share or act on\n\n---\n\n## Common Questions\n\n### Can I move a source to a different notebook?\nNot directly. Each source is tied to one notebook. If you want it in multiple notebooks, add it again (uploads are fast if it's already processed).\n\n### Can a note reference sources from a different notebook?\nNo. Notes stay within their notebook and reference sources within that notebook. This keeps boundaries clean.\n\n### What if I want to group sources within a notebook?\nUse tags. You can tag sources (\"primary research,\" \"background,\" \"methodology\") and filter by tags.\n\n### Can I merge two notebooks?\nNot built-in, but you can manually copy sources from one notebook to another by re-uploading them.\n\n---\n\n## Summary\n\n| Concept | Purpose | Lifecycle | Scope |\n|---------|---------|-----------|-------|\n| **Notebook** | Container + context | Create once, configure | All its sources + notes |\n| **Source** | Raw material | Add → Process → Store | One notebook |\n| **Note** | Processed output | Create/capture → Edit → Share | One notebook |\n\nThis three-layer model gives you:\n- **Clear organization** (everything scoped to projects)\n- **Privacy control** (isolated notebooks)\n- **Audit trails** (notes cite sources)\n- **Flexibility** (notes can be manual or AI-generated)\n" }, { "path": "docs/2-CORE-CONCEPTS/podcasts-explained.md", "content": "# Podcasts Explained - Research as Audio Dialogue\n\nPodcasts are Open Notebook's highest-level transformation: converting your research into audio dialogue for a different consumption pattern.\n\n---\n\n## Why Podcasts Matter\n\n### The Problem\nResearch naturally accumulates as text: PDFs, articles, web pages, notes. This creates a friction point:\n\n**To consume research, you must:**\n- Sit down at a desk\n- Focus intently\n- Read actively\n- Take notes\n- Set aside dedicated time\n\n**But much of life is passive time:**\n- Commuting\n- Exercising\n- Doing dishes\n- Driving\n- Walking\n- Idle moments\n\n### The Solution\nConvert your research into audio dialogue so you can consume it passively.\n\n```\nBefore (Text-based):\n Research pile → Must schedule reading time → Requires focus\n\nAfter (Podcast):\n Research pile → Podcast → Can listen while commuting\n → Absorb while exercising\n → Understand while walking\n → Engage without screen time\n```\n\n---\n\n## What Makes It Special: Open Notebook vs. Competitors\n\n### Google Notebook LM Podcasts\n- **Fixed format**: 2 hosts, always conversational\n- **Limited customization**: You can't choose who the \"hosts\" are\n- **One TTS voice per speaker**: Can't customize voices\n- **Only uses cloud services**: No local options\n\n### Open Notebook Podcasts\n- **Customizable format**: 1-4 speakers, you design them\n- **Rich speaker profiles**: Create personas with backstories and expertise\n- **Multiple TTS options**:\n - OpenAI (natural, fast)\n - Google TTS (high quality)\n - ElevenLabs (beautiful voices, accents)\n - Local TTS (privacy-first, no API calls)\n- **Async generation**: Doesn't block your work\n- **Full control**: Choose outline structure, tone, depth\n\n---\n\n## How Podcast Generation Works\n\n### Stage 1: Content Selection\n\nYou choose what goes into the podcast:\n```\nNotebook content → Which sources? → Which notes?\n → Which topics to focus on?\n → Depth of coverage?\n```\n\n### Stage 2: Episode Profile\n\nYou define how you want the podcast structured:\n```\nEpisode Profile\n├─ Topic: \"AI Safety Approaches\"\n├─ Length: 20 minutes\n├─ Tone: Academic but accessible\n├─ Format: Debate (2 speakers with opposing views)\n├─ Audience: Researchers new to the field\n└─ Focus areas: Main approaches, pros/cons, open questions\n```\n\n### Stage 3: Speaker Configuration\n\nYou create speaker personas (1-4 speakers):\n\n```\nSpeaker 1: \"Expert Alex\"\n├─ Expertise: \"Deep knowledge of alignment research\"\n├─ Personality: \"Rigorous, academic, patient with explanation\"\n├─ Accent: (Optional) \"British English\"\n└─ Voice Model: Selected from model registry (e.g., OpenAI TTS)\n └─ Optional per-speaker override of the episode's default voice model\n\nSpeaker 2: \"Researcher Sam\"\n├─ Expertise: \"Field observer, pragmatic perspective\"\n├─ Personality: \"Curious, asks clarifying questions\"\n├─ Accent: \"American English\"\n└─ Voice Model: Selected from model registry (e.g., ElevenLabs TTS)\n```\n\n### Stage 4: Outline Generation\n\nSystem generates episode outline:\n```\nEPISODE: \"AI Safety Approaches\"\n\n1. Introduction (2 min)\n Alex: Introduces topic and speakers\n Sam: What will we cover today?\n\n2. Main Approaches (8 min)\n Alex: Explains top 3 approaches\n Sam: Asks about tradeoffs\n\n3. Debate: Best approach? (6 min)\n Alex: Advocates for approach A\n Sam: Argues for approach B\n\n4. Open Questions (3 min)\n Both: What's unsolved?\n\n5. Conclusion (1 min)\n Recap and where to learn more\n```\n\n### Stage 5: Dialogue Generation\n\nSystem generates dialogue based on outline:\n```\nAlex: \"Today we're exploring three major approaches to AI alignment...\"\n\nSam: \"That's a great start. Can you break down what we mean by alignment?\"\n\nAlex: \"Good question. Alignment means ensuring AI systems pursue the goals\n we actually want them to pursue, not just what we literally asked for.\n There's a classic example of a paperclip maximizer...\"\n\nSam: \"Interesting. So it's about solving the intention problem?\"\n\nAlex: \"Exactly. And that's where the three approaches come in...\"\n```\n\n### Stage 6: Text-to-Speech\n\nSystem converts dialogue to audio using the voice models configured in the model registry. Credentials are automatically resolved from each model's configuration.\n```\nAlex's text → Voice model (from registry) → Alex's voice (audio file)\nSam's text → Voice model (from registry) → Sam's voice (audio file)\nAudio files → Mix together → Final podcast MP3\n```\n\n---\n\n## When Things Go Wrong: Failures & Retry\n\nPodcast generation involves multiple steps (outline, transcript, TTS) and depends on external AI providers. Sometimes things fail.\n\n### What Happens on Failure\n\nWhen podcast generation fails (e.g., wrong model configured, API key expired, provider outage):\n\n- The episode is marked as **Failed** with a red badge\n- The **error message** from the AI provider is displayed so you can understand what went wrong\n- No duplicate episodes are created — automatic retries are disabled to prevent confusion\n\n### How to Retry a Failed Episode\n\n1. Go to the podcast's **Episodes** tab\n2. Find the failed episode — it shows a red \"FAILED\" badge and an error details box\n3. Click the **Retry** button\n4. The failed episode is deleted and a new generation job is submitted\n5. The new episode appears with \"pending\" status\n\n### Common Failure Causes\n\n| Error | What to Do |\n|-------|-----------|\n| Invalid API key | Check Settings -> Credentials for the TTS and language model providers |\n| Model not found | Verify the model exists in the model registry and has valid credentials configured |\n| Rate limit exceeded | Wait a few minutes and retry |\n| Provider unavailable | Check provider status page; retry later |\n\n---\n\n## Key Architecture Decisions\n\n### 1. Asynchronous Processing\nPodcasts are generated in the background. You upload → system processes → you download when ready.\n\n**Why?** Podcast generation takes time (10+ minutes for a 30-minute episode). Blocking would lock up your interface.\n\n### 2. Multi-Speaker Support\nUnlike Google Notebook LM (always 2 hosts), you choose 1-4 speakers.\n\n**Why?** Different discussions work better with different formats:\n- Expert monologue (1 speaker)\n- Interview (2 speakers: host + expert)\n- Debate (2 speakers: opposing views)\n- Panel discussion (3-4 speakers: different expertise)\n\n### 3. Speaker Customization\nYou create rich speaker profiles, not just \"Host A\" and \"Host B\".\n\n**Why?** Makes podcasts more engaging and authentic. Different speakers bring different perspectives.\n\n### 4. Multiple TTS Providers\nYou're not locked into one voice provider.\n\n**Why?**\n- Cost optimization (some providers cheaper)\n- Quality preferences (some voices more natural)\n- Privacy options (local TTS for sensitive content)\n- Accessibility (different accents, genders, styles)\n\n### 5. Local TTS Option\nCan generate podcasts entirely offline with local text-to-speech.\n\n**Why?** For sensitive research, never send audio to external APIs.\n\n---\n\n## Use Cases Show Why This Matters\n\n### Academic Publishing\n```\nTraditional: Academic paper → PDF\nProblem: Hard to consume, linear reading required\n\nOpen Notebook:\nResearch materials → Podcast (expert explaining methodology)\n → Podcast (debate format: different interpretations)\n → Different consumption for different audiences\n```\n\n### Content Creation\n```\nBlog creator: Has research pile on a topic\nProblem: Doesn't have time to write the article\n\nSolution:\nAdd research → Create podcast → Transcribe → Becomes article\nOR: Podcast BECOMES the content (upload to podcast platforms)\n```\n\n### Educational Content\n```\nEducator: Has reading materials for a course\nProblem: Students don't read the papers\n\nSolution:\nCreate podcast with expert explaining papers\nStudents listen → Better engagement → Discussions can reference podcast\n```\n\n### Market Research\n```\nProduct manager: Has interviews with customers\nProblem: Too many hours of audio to review\n\nSolution:\nCreate podcast with debate format (customer perspective vs. team perspective)\nMuch more engaging than raw transcripts\n```\n\n### Knowledge Transfer\n```\nDomain expert: Leaving the organization\nProblem: How to preserve expertise?\n\nSolution:\nCreate expert-mode podcast explaining frameworks, decision-making, context\nNew team member listens, gets context faster than reading 100 documents\n```\n\n---\n\n## The Difference: Active vs. Passive Learning\n\n### Text-Based Research (Active)\n- **Effort**: High (must focus, read, synthesize)\n- **When**: Dedicated study time\n- **Cost**: Time is expensive (can't multitask)\n- **Best for**: Deep dives, precise information\n- **Format**: Whatever you write (notes, articles, books)\n\n### Audio Podcast (Passive)\n- **Effort**: Low (just listen)\n- **When**: Anywhere, anytime\n- **Cost**: Low (can multitask)\n- **Best for**: Overview, context, exploration\n- **Format**: Dialogue (more engaging than narration)\n\n**They complement each other:**\n1. **First encounter**: Listen to podcast (passive, get context)\n2. **Deep dive**: Read source materials (active, precise)\n3. **Mastery**: Both together (understand big picture + details)\n\n---\n\n## How Podcasts Fit Into Your Workflow\n\n```\n1. Build notebook (add sources)\n ↓\n2. Apply transformations (extract insights)\n ↓\n3. Chat/Ask (explore content)\n ↓\n4. Decide on podcast\n ├─→ Create speaker profiles\n ├─→ Define episode profile\n ├─→ Configure voice models (from model registry)\n └─→ Generate podcast\n ↓\n5. Listen while commuting/exercising\n ↓\n6. Reference sources for deep dive\n ↓\n7. Repeat for different formats/speakers/focus\n```\n\n---\n\n## Advanced: Multiple Podcasts from Same Research\n\nYou can create different podcasts from the same sources:\n\n### Example: AI Safety Research\n```\nPodcast 1: \"Expert Monologue\"\n Speaker: Researcher explaining field\n Format: Educational, comprehensive\n Audience: Students new to field\n\nPodcast 2: \"Debate Format\"\n Speakers: Optimist vs. skeptic\n Format: Discussion of tradeoffs\n Audience: Advanced researchers\n\nPodcast 3: \"Interview Format\"\n Speakers: Journalist + expert\n Format: Q&A about practical applications\n Audience: Industry practitioners\n```\n\nEach tells the same story from different angles.\n\n---\n\n## Privacy & Data Considerations\n\n### Where Your Data Goes\n\n**Option 1: Cloud TTS (Faster, Higher Quality)**\n```\nYour outline → API call to TTS provider\n → Audio returned\n → Stored in your notebook\n\nProvider sees: Your outlined script (not raw sources)\nPrivacy level: Medium (outline is shared, sources aren't)\n```\n\n**Option 2: Local TTS (Slower, Maximum Privacy)**\n```\nYour outline → Local TTS engine (runs on your machine)\n → Audio generated locally\n → Stored in your notebook\n\nProvider sees: Nothing\nPrivacy level: Maximum (everything local)\n```\n\n### Recommendation\n- **Sensitive research**: Use local TTS, no API calls\n- **Less sensitive**: Use ElevenLabs or Google (both handle audio data professionally)\n- **Mixed**: Use local TTS for speakers reading sensitive content\n\n---\n\n## Cost Considerations\n\n### Cloud TTS Costs\n| Provider | Cost | Quality | Speed |\n|----------|------|---------|-------|\n| OpenAI | ~$0.015 per minute | Good | Fast |\n| Google | ~$0.004 per minute | Excellent | Fast |\n| ElevenLabs | ~$0.10 per minute | Exceptional | Medium |\n| Local TTS | Free | Basic | Slow |\n\nA 30-minute podcast costs:\n- OpenAI: ~$0.45\n- Google: ~$0.12\n- ElevenLabs: ~$3.00\n- Local: Free (but slow)\n\n---\n\n## Summary: Why Podcasts Are Special\n\n**Podcasts transform your research consumption:**\n\n| Aspect | Text | Podcast |\n|--------|------|---------|\n| **How consumed?** | Active reading | Passive listening |\n| **Where consumed?** | Desk | Anywhere |\n| **Multitasking** | Hard | Easy |\n| **Time commitment** | Scheduled | Flexible |\n| **Format** | Whatever | Natural dialogue |\n| **Engagement** | Academic | Conversational |\n| **Accessibility** | Text-based | Audio-based |\n\n**In Open Notebook specifically:**\n- **Full customization** — you create speakers and format\n- **Privacy options** — local TTS for sensitive content\n- **Cost control** — choose TTS provider based on budget\n- **Non-blocking** — generates in background\n- **Multiple versions** — create different podcasts from same research\n\nThis is why podcasts matter: they change *when* and *how* you can consume your research.\n" }, { "path": "docs/3-USER-GUIDE/adding-sources.md", "content": "# Adding Sources - Getting Content Into Your Notebook\n\nSources are the raw materials of your research. This guide covers how to add different types of content.\n\n---\n\n## Quick-Start: Add Your First Source\n\n### Option 1: Upload a File (PDF, Word, etc.)\n\n```\n1. In your notebook, click \"Add Source\"\n2. Select \"Upload File\"\n3. Choose a file from your computer\n4. Click \"Upload\"\n5. Wait 30-60 seconds for processing\n6. Done! Source appears in your notebook\n```\n\n### Option 2: Add a Web Link\n\n```\n1. Click \"Add Source\"\n2. Select \"Web Link\"\n3. Paste URL: https://example.com/article\n4. Click \"Add\"\n5. Wait for processing (usually faster than files)\n6. Done!\n```\n\n### Option 3: Paste Text\n\n```\n1. Click \"Add Source\"\n2. Select \"Text\"\n3. Paste or type your content\n4. Click \"Save\"\n5. Done! Immediately available\n```\n\n---\n\n## Supported File Types\n\n### Documents\n- **PDF** (.pdf) — Best support, including scanned PDFs with OCR\n- **Word** (.docx, .doc) — Full support\n- **PowerPoint** (.pptx) — Slides converted to text\n- **Excel** (.xlsx, .xls) — Spreadsheet data\n- **EPUB** (.epub) — eBook files\n- **Markdown** (.md, .txt) — Plain text formats\n- **HTML** (.html, .htm) — Web page files\n\n**File size limits:** Up to ~100MB (varies by system)\n\n**Processing time:** 10 seconds - 2 minutes (depending on length and file type)\n\n### Audio & Video\n- **Audio**: MP3, WAV, M4A, OGG, FLAC (~30 seconds - 3 minutes per hour)\n- **Video**: MP4, AVI, MOV, MKV, WebM (~3-10 minutes per hour)\n- **YouTube**: Direct URL support\n- **Podcasts**: RSS feed URL\n\n**Automatic transcription**: Audio/video is transcribed to text automatically. This requires enabling speech-to-text in settings.\n\n### Web Content\n- **Articles**: Blog posts, news articles, Medium\n- **YouTube**: Full videos or playlists\n- **PDFs online**: Direct PDF links\n- **News**: News site articles\n\n**Just paste the URL** in \"Web Link\" section.\n\n### What Doesn't Work\n- Paywalled content (WSJ, FT, etc.) — Can't extract\n- Password-protected PDFs — Can't open\n- Pure image files (.jpg, .png) — Except scanned PDFs which have OCR\n- Very large files (>100MB) — Timeout\n\n---\n\n## What Happens When You Add a Source\n\nThe system automatically does four things:\n\n```\n1. EXTRACT TEXT\n File/URL → Readable text\n (PDFs get OCR if scanned)\n (Videos get transcribed if enabled)\n\n2. BREAK INTO CHUNKS\n Long text → ~500-word pieces\n (So search finds specific parts, not whole document)\n\n3. CREATE EMBEDDINGS\n Each chunk → Vector representation\n (Enables semantic/concept search)\n\n4. INDEX & STORE\n Everything → Database\n (Ready to search and retrieve)\n```\n\n**Time to use:** After the progress bar completes, the source is ready immediately. Embeddings are created in the background.\n\n---\n\n## Step-by-Step for Different Types\n\n### PDFs\n\n**Best practices:**\n```\nClean PDFs:\n 1. Upload → Done\n 2. Processing time: ~30-60 seconds\n\nScanned/Image PDFs:\n 1. Upload same way\n 2. System auto-detects and uses OCR\n 3. Processing time: ~2-3 minutes\n 4. (Higher, due to OCR overhead)\n\nLarge PDFs (50+ pages):\n 1. Consider splitting into smaller files\n 2. Or upload as-is (system handles it)\n 3. Processing time scales with size\n```\n\n**Common issues:**\n- \"Can't extract text\" → PDF is corrupted or has copy protection\n- Solution: Try opening in Adobe. If it won't, the PDF is likely protected.\n\n### Web Links / Articles\n\n**Best practices:**\n```\n1. Copy full URL from browser: https://example.com/article-title\n2. Paste in \"Web Link\"\n3. Click Add\n4. Wait for extraction\n\nProcessing time: Usually 5-15 seconds\n```\n\n**What works:**\n- Standard web articles\n- Blog posts\n- News articles\n- Wikipedia pages\n- Medium posts\n- Substack articles\n\n**What doesn't work:**\n- Twitter threads (unreliable)\n- Paywalled articles (can't access)\n- JavaScript-heavy sites (content not extracted)\n\n**Pro tip:** If it doesn't work, copy the article text and paste as \"Text\" instead.\n\n### Audio Files\n\n**Best practices:**\n```\n1. Ensure speech-to-text is enabled in Settings\n2. Upload MP3, WAV, or M4A file\n3. System automatically transcribes to text\n4. Processing time: ~1 minute per 5 minutes of audio\n\nExample:\n - 1-hour podcast → 12 minutes processing\n - 10-minute recording → 2 minutes processing\n```\n\n**Quality matters:**\n- Clear audio: Fast transcription\n- Muffled/noisy audio: Slower, less accurate transcription\n- Background noise: Try to minimize before uploading\n\n**Tip:** If audio quality is poor, the AI might misinterpret content. You can manually correct transcription if needed.\n\n### YouTube Videos\n\n**Best practices:**\n```\nTwo ways to add:\n\nMethod 1: Direct URL\n 1. Copy YouTube URL: https://www.youtube.com/watch?v=...\n 2. Paste in \"Web Link\"\n 3. Click Add\n 4. System extracts captions (if available) + transcript\n\nMethod 2: Playlist\n 1. Paste playlist URL\n 2. System adds all videos as separate sources\n 3. Each video processed separately\n 4. Takes longer (multiple videos)\n```\n\n**What's extracted:**\n- Captions/subtitles (if available)\n- Transcription (if captions aren't available)\n- Basic metadata (title, channel, length)\n\n**Processing:**\n- 10-minute video: ~2-3 minutes\n- 1-hour video: ~10-15 minutes\n\n### Text / Paste Content\n\n**Best practices:**\n```\n1. Select \"Text\" when adding source\n2. Paste or type content\n3. System processes immediately\n4. No wait time needed\n\nGood for:\n - Notes you want to reference\n - Quotes from books\n - Transcripts you have handy\n - Quick research snippets\n```\n\n---\n\n## Managing Your Sources\n\n### Viewing Source Details\n\n```\nClick on source → See:\n - Original file name/title\n - When it was added\n - Size and format\n - Processing status\n - Number of chunks\n```\n\n### Organizing with Metadata\n\nYou can add to each source:\n- **Title**: Better name than original filename\n- **Tags**: Category labels (\"primary research\", \"background\", \"competitor analysis\")\n- **Description**: A few notes about what it contains\n\n**Why this matters:**\n- Makes sources easier to find\n- Helps when contextualizing for Chat\n- Useful for organizing large notebooks\n\n### Searching Within Sources\n\n```\nAfter sources are added, you can:\n\nText search: \"Find exact phrase\"\nVector search: \"Find conceptually similar\"\n\nBoth search across all sources in notebook.\nResults show:\n - Which source\n - Which section\n - Relevance score\n```\n\n---\n\n## Context Management: How Sources Get Used\n\nYou control how AI accesses sources:\n\n### Three Levels (for Chat)\n\n**Full Content:**\n```\nAI sees: Complete source text\nCost: 100% of tokens\nUse when: Analyzing in detail, need precise citations\nExample: \"Analyze this methodology paper closely\"\n```\n\n**Summary Only:**\n```\nAI sees: AI-generated summary (not full text)\nCost: ~10-20% of tokens\nUse when: Background material, reference context\nExample: \"Use this as context but focus on the main source\"\n```\n\n**Not in Context:**\n```\nAI sees: Nothing (excluded)\nCost: 0 tokens\nUse when: Confidential, not relevant, or archived\nExample: \"Keep this in notebook but don't use in this conversation\"\n```\n\n### How to Set Context (in Chat)\n\n```\n1. Go to Chat\n2. Click \"Select Context Sources\"\n3. For each source:\n - Toggle ON/OFF (include/exclude)\n - Choose level (Full/Summary/Excluded)\n4. Click \"Save\"\n5. Now chat uses these settings\n```\n\n---\n\n## Common Mistakes\n\n| Mistake | What Happens | How to Fix |\n|---------|--------------|-----------|\n| Upload 200 sources at once | System gets slow, processing stalls | Add 10-20 at a time, wait for processing |\n| Use full content for all sources | Token usage skyrockets, expensive | Use \"Summary\" or \"Excluded\" for background material |\n| Add huge PDFs without splitting | Processing is slow, search results less precise | Consider splitting large PDFs into chapters |\n| Forget source titles | Can't distinguish between similar sources | Rename sources with descriptive titles right after uploading |\n| Don't tag sources | Hard to find and organize later | Add tags immediately: \"primary\", \"background\", etc. |\n| Mix languages in one source | Transcription/embedding quality drops | Keep each language in separate sources |\n| Use same source multiple times | Takes up space, creates confusion | Add once; reuse in multiple chats/notebooks |\n\n---\n\n## Processing Status & Troubleshooting\n\n### What the Status Indicators Mean\n\n```\n🟡 Processing\n → Source is being extracted and embedded\n → Wait 30 seconds - 3 minutes depending on size\n → Don't use in Chat yet\n\n🟢 Ready\n → Source is processed and searchable\n → Can use immediately in Chat\n → Can apply transformations\n\n🔴 Error\n → Something went wrong\n → Common reasons:\n - Unsupported file format\n - File too large or corrupted\n - Network timeout\n\n⚪ Not in Context\n → Source added but excluded from Chat\n → Still searchable, not sent to AI\n```\n\n### Common Errors & Solutions\n\n**\"Unsupported file type\"**\n- You tried to upload a format not in the list (e.g., `.webp` image)\n- Solution: Convert to supported format (PDF for documents, MP3 for audio)\n\n**\"Processing timeout\"**\n- Very large file (>100MB) or very long audio\n- Solution: Split into smaller pieces or try uploading again\n\n**\"Transcription failed\"**\n- Audio quality too poor or language not detected\n- Solution: Re-record with better quality, or paste text transcript manually\n\n**\"Web link won't extract\"**\n- Website blocks automated access or uses JavaScript for content\n- Solution: Copy the article text and paste as \"Text\" instead\n\n---\n\n## Tips for Best Results\n\n### For PDFs\n- Clean, digital PDFs work best\n- Remove copy protection if present (legally)\n- Scanned PDFs work but take longer\n\n### For Web Articles\n- Use full URL including domain\n- Avoid cookie/popup-laden sites\n- If extraction fails, copy-paste text instead\n\n### For Audio\n- Clear, well-recorded audio transcribes better\n- Remove background noise if possible\n- YouTube videos usually have good transcriptions built-in\n\n### For Large Documents\n- Consider splitting into smaller sources\n- Gives more precise search results\n- Processing is faster for smaller pieces\n\n### For Organization\n- Name sources clearly (not \"document_2.pdf\")\n- Add tags immediately after uploading\n- Use descriptions for complex documents\n\n---\n\n## What Comes After: Using Your Sources\n\nOnce you've added sources, you can:\n\n- **Chat** → Ask questions (see [Chat Effectively](chat-effectively.md))\n- **Search** → Find specific content (see [Search Effectively](search.md))\n- **Transformations** → Extract structured insights (see [Working with Notes](working-with-notes.md))\n- **Ask** → Get comprehensive answers (see [Search Effectively](search.md))\n- **Podcasts** → Turn into audio (see [Creating Podcasts](creating-podcasts.md))\n\n---\n\n## Summary Checklist\n\nBefore adding sources, confirm:\n\n- [ ] File is in supported format\n- [ ] File is under 100MB (or splitting large ones)\n- [ ] Web links are full URLs (not shortened)\n- [ ] Audio files have clear speech (if transcription-dependent)\n- [ ] You've named source clearly\n- [ ] You've added tags for organization\n- [ ] You understand context levels (Full/Summary/Excluded)\n\nDone! Sources are now ready for Chat, Search, Transformations, and more.\n" }, { "path": "docs/3-USER-GUIDE/api-configuration.md", "content": "# API Configuration\n\nConfigure AI provider credentials through the Settings UI. No file editing required.\n\n> **Credential System**: Open Notebook uses encrypted credentials stored in the database. Each credential connects to a provider and allows you to discover, register, and test models.\n\n---\n\n## Overview\n\nOpen Notebook manages AI provider access through a **credential-based system**:\n\n1. You create a **credential** for each provider (API key + settings)\n2. Credentials are **encrypted** and stored in the database\n3. You **test connections** to verify credentials work\n4. You **discover and register models** from each credential\n5. Models are linked to credentials for direct configuration\n\n---\n\n## Encryption Setup\n\nBefore storing credentials, you must configure an encryption key.\n\n### Setting the Encryption Key\n\nAdd `OPEN_NOTEBOOK_ENCRYPTION_KEY` to your docker-compose.yml:\n\n```yaml\nenvironment:\n - OPEN_NOTEBOOK_ENCRYPTION_KEY=my-secret-passphrase\n```\n\nAny string works as a key — it will be securely derived via SHA-256 internally.\n\n> **Warning**: If you change or lose the encryption key, **all stored credentials become unreadable**. Back up your encryption key securely and separately from your database backups.\n\n### Docker Secrets Support\n\nBoth password and encryption key support Docker secrets:\n\n```yaml\n# docker-compose.yml\nservices:\n open_notebook:\n environment:\n - OPEN_NOTEBOOK_PASSWORD_FILE=/run/secrets/app_password\n - OPEN_NOTEBOOK_ENCRYPTION_KEY_FILE=/run/secrets/encryption_key\n secrets:\n - app_password\n - encryption_key\n\nsecrets:\n app_password:\n file: ./secrets/password.txt\n encryption_key:\n file: ./secrets/encryption_key.txt\n```\n\n### Encryption Details\n\nAPI keys stored in the database are encrypted using Fernet (AES-128-CBC + HMAC-SHA256).\n\n| Configuration | Behavior |\n|---------------|----------|\n| Encryption key set | Keys encrypted with your key |\n| No encryption key set | Storing credentials is disabled |\n\n---\n\n## Accessing Credential Configuration\n\n1. Click **Settings** in the navigation bar\n2. Select **API Keys** tab\n3. You'll see existing credentials and an **Add Credential** button\n\n```\nNavigation: Settings → API Keys\n```\n\n---\n\n## Supported Providers\n\n### Cloud Providers\n\n| Provider | Required Fields | Optional Fields |\n|----------|-----------------|-----------------|\n| OpenAI | API Key | — |\n| Anthropic | API Key | — |\n| Google Gemini | API Key | — |\n| Groq | API Key | — |\n| Mistral | API Key | — |\n| DeepSeek | API Key | — |\n| xAI | API Key | — |\n| OpenRouter | API Key | — |\n| Voyage AI | API Key | — |\n| ElevenLabs | API Key | — |\n\n### Local/Self-Hosted\n\n| Provider | Required Fields | Notes |\n|----------|-----------------|-------|\n| Ollama | Base URL | Typically `http://localhost:11434` or `http://ollama:11434` |\n\n### Enterprise\n\n| Provider | Required Fields | Optional Fields |\n|----------|-----------------|-----------------|\n| Azure OpenAI | API Key, Endpoint, API Version | Service-specific endpoints (LLM, Embedding, STT, TTS) |\n| OpenAI-Compatible | Base URL | API Key, Service-specific configs |\n| Vertex AI | Project ID, Location, Credentials Path | — |\n\n---\n\n## Creating a Credential\n\n### Step 1: Add Credential\n\n1. Go to **Settings** → **API Keys**\n2. Click **Add Credential**\n3. Select your provider\n4. Give it a descriptive name (e.g., \"My OpenAI Key\", \"Work Anthropic\")\n5. Fill in the required fields (API key, base URL, etc.)\n6. Click **Save**\n\n### Step 2: Test Connection\n\n1. On your new credential card, click **Test Connection**\n2. Wait for the result:\n\n| Result | Meaning |\n|--------|---------|\n| Success | Key is valid, provider accessible |\n| Invalid API key | Check key format and value |\n| Connection failed | Check URL, network, firewall |\n\n### Step 3: Discover Models\n\n1. Click **Discover Models** on the credential card\n2. The system queries the provider for available models\n3. Review the discovered models\n\n### Step 4: Register Models\n\n1. Select the models you want to use\n2. Click **Register Models**\n3. The models are now available throughout Open Notebook\n\n---\n\n## Multi-Credential Support\n\nEach provider can have **multiple credentials**. This is useful when:\n- You have different API keys for different projects\n- You want to test with different endpoints\n- Multiple team members need separate credentials\n\n### Creating Multiple Credentials\n\n1. Click **Add Credential** again\n2. Select the same provider\n3. Fill in different credentials\n4. Each credential can discover and register its own models\n\n### How Models Link to Credentials\n\nWhen you register models from a credential, those models are linked to that specific credential. This means:\n- Each model knows which API key to use\n- You can have models from different credentials for the same provider\n- Deleting a credential removes its linked models\n\n---\n\n## Testing Connections\n\nClick **Test Connection** to verify your credential:\n\n| Result | Meaning |\n|--------|---------|\n| Success | Key is valid, provider accessible |\n| Invalid API key | Check key format and value |\n| Connection failed | Check URL, network, firewall |\n| Model not available | Key valid but model access restricted |\n\nTest uses inexpensive models (e.g., `gpt-3.5-turbo`, `claude-3-haiku`) to minimize cost.\n\n---\n\n## Configuring Specific Providers\n\n### Simple Providers (API Key Only)\n\nFor OpenAI, Anthropic, Google, Groq, Mistral, DeepSeek, xAI, OpenRouter:\n\n1. Add credential with your API key\n2. Test connection\n3. Discover and register models\n\n### Ollama (URL-Based)\n\n1. Add credential with provider **Ollama**\n2. Enter the base URL (e.g., `http://ollama:11434`)\n3. Test connection\n4. Discover and register models\n\nOllama allows localhost and private IPs since it runs locally.\n\n### Azure OpenAI\n\nAzure requires multiple fields:\n\n| Field | Example | Required |\n|-------|---------|----------|\n| API Key | `abc123...` | Yes |\n| Endpoint | `https://myresource.openai.azure.com` | Yes |\n| API Version | `2024-02-15-preview` | Yes |\n| LLM Endpoint | `https://myresource-llm.openai.azure.com` | No |\n| Embedding Endpoint | `https://myresource-embed.openai.azure.com` | No |\n\nService-specific endpoints override the main endpoint for that service type.\n\n### OpenAI-Compatible\n\nFor custom OpenAI-compatible servers (LM Studio, vLLM, etc.):\n\n1. Add credential with provider **OpenAI-Compatible**\n2. Enter the base URL\n3. Enter API key (if required)\n4. Optionally configure per-service URLs\n\nSupports separate configurations for:\n- LLM (language models)\n- Embedding\n- STT (speech-to-text)\n- TTS (text-to-speech)\n\n### Vertex AI\n\nGoogle Cloud's enterprise AI platform:\n\n| Field | Example |\n|-------|---------|\n| Project ID | `my-gcp-project` |\n| Location | `us-central1` |\n| Credentials Path | `/path/to/service-account.json` |\n\n---\n\n## Migrating from Environment Variables\n\nIf you have existing API keys in environment variables (from a previous version):\n\n1. Open **Settings → API Keys**\n2. A banner appears: \"Environment variables detected\"\n3. Click **Migrate to Database**\n4. Keys are copied to the database (encrypted)\n5. Original environment variables remain unchanged\n\n### Migration Behavior\n\n| Scenario | Action |\n|----------|--------|\n| Key in env only | Migrated to database |\n| Key in database only | No change |\n| Key in both | Database version kept (skipped) |\n\n### After Migration\n\n- Database credentials are used for all operations\n- You can remove the API key environment variables from your docker-compose.yml\n- Keep `OPEN_NOTEBOOK_ENCRYPTION_KEY` — it's still required\n\n### Migration Banner Visibility\n\nThe migration banner only appears when:\n- You have environment variables configured\n- Those providers are **not** already in the database\n- If all env providers are already migrated, the banner won't show\n\n---\n\n## Migrating from ProviderConfig (v1.1 → v1.2)\n\nIf you're upgrading from an older version that used the ProviderConfig system:\n\n- The migration happens automatically on first startup\n- Your existing configurations are converted to credentials\n- Check **Settings → API Keys** to verify the migration succeeded\n- If you see issues, check the API logs for migration messages\n\n---\n\n## Key Storage Security\n\n### Encryption\n\nAPI keys stored in the database are encrypted using Fernet (AES-128-CBC + HMAC-SHA256).\n\n| Configuration | Behavior |\n|---------------|----------|\n| Encryption key set | Keys encrypted with your key |\n| No encryption key set | Storing API keys in database is disabled |\n\n### Default Credentials\n\n| Setting | Default Value | Production Recommendation |\n|---------|---------------|---------------------------|\n| Password | `open-notebook-change-me` | Set `OPEN_NOTEBOOK_PASSWORD` |\n| Encryption Key | None (must be set) | Set `OPEN_NOTEBOOK_ENCRYPTION_KEY` to any secret string |\n\n**For production deployments, always set custom credentials.**\n\n---\n\n## Deleting Credentials\n\n1. Click the **Delete** button on the credential card\n2. Confirm deletion\n3. Credential and all its linked models are removed from the database\n\n---\n\n## Troubleshooting\n\n### Credential Not Saving\n\n| Symptom | Cause | Solution |\n|---------|-------|----------|\n| Save button disabled | Empty or invalid input | Enter a valid key |\n| Error on save | Encryption key not set | Set `OPEN_NOTEBOOK_ENCRYPTION_KEY` in docker-compose.yml |\n| Error on save | Database connection issue | Check database status |\n\n### Test Connection Fails\n\n| Error | Cause | Solution |\n|-------|-------|----------|\n| Invalid API key | Wrong key or format | Verify key from provider dashboard |\n| Connection refused | Wrong URL | Check base URL format |\n| Timeout | Network issue | Check firewall, proxy settings |\n| 403 Forbidden | IP restriction | Whitelist your server IP |\n\n### Migration Issues\n\n| Problem | Solution |\n|---------|----------|\n| No migration banner | No env vars detected, or already migrated |\n| Partial migration | Check error list, fix and retry |\n| Keys not working after migration | Clear browser cache, restart services |\n\n### Provider Shows \"Not Configured\"\n\n1. Check if a credential exists for this provider (Settings → API Keys)\n2. Test the credential connection\n3. Verify key format matches provider requirements\n4. Re-discover and register models if needed\n\n---\n\n## Provider-Specific Notes\n\n### OpenAI\n- Keys start with `sk-proj-` (project keys) or `sk-` (legacy)\n- Requires billing enabled on account\n\n### Anthropic\n- Keys start with `sk-ant-`\n- Check account has API access enabled\n\n### Google Gemini\n- Keys start with `AIzaSy`\n- Free tier has rate limits\n\n### Ollama\n- No API key required\n- Default URL: `http://localhost:11434` (local) or `http://ollama:11434` (Docker)\n- Ensure Ollama server is running\n\n### Azure OpenAI\n- Endpoint format: `https://{resource-name}.openai.azure.com`\n- API version format: `YYYY-MM-DD` or `YYYY-MM-DD-preview`\n- Deployment names configured separately when registering models via the credential's Discover Models dialog\n\n---\n\n## Related\n\n- **[AI Providers](../5-CONFIGURATION/ai-providers.md)** — Provider setup instructions and recommendations\n- **[Security](../5-CONFIGURATION/security.md)** — Password and encryption configuration\n- **[Environment Reference](../5-CONFIGURATION/environment-reference.md)** — All configuration options\n" }, { "path": "docs/3-USER-GUIDE/chat-effectively.md", "content": "# Chat Effectively - Conversations with Your Research\n\nChat is your main tool for exploratory questions and back-and-forth dialogue. This guide covers how to use it effectively.\n\n---\n\n## Quick-Start: Your First Chat\n\n```\n1. Go to your notebook\n2. Click \"Chat\"\n3. Select which sources to include (context)\n4. Type your question\n5. Click \"Send\"\n6. Read the response\n7. Ask a follow-up (context stays same)\n8. Repeat until satisfied\n```\n\nThat's it! But doing it *well* requires understanding how context works.\n\n---\n\n## Context Management: The Key to Good Chat\n\nContext controls **what the AI is allowed to see**. This is your most important control.\n\n### The Three Levels Explained\n\n**FULL CONTENT**\n- AI sees: Complete source text\n- Cost: 100 tokens per 1K tokens of source\n- Best for: Detailed analysis, precise citations\n- Example: \"Analyze this research paper closely\"\n\n```\nYou set: Paper A → Full Content\nAI sees: Every word of Paper A\nAI can: Cite specific sentences, notice nuances\nResult: Precise, detailed answers (higher cost)\n```\n\n**SUMMARY ONLY**\n- AI sees: AI-generated 200-word summary (not full text)\n- Cost: ~10-20% of full content cost\n- Best for: Background material, reference context\n- Example: \"Use this for background, focus on the main paper\"\n\n```\nYou set: Paper B → Summary Only\nAI sees: Condensed summary, key points\nAI can: Reference main ideas but not details\nResult: Faster, cheaper answers (loses precision)\n```\n\n**NOT IN CONTEXT**\n- AI sees: Nothing\n- Cost: 0 tokens\n- Best for: Confidential, irrelevant, archived content\n- Example: \"Keep this in notebook but don't use now\"\n\n```\nYou set: Paper C → Not in Context\nAI sees: Nothing (completely excluded)\nAI can: Never reference it\nResult: No cost, no privacy risk for that source\n```\n\n### Setting Context (Step by Step)\n\n```\n1. Click \"Select Sources\"\n (Shows list of all sources in notebook)\n\n2. For each source:\n □ Checkbox: Include or exclude\n\n Level dropdown:\n ├─ Full Content\n ├─ Summary Only\n └─ Excluded\n\n3. Check your selections\n Example:\n ✓ Paper A (Full Content) - \"Main focus\"\n ✓ Paper B (Summary Only) - \"Background\"\n ✓ Paper C (Excluded) - \"Keep private\"\n □ Paper D (Not included) - \"Not relevant\"\n\n4. Click \"Save Context\"\n\n5. Now chat uses these settings\n```\n\n### Context Strategies\n\n**Strategy 1: Minimalist**\n- Main source: Full Content\n- Everything else: Excluded\n- Result: Focused, cheap, precise\n\n```\nUse when:\n - Analyzing one source deeply\n - Budget-conscious\n - Want focused answers\n```\n\n**Strategy 2: Comprehensive**\n- All sources: Full Content\n- Result: All context considered, expensive\n\n```\nUse when:\n - Comprehensive analysis\n - Unlimited budget\n - Want AI to see everything\n```\n\n**Strategy 3: Tiered**\n- Primary sources: Full Content\n- Secondary sources: Summary Only\n- Background/reference: Excluded\n- Result: Balanced cost/quality\n\n```\nUse when:\n - Mix of important and reference material\n - Want thorough but not expensive\n - Most common strategy\n```\n\n**Strategy 4: Privacy-First**\n- Sensitive docs: Excluded\n- Public research: Full Content\n- Result: Never send confidential data\n\n```\nUse when:\n - Company confidential materials\n - Personal sensitive data\n - Complying with data protection\n```\n\n---\n\n## Asking Effective Questions\n\n### Good Questions vs. Poor Questions\n\n**Poor Question**\n```\n\"What do you think?\"\n\nProblems:\n- Too vague (about what?)\n- No context (what am I analyzing?)\n- Can't verify answer (citing what?)\n\nResult: Generic, shallow answer\n```\n\n**Good Question**\n```\n\"Based on the paper's methodology section,\nwhat are the three main limitations the authors acknowledge?\nPlease cite which pages mention each one.\"\n\nStrengths:\n- Specific about what you want\n- Clear scope (methodology section)\n- Asks for citations\n- Requires deep reading\n\nResult: Precise, verifiable, useful answer\n```\n\n### Question Patterns That Work\n\n**Factual Questions**\n```\n\"What does the paper say about X?\"\n\"Who are the authors?\"\n\"What year was this published?\"\n\nResult: Simple, factual answers with citations\n```\n\n**Analysis Questions**\n```\n\"How does this approach differ from the traditional method?\"\n\"What are the main assumptions underlying this argument?\"\n\"Why do you think the author chose this methodology?\"\n\nResult: Deeper thinking, comparison, critique\n```\n\n**Synthesis Questions**\n```\n\"How do these two sources approach the problem differently?\"\n\"What's the common theme across all three papers?\"\n\"If we combine these approaches, what would we get?\"\n\nResult: Cross-source insights, connections\n```\n\n**Actionable Questions**\n```\n\"What are the practical implications of this research?\"\n\"How could we apply these findings to our situation?\"\n\"What's the next logical research direction?\"\n\nResult: Practical, forward-looking answers\n```\n\n### The SPECIFIC Formula\n\nGood questions have:\n\n1. **SCOPE** - What are you analyzing?\n \"In this research paper...\"\n \"Looking at these three articles...\"\n \"Based on your experience...\"\n\n2. **SPECIFICITY** - Exactly what do you want?\n \"...the methodology...\"\n \"...main findings...\"\n \"...recommended next steps...\"\n\n3. **CONSTRAINT** - Any limits?\n \"...in 3 bullet points...\"\n \"...with citations to page numbers...\"\n \"...comparing these two approaches...\"\n\n4. **VERIFICATION** - How can you check it?\n \"...with specific quotes...\"\n \"...cite your sources...\"\n \"...link to the relevant section...\"\n\n**Example:**\n```\nPoor: \"What about transformers?\"\nGood: \"In this research paper on machine learning,\n explain the transformer architecture in 2-3 sentences,\n then cite which page describes the attention mechanism.\"\n```\n\n---\n\n## Follow-Up Questions (The Real Power of Chat)\n\nChat's strength is dialogue. You ask, get an answer, ask more.\n\n### Building on Responses\n\n```\nFirst question:\n\"What's the main finding?\"\n\nAI: \"The study shows X [citation]\"\n\nFollow-up question:\n\"How does that compare to Y research?\"\n\nAI: \"The key difference is Z [citation]\"\n\nNext question:\n\"Why do you think that difference matters?\"\n\nAI: \"Because it affects A, B, C [explained]\"\n```\n\n### Iterating Toward Understanding\n\n```\nRound 1: Get overview\n\"What's this source about?\"\n\nRound 2: Get details\n\"What's the most important part?\"\n\nRound 3: Compare\n\"How does it relate to my notes on X?\"\n\nRound 4: Apply\n\"What should I do with this information?\"\n```\n\n### Changing Direction\n\n```\nContext stays same, but you ask new questions:\n\nQuestion 1: \"What's the methodology?\"\nQuestion 2: \"What are the limitations?\"\nQuestion 3: \"What about the ethical implications?\"\nQuestion 4: \"Who else has done similar work?\"\n\nAll in one conversation, reusing context.\n```\n\n### Adjusting Context Between Rounds\n\n```\nAfter question 3, you realize:\n\"I need more context from another source\"\n\n1. Click \"Adjust Context\"\n2. Add new source or change context level\n3. Your conversation history stays\n4. Continue asking with new context\n```\n\n---\n\n## Citations and Verification\n\nCitations are how you verify that the AI's answer is accurate.\n\n### Understanding Citations\n\n```\nAI Response with Citation:\n\"The paper reports a 95% accuracy rate [see page 12]\"\n\nWhat this means:\n✓ The claim \"95% accuracy rate\" is from page 12\n✓ You can verify by reading page 12\n✓ If page 12 doesn't say that, the AI hallucinated\n```\n\n### Requesting Better Citations\n\n```\nIf you get a response without citations:\n\nAsk: \"Please cite the page number for that claim\"\nor: \"Show me where you found that information\"\n\nAI will:\n- Find the citation\n- Provide page numbers\n- Show you the source\n```\n\n### Verification Workflow\n\n```\n1. Get answer from Chat\n2. Check citation (which source? which page?)\n3. Click citation link (if available)\n4. See the actual text in source\n5. Does it really say what AI claimed?\n\nIf YES: Great, you can use this answer\nIf NO: The AI hallucinated, ask for correction\n```\n\n---\n\n## Common Chat Patterns\n\n### Pattern 1: Deep Dive into One Source\n\n```\n1. Set context: One source (Full Content)\n2. Question 1: Overview\n3. Question 2: Main argument\n4. Question 3: Evidence for argument\n5. Question 4: Limitations\n6. Question 5: Next steps\n\nResult: Complete understanding of one source\n```\n\n### Pattern 2: Comparative Analysis\n\n```\n1. Set context: 2-3 sources (all Full Content)\n2. Question 1: What does each source say about X?\n3. Question 2: How do they agree?\n4. Question 3: How do they disagree?\n5. Question 4: Which approach is stronger?\n\nResult: Understanding differences and trade-offs\n```\n\n### Pattern 3: Research Exploration\n\n```\n1. Set context: Many sources (mix of Full/Summary)\n2. Question 1: What are the main perspectives?\n3. Question 2: What's missing from these views?\n4. Question 3: What questions does this raise?\n5. Question 4: What should I research next?\n\nResult: Understanding landscape and gaps\n```\n\n### Pattern 4: Problem Solving\n\n```\n1. Set context: Relevant sources (Full Content)\n2. Question 1: What's the problem?\n3. Question 2: What approaches exist?\n4. Question 3: Pros and cons of each?\n5. Question 4: Which would work best for [my situation]?\n\nResult: Decision-making informed by research\n```\n\n---\n\n## Optimizing for Cost\n\nChat uses tokens for every response. Here's how to use efficiently:\n\n### Reduce Token Usage\n\n**Minimize context**\n```\nOption A: All sources, Full Content\n Cost per response: 5,000 tokens\n\nOption B: Only relevant sources, Summary Only\n Cost per response: 1,000 tokens\n\nSavings: 80% cheaper, same conversation\n```\n\n**Shorter questions**\n```\nVerbose: \"Could you please analyze the methodology\n section of this paper and explain in detail\n what the authors did?\"\n\nConcise: \"Summarize the methodology in 2-3 points.\"\n\nSavings: 20-30% per response\n```\n\n**Use cheaper models**\n```\nGPT-4o: $0.15 per 1M input tokens\nGPT-4o-mini: $0.03 per 1M input tokens\nClaude Sonnet: $0.90 per 1M input tokens\n\nFor chat: Mini/Haiku models are usually fine\nFor deep analysis: Sonnet/Opus worth the cost\n```\n\n### Budget Strategies\n\n**Exploration budget**\n- Use cheap model\n- Broad context (understand landscape)\n- Short questions\n- Result: Low cost, good overview\n\n**Analysis budget**\n- Use powerful model\n- Focused context (main source only)\n- Detailed questions\n- Result: Higher cost, deep insights\n\n**Synthesis budget**\n- Use powerful model for final synthesis\n- Multiple sources (Full Content)\n- Complex comparative questions\n- Result: Expensive but valuable output\n\n---\n\n## Troubleshooting Chat Issues\n\n### Poor Responses\n\n| Problem | Cause | Solution |\n|---------|-------|----------|\n| Generic answers | Vague question | Be specific (see question patterns) |\n| Missing context | Not enough in context | Add sources or change to Full Content |\n| Incorrect info | Source not in context | Add the relevant source |\n| Hallucinating | Model confused | Ask for citations, verify claims |\n| Shallow analysis | Wrong model | Switch to more powerful model |\n\n### High Costs\n\n| Problem | Cause | Solution |\n|---------|-------|----------|\n| Expensive per response | Too much context | Use Summary Only or exclude sources |\n| Many follow-ups | Exploratory chat | Use Ask instead for single comprehensive answer |\n| Long conversations | Keeping history | Archive old chats, start fresh |\n| Large sources | Full text in context | Use Summary Only for large documents |\n\n---\n\n## Best Practices\n\n### Before You Chat\n\n- [ ] Add sources you'll need\n- [ ] Decide context strategy (Tiered is usually best)\n- [ ] Choose model (cheaper for exploration, powerful for analysis)\n- [ ] Have a question in mind\n\n### During Chat\n\n- [ ] Ask specific questions (use SPECIFIC formula)\n- [ ] Check citations for factual claims\n- [ ] Follow up on unclear points\n- [ ] Adjust context if you need different sources\n\n### After Chat\n\n- [ ] Save good responses as notes\n- [ ] Archive conversation if you're done\n- [ ] Organize notes for future reference\n- [ ] Use insights in other features (Ask, Transformations, Podcasts)\n\n---\n\n## When to Use Chat vs. Ask\n\n**Use CHAT when:**\n- You want a dialogue\n- You're exploring a topic\n- You'll ask multiple related questions\n- You want to adjust context during conversation\n- You're not sure exactly what you need\n\n**Use ASK when:**\n- You have one specific question\n- You want a comprehensive answer\n- You want the system to auto-search\n- You want one response, not dialogue\n- You want maximum tokens spent on search\n\n---\n\n## Summary: Chat as Conversation\n\nChat is fundamentally different from asking ChatGPT directly:\n\n| Aspect | ChatGPT | Open Notebook Chat |\n|--------|---------|-------------------|\n| **Source control** | None (uses training) | You control which sources are visible |\n| **Cost control** | Per token | Per token, but context is your choice |\n| **Iteration** | Works | Works, with your sources changing dynamically |\n| **Citations** | Made up often | Tied to your sources (verifiable) |\n| **Privacy** | Your data to OpenAI | Your data stays local (unless you choose) |\n\nThe key insight: **Chat is retrieval-augmented generation.** AI sees only what you put in context. You control the conversation and the information flow.\n\nThat's why Chat is powerful for research. You're not just talking to an AI; you're having a conversation with your research itself.\n" }, { "path": "docs/3-USER-GUIDE/citations.md", "content": "# Citations - Verify and Trust AI Responses\n\nCitations connect AI responses to your source materials. This guide covers how to use and verify them.\n\n---\n\n## Why Citations Matter\n\nEvery AI-generated response in Open Notebook includes citations to your sources. This lets you:\n\n- **Verify claims** - Check that AI actually read what it claims\n- **Find original context** - See the full passage around a quote\n- **Catch hallucinations** - Spot when AI makes things up\n- **Build credibility** - Your notes have traceable sources\n\n---\n\n## Quick Start: Using Citations\n\n### Reading Citations\n\n```\nAI Response:\n\"The study found a 95% accuracy rate [1] using the proposed method.\"\n\n[1] = Click to see source\n\nWhat happens when you click:\n→ Opens the source document\n→ Highlights the relevant section\n→ You can verify the claim\n```\n\n### Requesting Better Citations\n\nIf a response lacks citations, ask:\n\n```\n\"Please cite the specific page or section for that claim.\"\n\"Where in the document does it say that?\"\n\"Can you quote the exact text?\"\n```\n\n---\n\n## How Citations Work\n\n### Automatic Generation\n\nWhen AI references your sources, citations are generated automatically:\n\n```\n1. AI analyzes your question\n2. Retrieves relevant source chunks\n3. Generates response with inline citations\n4. Links citations to original source locations\n```\n\n### Citation Format\n\n```\nInline format:\n\"The researchers concluded X [1] and Y [2].\"\n\nReference list:\n[1] Paper Title - Section 3.2\n[2] Report Name - Page 15\n\nClickable: Each [number] links to the source\n```\n\n---\n\n## Verifying Citations\n\n### The Verification Workflow\n\n```\nStep 1: Read AI response\n \"The model achieved 95% accuracy [1]\"\n\nStep 2: Click citation [1]\n → Opens source document\n → Shows relevant passage\n\nStep 3: Verify the claim\n Does source actually say 95%?\n Is context correct?\n Any nuance missed?\n\nStep 4: Trust or correct\n ✓ Accurate → Use the insight\n ✗ Wrong → Ask AI to correct\n```\n\n### What to Check\n\n| Check | Why |\n|-------|-----|\n| **Exact numbers** | AI sometimes rounds or misremembers |\n| **Context** | Quote might mean something different in context |\n| **Attribution** | Is this the source's claim or someone they cited? |\n| **Completeness** | Did AI miss important caveats? |\n\n---\n\n## Citations in Different Features\n\n### Chat Citations\n\n```\nContext: Sources you selected\nCitations: Reference chunks used in response\nVerification: Click to see original text\nSave: Citations preserved when saving as note\n```\n\n### Ask Feature Citations\n\n```\nContext: Auto-searched across all sources\nCitations: Multiple sources synthesized\nVerification: Each source linked separately\nQuality: Often more comprehensive than Chat\n```\n\n### Transformation Citations\n\n```\nContext: Single source being transformed\nCitations: Points back to original document\nVerification: Compare output to source\nUse: When you need structured extraction\n```\n\n---\n\n## Saving Citations\n\n### In Notes\n\nWhen you save an AI response as a note, citations are preserved:\n\n```\nOriginal response:\n\"According to the paper [1], the method works by...\"\n\nSaved note includes:\n- The text\n- The citation link\n- Reference to source document\n```\n\n### Exporting\n\nCitations work in exports:\n\n| Format | Citation Behavior |\n|--------|-------------------|\n| **Markdown** | Links preserved as `[text](link)` |\n| **Copy/Paste** | Plain text with reference numbers |\n| **PDF** | Clickable references (if supported) |\n\n---\n\n## Citation Quality Tips\n\n### Get Better Citations\n\n**Be specific in questions:**\n```\nPoor: \"What does it say about X?\"\nGood: \"What does page 15 say about X? Please quote directly.\"\n```\n\n**Request citation format:**\n```\n\"Include page numbers for each claim.\"\n\"Cite specific sections, not just document names.\"\n```\n\n**Use Full Content context:**\n```\nSummary Only → Less precise citations\nFull Content → Exact quotes possible\n```\n\n### When Citations Are Missing\n\n| Situation | Cause | Solution |\n|-----------|-------|----------|\n| No citations | AI used general knowledge | Ask: \"Base your answer only on my sources\" |\n| Vague citations | Source not in Full Content | Change context level |\n| Wrong citations | AI confused sources | Ask to verify with quotes |\n\n---\n\n## Common Issues\n\n### \"Citation doesn't match claim\"\n\n```\nProblem: AI says X, but source says Y\n\nWhat happened:\n- AI paraphrased incorrectly\n- AI combined multiple sources confusingly\n- Source was taken out of context\n\nSolution:\n1. Click citation to see original\n2. Note the discrepancy\n3. Ask AI: \"The source says Y, not X. Please correct.\"\n```\n\n### \"Can't find cited section\"\n\n```\nProblem: Citation link doesn't show relevant text\n\nWhat happened:\n- Source was chunked differently than expected\n- Information spread across multiple sections\n- Processing missed some content\n\nSolution:\n1. Search within source for key terms\n2. Ask AI for more specific location\n3. Re-process source if needed\n```\n\n### \"No citations at all\"\n\n```\nProblem: AI response has no source references\n\nWhat happened:\n- Sources not in context\n- Question asked for opinion/general knowledge\n- Model didn't find relevant content\n\nSolution:\n1. Check context settings\n2. Rephrase: \"Based on my sources, what...\"\n3. Add more relevant sources\n```\n\n---\n\n## Best Practices\n\n### For Research Integrity\n\n1. **Always verify important claims** - Don't trust AI blindly\n2. **Check context** - Quotes can be misleading out of context\n3. **Note limitations** - AI might miss nuance\n4. **Keep source access** - Don't delete sources you cite\n\n### For Academic Work\n\n1. **Use Full Content** for documents you'll cite\n2. **Request specific page numbers**\n3. **Cross-check with original sources**\n4. **Document your verification process**\n\n### For Professional Use\n\n1. **Verify before sharing** - Check claims clients will see\n2. **Keep citation trail** - Save notes with sources linked\n3. **Be transparent** - Note when insights are AI-assisted\n\n---\n\n## Summary\n\n```\nCitations = Your verification system\n\nHow to use:\n1. Read AI response\n2. Note citation markers [1], [2], etc.\n3. Click to see original source\n4. Verify claim matches source\n5. Trust verified insights\n\nWhen citations fail:\n- Ask for specific quotes\n- Change to Full Content\n- Request page numbers\n- Verify manually\n\nWhy it matters:\n- AI can hallucinate\n- Context can change meaning\n- Trust requires verification\n- Good research needs sources\n```\n\nCitations aren't just references — they're your quality control. Use them to build research you can trust.\n" }, { "path": "docs/3-USER-GUIDE/creating-podcasts.md", "content": "# Creating Podcasts - Turn Research into Audio\n\nPodcasts let you consume your research passively. This guide covers the complete workflow from setup to download.\n\n---\n\n## Quick-Start: Your First Podcast (5 Minutes)\n\n```\n1. Go to your notebook\n2. Click \"Generate Podcast\"\n3. Select sources to include\n4. Choose a speaker profile (or use default)\n5. Click \"Generate\"\n6. Wait 3-10 minutes (non-blocking)\n7. Download MP3 when ready\n8. Done!\n```\n\nThat's the minimum. Let's make it better.\n\n---\n\n## Step-by-Step: The Complete Workflow\n\n### Step 1: Prepare Your Notebook\n\n```\nBefore generating, make sure:\n\n✓ You have sources added\n (At least 1-2 sources)\n\n✓ Sources have been processed\n (Green \"Ready\" status)\n\n✓ Notes are organized\n (If you want notes included)\n\n✓ You know your message\n (What's the main story?)\n\nTypical preparation: 5-10 minutes\n```\n\n### Step 2: Choose Content\n\n```\nClick \"Generate Podcast\"\n\nYou'll see:\n- List of all sources in notebook\n- List of all notes\n\nSelect which to include:\n☑ Paper A (primary source)\n☑ Paper B (supporting source)\n☐ Old note (not relevant)\n✓ Analysis note (important)\n\nWhat to include:\n- Primary sources: Always include\n- Supporting sources: Usually include\n- Notes: Include your analysis/insights\n- Everything: Can overload podcast\n\nRecommended: 3-5 sources per podcast\n```\n\n### Step 3: Choose Episode Profile\n\nAn episode profile defines the structure and tone.\n\n**Option A: Use Preset Profile**\n\n```\nOpen Notebook provides preset profiles:\n\nAcademic Presentation (Monologue)\n├─ 1 speaker\n├─ Tone: Educational\n└─ Format: Expert explaining topic\n\nExpert Interview (2-speaker)\n├─ 2 speakers: Host + Expert\n├─ Tone: Q&A, conversational\n└─ Format: Interview with expert\n\nDebate Format (2-speaker)\n├─ 2 speakers: Pro vs. Con\n├─ Tone: Discussion, disagreement\n└─ Format: Debate about the topic\n\nPanel Discussion (3-4 speaker)\n├─ 3-4 speakers: Different perspectives\n├─ Tone: Thoughtful discussion\n└─ Format: Each brings different expertise\n\nSolo Explanation (Monologue)\n├─ 1 speaker\n├─ Tone: Conversational, friendly\n└─ Format: Personal explanation\n```\n\n**Pick based on your content:**\n- One main idea → Academic Presentation\n- You want to explain → Solo Explanation\n- Two competing views → Debate Format\n- Multiple perspectives → Panel Discussion\n- Want to explore → Expert Interview\n\n### Step 4: Customize Episode Profile (Optional)\n\nIf presets don't fit, customize:\n\n```\nEpisode Profile\n├─ Title: \"AI Safety in 2026\"\n├─ Description: \"Exploring current approaches\"\n├─ Length target: 20 minutes\n├─ Tone: \"Academic but accessible\"\n├─ Focus areas:\n│ ├─ Main approaches to alignment\n│ ├─ Pros and cons comparison\n│ └─ Open questions\n├─ Audience: \"Researchers new to field\"\n└─ Format: \"Debate between two perspectives\"\n\nHow to set:\n1. Click \"Customize\"\n2. Edit each field\n3. Click \"Save Profile\"\n4. System uses your profile for outline generation\n```\n\n### Step 5: Create or Select Speakers\n\nSpeakers are the \"voice\" of your podcast.\n\n**Option A: Use Preset Speakers**\n\n```\nOpen Notebook provides preset profiles:\n\n\"Expert Alex\"\n- Expertise: Deep knowledge\n- Personality: Rigorous, patient\n- Voice Model: Selected from model registry\n\n\"Curious Sam\"\n- Expertise: Curious newcomer\n- Personality: Asks questions\n- Voice Model: Selected from model registry\n\n\"Skeptic Jordan\"\n- Expertise: Critical perspective\n- Personality: Challenges assumptions\n- Voice Model: Selected from model registry\n\nFor your first podcast: Use presets\nFor custom podcast: Create your own\n```\n\n**Option B: Create Custom Speakers**\n\n```\nClick \"Add Speaker\"\n\nFill in:\n\nName: \"Dr. Research Expert\"\n\nExpertise:\n\"20 years in AI safety research,\n deep knowledge of alignment approaches\"\n\nPersonality:\n\"Rigorous, academic style,\n explains clearly, asks good questions\"\n\nVoice Configuration:\n- Voice Model: Select from model registry (e.g., OpenAI TTS, Google TTS, ElevenLabs)\n- Voice: Choose from available voices for the selected model\n- Per-speaker override: Each speaker can optionally use a different voice model\n\nCredentials are automatically resolved from the model configuration.\n\nExample:\nName: Dr. Research Expert\nExpertise: AI safety alignment research\nPersonality: Rigorous, academic but accessible\nVoice Model: ElevenLabs TTS (from registry), Voice: professional male\n```\n\n### Step 6: Generate Podcast\n\n```\n1. Review your setup:\n Sources: ✓ Selected\n Profile: ✓ Episode profile chosen\n Speakers: ✓ Speakers configured\n\n2. Click \"Generate Podcast\"\n\n3. System begins:\n - Analyzing your content\n - Creating outline\n - Writing dialogue\n - Generating audio\n - Mixing speakers\n\n4. Status shows progress:\n 20% Outline generation\n 40% Dialogue writing\n 60% Audio synthesis\n 80% Mixing\n 100% Complete\n\nProcessing time:\n- 5 minutes of content: 3-5 minutes\n- 15 minutes of content: 5-10 minutes\n- 30 minutes of content: 10-20 minutes\n```\n\n### Step 7: Review and Download\n\n```\nWhen complete:\n\nPreview:\n- Play audio sample\n- Review transcript\n- Check duration\n\nOptions:\n✓ Download as MP3 - Save to computer\n✓ Stream directly - Listen in browser\n✓ Share link - Get shareable URL (if public)\n✓ Regenerate - Try different speakers/profile\n\nDownload:\n1. Click \"Download as MP3\"\n2. Choose quality: 128kbps / 192kbps / 320kbps\n3. Save file: podcast_[notebook]_[date].mp3\n4. Listen!\n```\n\n---\n\n## Understanding What Happens Behind the Scenes\n\n### The Generation Pipeline\n\n```\nStage 1: CONTENT ANALYSIS (1 minute)\n Your sources → What's the main story?\n → Key themes?\n → Debate points?\n\nStage 2: OUTLINE CREATION (2-3 minutes)\n Themes → Episode structure\n → Section breakdown\n → Talking points\n\nStage 3: DIALOGUE WRITING (2-3 minutes)\n Outline → Convert to natural dialogue\n → Add speaker personalities\n → Create flow and transitions\n\nStage 4: AUDIO SYNTHESIS (3-5 minutes per speaker)\n Script + Speaker → Text-to-speech\n → Individual audio files\n → High quality audio\n\nStage 5: MIXING & MASTERING (1-2 minutes)\n Multiple audio → Combine speakers\n → Level audio\n → Add polish\n → Final MP3\n\nTotal: 10-20 minutes for typical podcast\n```\n\n---\n\n## Text-to-Speech Providers\n\nDifferent providers, different qualities.\n\n### OpenAI (Recommended)\n\n```\nVoices: 5 options (Alloy, Echo, Fable, Onyx, Shimmer)\nQuality: Good, natural sounding\nSpeed: Fast\nCost: ~$0.015 per minute\nBest for: General purpose, natural speech\nExample: \"I have to say, the research shows...\"\n```\n\n### Google TTS\n\n```\nVoices: Many options, various accents\nQuality: Excellent, very natural\nSpeed: Fast\nCost: ~$0.004 per minute\nBest for: High quality output, accents\nExample: \"The research demonstrates that...\"\n```\n\n### ElevenLabs\n\n```\nVoices: 100+ voices, highly customizable\nQuality: Exceptional, very expressive\nSpeed: Slower (5-10 seconds per phrase)\nCost: ~$0.10 per minute\nBest for: Premium quality, emotional range\nExample: [Can convey emotion and tone]\n```\n\n### Local TTS (Free)\n\n```\nVoices: Limited, basic options\nQuality: Basic, robotic\nSpeed: Depends on hardware (slow)\nCost: Free (local processing)\nBest for: Privacy, testing, offline use\nExample: \"The research shows...\"\nPrivacy: Everything stays on your computer\n```\n\n### Which Provider to Choose?\n\n```\nFor your first podcast: Google (quality/cost balance)\nFor privacy-sensitive: Local TTS (free, private)\nFor premium quality: ElevenLabs (best voices)\nFor budget: Google (cheapest quality option)\nFor speed: OpenAI (fast generation)\n```\n\n---\n\n## Tips for Better Podcasts\n\n### Choose Right Profile\n\n```\nSingle source analysis → Academic Presentation\n \"Explaining one paper to someone new\"\n\nComparing two approaches → Debate Format\n \"Pros and cons of different methods\"\n\nMultiple sources + insights → Panel Discussion\n \"Different experts discussing topic\"\n\nNarrative exploration → Expert Interview\n \"Host interviewing research expert\"\n\nPersonal take → Solo Explanation\n \"You explaining your analysis\"\n```\n\n### Create Good Speakers\n\n```\nGood Speaker:\n✓ Clear expertise (know what they're talking about)\n✓ Distinct personality (not generic)\n✓ Good voice choice (matches personality)\n✓ Realistic backstory (feels like real person)\n\nBad Speaker:\n✗ Generic expertise (\"good at research\")\n✗ No personality (\"just reads\")\n✗ Mismatched voice (deep voice for young person)\n✗ Contradicts personality (serious person uses casual voice)\n```\n\n### Focus Content\n\n```\nBetter: Podcast on ONE specific topic\n \"How transformers work\" (15 minutes, focused)\n\nWorse: Podcast on everything\n \"All of AI 2025\" (2 hours, unfocused)\n\nGuideline:\n- 5-10 minutes: One narrow topic\n- 15-20 minutes: One broad topic\n- 30+ minutes: Multiple related subtopics\n\nShorter is usually better for podcasts.\n```\n\n### Optimize Source Selection\n\n```\nToo much content:\n \"Here are all 20 papers\"\n → Podcast becomes 2+ hours\n → Unfocused\n → Low quality\n\nRight amount:\n \"Here are 3 key papers\"\n → Podcast is 15-20 minutes\n → Focused\n → High quality\n\nRule: 3-5 sources per podcast\n Remove long background papers\n Keep focused on main topic\n```\n\n---\n\n## Quality Troubleshooting\n\n### Audio Sounds Robotic\n\n**Problem**: TTS voice sounds unnatural\n\n**Solutions**:\n```\n1. Switch provider: Try Google or ElevenLabs instead\n2. Choose different voice: Some voices more natural\n3. Shorter sentences: Very long sentences sound robotic\n4. Adjust pacing: Ask for \"natural, conversational pacing\"\n```\n\n### Audio Sounds Unclear\n\n**Problem**: Hard to understand what's being said\n\n**Solutions**:\n```\n1. Re-generate with different speaker\n2. Try different TTS provider\n3. Use speakers with clear accents\n4. Lower background noise (if any)\n5. Increase speech rate (if too slow)\n```\n\n### Missing Content\n\n**Problem**: Important information isn't in podcast\n\n**Solutions**:\n```\n1. Include that source in content selection\n2. Review generated outline (check before generating)\n3. Regenerate with clearer profile instructions\n4. Try different model (more thorough model)\n```\n\n### Speakers Don't Match\n\n**Problem**: Speakers sound like same person\n\n**Solutions**:\n```\n1. Choose different voice models from the registry for each speaker\n2. Choose very different voice options\n3. Increase personality differences in profile\n4. Try different speaker count (2 vs 3 vs 4)\n```\n\n### Generation Failed\n\n**Problem**: \"Podcast generation failed\"\n\n**Solutions**:\n```\n1. Check internet connection (especially TTS)\n2. Try again (might be temporary issue)\n3. Use local TTS (doesn't need internet)\n4. Reduce source count (less to process)\n5. Contact support if persistent\n```\n\n---\n\n## Advanced: Multiple Podcasts from Same Research\n\nYou can generate different podcasts from one notebook:\n\n```\nPodcast 1: Overview\n Profile: Academic Presentation\n Sources: Papers A, B, C\n Speakers: One expert\n Length: 15 minutes\n\n→ Use for \"What's this about?\" understanding\n\nPodcast 2: Deep Dive\n Profile: Expert Interview\n Sources: Paper A (Full) + B, C (Summary)\n Speakers: Expert + Interviewer\n Length: 30 minutes\n\n→ Use for detailed exploration\n\nPodcast 3: Debate\n Profile: Debate Format\n Sources: Papers A vs B (different approaches)\n Speakers: Pro-A speaker + Pro-B speaker\n Length: 20 minutes\n\n→ Use for comparing approaches\n```\n\nEach tells the same story from different angles.\n\n---\n\n## Exporting and Sharing\n\n### Download MP3\n\n```\n1. Generation complete\n2. Click \"Download\"\n3. Choose quality:\n - 128 kbps: Smallest file, lower quality\n - 192 kbps: Balanced (recommended)\n - 320 kbps: Highest quality, largest file\n4. Save to computer\n5. Use in podcast app, upload to platform, etc.\n```\n\n### Export Transcript\n\n```\n1. Click \"Export Transcript\"\n2. Get full dialogue as text\n3. Useful for:\n - Blog post content\n - Show notes\n - Searchable text version\n - Accessibility\n```\n\n### Share Link\n\n```\nIf podcast is public:\n1. Click \"Share\"\n2. Get shareable link\n3. Others can listen/download\n4. Useful for:\n - Sharing with team\n - Public distribution\n - Embedding on website\n```\n\n### Publish to Podcast Platforms\n\n```\nIf you want to distribute (future feature):\n1. Download MP3\n2. Upload to platform (Spotify, Apple Podcasts, etc.)\n3. Add metadata (title, description, episode notes)\n4. Your research becomes a published podcast!\n```\n\n---\n\n## Best Practices\n\n### Before Generation\n- [ ] Sources are processed and ready\n- [ ] You've chosen content to include\n- [ ] You have a clear episode profile\n- [ ] Speakers are well-defined\n- [ ] Content is focused (3-5 sources max)\n\n### During Generation\n- Don't close the browser (use background processing)\n- Check back in 5-15 minutes\n- Review transcript when complete\n- Listen to sample before downloading\n\n### After Generation\n- [ ] Download MP3 to computer\n- [ ] Save in organized folder\n- [ ] Add metadata (title, description, date)\n- [ ] Test listening in podcast app\n- [ ] Share with colleagues for feedback\n\n---\n\n## Use Cases\n\n### Academic Researcher\n```\nPodcast: Explaining your dissertation\nSpeakers: You + colleague\nContent: Your papers + supporting research\nUse: Share with advisors, test explanations\n```\n\n### Content Creator\n```\nPodcast: Research-to-podcast article\nSpeakers: Narrator + expert\nContent: Articles you've researched\nUse: Transform article into podcast version\n```\n\n### Team Research\n```\nPodcast: Weekly research updates\nSpeakers: Multiple team members\nContent: This week's papers\nUse: Team updates, knowledge sharing\n```\n\n### Learning/Teaching\n```\nPodcast: Teaching material\nSpeakers: Teacher + inquisitive student\nContent: Textbook + examples\nUse: Students learn while commuting\n```\n\n---\n\n## Cost Breakdown Example\n\n### Generate 15-minute podcast with ElevenLabs\n\n```\nGeneration (outline + dialogue):\n No charge (included in service)\n\nText-to-speech:\n 2 speakers × 15 minutes = 30 minutes TTS\n ElevenLabs: $0.10 per minute\n Cost: 30 × $0.10 = $3.00\n\nProcessing:\n Included (no additional cost)\n\nTotal: $3.00 per podcast\n\nCheaper options:\n With Google TTS: ~$0.12\n With OpenAI: ~$0.45\n With Local TTS: ~$0.00\n```\n\n---\n\n## Summary: Podcasts as Research Tool\n\nPodcasts transform how you consume research:\n\n```\nBefore: Reading papers takes time, focus\nAfter: Listen while commuting, exercising, doing chores\n\nBefore: Can't share complex research easily\nAfter: Share audio of your analysis\n\nBefore: Different consumption styles isolated\nAfter: Same research, multiple formats (read/listen)\n```\n\nPodcasts aren't just for entertainment—they're a tool for making research more accessible, shareable, and consumable.\n\nThat's why they're important for Open Notebook.\n" }, { "path": "docs/3-USER-GUIDE/index.md", "content": "# User Guide - How to Use Open Notebook\n\nThis guide covers practical, step-by-step usage of Open Notebook features. You already understand the concepts; now learn how to actually use them.\n\n> **Prerequisite**: Review [2-CORE-CONCEPTS](../2-CORE-CONCEPTS/index.md) first to understand the mental models (notebooks, sources, notes, chat, transformations, podcasts).\n\n---\n\n## Start Here\n\n### [Interface Overview](interface-overview.md)\nLearn the layout before diving in. Understand the three-panel design and where everything is.\n\n---\n\n## Eight Core Features\n\n### 1. [Adding Sources](adding-sources.md)\nHow to bring content into your notebook. Supports PDFs, web links, audio, video, text, and more.\n\n**Quick links:**\n- Upload a PDF or document\n- Add a web link or article\n- Transcribe audio or video\n- Paste text directly\n- Common mistakes + fixes\n\n---\n\n### 2. [Working with Notes](working-with-notes.md)\nCreating, organizing, and using notes (both manual and AI-generated).\n\n**Quick links:**\n- Create a manual note\n- Save AI responses as notes\n- Apply transformations to generate insights\n- Organize with tags and naming\n- Use notes across your notebook\n\n---\n\n### 3. [Chat Effectively](chat-effectively.md)\nHave conversations with AI about your sources. Manage context to control what AI sees.\n\n**Quick links:**\n- Start your first chat\n- Select which sources go in context\n- Ask effective questions\n- Use follow-ups productively\n- Understand citations and verify claims\n\n---\n\n### 4. [Creating Podcasts](creating-podcasts.md)\nConvert your research into audio dialogue for passive consumption.\n\n**Quick links:**\n- Create your first podcast\n- Choose or customize speakers\n- Select TTS provider\n- Generate and download\n- Common audio quality fixes\n\n---\n\n### 5. [Search Effectively](search.md)\nTwo search modes: text-based (keyword) and vector-based (semantic). Know when to use each.\n\n**Quick links:**\n- Text search vs vector search (when to use)\n- Running effective searches\n- Using the Ask feature for comprehensive answers\n- Saving search results as notes\n- Troubleshooting poor results\n\n---\n\n### 6. [Transformations](transformations.md)\nBatch-process sources with predefined templates. Extract the same insights from multiple documents.\n\n**Quick links:**\n- Built-in transformation templates\n- Creating custom transformations\n- Applying to single or multiple sources\n- Managing transformation output\n\n---\n\n### 7. [Citations](citations.md)\nVerify AI claims by tracing them back to source material. Understand the citation system.\n\n**Quick links:**\n- Reading and clicking citations\n- Verifying claims against sources\n- Requesting better citations\n- Saving cited content as notes\n\n---\n\n### 8. [API Configuration](api-configuration.md)\nConfigure AI provider API keys directly through the Settings UI.\n\n**Quick links:**\n- Add API keys without editing files\n- Test provider connections\n- Migrate from environment variables\n- Manage Azure and OpenAI-compatible providers\n- Understand key storage and encryption\n\n---\n\n## Which Feature for Which Task?\n\n```\nTask: \"I want to explore a topic with follow-ups\"\n→ Use: Chat (add sources, select context, have conversation)\n\nTask: \"I want one comprehensive answer\"\n→ Use: Search / Ask (system finds relevant content)\n\nTask: \"I want to extract the same info from many sources\"\n→ Use: Transformations (define template, apply to all)\n\nTask: \"I want summaries of all my sources\"\n→ Use: Transformations (with built-in summary template)\n\nTask: \"I want to share my research in audio form\"\n→ Use: Podcasts (create speakers, generate episode)\n\nTask: \"I want to find that quote I remember\"\n→ Use: Search / Text Search (keyword matching)\n\nTask: \"I'm exploring a concept without knowing exact words\"\n→ Use: Search / Vector Search (semantic similarity)\n\nTask: \"I need to add or change my AI provider API keys\"\n→ Use: Settings / API Keys (configure providers without editing files)\n```\n\n---\n\n## Quick-Start Checklist: First 15 Minutes\n\n**Step 1: Create a Notebook (1 min)**\n- Name: Something descriptive (\"Q1 Market Research\", \"AI Safety Papers\", etc.)\n- Description: 1-2 sentences about what you're researching\n- This is your research container\n\n**Step 2: Add Your First Source (3 min)**\n- Pick one: PDF, web link, or text\n- Follow [Adding Sources](adding-sources.md)\n- Wait for processing (usually 30-60 seconds)\n\n**Step 3: Chat About It (3 min)**\n- Go to Chat\n- Select your source (set context to \"Full Content\")\n- Ask a simple question: \"What are the main points?\"\n- See AI respond with citations\n\n**Step 4: Save Insight as Note (2 min)**\n- Good response? Click \"Save as Note\"\n- Name it something useful (\"Main points from source X\")\n- Now you have a captured insight\n\n**Step 5: Explore More (6 min)**\n- Add another source\n- Chat about both together\n- Ask a question that compares them\n- Follow up with clarifying questions\n\n**Done!** You've used the core workflow: notebook → sources → chat → notes\n\n---\n\n## Common Mistakes to Avoid\n\n| Mistake | Problem | Fix |\n|---------|---------|-----|\n| Adding everything to one notebook | No isolation between projects | Create separate notebooks for different topics |\n| Expecting AI to know your context | Questions get generic answers | Describe your research focus in chat context |\n| Forgetting to cite sources | You can't verify claims | Click citations to check source chunks |\n| Using Chat for one-time questions | Slower than Ask | Use Ask for comprehensive Q&A, Chat for exploration |\n| Adding huge PDFs without chunking | Slow processing, poor search | Break into multiple smaller sources if possible |\n| Using same context for all chats | Expensive, unfocused | Adjust context level for each chat |\n| Ignoring vector search | Only finding exact keywords | Use vector search to explore conceptually |\n\n---\n\n## Next Steps\n\n1. **Follow each guide** in order (sources → notes → chat → podcasts → search)\n2. **Create your first notebook** with real content\n3. **Practice each feature** with your own research\n4. **Return to CORE-CONCEPTS** if you need to understand the \"why\"\n\n---\n\n## Getting Help\n\n- **Feature not working?** → Check the feature's guide (look for \"Troubleshooting\" section)\n- **Error message?** → Check [6-TROUBLESHOOTING](../6-TROUBLESHOOTING/index.md)\n- **Understanding how something works?** → Check [2-CORE-CONCEPTS](../2-CORE-CONCEPTS/index.md)\n- **Setting up for the first time?** → Go back to [1-INSTALLATION](../1-INSTALLATION/index.md)\n- **For developers** → See [7-DEVELOPMENT](../7-DEVELOPMENT/index.md)\n\n---\n\n**Ready to start?** Pick the guide for what you want to do first!\n" }, { "path": "docs/3-USER-GUIDE/interface-overview.md", "content": "# Interface Overview - Finding Your Way Around\n\nOpen Notebook uses a clean three-panel layout. This guide shows you where everything is.\n\n---\n\n## The Main Layout\n\n```\n┌─────────────────────────────────────────────────────────────┐\n│ [Logo] Notebooks Search Podcasts Models Settings │\n├──────────────┬──────────────┬───────────────────────────────┤\n│ │ │ │\n│ SOURCES │ NOTES │ CHAT │\n│ │ │ │\n│ Your docs │ Your │ Talk to AI about │\n│ PDFs, URLs │ insights │ your sources │\n│ Videos │ summaries │ │\n│ │ │ │\n│ [+Add] │ [+Write] │ [Type here...] │\n│ │ │ │\n└──────────────┴──────────────┴───────────────────────────────┘\n```\n\n---\n\n## Navigation Bar\n\nThe top navigation takes you to main sections:\n\n| Icon | Page | What It Does |\n|------|------|--------------|\n| **Notebooks** | Main workspace | Your research projects |\n| **Search** | Ask & Search | Query across all notebooks |\n| **Podcasts** | Audio generation | Manage podcast profiles |\n| **Models** | AI configuration | Set up providers and models |\n| **Settings** | Preferences | App configuration |\n\n---\n\n## Left Panel: Sources\n\nYour research materials live here.\n\n### What You'll See\n\n```\n┌─────────────────────────┐\n│ Sources (5) │\n│ [+ Add Source] │\n├─────────────────────────┤\n│ ┌─────────────────┐ │\n│ │ 📄 Paper.pdf │ │\n│ │ 🟢 Full Content │ │\n│ │ [⋮ Menu] │ │\n│ └─────────────────┘ │\n│ │\n│ ┌─────────────────┐ │\n│ │ 🔗 Article URL │ │\n│ │ 🟡 Summary Only │ │\n│ │ [⋮ Menu] │ │\n│ └─────────────────┘ │\n└─────────────────────────┘\n```\n\n### Source Card Elements\n\n- **Icon** - File type (PDF, URL, video, etc.)\n- **Title** - Document name\n- **Context indicator** - What AI can see:\n - 🟢 Full Content\n - 🟡 Summary Only\n - ⛔ Not in Context\n- **Menu (⋮)** - Edit, transform, delete\n\n### Add Source Button\n\nClick to add:\n- File upload (PDF, DOCX, etc.)\n- Web URL\n- YouTube video\n- Plain text\n\n---\n\n## Middle Panel: Notes\n\nYour insights and AI-generated content.\n\n### What You'll See\n\n```\n┌─────────────────────────┐\n│ Notes (3) │\n│ [+ Write Note] │\n├─────────────────────────┤\n│ ┌─────────────────┐ │\n│ │ 📝 My Analysis │ │\n│ │ Manual note │ │\n│ │ Jan 3, 2026 │ │\n│ └─────────────────┘ │\n│ │\n│ ┌─────────────────┐ │\n│ │ 🤖 Summary │ │\n│ │ From transform │ │\n│ │ Jan 2, 2026 │ │\n│ └─────────────────┘ │\n└─────────────────────────┘\n```\n\n### Note Card Elements\n\n- **Icon** - Note type (manual 📝 or AI 🤖)\n- **Title** - Note name\n- **Origin** - How it was created\n- **Date** - When created\n\n### Write Note Button\n\nClick to:\n- Create manual note\n- Add your own insights\n- Markdown supported\n\n---\n\n## Right Panel: Chat\n\nYour AI conversation space.\n\n### What You'll See\n\n```\n┌───────────────────────────────┐\n│ Chat │\n│ Session: Research Discussion │\n│ [+ New Session] [Sessions ▼] │\n├───────────────────────────────┤\n│ │\n│ You: What's the main │\n│ finding? │\n│ │\n│ AI: Based on the paper [1], │\n│ the main finding is... │\n│ [Save as Note] │\n│ │\n│ You: Tell me more about │\n│ the methodology. │\n│ │\n├───────────────────────────────┤\n│ Context: 3 sources (12K tok) │\n├───────────────────────────────┤\n│ [Type your message...] [↑] │\n└───────────────────────────────┘\n```\n\n### Chat Elements\n\n- **Session selector** - Switch between conversations\n- **Message history** - Your conversation\n- **Save as Note** - Keep good responses\n- **Context indicator** - What AI can see\n- **Input field** - Type your questions\n\n---\n\n## Context Indicators\n\nThese show what AI can access:\n\n### Token Counter\n\n```\nContext: 3 sources (12,450 tokens)\n ↑ ↑\n Sources Approximate cost indicator\n included\n```\n\n### Per-Source Indicators\n\n| Indicator | Meaning | AI Access |\n|-----------|---------|-----------|\n| 🟢 Full Content | Complete text | Everything |\n| 🟡 Summary Only | AI summary | Key points only |\n| ⛔ Not in Context | Excluded | Nothing |\n\nClick any source to change its context level.\n\n---\n\n## Podcasts Tab\n\nInside a notebook, switch to Podcasts:\n\n```\n┌───────────────────────────────┐\n│ [Chat] [Podcasts] │\n├───────────────────────────────┤\n│ Episode Profile: [Select ▼] │\n│ │\n│ Speakers: │\n│ ├─ Host: Alex (voice model) │\n│ └─ Guest: Sam (voice model) │\n│ │\n│ Include: │\n│ ☑ Paper.pdf │\n│ ☑ My Analysis (note) │\n│ ☐ Background article │\n│ │\n│ [Generate Podcast] │\n└───────────────────────────────┘\n```\n\n---\n\n## Settings Page\n\nAccess via navigation bar → Settings:\n\n### Key Sections\n\n| Section | What It Controls |\n|---------|------------------|\n| **Processing** | Document and URL extraction engines |\n| **Embedding** | Auto-embed settings |\n| **Files** | Auto-delete uploads after processing |\n| **YouTube** | Preferred transcript languages |\n\n---\n\n## Models Page\n\nConfigure AI providers:\n\n```\n┌───────────────────────────────────────┐\n│ Models │\n├───────────────────────────────────────┤\n│ Language Models │\n│ ┌─────────────────────────────────┐ │\n│ │ GPT-4o (OpenAI) [Edit] │ │\n│ │ Claude Sonnet (Anthropic) │ │\n│ │ Llama 3.3 (Ollama) [⭐] │ │\n│ └─────────────────────────────────┘ │\n│ [+ Add Model] │\n│ │\n│ Embedding Models │\n│ ┌─────────────────────────────────┐ │\n│ │ text-embedding-3-small [⭐] │ │\n│ └─────────────────────────────────┘ │\n│ │\n│ Text-to-Speech │\n│ ┌─────────────────────────────────┐ │\n│ │ OpenAI TTS [⭐] │ │\n│ │ Google TTS │ │\n│ └─────────────────────────────────┘ │\n└───────────────────────────────────────┘\n```\n\n- **⭐** = Default model for that category\n- **[Edit]** = Modify configuration\n- **[+ Add]** = Add new model\n\n---\n\n## Search Page\n\nQuery across all notebooks:\n\n```\n┌───────────────────────────────────────┐\n│ Search │\n├───────────────────────────────────────┤\n│ [What are you looking for? ] [🔍] │\n│ │\n│ Search type: [Text ▼] [Vector ▼] │\n│ Search in: [Sources] [Notes] │\n├───────────────────────────────────────┤\n│ Results (15) │\n│ │\n│ 📄 Paper.pdf - Notebook: Research │\n│ \"...the transformer model...\" │\n│ │\n│ 📝 My Analysis - Notebook: Research │\n│ \"...key findings include...\" │\n└───────────────────────────────────────┘\n```\n\n---\n\n## Common Actions\n\n### Create a Notebook\n\n```\nNotebooks page → [+ New Notebook] → Enter name → Create\n```\n\n### Add a Source\n\n```\nInside notebook → [+ Add Source] → Choose type → Upload/paste → Wait for processing\n```\n\n### Ask a Question\n\n```\nInside notebook → Chat panel → Type question → Enter → Read response\n```\n\n### Save AI Response\n\n```\nGet good response → Click [Save as Note] → Edit title → Save\n```\n\n### Change Context Level\n\n```\nClick source → Context dropdown → Select level → Changes apply immediately\n```\n\n### Generate Podcast\n\n```\nPodcasts tab → Select profile → Choose sources → [Generate] → Wait → Download\n```\n\n---\n\n## Keyboard Shortcuts\n\n| Key | Action |\n|-----|--------|\n| `Enter` | Send chat message |\n| `Shift + Enter` | New line in chat |\n| `Escape` | Close dialogs |\n| `Ctrl/Cmd + F` | Browser find |\n\n---\n\n## Mobile View\n\nOn smaller screens, the three-panel layout stacks vertically:\n\n```\n┌─────────────────┐\n│ SOURCES │\n│ (tap to expand)\n├─────────────────┤\n│ NOTES │\n│ (tap to expand)\n├─────────────────┤\n│ CHAT │\n│ (always visible)\n└─────────────────┘\n```\n\n- Panels collapse to save space\n- Tap headers to expand/collapse\n- Chat remains accessible\n- Full functionality preserved\n\n---\n\n## Tips for Efficient Navigation\n\n1. **Use keyboard** - Enter sends messages, Escape closes dialogs\n2. **Context first** - Set source context before chatting\n3. **Sessions** - Create new sessions for different topics\n4. **Search globally** - Use Search page to find across all notebooks\n5. **Models page** - Bookmark your preferred models\n\n---\n\nNow you know where everything is. Start with [Adding Sources](adding-sources.md) to begin your research!\n" }, { "path": "docs/3-USER-GUIDE/search.md", "content": "# Search Effectively - Finding What You Need\n\nSearch is your gateway into your research. This guide covers two search modes and when to use each.\n\n---\n\n## Quick-Start: Find Something\n\n### Simple Search\n\n```\n1. Go to your notebook\n2. Type in search box\n3. See results (both sources and notes)\n4. Click result to view source/note\n5. Done!\n\nThat works for basic searches.\nBut you can do much better...\n```\n\n---\n\n## Two Search Modes Explained\n\nOpen Notebook has two fundamentally different search approaches.\n\n### Search Type 1: TEXT SEARCH (Keyword Matching)\n\n**How it works:**\n- You search for words: \"transformer\"\n- System finds chunks containing \"transformer\"\n- Ranked by relevance: frequency, position, context\n\n**Speed:** Very fast (instant)\n\n**When to use:**\n- You remember exact words or phrases\n- You're looking for specific terms\n- You want precise keyword matches\n- You need exact quotes\n\n**Example:**\n```\nSearch: \"attention mechanism\"\nResults:\n 1. \"The attention mechanism allows...\" (perfect match)\n 2. \"Attention and other mechanisms...\" (partial match)\n 3. \"How mechanisms work in attention...\" (includes words separately)\n\nAll contain \"attention\" AND \"mechanism\"\nRanked by how close together they are\n```\n\n**What it finds:**\n- Exact phrases: \"transformer model\"\n- Individual words: transformer OR model (too broad)\n- Names: \"Vaswani et al.\"\n- Numbers: \"1994\", \"GPT-4\"\n- Technical terms: \"LSTM\", \"convolution\"\n\n**What it doesn't find:**\n- Similar words: searching \"attention\" won't find \"focus\"\n- Synonyms: searching \"large\" won't find \"big\"\n- Concepts: searching \"similarity\" won't find \"likeness\"\n\n---\n\n### Search Type 2: VECTOR SEARCH (Semantic/Concept Matching)\n\n**How it works:**\n- Your search converted to embedding (vector)\n- All chunks converted to embeddings\n- System finds most similar embeddings\n- Ranked by semantic similarity\n\n**Speed:** A bit slower (1-2 seconds)\n\n**When to use:**\n- You're exploring a concept\n- You don't know exact words\n- You want semantically similar content\n- You're discovering, not searching\n\n**Example:**\n```\nSearch: \"What's the mechanism for understanding in models?\"\n(Notice: No chunk likely says exactly that)\n\nResults:\n 1. \"Mechanistic interpretability allows understanding...\" (semantic match)\n 2. \"Feature attribution reveals how models work...\" (conceptually similar)\n 3. \"Attention visualization shows model decisions...\" (same topic)\n\nNone contain your exact words\nBut all are semantically related\n```\n\n**What it finds:**\n- Similar concepts: \"understanding\" + \"interpretation\" + \"explainability\" (all related)\n- Paraphrases: \"big\" and \"large\" (same meaning)\n- Related ideas: \"safety\" relates to \"alignment\" (connected concepts)\n- Analogies: content about biological learning when searching \"learning\"\n\n**What it doesn't find:**\n- Exact keywords: if you search a rare word, vector search might miss it\n- Specific numbers: \"1994\" vs \"1993\" are semantically different\n- Technical jargon: \"LSTM\" and \"RNN\" are different even if related\n\n---\n\n## Decision: Text Search vs. Vector Search?\n\n```\nQuestion: \"Do I remember the exact words?\"\n\n→ YES: Use TEXT SEARCH\n Example: \"I remember the paper said 'attention is all you need'\"\n\n→ NO: Use VECTOR SEARCH\n Example: \"I'm looking for content about how models process information\"\n\n→ UNSURE: Try TEXT SEARCH first (faster)\n If no results, try VECTOR SEARCH\n\nText search: \"I know what I'm looking for\"\nVector search: \"I'm exploring an idea\"\n```\n\n---\n\n## Step-by-Step: Using Each Search\n\n### Text Search\n\n```\n1. Go to search box\n2. Type your keywords: \"transformer\", \"attention\", \"2017\"\n3. Press Enter\n4. Results appear (usually instant)\n5. Click result to see context\n\nResults show:\n - Which source contains it\n - How many times it appears\n - Relevance score\n - Preview of surrounding text\n```\n\n### Vector Search\n\n```\n1. Go to search box\n2. Type your concept: \"How do models understand language?\"\n3. Choose \"Vector Search\" from dropdown\n4. Press Enter\n5. Results appear (1-2 seconds)\n6. Click result to see context\n\nResults show:\n - Semantically related chunks\n - Similarity score (higher = more related)\n - Preview of surrounding text\n - Different sources mixed together\n```\n\n---\n\n## The Ask Feature (Automated Search)\n\nAsk is different from simple search. It automatically searches, synthesizes, and answers.\n\n### How Ask Works\n\n```\nStage 1: QUESTION UNDERSTANDING\n \"Compare the approaches in my papers\"\n → System: \"This asks for comparison\"\n\nStage 2: SEARCH STRATEGY\n → System: \"I should search for each approach separately\"\n\nStage 3: PARALLEL SEARCHES\n → Search 1: \"Approach in paper A\"\n → Search 2: \"Approach in paper B\"\n (Multiple searches happen at once)\n\nStage 4: ANALYSIS & SYNTHESIS\n → Per-result analysis: \"Based on paper A, the approach is...\"\n → Per-result analysis: \"Based on paper B, the approach is...\"\n → Final synthesis: \"Comparing A and B: A differs from B in...\"\n\nResult: Comprehensive answer, not just search results\n```\n\n### When to Use Ask vs. Simple Search\n\n| Task | Use | Why |\n|------|-----|-----|\n| \"Find the quote about X\" | **TEXT SEARCH** | Need exact words |\n| \"What does source A say about X?\" | **TEXT SEARCH** | Direct, fast answer |\n| \"Find content about X\" | **VECTOR SEARCH** | Semantic discovery |\n| \"Compare A and B\" | **ASK** | Comprehensive synthesis |\n| \"What's the big picture?\" | **ASK** | Full analysis needed |\n| \"How do these sources relate?\" | **ASK** | Cross-source synthesis |\n| \"I remember something about X\" | **TEXT SEARCH** | Recall memory |\n| \"I'm exploring the topic of X\" | **VECTOR SEARCH** | Discovery mode |\n\n---\n\n## Advanced Search Strategies\n\n### Strategy 1: Simple Search with Follow-Up\n\n```\n1. Text search: \"attention mechanism\"\n Results: 50 matches\n\n2. Too many. Follow up with vector search:\n \"Why is attention useful?\" (concept search)\n Results: Most relevant papers/notes\n\n3. Better results with less noise\n```\n\n### Strategy 2: Ask for Comprehensive, Then Search for Details\n\n```\n1. Ask: \"What are the main approaches to X?\"\n Result: Comprehensive answer about A, B, C\n\n2. Use that to identify specific sources\n\n3. Text search in those specific sources:\n \"Why did they choose method X?\"\n Result: Detailed information\n```\n\n### Strategy 3: Vector Search for Discovery, Text for Verification\n\n```\n1. Vector search: \"How do transformers generalize?\"\n Results: Related conceptual papers\n\n2. Skim to understand landscape\n\n3. Text search in promising sources:\n \"generalization\", \"extrapolation\", \"transfer\"\n Results: Specific passages to read carefully\n```\n\n### Strategy 4: Combine Search with Chat\n\n```\n1. Vector search: \"What's new in AI 2026?\"\n Results: Latest papers\n\n2. Go to Chat\n3. Add those papers to context\n4. Ask detailed follow-up questions\n5. Get deep analysis of results\n```\n\n---\n\n## Search Quality Issues & Fixes\n\n### Getting No Results\n\n| Problem | Cause | Solution |\n|---------|-------|----------|\n| Text search: no results | Word doesn't appear | Try vector search instead |\n| Vector search: no results | Concept not in content | Try broader search term |\n| Both empty | Content not in notebook | Add sources to notebook |\n| | Sources not processed | Wait for processing to complete |\n\n### Getting Too Many Results\n\n| Problem | Cause | Solution |\n|---------|-------|----------|\n| 1000+ results | Search too broad | Be more specific |\n| | All sources | Filter by source |\n| | Keyword matches rare words | Use vector search instead |\n\n### Getting Wrong Results\n\n| Problem | Cause | Solution |\n|---------|-------|----------|\n| Results irrelevant | Search term has multiple meanings | Provide more context |\n| | Using text search for concepts | Try vector search |\n| Different meaning | Homonym (word means multiple things) | Add context (e.g., \"attention mechanism\") |\n\n### Getting Low Quality Results\n\n| Problem | Cause | Solution |\n|---------|-------|----------|\n| Results don't match intent | Vague search term | Be specific (\"Who invented X?\" vs \"X\") |\n| | Concept not well-represented | Add more sources on that topic |\n| | Vector embedding not trained on domain | Use text search as fallback |\n\n---\n\n## Tips for Better Searches\n\n### For Text Search\n1. **Be specific** — \"attention mechanism\" not just \"attention\"\n2. **Use exact phrases** — Put quotes around: \"attention is all you need\"\n3. **Include context** — \"LSTM vs attention\" not just \"attention\"\n4. **Use technical terms** — These are usually more precise\n5. **Try synonyms** — If first search fails, try related terms\n\n### For Vector Search\n1. **Ask a question** — \"What's the best way to X?\" is better than \"best way\"\n2. **Use natural language** — Explain what you're looking for\n3. **Be specific about intent** — \"Compare X and Y\" not \"X and Y\"\n4. **Include context** — \"In machine learning, how...\" vs just \"how...\"\n5. **Think conceptually** — What idea are you exploring?\n\n### General Tips\n1. **Start broad, then narrow** — \"AI papers\" → \"transformers\" → \"attention mechanism\"\n2. **Try both search types** — Each finds different things\n3. **Use Ask for complex questions** — Don't just search\n4. **Save good results as notes** — Create knowledge base\n5. **Filter by source if needed** — \"Search in Paper A only\"\n\n---\n\n## Search Examples\n\n### Example 1: Finding a Specific Fact\n\n**Goal:** \"Find the date the transformer was introduced\"\n\n```\nStep 1: Text search\n \"transformer 2017\" (or year you remember)\n\nIf that works: Done!\n\nIf no results: Try\n \"attention is all you need\" (famous paper title)\n\nCheck result for exact date\n```\n\n### Example 2: Exploring a Concept\n\n**Goal:** \"Find content about alignment interpretability\"\n\n```\nStep 1: Vector search\n \"How do we make AI interpretable?\"\n\nResults: Papers on interpretability, transparency, alignment\n\nStep 2: Review results\n See which papers are most relevant\n\nStep 3: Deep dive\n Go to Chat, add top 2-3 papers\n Ask detailed questions about alignment\n```\n\n### Example 3: Comprehensive Answer\n\n**Goal:** \"How do different approaches to AI safety compare?\"\n\n```\nStep 1: Ask\n \"Compare the main approaches to AI safety in my sources\"\n\nResult: Comprehensive analysis comparing approaches\n\nStep 2: Identify sources\n From answer, see which papers were most relevant\n\nStep 3: Deep dive\n Text search in those papers:\n \"limitations\", \"critiques\", \"open problems\"\n\nStep 4: Save as notes\n Create comparison note from Ask result\n```\n\n### Example 4: Finding Pattern\n\n**Goal:** \"Find all papers mentioning transformers\"\n\n```\nStep 1: Text search\n \"transformer\"\n\nResults: All papers mentioning \"transformer\"\n\nStep 2: Vector search\n \"neural network architecture for sequence processing\"\n\nResults: Papers that don't say \"transformer\" but discuss similar concept\n\nStep 3: Combine\n Union of text + vector results shows full landscape\n\nStep 4: Analyze\n Go to Chat with all results\n Ask: \"What's common across all these?\"\n```\n\n---\n\n## Search in the Workflow\n\nHow search fits with other features:\n\n```\nSOURCES\n ↓\nSEARCH (find what matters)\n ├─ Text search (precise)\n ├─ Vector search (exploration)\n └─ Ask (comprehensive)\n ↓\nCHAT (explore with follow-ups)\n ↓\nTRANSFORMATIONS (batch extract)\n ↓\nNOTES (save insights)\n```\n\n### Workflow Example\n\n```\n1. Add 10 papers to notebook\n\n2. Search: \"What's the state of the art?\"\n (Vector search explores landscape)\n\n3. Ask: \"Compare these 3 approaches\"\n (Comprehensive synthesis)\n\n4. Chat: Deep questions about winner\n (Follow-up exploration)\n\n5. Save best insights as notes\n (Knowledge capture)\n\n6. Transform remaining papers\n (Batch extraction for later)\n\n7. Create podcast from notes + sources\n (Share findings)\n```\n\n---\n\n## Summary: Know Your Search\n\n**TEXT SEARCH** — \"I know what I'm looking for\"\n- Fast, precise, keyword-based\n- Use when you remember exact words/phrases\n- Best for: Finding specific facts, quotes, technical terms\n- Speed: Instant\n\n**VECTOR SEARCH** — \"I'm exploring an idea\"\n- Slow-ish, concept-based, semantic\n- Use when you're discovering connections\n- Best for: Concept exploration, related ideas, synonyms\n- Speed: 1-2 seconds\n\n**ASK** — \"I want a comprehensive answer\"\n- Auto-searches, auto-analyzes, synthesizes\n- Use for complex questions needing multiple sources\n- Best for: Comparisons, big-picture questions, synthesis\n- Speed: 10-30 seconds\n\nPick the right tool for your search goal, and you'll find what you need faster.\n" }, { "path": "docs/3-USER-GUIDE/transformations.md", "content": "# Transformations - Batch Processing Your Sources\n\nTransformations apply the same analysis to multiple sources at once. Instead of asking the same question repeatedly, define a template and run it across your content.\n\n---\n\n## When to Use Transformations\n\n| Use Transformations When | Use Chat Instead When |\n|-------------------------|----------------------|\n| Same analysis on many sources | One-off questions |\n| Consistent output format needed | Exploratory conversation |\n| Batch processing | Follow-up questions needed |\n| Creating structured notes | Context changes between questions |\n\n**Example**: You have 10 papers and want a summary of each. Transformation does it in one operation.\n\n---\n\n## Quick Start: Your First Transformation\n\n```\n1. Go to your notebook\n2. Click \"Transformations\" in navigation\n3. Select a built-in template (e.g., \"Summary\")\n4. Select sources to transform\n5. Click \"Apply\"\n6. Wait for processing\n7. New notes appear automatically\n```\n\n---\n\n## Built-in Transformations\n\nOpen Notebook includes ready-to-use templates:\n\n### Summary\n\n```\nWhat it does: Creates a 200-300 word overview\nOutput: Key points, main arguments, conclusions\nBest for: Quick reference, getting the gist\n```\n\n### Key Concepts\n\n```\nWhat it does: Extracts main ideas and terminology\nOutput: List of concepts with explanations\nBest for: Learning new topics, building vocabulary\n```\n\n### Methodology\n\n```\nWhat it does: Extracts research approach\nOutput: How the study was conducted\nBest for: Academic papers, research review\n```\n\n### Takeaways\n\n```\nWhat it does: Extracts actionable insights\nOutput: What you should do with this information\nBest for: Business documents, practical guides\n```\n\n### Questions\n\n```\nWhat it does: Generates questions the source raises\nOutput: Open questions, gaps, follow-up research\nBest for: Literature review, research planning\n```\n\n---\n\n## Creating Custom Transformations\n\n### Step-by-Step\n\n```\n1. Go to \"Transformations\" page\n2. Click \"Create New\"\n3. Enter a name: \"Academic Paper Analysis\"\n4. Write your prompt template:\n\n \"Analyze this academic paper and extract:\n\n 1. **Research Question**: What problem does this address?\n 2. **Hypothesis**: What did they predict?\n 3. **Methodology**: How did they test it?\n 4. **Key Findings**: What did they discover? (numbered list)\n 5. **Limitations**: What caveats do the authors mention?\n 6. **Future Work**: What do they suggest next?\n\n Be specific and cite page numbers where possible.\"\n\n5. Click \"Save\"\n6. Your transformation appears in the list\n```\n\n### Prompt Template Tips\n\n**Be specific about format:**\n```\nGood: \"List 5 key points as bullet points\"\nBad: \"What are the key points?\"\n```\n\n**Request structure:**\n```\nGood: \"Create sections for: Summary, Methods, Results\"\nBad: \"Tell me about this paper\"\n```\n\n**Ask for citations:**\n```\nGood: \"Cite page numbers for each claim\"\nBad: (no citation request)\n```\n\n**Set length expectations:**\n```\nGood: \"In 200-300 words, summarize...\"\nBad: \"Summarize this\"\n```\n\n---\n\n## Applying Transformations\n\n### To a Single Source\n\n```\n1. In Sources panel, click source menu (⋮)\n2. Select \"Transform\"\n3. Choose transformation template\n4. Click \"Apply\"\n5. Note appears when done\n```\n\n### To Multiple Sources (Batch)\n\n```\n1. Go to Transformations page\n2. Select your template\n3. Check multiple sources\n4. Click \"Apply to Selected\"\n5. Processing runs in parallel\n6. One note per source created\n```\n\n### Processing Time\n\n| Sources | Typical Time |\n|---------|--------------|\n| 1 source | 30 seconds - 1 minute |\n| 5 sources | 2-3 minutes |\n| 10 sources | 4-5 minutes |\n| 20+ sources | 8-10 minutes |\n\nProcessing runs in background. You can continue working.\n\n---\n\n## Transformation Examples\n\n### Literature Review Template\n\n```\nName: Literature Review Entry\n\nPrompt:\n\"For this research paper, create a literature review entry:\n\n**Citation**: [Author(s), Year, Title, Journal]\n**Research Question**: What problem is addressed?\n**Methodology**: What approach was used?\n**Sample**: What population/data was studied?\n**Key Findings**:\n1. [Finding with page citation]\n2. [Finding with page citation]\n3. [Finding with page citation]\n**Strengths**: What did this study do well?\n**Limitations**: What are the gaps?\n**Relevance**: How does this connect to my research?\n\nKeep each section to 2-3 sentences.\"\n```\n\n### Meeting Notes Template\n\n```\nName: Meeting Summary\n\nPrompt:\n\"From this meeting transcript, extract:\n\n**Attendees**: Who was present\n**Date/Time**: When it occurred\n**Key Decisions**: What was decided (numbered)\n**Action Items**:\n- [ ] Task (Owner, Due Date)\n**Open Questions**: Unresolved issues\n**Next Steps**: What happens next\n\nFormat as clear, scannable notes.\"\n```\n\n### Competitor Analysis Template\n\n```\nName: Competitor Analysis\n\nPrompt:\n\"Analyze this company/product document:\n\n**Company**: Name and overview\n**Products/Services**: What they offer\n**Target Market**: Who they serve\n**Pricing**: If available\n**Strengths**: Competitive advantages\n**Weaknesses**: Gaps or limitations\n**Opportunities**: How we compare\n**Threats**: What they do better\n\nBe objective and cite specific details.\"\n```\n\n### Technical Documentation Template\n\n```\nName: API Documentation Summary\n\nPrompt:\n\"Extract from this technical document:\n\n**Overview**: What does this do? (1-2 sentences)\n**Authentication**: How to authenticate\n**Key Endpoints**:\n- Endpoint 1: [method] [path] - [purpose]\n- Endpoint 2: ...\n**Common Parameters**: Frequently used params\n**Rate Limits**: If mentioned\n**Error Codes**: Key error responses\n**Example Usage**: Simple code example if possible\n\nKeep technical but concise.\"\n```\n\n---\n\n## Managing Transformations\n\n### Edit a Transformation\n\n```\n1. Go to Transformations page\n2. Find your template\n3. Click \"Edit\"\n4. Modify the prompt\n5. Click \"Save\"\n```\n\n### Delete a Transformation\n\n```\n1. Go to Transformations page\n2. Find the template\n3. Click \"Delete\"\n4. Confirm\n```\n\n### Reorder/Organize\n\nBuilt-in transformations appear first, then custom ones alphabetically.\n\n---\n\n## Transformation Output\n\n### Where Results Go\n\n- Each source produces one note\n- Notes appear in your notebook's Notes panel\n- Notes are tagged with transformation name\n- Original source is linked\n\n### Note Naming\n\n```\nDefault: \"[Transformation Name] - [Source Title]\"\nExample: \"Summary - Research Paper 2025.pdf\"\n```\n\n### Editing Output\n\n```\n1. Click the generated note\n2. Click \"Edit\"\n3. Refine the content\n4. Save\n```\n\n---\n\n## Best Practices\n\n### Template Design\n\n1. **Start specific** - Vague prompts give vague results\n2. **Use formatting** - Headings, bullets, numbered lists\n3. **Request citations** - Make results verifiable\n4. **Set length** - Prevent overly long or short output\n5. **Test first** - Run on one source before batch\n\n### Source Selection\n\n1. **Similar content** - Same transformation on similar sources\n2. **Reasonable size** - Very long sources may need splitting\n3. **Processed status** - Ensure sources are fully processed\n\n### Quality Control\n\n1. **Review samples** - Check first few outputs before trusting batch\n2. **Edit as needed** - Transformations are starting points\n3. **Iterate prompts** - Refine based on results\n\n---\n\n## Common Issues\n\n### Generic Output\n\n**Problem**: Results are too vague\n**Solution**: Make prompt more specific, add format requirements\n\n### Missing Information\n\n**Problem**: Key details not extracted\n**Solution**: Explicitly ask for what you need in prompt\n\n### Inconsistent Format\n\n**Problem**: Each note looks different\n**Solution**: Add clear formatting instructions to prompt\n\n### Too Long/Short\n\n**Problem**: Output doesn't match expectations\n**Solution**: Specify word count or section lengths\n\n### Processing Fails\n\n**Problem**: Transformation doesn't complete\n**Solution**:\n- Check source is processed\n- Try shorter/simpler prompt\n- Process sources individually\n\n---\n\n## Transformations vs. Chat vs. Ask\n\n| Feature | Transformations | Chat | Ask |\n|---------|----------------|------|-----|\n| **Input** | Predefined template | Your questions | Your question |\n| **Scope** | One source at a time | Selected sources | Auto-searched |\n| **Output** | Structured note | Conversation | Comprehensive answer |\n| **Best for** | Batch processing | Exploration | One-shot answers |\n| **Follow-up** | Run again | Ask more | New query |\n\n---\n\n## Summary\n\n```\nTransformations = Batch AI Processing\n\nHow to use:\n1. Define template (or use built-in)\n2. Select sources\n3. Apply transformation\n4. Get structured notes\n\nWhen to use:\n- Same analysis on many sources\n- Consistent output needed\n- Building structured knowledge base\n- Saving time on repetitive tasks\n\nTips:\n- Be specific in prompts\n- Request formatting\n- Test before batch\n- Edit output as needed\n```\n\nTransformations turn repetitive analysis into one-click operations. Define once, apply many times.\n" }, { "path": "docs/3-USER-GUIDE/working-with-notes.md", "content": "# Working with Notes - Capturing and Organizing Insights\n\nNotes are your processed knowledge. This guide covers how to create, organize, and use them effectively.\n\n---\n\n## What Are Notes?\n\nNotes are your **research output** — the insights you capture from analyzing sources. They can be:\n\n- **Manual** — You write them yourself\n- **AI-Generated** — From Chat responses, Ask results, or Transformations\n- **Hybrid** — AI insight + your edits and additions\n\nUnlike sources (which never change), notes are mutable — you edit, refine, and organize them.\n\n---\n\n## Quick-Start: Create Your First Note\n\n### Method 1: Manual Note (Write Yourself)\n\n```\n1. In your notebook, go to \"Notes\" section\n2. Click \"Create New Note\"\n3. Give it a title: \"Key insights from source X\"\n4. Write your content (markdown supported)\n5. Click \"Save\"\n6. Done! Note appears in your notebook\n```\n\n### Method 2: Save from Chat\n\n```\n1. Have a Chat conversation\n2. Get a good response from AI\n3. Click \"Save as Note\" button under response\n4. Give the note a title\n5. Add any additional context\n6. Click \"Save\"\n7. Done! Note appears in your notebook\n```\n\n### Method 3: Apply Transformation\n\n```\n1. Go to \"Transformations\"\n2. Select a template (or create custom)\n3. Click \"Apply to sources\"\n4. Select which sources to transform\n5. Wait for processing\n6. New notes automatically appear\n7. Done! Each source produces one note\n```\n\n---\n\n## Creating Manual Notes\n\n### Basic Structure\n\n```\nTitle: \"What you're capturing\"\n (Make it descriptive)\n\nContent:\n - Main points\n - Your analysis\n - Questions raised\n - Next steps\n\nMetadata:\n - Tags: How to categorize\n - Related sources: Which documents influenced this\n - Date: Auto-added when created\n```\n\n### Markdown Support\n\nYou can format notes with markdown:\n\n```markdown\n# Heading\n## Subheading\n### Sub-subheading\n\n**Bold text** for emphasis\n*Italic text* for secondary emphasis\n\n- Bullet lists\n- Like this\n\n1. Numbered lists\n2. Like this\n\n> Quotes and important callouts\n\n[Links work](https://example.com)\n```\n\n### Example Note Structure\n\n```markdown\n# Key Findings from \"AI Safety Paper 2025\"\n\n## Main Argument\nThe paper argues that X approach is better than Y because...\n\n## Methodology\nThe authors use [methodology] to test this hypothesis.\n\n## Key Results\n- Result 1: [specific finding with citation]\n- Result 2: [specific finding with citation]\n- Result 3: [specific finding with citation]\n\n## Gaps & Limitations\n1. The paper assumes X, which might not hold in Y scenario\n2. Limited to Z population/domain\n3. Future work needed on A, B, C\n\n## My Thoughts\n- This connects to previous research on...\n- Potential application in...\n\n## Next Steps\n- [ ] Read the referenced paper on X\n- [ ] Find similar studies on Y\n- [ ] Discuss implications with team\n```\n\n---\n\n## AI-Generated Notes: Three Sources\n\n### 1. Save from Chat\n\n```\nWorkflow:\n Chat → Good response → \"Save as Note\"\n → Edit if needed → Save\n\nWhen to use:\n - AI response answers your question well\n - You want to keep the answer for reference\n - You're building a knowledge base from conversations\n\nQuality:\n - Quality = quality of your Chat question\n - Better context = better responses = better notes\n - Ask specific questions for useful notes\n```\n\n### 2. Save from Ask\n\n```\nWorkflow:\n Ask → Comprehensive answer → \"Save as Note\"\n → Edit if needed → Save\n\nWhen to use:\n - You need a one-time comprehensive answer\n - You want to save the synthesized result\n - Building a knowledge base of comprehensive answers\n\nQuality:\n - System automatically found relevant sources\n - Results already have citations\n - Often higher quality than Chat (more thorough)\n```\n\n### 3. Transformations (Batch Processing)\n\n```\nWorkflow:\n Define transformation → Apply to sources → Notes auto-created\n → Review & edit → Organize\n\nExample Transformation:\n Template: \"Extract: main argument, methodology, key findings\"\n Apply to: 5 sources\n Result: 5 new notes with consistent structure\n\nWhen to use:\n - Same extraction from many sources\n - Building structured knowledge base\n - Creating consistent summaries\n```\n\n---\n\n## Using Transformations for Batch Insights\n\n### Built-in Transformations\n\nOpen Notebook comes with presets:\n\n**Summary**\n```\nExtracts: Main points, key arguments, conclusions\nOutput: 200-300 word summary of source\nBest for: Quick reference summaries\n```\n\n**Key Concepts**\n```\nExtracts: Main ideas, concepts, terminology\nOutput: List of concepts with explanations\nBest for: Learning and terminology\n```\n\n**Methodology**\n```\nExtracts: Research approach, methods, data\nOutput: How the research was conducted\nBest for: Academic sources, methodology review\n```\n\n**Takeaways**\n```\nExtracts: Actionable insights, recommendations\nOutput: What you should do with this information\nBest for: Practical/business sources\n```\n\n### How to Apply Transformation\n\n```\n1. Go to \"Transformations\"\n2. Select a template\n3. Click \"Apply\"\n4. Select which sources (one or many)\n5. Wait for processing (usually 30 seconds - 2 minutes)\n6. New notes appear in your notebook\n7. Edit if needed\n```\n\n### Create Custom Transformation\n\n```\n1. Click \"Create Custom Transformation\"\n2. Write your extraction template:\n\n Example:\n \"For this academic paper, extract:\n - Central research question\n - Hypothesis tested\n - Methodology used\n - Key findings (numbered)\n - Limitations acknowledged\n - Recommendations for future work\"\n\n3. Click \"Save Template\"\n4. Apply to one or many sources\n5. System generates notes with consistent structure\n```\n\n---\n\n## Organizing Notes\n\n### Naming Conventions\n\n**Option 1: Date-based**\n```\n2026-01-03 - Key points from X source\n2026-01-04 - Comparison between A and B\nBenefit: Easy to see what you did when\n```\n\n**Option 2: Topic-based**\n```\nAI Safety - Alignment approaches\nAI Safety - Interpretability research\nBenefit: Groups by subject matter\n```\n\n**Option 3: Type-based**\n```\nSUMMARY: Paper on X\nQUESTION: What about Y?\nINSIGHT: Connection between Z and W\nBenefit: Easy to filter by type\n```\n\n**Option 4: Source-based**\n```\nFrom: Paper A - Main insights\nFrom: Video B - Interesting implications\nBenefit: Easy to trace back to sources\n```\n\n**Best practice:** Combine approaches\n```\n[Date] [Source] - [Topic] - [Type]\n2026-01-03 - Paper A - AI Safety - Takeaways\n```\n\n### Using Tags\n\nTags are labels for categorization. Add them when creating notes:\n\n```\nExample tags:\n - \"primary-research\" (direct source analysis)\n - \"background\" (supporting material)\n - \"methodology\" (about research methods)\n - \"insights\" (your original thinking)\n - \"questions\" (open questions raised)\n - \"follow-up\" (needs more work)\n - \"published\" (ready to share/use)\n```\n\n**Benefits of tags:**\n- Filter notes by tag\n- Find all notes of a type\n- Organize workflow (e.g., find all \"follow-up\" notes)\n\n### Note Linking & References\n\nYou can reference sources within notes:\n\n```markdown\n# Analysis of Paper A\n\nAs shown in Paper A (see \"main argument\" section),\nthe authors argue that...\n\n## Related Sources\n- Paper B discusses similar approach\n- Video C shows practical application\n- My note on \"Comparative analysis\" has more\n```\n\n---\n\n## Editing and Refining Notes\n\n### Improving AI-Generated Notes\n\n```\nAI Note:\n \"The paper discusses machine learning\"\n\nWhat you might change:\n \"The paper proposes a supervised learning approach\n to classification problems, using neural networks\n with attention mechanisms (see pp. 15-18).\"\n\nHow to edit:\n 1. Click note\n 2. Click \"Edit\"\n 3. Refine the content\n 4. Click \"Save\"\n```\n\n### Adding Citations\n\n```\nWhen saving from Chat/Ask:\n - Citations auto-added\n - Shows which sources informed answer\n - You can verify by clicking\n\nWhen manual notes:\n - Add manually: \"From Paper A, page 15: ...\"\n - Or reference: \"As discussed in [source]\"\n```\n\n---\n\n## Searching Your Notes\n\nNotes are fully searchable:\n\n### Text Search\n```\nFind exact phrase: \"attention mechanism\"\nResults: All notes containing that phrase\nUse when: Looking for specific terms or quotes\n```\n\n### Vector/Semantic Search\n```\nFind concept: \"How do models understand?\"\nResults: Notes about interpretability, mechanistic understanding, etc.\nUse when: Exploring conceptually (words not exact)\n```\n\n### Combined Search\n```\nText search notes → Find keyword matches\nVector search notes → Find conceptual matches\nBoth work across sources + notes together\n```\n\n---\n\n## Exporting and Sharing Notes\n\n### Options\n\n**Copy to clipboard**\n```\nClick \"Share\" → \"Copy\" → Paste anywhere\nGood for: Sharing one note via email/chat\n```\n\n**Export as Markdown**\n```\nClick \"Share\" → \"Export as MD\" → Saves as .md file\nGood for: Sharing with others, version control\n```\n\n**Create note collection**\n```\nSelect multiple notes → \"Export collection\"\n→ Creates organized markdown document\nGood for: Sharing a topic overview\n```\n\n**Publish to web**\n```\nClick \"Publish\" → Get shareable link\nGood for: Publishing publicly (if desired)\n```\n\n---\n\n## Organizing Your Notebook's Notes\n\n### By Research Phase\n\n**Phase 1: Discovery**\n- Initial summaries\n- Questions raised\n- Interesting findings\n\n**Phase 2: Deep Dive**\n- Detailed analysis\n- Comparative insights\n- Methodology reviews\n\n**Phase 3: Synthesis**\n- Connections across sources\n- Original thinking\n- Conclusions\n\n### By Content Type\n\n**Summaries**\n- High-level overviews\n- Generated by transformations\n- Quick reference\n\n**Questions**\n- Open questions\n- Things to research more\n- Gaps to fill\n\n**Insights**\n- Your original analysis\n- Connections made\n- Conclusions reached\n\n**Tasks**\n- Follow-up research\n- Sources to add\n- People to contact\n\n---\n\n## Using Notes in Other Features\n\n### In Chat\n\n```\nYou can reference notes:\n\"Based on my note 'Key findings from A',\nhow does this compare to B?\"\n\nNotes become part of context.\nTreated like sources but smaller/more focused.\n```\n\n### In Transformations\n\n```\nNotes can be transformed:\n1. Select notes as input\n2. Apply transformation\n3. Get new derived notes\n\nExample: Transform 5 analysis notes → Create synthesis\n```\n\n### In Podcasts\n\n```\nNotes are used to create podcast content:\n1. Generate podcast for notebook\n2. System includes notes in content selection\n3. Notes become part of episode outline\n```\n\n---\n\n## Best Practices\n\n### For Manual Notes\n1. **Write clearly** — Future you will appreciate it\n2. **Add context** — Why this matters, not just what it says\n3. **Link to sources** — You can verify later\n4. **Date them** — Track your thinking over time\n5. **Tag immediately** — Don't defer organization\n\n### For AI-Generated Notes\n1. **Review before saving** — Verify quality\n2. **Edit for clarity** — AI might miss nuance\n3. **Add your thoughts** — Make it your own\n4. **Include citations** — Understand sources\n5. **Organize right away** — While context is fresh\n\n### For Organization\n1. **Consistent naming** — Your future self will thank you\n2. **Tag everything** — Makes filtering later much easier\n3. **Link related notes** — Create knowledge network\n4. **Review periodically** — Refactor as understanding evolves\n5. **Archive old notes** — Keep working space clean\n\n---\n\n## Common Mistakes\n\n| Mistake | Problem | Solution |\n|---------|---------|----------|\n| Save every Chat response | Notebook becomes cluttered with low-quality notes | Only save good responses that answer your questions |\n| Don't add tags | Can't find notes later | Tag immediately when creating |\n| Poor note titles | Can't remember what's in them | Use descriptive titles, include key concept |\n| Never link notes together | Miss connections between ideas | Add references to related notes |\n| Forget the source | Can't verify claims later | Always link back to source |\n| Never edit AI notes | Keep generic AI responses | Refine for clarity and context |\n| Create one giant note | Too long to be useful | Split into focused notes by subtopic |\n\n---\n\n## Summary: Note Lifecycle\n\n```\n1. CREATE\n ├─ Manual: Write from scratch\n ├─ From Chat: Save good response\n ├─ From Ask: Save synthesis\n └─ From Transform: Batch process\n\n2. EDIT & REFINE\n ├─ Improve clarity\n ├─ Add context\n ├─ Fix AI mistakes\n └─ Add citations\n\n3. ORGANIZE\n ├─ Name clearly\n ├─ Add tags\n ├─ Link related\n └─ Categorize\n\n4. USE\n ├─ Reference in Chat\n ├─ Transform for synthesis\n ├─ Export for sharing\n └─ Build on with new questions\n\n5. MAINTAIN\n ├─ Periodically review\n ├─ Update as understanding grows\n ├─ Archive when done\n └─ Learn from organized knowledge\n```\n\nYour notes become your actual knowledge base. The more you invest in organizing them, the more valuable they become.\n" }, { "path": "docs/4-AI-PROVIDERS/index.md", "content": "# AI Providers - Comparison & Selection Guide\n\nOpen Notebook supports 15+ AI providers. This guide helps you **choose the right provider** for your needs.\n\n> 💡 **Just want to set up a provider?** Skip to the [Configuration Guide](../5-CONFIGURATION/ai-providers.md) for detailed setup instructions.\n\n---\n\n## Quick Decision: Which Provider?\n\n### Cloud Providers (Easiest)\n\n**OpenAI (Recommended)**\n- Cost: ~$0.03-0.15 per 1K tokens\n- Speed: Very fast\n- Quality: Excellent\n- Best for: Most users (best quality/price balance)\n\n→ [Setup Guide](../5-CONFIGURATION/ai-providers.md#openai)\n\n**Anthropic (Claude)**\n- Cost: ~$0.80-3.00 per 1M tokens\n- Speed: Fast\n- Quality: Excellent\n- Best for: Long context (200K tokens), reasoning, latest AI\n- Advantage: Superior long-context handling\n\n→ [Setup Guide](../5-CONFIGURATION/ai-providers.md#anthropic-claude)\n\n**Google Gemini**\n- Cost: ~$0.075-0.30 per 1K tokens\n- Speed: Very fast\n- Quality: Good to excellent\n- Best for: Multimodal (images, audio, video)\n- Advantage: Longest context (up to 2M tokens)\n\n→ [Setup Guide](../5-CONFIGURATION/ai-providers.md#google-gemini)\n\n**Groq (Ultra-Fast)**\n- Cost: ~$0.05 per 1M tokens (cheapest)\n- Speed: Ultra-fast (fastest available)\n- Quality: Good\n- Best for: Budget-conscious, transformations, speed-critical tasks\n- Disadvantage: Limited model selection\n\n→ [Setup Guide](../5-CONFIGURATION/ai-providers.md#groq)\n\n**OpenRouter (100+ Models)**\n- Cost: Pay-per-model (varies widely)\n- Speed: Varies by model\n- Quality: Varies by model\n- Best for: Model comparison, testing, unified billing\n- Advantage: One API key for 100+ models from different providers\n\n→ [Setup Guide](../5-CONFIGURATION/ai-providers.md#openrouter)\n\n### Local / Self-Hosted (Free)\n\n**Ollama (Recommended for Local)**\n- Cost: Free (electricity only)\n- Speed: Depends on hardware (slow on CPU, fast on GPU)\n- Quality: Good (open-source models)\n- Setup: 10 minutes\n- Best for: Privacy-first, offline use\n- Privacy: 100% local, nothing leaves your machine\n\n→ [Setup Guide](../5-CONFIGURATION/ai-providers.md#ollama-recommended-for-local)\n\n**LM Studio (Alternative)**\n- Cost: Free (electricity only)\n- Speed: Depends on hardware\n- Quality: Good (same models as Ollama)\n- Setup: 15 minutes (GUI interface)\n- Best for: Non-technical users who prefer GUI over CLI\n- Privacy: 100% local\n\n→ [Setup Guide](../5-CONFIGURATION/ai-providers.md#lm-studio-local-alternative)\n\n### Enterprise\n\n**Azure OpenAI**\n- Cost: Same as OpenAI (usage-based)\n- Speed: Very fast\n- Quality: Excellent (same models as OpenAI)\n- Setup: 10 minutes (more complex)\n- Best for: Enterprise, compliance (HIPAA, SOC2), VPC integration\n\n→ [Setup Guide](../5-CONFIGURATION/ai-providers.md#azure-openai)\n\n---\n\n## Comparison Table\n\n| Provider | Speed | Cost | Quality | Privacy | Setup | Context |\n|----------|-------|------|---------|---------|-------|---------|\n| **OpenAI** | Very Fast | $$ | Excellent | Low | 5 min | 128K |\n| **Anthropic** | Fast | $$ | Excellent | Low | 5 min | 200K |\n| **Google** | Very Fast | $$ | Good-Excellent | Low | 5 min | 2M |\n| **Groq** | Ultra Fast | $ | Good | Low | 5 min | 32K |\n| **OpenRouter** | Varies | Varies | Varies | Low | 5 min | Varies |\n| **Ollama** | Slow-Medium | Free | Good | Max | 10 min | Varies |\n| **LM Studio** | Slow-Medium | Free | Good | Max | 15 min | Varies |\n| **Azure** | Very Fast | $$ | Excellent | High | 10 min | 128K |\n\n---\n\n## Choosing Your Provider\n\n### I want the easiest setup\n→ **OpenAI** — Most popular, best community support\n\n### I have unlimited budget\n→ **OpenAI** — Best quality\n\n### I want to save money\n→ **Groq** — Cheapest cloud ($0.05 per 1M tokens)\n\n### I want privacy/offline\n→ **Ollama** — Free, local, private\n\n### I want a GUI (not CLI)\n→ **LM Studio** — Desktop app\n\n### I'm in an enterprise\n→ **Azure OpenAI** — Compliance, support\n\n### I need long context (200K+ tokens)\n→ **Anthropic** — Best long-context model\n\n### I need multimodal (images, audio, video)\n→ **Google Gemini** — Best multimodal support\n\n### I want access to many models with one API key\n→ **OpenRouter** — 100+ models, unified billing\n\n---\n\n## Ready to Set Up Your Provider?\n\nNow that you've chosen a provider, follow the detailed setup instructions:\n\n→ **[AI Providers Configuration Guide](../5-CONFIGURATION/ai-providers.md)**\n\nThis guide includes:\n- Step-by-step setup instructions for each provider via the Settings UI\n- How to add credentials, test connections, and discover models\n- Model selection and recommendations\n- Provider-specific troubleshooting\n- Hardware requirements (for local providers)\n- Cost optimization tips\n\n---\n\n## Cost Estimator\n\n### OpenAI\n```\nLight use (10 chats/day): $1-5/month\nMedium use (50 chats/day): $10-30/month\nHeavy use (all-day use): $50-100+/month\n```\n\n### Anthropic\n```\nLight use: $1-3/month\nMedium use: $5-20/month\nHeavy use: $20-50+/month\n```\n\n### Groq\n```\nLight use: $0-1/month\nMedium use: $2-5/month\nHeavy use: $5-20/month\n```\n\n### Ollama\n```\nAny use: Free (electricity only)\n8GB GPU running 24/7: ~$10/month electricity\n```\n\n---\n\n## Next Steps\n\n1. **You've chosen a provider** (from this comparison guide)\n2. **Follow the setup guide**: [AI Providers Configuration](../5-CONFIGURATION/ai-providers.md)\n3. **Add your credential** in Settings → API Keys\n4. **Test your connection** and discover models\n5. **Start using Open Notebook!**\n\n---\n\n## Need Help?\n\n- **Setup issues?** See [AI Providers Configuration](../5-CONFIGURATION/ai-providers.md) for detailed troubleshooting per provider\n- **General problems?** Check [Troubleshooting Guide](../6-TROUBLESHOOTING/index.md)\n- **Questions?** Join [Discord community](https://discord.gg/37XJPXfz2w)\n" }, { "path": "docs/5-CONFIGURATION/advanced.md", "content": "# Advanced Configuration\n\nPerformance tuning, debugging, and advanced features.\n\n---\n\n## Performance Tuning\n\n### Concurrency Control\n\n```env\n# Max concurrent database operations (default: 5)\n# Increase: Faster processing, more conflicts\n# Decrease: Slower, fewer conflicts\nSURREAL_COMMANDS_MAX_TASKS=5\n```\n\n**Guidelines:**\n- CPU: 2 cores → 2-3 tasks\n- CPU: 4 cores → 5 tasks (default)\n- CPU: 8+ cores → 10-20 tasks\n\nHigher concurrency = more throughput but more database conflicts (retries handle this).\n\n### Retry Strategy\n\n```env\n# How to wait between retries\nSURREAL_COMMANDS_RETRY_WAIT_STRATEGY=exponential_jitter\n\n# Options:\n# - exponential_jitter (recommended)\n# - exponential\n# - fixed\n# - random\n```\n\nFor high-concurrency deployments, use `exponential_jitter` to prevent thundering herd.\n\n### Timeout Tuning\n\n```env\n# Client timeout (default: 300 seconds)\nAPI_CLIENT_TIMEOUT=300\n\n# LLM timeout (default: 60 seconds)\nESPERANTO_LLM_TIMEOUT=60\n```\n\n**Guideline:** Set `API_CLIENT_TIMEOUT` > `ESPERANTO_LLM_TIMEOUT` + buffer\n\n```\nExample:\n ESPERANTO_LLM_TIMEOUT=120\n API_CLIENT_TIMEOUT=180 # 120 + 60 second buffer\n```\n\n---\n\n## Batching\n\n### TTS Batch Size\n\nFor podcast generation, control concurrent TTS requests:\n\n```env\n# Default: 5\nTTS_BATCH_SIZE=2\n```\n\n**Providers and recommendations:**\n- OpenAI: 5 (can handle many concurrent)\n- Google: 4 (good concurrency)\n- ElevenLabs: 2 (limited concurrent requests)\n- Local TTS: 1 (single-threaded)\n\nLower = slower but more stable. Higher = faster but more load on provider.\n\n---\n\n## Logging & Debugging\n\n### Enable Detailed Logging\n\n```bash\n# Start with debug logging\nRUST_LOG=debug # For Rust components\nLOGLEVEL=DEBUG # For Python components\n```\n\n### Debug Specific Components\n\n```bash\n# Only surreal operations\nRUST_LOG=surrealdb=debug\n\n# Only langchain\nLOGLEVEL=langchain:debug\n\n# Only specific module\nRUST_LOG=open_notebook::database=debug\n```\n\n### LangSmith Tracing\n\nFor debugging LLM workflows:\n\n```env\nLANGCHAIN_TRACING_V2=true\nLANGCHAIN_ENDPOINT=\"https://api.smith.langchain.com\"\nLANGCHAIN_API_KEY=your-key\nLANGCHAIN_PROJECT=\"Open Notebook\"\n```\n\nThen visit https://smith.langchain.com to see traces.\n\n---\n\n## Port Configuration\n\n### Default Ports\n\n```\nFrontend: 8502 (Docker deployment)\nFrontend: 3000 (Development from source)\nAPI: 5055\nSurrealDB: 8000\n```\n\n### Changing Frontend Port\n\nEdit `docker-compose.yml`:\n\n```yaml\nservices:\n open-notebook:\n ports:\n - \"8001:8502\" # Change from 8502 to 8001\n```\n\nAccess at: `http://localhost:8001`\n\nAPI auto-detects to: `http://localhost:5055` ✓\n\n### Changing API Port\n\n```yaml\nservices:\n open-notebook:\n ports:\n - \"127.0.0.1:8502:8502\" # Frontend\n - \"5056:5055\" # Change API from 5055 to 5056\n environment:\n - API_URL=http://localhost:5056 # Update API_URL\n```\n\nAccess API directly: `http://localhost:5056/docs`\n\n**Note:** When changing API port, you must set `API_URL` explicitly since auto-detection assumes port 5055.\n\n### Changing SurrealDB Port\n\n```yaml\nservices:\n surrealdb:\n ports:\n - \"8001:8000\" # Change from 8000 to 8001\n environment:\n - SURREAL_URL=ws://surrealdb:8001/rpc # Update connection URL\n```\n\n**Important:** Internal Docker network uses container name (`surrealdb`), not `localhost`.\n\n---\n\n## SSL/TLS Configuration\n\n### Custom CA Certificate\n\nFor self-signed certs on local providers:\n\n```env\nESPERANTO_SSL_CA_BUNDLE=/path/to/ca-bundle.pem\n```\n\n### Disable Verification (Development Only)\n\n```env\n# WARNING: Only for testing/development\n# Vulnerable to MITM attacks\nESPERANTO_SSL_VERIFY=false\n```\n\n---\n\n## Multi-Provider Setup\n\n### Use Different Providers for Different Tasks\n\nConfigure multiple AI providers via **Settings → API Keys**. Each provider gets its own credential:\n\n1. Add a credential for your main language model provider (e.g., OpenAI, Anthropic)\n2. Add a credential for embeddings (e.g., Voyage AI, or use the same provider)\n3. Add a credential for TTS (e.g., ElevenLabs, or OpenAI-Compatible for local Speaches)\n4. Each credential's models are registered and available independently\n\n### Multiple Endpoints for OpenAI-Compatible\n\nWhen using OpenAI-Compatible providers, you can configure per-service URLs in a single credential:\n\n1. Go to **Settings** → **API Keys**\n2. Click **Add Credential** → Select **OpenAI-Compatible**\n3. Configure separate URLs for LLM, Embedding, TTS, and STT\n4. Click **Save**, then **Test Connection**\n\n---\n\n## Security Hardening\n\n### Change Default Credentials\n\n```env\n# Don't use defaults in production\nSURREAL_USER=your_secure_username\nSURREAL_PASSWORD=$(openssl rand -base64 32) # Generate secure password\n```\n\n### Add Password Protection\n\n```env\n# Protect your Open Notebook instance\nOPEN_NOTEBOOK_PASSWORD=your_secure_password\n```\n\n### Use HTTPS\n\n```env\n# Always use HTTPS in production\nAPI_URL=https://mynotebook.example.com\n```\n\n### Firewall Rules\n\nRestrict access to your Open Notebook:\n- Port 8502 (frontend): Only from your IP\n- Port 5055 (API): Only from frontend\n- Port 8000 (SurrealDB): Never expose to internet\n\n---\n\n## Web Scraping & Content Extraction\n\nOpen Notebook uses multiple services for content extraction:\n\n### Firecrawl\n\nFor advanced web scraping:\n\n```env\nFIRECRAWL_API_KEY=your-key\n```\n\nGet key from: https://firecrawl.dev/\n\n### Jina AI\n\nAlternative web extraction:\n\n```env\nJINA_API_KEY=your-key\n```\n\nGet key from: https://jina.ai/\n\n---\n\n## Environment Variable Groups\n\n### Credential Storage (Required)\n```env\nOPEN_NOTEBOOK_ENCRYPTION_KEY # Required for storing credentials\n```\n\nAI provider API keys are configured via **Settings → API Keys** (not environment variables).\n\n### Database\n```env\nSURREAL_URL\nSURREAL_USER\nSURREAL_PASSWORD\nSURREAL_NAMESPACE\nSURREAL_DATABASE\n```\n\n### Performance\n```env\nSURREAL_COMMANDS_MAX_TASKS\nSURREAL_COMMANDS_RETRY_ENABLED\nSURREAL_COMMANDS_RETRY_MAX_ATTEMPTS\nSURREAL_COMMANDS_RETRY_WAIT_STRATEGY\nSURREAL_COMMANDS_RETRY_WAIT_MIN\nSURREAL_COMMANDS_RETRY_WAIT_MAX\n```\n\n### API Settings\n```env\nAPI_URL\nINTERNAL_API_URL\nAPI_CLIENT_TIMEOUT\nESPERANTO_LLM_TIMEOUT\n```\n\n### Audio/TTS\n```env\nTTS_BATCH_SIZE\n```\n\n> **Note:** `ELEVENLABS_API_KEY` is deprecated. Configure ElevenLabs via **Settings → API Keys**.\n\n### Debugging\n```env\nLANGCHAIN_TRACING_V2\nLANGCHAIN_ENDPOINT\nLANGCHAIN_API_KEY\nLANGCHAIN_PROJECT\n```\n\n---\n\n## Testing Configuration\n\n### Quick Test\n\n```bash\n# Test API health\ncurl http://localhost:5055/health\n\n# Test with sample (requires configured credential and registered models)\ncurl -X POST http://localhost:5055/api/chat \\\n -H \"Content-Type: application/json\" \\\n -d '{\"message\":\"Hello\"}'\n```\n\n### Validate Config\n\n```bash\n# Check environment variables are set\nenv | grep OPEN_NOTEBOOK_ENCRYPTION_KEY\n\n# Verify database connection\npython -c \"import os; print(os.getenv('SURREAL_URL'))\"\n```\n\n---\n\n## Troubleshooting Performance\n\n### High Memory Usage\n\n```env\n# Reduce concurrency\nSURREAL_COMMANDS_MAX_TASKS=2\n\n# Reduce TTS batch size\nTTS_BATCH_SIZE=1\n```\n\n### High CPU Usage\n\n```env\n# Check worker count\nSURREAL_COMMANDS_MAX_TASKS\n\n# Reduce if maxed out:\nSURREAL_COMMANDS_MAX_TASKS=5\n```\n\n### Slow Responses\n\n```env\n# Check timeout settings\nAPI_CLIENT_TIMEOUT=300\n\n# Check retry config\nSURREAL_COMMANDS_RETRY_MAX_ATTEMPTS=3\n```\n\n### Database Conflicts\n\n```env\n# Reduce concurrency\nSURREAL_COMMANDS_MAX_TASKS=3\n\n# Use jitter strategy\nSURREAL_COMMANDS_RETRY_WAIT_STRATEGY=exponential_jitter\n```\n\n---\n\n## Backup & Restore\n\n### Data Locations\n\n| Path | Contents |\n|------|----------|\n| `./data` or `/app/data` | Uploads, podcasts, checkpoints |\n| `./surreal_data` or `/mydata` | SurrealDB database files |\n\n### Quick Backup\n\n```bash\n# Stop services (recommended for consistency)\ndocker compose down\n\n# Create timestamped backup\ntar -czf backup-$(date +%Y%m%d-%H%M%S).tar.gz \\\n notebook_data/ surreal_data/\n\n# Restart services\ndocker compose up -d\n```\n\n### Automated Backup Script\n\n```bash\n#!/bin/bash\n# backup.sh - Run daily via cron\n\nBACKUP_DIR=\"/path/to/backups\"\nDATE=$(date +%Y%m%d-%H%M%S)\n\n# Create backup\ntar -czf \"$BACKUP_DIR/open-notebook-$DATE.tar.gz\" \\\n /path/to/notebook_data \\\n /path/to/surreal_data\n\n# Keep only last 7 days\nfind \"$BACKUP_DIR\" -name \"open-notebook-*.tar.gz\" -mtime +7 -delete\n\necho \"Backup complete: open-notebook-$DATE.tar.gz\"\n```\n\nAdd to cron:\n```bash\n# Daily backup at 2 AM\n0 2 * * * /path/to/backup.sh >> /var/log/open-notebook-backup.log 2>&1\n```\n\n### Restore\n\n```bash\n# Stop services\ndocker compose down\n\n# Remove old data (careful!)\nrm -rf notebook_data/ surreal_data/\n\n# Extract backup\ntar -xzf backup-20240115-120000.tar.gz\n\n# Restart services\ndocker compose up -d\n```\n\n### Migration Between Servers\n\n```bash\n# On source server\ndocker compose down\ntar -czf open-notebook-migration.tar.gz notebook_data/ surreal_data/\n\n# Transfer to new server\nscp open-notebook-migration.tar.gz user@newserver:/path/\n\n# On new server\ntar -xzf open-notebook-migration.tar.gz\ndocker compose up -d\n```\n\n---\n\n## Container Management\n\n### Common Commands\n\n```bash\n# Start services\ndocker compose up -d\n\n# Stop services\ndocker compose down\n\n# View logs (all services)\ndocker compose logs -f\n\n# View logs (specific service)\ndocker compose logs -f api\n\n# Restart specific service\ndocker compose restart api\n\n# Update to latest version\ndocker compose down\ndocker compose pull\ndocker compose up -d\n\n# Check resource usage\ndocker stats\n\n# Check service health\ndocker compose ps\n```\n\n### Clean Up\n\n```bash\n# Remove stopped containers\ndocker compose rm\n\n# Remove unused images\ndocker image prune\n\n# Full cleanup (careful!)\ndocker system prune -a\n```\n\n---\n\n## Summary\n\n**Most deployments need:**\n- One AI provider API key\n- Default database settings\n- Default timeouts\n\n**Tune performance only if:**\n- You have specific bottlenecks\n- High-concurrency workload\n- Custom hardware (very fast or very slow)\n\n**Advanced features:**\n- Firecrawl for better web scraping\n- LangSmith for debugging workflows\n- Custom CA bundles for self-signed certs\n" }, { "path": "docs/5-CONFIGURATION/ai-providers.md", "content": "# AI Providers - Configuration Guide\n\nComplete setup instructions for each AI provider via the **Settings UI**.\n\n> **New in v1.2**: All AI provider credentials are now managed through the Settings UI. Environment variables for API keys are deprecated.\n\n---\n\n## How Provider Setup Works\n\nOpen Notebook uses a **credential-based system** for managing AI providers:\n\n1. **Get your API key** from the provider's website\n2. **Open Settings** → **API Keys** → **Add Credential**\n3. **Test the connection** to verify it works\n4. **Discover & Register Models** to make them available\n5. **Start using** the provider in your notebooks\n\n> **Prerequisite**: You must set `OPEN_NOTEBOOK_ENCRYPTION_KEY` in your docker-compose.yml before storing credentials. See [API Configuration](../3-USER-GUIDE/api-configuration.md#encryption-setup) for details.\n\n---\n\n## Cloud Providers (Recommended for Most)\n\n### OpenAI\n\n**Cost:** ~$0.03-0.15 per 1K tokens (varies by model)\n\n**Get Your API Key:**\n1. Go to https://platform.openai.com/api-keys\n2. Create account (if needed)\n3. Create new API key (starts with \"sk-proj-\")\n4. Add $5+ credits to account\n\n**Configure in Open Notebook:**\n1. Go to **Settings** → **API Keys**\n2. Click **Add Credential**\n3. Select provider: **OpenAI**\n4. Give it a name (e.g., \"My OpenAI Key\")\n5. Paste your API key\n6. Click **Save**, then **Test Connection**\n7. Click **Discover Models** to find available models\n8. Click **Register Models** to make them available\n\n**Available Models (in Open Notebook):**\n- `gpt-4o` — Best quality, fast (latest version)\n- `gpt-4o-mini` — Fast, cheap, good for testing\n- `o1` — Advanced reasoning model (slower, more expensive)\n- `o1-mini` — Faster reasoning model\n\n**Recommended:**\n- For general use: `gpt-4o` (best balance)\n- For testing/cheap: `gpt-4o-mini` (90% cheaper)\n- For complex reasoning: `o1` (best for hard problems)\n\n**Cost Estimate:**\n```\nLight use: $1-5/month\nMedium use: $10-30/month\nHeavy use: $50-100+/month\n```\n\n**Troubleshooting:**\n- \"Invalid API key\" → Check key starts with \"sk-proj-\" and test the connection in Settings\n- \"Rate limit exceeded\" → Wait or upgrade account\n- \"Model not available\" → Try gpt-4o-mini instead, or re-discover models\n\n---\n\n### Anthropic (Claude)\n\n**Cost:** ~$0.80-3.00 per 1M tokens (cheaper than OpenAI for long context)\n\n**Get Your API Key:**\n1. Go to https://console.anthropic.com/\n2. Create account or login\n3. Go to API keys section\n4. Create new API key (starts with \"sk-ant-\")\n\n**Configure in Open Notebook:**\n1. Go to **Settings** → **API Keys**\n2. Click **Add Credential**\n3. Select provider: **Anthropic**\n4. Give it a name, paste your API key\n5. Click **Save**, then **Test Connection**\n6. Click **Discover Models** → **Register Models**\n\n**Available Models:**\n- `claude-sonnet-4-5-20250929` — Latest, best quality (recommended)\n- `claude-3-5-sonnet-20241022` — Previous generation, still excellent\n- `claude-3-5-haiku-20241022` — Fast, cheap\n- `claude-opus-4-5-20251101` — Most powerful, expensive\n\n**Recommended:**\n- For general use: `claude-sonnet-4-5` (best overall, latest)\n- For cheap: `claude-3-5-haiku` (80% cheaper)\n- For complex: `claude-opus-4-5` (most capable)\n\n**Cost Estimate:**\n```\nSonnet: $3-20/month (typical use)\nHaiku: $0.50-3/month\nOpus: $10-50+/month\n```\n\n**Advantages:**\n- Great long-context support (200K tokens)\n- Excellent reasoning\n- Fast processing\n\n**Troubleshooting:**\n- \"Invalid API key\" → Check it starts with \"sk-ant-\" and test in Settings\n- \"Overloaded\" → Anthropic is busy, retry later\n- \"Model unavailable\" → Re-discover models from the credential\n\n---\n\n### Google Gemini\n\n**Cost:** ~$0.075-0.30 per 1K tokens (competitive with OpenAI)\n\n**Get Your API Key:**\n1. Go to https://aistudio.google.com/app/apikey\n2. Create account or login\n3. Create new API key\n\n**Configure in Open Notebook:**\n1. Go to **Settings** → **API Keys**\n2. Click **Add Credential**\n3. Select provider: **Google Gemini**\n4. Give it a name, paste your API key\n5. Click **Save**, then **Test Connection**\n6. Click **Discover Models** → **Register Models**\n\n**Available Models:**\n- `gemini-2.0-flash-exp` — Latest experimental, fastest (recommended)\n- `gemini-2.0-flash` — Stable version, fast, cheap\n\n**Recommended:**\n- For general use: `gemini-2.0-flash-exp` (best value, latest)\n- For cheap: `gemini-1.5-flash` (very cheap)\n- For complex/long context: `gemini-1.5-pro-latest` (2M token context)\n\n**Advantages:**\n- Very long context (1M tokens)\n- Multimodal (images, audio, video)\n- Good for podcasts\n\n**Troubleshooting:**\n- \"API key invalid\" → Get fresh key from aistudio.google.com\n- \"Quota exceeded\" → Free tier limited, upgrade account\n- \"Model not found\" → Re-discover models from the credential\n\n---\n\n### Groq\n\n**Cost:** ~$0.05 per 1M tokens (cheapest, but limited models)\n\n**Get Your API Key:**\n1. Go to https://console.groq.com/keys\n2. Create account or login\n3. Create new API key\n\n**Configure in Open Notebook:**\n1. Go to **Settings** → **API Keys**\n2. Click **Add Credential**\n3. Select provider: **Groq**\n4. Give it a name, paste your API key\n5. Click **Save**, then **Test Connection**\n6. Click **Discover Models** → **Register Models**\n\n**Available Models:**\n- `llama-3.3-70b-versatile` — Best on Groq (recommended)\n- `llama-3.1-70b-versatile` — Fast, capable\n- `mixtral-8x7b-32768` — Good alternative\n- `gemma2-9b-it` — Small, very fast\n\n**Recommended:**\n- For quality: `llama-3.3-70b-versatile` (best overall)\n- For speed: `gemma2-9b-it` (ultra-fast)\n- For balance: `llama-3.1-70b-versatile`\n\n**Advantages:**\n- Ultra-fast inference\n- Very cheap\n- Great for transformations/batch work\n\n**Disadvantages:**\n- Limited model selection\n- Smaller models than OpenAI/Anthropic\n\n**Troubleshooting:**\n- \"Rate limited\" → Free tier has limits, upgrade\n- \"Model not available\" → Re-discover models from the credential\n\n---\n\n### OpenRouter\n\n**Cost:** Varies by model ($0.05-15 per 1M tokens)\n\n**Get Your API Key:**\n1. Go to https://openrouter.ai/keys\n2. Create account or login\n3. Add credits to your account\n4. Create new API key\n\n**Configure in Open Notebook:**\n1. Go to **Settings** → **API Keys**\n2. Click **Add Credential**\n3. Select provider: **OpenRouter**\n4. Give it a name, paste your API key\n5. Click **Save**, then **Test Connection**\n6. Click **Discover Models** → **Register Models**\n\n**Available Models (100+ options):**\n- OpenAI: `openai/gpt-4o`, `openai/o1`\n- Anthropic: `anthropic/claude-sonnet-4.5`, `anthropic/claude-3.5-haiku`\n- Google: `google/gemini-2.0-flash-exp`, `google/gemini-1.5-pro`\n- Meta: `meta-llama/llama-3.3-70b-instruct`, `meta-llama/llama-3.1-405b-instruct`\n- Mistral: `mistralai/mistral-large-2411`\n- DeepSeek: `deepseek/deepseek-chat`\n- And many more...\n\n**Recommended:**\n- For quality: `anthropic/claude-sonnet-4.5` (best overall)\n- For speed/cost: `google/gemini-2.0-flash-exp` (very fast, cheap)\n- For open-source: `meta-llama/llama-3.3-70b-instruct`\n- For reasoning: `openai/o1`\n\n**Advantages:**\n- One API key for 100+ models\n- Unified billing\n- Easy model comparison\n- Access to models that may have waitlists elsewhere\n\n**Cost Estimate:**\n```\nLight use: $1-5/month\nMedium use: $10-30/month\nHeavy use: Depends on models chosen\n```\n\n**Troubleshooting:**\n- \"Invalid API key\" → Check it starts with \"sk-or-\"\n- \"Insufficient credits\" → Add credits at openrouter.ai\n- \"Model not available\" → Check model ID spelling (use full path)\n\n---\n\n## Self-Hosted / Local\n\n### Ollama (Recommended for Local)\n\n**Cost:** Free (electricity only)\n\n**Setup Ollama:**\n1. Install Ollama: https://ollama.ai\n2. Run Ollama in background: `ollama serve`\n3. Download a model: `ollama pull mistral`\n\n**Configure in Open Notebook:**\n1. Go to **Settings** → **API Keys**\n2. Click **Add Credential**\n3. Select provider: **Ollama**\n4. Give it a name (e.g., \"Local Ollama\")\n5. Enter the base URL:\n - Same machine (non-Docker): `http://localhost:11434`\n - Docker with Ollama on host: `http://host.docker.internal:11434`\n - Docker with Ollama container: `http://ollama:11434`\n6. Click **Save**, then **Test Connection**\n7. Click **Discover Models** → **Register Models**\n\nSee [Ollama Setup Guide](ollama.md) for detailed network configuration.\n\n**Available Models:**\n- `llama3.3:70b` — Best quality (requires 40GB+ RAM)\n- `llama3.1:8b` — Recommended, balanced (8GB RAM)\n- `qwen2.5:7b` — Excellent for code and reasoning\n- `mistral:7b` — Good general purpose\n- `phi3:3.8b` — Small, fast (4GB RAM)\n- `gemma2:9b` — Google's model, balanced\n- Many more: `ollama list` to see available\n\n**Recommended:**\n- For quality (with GPU): `llama3.3:70b` (best)\n- For general use: `llama3.1:8b` (best balance)\n- For speed/low memory: `phi3:3.8b` (very fast)\n- For coding: `qwen2.5:7b` (excellent at code)\n\n**Hardware Requirements:**\n```\nGPU (NVIDIA/AMD):\n 8GB VRAM: Runs most models fine\n 6GB VRAM: Works, slower\n 4GB VRAM: Small models only\n\nCPU-only:\n 16GB+ RAM: Slow but works\n 8GB RAM: Very slow\n 4GB RAM: Not recommended\n```\n\n**Advantages:**\n- Completely private (runs locally)\n- Free (electricity only)\n- No API key needed\n- Works offline\n\n**Disadvantages:**\n- Slower than cloud (unless on GPU)\n- Smaller models than cloud\n- Requires local hardware\n\n**Troubleshooting:**\n- \"Connection refused\" → Ollama not running or wrong URL in credential\n- \"Model not found\" → Download it: `ollama pull modelname`\n- \"Out of memory\" → Use smaller model or add more RAM\n\n---\n\n### LM Studio (Local Alternative)\n\n**Cost:** Free\n\n**Setup LM Studio:**\n1. Download LM Studio: https://lmstudio.ai\n2. Open app\n3. Download a model from library\n4. Go to \"Local Server\" tab\n5. Start server (default port: 1234)\n\n**Configure in Open Notebook:**\n1. Go to **Settings** → **API Keys**\n2. Click **Add Credential**\n3. Select provider: **OpenAI-Compatible**\n4. Give it a name (e.g., \"LM Studio\")\n5. Enter the base URL: `http://host.docker.internal:1234/v1` (Docker) or `http://localhost:1234/v1` (local)\n6. API key: `lm-studio` (placeholder, LM Studio doesn't require one)\n7. Click **Save**, then **Test Connection**\n\n**Advantages:**\n- GUI interface (easier than Ollama CLI)\n- Good model selection\n- Privacy-focused\n- Works offline\n\n**Disadvantages:**\n- Desktop only (Mac/Windows/Linux)\n- Slower than cloud\n- Requires local GPU\n\n---\n\n### Custom OpenAI-Compatible\n\nFor Text Generation UI, vLLM, or other OpenAI-compatible endpoints:\n\n1. Go to **Settings** → **API Keys**\n2. Click **Add Credential**\n3. Select provider: **OpenAI-Compatible**\n4. Enter the base URL for your endpoint (e.g., `http://localhost:8000/v1`)\n5. Enter API key if required\n6. Optionally configure per-service URLs (LLM, Embedding, TTS, STT)\n7. Click **Save**, then **Test Connection**\n\nSee [OpenAI-Compatible Setup](openai-compatible.md) for detailed instructions.\n\n---\n\n## Enterprise\n\n### Azure OpenAI\n\n**Cost:** Same as OpenAI (usage-based)\n\n**Configure in Open Notebook:**\n1. Create Azure OpenAI service in Azure portal\n2. Deploy GPT-4/3.5-turbo model\n3. Get your endpoint and key\n4. Go to **Settings** → **API Keys**\n5. Click **Add Credential**\n6. Select provider: **Azure OpenAI**\n7. Fill in: API Key, Endpoint, API Version\n8. Optionally configure service-specific endpoints (LLM, Embedding)\n9. Click **Save**, then **Test Connection**\n\n**Advantages:**\n- Enterprise support\n- VPC integration\n- Compliance (HIPAA, SOC2, etc.)\n\n**Disadvantages:**\n- More complex setup\n- Higher overhead\n- Requires Azure account\n\n---\n\n## Embeddings (For Search/Semantic Features)\n\nBy default, Open Notebook uses the LLM provider's embeddings. Embedding models are discovered and registered through the same credential system — when you discover models from a credential, embedding models are included alongside language models.\n\n---\n\n## Choosing Your Provider\n\n**1. Don't want to run locally and don't want to mess around with different providers:**\n\nUse OpenAI\n- Cloud-based\n- Good quality\n- Reasonable cost\n- Simplest setup, supports all modes (text, embedding, tts, stt, etc)\n\n**For budget-conscious:** Groq, OpenRouter or Ollama\n- Groq: Super cheap cloud\n- Ollama: Free, but local\n- OpenRouter: many open source models very accessible\n\n**For privacy-first:** Ollama or LM Studio and Speaches ([TTS](local-tts.md), [STT](local-stt.md))\n- Everything stays local\n- Works offline\n- No API keys sent anywhere\n\n**For enterprise:** Azure OpenAI\n- Compliance\n- VPC integration\n- Support\n\n---\n\n## Next Steps\n\n1. **Choose your provider** from above\n2. **Get API key** (if cloud) or install locally (if Ollama)\n3. **Set `OPEN_NOTEBOOK_ENCRYPTION_KEY`** in your docker-compose.yml (required for storing credentials)\n4. **Open Settings** → **API Keys** → **Add Credential**\n5. **Test Connection** to verify it works\n6. **Discover & Register Models** to make them available\n7. **Verify it works** with a test chat\n\n> **Multiple providers**: You can add credentials for as many providers as you want. Create separate credentials for different projects or team members.\n\nDone!\n\n---\n\n## Legacy: Environment Variables (Deprecated)\n\n> **Deprecated**: Configuring AI provider API keys via environment variables is deprecated. Use the Settings UI instead. Environment variables may still work as a fallback but are no longer the recommended approach.\n\nIf you are migrating from an older version that used environment variables, go to **Settings** → **API Keys** and click the **Migrate to Database** button to import your existing keys into the credential system.\n\n---\n\n## Related\n\n- **[API Configuration](../3-USER-GUIDE/api-configuration.md)** — Detailed credential management guide\n- **[Environment Reference](environment-reference.md)** - Complete list of all environment variables\n- **[Advanced Configuration](advanced.md)** - Timeouts, SSL, performance tuning\n- **[Ollama Setup](ollama.md)** - Detailed Ollama configuration guide\n- **[OpenAI-Compatible](openai-compatible.md)** - LM Studio and other compatible providers\n- **[Local TTS Setup](local-tts.md)** - Text-to-speech with Speaches\n- **[Local STT Setup](local-stt.md)** - Speech-to-text with Speaches\n- **[Troubleshooting](../6-TROUBLESHOOTING/quick-fixes.md)** - Common issues and fixes\n" }, { "path": "docs/5-CONFIGURATION/database.md", "content": "# Database - SurrealDB Configuration\n\nOpen Notebook uses SurrealDB for its database needs. \n\n---\n\n## Default Configuration\n\nOpen Notebook should work out of the box with SurrealDB as long as the environment variables are correctly setup. \n\n\n### DB running in the same docker compose as Open Notebook (recommended)\n\nThe example above is for when you are running SurrealDB as a separate docker container, which is the method described [here](../1-INSTALLATION/docker-compose.md) (and our recommended method). \n\n```env\nSURREAL_URL=\"ws://surrealdb:8000/rpc\"\nSURREAL_USER=\"root\"\nSURREAL_PASSWORD=\"root\"\nSURREAL_NAMESPACE=\"open_notebook\"\nSURREAL_DATABASE=\"open_notebook\"\n```\n\n### DB running in the host machine and Open Notebook running in Docker\n\nIf ON is running in docker and SurrealDB is on your host machine, you need to point to it. \n\n```env\nSURREAL_URL=\"ws://your-machine-ip:8000/rpc\" #or host.docker.internal\nSURREAL_USER=\"root\"\nSURREAL_PASSWORD=\"root\"\nSURREAL_NAMESPACE=\"open_notebook\"\nSURREAL_DATABASE=\"open_notebook\"\n```\n\n### Open Notebook and Surreal are running on the same machine\n\nIf you are running both services locally or if you are using the deprecated [single container setup](../1-INSTALLATION/single-container.md)\n\n```env\nSURREAL_URL=\"ws://localhost:8000/rpc\"\nSURREAL_USER=\"root\"\nSURREAL_PASSWORD=\"root\"\nSURREAL_NAMESPACE=\"open_notebook\"\nSURREAL_DATABASE=\"open_notebook\"\n```\n\n## Multiple databases\n\nYou can have multiple namespaces in one SurrealDB instance and you can also have multiple databases in one instance. So, if you want to setup multiple open noteobok deployments for different users, you don't need to deploy multiple databases. \n" }, { "path": "docs/5-CONFIGURATION/environment-reference.md", "content": "# Complete Environment Reference\n\nComprehensive list of all environment variables available in Open Notebook.\n\n---\n\n## API Configuration\n\n| Variable | Required? | Default | Description |\n|----------|-----------|---------|-------------|\n| `API_URL` | No | Auto-detected | URL where frontend reaches API (e.g., http://localhost:5055) |\n| `INTERNAL_API_URL` | No | http://localhost:5055 | Internal API URL for Next.js server-side proxying |\n| `API_CLIENT_TIMEOUT` | No | 300 | Client timeout in seconds (how long to wait for API response) |\n| `OPEN_NOTEBOOK_PASSWORD` | No | None | Password to protect Open Notebook instance |\n| `OPEN_NOTEBOOK_ENCRYPTION_KEY` | **Yes** | None | Secret string to encrypt credentials stored in database (any string works). **Required** for the credential system. Supports Docker secrets via `_FILE` suffix. |\n| `HOSTNAME` | No | `0.0.0.0` (in Docker) | Network interface for Next.js to bind to. Default `0.0.0.0` ensures accessibility from reverse proxies |\n\n> **Important**: `OPEN_NOTEBOOK_ENCRYPTION_KEY` is required for storing AI provider credentials via the Settings UI. Without it, you cannot save credentials. If you change or lose this key, all stored credentials become unreadable.\n\n---\n\n## Database: SurrealDB\n\n| Variable | Required? | Default | Description |\n|----------|-----------|---------|-------------|\n| `SURREAL_URL` | Yes | ws://surrealdb:8000/rpc | SurrealDB WebSocket connection URL |\n| `SURREAL_USER` | Yes | root | SurrealDB username |\n| `SURREAL_PASSWORD` | Yes | root | SurrealDB password |\n| `SURREAL_NAMESPACE` | Yes | open_notebook | SurrealDB namespace |\n| `SURREAL_DATABASE` | Yes | open_notebook | SurrealDB database name |\n\n---\n\n## Database: Retry Configuration\n\n| Variable | Required? | Default | Description |\n|----------|-----------|---------|-------------|\n| `SURREAL_COMMANDS_RETRY_ENABLED` | No | true | Enable retries on failure |\n| `SURREAL_COMMANDS_RETRY_MAX_ATTEMPTS` | No | 3 | Maximum retry attempts |\n| `SURREAL_COMMANDS_RETRY_WAIT_STRATEGY` | No | exponential_jitter | Retry wait strategy (exponential_jitter/exponential/fixed/random) |\n| `SURREAL_COMMANDS_RETRY_WAIT_MIN` | No | 1 | Minimum wait time between retries (seconds) |\n| `SURREAL_COMMANDS_RETRY_WAIT_MAX` | No | 30 | Maximum wait time between retries (seconds) |\n\n---\n\n## Database: Concurrency\n\n| Variable | Required? | Default | Description |\n|----------|-----------|---------|-------------|\n| `SURREAL_COMMANDS_MAX_TASKS` | No | 5 | Maximum concurrent database tasks |\n\n---\n\n## LLM Timeouts\n\n| Variable | Required? | Default | Description |\n|----------|-----------|---------|-------------|\n| `ESPERANTO_LLM_TIMEOUT` | No | 60 | LLM inference timeout in seconds |\n| `ESPERANTO_SSL_VERIFY` | No | true | Verify SSL certificates (false = development only) |\n| `ESPERANTO_SSL_CA_BUNDLE` | No | None | Path to custom CA certificate bundle |\n\n---\n\n## Text-to-Speech (TTS)\n\n| Variable | Required? | Default | Description |\n|----------|-----------|---------|-------------|\n| `TTS_BATCH_SIZE` | No | 5 | Concurrent TTS requests (1-5, depends on provider) |\n\n---\n\n## Content Extraction\n\n| Variable | Required? | Default | Description |\n|----------|-----------|---------|-------------|\n| `FIRECRAWL_API_KEY` | No | None | Firecrawl API key for advanced web scraping |\n| `JINA_API_KEY` | No | None | Jina AI API key for web extraction |\n\n**Setup:**\n- Firecrawl: https://firecrawl.dev/\n- Jina: https://jina.ai/\n\n---\n\n## Network / Proxy\n\n| Variable | Required? | Default | Description |\n|----------|-----------|---------|-------------|\n| `HTTP_PROXY` | No | None | HTTP proxy URL for outbound HTTP requests |\n| `HTTPS_PROXY` | No | None | HTTPS proxy URL for outbound HTTPS requests |\n| `NO_PROXY` | No | None | Comma-separated list of hosts to bypass proxy |\n\nRoute all outbound HTTP requests through a proxy server. Useful for corporate/firewalled environments.\n\nThe underlying libraries (esperanto, content-core, podcast-creator) automatically detect proxy settings from these standard environment variables.\n\n**Affects:**\n- AI provider API calls (OpenAI, Anthropic, Google, Groq, etc.)\n- Content extraction from URLs (web scraping, YouTube transcripts)\n- Podcast generation (LLM and TTS provider calls)\n\n**Format:** `http://[user:pass@]host:port` or `https://[user:pass@]host:port`\n\n**Examples:**\n```bash\n# Basic proxy\nHTTP_PROXY=http://proxy.corp.com:8080\nHTTPS_PROXY=http://proxy.corp.com:8080\n\n# Authenticated proxy\nHTTP_PROXY=http://user:password@proxy.corp.com:8080\nHTTPS_PROXY=http://user:password@proxy.corp.com:8080\n\n# Bypass proxy for local hosts\nNO_PROXY=localhost,127.0.0.1,.local\n```\n\n---\n\n## Debugging & Monitoring\n\n| Variable | Required? | Default | Description |\n|----------|-----------|---------|-------------|\n| `LANGCHAIN_TRACING_V2` | No | false | Enable LangSmith tracing |\n| `LANGCHAIN_ENDPOINT` | No | https://api.smith.langchain.com | LangSmith endpoint |\n| `LANGCHAIN_API_KEY` | No | None | LangSmith API key |\n| `LANGCHAIN_PROJECT` | No | Open Notebook | LangSmith project name |\n\n**Setup:** https://smith.langchain.com/\n\n---\n\n## Environment Variables by Use Case\n\n### Minimal Setup (New Installation)\n```\nOPEN_NOTEBOOK_ENCRYPTION_KEY=my-secret-key\nSURREAL_URL=ws://surrealdb:8000/rpc\nSURREAL_USER=root\nSURREAL_PASSWORD=password\nSURREAL_NAMESPACE=open_notebook\nSURREAL_DATABASE=open_notebook\n```\nThen configure AI providers via **Settings → API Keys** in the browser.\n\n### Production Deployment\n```\nOPEN_NOTEBOOK_ENCRYPTION_KEY=your-strong-secret-key\nOPEN_NOTEBOOK_PASSWORD=your-secure-password\nAPI_URL=https://mynotebook.example.com\nSURREAL_USER=production_user\nSURREAL_PASSWORD=secure_password\n```\n\n### Self-Hosted Behind Reverse Proxy\n```\nOPEN_NOTEBOOK_ENCRYPTION_KEY=your-secret-key\nAPI_URL=https://mynotebook.example.com\n```\n\n### Corporate Environment (Behind Proxy)\n```\nOPEN_NOTEBOOK_ENCRYPTION_KEY=your-secret-key\nHTTP_PROXY=http://proxy.corp.com:8080\nHTTPS_PROXY=http://proxy.corp.com:8080\nNO_PROXY=localhost,127.0.0.1\n```\n\n### High-Performance Deployment\n```\nOPEN_NOTEBOOK_ENCRYPTION_KEY=your-secret-key\nSURREAL_COMMANDS_MAX_TASKS=10\nTTS_BATCH_SIZE=5\nAPI_CLIENT_TIMEOUT=600\n```\n\n### Debugging\n```\nOPEN_NOTEBOOK_ENCRYPTION_KEY=your-secret-key\nLANGCHAIN_TRACING_V2=true\nLANGCHAIN_API_KEY=your-key\n```\n\n---\n\n## Validation\n\nCheck if a variable is set:\n\n```bash\n# Check single variable\necho $OPEN_NOTEBOOK_ENCRYPTION_KEY\n\n# Check multiple\nenv | grep -E \"OPEN_NOTEBOOK|API_URL\"\n\n# Print all config\nenv | grep -E \"^[A-Z_]+=\" | sort\n```\n\n---\n\n## Notes\n\n- **Case-sensitive:** `OPEN_NOTEBOOK_ENCRYPTION_KEY` ≠ `open_notebook_encryption_key`\n- **No spaces:** `OPEN_NOTEBOOK_ENCRYPTION_KEY=my-key` not `OPEN_NOTEBOOK_ENCRYPTION_KEY = my-key`\n- **Quote values:** Use quotes for values with spaces: `API_URL=\"http://my server:5055\"`\n- **Restart required:** Changes take effect after restarting services\n- **Secrets:** Don't commit encryption keys or passwords to git\n- **AI Providers:** Configure via **Settings → API Keys** in the browser (not via env vars)\n- **Migration:** Use Settings UI to migrate existing env vars to the credential system. See [API Configuration](../3-USER-GUIDE/api-configuration.md#migrating-from-environment-variables)\n\n---\n\n## Quick Setup Checklist\n\n- [ ] Set `OPEN_NOTEBOOK_ENCRYPTION_KEY` in docker-compose.yml\n- [ ] Set database credentials (`SURREAL_*`)\n- [ ] Start services\n- [ ] Open browser → Go to **Settings → API Keys**\n- [ ] **Add Credential** for your AI provider\n- [ ] **Test Connection** to verify\n- [ ] **Discover & Register Models**\n- [ ] Set `API_URL` if behind reverse proxy\n- [ ] Change `SURREAL_PASSWORD` in production\n- [ ] Try a test chat\n\nDone!\n\n---\n\n## Legacy: AI Provider Environment Variables (Deprecated)\n\n> **Deprecated**: The following AI provider API key environment variables are deprecated. Configure providers via the Settings UI instead. These variables may still work as a fallback but are no longer recommended.\n\nIf you have these variables configured from a previous installation, click the **Migrate to Database** button in **Settings → API Keys** to import them into the credential system, then remove them from your configuration.\n\n| Variable | Provider | Replacement |\n|----------|----------|-------------|\n| `OPENAI_API_KEY` | OpenAI | Settings → API Keys → Add OpenAI Credential |\n| `ANTHROPIC_API_KEY` | Anthropic | Settings → API Keys → Add Anthropic Credential |\n| `GOOGLE_API_KEY` | Google Gemini | Settings → API Keys → Add Google Credential |\n| `GEMINI_API_BASE_URL` | Google Gemini | Configure in Google Gemini credential |\n| `VERTEX_PROJECT` | Vertex AI | Settings → API Keys → Add Vertex AI Credential |\n| `VERTEX_LOCATION` | Vertex AI | Configure in Vertex AI credential |\n| `GOOGLE_APPLICATION_CREDENTIALS` | Vertex AI | Configure in Vertex AI credential |\n| `GROQ_API_KEY` | Groq | Settings → API Keys → Add Groq Credential |\n| `MISTRAL_API_KEY` | Mistral | Settings → API Keys → Add Mistral Credential |\n| `DEEPSEEK_API_KEY` | DeepSeek | Settings → API Keys → Add DeepSeek Credential |\n| `XAI_API_KEY` | xAI | Settings → API Keys → Add xAI Credential |\n| `OLLAMA_API_BASE` | Ollama | Settings → API Keys → Add Ollama Credential |\n| `OPENROUTER_API_KEY` | OpenRouter | Settings → API Keys → Add OpenRouter Credential |\n| `OPENROUTER_BASE_URL` | OpenRouter | Configure in OpenRouter credential |\n| `VOYAGE_API_KEY` | Voyage AI | Settings → API Keys → Add Voyage AI Credential |\n| `ELEVENLABS_API_KEY` | ElevenLabs | Settings → API Keys → Add ElevenLabs Credential |\n| `OPENAI_COMPATIBLE_BASE_URL` | OpenAI-Compatible | Settings → API Keys → Add OpenAI-Compatible Credential |\n| `OPENAI_COMPATIBLE_API_KEY` | OpenAI-Compatible | Configure in OpenAI-Compatible credential |\n| `OPENAI_COMPATIBLE_BASE_URL_LLM` | OpenAI-Compatible | Configure per-service URL in credential |\n| `OPENAI_COMPATIBLE_API_KEY_LLM` | OpenAI-Compatible | Configure per-service key in credential |\n| `OPENAI_COMPATIBLE_BASE_URL_EMBEDDING` | OpenAI-Compatible | Configure per-service URL in credential |\n| `OPENAI_COMPATIBLE_API_KEY_EMBEDDING` | OpenAI-Compatible | Configure per-service key in credential |\n| `OPENAI_COMPATIBLE_BASE_URL_STT` | OpenAI-Compatible | Configure per-service URL in credential |\n| `OPENAI_COMPATIBLE_API_KEY_STT` | OpenAI-Compatible | Configure per-service key in credential |\n| `OPENAI_COMPATIBLE_BASE_URL_TTS` | OpenAI-Compatible | Configure per-service URL in credential |\n| `OPENAI_COMPATIBLE_API_KEY_TTS` | OpenAI-Compatible | Configure per-service key in credential |\n| `AZURE_OPENAI_API_KEY` | Azure OpenAI | Settings → API Keys → Add Azure OpenAI Credential |\n| `AZURE_OPENAI_ENDPOINT` | Azure OpenAI | Configure in Azure OpenAI credential |\n| `AZURE_OPENAI_API_VERSION` | Azure OpenAI | Configure in Azure OpenAI credential |\n| `AZURE_OPENAI_API_KEY_LLM` | Azure OpenAI | Configure per-service in credential |\n| `AZURE_OPENAI_ENDPOINT_LLM` | Azure OpenAI | Configure per-service in credential |\n| `AZURE_OPENAI_API_VERSION_LLM` | Azure OpenAI | Configure per-service in credential |\n| `AZURE_OPENAI_API_KEY_EMBEDDING` | Azure OpenAI | Configure per-service in credential |\n| `AZURE_OPENAI_ENDPOINT_EMBEDDING` | Azure OpenAI | Configure per-service in credential |\n| `AZURE_OPENAI_API_VERSION_EMBEDDING` | Azure OpenAI | Configure per-service in credential |\n" }, { "path": "docs/5-CONFIGURATION/index.md", "content": "# Configuration - Essential Settings\n\nConfiguration is how you customize Open Notebook for your specific setup. This section covers what you need to know.\n\n---\n\n## What Needs Configuration?\n\nThree things:\n\n1. **AI Provider** — Which LLM/embedding service you're using (OpenAI, Anthropic, Ollama, etc.)\n2. **Database** — How to connect to SurrealDB (usually pre-configured)\n3. **Server** — API URL, ports, timeouts (usually auto-detected)\n\n---\n\n## Quick Decision: Which Provider?\n\n### Option 1: Cloud Provider (Fastest)\n- **OpenRouter (recommended)** (access to all models with one key)\n- **OpenAI** (GPT)\n- **Anthropic** (Claude)\n- **Google Gemini** (multi-modal, long context)\n- **Groq** (ultra-fast inference)\n\nSetup: Get API key → Add credential in Settings UI → Done\n\n→ Go to **[AI Providers Guide](ai-providers.md)**\n\n### Option 2: Local (Free & Private)\n- **Ollama** (open-source models, on your machine)\n\n→ Go to **[Ollama Setup](ollama.md)**\n\n### Option 3: OpenAI-Compatible\n- **LM Studio** (local)\n- **Custom endpoints**\n\n→ Go to **[OpenAI-Compatible Guide](openai-compatible.md)**\n\n---\n\n## Configuration File\n\nUse the right file depending on your setup.\n\n### `.env` (Local Development)\n\nYou will only use .env if you are running Open Notebook locally.\n\n```\nLocated in: project root\nUse for: Development on your machine\nFormat: KEY=value, one per line\n```\n\n### `docker.env` (Docker Deployment)\n\nYou will use this file to hold your environment variables if you are using docker-compose and prefer not to put the variables directly in the compose file. \n```\nLocated in: project root (or ./docker)\nUse for: Docker deployments\nFormat: Same as .env\nLoaded by: docker-compose.yml\n```\n\n---\n\n## Most Important Settings\n\nAll of the settings provided below are to be placed inside your environment file (.env or docker.env depending on your setup).\n\n\n### Surreal Database\n\nThis is the database used by the app.\n\n```\nSURREAL_URL=ws://surrealdb:8000/rpc\nSURREAL_USER=root\nSURREAL_PASSWORD=root # Change in production!\nSURREAL_NAMESPACE=open_notebook\nSURREAL_DATABASE=open_notebook\n```\n\n> The only thing that is critical to not miss is the hostname in the `SURREAL_URL`. Check what URL to use based on your deployment, [here](database.md).\n\n\n### AI Provider (Credentials)\n\nWe need access to LLMs in order for the app to work. AI provider credentials are configured via the **Settings UI**:\n\n1. Set `OPEN_NOTEBOOK_ENCRYPTION_KEY` in your environment (required for storing credentials)\n2. Start services\n3. Go to **Settings → API Keys → Add Credential**\n4. Select your provider, paste your API key\n5. **Test Connection** → **Discover Models** → **Register Models**\n\n```\n# Required in your .env or docker-compose.yml:\nOPEN_NOTEBOOK_ENCRYPTION_KEY=my-secret-key\n```\n\n> **Ollama users**: Add an Ollama credential in Settings → API Keys with the correct base URL. See [Ollama Setup](ollama.md) for network configuration help.\n\n> **LM Studio / OpenAI-Compatible**: Add an OpenAI-Compatible credential in Settings → API Keys. See [OpenAI-Compatible Guide](openai-compatible.md).\n\n\n### API URL (If Behind Reverse Proxy)\nYou only need to worry about this if you are deploying on a proxy or if you are changing port information. Otherwise, skip this.\n\n```\nAPI_URL=https://your-domain.com\n# Usually auto-detected. Only set if needed.\n```\n\nAuto-detection works for most setups.\n\n---\n\n## Configuration by Scenario\n\n### Scenario 1: Docker on Localhost (Default)\n```env\n# In docker.env:\nOPEN_NOTEBOOK_ENCRYPTION_KEY=my-secret-key\n# Everything else uses defaults\n# Then configure AI provider in Settings → API Keys\n```\n\n### Scenario 2: Docker on Remote Server\n```env\n# In docker.env:\nOPEN_NOTEBOOK_ENCRYPTION_KEY=my-secret-key\nAPI_URL=http://your-server-ip:5055\n```\n\n### Scenario 3: Behind Reverse Proxy (Nginx/Cloudflare)\n```env\n# In docker.env:\nOPEN_NOTEBOOK_ENCRYPTION_KEY=my-secret-key\nAPI_URL=https://your-domain.com\n# The reverse proxy handles HTTPS\n```\n\n### Scenario 4: Using Ollama Locally\n```env\n# In .env:\nOPEN_NOTEBOOK_ENCRYPTION_KEY=my-secret-key\n# Then add Ollama credential in Settings → API Keys\n```\n\n### Scenario 5: Using Azure OpenAI\n```env\n# In docker.env:\nOPEN_NOTEBOOK_ENCRYPTION_KEY=my-secret-key\n# Then add Azure OpenAI credential in Settings → API Keys\n```\n\n---\n\n## Configuration Sections\n\n### [AI Providers](ai-providers.md)\n- OpenAI configuration\n- Anthropic configuration\n- Google Gemini configuration\n- Groq configuration\n- Ollama configuration\n- Azure OpenAI configuration\n- OpenAI-compatible configuration\n\n### [Database](database.md)\n- SurrealDB setup\n- Connection strings\n- Database vs. namespace\n- Running your own SurrealDB\n\n### [Advanced](advanced.md)\n- Ports and networking\n- Timeouts and concurrency\n- SSL/security\n- Retry configuration\n- Worker concurrency\n- Language models & embeddings\n- Speech-to-text & text-to-speech\n- Debugging and logging\n\n### [Reverse Proxy](reverse-proxy.md)\n- Nginx, Caddy, Traefik configs\n- Custom domain setup\n- SSL/HTTPS configuration\n- Coolify and other platforms\n\n### [Security](security.md)\n- Password protection\n- API authentication\n- Production hardening\n- Firewall configuration\n\n### [Local TTS](local-tts.md)\n- Speaches setup for local text-to-speech\n- GPU acceleration\n- Voice options\n- Docker networking\n\n### [Local STT](local-stt.md)\n- Speaches setup for local speech-to-text\n- Whisper model options\n- GPU acceleration\n- Docker networking\n\n### [Ollama](ollama.md)\n- Setting up and pointing to an Ollama server\n- Downloading models\n- Using embedding\n\n### [OpenAI-Compatible Providers](openai-compatible.md)\n- LM Studio, vLLM, Text Generation WebUI\n- Connection configuration\n- Docker networking\n- Troubleshooting\n\n### [Complete Reference](environment-reference.md)\n- All environment variables\n- Grouped by category\n- What each one does\n- Default values\n\n---\n\n## How to Add Configuration\n\n### Method 1: Settings UI (For AI Provider Credentials)\n\nThe recommended way to configure AI providers:\n\n```\n1. Open Open Notebook in your browser\n2. Go to Settings → API Keys\n3. Click \"Add Credential\"\n4. Select provider, enter API key\n5. Click Save, then Test Connection\n6. Click Discover Models → Register Models\n```\n\nNo file editing, no restarts. Credentials stored securely (encrypted) in database.\n\n→ **[Full Guide: API Configuration](../3-USER-GUIDE/api-configuration.md)**\n\n### Method 2: Edit `.env` File (Infrastructure Settings)\n\nFor database, network, and encryption key settings:\n\n```bash\n1. Open .env in your editor\n2. Set OPEN_NOTEBOOK_ENCRYPTION_KEY and database vars\n3. Save\n4. Restart services\n```\n\n### Method 3: Set Docker Environment (Deployment)\n\n```bash\n# In docker-compose.yml:\nservices:\n api:\n environment:\n - OPEN_NOTEBOOK_ENCRYPTION_KEY=my-secret-key\n - API_URL=https://your-domain.com\n```\n\n---\n\n## Verification\n\nAfter configuration, verify it works:\n\n```\n1. Open your notebook\n2. Go to Settings → Models\n3. You should see your configured provider\n4. Try a simple Chat question\n5. If it responds, configuration is correct!\n```\n\n---\n\n## Common Mistakes\n\n| Mistake | Problem | Fix |\n|---------|---------|-----|\n| No credential configured | Models not available | Add credential in Settings → API Keys |\n| Missing encryption key | Can't save credentials | Set OPEN_NOTEBOOK_ENCRYPTION_KEY |\n| Wrong database URL | Can't start API | Check SURREAL_URL format |\n| Expose port 5055 | \"Can't connect to server\" | Expose 5055 in docker-compose |\n| Typo in env var | Settings ignored | Check spelling (case-sensitive!) |\n| Don't restart | Old config still used | Restart services after env changes |\n\n---\n\n## What Comes After Configuration\n\nOnce configured:\n\n1. **[Quick Start](../0-START-HERE/index.md)** — Run your first notebook\n2. **[Installation](../1-INSTALLATION/index.md)** — Multi-route deployment guides\n3. **[User Guide](../3-USER-GUIDE/index.md)** — How to use each feature\n\n---\n\n## Getting Help\n\n- **Configuration error?** → Check [Troubleshooting](../6-TROUBLESHOOTING/quick-fixes.md)\n- **Provider-specific issue?** → Check [AI Providers](ai-providers.md)\n- **Need complete reference?** → See [Environment Reference](environment-reference.md)\n\n---\n\n## Summary\n\n**Minimal configuration to run:**\n1. Set `OPEN_NOTEBOOK_ENCRYPTION_KEY` in your environment\n2. Start services\n3. Add AI provider credential in Settings → API Keys\n4. Done!\n\nEverything else is optional optimization.\n" }, { "path": "docs/5-CONFIGURATION/local-stt.md", "content": "# Local Speech-to-Text Setup\n\nRun speech-to-text locally for free, private audio/video transcription using OpenAI-compatible STT servers.\n\n---\n\n## Why Local STT?\n\n| Benefit | Description |\n|---------|-------------|\n| **Free** | No per-minute costs after setup |\n| **Private** | Audio never leaves your machine |\n| **Unlimited** | No rate limits or quotas |\n| **Offline** | Works without internet |\n\n---\n\n## Quick Start with Speaches\n\n[Speaches](https://github.com/speaches-ai/speaches) is an open-source, OpenAI-compatible server that supports both TTS and STT. It uses [faster-whisper](https://github.com/SYSTRAN/faster-whisper) for transcription.\n\n> **💡 Ready-made Docker Compose files available:**\n> - **[docker-compose-speaches.yml](../../examples/docker-compose-speaches.yml)** - Speaches + Open Notebook\n> - **[docker-compose-full-local.yml](../../examples/docker-compose-full-local.yml)** - Speaches + Ollama (100% local setup)\n>\n> These include complete setup instructions and configuration examples. Just copy and run!\n\n### Step 1: Create Docker Compose File\n\nCreate a folder and add `docker-compose.yml`:\n\n```yaml\nservices:\n speaches:\n image: ghcr.io/speaches-ai/speaches:latest-cpu\n container_name: speaches\n ports:\n - \"8969:8000\"\n volumes:\n - hf-hub-cache:/home/ubuntu/.cache/huggingface/hub\n restart: unless-stopped\n\nvolumes:\n hf-hub-cache:\n```\n\n### Step 2: Start and Download Model\n\n```bash\n# Start Speaches\ndocker compose up -d\n\n# Wait for startup\nsleep 10\n\n# Download Whisper model (~500MB for small)\ndocker compose exec speaches uv tool run speaches-cli model download Systran/faster-whisper-small\n```\n\nModels can also be downloaded automatically on first use, but pre-downloading avoids delays.\n\n### Step 3: Test\n\n```bash\n# Create a test audio file (or use your own)\n# Then transcribe it:\ncurl \"http://localhost:8969/v1/audio/transcriptions\" \\\n -F \"file=@test.mp3\" \\\n -F \"model=Systran/faster-whisper-small\"\n```\n\nYou should see the transcribed text in the response.\n\n### Step 4: Configure Open Notebook\n\n**Via Settings UI (Recommended):**\n1. Go to **Settings** → **API Keys**\n2. Click **Add Credential** → Select **OpenAI-Compatible**\n3. Enter base URL for STT: `http://host.docker.internal:8969/v1` (Docker) or `http://localhost:8969/v1` (local)\n4. Click **Save**, then **Test Connection**\n\n**Legacy (Deprecated) — Environment variables:**\n```yaml\n# In your Open Notebook docker-compose.yml\nenvironment:\n - OPENAI_COMPATIBLE_BASE_URL_STT=http://host.docker.internal:8969/v1\n```\n\n```bash\n# Local development\nexport OPENAI_COMPATIBLE_BASE_URL_STT=http://localhost:8969/v1\n```\n\n### Step 5: Add Model in Open Notebook\n\n1. Go to **Settings** → **Models**\n2. Click **Add Model** in Speech-to-Text section\n3. Configure:\n - **Provider**: `openai_compatible`\n - **Model Name**: `Systran/faster-whisper-small`\n - **Display Name**: `Local Whisper`\n4. Click **Save**\n5. Set as default if desired\n\n---\n\n## Available Models\n\nSpeaches supports various Whisper model sizes. Larger models are more accurate but slower:\n\n| Model | Size | Speed | Accuracy | VRAM (GPU) |\n|-------|------|-------|----------|------------|\n| `Systran/faster-whisper-tiny` | ~75 MB | Fastest | Basic | ~1 GB |\n| `Systran/faster-whisper-base` | ~150 MB | Fast | Good | ~1 GB |\n| `Systran/faster-whisper-small` | ~500 MB | Medium | Better | ~2 GB |\n| `Systran/faster-whisper-medium` | ~1.5 GB | Slow | Great | ~5 GB |\n| `Systran/faster-whisper-large-v3` | ~3 GB | Slowest | Best | ~10 GB |\n| `Systran/faster-distil-whisper-small.en` | ~400 MB | Fast | Good (English only) | ~2 GB |\n\n### List Available Models\n\n```bash\ndocker compose exec speaches uv tool run speaches-cli registry ls --task automatic-speech-recognition\n```\n\n### Recommended Models\n\n- **For speed**: `Systran/faster-whisper-tiny` or `Systran/faster-whisper-base`\n- **For balance**: `Systran/faster-whisper-small` (recommended)\n- **For accuracy**: `Systran/faster-whisper-large-v3`\n\n---\n\n## GPU Acceleration\n\nFor faster transcription with NVIDIA GPUs:\n\n```yaml\nservices:\n speaches:\n image: ghcr.io/speaches-ai/speaches:latest-cuda\n container_name: speaches\n ports:\n - \"8969:8000\"\n volumes:\n - hf-hub-cache:/home/ubuntu/.cache/huggingface/hub\n environment:\n - WHISPER__TTL=-1 # Keep model in VRAM (recommended if you have enough memory)\n restart: unless-stopped\n deploy:\n resources:\n reservations:\n devices:\n - driver: nvidia\n count: 1\n capabilities: [gpu]\n\nvolumes:\n hf-hub-cache:\n```\n\n### Keep Model in Memory\n\nBy default, Speaches unloads models after some time. To keep the Whisper model loaded for instant transcription:\n\n```yaml\nenvironment:\n - WHISPER__TTL=-1 # Never unload\n```\n\nThis is recommended if you have enough RAM/VRAM, as loading the model can take a few seconds.\n\n---\n\n## Docker Networking\n\nWhen configuring your OpenAI-Compatible credential in **Settings → API Keys**, use the appropriate STT base URL for your setup:\n\n### Open Notebook in Docker (macOS/Windows)\n\n**STT Base URL:** `http://host.docker.internal:8969/v1`\n\n### Open Notebook in Docker (Linux)\n\n**STT Base URL (Option 1 — Docker bridge IP):** `http://172.17.0.1:8969/v1`\n\n**Option 2:** Use host networking mode (`docker run --network host ...`), then use: `http://localhost:8969/v1`\n\n### Remote Server\n\nRun Speaches on a different machine:\n\n**STT Base URL:** `http://server-ip:8969/v1` (replace with your server's IP)\n\n---\n\n## Language Support\n\nWhisper supports 99+ languages. Specify the language for better accuracy:\n\n```bash\ncurl \"http://localhost:8969/v1/audio/transcriptions\" \\\n -F \"file=@audio.mp3\" \\\n -F \"model=Systran/faster-whisper-small\" \\\n -F \"language=ru\"\n```\n\nCommon language codes:\n- `en` - English\n- `ru` - Russian\n- `es` - Spanish\n- `fr` - French\n- `de` - German\n- `zh` - Chinese\n- `ja` - Japanese\n\n---\n\n## Troubleshooting\n\n### Service Won't Start\n\n```bash\n# Check logs\ndocker compose logs speaches\n\n# Verify port available\nlsof -i :8969\n\n# Restart\ndocker compose down && docker compose up -d\n```\n\n### Connection Refused\n\n```bash\n# Test Speaches is running\ncurl http://localhost:8969/v1/models\n\n# From inside Open Notebook container\ndocker exec -it open-notebook curl http://host.docker.internal:8969/v1/models\n```\n\n### Model Download Fails\n\nModels are downloaded automatically on first use. If download fails:\n\n```bash\n# Check available disk space\ndf -h\n\n# Check Docker logs for errors\ndocker compose logs speaches\n\n# Restart and try again\ndocker compose restart speaches\n```\n\n### Poor Transcription Quality\n\n- Use a larger model (`faster-whisper-medium` or `large-v3`)\n- Specify the correct language\n- Ensure audio quality is good (clear speech, minimal background noise)\n- Try different audio formats (WAV often works better than MP3)\n\n### Slow Transcription\n\n| Solution | How |\n|----------|-----|\n| Use GPU | Switch to `latest-cuda` image |\n| Smaller model | Use `faster-whisper-tiny` or `base` |\n| More CPU | Allocate more cores in Docker |\n| SSD storage | Move Docker volumes to SSD |\n\n---\n\n## Performance Tips\n\n### Recommended Specs\n\n| Component | Minimum | Recommended |\n|-----------|---------|-------------|\n| CPU | 2 cores | 4+ cores |\n| RAM | 2 GB | 8+ GB |\n| Storage | 5 GB | 10 GB (for multiple models) |\n| GPU | None | NVIDIA (optional, much faster) |\n\n### Resource Limits\n\n```yaml\nservices:\n speaches:\n # ... other config\n mem_limit: 4g\n cpus: 2\n```\n\n### Monitor Usage\n\n```bash\ndocker stats speaches\n```\n\n---\n\n## Comparison: Local vs Cloud\n\n| Aspect | Local (Speaches) | Cloud (OpenAI Whisper) |\n|--------|------------------|------------------------|\n| **Cost** | Free | $0.006/min |\n| **Privacy** | Complete | Data sent to provider |\n| **Speed** | Depends on hardware | Usually faster |\n| **Quality** | Excellent (same Whisper) | Excellent |\n| **Setup** | Moderate | Simple API key |\n| **Offline** | Yes | No |\n| **Languages** | 99+ | 99+ |\n\n### When to Use Local\n\n- Privacy-sensitive content\n- High-volume transcription\n- Development/testing\n- Offline environments\n- Cost control\n\n### When to Use Cloud\n\n- Limited hardware\n- Time-sensitive projects\n- No GPU available\n- Simple setup preferred\n\n---\n\n## Using Both TTS and STT\n\nSpeaches supports both TTS and STT in one server. In **Settings → API Keys**, add a single **OpenAI-Compatible** credential and configure both the TTS and STT base URLs to point to the same Speaches server (e.g., `http://localhost:8969/v1`).\n\nSee **[Local TTS Setup](local-tts.md)** for TTS configuration.\n\n---\n\n## Other Local STT Options\n\nAny OpenAI-compatible STT server works:\n\n| Server | Description |\n|--------|-------------|\n| [Speaches](https://github.com/speaches-ai/speaches) | TTS + STT in one (recommended) |\n| [faster-whisper-server](https://github.com/fedirz/faster-whisper-server) | Lightweight STT only |\n| [whisper.cpp](https://github.com/ggerganov/whisper.cpp) | C++ implementation with server mode |\n| [LocalAI](https://github.com/mudler/LocalAI) | Multi-model local AI server |\n\nThe key requirements:\n\n1. Server implements `/v1/audio/transcriptions` endpoint\n2. Add an OpenAI-Compatible credential in **Settings → API Keys** with the STT base URL\n3. Add model with provider `openai_compatible`\n\n---\n\n## Related\n\n- **[Local TTS Setup](local-tts.md)** - Text-to-speech with Speaches\n- **[OpenAI-Compatible Providers](openai-compatible.md)** - General compatible provider setup\n- **[AI Providers](ai-providers.md)** - All provider configuration" }, { "path": "docs/5-CONFIGURATION/local-tts.md", "content": "# Local Text-to-Speech Setup\n\nRun text-to-speech locally for free, private podcast generation using OpenAI-compatible TTS servers.\n\n---\n\n## Why Local TTS?\n\n| Benefit | Description |\n|---------|-------------|\n| **Free** | No per-character costs after setup |\n| **Private** | Audio never leaves your machine |\n| **Unlimited** | No rate limits or quotas |\n| **Offline** | Works without internet |\n\n---\n\n## Quick Start with Speaches\n\n[Speaches](https://github.com/speaches-ai/speaches) is an open-source, OpenAI-compatible TTS server.\n\n> **💡 Ready-made Docker Compose files available:**\n> - **[docker-compose-speaches.yml](../../examples/docker-compose-speaches.yml)** - Speaches + Open Notebook\n> - **[docker-compose-full-local.yml](../../examples/docker-compose-full-local.yml)** - Speaches + Ollama (100% local setup)\n>\n> These include complete setup instructions and configuration examples. Just copy and run!\n\n### Step 1: Create Docker Compose File\n\nCreate a folder and add `docker-compose.yml`:\n\n```yaml\nservices:\n speaches:\n image: ghcr.io/speaches-ai/speaches:latest-cpu\n container_name: speaches\n ports:\n - \"8969:8000\"\n volumes:\n - hf-hub-cache:/home/ubuntu/.cache/huggingface/hub\n restart: unless-stopped\n\nvolumes:\n hf-hub-cache:\n```\n\n### Step 2: Start and Download Model\n\n```bash\n# Start Speaches\ndocker compose up -d\n\n# Wait for startup\nsleep 10\n\n# Download voice model (~500MB)\ndocker compose exec speaches uv tool run speaches-cli model download speaches-ai/Kokoro-82M-v1.0-ONNX\n```\n\n### Step 3: Test\n\n```bash\ncurl \"http://localhost:8969/v1/audio/speech\" -s \\\n -H \"Content-Type: application/json\" \\\n --output test.mp3 \\\n --data '{\n \"input\": \"Hello! Local TTS is working.\",\n \"model\": \"speaches-ai/Kokoro-82M-v1.0-ONNX\",\n \"voice\": \"af_bella\"\n }'\n```\n\nPlay `test.mp3` to verify.\n\n### Step 4: Configure Open Notebook\n\n**Via Settings UI (Recommended):**\n1. Go to **Settings** → **API Keys**\n2. Click **Add Credential** → Select **OpenAI-Compatible**\n3. Enter base URL for TTS: `http://host.docker.internal:8969/v1` (Docker) or `http://localhost:8969/v1` (local)\n4. Click **Save**, then **Test Connection**\n\n**Legacy (Deprecated) — Environment variables:**\n```yaml\n# In your Open Notebook docker-compose.yml\nenvironment:\n - OPENAI_COMPATIBLE_BASE_URL_TTS=http://host.docker.internal:8969/v1\n```\n\n```bash\n# Local development\nexport OPENAI_COMPATIBLE_BASE_URL_TTS=http://localhost:8969/v1\n```\n\n### Step 5: Add Model in Open Notebook\n\n1. Go to **Settings** → **Models**\n2. Click **Add Model** in Text-to-Speech section\n3. Configure:\n - **Provider**: `openai_compatible`\n - **Model Name**: `speaches-ai/Kokoro-82M-v1.0-ONNX`\n - **Display Name**: `Local TTS`\n4. Click **Save**\n5. Set as default if desired\n\n---\n\n## Available Voices\n\nThe Kokoro model includes multiple voices:\n\n### Female Voices\n| Voice ID | Description |\n|----------|-------------|\n| `af_bella` | Clear, professional |\n| `af_sarah` | Warm, friendly |\n| `af_nicole` | Energetic, expressive |\n\n### Male Voices\n| Voice ID | Description |\n|----------|-------------|\n| `am_adam` | Deep, authoritative |\n| `am_michael` | Friendly, conversational |\n\n### British Accents\n| Voice ID | Description |\n|----------|-------------|\n| `bf_emma` | British female, professional |\n| `bm_george` | British male, formal |\n\n### Test Different Voices\n\n```bash\nfor voice in af_bella af_sarah am_adam am_michael; do\n curl \"http://localhost:8969/v1/audio/speech\" -s \\\n -H \"Content-Type: application/json\" \\\n --output \"test_${voice}.mp3\" \\\n --data \"{\n \\\"input\\\": \\\"Hello, this is the ${voice} voice.\\\",\n \\\"model\\\": \\\"speaches-ai/Kokoro-82M-v1.0-ONNX\\\",\n \\\"voice\\\": \\\"${voice}\\\"\n }\"\ndone\n```\n\n---\n\n## GPU Acceleration\n\nFor faster generation with NVIDIA GPUs:\n\n```yaml\nservices:\n speaches:\n image: ghcr.io/speaches-ai/speaches:latest-cuda\n container_name: speaches\n ports:\n - \"8969:8000\"\n volumes:\n - hf-hub-cache:/home/ubuntu/.cache/huggingface/hub\n restart: unless-stopped\n deploy:\n resources:\n reservations:\n devices:\n - driver: nvidia\n count: 1\n capabilities: [gpu]\n\nvolumes:\n hf-hub-cache:\n```\n\n---\n\n## Docker Networking\n\nWhen configuring your OpenAI-Compatible credential in **Settings → API Keys**, use the appropriate TTS base URL for your setup:\n\n### Open Notebook in Docker (macOS/Windows)\n\n**TTS Base URL:** `http://host.docker.internal:8969/v1`\n\n### Open Notebook in Docker (Linux)\n\n**TTS Base URL (Option 1 — Docker bridge IP):** `http://172.17.0.1:8969/v1`\n\n**Option 2:** Use host networking mode (`docker run --network host ...`), then use: `http://localhost:8969/v1`\n\n### Remote Server\n\nRun Speaches on a different machine:\n\n**TTS Base URL:** `http://server-ip:8969/v1` (replace with your server's IP)\n\n---\n\n## Multi-Speaker Podcasts\n\nConfigure different voices for each speaker:\n\n```\nSpeaker 1 (Host):\n Model: speaches-ai/Kokoro-82M-v1.0-ONNX\n Voice: af_bella\n\nSpeaker 2 (Guest):\n Model: speaches-ai/Kokoro-82M-v1.0-ONNX\n Voice: am_adam\n\nSpeaker 3 (Narrator):\n Model: speaches-ai/Kokoro-82M-v1.0-ONNX\n Voice: bf_emma\n```\n\n---\n\n## Troubleshooting\n\n### Service Won't Start\n\n```bash\n# Check logs\ndocker compose logs speaches\n\n# Verify port available\nlsof -i :8969\n\n# Restart\ndocker compose down && docker compose up -d\n```\n\n### Connection Refused\n\n```bash\n# Test Speaches is running\ncurl http://localhost:8969/v1/models\n\n# From inside Open Notebook container\ndocker exec -it open-notebook curl http://host.docker.internal:8969/v1/models\n```\n\n### Model Not Found\n\n```bash\n# List downloaded models\ndocker compose exec speaches uv tool run speaches-cli model list\n\n# Download if missing\ndocker compose exec speaches uv tool run speaches-cli model download speaches-ai/Kokoro-82M-v1.0-ONNX\n```\n\n### Poor Audio Quality\n\n- Try different voices\n- Adjust speed: `\"speed\": 0.9` to `1.2`\n- Check model downloaded completely\n- Allocate more memory\n\n### Slow Generation\n\n| Solution | How |\n|----------|-----|\n| Use GPU | Switch to `latest-cuda` image |\n| More CPU | Allocate more cores in Docker |\n| Faster model | Use smaller/quantized models |\n| SSD storage | Move Docker volumes to SSD |\n\n---\n\n## Performance Tips\n\n### Recommended Specs\n\n| Component | Minimum | Recommended |\n|-----------|---------|-------------|\n| CPU | 2 cores | 4+ cores |\n| RAM | 2 GB | 4+ GB |\n| Storage | 5 GB | 10 GB (for multiple models) |\n| GPU | None | NVIDIA (optional) |\n\n### Resource Limits\n\n```yaml\nservices:\n speaches:\n # ... other config\n mem_limit: 4g\n cpus: 2\n```\n\n### Monitor Usage\n\n```bash\ndocker stats speaches\n```\n\n---\n\n## Comparison: Local vs Cloud\n\n| Aspect | Local (Speaches) | Cloud (OpenAI/ElevenLabs) |\n|--------|------------------|---------------------------|\n| **Cost** | Free | $0.015-0.10/min |\n| **Privacy** | Complete | Data sent to provider |\n| **Speed** | Depends on hardware | Usually faster |\n| **Quality** | Good | Excellent |\n| **Setup** | Moderate | Simple API key |\n| **Offline** | Yes | No |\n| **Voices** | Limited | Many options |\n\n### When to Use Local\n\n- Privacy-sensitive content\n- High-volume generation\n- Development/testing\n- Offline environments\n- Cost control\n\n### When to Use Cloud\n\n- Premium quality needs\n- Multiple languages\n- Time-sensitive projects\n- Limited hardware\n\n---\n\n## Other Local TTS Options\n\nAny OpenAI-compatible TTS server works. The key is:\n\n1. Server implements `/v1/audio/speech` endpoint\n2. Add an OpenAI-Compatible credential in **Settings → API Keys** with the TTS base URL\n3. Add model with provider `openai_compatible`\n\n---\n\n## Related\n\n- **[Local STT Setup](local-stt.md)** - Speech-to-text with Speaches\n- **[OpenAI-Compatible Providers](openai-compatible.md)** - General compatible provider setup\n- **[AI Providers](ai-providers.md)** - All provider configuration\n- **[Creating Podcasts](../3-USER-GUIDE/creating-podcasts.md)** - Using TTS for podcasts\n" }, { "path": "docs/5-CONFIGURATION/mcp-integration.md", "content": "# Model Context Protocol (MCP) Integration\n\nOpen Notebook can be seamlessly integrated into your AI workflows using the **Model Context Protocol (MCP)**, enabling direct access to your notebooks, sources, and chat functionality from AI assistants like Claude Desktop and VS Code extensions.\n\n## What is MCP?\n\nThe [Model Context Protocol](https://modelcontextprotocol.io) is an open standard that allows AI applications to securely connect to external data sources and tools. With the Open Notebook MCP server, you can:\n\n- 📚 **Access your notebooks** directly from Claude Desktop or VS Code\n- 🔍 **Search your research content** without leaving your AI assistant\n- 💬 **Create and manage chat sessions** with your research as context\n- 📝 **Generate notes** and insights on-the-fly\n- 🤖 **Automate workflows** using the full Open Notebook API\n\n## Quick Setup\n\n### For Claude Desktop\n\n1. **Install the MCP server** (automatically from PyPI):\n\n ```bash\n # No manual installation needed! Claude Desktop will use uvx to run it automatically\n ```\n\n2. **Configure Claude Desktop**:\n\n **macOS/Linux**: Edit `~/Library/Application Support/Claude/claude_desktop_config.json`\n\n ```json\n {\n \"mcpServers\": {\n \"open-notebook\": {\n \"command\": \"uvx\",\n \"args\": [\"open-notebook-mcp\"],\n \"env\": {\n \"OPEN_NOTEBOOK_URL\": \"http://localhost:5055\",\n \"OPEN_NOTEBOOK_PASSWORD\": \"your_password_here\"\n }\n }\n }\n }\n ```\n\n **Windows**: Edit `%APPDATA%\\Claude\\claude_desktop_config.json`\n\n ```json\n {\n \"mcpServers\": {\n \"open-notebook\": {\n \"command\": \"uvx\",\n \"args\": [\"open-notebook-mcp\"],\n \"env\": {\n \"OPEN_NOTEBOOK_URL\": \"http://localhost:5055\",\n \"OPEN_NOTEBOOK_PASSWORD\": \"your_password_here\"\n }\n }\n }\n }\n ```\n\n3. **Restart Claude Desktop** and start using your notebooks in conversations!\n\n### For VS Code (Cline and other MCP-compatible extensions)\n\nAdd to your VS Code settings or `.vscode/mcp.json`:\n\n```json\n{\n \"servers\": {\n \"open-notebook\": {\n \"command\": \"uvx\",\n \"args\": [\"open-notebook-mcp\"],\n \"env\": {\n \"OPEN_NOTEBOOK_URL\": \"http://localhost:5055\",\n \"OPEN_NOTEBOOK_PASSWORD\": \"your_password_here\"\n }\n }\n }\n}\n```\n\n## Configuration\n\n- **OPEN_NOTEBOOK_URL**: URL to your Open Notebook API (default: `http://localhost:5055`)\n- **OPEN_NOTEBOOK_PASSWORD**: Optional - only needed if you've enabled password protection\n\n### For Remote Servers\n\nIf your Open Notebook instance is running on a remote server, update the URL accordingly:\n\n```json\n\"OPEN_NOTEBOOK_URL\": \"http://192.168.1.100:5055\"\n```\n\nOr with a domain:\n\n```json\n\"OPEN_NOTEBOOK_URL\": \"https://notebook.yourdomain.com/api\"\n```\n\n## What You Can Do\n\nOnce connected, you can ask Claude or your AI assistant to:\n\n- _\"Search my research notebooks for information about [topic]\"_\n- _\"Create a new note summarizing the key points from our conversation\"_\n- _\"List all my notebooks\"_\n- _\"Start a chat session about [specific source or topic]\"_\n- _\"What sources do I have in my [notebook name] notebook?\"_\n- _\"Add this PDF to my research notebook\"_\n- _\"Show me all notes in [notebook name]\"_\n\nThe MCP server provides full access to Open Notebook's capabilities, allowing you to manage your research seamlessly from within your AI assistant.\n\n## Available Tools\n\nThe Open Notebook MCP server exposes these capabilities:\n\n### Notebooks\n\n- List notebooks\n- Get notebook details\n- Create new notebooks\n- Update notebook information\n- Delete notebooks\n\n### Sources\n\n- List sources in a notebook\n- Get source details\n- Add new sources (links, files, text)\n- Update source metadata\n- Delete sources\n\n### Notes\n\n- List notes in a notebook\n- Get note details\n- Create new notes\n- Update notes\n- Delete notes\n\n### Chat\n\n- Create chat sessions\n- Send messages to chat sessions\n- Get chat history\n- List chat sessions\n\n### Search\n\n- Vector search across content\n- Text search across content\n- Filter by notebook\n\n### Models\n\n- List configured AI models\n- Get model details\n- Create model configurations\n- Update model settings\n\n### Settings\n\n- Get application settings\n- Update settings\n\n## MCP Server Repository\n\nThe Open Notebook MCP server is developed and maintained by the Epochal team:\n\n**🔗 GitHub**: [Epochal-dev/open-notebook-mcp](https://github.com/Epochal-dev/open-notebook-mcp)\n\nContributions, issues, and feature requests are welcome!\n\n## Finding the Server\n\nThe Open Notebook MCP server is published to the official MCP Registry:\n\n- **Registry**: Search for \"open-notebook\" at [registry.modelcontextprotocol.io](https://registry.modelcontextprotocol.io)\n- **PyPI**: [pypi.org/project/open-notebook-mcp](https://pypi.org/project/open-notebook-mcp)\n- **GitHub**: [Epochal-dev/open-notebook-mcp](https://github.com/Epochal-dev/open-notebook-mcp)\n\n## Troubleshooting\n\n### Connection Errors\n\n1. Verify the `OPEN_NOTEBOOK_URL` is correct and accessible\n2. If using password protection, ensure `OPEN_NOTEBOOK_PASSWORD` is set correctly\n3. For remote servers, make sure port 5055 is accessible from your machine\n4. Check firewall settings if connecting to a remote server\n\n## Using with Other MCP Clients\n\nThe Open Notebook MCP server follows the standard MCP protocol and can be used with any MCP-compatible client. Check your client's documentation for configuration details.\n\n## Learn More\n\n- [Model Context Protocol Documentation](https://modelcontextprotocol.io)\n" }, { "path": "docs/5-CONFIGURATION/ollama.md", "content": "# Ollama Setup Guide\n\nOllama provides free, local AI models that run on your own hardware. This guide covers everything you need to know about setting up Ollama with Open Notebook, including different deployment scenarios and network configurations.\n\n## Why Choose Ollama?\n\n- **🆓 Completely Free**: No API costs after initial setup\n- **🔒 Full Privacy**: Your data never leaves your local network\n- **📱 Offline Capable**: Works without internet connection\n- **🚀 Fast**: Local inference with no network latency\n- **🧠 Reasoning Models**: Support for advanced reasoning models like DeepSeek-R1\n- **💾 Model Variety**: Access to hundreds of open-source models\n\n## Quick Start\n\n### 1. Install Ollama\n\n**Linux/macOS:**\n```bash\ncurl -fsSL https://ollama.ai/install.sh | sh\n```\n\n**Windows:**\nDownload and install from [ollama.ai](https://ollama.ai/download)\n\n### 2. Pull Required Models\n\n```bash\n# Language models (choose one or more)\nollama pull qwen3 # Excellent general purpose, 7B parameters\nollama pull gemma3 # Google's model, good performance\nollama pull deepseek-r1 # Advanced reasoning model\nollama pull phi4 # Microsoft's efficient model\n\n# Embedding model (required for search)\nollama pull mxbai-embed-large # Best embedding model for Ollama\n```\n\n### 3. Configure Open Notebook\n\n**Via Settings UI (Recommended):**\n1. Go to **Settings** → **API Keys**\n2. Click **Add Credential** → Select **Ollama**\n3. Enter the base URL (see [Network Configuration](#network-configuration-guide) below for correct URL)\n4. Click **Save**, then **Test Connection**\n5. Click **Discover Models** → **Register Models**\n\n**Legacy (Deprecated) — Environment variables:**\n```bash\n# For local installation:\nexport OLLAMA_API_BASE=http://localhost:11434\n# For Docker installation:\nexport OLLAMA_API_BASE=http://host.docker.internal:11434\n```\n\n> **Note**: The `OLLAMA_API_BASE` environment variable is deprecated. Configure Ollama via Settings → API Keys instead.\n\n## Network Configuration Guide\n\nWhen adding an Ollama credential in **Settings → API Keys**, you need to enter the correct base URL. The correct URL depends on your deployment scenario:\n\n### Scenario 1: Local Installation (Same Machine)\n\nWhen both Open Notebook and Ollama run directly on your machine:\n\n**Base URL to enter in Settings → API Keys:** `http://localhost:11434`\n\nAlternative: `http://127.0.0.1:11434` (use if you have DNS resolution issues with localhost)\n\n### Scenario 2: Open Notebook in Docker, Ollama on Host\n\nWhen Open Notebook runs in Docker but Ollama runs on your host machine:\n\n**Base URL to enter in Settings → API Keys:** `http://host.docker.internal:11434`\n\n**⚠️ CRITICAL: Ollama must accept external connections:**\n```bash\n# Start Ollama with external access enabled\nexport OLLAMA_HOST=0.0.0.0:11434\nollama serve\n```\n\n**⚠️ LINUX USERS: Extra configuration required!**\n\nOn Linux, `host.docker.internal` doesn't resolve automatically like it does on macOS/Windows. You must add `extra_hosts` to your docker-compose.yml:\n\n```yaml\nservices:\n open_notebook:\n image: lfnovo/open_notebook:v1-latest-single\n # ... other settings ...\n extra_hosts:\n - \"host.docker.internal:host-gateway\"\n```\n\nWithout this, you'll get connection errors like:\n```\nhttpcore.ConnectError: [Errno -2] Name or service not known\n```\n\n**Why `host.docker.internal`?**\n- Docker containers can't reach `localhost` on the host\n- `host.docker.internal` is Docker's special hostname for the host machine\n- Available on Docker Desktop for Mac/Windows; **requires `extra_hosts` on Linux**\n\n**Why `OLLAMA_HOST=0.0.0.0:11434`?**\n- By default, Ollama only binds to localhost and rejects external connections\n- Docker containers are considered \"external\" even when running on the same machine\n- Setting `OLLAMA_HOST=0.0.0.0:11434` allows connections from Docker containers\n\n### Scenario 3: Both in Docker (Same Compose)\n\nWhen both Open Notebook and Ollama run in the same Docker Compose stack:\n\n**Base URL to enter in Settings → API Keys:** `http://ollama:11434`\n\n**Docker Compose Example:**\n\n```yaml\nversion: '3.8'\nservices:\n open-notebook:\n image: lfnovo/open_notebook:v1-latest-single\n pull_policy: always\n ports:\n - \"8502:8502\"\n - \"5055:5055\"\n environment:\n - OPEN_NOTEBOOK_ENCRYPTION_KEY=change-me-to-a-secret-string\n volumes:\n - ./notebook_data:/app/data\n - ./surreal_data:/mydata\n depends_on:\n - ollama\n\n ollama:\n image: ollama/ollama:latest\n ports:\n - \"11434:11434\"\n volumes:\n - ollama_data:/root/.ollama\n # Optional: GPU support\n deploy:\n resources:\n reservations:\n devices:\n - driver: nvidia\n count: 1\n capabilities: [gpu]\n\nvolumes:\n ollama_data:\n```\n\n### Scenario 4: Remote Ollama Server\n\nWhen Ollama runs on a different machine in your network:\n\n**Base URL to enter in Settings → API Keys:** `http://192.168.1.100:11434` (replace with your Ollama server's IP)\n\n**Security Note:** Only use this in trusted networks. Ollama doesn't have built-in authentication.\n\n### Scenario 5: Ollama with Custom Port\n\nIf you've configured Ollama to use a different port:\n\n```bash\n# Start Ollama on custom port\nOLLAMA_HOST=0.0.0.0:8080 ollama serve\n```\n\n**Base URL to enter in Settings → API Keys:** `http://localhost:8080`\n\n## Model Recommendations\n\n### Language Models\n\n| Model | Size | Best For | Quality | Speed |\n|-------|------|----------|---------|-------|\n| **qwen3** | 7B | General purpose, coding | Excellent | Fast |\n| **deepseek-r1** | 7B | Reasoning, problem-solving | Exceptional | Medium |\n| **gemma3** | 7B | Balanced performance | Very Good | Fast |\n| **phi4** | 14B | Efficiency on small hardware | Good | Very Fast |\n| **llama3** | 8B | General purpose | Very Good | Medium |\n\n### Embedding Models\n\n| Model | Best For | Performance |\n|-------|----------|-------------|\n| **mxbai-embed-large** | General search | Excellent |\n| **nomic-embed-text** | Document similarity | Good |\n| **all-minilm** | Lightweight option | Fair |\n\n### Installation Commands\n\n```bash\n# Essential models\nollama pull qwen3 # Primary language model\nollama pull mxbai-embed-large # Search embeddings\n\n# Optional reasoning model\nollama pull deepseek-r1 # Advanced reasoning\n\n# Alternative language models\nollama pull gemma3 # Google's model\nollama pull phi4 # Microsoft's efficient model\n```\n\n## Hardware Requirements\n\n### Minimum Requirements\n- **RAM**: 8GB (for 7B models)\n- **Storage**: 10GB free space per model\n- **CPU**: Modern multi-core processor\n\n### Recommended Setup\n- **RAM**: 16GB+ (for multiple models)\n- **Storage**: SSD with 50GB+ free space\n- **GPU**: NVIDIA GPU with 8GB+ VRAM (optional but faster)\n\n### GPU Acceleration\n\n**NVIDIA GPU (CUDA):**\n```bash\n# Install NVIDIA Container Toolkit for Docker\n# Then use the Docker Compose example above with GPU support\n\n# For local installation, Ollama auto-detects CUDA\nollama pull qwen3\n```\n\n**Apple Silicon (M1/M2/M3):**\n```bash\n# Ollama automatically uses Metal acceleration\n# No additional setup required\nollama pull qwen3\n```\n\n**AMD GPUs:**\n```bash\n# ROCm support varies by model and system\n# Check Ollama documentation for latest compatibility\n```\n\n## Troubleshooting\n\n### Model Name Configuration (Critical)\n\n**⚠️ IMPORTANT: Model names must exactly match the output of `ollama list`**\n\nThis is the most common cause of \"Failed to send message\" errors. Open Notebook requires the **exact model name** as it appears in Ollama.\n\n**Step 1: Get the exact model name**\n```bash\nollama list\n```\n\nExample output:\n```\nNAME ID SIZE MODIFIED\nmxbai-embed-large:latest 468836162de7 669 MB 7 minutes ago\ngemma3:12b f4031aab637d 8.1 GB 2 months ago\nqwen3:32b 030ee887880f 20 GB 9 days ago\n```\n\n**Step 2: Use the exact name when adding the model in Open Notebook**\n\n| ✅ Correct | ❌ Wrong |\n|-----------|----------|\n| `gemma3:12b` | `gemma3` (missing tag) |\n| `qwen3:32b` | `qwen3-32b` (wrong format) |\n| `mxbai-embed-large:latest` | `mxbai-embed-large` (missing tag) |\n\n**Note:** Some models use `:latest` as the default tag. If `ollama list` shows `model:latest`, you must use `model:latest` in Open Notebook, not just `model`.\n\n**Step 3: Configure in Open Notebook**\n\n1. Go to **Settings → Models**\n2. Click **Add Model**\n3. Enter the **exact name** from `ollama list`\n4. Select provider: `ollama`\n5. Select type: `language` (for chat) or `embedding` (for search)\n6. Save the model\n7. Set it as the default for the appropriate task (chat, transformation, etc.)\n\n### Common Issues\n\n**1. \"Ollama unavailable\" in Open Notebook**\n\n**Check Ollama is running:**\n```bash\ncurl http://localhost:11434/api/tags\n```\n\n**Verify credential is configured:**\nCheck **Settings → API Keys** for an Ollama credential with the correct base URL.\n\n**⚠️ IMPORTANT: Enable external connections (most common fix):**\n```bash\n# If Open Notebook runs in Docker or on a different machine,\n# Ollama must bind to all interfaces, not just localhost\nexport OLLAMA_HOST=0.0.0.0:11434\nollama serve\n```\n> **Why this is needed:** By default, Ollama only accepts connections from `localhost` (127.0.0.1). When Open Notebook runs in Docker or on a different machine, it can't reach Ollama unless you configure `OLLAMA_HOST=0.0.0.0:11434` to accept external connections.\n\n**Restart Ollama:**\n```bash\n# Linux/macOS\nsudo systemctl restart ollama\n# or\nollama serve\n\n# Windows\n# Restart from system tray or Services\n```\n\n**2. Docker networking issues**\n\n**From inside Open Notebook container, test Ollama:**\n```bash\n# Get into container\ndocker exec -it open-notebook bash\n\n# Test connection\ncurl http://host.docker.internal:11434/api/tags\n```\n\n**If this fails on Linux** with \"Name or service not known\", you need to add `extra_hosts` to your docker-compose.yml. See the [Docker-Specific Troubleshooting](#docker-specific-troubleshooting) section below.\n\n**3. Models not downloading**\n\n**Check disk space:**\n```bash\ndf -h\n```\n\n**Manual model pull:**\n```bash\nollama pull qwen3 --verbose\n```\n\n**Clear failed downloads:**\n```bash\nollama rm qwen3\nollama pull qwen3\n```\n\n**4. Slow performance**\n\n**Check model size vs available RAM:**\n```bash\nollama ps # Show running models\nfree -h # Check available memory\n```\n\n**Use smaller models:**\n```bash\nollama pull phi4 # Instead of larger models\nollama pull gemma3:2b # 2B parameter variant\n```\n\n**5. Port conflicts**\n\n**Check what's using port 11434:**\n```bash\nlsof -i :11434\nnetstat -tulpn | grep 11434\n```\n\n**Use custom port:**\n```bash\nOLLAMA_HOST=0.0.0.0:8080 ollama serve\n```\nThen update the base URL in **Settings → API Keys** to `http://localhost:8080`\n\n**6. \"Failed to send message\" in Chat**\n\n**Symptom:** Chat shows \"Failed to send message\" toast notification. Logs may show:\n```\nError executing chat: Model is not a LanguageModel: None\n```\n\n**Causes (in order of likelihood):**\n\n1. **Model name mismatch**: The model name in Open Notebook doesn't exactly match `ollama list`\n2. **No default model configured**: You haven't set a default chat model in Settings → Models\n3. **Model was deleted**: You removed the model from Ollama but didn't update Open Notebook's defaults\n4. **Model record deleted**: The model was removed from Open Notebook but is still set as default\n\n**Solutions:**\n\n**Check 1: Verify model names match exactly**\n```bash\n# Get exact model names from Ollama\nollama list\n\n# Compare with what's configured in Open Notebook\n# Go to Settings → Models and verify the names match EXACTLY\n```\n\n**Check 2: Verify default models are set**\n1. Go to **Settings → Models**\n2. Scroll to **Default Models** section\n3. Ensure **Default Chat Model** has a value selected\n4. If empty, select an available language model\n\n**Check 3: Refresh after changes**\nIf you've added/removed models in Ollama:\n1. Refresh the Open Notebook page\n2. Go to Settings → Models\n3. Re-add any missing models with exact names from `ollama list`\n4. Re-select default models if needed\n\n**Check 4: Test the model directly**\n```bash\n# Verify Ollama can use the model\nollama run gemma3:12b \"Hello, world\"\n```\n\n### Docker-Specific Troubleshooting\n\n**1. Linux: `host.docker.internal` not resolving (Most Common)**\n\nIf you see `Name or service not known` errors on Linux, add `extra_hosts` to your docker-compose.yml:\n\n```yaml\nservices:\n open_notebook:\n image: lfnovo/open_notebook:v1-latest-single\n extra_hosts:\n - \"host.docker.internal:host-gateway\"\n environment:\n # ... rest of your config\n```\n\nThen in **Settings → API Keys**, use base URL: `http://host.docker.internal:11434`\n\nThis maps `host.docker.internal` to your host machine's IP. macOS/Windows Docker Desktop does this automatically, but Linux requires explicit configuration.\n\n**2. Host networking on Linux (alternative):**\n```bash\n# Use host networking if host.docker.internal doesn't work\ndocker run --network host lfnovo/open_notebook:v1-latest-single\n```\nThen in **Settings → API Keys**, use base URL: `http://localhost:11434`\n\n**3. Custom bridge network:**\n```yaml\nversion: '3.8'\nnetworks:\n ollama_network:\n driver: bridge\n\nservices:\n open-notebook:\n networks:\n - ollama_network\n environment:\n ollama:\n networks:\n - ollama_network\n```\n\nThen in **Settings → API Keys**, use base URL: `http://ollama:11434`\n\n**4. Firewall issues:**\n```bash\n# Allow Ollama port through firewall\nsudo ufw allow 11434\n# or\nsudo firewall-cmd --add-port=11434/tcp --permanent\n```\n\n## Performance Optimization\n\n### Model Management\n\n**List installed models:**\n```bash\nollama list\n```\n\n**Remove unused models:**\n```bash\nollama rm model_name\n```\n\n**Show running models:**\n```bash\nollama ps\n```\n\n**Preload models for faster startup:**\n```bash\n# Keep model in memory\ncurl http://localhost:11434/api/generate -d '{\n \"model\": \"qwen3\",\n \"prompt\": \"test\",\n \"keep_alive\": -1\n}'\n```\n\n### System Optimization\n\n**Linux: Increase file limits:**\n```bash\necho \"* soft nofile 65536\" >> /etc/security/limits.conf\necho \"* hard nofile 65536\" >> /etc/security/limits.conf\n```\n\n**macOS: Increase memory limits:**\n```bash\n# Add to ~/.zshrc or ~/.bash_profile\nexport OLLAMA_MAX_LOADED_MODELS=2\nexport OLLAMA_NUM_PARALLEL=4\n```\n\n**Docker: Resource allocation:**\n```yaml\nservices:\n ollama:\n deploy:\n resources:\n limits:\n memory: 8G\n cpus: '4'\n```\n\n## Advanced Configuration\n\n### Environment Variables\n\n```bash\n# Ollama server configuration\nexport OLLAMA_HOST=0.0.0.0:11434 # Bind to all interfaces\nexport OLLAMA_KEEP_ALIVE=5m # Keep models in memory\nexport OLLAMA_MAX_LOADED_MODELS=3 # Max concurrent models\nexport OLLAMA_MAX_QUEUE=512 # Request queue size\nexport OLLAMA_NUM_PARALLEL=4 # Parallel request handling\nexport OLLAMA_FLASH_ATTENTION=1 # Enable flash attention (if supported)\n\n# Open Notebook configuration (configure via Settings → API Keys instead)\n# OLLAMA_API_BASE=http://localhost:11434 # Deprecated — use Settings UI\n```\n\n### SSL Configuration (Self-Signed Certificates)\n\nIf you're running Ollama behind a reverse proxy with self-signed SSL certificates (e.g., Caddy, nginx with custom certs), you may encounter SSL verification errors:\n\n```\n[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: unable to get local issuer certificate\n```\n\n**Solutions:**\n\n**Option 1: Use a custom CA bundle (recommended)**\n```bash\n# Point to your CA certificate file\nexport ESPERANTO_SSL_CA_BUNDLE=/path/to/your/ca-bundle.pem\n```\n\n**Option 2: Disable SSL verification (development only)**\n```bash\n# WARNING: Only use in trusted development environments\nexport ESPERANTO_SSL_VERIFY=false\n```\n\n**Docker Compose example with SSL configuration:**\n```yaml\nservices:\n open-notebook:\n image: lfnovo/open_notebook:v1-latest-single\n pull_policy: always\n environment:\n - OPEN_NOTEBOOK_ENCRYPTION_KEY=change-me-to-a-secret-string\n # Option 1: Custom CA bundle (if Ollama uses self-signed SSL)\n - ESPERANTO_SSL_CA_BUNDLE=/certs/ca-bundle.pem\n # Option 2: Disable verification (dev only)\n # - ESPERANTO_SSL_VERIFY=false\n volumes:\n - /path/to/your/ca-bundle.pem:/certs/ca-bundle.pem:ro\n```\n\n> **Security Note:** Disabling SSL verification exposes you to man-in-the-middle attacks. Always prefer using a custom CA bundle in production environments.\n\n### Custom Model Imports\n\n**Import custom models:**\n```bash\n# Create Modelfile\ncat > Modelfile << EOF\nFROM qwen3\nPARAMETER temperature 0.7\nPARAMETER top_p 0.9\nSYSTEM \"You are a helpful research assistant.\"\nEOF\n\n# Create custom model\nollama create my-research-model -f Modelfile\n```\n\n**Use in Open Notebook:**\n1. Go to Models\n2. Add new model: `my-research-model`\n3. Set as default for specific tasks\n\n### Monitoring and Logging\n\n**Monitor Ollama logs:**\n```bash\n# Linux (systemd)\njournalctl -u ollama -f\n\n# Docker\ndocker logs -f ollama\n\n# Manual run with verbose logging\nOLLAMA_DEBUG=1 ollama serve\n```\n\n**Resource monitoring:**\n```bash\n# CPU and memory usage\nhtop\n\n# GPU usage (NVIDIA)\nnvidia-smi -l 1\n\n# Model-specific metrics\nollama ps\n```\n\n## Integration Examples\n\n### Python Script Integration\n\n```python\nimport requests\nimport os\n\n# Test Ollama connection\nollama_base = os.environ.get('OLLAMA_API_BASE', 'http://localhost:11434')\nresponse = requests.get(f'{ollama_base}/api/tags')\nprint(f\"Available models: {response.json()}\")\n\n# Generate text\npayload = {\n \"model\": \"qwen3\",\n \"prompt\": \"Explain quantum computing\",\n \"stream\": False\n}\nresponse = requests.post(f'{ollama_base}/api/generate', json=payload)\nprint(response.json()['response'])\n```\n\n### Health Check Script\n\n```bash\n#!/bin/bash\n# ollama-health-check.sh\n\nOLLAMA_API_BASE=${OLLAMA_API_BASE:-\"http://localhost:11434\"}\n\necho \"Checking Ollama health...\"\nif curl -s \"${OLLAMA_API_BASE}/api/tags\" > /dev/null; then\n echo \"✅ Ollama is running\"\n echo \"Available models:\"\n curl -s \"${OLLAMA_API_BASE}/api/tags\" | jq -r '.models[].name'\nelse\n echo \"❌ Ollama is not accessible at ${OLLAMA_API_BASE}\"\n exit 1\nfi\n```\n\n## Migration from Other Providers\n\n### Coming from OpenAI\n\n**Similar performance models:**\n- GPT-4 → `qwen3` or `deepseek-r1`\n- GPT-3.5 → `gemma3` or `phi4`\n- text-embedding-ada-002 → `mxbai-embed-large`\n\n**Cost comparison:**\n- OpenAI: $0.01-0.06 per 1K tokens\n- Ollama: $0 after hardware investment\n\n### Coming from Anthropic\n\n**Claude replacement suggestions:**\n- Claude 3.5 Sonnet → `deepseek-r1` (reasoning)\n- Claude 3 Haiku → `phi4` (speed)\n\n## Best Practices\n\n### Security\n\n1. **Network Security:**\n - Run Ollama only on trusted networks\n - Use firewall rules to limit access\n - Consider VPN for remote access\n\n2. **Model Verification:**\n - Only pull models from trusted sources\n - Verify model checksums when possible\n\n3. **Resource Limits:**\n - Set memory and CPU limits in production\n - Monitor resource usage regularly\n\n### Performance\n\n1. **Model Selection:**\n - Use appropriate model size for your hardware\n - Smaller models for simple tasks\n - Reasoning models only when needed\n\n2. **Resource Management:**\n - Preload frequently used models\n - Remove unused models regularly\n - Monitor system resources\n\n3. **Network Optimization:**\n - Use local networks for better latency\n - Consider SSD storage for faster model loading\n\n## Getting Help\n\n**Community Resources:**\n- [Ollama GitHub](https://github.com/jmorganca/ollama) - Official repository\n- [Ollama Discord](https://discord.gg/ollama) - Community support\n- [Open Notebook Discord](https://discord.gg/37XJPXfz2w) - Integration help\n\n**Debugging Resources:**\n- Check Ollama logs for error messages\n- Test connection with curl commands\n- Verify environment variables\n- Monitor system resources\n\nThis comprehensive guide should help you successfully deploy and optimize Ollama with Open Notebook. Start with the Quick Start section and refer to specific scenarios as needed." }, { "path": "docs/5-CONFIGURATION/openai-compatible.md", "content": "# OpenAI-Compatible Providers\n\nUse any server that implements the OpenAI API format with Open Notebook. This includes LM Studio, Text Generation WebUI, vLLM, and many others.\n\n---\n\n## What is OpenAI-Compatible?\n\nMany AI tools implement the same API format as OpenAI:\n\n```\nPOST /v1/chat/completions\nPOST /v1/embeddings\nPOST /v1/audio/speech\n```\n\nOpen Notebook can connect to any server using this format.\n\n---\n\n## Common Compatible Servers\n\n| Server | Use Case | URL |\n|--------|----------|-----|\n| **LM Studio** | Desktop GUI for local models | https://lmstudio.ai |\n| **Text Generation WebUI** | Full-featured local inference | https://github.com/oobabooga/text-generation-webui |\n| **vLLM** | High-performance serving | https://github.com/vllm-project/vllm |\n| **Ollama** | Simple local models | (Use native Ollama provider instead) |\n| **LocalAI** | Local AI inference | https://github.com/mudler/LocalAI |\n| **llama.cpp server** | Lightweight inference | https://github.com/ggerganov/llama.cpp |\n\n---\n\n## Quick Setup: LM Studio\n\n### Step 1: Install and Start LM Studio\n\n1. Download from https://lmstudio.ai\n2. Install and launch\n3. Download a model (e.g., Llama 3)\n4. Start the local server (default: port 1234)\n\n### Step 2: Configure in Settings UI (Recommended)\n\n1. Go to **Settings** → **API Keys**\n2. Click **Add Credential** → Select **OpenAI-Compatible**\n3. Enter base URL: `http://host.docker.internal:1234/v1` (Docker) or `http://localhost:1234/v1` (local)\n4. API key: `lm-studio` (placeholder, LM Studio doesn't require one)\n5. Click **Save**, then **Test Connection**\n\n**Legacy (Deprecated) — Environment variables:**\n```bash\nexport OPENAI_COMPATIBLE_BASE_URL=http://localhost:1234/v1\nexport OPENAI_COMPATIBLE_API_KEY=not-needed\n```\n\n### Step 3: Add Model in Open Notebook\n\n1. Go to **Settings** → **Models**\n2. Click **Add Model**\n3. Configure:\n - **Provider**: `openai_compatible`\n - **Model Name**: Your model name from LM Studio\n - **Display Name**: `LM Studio - Llama 3`\n4. Click **Save**\n\n---\n\n## Configuration via Settings UI\n\nThe recommended way to configure OpenAI-compatible providers is through the Settings UI:\n\n1. Go to **Settings** → **API Keys**\n2. Click **Add Credential** → Select **OpenAI-Compatible**\n3. Enter your base URL and API key (if needed)\n4. Optionally configure per-service URLs for LLM, Embedding, TTS, and STT\n5. Click **Save**, then **Test Connection**\n\n## Legacy: Environment Variables (Deprecated)\n\n> **Deprecated**: These environment variables are deprecated. Use the Settings UI instead.\n\n### Language Models (Chat)\n\n```bash\nOPENAI_COMPATIBLE_BASE_URL=http://localhost:1234/v1\nOPENAI_COMPATIBLE_API_KEY=optional-api-key\n```\n\n### Embeddings\n\n```bash\nOPENAI_COMPATIBLE_BASE_URL_EMBEDDING=http://localhost:1234/v1\nOPENAI_COMPATIBLE_API_KEY_EMBEDDING=optional-api-key\n```\n\n### Text-to-Speech\n\n```bash\nOPENAI_COMPATIBLE_BASE_URL_TTS=http://localhost:8969/v1\nOPENAI_COMPATIBLE_API_KEY_TTS=optional-api-key\n```\n\n### Speech-to-Text\n\n```bash\nOPENAI_COMPATIBLE_BASE_URL_STT=http://localhost:9000/v1\nOPENAI_COMPATIBLE_API_KEY_STT=optional-api-key\n```\n\n---\n\n## Docker Networking\n\nWhen Open Notebook runs in Docker and your compatible server runs on the host, use the appropriate base URL when adding your credential in **Settings → API Keys**:\n\n### macOS / Windows\n\n**Base URL:** `http://host.docker.internal:1234/v1`\n\n### Linux\n\n**Base URL (Option 1 — Docker bridge IP):** `http://172.17.0.1:1234/v1`\n\n**Option 2:** Use host networking mode: `docker run --network host ...`\nThen use base URL: `http://localhost:1234/v1`\n\n### Same Docker Network\n\n```yaml\n# docker-compose.yml\nservices:\n open-notebook:\n # ...\n\n lm-studio:\n # your LM Studio container\n ports:\n - \"1234:1234\"\n```\n\n**Base URL in Settings → API Keys:** `http://lm-studio:1234/v1`\n\n---\n\n## Text Generation WebUI Setup\n\n### Start with API Enabled\n\n```bash\npython server.py --api --listen\n```\n\n### Configure Open Notebook\n\nIn **Settings → API Keys**, add an **OpenAI-Compatible** credential with base URL: `http://localhost:5000/v1`\n\n### Docker Compose Example\n\n```yaml\nservices:\n text-gen:\n image: atinoda/text-generation-webui:default\n ports:\n - \"5000:5000\"\n - \"7860:7860\"\n volumes:\n - ./models:/app/models\n command: --api --listen\n\n open-notebook:\n image: lfnovo/open_notebook:v1-latest-single\n pull_policy: always\n depends_on:\n - text-gen\n```\n\nThen in **Settings → API Keys**, add an **OpenAI-Compatible** credential with base URL: `http://text-gen:5000/v1`\n\n---\n\n## vLLM Setup\n\n### Start vLLM Server\n\n```bash\npython -m vllm.entrypoints.openai.api_server \\\n --model meta-llama/Llama-3.1-8B-Instruct \\\n --port 8000\n```\n\n### Configure Open Notebook\n\nIn **Settings → API Keys**, add an **OpenAI-Compatible** credential with base URL: `http://localhost:8000/v1`\n\n### Docker Compose with GPU\n\n```yaml\nservices:\n vllm:\n image: vllm/vllm-openai:latest\n command: --model meta-llama/Llama-3.1-8B-Instruct\n ports:\n - \"8000:8000\"\n volumes:\n - ~/.cache/huggingface:/root/.cache/huggingface\n deploy:\n resources:\n reservations:\n devices:\n - driver: nvidia\n count: 1\n capabilities: [gpu]\n\n open-notebook:\n image: lfnovo/open_notebook:v1-latest-single\n pull_policy: always\n depends_on:\n - vllm\n```\n\nThen in **Settings → API Keys**, add an **OpenAI-Compatible** credential with base URL: `http://vllm:8000/v1`\n\n---\n\n## Adding Models in Open Notebook\n\n### Via Settings UI\n\n1. Go to **Settings** → **Models**\n2. Click **Add Model** in appropriate section\n3. Select **Provider**: `openai_compatible`\n4. Enter **Model Name**: exactly as the server expects\n5. Enter **Display Name**: your preferred name\n6. Click **Save**\n\n### Model Name Format\n\nThe model name must match what your server expects:\n\n| Server | Model Name Format |\n|--------|-------------------|\n| LM Studio | As shown in LM Studio UI |\n| vLLM | HuggingFace model path |\n| Text Gen WebUI | As loaded in UI |\n| llama.cpp | Model file name |\n\n---\n\n## Testing Connection\n\n### Test API Endpoint\n\n```bash\n# Test chat completions\ncurl http://localhost:1234/v1/chat/completions \\\n -H \"Content-Type: application/json\" \\\n -d '{\n \"model\": \"your-model-name\",\n \"messages\": [{\"role\": \"user\", \"content\": \"Hello\"}]\n }'\n```\n\n### Test from Inside Docker\n\n```bash\ndocker exec -it open-notebook curl http://host.docker.internal:1234/v1/models\n```\n\n---\n\n## Troubleshooting\n\n### Connection Refused\n\n```\nProblem: Cannot connect to server\n\nSolutions:\n1. Verify server is running\n2. Check port is correct\n3. Test with curl directly\n4. Check Docker networking (use host.docker.internal)\n5. Verify firewall allows connection\n```\n\n### Model Not Found\n\n```\nProblem: Server returns \"model not found\"\n\nSolutions:\n1. Check model is loaded in server\n2. Verify exact model name spelling\n3. List available models: curl http://localhost:1234/v1/models\n4. Update model name in Open Notebook\n```\n\n### Slow Responses\n\n```\nProblem: Requests take very long\n\nSolutions:\n1. Check server resources (RAM, GPU)\n2. Use smaller/quantized model\n3. Reduce context length\n4. Enable GPU acceleration if available\n```\n\n### Authentication Errors\n\n```\nProblem: 401 or authentication failed\n\nSolutions:\n1. Check if server requires API key\n2. Set the API key in your credential (Settings → API Keys)\n3. Some servers need any non-empty key (use a placeholder like \"not-needed\")\n```\n\n### Timeout Errors\n\n```\nProblem: Request times out\n\nSolutions:\n1. Model may be loading (first request slow)\n2. Increase timeout settings\n3. Check server logs for errors\n4. Reduce request size\n```\n\n---\n\n## Multiple Compatible Endpoints\n\nYou can use different compatible servers for different purposes. When adding an **OpenAI-Compatible** credential in **Settings → API Keys**, you can configure per-service URLs:\n\n- **LLM URL**: e.g., `http://localhost:1234/v1` (LM Studio)\n- **Embedding URL**: e.g., `http://localhost:8080/v1` (different server)\n- **TTS URL**: e.g., `http://localhost:8969/v1` (Speaches)\n- **STT URL**: e.g., `http://localhost:9000/v1` (Speaches)\n\nAlternatively, add each as a separate credential with its own base URL.\n\n---\n\n## Performance Tips\n\n### Model Selection\n\n| Model Size | RAM Needed | Speed |\n|------------|------------|-------|\n| 7B | 8GB | Fast |\n| 13B | 16GB | Medium |\n| 70B | 64GB+ | Slow |\n\n### Quantization\n\nUse quantized models (Q4, Q5) for faster inference with less RAM:\n\n```\nllama-3-8b-q4_k_m.gguf → ~4GB RAM, fast\nllama-3-8b-f16.gguf → ~16GB RAM, slower\n```\n\n### GPU Acceleration\n\nEnable GPU in your server for much faster inference:\n- LM Studio: Settings → GPU layers\n- vLLM: Automatic with CUDA\n- llama.cpp: `--n-gpu-layers 35`\n\n---\n\n## Comparison: Native vs Compatible\n\n| Aspect | Native Provider | OpenAI Compatible |\n|--------|-----------------|-------------------|\n| **Setup** | API key only | Server + configuration |\n| **Models** | Provider's models | Any compatible model |\n| **Cost** | Pay per token | Free (local) |\n| **Speed** | Usually fast | Depends on hardware |\n| **Features** | Full support | Basic features |\n\nUse OpenAI-compatible when:\n- Running local models\n- Using custom/fine-tuned models\n- Privacy requirements\n- Cost control\n\n---\n\n## Related\n\n- **[Local TTS Setup](local-tts.md)** - Text-to-speech with Speaches\n- **[Local STT Setup](local-stt.md)** - Speech-to-text with Speaches\n- **[AI Providers](ai-providers.md)** - All provider options\n- **[Ollama Setup](ollama.md)** - Native Ollama integration\n" }, { "path": "docs/5-CONFIGURATION/reverse-proxy.md", "content": "# Reverse Proxy Configuration\n\nDeploy Open Notebook behind nginx, Caddy, Traefik, or other reverse proxies with custom domains and HTTPS.\n\n---\n\n## Simplified Setup (v1.1+)\n\nStarting with v1.1, Open Notebook uses Next.js rewrites to simplify configuration. **You only need to proxy to one port** - Next.js handles internal API routing automatically.\n\n### How It Works\n\n```\nBrowser → Reverse Proxy → Port 8502 (Next.js)\n ↓ (internal proxy)\n Port 5055 (FastAPI)\n```\n\nNext.js automatically forwards `/api/*` requests to the FastAPI backend, so your reverse proxy only needs one port!\n\n---\n\n## Quick Configuration Examples\n\n### Nginx (Recommended)\n\n```nginx\nserver {\n listen 443 ssl http2;\n server_name notebook.example.com;\n\n ssl_certificate /etc/nginx/ssl/fullchain.pem;\n ssl_certificate_key /etc/nginx/ssl/privkey.pem;\n\n # Allow file uploads up to 100MB\n client_max_body_size 100M;\n\n # Single location block - that's it!\n location / {\n proxy_pass http://open-notebook:8502;\n proxy_http_version 1.1;\n proxy_set_header Host $host;\n proxy_set_header X-Real-IP $remote_addr;\n proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;\n proxy_set_header X-Forwarded-Proto $scheme;\n proxy_set_header Upgrade $http_upgrade;\n proxy_set_header Connection 'upgrade';\n proxy_cache_bypass $http_upgrade;\n }\n}\n\n# HTTP to HTTPS redirect\nserver {\n listen 80;\n server_name notebook.example.com;\n return 301 https://$server_name$request_uri;\n}\n```\n\n### Caddy\n\n```caddy\nnotebook.example.com {\n reverse_proxy open-notebook:8502 {\n transport http {\n read_timeout 600s\n write_timeout 600s\n }\n }\n}\n```\n\nCaddy handles HTTPS automatically. The timeout settings ensure long-running operations (transformations, podcast generation) don't fail.\n\n### Traefik\n\n```yaml\nservices:\n open-notebook:\n image: lfnovo/open_notebook:v1-latest-single\n pull_policy: always\n environment:\n - API_URL=https://notebook.example.com\n labels:\n - \"traefik.enable=true\"\n - \"traefik.http.routers.notebook.rule=Host(`notebook.example.com`)\"\n - \"traefik.http.routers.notebook.entrypoints=websecure\"\n - \"traefik.http.routers.notebook.tls.certresolver=myresolver\"\n - \"traefik.http.services.notebook.loadbalancer.server.port=8502\"\n # Timeout for long-running operations (transformations, podcasts)\n - \"traefik.http.services.notebook.loadbalancer.responseforwarding.flushinterval=100ms\"\n networks:\n - traefik-network\n```\n\n**Note**: For Traefik v2+, you may also need to configure `serversTransport` timeouts in your static configuration:\n\n```yaml\n# traefik.yml (static configuration)\nserversTransport:\n forwardingTimeouts:\n dialTimeout: 30s\n responseHeaderTimeout: 600s\n idleConnTimeout: 90s\n```\n\n### Coolify\n\n1. Create new service with `lfnovo/open_notebook:v1-latest-single`\n2. Set port to **8502**\n3. Add environment: `API_URL=https://your-domain.com`\n4. Enable HTTPS in Coolify\n5. Done!\n\n---\n\n## Environment Variables\n\n```bash\n# Required for reverse proxy setups\nAPI_URL=https://your-domain.com\n\n# Optional: For multi-container deployments\n# INTERNAL_API_URL=http://api-service:5055\n```\n\n**Important**: Set `API_URL` to your public URL (with https://).\n\n**Note on HOSTNAME**: The Docker images set `HOSTNAME=0.0.0.0` by default, which ensures Next.js binds to all interfaces and is accessible from reverse proxies. You typically don't need to set this manually.\n\n---\n\n## Understanding API_URL\n\nThe frontend uses a three-tier priority system to determine the API URL:\n\n1. **Runtime Configuration** (Highest Priority): `API_URL` environment variable set at container runtime\n2. **Build-time Configuration**: `NEXT_PUBLIC_API_URL` baked into the Docker image\n3. **Auto-detection** (Fallback): Infers from the incoming HTTP request headers\n\n### Auto-Detection Details\n\nWhen `API_URL` is not set, the Next.js frontend:\n- Analyzes the incoming HTTP request\n- Extracts the hostname from the `host` header\n- Respects the `X-Forwarded-Proto` header (for HTTPS behind reverse proxies)\n- Constructs the API URL as `{protocol}://{hostname}:5055`\n- Example: Request to `http://10.20.30.20:8502` → API URL becomes `http://10.20.30.20:5055`\n\n**Why set API_URL explicitly?**\n- **Reliability**: Auto-detection can fail with complex proxy setups\n- **HTTPS**: Ensures frontend uses `https://` when behind SSL-terminating proxy\n- **Custom domains**: Works correctly with domain names instead of IP addresses\n- **Port mapping**: Avoids exposing port 5055 in the URL when using reverse proxy\n\n**Important**: Don't include `/api` at the end - the system adds this automatically!\n\n---\n\n## Complete Docker Compose Example\n\n```yaml\nservices:\n open-notebook:\n image: lfnovo/open_notebook:v1-latest-single\n pull_policy: always\n container_name: open-notebook\n environment:\n - API_URL=https://notebook.example.com\n - OPEN_NOTEBOOK_ENCRYPTION_KEY=${OPEN_NOTEBOOK_ENCRYPTION_KEY}\n - OPEN_NOTEBOOK_PASSWORD=${OPEN_NOTEBOOK_PASSWORD}\n volumes:\n - ./notebook_data:/app/data\n - ./surreal_data:/mydata\n # Only expose to localhost (nginx handles public access)\n ports:\n - \"127.0.0.1:8502:8502\"\n restart: unless-stopped\n\n nginx:\n image: nginx:alpine\n container_name: nginx-proxy\n ports:\n - \"80:80\"\n - \"443:443\"\n volumes:\n - ./nginx.conf:/etc/nginx/nginx.conf:ro\n - ./ssl:/etc/nginx/ssl:ro\n depends_on:\n - open-notebook\n restart: unless-stopped\n```\n\n---\n\n## Full Nginx Configuration\n\n```nginx\nevents {\n worker_connections 1024;\n}\n\nhttp {\n upstream notebook {\n server open-notebook:8502;\n }\n\n # HTTP redirect\n server {\n listen 80;\n server_name notebook.example.com;\n return 301 https://$server_name$request_uri;\n }\n\n # HTTPS server\n server {\n listen 443 ssl http2;\n server_name notebook.example.com;\n\n ssl_certificate /etc/nginx/ssl/fullchain.pem;\n ssl_certificate_key /etc/nginx/ssl/privkey.pem;\n ssl_protocols TLSv1.2 TLSv1.3;\n ssl_ciphers HIGH:!aNULL:!MD5;\n\n # Allow file uploads up to 100MB\n client_max_body_size 100M;\n\n # Security headers\n add_header X-Frame-Options DENY;\n add_header X-Content-Type-Options nosniff;\n add_header X-XSS-Protection \"1; mode=block\";\n add_header Strict-Transport-Security \"max-age=31536000; includeSubDomains\";\n\n # Proxy settings\n location / {\n proxy_pass http://notebook;\n proxy_http_version 1.1;\n proxy_set_header Host $host;\n proxy_set_header X-Real-IP $remote_addr;\n proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;\n proxy_set_header X-Forwarded-Proto $scheme;\n proxy_set_header Upgrade $http_upgrade;\n proxy_set_header Connection 'upgrade';\n proxy_cache_bypass $http_upgrade;\n\n # Timeouts for long-running operations (transformations, podcasts, etc.)\n # 600s matches the frontend timeout for slow LLM operations\n proxy_read_timeout 600s;\n proxy_connect_timeout 60s;\n proxy_send_timeout 600s;\n }\n }\n}\n```\n\n---\n\n## Direct API Access (Optional)\n\nIf external scripts or integrations need direct API access, route `/api/*` directly:\n\n```nginx\n# Direct API access (for external integrations)\nlocation /api/ {\n proxy_pass http://open-notebook:5055/api/;\n proxy_http_version 1.1;\n proxy_set_header Host $host;\n proxy_set_header X-Real-IP $remote_addr;\n proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;\n proxy_set_header X-Forwarded-Proto $scheme;\n}\n\n# Frontend (handles all other traffic)\nlocation / {\n proxy_pass http://open-notebook:8502;\n # ... same headers as above\n}\n```\n\n**Note**: This is only needed for external API integrations. Browser traffic works fine with single-port setup.\n\n---\n\n## Advanced Scenarios\n\n### Remote Server Access (LAN/VPS)\n\nAccessing Open Notebook from a different machine on your network:\n\n**Step 1: Get your server IP**\n```bash\n# On the server running Open Notebook:\nhostname -I\n# or\nifconfig | grep \"inet \"\n# Note the IP (e.g., 192.168.1.100)\n```\n\n**Step 2: Configure API_URL**\n```bash\n# In docker-compose.yml or .env:\nAPI_URL=http://192.168.1.100:5055\n```\n\n**Step 3: Expose ports**\n```yaml\nservices:\n open-notebook:\n image: lfnovo/open_notebook:v1-latest-single\n pull_policy: always\n environment:\n - API_URL=http://192.168.1.100:5055\n ports:\n - \"8502:8502\"\n - \"5055:5055\"\n```\n\n**Step 4: Access from client machine**\n```bash\n# In browser on other machine:\nhttp://192.168.1.100:8502\n```\n\n**Troubleshooting**:\n- Check firewall: `sudo ufw allow 8502 && sudo ufw allow 5055`\n- Verify connectivity: `ping 192.168.1.100` from client machine\n- Test port: `telnet 192.168.1.100 8502` from client machine\n\n---\n\n### API on Separate Subdomain\n\nHost the API and frontend on different subdomains:\n\n**docker-compose.yml:**\n```yaml\nservices:\n open-notebook:\n image: lfnovo/open_notebook:v1-latest-single\n pull_policy: always\n environment:\n - API_URL=https://api.notebook.example.com\n - OPEN_NOTEBOOK_ENCRYPTION_KEY=${OPEN_NOTEBOOK_ENCRYPTION_KEY}\n # Don't expose ports (nginx handles routing)\n```\n\n**nginx.conf:**\n```nginx\n# Frontend server\nserver {\n listen 443 ssl http2;\n server_name notebook.example.com;\n\n ssl_certificate /etc/nginx/ssl/fullchain.pem;\n ssl_certificate_key /etc/nginx/ssl/privkey.pem;\n\n location / {\n proxy_pass http://open-notebook:8502;\n proxy_http_version 1.1;\n proxy_set_header Host $host;\n proxy_set_header X-Real-IP $remote_addr;\n proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;\n proxy_set_header X-Forwarded-Proto $scheme;\n proxy_set_header Upgrade $http_upgrade;\n proxy_set_header Connection 'upgrade';\n proxy_cache_bypass $http_upgrade;\n }\n}\n\n# API server (separate subdomain)\nserver {\n listen 443 ssl http2;\n server_name api.notebook.example.com;\n\n ssl_certificate /etc/nginx/ssl/fullchain.pem;\n ssl_certificate_key /etc/nginx/ssl/privkey.pem;\n\n location / {\n proxy_pass http://open-notebook:5055;\n proxy_http_version 1.1;\n proxy_set_header Host $host;\n proxy_set_header X-Real-IP $remote_addr;\n proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;\n proxy_set_header X-Forwarded-Proto $scheme;\n }\n}\n```\n\n**Use case**: Separate DNS records, different rate limiting, or isolated API access control.\n\n---\n\n### Multi-Container Deployment (Advanced)\n\nFor complex deployments with separate frontend and API containers:\n\n**docker-compose.yml:**\n```yaml\nservices:\n frontend:\n image: lfnovo/open_notebook_frontend:v1-latest\n pull_policy: always\n environment:\n - API_URL=https://notebook.example.com\n ports:\n - \"8502:8502\"\n\n api:\n image: lfnovo/open_notebook_api:v1-latest\n pull_policy: always\n environment:\n - OPEN_NOTEBOOK_ENCRYPTION_KEY=${OPEN_NOTEBOOK_ENCRYPTION_KEY}\n ports:\n - \"5055:5055\"\n depends_on:\n - surrealdb\n\n surrealdb:\n image: surrealdb/surrealdb:latest\n command: start --log trace --user root --pass root file:/mydata/database.db\n ports:\n - \"8000:8000\"\n volumes:\n - ./surreal_data:/mydata\n```\n\n**nginx.conf:**\n```nginx\nhttp {\n upstream frontend {\n server frontend:8502;\n }\n\n upstream api {\n server api:5055;\n }\n\n server {\n listen 443 ssl http2;\n server_name notebook.example.com;\n\n # API routes\n location /api/ {\n proxy_pass http://api/api/;\n proxy_http_version 1.1;\n proxy_set_header Host $host;\n proxy_set_header X-Real-IP $remote_addr;\n proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;\n proxy_set_header X-Forwarded-Proto $scheme;\n }\n\n # Frontend (catch-all)\n location / {\n proxy_pass http://frontend;\n proxy_http_version 1.1;\n proxy_set_header Host $host;\n proxy_set_header X-Real-IP $remote_addr;\n proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;\n proxy_set_header X-Forwarded-Proto $scheme;\n proxy_set_header Upgrade $http_upgrade;\n proxy_set_header Connection 'upgrade';\n proxy_cache_bypass $http_upgrade;\n }\n }\n}\n```\n\n**Note**: Most users should use the single-container approach (`v1-latest-single`). Multi-container is only needed for custom scaling or isolation requirements.\n\n---\n\n## SSL Certificates\n\n### Let's Encrypt with Certbot\n\n```bash\n# Install certbot\nsudo apt install certbot python3-certbot-nginx\n\n# Get certificate\nsudo certbot --nginx -d notebook.example.com\n\n# Auto-renewal (usually configured automatically)\nsudo certbot renew --dry-run\n```\n\n### Let's Encrypt with Caddy\n\nCaddy handles SSL automatically - no configuration needed!\n\n### Self-Signed (Development Only)\n\n```bash\nopenssl req -x509 -nodes -days 365 -newkey rsa:2048 \\\n -keyout ssl/privkey.pem \\\n -out ssl/fullchain.pem \\\n -subj \"/CN=localhost\"\n```\n\n---\n\n## Troubleshooting\n\n### \"Unable to connect to server\"\n\n1. **Check API_URL is set**:\n ```bash\n docker exec open-notebook env | grep API_URL\n ```\n\n2. **Verify reverse proxy reaches container**:\n ```bash\n curl -I http://localhost:8502\n ```\n\n3. **Check browser console** (F12):\n - Look for connection errors\n - Check what URL it's trying to reach\n\n### Mixed Content Errors\n\nFrontend using HTTPS but trying to reach HTTP API:\n\n```bash\n# Ensure API_URL uses https://\nAPI_URL=https://notebook.example.com # Not http://\n```\n\n### WebSocket Issues\n\nEnsure your proxy supports WebSocket upgrades:\n\n```nginx\nproxy_http_version 1.1;\nproxy_set_header Upgrade $http_upgrade;\nproxy_set_header Connection 'upgrade';\n```\n\n### 502 Bad Gateway\n\n1. Check container is running: `docker ps`\n2. Check container logs: `docker logs open-notebook`\n3. Verify nginx can reach container (same network)\n\n### Timeout Errors\n\n**Symptoms:**\n- `socket hang up` or `ECONNRESET` errors\n- `Timeout after 30000ms` errors\n- Operations fail after exactly 30 seconds\n\n**Cause:** Your reverse proxy has a default timeout (often 30s) that's shorter than Open Notebook's operations.\n\n**Solutions by proxy:**\n\n**Nginx:**\n```nginx\nproxy_read_timeout 600s;\nproxy_send_timeout 600s;\n```\n\n**Caddy:**\n```caddy\nreverse_proxy open-notebook:8502 {\n transport http {\n read_timeout 600s\n write_timeout 600s\n }\n}\n```\n\n**Traefik (static config):**\n```yaml\nserversTransport:\n forwardingTimeouts:\n responseHeaderTimeout: 600s\n```\n\n**Application-level timeouts:**\n\nIf you still experience timeouts after configuring your proxy, you can also adjust the application timeouts:\n\n```bash\n# In .env file:\nAPI_CLIENT_TIMEOUT=600 # API client timeout (default: 300s)\nESPERANTO_LLM_TIMEOUT=180 # LLM inference timeout (default: 60s)\n```\n\nSee [Advanced Configuration](advanced.md) for more timeout options.\n\n---\n\n### How to Debug Configuration Issues\n\n**Step 1: Check browser console** (F12 → Console tab)\n```\nLook for messages starting with 🔧 [Config]\nThese show the configuration detection process\nYou'll see which API URL is being used\n```\n\n**Example good output:**\n```\n✅ [Config] Runtime API URL from server: https://your-domain.com\n```\n\n**Example bad output:**\n```\n❌ [Config] Failed to fetch runtime config\n⚠️ [Config] Using auto-detected URL: http://localhost:5055\n```\n\n**Step 2: Test API directly**\n```bash\n# Should return JSON config\ncurl https://your-domain.com/api/config\n\n# Expected output:\n{\"status\":\"ok\",\"credentials_configured\":true,...}\n```\n\n**Step 3: Check Docker logs**\n```bash\ndocker logs open-notebook\n\n# Look for:\n# - Frontend startup: \"▲ Next.js ready on http://0.0.0.0:8502\"\n# - API startup: \"INFO: Uvicorn running on http://0.0.0.0:5055\"\n# - Connection errors or CORS issues\n```\n\n**Step 4: Verify environment variable**\n```bash\ndocker exec open-notebook env | grep API_URL\n\n# Should show:\n# API_URL=https://your-domain.com\n```\n\n---\n\n### Frontend Adds `:5055` to URL (Versions ≤ 1.0.10)\n\n**Symptoms** (only in older versions):\n- You set `API_URL=https://your-domain.com`\n- Browser console shows: \"Attempted URL: https://your-domain.com:5055/api/config\"\n- CORS errors with \"Status code: (null)\"\n\n**Root Cause:**\nIn versions ≤ 1.0.10, the frontend's config endpoint was at `/api/runtime-config`, which got intercepted by reverse proxies routing all `/api/*` requests to the backend. This prevented the frontend from reading the `API_URL` environment variable.\n\n**Solution:**\nUpgrade to version 1.0.11 or later. The config endpoint has been moved to `/config` which avoids the `/api/*` routing conflict.\n\n**Verification:**\nCheck browser console (F12) - should see: `✅ [Config] Runtime API URL from server: https://your-domain.com`\n\n**If you can't upgrade**, explicitly configure the `/config` route:\n```nginx\n# Only needed for versions ≤ 1.0.10\nlocation = /config {\n proxy_pass http://open-notebook:8502;\n proxy_http_version 1.1;\n proxy_set_header Host $host;\n proxy_set_header X-Forwarded-Proto $scheme;\n}\n```\n\n---\n\n### File Upload Errors (413 Payload Too Large)\n\n**Symptoms:**\n```\nCORS header 'Access-Control-Allow-Origin' missing. Status code: 413.\nError creating source. Please try again.\n```\n\n**Root Cause:**\nWhen uploading files, your reverse proxy may reject the request due to body size limits *before* it reaches the application. Since the error happens at the proxy level, CORS headers are not included in the response.\n\n**Version Requirement:**\n- **Open Notebook v1.3.2+** is required for file uploads >10MB\n- Uses Next.js 16+ which supports the `proxyClientMaxBodySize` configuration option\n- Check your version: Settings → About (bottom of settings page)\n\n**Solutions:**\n\n1. **Nginx - Increase body size limit**:\n ```nginx\n server {\n # Allow larger file uploads (default is 1MB)\n client_max_body_size 100M;\n\n # Add CORS headers to error responses\n error_page 413 = @cors_error_413;\n\n location @cors_error_413 {\n add_header 'Access-Control-Allow-Origin' '*' always;\n add_header 'Access-Control-Allow-Methods' 'GET, POST, PUT, DELETE, OPTIONS' always;\n add_header 'Access-Control-Allow-Headers' '*' always;\n return 413 '{\"detail\": \"File too large. Maximum size is 100MB.\"}';\n }\n\n location / {\n # ... your existing proxy configuration\n }\n }\n ```\n\n2. **Traefik - Increase buffer size**:\n ```yaml\n # In your traefik configuration\n http:\n middlewares:\n large-body:\n buffering:\n maxRequestBodyBytes: 104857600 # 100MB\n ```\n\n Apply middleware to your router:\n ```yaml\n labels:\n - \"traefik.http.routers.notebook.middlewares=large-body\"\n ```\n\n3. **Kubernetes Ingress (nginx-ingress)**:\n ```yaml\n apiVersion: networking.k8s.io/v1\n kind: Ingress\n metadata:\n name: open-notebook\n annotations:\n nginx.ingress.kubernetes.io/proxy-body-size: \"100m\"\n # Add CORS headers for error responses\n nginx.ingress.kubernetes.io/configuration-snippet: |\n more_set_headers \"Access-Control-Allow-Origin: *\";\n ```\n\n4. **Caddy**:\n ```caddy\n notebook.example.com {\n request_body {\n max_size 100MB\n }\n reverse_proxy open-notebook:8502 {\n transport http {\n read_timeout 600s\n write_timeout 600s\n }\n }\n }\n ```\n\n**Note:** Open Notebook's API includes CORS headers in error responses, but this only works for errors that reach the application. Proxy-level errors (like 413 from nginx) need to be configured at the proxy level.\n\n---\n\n### CORS Errors\n\n**Symptoms:**\n```\nAccess-Control-Allow-Origin header is missing\nCross-Origin Request Blocked\nResponse to preflight request doesn't pass access control check\n```\n\n**Possible Causes:**\n\n1. **Missing proxy headers**:\n ```nginx\n # Make sure these are set:\n proxy_set_header X-Forwarded-Proto $scheme;\n proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;\n proxy_set_header Host $host;\n ```\n\n2. **API_URL protocol mismatch**:\n ```bash\n # Frontend is HTTPS, but API_URL is HTTP:\n API_URL=http://notebook.example.com # ❌ Wrong\n API_URL=https://notebook.example.com # ✅ Correct\n ```\n\n3. **Reverse proxy not forwarding `/api/*` correctly**:\n ```nginx\n # Make sure this works:\n location /api/ {\n proxy_pass http://open-notebook:5055/api/; # Note the trailing slash!\n }\n ```\n\n---\n\n### Missing Authorization Header\n\n**Symptoms:**\n```json\n{\"detail\": \"Missing authorization header\"}\n```\n\nThis happens when:\n- You have set `OPEN_NOTEBOOK_PASSWORD` for authentication\n- You're trying to access `/api/config` directly without logging in first\n\n**Solution:**\nThis is **expected behavior**! The frontend handles authentication automatically. Just:\n1. Access the frontend URL (not `/api/` directly)\n2. Log in through the UI\n3. The frontend will handle authorization headers for all API calls\n\n**For API integrations:** Include the password in the Authorization header:\n```bash\ncurl -H \"Authorization: Bearer your-password-here\" \\\n https://your-domain.com/api/config\n```\n\n---\n\n### SSL/TLS Certificate Errors\n\n**Symptoms:**\n- Browser shows \"Your connection is not private\"\n- Certificate warnings\n- Mixed content errors\n\n**Solutions:**\n\n1. **Use Let's Encrypt** (recommended):\n ```bash\n sudo certbot --nginx -d notebook.example.com\n ```\n\n2. **Check certificate paths** in nginx:\n ```nginx\n ssl_certificate /etc/nginx/ssl/fullchain.pem; # Full chain\n ssl_certificate_key /etc/nginx/ssl/privkey.pem; # Private key\n ```\n\n3. **Verify certificate is valid**:\n ```bash\n openssl x509 -in /etc/nginx/ssl/fullchain.pem -text -noout\n ```\n\n4. **For development**, use self-signed (not for production):\n ```bash\n openssl req -x509 -nodes -days 365 -newkey rsa:2048 \\\n -keyout ssl/privkey.pem -out ssl/fullchain.pem \\\n -subj \"/CN=localhost\"\n ```\n\n---\n\n## Best Practices\n\n1. **Always use HTTPS** in production\n2. **Set API_URL explicitly** when using reverse proxies to avoid auto-detection issues\n3. **Bind to localhost** (`127.0.0.1:8502`) and let proxy handle public access for security\n4. **Enable security headers** (HSTS, X-Frame-Options, X-Content-Type-Options, X-XSS-Protection)\n5. **Set up certificate renewal** for Let's Encrypt (usually automatic with certbot)\n6. **Keep ports 5055 and 8502 accessible** from your reverse proxy container (use Docker networks)\n7. **Use environment files** (`.env` or `docker.env`) to manage configuration securely\n8. **Test your configuration** before going live:\n - Check browser console for config messages\n - Test API: `curl https://your-domain.com/api/config`\n - Verify authentication works\n - Check long-running operations (podcast generation)\n9. **Monitor logs** regularly: `docker logs open-notebook`\n10. **Don't include `/api` in API_URL** - the system adds this automatically\n\n---\n\n## Legacy Configurations (Pre-v1.1)\n\nIf you're running Open Notebook **version 1.0.x or earlier**, you may need to use the legacy two-port configuration where you explicitly route `/api/*` to port 5055.\n\n**Check your version:**\n```bash\ndocker exec open-notebook cat /app/package.json | grep version\n```\n\n**If version < 1.1.0**, you may need:\n- Explicit `/api/*` routing to port 5055 in reverse proxy\n- Explicit `/config` endpoint routing for versions ≤ 1.0.10\n- See the \"Frontend Adds `:5055` to URL\" troubleshooting section above\n\n**Recommendation:** Upgrade to v1.1+ for simplified configuration and better performance.\n\n---\n\n## Related\n\n- **[Security Configuration](security.md)** - Password protection and hardening\n- **[Advanced Configuration](advanced.md)** - Ports, timeouts, and SSL settings\n- **[Troubleshooting](../6-TROUBLESHOOTING/connection-issues.md)** - Connection problems\n- **[Docker Deployment](../1-INSTALLATION/docker-compose.md)** - Complete deployment guide\n" }, { "path": "docs/5-CONFIGURATION/security.md", "content": "# Security Configuration\n\nProtect your Open Notebook deployment with password authentication and production hardening.\n\n---\n\n## API Key Encryption\n\nOpen Notebook encrypts API keys stored in the database using Fernet symmetric encryption (AES-128-CBC with HMAC-SHA256).\n\n### Configuration Methods\n\n| Method | Documentation |\n|--------|---------------|\n| **Settings UI** | [API Configuration Guide](../3-USER-GUIDE/api-configuration.md) |\n| **Environment Variables** | This page (below) |\n\n### Setup\n\nSet the encryption key to any secret string:\n\n```bash\n# .env or docker.env\nOPEN_NOTEBOOK_ENCRYPTION_KEY=my-secret-passphrase\n```\n\nAny string works — it will be securely derived via SHA-256 internally. Use a strong passphrase for production deployments.\n\n### Default Credentials\n\n| Setting | Default | Security Level |\n|---------|---------|----------------|\n| Password | `open-notebook-change-me` | Development only |\n| Encryption Key | **None** (must be configured) | Required for API key storage |\n\n**The encryption key has no default.** You must set `OPEN_NOTEBOOK_ENCRYPTION_KEY` before using the API key configuration feature. Without it, encrypting/decrypting API keys will fail.\n\n### Docker Secrets Support\n\nBoth settings support Docker secrets via `_FILE` suffix:\n\n```yaml\nenvironment:\n - OPEN_NOTEBOOK_PASSWORD_FILE=/run/secrets/app_password\n - OPEN_NOTEBOOK_ENCRYPTION_KEY_FILE=/run/secrets/encryption_key\n```\n\n### Security Notes\n\n| Scenario | Behavior |\n|----------|----------|\n| Key configured | API keys encrypted with your key |\n| No key configured | Encryption/decryption will fail (key is required) |\n| Key changed | Old encrypted keys become unreadable |\n| Legacy data | Unencrypted keys still work (graceful fallback) |\n\n### Key Management\n\n- **Keep secret**: Never commit the encryption key to version control\n- **Backup securely**: Store the key separately from database backups\n- **No rotation yet**: Changing the key requires re-saving all API keys\n- **Per-deployment**: Each instance should have its own encryption key\n\n---\n\n## When to Use Password Protection\n\n### Use it for:\n- Public cloud deployments (PikaPods, Railway, DigitalOcean)\n- Shared network environments\n- Any deployment accessible beyond localhost\n\n### You can skip it for:\n- Local development on your machine\n- Private, isolated networks\n- Single-user local setups\n\n---\n\n## Quick Setup\n\n### Docker Deployment\n\n```yaml\n# docker-compose.yml\nservices:\n open_notebook:\n image: lfnovo/open_notebook:v1-latest-single\n pull_policy: always\n environment:\n - OPEN_NOTEBOOK_ENCRYPTION_KEY=your-secret-encryption-key\n - OPEN_NOTEBOOK_PASSWORD=your_secure_password\n # ... rest of config\n```\n\nOr using environment file:\n\n```bash\n# docker.env\nOPEN_NOTEBOOK_ENCRYPTION_KEY=your-secret-encryption-key\nOPEN_NOTEBOOK_PASSWORD=your_secure_password\n```\n\n> **Important**: The encryption key is **required** for credential storage. Without it, you cannot save AI provider credentials via the Settings UI. If you change or lose the encryption key, all stored credentials become unreadable.\n\n### Development Setup\n\n```bash\n# .env\nOPEN_NOTEBOOK_PASSWORD=your_secure_password\n```\n\n---\n\n## Password Requirements\n\n### Good Passwords\n\n```bash\n# Strong: 20+ characters, mixed case, numbers, symbols\nOPEN_NOTEBOOK_PASSWORD=MySecure2024!Research#Tool\nOPEN_NOTEBOOK_PASSWORD=Notebook$Dev$2024$Strong!\n\n# Generated (recommended)\nOPEN_NOTEBOOK_PASSWORD=$(openssl rand -base64 24)\n```\n\n### Bad Passwords\n\n```bash\n# DON'T use these\nOPEN_NOTEBOOK_PASSWORD=password123\nOPEN_NOTEBOOK_PASSWORD=opennotebook\nOPEN_NOTEBOOK_PASSWORD=admin\n```\n\n---\n\n## How It Works\n\n### Frontend Protection\n\n1. Login form appears on first visit\n2. Password stored in browser session\n3. Session persists until browser closes\n4. Clear browser data to log out\n\n### API Protection\n\nAll API endpoints require authentication:\n\n```bash\n# Authenticated request\ncurl -H \"Authorization: Bearer your_password\" \\\n http://localhost:5055/api/notebooks\n\n# Unauthenticated (will fail)\ncurl http://localhost:5055/api/notebooks\n# Returns: {\"detail\": \"Missing authorization header\"}\n```\n\n### Unprotected Endpoints\n\nThese work without authentication:\n\n- `/health` - System health check\n- `/docs` - API documentation\n- `/openapi.json` - OpenAPI spec\n\n---\n\n## API Authentication Examples\n\n### curl\n\n```bash\n# List notebooks\ncurl -H \"Authorization: Bearer your_password\" \\\n http://localhost:5055/api/notebooks\n\n# Create notebook\ncurl -X POST \\\n -H \"Authorization: Bearer your_password\" \\\n -H \"Content-Type: application/json\" \\\n -d '{\"name\": \"My Notebook\", \"description\": \"Research notes\"}' \\\n http://localhost:5055/api/notebooks\n\n# Upload file\ncurl -X POST \\\n -H \"Authorization: Bearer your_password\" \\\n -F \"file=@document.pdf\" \\\n http://localhost:5055/api/sources/upload\n```\n\n### Python\n\n```python\nimport requests\n\nclass OpenNotebookClient:\n def __init__(self, base_url: str, password: str):\n self.base_url = base_url\n self.headers = {\"Authorization\": f\"Bearer {password}\"}\n\n def get_notebooks(self):\n response = requests.get(\n f\"{self.base_url}/api/notebooks\",\n headers=self.headers\n )\n return response.json()\n\n def create_notebook(self, name: str, description: str = None):\n response = requests.post(\n f\"{self.base_url}/api/notebooks\",\n headers=self.headers,\n json={\"name\": name, \"description\": description}\n )\n return response.json()\n\n# Usage\nclient = OpenNotebookClient(\"http://localhost:5055\", \"your_password\")\nnotebooks = client.get_notebooks()\n```\n\n### JavaScript/TypeScript\n\n```javascript\nconst API_URL = 'http://localhost:5055';\nconst PASSWORD = 'your_password';\n\nasync function getNotebooks() {\n const response = await fetch(`${API_URL}/api/notebooks`, {\n headers: {\n 'Authorization': `Bearer ${PASSWORD}`\n }\n });\n return response.json();\n}\n```\n\n---\n\n## Production Hardening\n\n### Docker Security\n\n```yaml\nservices:\n open_notebook:\n image: lfnovo/open_notebook:v1-latest-single\n pull_policy: always\n ports:\n - \"127.0.0.1:8502:8502\" # Bind to localhost only\n environment:\n - OPEN_NOTEBOOK_PASSWORD=your_secure_password\n security_opt:\n - no-new-privileges:true\n deploy:\n resources:\n limits:\n memory: 2G\n cpus: \"1.0\"\n restart: always\n```\n\n### Firewall Configuration\n\n```bash\n# UFW (Ubuntu)\nsudo ufw allow ssh\nsudo ufw allow 80/tcp\nsudo ufw allow 443/tcp\nsudo ufw deny 8502/tcp # Block direct access\nsudo ufw deny 5055/tcp # Block direct API access\nsudo ufw enable\n\n# iptables\niptables -A INPUT -p tcp --dport 22 -j ACCEPT\niptables -A INPUT -p tcp --dport 80 -j ACCEPT\niptables -A INPUT -p tcp --dport 443 -j ACCEPT\niptables -A INPUT -p tcp --dport 8502 -j DROP\niptables -A INPUT -p tcp --dport 5055 -j DROP\n```\n\n### Reverse Proxy with SSL\n\nSee [Reverse Proxy Configuration](reverse-proxy.md) for complete nginx/Caddy/Traefik setup with HTTPS.\n\n---\n\n## Security Limitations\n\nOpen Notebook's password protection provides **basic access control**, not enterprise-grade security:\n\n| Feature | Status |\n|---------|--------|\n| Password transmission | Plain text (use HTTPS!) |\n| Password storage | In memory |\n| User management | Single password for all |\n| Session timeout | None (until browser close) |\n| Rate limiting | None |\n| Audit logging | None |\n\n### Risk Mitigation\n\n1. **Always use HTTPS** - Encrypt traffic with TLS\n2. **Strong passwords** - 20+ characters, complex\n3. **Network security** - Firewall, VPN for sensitive deployments\n4. **Regular updates** - Keep containers and dependencies updated\n5. **Monitoring** - Check logs for suspicious activity\n6. **Backups** - Regular backups of data\n\n---\n\n## Enterprise Considerations\n\nFor deployments requiring advanced security:\n\n| Need | Solution |\n|------|----------|\n| SSO/OAuth | Implement OAuth2/SAML proxy |\n| Role-based access | Custom middleware |\n| Audit logging | Log aggregation service |\n| Rate limiting | API gateway or nginx |\n| Data encryption | Encrypt volumes at rest |\n| Network segmentation | Docker networks, VPC |\n\n---\n\n## Troubleshooting\n\n### Password Not Working\n\n```bash\n# Check env var is set\ndocker exec open-notebook env | grep OPEN_NOTEBOOK_PASSWORD\n\n# Check logs\ndocker logs open-notebook | grep -i auth\n\n# Test API directly\ncurl -H \"Authorization: Bearer your_password\" \\\n http://localhost:5055/health\n```\n\n### 401 Unauthorized Errors\n\n```bash\n# Check header format\ncurl -v -H \"Authorization: Bearer your_password\" \\\n http://localhost:5055/api/notebooks\n\n# Verify password matches\necho \"Password length: $(echo -n $OPEN_NOTEBOOK_PASSWORD | wc -c)\"\n```\n\n### Cannot Access After Setting Password\n\n1. Clear browser cache and cookies\n2. Try incognito/private mode\n3. Check browser console for errors\n4. Verify password is correct in environment\n\n### Security Testing\n\n```bash\n# Without password (should fail)\ncurl http://localhost:5055/api/notebooks\n# Expected: {\"detail\": \"Missing authorization header\"}\n\n# With correct password (should succeed)\ncurl -H \"Authorization: Bearer your_password\" \\\n http://localhost:5055/api/notebooks\n\n# Health check (should work without password)\ncurl http://localhost:5055/health\n```\n\n---\n\n## Reporting Security Issues\n\nIf you discover security vulnerabilities:\n\n1. **Do NOT open public issues**\n2. Contact maintainers directly\n3. Provide detailed information\n4. Allow time for fixes before disclosure\n\n---\n\n## Related\n\n- **[Reverse Proxy](reverse-proxy.md)** - HTTPS and SSL setup\n- **[Advanced Configuration](advanced.md)** - Ports, timeouts, and SSL settings\n- **[Environment Reference](environment-reference.md)** - All configuration options\n" }, { "path": "docs/6-TROUBLESHOOTING/ai-chat-issues.md", "content": "# AI & Chat Issues - Model Configuration & Quality\n\nProblems with AI models, chat, and response quality.\n\n> **Note:** Open Notebook now shows descriptive error messages for AI provider failures. Instead of a generic \"An unexpected error occurred\", you'll see specific messages like \"Authentication failed. Please check your API key\" or \"Rate limit exceeded. Please wait a moment and try again.\" These messages help you diagnose and fix issues faster.\n\n---\n\n## \"Failed to send message\" Error\n\n**Symptom:** Chat shows \"Failed to send message\" toast. Logs show:\n```\nError executing chat: Model is not a LanguageModel: None\n```\n\n**Cause:** No valid language model configured for chat\n\n**Solutions:**\n\n### Solution 1: Check Default Model Configuration\n```\n1. Go to Settings → Models\n2. Scroll to \"Default Models\" section\n3. Verify \"Default Chat Model\" has a model selected\n4. If empty, select an available language model\n5. Click Save\n```\n\n### Solution 2: Verify Model Names (Ollama Users)\n```bash\n# Get exact model names\nollama list\n\n# Example output:\n# NAME SIZE MODIFIED\n# gemma3:12b 8.1 GB 2 months ago\n\n# The model name in Open Notebook must be EXACTLY \"gemma3:12b\"\n# NOT \"gemma3\" or \"gemma3-12b\"\n```\n\n### Solution 3: Re-add Missing Models\n```\n1. Note the exact model names from your provider\n2. Go to Settings → Models\n3. Delete any misconfigured models\n4. Add models with exact names\n5. Set new defaults\n```\n\n### Solution 4: Check Model Still Exists\n```bash\n# For Ollama: verify model is installed\nollama list\n\n# For cloud providers: verify API key is valid\n# and you have access to the model\n```\n\n> **Tip:** This error often occurs when you delete a model from Ollama but forget to update the default models in Open Notebook. Always re-configure defaults after removing models.\n\n---\n\n## \"Models not available\" or \"Models not showing\"\n\n**Symptom:** Settings → Models shows empty, or \"No models configured\"\n\n**Cause:** No credential configured, or credential has invalid API key\n\n**Solutions:**\n\n### Solution 1: Add Credential via Settings UI\n```\n1. Go to Settings → API Keys\n2. Click \"Add Credential\"\n3. Select your provider (e.g., OpenAI, Anthropic, Google)\n4. Enter your API key\n5. Click Save, then Test Connection\n6. Click Discover Models → Register Models\n7. Go to Settings → Models to verify\n```\n\n### Solution 2: Check Key is Valid\n```\n1. Go to Settings → API Keys\n2. Click \"Test Connection\" on your credential\n3. If it shows \"Invalid API key\":\n - Get a fresh key from the provider's website\n - Delete the credential and create a new one\n```\n\n### Solution 3: Switch Provider\n```\n1. Go to Settings → API Keys\n2. Add a credential for a different provider\n3. Test Connection → Discover Models → Register Models\n4. Go to Settings → Models to select the new provider's models\n```\n\n---\n\n## \"Invalid API key\" or \"Unauthorized\"\n\n**Symptom:** Error when trying to chat: \"Invalid API key\"\n\n**Cause:** Credential has wrong, expired, or revoked API key\n\n**Solutions:**\n\n### Step 1: Test Your Credential\n```\n1. Go to Settings → API Keys\n2. Click \"Test Connection\" on your credential\n3. If it fails, proceed to Step 2\n```\n\n### Step 2: Get Fresh Key\n```\nGo to provider's dashboard:\n- OpenAI: https://platform.openai.com/api-keys (starts with sk-proj-)\n- Anthropic: https://console.anthropic.com/ (starts with sk-ant-)\n- Google: https://aistudio.google.com/app/apikey (starts with AIzaSy)\n\nGenerate new key and copy exactly (no extra spaces)\n```\n\n### Step 3: Update Credential\n```\n1. Go to Settings → API Keys\n2. Delete the old credential\n3. Click \"Add Credential\" → select provider\n4. Paste the new key\n5. Click Save, then Test Connection\n6. Re-discover and register models if needed\n```\n\n### Step 4: Verify in UI\n```\n1. Go to Settings → Models\n2. Verify models are available\n3. Try a test chat\n```\n\n---\n\n## Chat Returns Generic/Bad Responses\n\n**Symptom:** AI responses are shallow, generic, or wrong\n\n**Cause:** Bad context, vague question, or wrong model\n\n**Solutions:**\n\n### Solution 1: Check Context\n```\n1. In Chat, click \"Select Sources\"\n2. Verify sources you want are CHECKED\n3. Set them to \"Full Content\" (not \"Summary Only\")\n4. Click \"Save\"\n5. Try chat again\n```\n\n### Solution 2: Ask Better Question\n```\nBad: \"What do you think?\"\nGood: \"Based on the paper's methodology, what are 3 limitations?\"\n\nBad: \"Tell me about X\"\nGood: \"Summarize X in 3 bullet points with page citations\"\n```\n\n### Solution 3: Use Stronger Model\n```\nOpenAI:\n Current: gpt-4o-mini → Switch to: gpt-4o\n\nAnthropic:\n Current: claude-3-5-haiku → Switch to: claude-3-5-sonnet\n\nTo change:\n1. Settings → Models\n2. Select model\n3. Try chat again\n```\n\n### Solution 4: Add More Sources\n```\nIf: \"Response seems incomplete\"\nTry: Add more relevant sources to provide context\n```\n\n---\n\n## Chat is Very Slow\n\n**Symptom:** Chat responses take minutes\n\n**Cause:** Large context, slow model, or overloaded API\n\n**Solutions:**\n\n### Solution 1: Use Faster Model\n```bash\nFastest: Groq (any model)\nFast: OpenAI gpt-4o-mini\nMedium: Anthropic claude-3-5-haiku\nSlow: Anthropic claude-3-5-sonnet\n\nSwitch in: Settings → Models\n```\n\n### Solution 2: Reduce Context\n```\n1. Chat → Select Sources\n2. Uncheck sources you don't need\n3. Or switch to \"Summary Only\" for background sources\n4. Save and try again\n```\n\n### Solution 3: Increase Timeout\n```bash\n# In .env:\nAPI_CLIENT_TIMEOUT=600 # 10 minutes\n\n# Restart:\ndocker compose restart\n```\n\n### Solution 4: Check System Load\n```bash\n# See if API is overloaded:\ndocker stats\n\n# If CPU >80% or memory >90%:\n# Reduce: SURREAL_COMMANDS_MAX_TASKS=2\n# Restart: docker compose restart\n```\n\n---\n\n## Chat Doesn't Remember History\n\n**Symptom:** Each message treated as separate, no context between questions\n\n**Cause:** Chat history not saved or new chat started\n\n**Solution:**\n\n```\n1. Make sure you're in same Chat (not new Chat)\n2. Check Chat title at top\n3. If it's blank, start new Chat with a title\n4. Each named Chat keeps its history\n5. If you start new Chat, history is separate\n```\n\n---\n\n## \"Rate limit exceeded\"\n\n**Symptom:** Error: \"Rate limit exceeded\" or \"Too many requests\"\n\n**Cause:** Hit provider's API rate limit\n\n**Solutions:**\n\n### For Cloud Providers (OpenAI, Anthropic, etc.)\n\n**Immediate:**\n- Wait 1-2 minutes\n- Try again\n\n**Short term:**\n- Use cheaper/smaller model\n- Reduce concurrent operations\n- Space out requests\n\n**Long term:**\n- Upgrade your account\n- Switch to different provider\n- Use Ollama (local, no limits)\n\n### Check Account Status\n```\nOpenAI: https://platform.openai.com/account/usage/overview\nAnthropic: https://console.anthropic.com/account/billing/overview\nGoogle: Google Cloud Console\n```\n\n### For Ollama (Local)\n- No rate limits\n- Use `ollama pull mistral` for best model\n- Restart if hitting resource limits\n\n---\n\n## \"Context length exceeded\" or \"Token limit\"\n\n**Symptom:** Error about too many tokens\n\n**Cause:** Sources too large for model\n\n**Solutions:**\n\n### Solution 1: Use Model with Longer Context\n```\nCurrent: GPT-4o (128K tokens) → Switch to: Claude (200K tokens)\nCurrent: Claude Haiku (200K) → Switch to: Gemini (1M tokens)\n\nTo change: Settings → Models\n```\n\n### Solution 2: Reduce Context\n```\n1. Select fewer sources\n2. Or use \"Summary Only\" instead of \"Full Content\"\n3. Or split large documents into smaller pieces\n```\n\n### Solution 3: For Ollama (Local)\n```bash\n# Use smaller model:\nollama pull phi # Very small\n# Instead of: ollama pull neural-chat # Large\n```\n\n---\n\n## \"API call failed\" or Timeout\n\n**Symptom:** Generic API error, response times out\n\n**Cause:** Provider API down, network issue, or slow service\n\n**Solutions:**\n\n### Check Provider Status\n```\nOpenAI: https://status.openai.com/\nAnthropic: Check website\nGoogle: Google Cloud Status\nGroq: Check website\n```\n\n### Retry Operation\n```\n1. Wait 30 seconds\n2. Try again\n```\n\n### Use Different Model/Provider\n```\n1. Settings → Models\n2. Try different provider\n3. If OpenAI down, use Anthropic\n```\n\n### Check Network\n```bash\n# Verify internet working:\nping google.com\n\n# Test API endpoint directly:\ncurl https://api.openai.com/v1/models \\\n -H \"Authorization: Bearer YOUR_KEY\"\n```\n\n---\n\n## Responses Include Hallucinations\n\n**Symptom:** AI makes up facts that aren't in sources\n\n**Cause:** Sources not in context, or model guessing\n\n**Solutions:**\n\n### Solution 1: Verify Context\n```\n1. Click citation in response\n2. Check source actually says that\n3. If not, sources weren't in context\n4. Add source to context and try again\n```\n\n### Solution 2: Request Citations\n```\nAsk: \"Answer this with citations to specific pages\"\n\nThe AI will be more careful if asked for citations\n```\n\n### Solution 3: Use Stronger Model\n```\nWeaker models hallucinate more\nSwitch to: GPT-4o or Claude Sonnet\n```\n\n---\n\n## High API Costs\n\n**Symptom:** API bills are higher than expected\n\n**Cause:** Using expensive model, large context, many requests\n\n**Solutions:**\n\n### Use Cheaper Model\n```\nExpensive: gpt-4o\nCheaper: gpt-4o-mini (10x cheaper)\n\nExpensive: Claude Sonnet\nCheaper: Claude Haiku (5x cheaper)\n\nGroq: Ultra cheap but fewer models\n```\n\n### Reduce Context\n```\nIn Chat:\n1. Select fewer sources\n2. Use \"Summary Only\" for background\n3. Ask more specific questions\n```\n\n### Switch to Ollama (Free)\n```bash\n# Install Ollama\n# Run: ollama serve\n# Download: ollama pull mistral\n# Set: OLLAMA_API_BASE=http://localhost:11434\n# Cost: Free!\n```\n\n---\n\n## Still Having Chat Issues?\n\n- Try [Quick Fixes](quick-fixes.md)\n- Try [Chat Effectively Guide](../3-USER-GUIDE/chat-effectively.md)\n- Check logs: `docker compose logs api | grep -i \"error\"`\n- Ask for help: [Troubleshooting Index](index.md#getting-help)\n" }, { "path": "docs/6-TROUBLESHOOTING/connection-issues.md", "content": "# Connection Issues - Network & API Problems\n\nFrontend can't reach API or services won't communicate.\n\n---\n\n## \"Cannot connect to server\" (Most Common)\n\n**What it looks like:**\n- Browser shows error page\n- \"Unable to reach API\"\n- \"Cannot connect to server\"\n- UI loads but can't create notebooks\n\n**Diagnosis:**\n\n```bash\n# Check if API is running\ndocker ps | grep api\n# Should see \"api\" service running\n\n# Check if API is responding\ncurl http://localhost:5055/health\n# Should show: {\"status\":\"ok\"}\n\n# Check if frontend is running\ndocker ps | grep frontend\n# Should see \"frontend\" or React service running\n```\n\n**Solutions:**\n\n### Solution 1: API Not Running\n```bash\n# Start API\ndocker compose up api -d\n\n# Wait 5 seconds\nsleep 5\n\n# Verify it's running\ndocker compose logs api | tail -20\n```\n\n### Solution 2: Port Not Exposed\n```bash\n# Check docker-compose.yml has port mapping:\n# api:\n# ports:\n# - \"5055:5055\"\n\n# If missing, add it and restart:\ndocker compose down\ndocker compose up -d\n```\n\n### Solution 3: API_URL Mismatch\n```bash\n# In .env, check API_URL:\ncat .env | grep API_URL\n\n# Should match your frontend URL:\n# Frontend: http://localhost:8502\n# API_URL: http://localhost:5055\n\n# If wrong, fix it:\n# API_URL=http://localhost:5055\n# Then restart:\ndocker compose restart frontend\n```\n\n### Solution 4: Firewall Blocking\n```bash\n# Verify port 5055 is accessible\nnetstat -tlnp | grep 5055\n# Should show port listening\n\n# If on different machine, try:\n# Instead of localhost, use your IP:\nAPI_URL=http://192.168.1.100:5055\n```\n\n### Solution 5: Services Not Started\n```bash\n# Restart everything\ndocker compose restart\n\n# Wait 10 seconds\nsleep 10\n\n# Check all services\ndocker compose ps\n# All should show \"Up\"\n```\n\n---\n\n## Connection Refused\n\n**What it looks like:**\n```\nConnection refused\nECONNREFUSED\nError: socket hang up\n```\n\n**Diagnosis:**\n- API port (5055) not open\n- API crashed\n- Wrong IP/hostname\n\n**Solution:**\n\n```bash\n# Step 1: Check if API is running\ndocker ps | grep api\n\n# Step 2: Check if port is listening\nlsof -i :5055\n# or\nnetstat -tlnp | grep 5055\n\n# Step 3: Check API logs\ndocker compose logs api | tail -30\n# Look for errors\n\n# Step 4: Restart API\ndocker compose restart api\ndocker compose logs api | grep -i \"error\"\n```\n\n---\n\n## Timeout / Slow Connection\n\n**What it looks like:**\n- Page loads slowly\n- Request times out\n- \"Gateway timeout\" error\n\n**Causes:**\n- API is overloaded\n- Network is slow\n- Reverse proxy issue\n\n**Solutions:**\n\n### Check API Performance\n```bash\n# See CPU/memory usage\ndocker stats\n\n# Check logs for slow operations\ndocker compose logs api | grep \"slow\\|timeout\"\n```\n\n### Reduce Load\n```bash\n# In .env:\nSURREAL_COMMANDS_MAX_TASKS=2\nAPI_CLIENT_TIMEOUT=600\n\n# Restart\ndocker compose restart\n```\n\n### Check Network\n```bash\n# Test latency\nping localhost\n\n# Test API directly\ntime curl http://localhost:5055/health\n\n# Should be < 100ms\n```\n\n---\n\n## 502 Bad Gateway (Reverse Proxy)\n\n**What it looks like:**\n```\n502 Bad Gateway\nThe server is temporarily unable to service the request\n```\n\n**Cause:** Reverse proxy can't reach API\n\n**Solutions:**\n\n### Check Backend is Running\n```bash\n# From the reverse proxy server\ncurl http://localhost:5055/health\n\n# Should work\n```\n\n### Check Reverse Proxy Config\n```nginx\n# Nginx example (correct):\nlocation /api {\n proxy_pass http://localhost:5055/api;\n proxy_http_version 1.1;\n}\n\n# Common mistake (wrong):\nlocation /api {\n proxy_pass http://localhost:5055; # Missing /api\n}\n```\n\n### Set API_URL for HTTPS\n```bash\n# In .env:\nAPI_URL=https://yourdomain.com\n\n# Restart\ndocker compose restart\n```\n\n---\n\n## Intermittent Disconnects\n\n**What it looks like:**\n- Works sometimes, fails other times\n- Sporadic \"cannot connect\" errors\n- Works then stops working\n\n**Cause:** Transient network issue or database conflicts\n\n**Solutions:**\n\n### Enable Retry Logic\n```bash\n# In .env:\nSURREAL_COMMANDS_RETRY_ENABLED=true\nSURREAL_COMMANDS_RETRY_MAX_ATTEMPTS=5\nSURREAL_COMMANDS_RETRY_WAIT_STRATEGY=exponential_jitter\n\n# Restart\ndocker compose restart\n```\n\n### Reduce Concurrency\n```bash\n# In .env:\nSURREAL_COMMANDS_MAX_TASKS=2\n\n# Restart\ndocker compose restart\n```\n\n### Check Network Stability\n```bash\n# Monitor connection\nping google.com\n\n# Long-running test\nping -c 100 google.com | grep \"packet loss\"\n# Should be 0% loss\n```\n\n---\n\n## Different Machine / Remote Access\n\n**You want to access Open Notebook from another computer**\n\n**Solution:**\n\n### Step 1: Get Your Machine IP\n```bash\n# On the server running Open Notebook:\nifconfig | grep \"inet \"\n# or\nhostname -I\n# Note the IP (e.g., 192.168.1.100)\n```\n\n### Step 2: Update API_URL\n```bash\n# In .env:\nAPI_URL=http://192.168.1.100:5055\n\n# Restart\ndocker compose restart\n```\n\n### Step 3: Access from Other Machine\n```bash\n# In browser on other machine:\nhttp://192.168.1.100:8502\n# (or your server IP)\n```\n\n### Step 4: Verify Port is Exposed\n```bash\n# On server:\ndocker compose ps\n\n# Should show port mapping:\n# 0.0.0.0:8502->8502/tcp\n# 0.0.0.0:5055->5055/tcp\n```\n\n### If Still Doesn't Work\n```bash\n# Check firewall on server\nsudo ufw status\n# May need to open ports:\nsudo ufw allow 8502\nsudo ufw allow 5055\n\n# Check on different machine:\ntelnet 192.168.1.100 5055\n# Should connect\n```\n\n---\n\n## CORS Error (Browser Console)\n\n**What it looks like:**\n```\nCross-Origin Request Blocked\nAccess-Control-Allow-Origin\n```\n\n**In browser console (F12):**\n```\nCORS policy: Response to preflight request doesn't pass access control check\n```\n\n**Cause:** Frontend and API URLs don't match\n\n**Solution:**\n\n```bash\n# Check browser console error for what URLs are being used\n# The error shows:\n# - Requesting from: http://localhost:8502\n# - Trying to reach: http://localhost:5055\n\n# Make sure API_URL matches:\nAPI_URL=http://localhost:5055\n\n# And protocol matches (http/https)\n# Restart\ndocker compose restart frontend\n```\n\n---\n\n## Testing Connection\n\n**Full diagnostic:**\n\n```bash\n# 1. Services running?\ndocker compose ps\n# All should show \"Up\"\n\n# 2. Ports listening?\nnetstat -tlnp | grep -E \"8502|5055|8000\"\n\n# 3. API responding?\ncurl http://localhost:5055/health\n\n# 4. Frontend accessible?\ncurl http://localhost:8502 | head\n\n# 5. Network OK?\nping google.com\n\n# 6. No firewall?\nsudo ufw status | grep -E \"5055|8502|8000\"\n```\n\n---\n\n## Checklist for Remote Access\n\n- [ ] Server IP noted (e.g., 192.168.1.100)\n- [ ] Ports 8502, 5055, 8000 exposed in docker-compose\n- [ ] API_URL set to server IP\n- [ ] Firewall allows ports 8502, 5055, 8000\n- [ ] Can reach server from client machine (ping IP)\n- [ ] All services running (docker compose ps)\n- [ ] Can curl API from client (curl http://IP:5055/health)\n\n---\n\n## SSL Certificate Errors\n\n**What it looks like:**\n```\n[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed\nConnection error when using HTTPS endpoints\nWorks with HTTP but fails with HTTPS\n```\n\n**Cause:** Self-signed certificates not trusted by Python's SSL verification\n\n**Solutions:**\n\n### Solution 1: Use Custom CA Bundle (Recommended)\n```bash\n# In .env:\nESPERANTO_SSL_CA_BUNDLE=/path/to/your/ca-bundle.pem\n\n# For Docker, mount the certificate:\n# In docker-compose.yml:\nvolumes:\n - /path/to/your/ca-bundle.pem:/certs/ca-bundle.pem:ro\nenvironment:\n - ESPERANTO_SSL_CA_BUNDLE=/certs/ca-bundle.pem\n```\n\n### Solution 2: Disable SSL Verification (Development Only)\n```bash\n# WARNING: Only use in trusted development environments\n# In .env:\nESPERANTO_SSL_VERIFY=false\n```\n\n### Solution 3: Use HTTP Instead\nIf services are on a trusted local network, HTTP is acceptable:\n```\nChange the base URL in your credential (Settings → API Keys) from https:// to http://\nExample: http://localhost:1234/v1\n```\n\n> **Security Note:** Disabling SSL verification exposes you to man-in-the-middle attacks. Always prefer custom CA bundle or HTTP on trusted networks.\n\n---\n\n## Still Having Issues?\n\n- Check [Quick Fixes](quick-fixes.md)\n- Check [FAQ](faq.md)\n- Check logs: `docker compose logs`\n- Try restart: `docker compose restart`\n- Check firewall: `sudo ufw status`\n- Ask for help on [Discord](https://discord.gg/37XJPXfz2w)\n" }, { "path": "docs/6-TROUBLESHOOTING/faq.md", "content": "# Frequently Asked Questions\n\nCommon questions about Open Notebook usage, configuration, and best practices.\n\n---\n\n## General Usage\n\n### What is Open Notebook?\n\nOpen Notebook is an open-source, privacy-focused alternative to Google's Notebook LM. It allows you to:\n- Create and manage research notebooks\n- Chat with your documents using AI\n- Generate podcasts from your content\n- Search across all your sources with semantic search\n- Transform and analyze your content\n\n### How is it different from Google Notebook LM?\n\n**Privacy**: Your data stays local by default. Only your chosen AI providers receive queries.\n**Flexibility**: Support for 15+ AI providers (OpenAI, Anthropic, Google, local models, etc.)\n**Customization**: Open source, so you can modify and extend functionality\n**Control**: You control your data, models, and processing\n\n### Can I use Open Notebook offline?\n\n**Partially**: The application runs locally, but requires internet for:\n- AI model API calls (unless using local models like Ollama)\n- Web content scraping\n\n**Fully offline**: Possible with local models (Ollama) for basic functionality.\n\n### What file types are supported?\n\n**Documents**: PDF, DOCX, TXT, Markdown\n**Web Content**: URLs, YouTube videos\n**Media**: MP3, WAV, M4A (audio), MP4, AVI, MOV (video)\n**Other**: Direct text input, CSV, code files\n\n### How much does it cost?\n\n**Software**: Free (open source)\n**AI API costs**: Pay-per-use to providers:\n- OpenAI: ~$0.50-5 per 1M tokens\n- Anthropic: ~$3-75 per 1M tokens\n- Google: Often free tier available\n- Local models: Free after initial setup\n\n**Typical monthly costs**: $5-50 for moderate usage.\n\n---\n\n## AI Models and Providers\n\n### Which AI provider should I choose?\n\n**For beginners**: OpenAI (reliable, well-documented)\n**For privacy**: Local models (Ollama) or European providers (Mistral)\n**For cost optimization**: Groq, Google (free tier), or OpenRouter\n**For long context**: Anthropic (200K tokens) or Google Gemini (1M tokens)\n\n### Can I use multiple providers?\n\n**Yes**: Configure different providers for different tasks:\n- OpenAI for chat\n- Google for embeddings\n- ElevenLabs for text-to-speech\n- Anthropic for complex reasoning\n\n### What are the best model combinations?\n\n**Budget-friendly**:\n- Language: `gpt-4o-mini` (OpenAI) or `deepseek-chat`\n- Embedding: `text-embedding-3-small` (OpenAI)\n\n**High-quality**:\n- Language: `claude-3-5-sonnet` (Anthropic) or `gpt-4o` (OpenAI)\n- Embedding: `text-embedding-3-large` (OpenAI)\n\n**Privacy-focused**:\n- Language: Local Ollama models (mistral, llama3)\n- Embedding: Local embedding models\n\n### How do I optimize AI costs?\n\n**Model selection**:\n- Use smaller models for simple tasks (gpt-4o-mini, claude-3-5-haiku)\n- Use larger models only for complex reasoning\n- Leverage free tiers when available\n\n**Usage optimization**:\n- Use \"Summary Only\" context for background sources\n- Ask more specific questions\n- Use local models (Ollama) for frequent tasks\n\n---\n\n## Data Management\n\n### Where is my data stored?\n\n**Local storage**: By default, all data is stored locally:\n- Database: SurrealDB files in `surreal_data/`\n- Uploads: Files in `data/uploads/`\n- Podcasts: Generated audio in `data/podcasts/`\n- No external data transmission (except to chosen AI providers)\n\n### How do I backup my data?\n\n```bash\n# Create backup\ntar -czf backup-$(date +%Y%m%d).tar.gz data/ surreal_data/\n\n# Restore backup\ntar -xzf backup-20240101.tar.gz\n```\n\n### Can I sync data between devices?\n\n**Currently**: No built-in sync functionality.\n**Workarounds**:\n- Use shared network storage for data directories\n- Manual backup/restore between devices\n\n### What happens if I delete a notebook?\n\n**Soft deletion**: Notebooks are marked as archived, not permanently deleted.\n**Recovery**: Archived notebooks can be restored from the database.\n\n---\n\n## Best Practices\n\n### How should I organize my notebooks?\n\n- **By topic**: Separate notebooks for different research areas\n- **By project**: One notebook per project or course\n- **By time period**: Monthly or quarterly notebooks\n\n**Recommended size**: 20-100 sources per notebook for best performance.\n\n### How do I get the best search results?\n\n- Use descriptive queries (\"data analysis methods\" not just \"data\")\n- Combine multiple related terms\n- Use natural language (ask questions as you would to a human)\n- Try both text search (keywords) and vector search (concepts)\n\n### How can I improve chat responses?\n\n- Provide context: Reference specific sources or topics\n- Be specific: Ask detailed questions rather than general ones\n- Request citations: \"Answer with page citations\"\n- Use follow-up questions: Build on previous responses\n\n### What are the security best practices?\n\n- Never share API keys publicly\n- Use `OPEN_NOTEBOOK_PASSWORD` for public deployments\n- Use HTTPS for production (via reverse proxy)\n- Keep Docker images updated\n- Encrypt backups if they contain sensitive data\n\n---\n\n## Technical Questions\n\n### Can I use Open Notebook programmatically?\n\n**Yes**: Open Notebook provides a REST API:\n- Full API documentation at `http://localhost:5055/docs`\n- Support for all UI functionality\n- Authentication via password header\n\n### Can I run Open Notebook in production?\n\n**Yes**: Designed for production use with:\n- Docker deployment\n- Security features (password protection)\n- Monitoring and logging\n- Reverse proxy support (nginx, Caddy, Traefik)\n\n### What are the system requirements?\n\n**Minimum**:\n- 4GB RAM\n- 2 CPU cores\n- 10GB disk space\n\n**Recommended**:\n- 8GB+ RAM\n- 4+ CPU cores\n- SSD storage\n- For local models: 16GB+ RAM, GPU recommended\n\n---\n\n## Timeout and Performance\n\n### Why do I get timeout errors?\n\n**Common causes**:\n- Large context (too many sources)\n- Slow AI provider\n- Local models on CPU (slow)\n- First request (model loading)\n\n**Solutions**:\n```bash\n# In .env:\nAPI_CLIENT_TIMEOUT=600 # 10 minutes for slow setups\nESPERANTO_LLM_TIMEOUT=180 # 3 minutes for model inference\n```\n\n### Recommended timeouts by setup:\n\n| Setup | API_CLIENT_TIMEOUT |\n|-------|-------------------|\n| Cloud APIs (OpenAI, Anthropic) | 300 (default) |\n| Local Ollama with GPU | 600 |\n| Local Ollama with CPU | 1200 |\n| Remote LM Studio | 900 |\n\n---\n\n## Getting Help\n\n### My question isn't answered here\n\n1. Check the troubleshooting guides in this section\n2. Search existing GitHub issues\n3. Ask in the Discord community\n4. Create a GitHub issue with detailed information\n\n### How do I report a bug?\n\nInclude:\n- Steps to reproduce\n- Expected vs actual behavior\n- Error messages and logs\n- System information\n- Configuration details (without API keys)\n\nSubmit to: [GitHub Issues](https://github.com/lfnovo/open-notebook/issues)\n\n### Where can I get help?\n\n- **Discord**: https://discord.gg/37XJPXfz2w (fastest)\n- **GitHub Issues**: Bug reports and feature requests\n- **Documentation**: This docs site\n\n---\n\n## Related\n\n- [Quick Fixes](quick-fixes.md) - Common issues with 1-minute solutions\n- [AI & Chat Issues](ai-chat-issues.md) - Model and chat problems\n- [Connection Issues](connection-issues.md) - Network and API problems\n" }, { "path": "docs/6-TROUBLESHOOTING/index.md", "content": "# Troubleshooting - Problem Solving Guide\n\nHaving issues? Use this guide to diagnose and fix problems.\n\n---\n\n## How to Use This Guide\n\n**Step 1: Identify your problem**\n- What's the symptom? (error message, behavior, something not working?)\n- When did it happen? (during install, while using, after update?)\n\n**Step 2: Find the right guide**\n- Look below for your symptom\n- Go to the specific troubleshooting guide\n\n**Step 3: Follow the steps**\n- Guides are organized by symptom, not by root cause\n- Each has diagnostic steps and solutions\n\n---\n\n## Quick Problem Map\n\n### During Installation\n\n- **Docker won't start** → [Quick Fixes](quick-fixes.md#9-services-wont-start-or-docker-error)\n- **Port already in use** → [Quick Fixes](quick-fixes.md#3-port-x-already-in-use)\n- **Permission denied** → [Quick Fixes](quick-fixes.md#9-services-wont-start-or-docker-error)\n- **Can't connect to database** → [Connection Issues](connection-issues.md)\n\n### When Starting\n\n- **API won't start** → [Quick Fixes](quick-fixes.md#9-services-wont-start-or-docker-error)\n- **Frontend won't load** → [Connection Issues](connection-issues.md)\n- **\"Cannot connect to server\" error** → [Connection Issues](connection-issues.md)\n\n### Settings / Configuration\n\n- **Models not showing** → [AI & Chat Issues](ai-chat-issues.md)\n- **\"Invalid API key\"** → [AI & Chat Issues](ai-chat-issues.md)\n- **Can't find Settings** → [Quick Fixes](quick-fixes.md)\n\n### Using Features\n\n- **Chat not working** → [AI & Chat Issues](ai-chat-issues.md)\n- **Chat responses are slow** → [AI & Chat Issues](ai-chat-issues.md)\n- **Chat gives bad answers** → [AI & Chat Issues](ai-chat-issues.md)\n\n### Adding Content\n\n- **Can't upload PDF** → [Quick Fixes](quick-fixes.md#4-cannot-process-file-or-unsupported-format)\n- **File won't process** → [Quick Fixes](quick-fixes.md#4-cannot-process-file-or-unsupported-format)\n- **Web link won't extract** → [Quick Fixes](quick-fixes.md#4-cannot-process-file-or-unsupported-format)\n\n### Search\n\n- **Search returns no results** → [Quick Fixes](quick-fixes.md#7-search-returns-nothing)\n- **Search returns wrong results** → [Quick Fixes](quick-fixes.md#7-search-returns-nothing)\n\n### Podcasts\n\n- **Can't generate podcast** → [Quick Fixes](quick-fixes.md#8-podcast-generation-failed)\n- **Podcast shows \"FAILED\" badge** → Check the error message displayed on the episode, then use the **Retry** button. See [Podcasts Explained](../2-CORE-CONCEPTS/podcasts-explained.md#when-things-go-wrong-failures--retry)\n- **Podcast audio is robotic** → [Quick Fixes](quick-fixes.md#8-podcast-generation-failed)\n- **Podcast generation times out** → [Quick Fixes](quick-fixes.md#8-podcast-generation-failed)\n\n---\n\n## Troubleshooting by Error Message\n\n### \"Cannot connect to server\"\n→ [Connection Issues](connection-issues.md) — Frontend can't reach API\n\n### \"Invalid API key\"\n→ [AI & Chat Issues](ai-chat-issues.md) — Wrong or missing API key\n\n### \"Models not available\"\n→ [AI & Chat Issues](ai-chat-issues.md) — Model not configured\n\n### \"Connection refused\"\n→ [Connection Issues](connection-issues.md) — Service not running or port wrong\n\n### \"Port already in use\"\n→ [Quick Fixes](quick-fixes.md#3-port-x-already-in-use) — Port conflict\n\n### \"Permission denied\"\n→ [Quick Fixes](quick-fixes.md#9-services-wont-start-or-docker-error) — File permissions issue\n\n### \"Unsupported file type\"\n→ [Quick Fixes](quick-fixes.md#4-cannot-process-file-or-unsupported-format) — File format not supported\n\n### \"Processing timeout\"\n→ [Quick Fixes](quick-fixes.md#5-chat-is-very-slow) — File too large or slow processing\n\n---\n\n## Troubleshooting by Component\n\n### Frontend (Browser/UI)\n- Can't access UI → [Connection Issues](connection-issues.md)\n- UI is slow → [Quick Fixes](quick-fixes.md)\n- Button/feature missing → [Quick Fixes](quick-fixes.md)\n\n### API (Backend)\n- API won't start → [Quick Fixes](quick-fixes.md#9-services-wont-start-or-docker-error)\n- API errors in logs → [Quick Fixes](quick-fixes.md#9-services-wont-start-or-docker-error)\n- API is slow → [Quick Fixes](quick-fixes.md)\n\n### Database\n- Can't connect to database → [Connection Issues](connection-issues.md)\n- Data lost after restart → [FAQ](faq.md#how-do-i-backup-my-data)\n\n### AI / Chat\n- Chat not working → [AI & Chat Issues](ai-chat-issues.md)\n- Bad responses → [AI & Chat Issues](ai-chat-issues.md)\n- Cost too high → [AI & Chat Issues](ai-chat-issues.md#high-api-costs)\n\n### Sources\n- Can't upload file → [Quick Fixes](quick-fixes.md#4-cannot-process-file-or-unsupported-format)\n- File won't process → [Quick Fixes](quick-fixes.md#4-cannot-process-file-or-unsupported-format)\n\n### Podcasts\n- Won't generate → [Quick Fixes](quick-fixes.md#8-podcast-generation-failed)\n- Bad audio quality → [Quick Fixes](quick-fixes.md#8-podcast-generation-failed)\n\n---\n\n## Diagnostic Checklist\n\n**When something isn't working:**\n\n- [ ] Check if services are running: `docker ps`\n- [ ] Check logs: `docker compose logs api` (or frontend, surrealdb)\n- [ ] Verify ports are exposed: `netstat -tlnp` or `lsof -i :5055`\n- [ ] Test connectivity: `curl http://localhost:5055/health`\n- [ ] Check environment variables: `docker inspect `\n- [ ] Try restarting: `docker compose restart`\n- [ ] Check firewall/antivirus isn't blocking\n\n---\n\n## Getting Help\n\nIf you can't find the answer here:\n\n1. **Check the relevant guide** — Read completely, try all steps\n2. **Check the FAQ** — [Frequently Asked Questions](faq.md)\n3. **Search our Discord** — Others may have had same issue\n4. **Check logs** — Most issues show error messages in logs\n5. **Report on GitHub** — Include error message, steps to reproduce\n\n### How to Report an Issue\n\nInclude:\n1. Error message (exact)\n2. Steps to reproduce\n3. Logs: `docker compose logs`\n4. Your setup: Docker/local, provider, OS\n5. What you've already tried\n\n→ [Report on GitHub](https://github.com/lfnovo/open-notebook/issues)\n\n---\n\n## Guides\n\n### [Quick Fixes](quick-fixes.md)\nTop 10 most common issues with 1-minute solutions.\n\n### [Connection Issues](connection-issues.md)\nFrontend can't reach API, network problems.\n\n### [AI & Chat Issues](ai-chat-issues.md)\nChat not working, bad responses, slow performance.\n\n### [FAQ](faq.md)\nFrequently asked questions about usage, costs, and best practices.\n\n---\n\n## Common Solutions\n\n**Service won't start?**\n```bash\n# Check logs\ndocker compose logs\n\n# Restart everything\ndocker compose restart\n\n# Nuclear option: rebuild\ndocker compose down\ndocker compose up --build\n```\n\n**Port conflict?**\n```bash\n# Find what's using port 5055\nlsof -i :5055\n# Kill it or use different port\n```\n\n**Can't connect?**\n```bash\n# Test API directly\ncurl http://localhost:5055/health\n# Should return: {\"status\":\"ok\"}\n```\n\n**Slow performance?**\n```bash\n# Check resource usage\ndocker stats\n\n# Reduce concurrency in .env\nSURREAL_COMMANDS_MAX_TASKS=2\n```\n\n**High costs?**\n```bash\n# Switch to cheaper model\n# In Settings → Models → Choose gpt-4o-mini (OpenAI)\n# Or use Ollama (free)\n```\n\n---\n\n## Still Stuck?\n\n**Before asking for help:**\n1. Read the relevant guide completely\n2. Try all the steps\n3. Check the logs\n4. Restart services\n5. Search existing issues on GitHub\n\n**Then:**\n- **Discord**: https://discord.gg/37XJPXfz2w (fastest response)\n- **GitHub Issues**: https://github.com/lfnovo/open-notebook/issues\n" }, { "path": "docs/6-TROUBLESHOOTING/quick-fixes.md", "content": "# Quick Fixes - Top 11 Issues & Solutions\n\nCommon problems with 1-minute solutions.\n\n---\n\n## #1: \"Cannot connect to server\"\n\n**Symptom:** Browser shows error \"Cannot connect to server\" or \"Unable to reach API\"\n\n**Cause:** Frontend can't reach API\n\n**Solution (1 minute):**\n\n```bash\n# Step 1: Check if API is running\ndocker ps | grep api\n\n# Step 2: Verify port 5055 is accessible\ncurl http://localhost:5055/health\n\n# Expected output: {\"status\":\"ok\"}\n\n# If that doesn't work:\n# Step 3: Restart services\ndocker compose restart\n\n# Step 4: Try again\n# Open http://localhost:8502 in browser\n```\n\n**If still broken:**\n- Check `API_URL` in .env (should match your frontend URL)\n- See [Connection Issues](connection-issues.md)\n\n---\n\n## #2: \"Invalid API key\" or \"Models not showing\"\n\n**Symptom:** Settings → Models shows \"No models available\"\n\n**Cause:** No credential configured, or credential has invalid API key\n\n**Solution (1 minute):**\n\n```\n1. Go to Settings → API Keys\n2. If no credential exists, click \"Add Credential\" and add one\n3. If a credential exists, click \"Test Connection\"\n4. If test fails, delete and re-create with correct key\n5. After test passes, click \"Discover Models\" → \"Register Models\"\n6. Go to Settings → Models to verify models appear\n```\n\n**If still broken:**\n- Make sure key has no extra spaces\n- Generate a fresh key from provider dashboard\n- Check that `OPEN_NOTEBOOK_ENCRYPTION_KEY` is set in docker-compose.yml\n- See [AI & Chat Issues](ai-chat-issues.md)\n\n---\n\n## #3: \"Port X already in use\"\n\n**Symptom:** Docker error \"Port 8502 is already allocated\"\n\n**Cause:** Another service using that port\n\n**Solution (1 minute):**\n\n```bash\n# Option 1: Stop the other service\n# Find what's using port 8502\nlsof -i :8502\n# Kill it or close the app\n\n# Option 2: Use different port\n# Edit docker-compose.yml\n# Change: - \"8502:8502\"\n# To: - \"8503:8502\"\n\n# Then restart\ndocker compose restart\n# Access at: http://localhost:8503\n```\n\n---\n\n## #4: \"Cannot process file\" or \"Unsupported format\"\n\n**Symptom:** Upload fails or says \"File format not supported\"\n\n**Cause:** File type not supported or too large\n\n**Solution (1 minute):**\n\n```bash\n# Check if file format is supported:\n# ✓ PDF, DOCX, PPTX, XLSX (documents)\n# ✓ MP3, WAV, M4A (audio)\n# ✓ MP4, AVI, MOV (video)\n# ✓ URLs/web links\n\n# ✗ Pure images (.jpg without OCR)\n# ✗ Files > 100MB\n\n# Try these:\n# - Convert to PDF if possible\n# - Split large files\n# - Try uploading again\n```\n\n---\n\n## #5: \"Chat is very slow\"\n\n**Symptom:** Chat responses take minutes or timeout\n\n**Cause:** Slow AI provider, large context, or overloaded system\n\n**Solution (1 minute):**\n\n```bash\n# Step 1: Check which model you're using\n# Settings → Models\n# Note the model name\n\n# Step 2: Try a cheaper/faster model\n# OpenAI: Switch to gpt-4o-mini (10x cheaper, slightly faster)\n# Anthropic: Switch to claude-3-5-haiku (fastest)\n# Groq: Use any model (ultra-fast)\n\n# Step 3: Reduce context\n# Chat: Select fewer sources\n# Use \"Summary Only\" instead of \"Full Content\"\n\n# Step 4: Check if API is overloaded\ndocker stats\n# Look at CPU/memory usage\n```\n\nFor deep dive: See [AI & Chat Issues](ai-chat-issues.md)\n\n---\n\n## #6: \"Chat gives bad responses\"\n\n**Symptom:** AI responses are generic, wrong, or irrelevant\n\n**Cause:** Bad context, vague question, or wrong model\n\n**Solution (1 minute):**\n\n```bash\n# Step 1: Make sure sources are in context\n# Click \"Select Sources\" in Chat\n# Verify relevant sources are checked and set to \"Full Content\"\n\n# Step 2: Ask a specific question\n# Bad: \"What do you think?\"\n# Good: \"Based on the paper's methodology section, what are the 3 main limitations?\"\n\n# Step 3: Try a more powerful model\n# OpenAI: Use gpt-4o (better reasoning)\n# Anthropic: Use claude-3-5-sonnet (best reasoning)\n\n# Step 4: Check citations\n# Click citations to verify AI actually saw those sources\n```\n\nFor detailed help: See [Chat Effectively](../3-USER-GUIDE/chat-effectively.md)\n\n---\n\n## #7: \"Search returns nothing\"\n\n**Symptom:** Search shows 0 results even though content exists\n\n**Cause:** Wrong search type or poor query\n\n**Solution (1 minute):**\n\n```bash\n# Try a different search type:\n\n# If you searched with KEYWORDS:\n# Try VECTOR SEARCH instead\n# (Concept-based, not keyword-based)\n\n# If you searched for CONCEPTS:\n# Try TEXT SEARCH instead\n# (Look for specific words in your query)\n\n# Try simpler search:\n# Instead of: \"How do transformers work in neural networks?\"\n# Try: \"transformers\" or \"neural networks\"\n\n# Check sources are processed:\n# Go to notebook\n# All sources should show green \"Ready\" status\n```\n\nFor detailed help: See [Search Effectively](../3-USER-GUIDE/search.md)\n\n---\n\n## #8: \"Podcast generation failed\"\n\n**Symptom:** \"Podcast generation failed\" error\n\n**Cause:** Insufficient content, API quota, or network issue\n\n**Solution (1 minute):**\n\n```bash\n# Step 1: Make sure you have content\n# Select at least 1-2 sources\n# Avoid single-sentence sources\n\n# Step 2: Try again\n# Sometimes it's a temporary API issue\n# Wait 30 seconds and retry\n\n# Step 3: Check your TTS provider has quota\n# OpenAI: Check account has credits\n# ElevenLabs: Check monthly quota\n# Google: Check API quota\n\n# Step 4: Try different TTS provider\n# In podcast generation, choose \"Google\" or \"Local\"\n# instead of \"ElevenLabs\"\n```\n\nFor detailed help: See [FAQ](faq.md)\n\n---\n\n## #9: \"Services won't start\" or Docker error\n\n**Symptom:** Docker error when running `docker compose up`\n\n**Cause:** Corrupt configuration, permission issue, or resource issue\n\n**Solution (1 minute):**\n\n```bash\n# Step 1: Check logs\ndocker compose logs\n\n# Step 2: Try restart\ndocker compose restart\n\n# Step 3: If that fails, rebuild\ndocker compose down\ndocker compose up --build\n\n# Step 4: Check disk space\ndf -h\n# Need at least 5GB free\n\n# Step 5: Check Docker has enough memory\n# Docker settings → Resources → Memory: 4GB+\n```\n\n---\n\n## #10: \"Database says 'too many connections'\"\n\n**Symptom:** Error about database connections\n\n**Cause:** Too many concurrent operations\n\n**Solution (1 minute):**\n\n```bash\n# In .env, reduce concurrency:\nSURREAL_COMMANDS_MAX_TASKS=2\n\n# Then restart:\ndocker compose restart\n\n# This makes it slower but more stable\n```\n\n---\n\n## #11: Slow Startup or Download Timeouts (China/Slow Networks)\n\n**Symptom:** Container crashes on startup, worker enters FATAL state, or pip/uv downloads fail\n\n**Cause:** Slow network or restricted access to Python package repositories\n\n**Solution:**\n\n### Increase Download Timeout\n```yaml\n# In docker-compose.yml environment:\nenvironment:\n - UV_HTTP_TIMEOUT=600 # 10 minutes (default is 30s)\n```\n\n### Use Chinese Mirrors (if in China)\n```yaml\nenvironment:\n - UV_HTTP_TIMEOUT=600\n - UV_INDEX_URL=https://pypi.tuna.tsinghua.edu.cn/simple\n - PIP_INDEX_URL=https://pypi.tuna.tsinghua.edu.cn/simple\n```\n\n**Alternative Chinese mirrors:**\n- Tsinghua: `https://pypi.tuna.tsinghua.edu.cn/simple`\n- Aliyun: `https://mirrors.aliyun.com/pypi/simple/`\n- Huawei: `https://repo.huaweicloud.com/repository/pypi/simple`\n\n**Note:** First startup may take several minutes while dependencies download. Subsequent starts will be faster.\n\n---\n\n## Quick Troubleshooting Checklist\n\nWhen something breaks:\n\n- [ ] **Restart services:** `docker compose restart`\n- [ ] **Check logs:** `docker compose logs`\n- [ ] **Verify connectivity:** `curl http://localhost:5055/health`\n- [ ] **Check .env:** API keys set? API_URL correct?\n- [ ] **Check resources:** `docker stats` (CPU/memory)\n- [ ] **Clear cache:** `docker system prune` (free space)\n- [ ] **Rebuild if needed:** `docker compose up --build`\n\n---\n\n## Nuclear Options (Last Resort)\n\n**Completely reset (will lose all data in Docker):**\n\n```bash\ndocker compose down -v\ndocker compose up --build\n```\n\n**Reset to defaults:**\n```bash\n# Backup your .env first!\ncp .env .env.backup\n\n# Reset to example\ncp .env.example .env\n\n# Edit with your API keys\n# Restart\ndocker compose up\n```\n\n---\n\n## Prevention Tips\n\n1. **Keep backups** — Export your notebooks regularly\n2. **Monitor logs** — Check `docker compose logs` periodically\n3. **Update regularly** — Pull latest image: `docker pull lfnovo/open_notebook:latest`\n4. **Document changes** — Keep notes on what you configured\n5. **Test after updates** — Verify everything works\n\n---\n\n## Still Stuck?\n\n- **Look up your exact error** in [Troubleshooting Index](index.md)\n- **Check the FAQ** in [FAQ](faq.md)\n- **Check logs:** `docker compose logs | head -50`\n- **Ask for help:** [Discord](https://discord.gg/37XJPXfz2w) or [GitHub Issues](https://github.com/lfnovo/open-notebook/issues)\n" }, { "path": "docs/7-DEVELOPMENT/api-reference.md", "content": "# API Reference\n\nComplete REST API for Open Notebook. All endpoints are served from the API backend (default: `http://localhost:5055`).\n\n**Base URL**: `http://localhost:5055` (development) or environment-specific production URL\n\n**Interactive Docs**: Use FastAPI's built-in Swagger UI at `http://localhost:5055/docs` for live testing and exploration. This is the primary reference for all endpoints, request/response schemas, and real-time testing.\n\n---\n\n## Quick Start\n\n### 1. Authentication\n\nSimple password-based (development only):\n\n```bash\ncurl http://localhost:5055/api/notebooks \\\n -H \"Authorization: Bearer your_password\"\n```\n\n**⚠️ Production**: Replace with OAuth/JWT. See [Security Configuration](../5-CONFIGURATION/security.md) for details.\n\n### 2. Base API Flow\n\nMost operations follow this pattern:\n1. Create a **Notebook** (container for research)\n2. Add **Sources** (PDFs, URLs, text)\n3. Query via **Chat** or **Search**\n4. View results and **Notes**\n\n### 3. Testing Endpoints\n\nInstead of memorizing endpoints, use the interactive API docs:\n- Navigate to `http://localhost:5055/docs`\n- Try requests directly in the browser\n- See request/response schemas in real-time\n- Test with your own data\n\n---\n\n## API Endpoints Overview\n\n### Main Resource Types\n\n**Notebooks** - Research projects containing sources and notes\n- `GET/POST /notebooks` - List and create\n- `GET/PUT/DELETE /notebooks/{id}` - Read, update, delete\n\n**Sources** - Content items (PDFs, URLs, text)\n- `GET/POST /sources` - List and add content\n- `GET /sources/{id}` - Fetch source details\n- `POST /sources/{id}/retry` - Retry failed processing\n- `GET /sources/{id}/download` - Download original file\n\n**Notes** - User-created or AI-generated research notes\n- `GET/POST /notes` - List and create\n- `GET/PUT/DELETE /notes/{id}` - Read, update, delete\n\n**Chat** - Conversational AI interface\n- `GET/POST /chat/sessions` - Manage chat sessions\n- `POST /chat/execute` - Send message and get response\n- `POST /chat/context/build` - Prepare context for chat\n\n**Search** - Find content by text or semantic similarity\n- `POST /search` - Full-text or vector search\n- `POST /ask` - Ask a question (search + synthesize)\n\n**Transformations** - Custom prompts for extracting insights\n- `GET/POST /transformations` - Create custom extraction rules\n- `POST /sources/{id}/insights` - Apply transformation to source\n\n**Models** - Configure AI providers\n- `GET /models` - Available models\n- `GET /models/defaults` - Current defaults\n- `POST /models/config` - Set defaults\n\n**Credentials** - Manage AI provider credentials\n- `GET/POST /credentials` - List and create credentials\n- `GET/PUT/DELETE /credentials/{id}` - CRUD operations\n- `POST /credentials/{id}/test` - Test connection\n- `POST /credentials/{id}/discover` - Discover models from provider\n- `POST /credentials/{id}/register-models` - Register discovered models\n- `GET /credentials/status` - Provider status overview\n- `GET /credentials/env-status` - Environment variable status\n- `POST /credentials/migrate-from-env` - Migrate env vars to credentials\n\n**Health & Status**\n- `GET /health` - Health check\n- `GET /commands/{id}` - Track async operations\n\n---\n\n## Authentication\n\n### Current (Development)\n\nAll requests require password header:\n\n```bash\ncurl -H \"Authorization: Bearer your_password\" http://localhost:5055/api/notebooks\n```\n\nPassword configured via `OPEN_NOTEBOOK_PASSWORD` environment variable.\n\n> **📖 See [Security Configuration](../5-CONFIGURATION/security.md)** for complete authentication setup, API examples, and production hardening.\n\n### Production\n\n**⚠️ Not secure.** Replace with:\n- OAuth 2.0 (recommended)\n- JWT tokens\n- API keys\n\nSee [Security Configuration](../5-CONFIGURATION/security.md) for production setup.\n\n---\n\n## Common Patterns\n\n### Pagination\n\n```bash\n# List sources with limit/offset\ncurl 'http://localhost:5055/sources?limit=20&offset=10'\n```\n\n### Filtering & Sorting\n\n```bash\n# Filter by notebook, sort by date\ncurl 'http://localhost:5055/sources?notebook_id=notebook:abc&sort_by=created&sort_order=asc'\n```\n\n### Async Operations\n\nSome operations (source processing, podcast generation) return immediately with a command ID:\n\n```bash\n# Submit async operation\ncurl -X POST http://localhost:5055/sources -F async_processing=true\n# Response: {\"id\": \"source:src001\", \"command_id\": \"command:cmd123\"}\n\n# Poll status\ncurl http://localhost:5055/commands/command:cmd123\n```\n\n### Streaming Responses\n\nThe `/ask` endpoint streams responses as Server-Sent Events:\n\n```bash\ncurl -N 'http://localhost:5055/ask' \\\n -H \"Content-Type: application/json\" \\\n -d '{\"question\": \"What is AI?\"}'\n\n# Outputs: data: {\"type\":\"strategy\",...}\n# data: {\"type\":\"answer\",...}\n# data: {\"type\":\"final_answer\",...}\n```\n\n### Multipart File Upload\n\n```bash\ncurl -X POST http://localhost:5055/sources \\\n -F \"type=upload\" \\\n -F \"notebook_id=notebook:abc\" \\\n -F \"file=@document.pdf\"\n```\n\n---\n\n## Error Handling\n\nAll errors return JSON with status code:\n\n```json\n{\"detail\": \"Notebook not found\"}\n```\n\n### Common Status Codes\n\n| Code | Meaning | Example |\n|------|---------|---------|\n| 200 | Success | Operation completed |\n| 400 | Bad Request | Invalid input |\n| 404 | Not Found | Resource doesn't exist |\n| 409 | Conflict | Resource already exists |\n| 500 | Server Error | Database/processing error |\n\n---\n\n## Tips for Developers\n\n1. **Start with interactive docs** (`http://localhost:5055/docs`) - this is the definitive reference\n2. **Enable logging** for debugging (check API logs: `docker logs`)\n3. **Streaming endpoints** require special handling (Server-Sent Events, not standard JSON)\n4. **Async operations** return immediately; always poll status before assuming completion\n5. **Vector search** requires embedding model configured (check `/models`)\n6. **Model overrides** are per-request; set in body, not config\n7. **CORS enabled** in development; configure for production\n\n---\n\n## Learning Path\n\n1. **Authentication**: Add `X-Password` header to all requests\n2. **Create a notebook**: `POST /notebooks` with name and description\n3. **Add a source**: `POST /sources` with file, URL, or text\n4. **Query your content**: `POST /chat/execute` to ask questions\n5. **Explore advanced features**: Search, transformations, streaming\n\n---\n\n## Production Considerations\n\n- Replace password auth with OAuth/JWT (see [Security](../5-CONFIGURATION/security.md))\n- Add rate limiting via reverse proxy (Nginx, CloudFlare, Kong)\n- Enable CORS restrictions (currently allows all origins)\n- Use HTTPS via reverse proxy (see [Reverse Proxy](../5-CONFIGURATION/reverse-proxy.md))\n- Set up API versioning strategy (currently implicit)\n\nSee [Security Configuration](../5-CONFIGURATION/security.md) and [Reverse Proxy Setup](../5-CONFIGURATION/reverse-proxy.md) for complete production setup.\n" }, { "path": "docs/7-DEVELOPMENT/architecture.md", "content": "# Open Notebook Architecture\n\n## High-Level Overview\n\nOpen Notebook follows a three-tier architecture with clear separation of concerns:\n\n```\n┌─────────────────────────────────────────────────────────┐\n│ Your Browser │\n│ Access: http://your-server-ip:8502 │\n└────────────────┬────────────────────────────────────────┘\n │\n ▼\n ┌───────────────┐\n │ Port 8502 │ ← Next.js Frontend (what you see)\n │ Frontend │ Also proxies API requests internally!\n └───────┬───────┘\n │ proxies /api/* requests ↓\n ▼\n ┌───────────────┐\n │ Port 5055 │ ← FastAPI Backend (handles requests)\n │ API │\n └───────┬───────┘\n │\n ▼\n ┌───────────────┐\n │ SurrealDB │ ← Database (internal, auto-configured)\n │ (Port 8000) │\n └───────────────┘\n```\n\n**Key Points:**\n- **v1.1+**: Next.js automatically proxies `/api/*` requests to the backend, simplifying reverse proxy setup\n- Your browser loads the frontend from port 8502\n- The frontend needs to know where to find the API - when accessing remotely, set: `API_URL=http://your-server-ip:5055`\n- **Behind reverse proxy?** You only need to proxy to port 8502 now! See [Reverse Proxy Configuration](../5-CONFIGURATION/reverse-proxy.md)\n\n---\n\n## Detailed Architecture\n\nOpen Notebook is built on a **three-tier, async-first architecture** designed for scalability, modularity, and multi-provider AI flexibility. The system separates concerns across frontend, API, and database layers, with LangGraph powering intelligent workflows and Esperanto enabling seamless integration with 8+ AI providers.\n\n**Core Philosophy**:\n- Privacy-first: Users control their data and AI provider choice\n- Async/await throughout: Non-blocking operations for responsive UX\n- Domain-Driven Design: Clear separation between domain models, repositories, and orchestrators\n- Multi-provider flexibility: Swap AI providers without changing application code\n- Self-hosted capable: All components deployable in isolated environments\n\n---\n\n## Three-Tier Architecture\n\n### Layer 1: Frontend (React/Next.js @ port 3000)\n\n**Purpose**: Responsive, interactive user interface for research, notes, chat, and podcast management.\n\n**Technology Stack**:\n- **Framework**: Next.js 15 with React 19\n- **Language**: TypeScript with strict type checking\n- **State Management**: Zustand (lightweight store) + TanStack Query (server state)\n- **Styling**: Tailwind CSS + Shadcn/ui component library\n- **Build Tool**: Webpack (bundled via Next.js)\n\n**Key Responsibilities**:\n- Render notebooks, sources, notes, chat sessions, and podcasts\n- Handle user interactions (create, read, update, delete operations)\n- Manage complex UI state (modals, file uploads, real-time search)\n- Stream responses from API (chat, podcast generation)\n- Display embeddings, vector search results, and insights\n\n**Communication Pattern**:\n- All data fetched via REST API (async requests to port 5055)\n- Configured base URL: `http://localhost:5055` (dev) or environment-specific (prod)\n- TanStack Query handles caching, refetching, and data synchronization\n- Zustand stores global state (user, notebooks, selected context)\n- CORS enabled on API side for cross-origin requests\n\n**Component Architecture**:\n- `/src/app/`: Next.js App Router (pages, layouts)\n- `/src/components/`: Reusable React components (buttons, forms, cards)\n- `/src/hooks/`: Custom hooks (useNotebook, useChat, useSearch)\n- `/src/lib/`: Utility functions, API clients, validators\n- `/src/styles/`: Global CSS, Tailwind config\n\n---\n\n### Layer 2: API (FastAPI @ port 5055)\n\n**Purpose**: RESTful backend exposing operations on notebooks, sources, notes, chat sessions, and AI models.\n\n**Technology Stack**:\n- **Framework**: FastAPI 0.104+ (async Python web framework)\n- **Language**: Python 3.11+\n- **Validation**: Pydantic v2 (request/response schemas)\n- **Logging**: Loguru (structured JSON logging)\n- **Testing**: Pytest (unit and integration tests)\n\n**Architecture**:\n```\nFastAPI App (main.py)\n ├── Routers (HTTP endpoints)\n │ ├── routers/notebooks.py (CRUD operations)\n │ ├── routers/sources.py (content ingestion, upload)\n │ ├── routers/notes.py (note management)\n │ ├── routers/chat.py (conversation sessions)\n │ ├── routers/search.py (full-text + vector search)\n │ ├── routers/transformations.py (custom transformations)\n │ ├── routers/models.py (AI model configuration)\n │ └── routers/*.py (11 additional routers)\n │\n ├── Services (business logic)\n │ ├── *_service.py (orchestration, graph invocation)\n │ ├── command_service.py (async job submission)\n │ └── middleware (auth, logging)\n │\n ├── Models (Pydantic schemas)\n │ └── models.py (validation, serialization)\n │\n └── Lifespan (startup/shutdown)\n └── AsyncMigrationManager (database schema migrations)\n```\n\n**Key Responsibilities**:\n1. **HTTP Interface**: Accept REST requests, validate, return JSON responses\n2. **Business Logic**: Orchestrate domain models, repository operations, and workflows\n3. **Async Job Queue**: Submit long-running tasks (podcast generation, source processing)\n4. **Database Migrations**: Run schema updates on startup\n5. **Error Handling**: Catch exceptions, return appropriate HTTP status codes\n6. **Logging**: Track operations for debugging and monitoring\n\n**Startup Flow**:\n1. Load `.env` environment variables\n2. Initialize FastAPI app with CORS + auth middleware\n3. Run AsyncMigrationManager (creates/updates database schema)\n4. Register all routers (20+ endpoints)\n5. Server ready on port 5055\n\n**Request-Response Cycle**:\n```\nHTTP Request → Router → Service → Domain/Repository → SurrealDB\n ↓\n LangGraph (optional)\n ↓\nResponse ← Pydantic serialization ← Service ← Result\n```\n\n---\n\n### Layer 3: Database (SurrealDB @ port 8000)\n\n**Purpose**: Graph database with built-in vector embeddings, semantic search, and relationship management.\n\n**Technology Stack**:\n- **Database**: SurrealDB (multi-model, ACID transactions)\n- **Query Language**: SurrealQL (SQL-like syntax with graph operations)\n- **Async Driver**: Async Rust client for Python\n- **Migrations**: Manual `.surql` files in `/migrations/` (auto-run on API startup)\n\n**Core Tables**:\n\n| Table | Purpose | Key Fields |\n|-------|---------|-----------|\n| `notebook` | Research project container | id, name, description, archived, created, updated |\n| `source` | Content item (PDF, URL, text) | id, title, full_text, topics, asset, created, updated |\n| `source_embedding` | Vector embeddings for semantic search | id, source, embedding, chunk_text, chunk_index |\n| `note` | User-created research notes | id, title, content, note_type (human/ai), created, updated |\n| `chat_session` | Conversation session | id, notebook_id, title, messages (JSON), created, updated |\n| `transformation` | Custom transformation rules | id, name, description, prompt, created, updated |\n| `source_insight` | Transformation output | id, source_id, insight_type, content, created, updated |\n| `reference` | Relationship: source → notebook | out (source), in (notebook) |\n| `artifact` | Relationship: note → notebook | out (note), in (notebook) |\n\n**Relationship Graph**:\n```\nNotebook\n ↓ (referenced_by)\nSource\n ├→ SourceEmbedding (1:many for chunked text)\n ├→ SourceInsight (1:many for transformation outputs)\n └→ Note (via artifact relationship)\n ├→ Embedding (semantic search)\n └→ Topics (tags)\n\nChatSession\n ├→ Notebook\n └→ Messages (stored as JSON array)\n```\n\n**Vector Search Capability**:\n- Embeddings stored natively in SurrealDB\n- Full-text search on `source.full_text` and `note.content`\n- Cosine similarity search on embedding vectors\n- Semantic search integrates with search endpoint\n\n**Connection Management**:\n- Async connection pooling (configurable size)\n- Transaction support for multi-record operations\n- Schema auto-validation via migrations\n- Query timeout protection (prevent infinite queries)\n\n---\n\n## Tech Stack Rationale\n\n### Why Python + FastAPI?\n\n**Python**:\n- Rich AI/ML ecosystem (LangChain, LangGraph, transformers, scikit-learn)\n- Rapid prototyping and deployment\n- Extensive async support (asyncio, async/await)\n- Strong type hints (Pydantic, mypy)\n\n**FastAPI**:\n- Modern, async-first framework\n- Automatic OpenAPI documentation (Swagger UI @ /docs)\n- Built-in request validation (Pydantic)\n- Excellent performance (benchmarked near C/Rust speeds)\n- Easy middleware/dependency injection\n\n### Why Next.js + React + TypeScript?\n\n**Next.js**:\n- Full-stack React framework with SSR/SSG\n- File-based routing (intuitive project structure)\n- Built-in API routes (optional backend co-location)\n- Optimized image/code splitting\n- Easy deployment (Vercel, Docker, self-hosted)\n\n**React 19**:\n- Component-based UI (reusable, testable)\n- Excellent tooling and community\n- Client-side state management (Zustand)\n- Server-side state sync (TanStack Query)\n\n**TypeScript**:\n- Type safety catches errors at compile time\n- Better IDE autocomplete and refactoring\n- Documentation via types (self-documenting code)\n- Easier onboarding for new contributors\n\n### Why SurrealDB?\n\n**SurrealDB**:\n- Native graph database (relationships are first-class)\n- Built-in vector embeddings (no separate vector DB)\n- ACID transactions (data consistency)\n- Multi-model (relational + document + graph)\n- Full-text search + semantic search in one query\n- Self-hosted (unlike managed Pinecone/Weaviate)\n- Flexible SurrealQL (SQL-like syntax)\n\n**Alternative Considered**: PostgreSQL + pgvector (more mature but separate extensions)\n\n### Why Esperanto for AI Providers?\n\n**Esperanto Library**:\n- Unified interface to 8+ LLM providers (OpenAI, Anthropic, Google, Groq, Ollama, Mistral, DeepSeek, xAI)\n- Multi-provider embeddings (OpenAI, Google, Ollama, Mistral, Voyage)\n- TTS/STT integration (OpenAI, Groq, ElevenLabs, Google)\n- Smart provider selection (fallback logic, cost optimization)\n- Per-request model override support\n- Local Ollama support (completely self-hosted option)\n\n**Alternative Considered**: LangChain's provider abstraction (more verbose, less flexible)\n\n---\n\n## LangGraph Workflows\n\nLangGraph is a state machine library that orchestrates multi-step AI workflows. Open Notebook uses five core workflows:\n\n### 1. **Source Processing Workflow** (`open_notebook/graphs/source.py`)\n\n**Purpose**: Ingest content (PDF, URL, text) and prepare for search/insights.\n\n**Flow**:\n```\nInput (file/URL/text)\n ↓\nExtract Content (content-core library)\n ↓\nClean & tokenize text\n ↓\nGenerate Embeddings (Esperanto)\n ↓\nCreate SourceEmbedding records (chunked + indexed)\n ↓\nExtract Topics (LLM summarization)\n ↓\nSave to SurrealDB\n ↓\nOutput (Source record with embeddings)\n```\n\n**State Dict**:\n```python\n{\n \"content_state\": {\"file_path\" | \"url\" | \"content\": str},\n \"source_id\": str,\n \"full_text\": str,\n \"embeddings\": List[Dict],\n \"topics\": List[str],\n \"notebook_ids\": List[str],\n}\n```\n\n**Invoked By**: Sources API (`POST /sources`)\n\n---\n\n### 2. **Chat Workflow** (`open_notebook/graphs/chat.py`)\n\n**Purpose**: Conduct multi-turn conversations with AI model, referencing notebook context.\n\n**Flow**:\n```\nUser Message\n ↓\nBuild Context (selected sources/notes)\n ↓\nAdd Message to Session\n ↓\nCreate Chat Prompt (system + history + context)\n ↓\nCall LLM (via Esperanto)\n ↓\nStream Response\n ↓\nSave AI Message to ChatSession\n ↓\nOutput (complete message)\n```\n\n**State Dict**:\n```python\n{\n \"session_id\": str,\n \"messages\": List[BaseMessage],\n \"context\": Dict[str, Any], # sources, notes, snippets\n \"response\": str,\n \"model_override\": Optional[str],\n}\n```\n\n**Key Features**:\n- Message history persisted in SurrealDB (SqliteSaver checkpoint)\n- Context building via `build_context_for_chat()` utility\n- Token counting to prevent overflow\n- Per-message model override support\n\n**Invoked By**: Chat API (`POST /chat/execute`)\n\n---\n\n### 3. **Ask Workflow** (`open_notebook/graphs/ask.py`)\n\n**Purpose**: Answer user questions by searching sources and synthesizing responses.\n\n**Flow**:\n```\nUser Question\n ↓\nPlan Search Strategy (LLM generates searches)\n ↓\nExecute Searches (vector + text search)\n ↓\nScore & Rank Results\n ↓\nProvide Answers (LLM synthesizes from results)\n ↓\nStream Responses\n ↓\nOutput (final answer)\n```\n\n**State Dict**:\n```python\n{\n \"question\": str,\n \"strategy\": SearchStrategy,\n \"answers\": List[str],\n \"final_answer\": str,\n \"sources_used\": List[Source],\n}\n```\n\n**Streaming**: Uses `astream()` to emit updates in real-time (strategy → answers → final answer)\n\n**Invoked By**: Search API (`POST /ask` with streaming)\n\n---\n\n### 4. **Transformation Workflow** (`open_notebook/graphs/transformation.py`)\n\n**Purpose**: Apply custom transformations to sources (extract summaries, key points, etc).\n\n**Flow**:\n```\nSource + Transformation Rule\n ↓\nGenerate Prompt (Jinja2 template)\n ↓\nCall LLM\n ↓\nParse Output\n ↓\nCreate SourceInsight record\n ↓\nOutput (insight with type + content)\n```\n\n**Example Transformations**:\n- Summary (5-sentence overview)\n- Key Points (bulleted list)\n- Quotes (notable excerpts)\n- Q&A (generated questions and answers)\n\n**Invoked By**: Sources API (`POST /sources/{id}/insights`)\n\n---\n\n### 5. **Prompt Workflow** (`open_notebook/graphs/prompt.py`)\n\n**Purpose**: Generic LLM task execution (e.g., auto-generate note titles, analyze content).\n\n**Flow**:\n```\nInput Text + Prompt\n ↓\nCall LLM (simple request-response)\n ↓\nOutput (completion)\n```\n\n**Used For**: Note title generation, content analysis, etc.\n\n---\n\n## AI Provider Integration Pattern\n\n### ModelManager: Centralized Factory\n\nLocated in `open_notebook/ai/models.py`, ModelManager handles:\n\n1. **Provider Detection**: Check environment variables for available providers\n2. **Model Selection**: Choose best model based on context size and task\n3. **Fallback Logic**: If primary provider unavailable, try backup\n4. **Cost Optimization**: Prefer cheaper models for simple tasks\n5. **Token Calculation**: Estimate cost before LLM call\n\n**Usage**:\n```python\nfrom open_notebook.ai.provision import provision_langchain_model\n\n# Get best LLM for context size\nmodel = await provision_langchain_model(\n task=\"chat\", # or \"search\", \"extraction\"\n model_override=\"anthropic/claude-opus-4\", # optional\n context_size=8000, # estimated tokens\n)\n\n# Invoke model\nresponse = await model.ainvoke({\"input\": prompt})\n```\n\n### Multi-Provider Support\n\n**LLM Providers**:\n- OpenAI (gpt-4, gpt-4-turbo, gpt-3.5-turbo)\n- Anthropic (claude-opus, claude-sonnet, claude-haiku)\n- Google (gemini-pro, gemini-1.5)\n- Groq (mixtral, llama-2)\n- Ollama (local models)\n- Mistral (mistral-large, mistral-medium)\n- DeepSeek (deepseek-chat)\n- xAI (grok)\n\n**Embedding Providers**:\n- OpenAI (text-embedding-3-large, text-embedding-3-small)\n- Google (embedding-001)\n- Ollama (local embeddings)\n- Mistral (mistral-embed)\n- Voyage (voyage-large-2)\n\n**TTS Providers**:\n- OpenAI (tts-1, tts-1-hd)\n- Groq (no TTS, fallback to OpenAI)\n- ElevenLabs (multilingual voices)\n- Google TTS (text-to-speech)\n\n### Per-Request Override\n\nEvery LangGraph invocation accepts a `config` parameter to override models:\n\n```python\nresult = await graph.ainvoke(\n input={...},\n config={\n \"configurable\": {\n \"model_override\": \"anthropic/claude-opus-4\" # Use Claude instead\n }\n }\n)\n```\n\n---\n\n## Design Patterns\n\n### 1. **Domain-Driven Design (DDD)**\n\n**Domain Objects** (`open_notebook/domain/`):\n- `Notebook`: Research container with relationships to sources/notes\n- `Source`: Content item (PDF, URL, text) with embeddings\n- `Note`: User-created or AI-generated research note\n- `ChatSession`: Conversation history for a notebook\n- `Transformation`: Custom rule for extracting insights\n\n**Repository Pattern**:\n- Database access layer (`open_notebook/database/repository.py`)\n- `repo_query()`: Execute SurrealQL queries\n- `repo_create()`: Insert records\n- `repo_upsert()`: Merge records\n- `repo_delete()`: Remove records\n\n**Entity Methods**:\n```python\n# Domain methods (business logic)\nnotebook = await Notebook.get(id)\nawait notebook.save()\nnotes = await notebook.get_notes()\nsources = await notebook.get_sources()\n```\n\n### 2. **Async-First Architecture**\n\n**All I/O is async**:\n- Database queries: `await repo_query(...)`\n- LLM calls: `await model.ainvoke(...)`\n- File I/O: `await upload_file.read()`\n- Graph invocations: `await graph.ainvoke(...)`\n\n**Benefits**:\n- Non-blocking request handling (FastAPI serves multiple concurrent requests)\n- Better resource utilization (I/O waiting doesn't block CPU)\n- Natural fit for Python async/await syntax\n\n**Example**:\n```python\n@router.post(\"/sources\")\nasync def create_source(source_data: SourceCreate):\n # All operations are non-blocking\n source = Source(title=source_data.title)\n await source.save() # async database operation\n await graph.ainvoke({...}) # async LangGraph invocation\n return SourceResponse(...)\n```\n\n### 3. **Service Pattern**\n\nServices orchestrate domain objects, repositories, and workflows:\n\n```python\n# api/notebook_service.py\nclass NotebookService:\n async def get_notebook_with_stats(notebook_id: str):\n notebook = await Notebook.get(notebook_id)\n sources = await notebook.get_sources()\n notes = await notebook.get_notes()\n return {\n \"notebook\": notebook,\n \"source_count\": len(sources),\n \"note_count\": len(notes),\n }\n```\n\n**Responsibilities**:\n- Validate inputs (Pydantic)\n- Orchestrate database operations\n- Invoke workflows (LangGraph graphs)\n- Handle errors and return appropriate status codes\n- Log operations\n\n### 4. **Streaming Pattern**\n\nFor long-running operations (ask workflow, podcast generation), stream results as Server-Sent Events:\n\n```python\n@router.post(\"/ask\", response_class=StreamingResponse)\nasync def ask(request: AskRequest):\n async def stream_response():\n async for chunk in ask_graph.astream(input={...}):\n yield f\"data: {json.dumps(chunk)}\\n\\n\"\n return StreamingResponse(stream_response(), media_type=\"text/event-stream\")\n```\n\n### 5. **Job Queue Pattern**\n\nFor async background tasks (source processing), use Surreal-Commands job queue:\n\n```python\n# Submit job\ncommand_id = await CommandService.submit_command_job(\n app=\"open_notebook\",\n command=\"process_source\",\n input={...}\n)\n\n# Poll status\nstatus = await source.get_status()\n```\n\n---\n\n## Service Communication Patterns\n\n### Frontend → API\n\n1. **REST requests** (HTTP GET/POST/PUT/DELETE)\n2. **JSON request/response bodies**\n3. **Standard HTTP status codes** (200, 400, 404, 500)\n4. **Optional streaming** (Server-Sent Events for long operations)\n\n**Example**:\n```typescript\n// Frontend\nconst response = await fetch(\"http://localhost:5055/sources\", {\n method: \"POST\",\n body: formData, // multipart/form-data for file upload\n});\nconst source = await response.json();\n```\n\n### API → SurrealDB\n\n1. **SurrealQL queries** (similar to SQL)\n2. **Async driver** with connection pooling\n3. **Type-safe record IDs** (record_id syntax)\n4. **Transaction support** for multi-step operations\n\n**Example**:\n```python\n# API\nresult = await repo_query(\n \"SELECT * FROM source WHERE notebook = $notebook_id\",\n {\"notebook_id\": ensure_record_id(notebook_id)}\n)\n```\n\n### API → AI Providers (via Esperanto)\n\n1. **Esperanto unified interface**\n2. **Per-request provider override**\n3. **Automatic fallback on failure**\n4. **Token counting and cost estimation**\n\n**Example**:\n```python\n# API\nmodel = await provision_langchain_model(task=\"chat\")\nresponse = await model.ainvoke({\"input\": prompt})\n```\n\n### API → Job Queue (Surreal-Commands)\n\n1. **Async job submission**\n2. **Fire-and-forget pattern**\n3. **Status polling via `/commands/{id}` endpoint**\n4. **Job completion callbacks (optional)**\n\n**Example**:\n```python\n# Submit async source processing\ncommand_id = await CommandService.submit_command_job(...)\n\n# Client polls status\nresponse = await fetch(f\"http://localhost:5055/commands/{command_id}\")\nstatus = await response.json() # returns { status: \"running|queued|completed|failed\" }\n```\n\n---\n\n## Database Schema Overview\n\n### Core Schema Structure\n\n**Tables** (20+):\n- Notebooks (with soft-delete via `archived` flag)\n- Sources (content + metadata)\n- SourceEmbeddings (vector chunks)\n- Notes (user-created + AI-generated)\n- ChatSessions (conversation history)\n- Transformations (custom rules)\n- SourceInsights (transformation outputs)\n- Relationships (notebook→source, notebook→note)\n\n**Migrations**:\n- Automatic on API startup\n- Located in `/migrations/` directory\n- Numbered sequentially (001_*.surql, 002_*.surql, etc)\n- Tracked in `_sbl_migrations` table\n- Rollback via `_down.surql` files (manual)\n\n### Relationship Model\n\n**Graph Relationships**:\n```\nNotebook\n ← reference ← Source (many:many)\n ← artifact ← Note (many:many)\n\nSource\n → source_embedding (one:many)\n → source_insight (one:many)\n → embedding (via source_embedding)\n\nChatSession\n → messages (JSON array in database)\n → notebook_id (reference to Notebook)\n\nTransformation\n → source_insight (one:many)\n```\n\n**Query Example** (get all sources in a notebook with counts):\n```sql\nSELECT id, title,\n count(<-reference.in) as note_count,\n count(<-embedding.in) as embedded_chunks\nFROM source\nWHERE notebook = $notebook_id\nORDER BY updated DESC\n```\n\n---\n\n## Key Architectural Decisions\n\n### 1. **Async Throughout**\n\nAll I/O operations are non-blocking to maximize concurrency and responsiveness.\n\n**Trade-off**: Slightly more complex code (async/await syntax) vs. high throughput.\n\n### 2. **Multi-Provider from Day 1**\n\nBuilt-in support for 8+ AI providers prevents vendor lock-in.\n\n**Trade-off**: Added complexity in ModelManager vs. flexibility and cost optimization.\n\n### 3. **Graph-First Workflows**\n\nLangGraph state machines for complex multi-step operations (ask, chat, transformations).\n\n**Trade-off**: Steeper learning curve vs. maintainable, debuggable workflows.\n\n### 4. **Self-Hosted Database**\n\nSurrealDB for graph + vector search in one system (no external dependencies).\n\n**Trade-off**: Operational responsibility vs. simplified architecture and cost savings.\n\n### 5. **Job Queue for Long-Running Tasks**\n\nAsync job submission (source processing, podcast generation) prevents request timeouts.\n\n**Trade-off**: Eventual consistency vs. responsive user experience.\n\n---\n\n## Important Quirks & Gotchas\n\n### API Startup\n\n- **Migrations run automatically** on every startup; check logs for errors\n- **SurrealDB must be running** before starting API (connection test in lifespan)\n- **Auth middleware is basic** (password-only); upgrade to OAuth/JWT for production\n\n### Database Operations\n\n- **Record IDs use SurrealDB syntax** (table:id format, e.g., \"notebook:abc123\")\n- **ensure_record_id()** helper prevents malformed IDs\n- **Soft deletes** via `archived` field (data not removed, just marked inactive)\n- **Timestamps in ISO 8601 format** (created, updated fields)\n\n### LangGraph Workflows\n\n- **State persistence** via SqliteSaver in `/data/sqlite-db/`\n- **No built-in timeout**; long workflows may block requests (use streaming for UX)\n- **Model fallback** automatic if primary provider unavailable\n- **Checkpoint IDs** must be unique per session (avoid collisions)\n\n### AI Provider Integration\n\n- **Esperanto library** handles all provider APIs (no direct API calls)\n- **Per-request override** via RunnableConfig (temporary, not persistent)\n- **Cost estimation** via token counting (not 100% accurate, use for guidance)\n- **Fallback logic** tries cheaper models if primary fails\n\n### File Uploads\n\n- **Stored in `/data/uploads/`** directory (not database)\n- **Unique filename generation** prevents overwrites (counter suffix)\n- **Content-core library** extracts text from 50+ file types\n- **Large files** may block API briefly (sync content extraction)\n\n---\n\n## Performance Considerations\n\n### Optimization Strategies\n\n1. **Connection Pooling**: SurrealDB async driver with configurable pool size\n2. **Query Caching**: TanStack Query on frontend (client-side caching)\n3. **Embedding Reuse**: Vector search uses pre-computed embeddings\n4. **Chunking**: Sources split into chunks for better search relevance\n5. **Async Operations**: Non-blocking I/O for high concurrency\n6. **Lazy Loading**: Frontend requests only needed data (pagination)\n\n### Bottlenecks\n\n1. **LLM Calls**: Latency depends on provider (typically 1-30 seconds)\n2. **Embedding Generation**: Time proportional to content size and provider\n3. **Vector Search**: Similarity computation over all embeddings\n4. **Content Extraction**: Sync operation in source processing\n\n### Monitoring\n\n- **API Logs**: Check loguru output for errors and slow operations\n- **Database Queries**: SurrealDB metrics available via admin UI\n- **Token Usage**: Estimated via `estimate_tokens()` utility\n- **Job Status**: Poll `/commands/{id}` for async operations\n\n---\n\n## Extension Points\n\n### Adding a New Workflow\n\n1. Create `open_notebook/graphs/workflow_name.py`\n2. Define StateDict and node functions\n3. Build graph with `.add_node()` / `.add_edge()`\n4. Create service in `api/workflow_service.py`\n5. Register router in `api/main.py`\n6. Add tests in `tests/test_workflow.py`\n\n### Adding a New Data Model\n\n1. Create model in `open_notebook/domain/model_name.py`\n2. Inherit from BaseModel (domain object)\n3. Implement `save()`, `get()`, `delete()` methods (CRUD)\n4. Add repository functions if complex queries needed\n5. Create database migration in `migrations/`\n6. Add API routes and models in `api/`\n\n### Adding a New AI Provider\n\n1. Configure Esperanto for new provider (see .env.example)\n2. ModelManager automatically detects via environment variables\n3. Override via per-request config (no code changes needed)\n4. Test fallback logic if provider unavailable\n\n---\n\n## Deployment Considerations\n\n### Development\n\n- All services on localhost (3000, 5055, 8000)\n- Auto-reload on file changes (Next.js, FastAPI)\n- Hot-reload database migrations\n- Open API docs at http://localhost:5055/docs\n\n### Production\n\n- **Frontend**: Deploy to Vercel, Netlify, or Docker\n- **API**: Docker container (see Dockerfile)\n- **Database**: SurrealDB container or managed service\n- **Environment**: Secure .env file with API keys\n- **SSL/TLS**: Reverse proxy (Nginx, CloudFlare)\n- **Rate Limiting**: Add at proxy layer\n- **Auth**: Replace PasswordAuthMiddleware with OAuth/JWT\n- **Monitoring**: Log aggregation (CloudWatch, DataDog, etc)\n\n---\n\n## Summary\n\nOpen Notebook's architecture provides a solid foundation for privacy-focused, AI-powered research. The separation of concerns (frontend/API/database), async-first design, and multi-provider flexibility enable rapid development and easy deployment. LangGraph workflows orchestrate complex AI tasks, while Esperanto abstracts provider details. The result is a scalable, maintainable system that puts users in control of their data and AI provider choice.\n" }, { "path": "docs/7-DEVELOPMENT/code-standards.md", "content": "# Code Standards\n\nThis document outlines coding standards and best practices for Open Notebook contributions. All code should follow these guidelines to ensure consistency, readability, and maintainability.\n\n## Python Standards\n\n### Code Formatting\n\nWe follow **PEP 8** with some specific guidelines:\n\n- Use **Ruff** for linting and formatting\n- Maximum line length: **88 characters**\n- Use **double quotes** for strings\n- Use **trailing commas** in multi-line structures\n\n### Type Hints\n\nAlways use type hints for function parameters and return values:\n\n```python\nfrom typing import List, Optional, Dict, Any\nfrom pydantic import BaseModel\n\nasync def process_content(\n content: str,\n options: Optional[Dict[str, Any]] = None\n) -> ProcessedContent:\n \"\"\"Process content with optional configuration.\"\"\"\n # Implementation\n```\n\n### Async/Await Patterns\n\nUse async/await consistently throughout the codebase:\n\n```python\n# Good\nasync def fetch_data(url: str) -> Dict[str, Any]:\n async with aiohttp.ClientSession() as session:\n async with session.get(url) as response:\n return await response.json()\n\n# Bad - mixing sync and async\ndef fetch_data(url: str) -> Dict[str, Any]:\n loop = asyncio.get_event_loop()\n return loop.run_until_complete(async_fetch(url))\n```\n\n### Error Handling\n\nUse structured error handling with custom exceptions:\n\n```python\nfrom open_notebook.exceptions import DatabaseOperationError, InvalidInputError\n\nasync def create_notebook(name: str, description: str) -> Notebook:\n \"\"\"Create a new notebook with validation.\"\"\"\n if not name.strip():\n raise InvalidInputError(\"Notebook name cannot be empty\")\n\n try:\n notebook = Notebook(name=name, description=description)\n await notebook.save()\n return notebook\n except Exception as e:\n raise DatabaseOperationError(f\"Failed to create notebook: {str(e)}\")\n```\n\n### Documentation (Google-style Docstrings)\n\nUse Google-style docstrings for all functions, classes, and modules:\n\n```python\nasync def vector_search(\n query: str,\n limit: int = 10,\n minimum_score: float = 0.2\n) -> List[SearchResult]:\n \"\"\"Perform vector search across embedded content.\n\n Args:\n query: Search query string\n limit: Maximum number of results to return\n minimum_score: Minimum similarity score for results\n\n Returns:\n List of search results sorted by relevance score\n\n Raises:\n InvalidInputError: If query is empty or limit is invalid\n DatabaseOperationError: If search operation fails\n \"\"\"\n # Implementation\n```\n\n#### Module Docstrings\n```python\n\"\"\"\nNotebook domain model and operations.\n\nThis module contains the core Notebook class and related operations for\nmanaging research notebooks within the Open Notebook system.\n\"\"\"\n```\n\n#### Class Docstrings\n```python\nclass Notebook(BaseModel):\n \"\"\"A research notebook containing sources, notes, and chat sessions.\n\n Notebooks are the primary organizational unit in Open Notebook, allowing\n users to group related research materials and maintain separate contexts\n for different projects.\n\n Attributes:\n name: The notebook's display name\n description: Optional description of the notebook's purpose\n archived: Whether the notebook is archived (default: False)\n created: Timestamp of creation\n updated: Timestamp of last update\n \"\"\"\n```\n\n#### Function Docstrings\n```python\nasync def create_notebook(\n name: str,\n description: str = \"\",\n user_id: Optional[str] = None\n) -> Notebook:\n \"\"\"Create a new notebook with validation.\n\n Args:\n name: The notebook name (required, non-empty)\n description: Optional notebook description\n user_id: Optional user ID for multi-user deployments\n\n Returns:\n The created notebook instance\n\n Raises:\n InvalidInputError: If name is empty or invalid\n DatabaseOperationError: If creation fails\n\n Example:\n ```python\n notebook = await create_notebook(\n name=\"AI Research\",\n description=\"Research on AI applications\"\n )\n ```\n \"\"\"\n```\n\n## FastAPI Standards\n\n### Router Organization\n\nOrganize endpoints by domain:\n\n```python\n# api/routers/notebooks.py\nfrom fastapi import APIRouter, HTTPException, Query\nfrom typing import List, Optional\n\nrouter = APIRouter()\n\n@router.get(\"/notebooks\", response_model=List[NotebookResponse])\nasync def get_notebooks(\n archived: Optional[bool] = Query(None, description=\"Filter by archived status\"),\n order_by: str = Query(\"updated desc\", description=\"Order by field and direction\"),\n):\n \"\"\"Get all notebooks with optional filtering and ordering.\"\"\"\n # Implementation\n```\n\n### Request/Response Models\n\nUse Pydantic models for validation:\n\n```python\nfrom pydantic import BaseModel, Field\nfrom typing import Optional\n\nclass NotebookCreate(BaseModel):\n name: str = Field(..., description=\"Name of the notebook\", min_length=1)\n description: str = Field(default=\"\", description=\"Description of the notebook\")\n\nclass NotebookResponse(BaseModel):\n id: str\n name: str\n description: str\n archived: bool\n created: str\n updated: str\n```\n\n### Error Handling\n\nUse consistent error responses:\n\n```python\nfrom fastapi import HTTPException\nfrom loguru import logger\n\ntry:\n result = await some_operation()\n return result\nexcept InvalidInputError as e:\n raise HTTPException(status_code=400, detail=str(e))\nexcept DatabaseOperationError as e:\n logger.error(f\"Database error: {str(e)}\")\n raise HTTPException(status_code=500, detail=\"Internal server error\")\n```\n\n### API Documentation\n\nUse FastAPI's automatic documentation features:\n\n```python\n@router.post(\n \"/notebooks\",\n response_model=NotebookResponse,\n summary=\"Create a new notebook\",\n description=\"Create a new notebook with the specified name and description.\",\n responses={\n 201: {\"description\": \"Notebook created successfully\"},\n 400: {\"description\": \"Invalid input data\"},\n 500: {\"description\": \"Internal server error\"}\n }\n)\nasync def create_notebook(notebook: NotebookCreate):\n \"\"\"Create a new notebook.\"\"\"\n # Implementation\n```\n\n## Database Standards\n\n### SurrealDB Patterns\n\nUse the repository pattern consistently:\n\n```python\nfrom open_notebook.database.repository import repo_create, repo_query, repo_update\n\n# Create records\nasync def create_notebook(data: Dict[str, Any]) -> Dict[str, Any]:\n \"\"\"Create a new notebook record.\"\"\"\n return await repo_create(\"notebook\", data)\n\n# Query with parameters\nasync def find_notebooks_by_user(user_id: str) -> List[Dict[str, Any]]:\n \"\"\"Find notebooks for a specific user.\"\"\"\n return await repo_query(\n \"SELECT * FROM notebook WHERE user_id = $user_id\",\n {\"user_id\": user_id}\n )\n\n# Update records\nasync def update_notebook(notebook_id: str, data: Dict[str, Any]) -> Dict[str, Any]:\n \"\"\"Update a notebook record.\"\"\"\n return await repo_update(\"notebook\", notebook_id, data)\n```\n\n### Schema Management\n\nUse migrations for schema changes:\n\n```surrealql\n-- migrations/8.surrealql\nDEFINE TABLE IF NOT EXISTS new_feature SCHEMAFULL;\nDEFINE FIELD IF NOT EXISTS name ON TABLE new_feature TYPE string;\nDEFINE FIELD IF NOT EXISTS description ON TABLE new_feature TYPE option;\nDEFINE FIELD IF NOT EXISTS created ON TABLE new_feature TYPE datetime DEFAULT time::now();\nDEFINE FIELD IF NOT EXISTS updated ON TABLE new_feature TYPE datetime DEFAULT time::now();\n```\n\n## TypeScript Standards\n\n### Basic Guidelines\n\nFollow TypeScript best practices:\n\n- Use strict mode enabled in `tsconfig.json`\n- Use proper type annotations for all variables and functions\n- Avoid using `any` type unless absolutely necessary\n- Use `interface` for object shapes, `type` for unions and other advanced types\n\n### Component Structure\n\n- Use functional components with hooks\n- Keep components focused and single-responsibility\n- Extract reusable logic into custom hooks\n- Use proper TypeScript types for props\n\n### Error Handling\n\n- Handle errors explicitly\n- Provide meaningful error messages\n- Log errors appropriately\n- Don't suppress errors silently\n\n## Code Quality Tools\n\nWe use these tools to maintain code quality:\n\n- **Ruff**: Linting and code formatting\n - Run with: `uv run ruff check . --fix`\n - Format with: `uv run ruff format .`\n\n- **MyPy**: Static type checking\n - Run with: `uv run python -m mypy .`\n\n- **Pytest**: Testing framework\n - Run with: `uv run pytest`\n\n## Common Patterns\n\n### Async Database Operations\n\n```python\nasync def get_notebook_with_sources(notebook_id: str) -> Notebook:\n \"\"\"Retrieve notebook with all related sources.\"\"\"\n notebook_data = await repo_query(\n \"SELECT * FROM notebook WHERE id = $id\",\n {\"id\": notebook_id}\n )\n if not notebook_data:\n raise InvalidInputError(f\"Notebook {notebook_id} not found\")\n\n sources_data = await repo_query(\n \"SELECT * FROM source WHERE notebook_id = $notebook_id\",\n {\"notebook_id\": notebook_id}\n )\n\n return Notebook(\n **notebook_data[0],\n sources=[Source(**s) for s in sources_data]\n )\n```\n\n### Model Validation\n\n```python\nfrom pydantic import BaseModel, validator\n\nclass NotebookInput(BaseModel):\n name: str\n description: str = \"\"\n\n @validator('name')\n def name_not_empty(cls, v):\n if not v.strip():\n raise ValueError('Name cannot be empty')\n return v.strip()\n```\n\n## Code Review Checklist\n\nBefore submitting code for review, ensure:\n\n- [ ] Code follows PEP 8 / TypeScript best practices\n- [ ] Type hints are present for all functions\n- [ ] Docstrings are complete and accurate\n- [ ] Error handling is appropriate\n- [ ] Tests are included and passing\n- [ ] No debug code (console.logs, print statements) left behind\n- [ ] Commit messages are clear and follow conventions\n- [ ] Documentation is updated if needed\n\n---\n\n**See also:**\n- [Testing Guide](testing.md) - How to write tests\n- [Contributing Guide](contributing.md) - Overall contribution workflow\n" }, { "path": "docs/7-DEVELOPMENT/contributing.md", "content": "# Contributing to Open Notebook\n\nThank you for your interest in contributing to Open Notebook! We welcome contributions from developers of all skill levels. This guide will help you understand our contribution workflow and what makes a good contribution.\n\n## 🚨 Issue-First Workflow\n\n**To maintain project coherence and avoid wasted effort, please follow this process:**\n\n1. **Create an issue first** - Before writing any code, create an issue describing the bug or feature\n2. **Propose your solution** - Explain how you plan to implement the fix or feature\n3. **Wait for assignment** - A maintainer will review and assign the issue to you if approved\n4. **Only then start coding** - This ensures your work aligns with the project's vision and architecture\n\n**Why this process?**\n- Prevents duplicate work\n- Ensures solutions align with our architecture and design principles\n- Saves your time by getting feedback before coding\n- Helps maintainers manage the project direction\n\n> ⚠️ **Pull requests without an assigned issue may be closed**, even if the code is good. We want to respect your time by making sure work is aligned before it starts.\n\n## Code of Conduct\n\nBy participating in this project, you are expected to uphold our Code of Conduct. Be respectful, constructive, and collaborative.\n\n## How Can I Contribute?\n\n### Reporting Bugs\n\n1. **Search existing issues** - Check if the bug was already reported\n2. **Create a bug report** - Use the [Bug Report template](https://github.com/lfnovo/open-notebook/issues/new?template=bug_report.yml)\n3. **Provide details** - Include:\n - Steps to reproduce\n - Expected vs actual behavior\n - Logs, screenshots, or error messages\n - Your environment (OS, Docker version, Open Notebook version)\n4. **Indicate if you want to fix it** - Check the \"I would like to work on this\" box if you're interested\n\n### Suggesting Features\n\n1. **Search existing issues** - Check if the feature was already suggested\n2. **Create a feature request** - Use the [Feature Request template](https://github.com/lfnovo/open-notebook/issues/new?template=feature_request.yml)\n3. **Explain the value** - Describe why this feature would be helpful\n4. **Propose implementation** - If you have ideas on how to implement it, share them\n5. **Indicate if you want to build it** - Check the \"I would like to work on this\" box if you're interested\n\n### Contributing Code (Pull Requests)\n\n**IMPORTANT: Follow the issue-first workflow above before starting any PR**\n\nOnce your issue is assigned:\n\n1. **Fork the repo** and create your branch from `main`\n2. **Understand our vision and principles** - Read [design-principles.md](design-principles.md) to understand what guides our decisions\n3. **Follow our architecture** - Refer to the architecture documentation to understand project structure\n4. **Write quality code** - Follow the standards outlined in [code-standards.md](code-standards.md)\n5. **Test your changes** - See [testing.md](testing.md) for test guidelines\n6. **Update documentation** - If you changed functionality, update the relevant docs\n7. **Create your PR**:\n - Reference the issue number (e.g., \"Fixes #123\")\n - Describe what changed and why\n - Include screenshots for UI changes\n - Keep PRs focused - one issue per PR\n\n### What Makes a Good Contribution?\n\n✅ **We love PRs that:**\n- Solve a real problem described in an issue\n- Follow our architecture and coding standards\n- Include tests and documentation\n- Are well-scoped (focused on one thing)\n- Have clear commit messages\n\n❌ **We may close PRs that:**\n- Don't have an associated approved issue\n- Introduce breaking changes without discussion\n- Conflict with our architectural vision\n- Lack tests or documentation\n- Try to solve multiple unrelated problems\n\n## Git Commit Messages\n\n- Use the present tense (\"Add feature\" not \"Added feature\")\n- Use the imperative mood (\"Move cursor to...\" not \"Moves cursor to...\")\n- Limit the first line to 72 characters or less\n- Reference issues and pull requests liberally after the first line\n\n## Development Workflow\n\n### Branch Strategy\n\nWe use a **feature branch workflow**:\n\n1. **Main Branch**: `main` - production-ready code\n2. **Feature Branches**: `feature/description` - new features\n3. **Bug Fixes**: `fix/description` - bug fixes\n4. **Documentation**: `docs/description` - documentation updates\n\n### Making Changes\n\n1. **Create a feature branch**:\n```bash\ngit checkout -b feature/amazing-new-feature\n```\n\n2. **Make your changes** following our coding standards\n\n3. **Test your changes**:\n```bash\n# Run tests\nuv run pytest\n\n# Run linting\nuv run ruff check .\n\n# Run formatting\nuv run ruff format .\n```\n\n4. **Commit your changes**:\n```bash\ngit add .\ngit commit -m \"feat: add amazing new feature\"\n```\n\n5. **Push and create PR**:\n```bash\ngit push origin feature/amazing-new-feature\n# Then create a Pull Request on GitHub\n```\n\n### Keeping Your Fork Updated\n\n```bash\n# Fetch upstream changes\ngit fetch upstream\n\n# Switch to main and merge\ngit checkout main\ngit merge upstream/main\n\n# Push to your fork\ngit push origin main\n```\n\n## Pull Request Process\n\nWhen you create a pull request:\n\n1. **Link your issue** - Reference the issue number in PR description\n2. **Describe your changes** - Explain what changed and why\n3. **Provide test evidence** - Screenshots, test results, or logs\n4. **Check PR template** - Ensure you've completed all required sections\n5. **Wait for review** - A maintainer will review your PR within a week\n\n### PR Review Expectations\n\n- Code review feedback is about the code, not the person\n- Be open to suggestions and alternative approaches\n- Address review comments with clarity and respect\n- Ask questions if feedback is unclear\n\n## Current Priority Areas\n\nWe're actively looking for contributions in these areas:\n\n1. **Frontend Enhancement** - Help improve the Next.js/React UI with real-time updates and better UX\n2. **Testing** - Expand test coverage across all components\n3. **Performance** - Async processing improvements and caching\n4. **Documentation** - API examples and user guides\n5. **Integrations** - New content sources and AI providers\n\n## Getting Help\n\n### Community Support\n\n- **Discord**: [Join our Discord server](https://discord.gg/37XJPXfz2w) for real-time help\n- **GitHub Discussions**: For longer-form questions and ideas\n- **GitHub Issues**: For bug reports and feature requests\n\n### Documentation References\n\n- [Design Principles](design-principles.md) - Understanding our project vision\n- [Code Standards](code-standards.md) - Coding guidelines by language\n- [Testing Guide](testing.md) - How to write tests\n- [Development Setup](development-setup.md) - Getting started locally\n\n## Recognition\n\nWe recognize contributions through:\n\n- **GitHub credits** on releases\n- **Community recognition** in Discord\n- **Contribution statistics** in project analytics\n- **Maintainer consideration** for active contributors\n\n---\n\nThank you for contributing to Open Notebook! Your contributions help make research more accessible and private for everyone.\n\nFor questions about this guide or contributing in general, please reach out on [Discord](https://discord.gg/37XJPXfz2w) or open a GitHub Discussion.\n" }, { "path": "docs/7-DEVELOPMENT/design-principles.md", "content": "# Design Principles & Project Vision\n\nThis document outlines the core principles, vision, and design philosophy that guide Open Notebook's development. All contributors should read and understand these principles before proposing changes or new features.\n\n## 🎯 Project Vision\n\nOpen Notebook aims to be a **privacy-focused, self-hosted alternative to Google's Notebook LM** that empowers users to:\n\n1. **Own their research data** - Full control over where data lives and who can access it\n2. **Choose their AI providers** - Freedom to use any AI provider or run models locally\n3. **Customize their workflows** - Flexibility to adapt the tool to different research needs\n4. **Access their work anywhere** - Through web UI, API, or integrations\n\n### What Open Notebook IS\n\n- A **research assistant** for managing and understanding content\n- A **platform** that connects various AI providers\n- A **privacy-first** tool that keeps your data under your control\n- An **extensible system** with APIs and customization options\n\n### What Open Notebook IS NOT\n\n- A document editor (use Google Docs, Notion, etc. for that)\n- A file storage system (use Dropbox, S3, etc. for that)\n- A general-purpose chatbot (use ChatGPT, Claude, etc. for that)\n- A replacement for your entire workflow (it's one tool in your toolkit)\n\n## 🏗️ Core Design Principles\n\n### 1. Privacy First\n\n**Principle**: User data and research should stay under user control by default.\n\n**In Practice**:\n- Self-hosted deployment is the primary use case\n- No telemetry or analytics without explicit opt-in\n- No hard dependency on specific cloud services\n- Clear documentation on what data goes where\n\n**Example Decisions**:\n- ✅ Support for local Ollama models\n- ✅ Configurable AI provider selection\n- ❌ Hard-coded cloud service integrations\n- ❌ Required external service dependencies\n\n### 2. Simplicity Over Features\n\n**Principle**: The tool should be easy to understand and use, even if it means fewer features.\n\n**In Practice**:\n- Clear, focused UI with well-defined sections\n- Sensible defaults that work for most users\n- Advanced features hidden behind optional configuration\n- Documentation written for non-technical users\n\n**Example Decisions**:\n- ✅ Three-column layout (Sources, Notes, Chat)\n- ✅ Default models that work out of the box\n- ❌ Overwhelming users with too many options upfront\n- ❌ Complex multi-step workflows for basic tasks\n\n### 3. API-First Architecture\n\n**Principle**: All functionality should be accessible via API, not just the UI.\n\n**In Practice**:\n- UI calls the same API that external clients use\n- Comprehensive REST API with OpenAPI documentation\n- No \"UI-only\" features that can't be automated\n- Clear separation between frontend and backend\n\n**Example Decisions**:\n- ✅ FastAPI backend with full API documentation\n- ✅ Consistent API patterns across all endpoints\n- ❌ Business logic in UI components\n- ❌ Features that require direct database access\n\n### 4. Multi-Provider Flexibility\n\n**Principle**: Users should never be locked into a single AI provider.\n\n**In Practice**:\n- Support for multiple AI providers through Esperanto library\n- Easy switching between providers and models\n- Clear documentation on provider limitations\n- Graceful degradation when providers are unavailable\n\n**Example Decisions**:\n- ✅ Support for 16+ AI providers\n- ✅ Per-feature model selection (chat, embeddings, TTS)\n- ❌ Features that only work with OpenAI\n- ❌ Hard-coded API endpoints for specific providers\n\n### 5. Extensibility Through Standards\n\n**Principle**: The system should be extensible through well-defined interfaces, not by forking.\n\n**In Practice**:\n- Plugin systems for transformations and commands\n- Standard data formats (JSON, Markdown)\n- Clear extension points in the architecture\n- Documentation for common customization scenarios\n\n**Example Decisions**:\n- ✅ Custom transformation templates\n- ✅ Background command system\n- ✅ Jinja2 prompt templates\n- ❌ Hard-coded business logic without extension points\n\n### 6. Async-First for Performance\n\n**Principle**: Long-running operations should not block the user interface or API.\n\n**In Practice**:\n- Async/await patterns throughout the backend\n- Background job processing for heavy workloads\n- Status updates and progress tracking\n- Graceful handling of slow AI provider responses\n\n**Example Decisions**:\n- ✅ AsyncIO for database operations\n- ✅ Background commands for podcast generation\n- ✅ Streaming responses for chat\n- ❌ Synchronous blocking operations in API endpoints\n\n## 🎨 UI/UX Principles\n\n### Focus on Content, Not Chrome\n\n- Minimize UI clutter and distractions\n- Content should occupy most of the screen space\n- Controls appear when needed, not always visible\n- Consistent layout across different views\n\n### Progressive Disclosure\n\n- Show simple options first, advanced options on demand\n- Don't overwhelm new users with every possible setting\n- Provide sensible defaults that work for 80% of use cases\n- Make power features discoverable but not intrusive\n\n### Responsive and Fast\n\n- UI should feel instant for common operations\n- Show loading states for operations that take time\n- Cache and optimize where possible\n- Degrade gracefully on slow connections\n\n## 🔧 Technical Principles\n\n### Clean Separation of Concerns\n\n**Layers should not leak**:\n- Frontend should not know about database structure\n- API should not contain business logic (delegate to domain layer)\n- Domain models should not know about HTTP requests\n- Database layer should not know about AI providers\n\n### Type Safety and Validation\n\n**Catch errors early**:\n- Use Pydantic models for all API boundaries\n- Type hints throughout Python codebase\n- TypeScript for frontend code\n- Validate data at system boundaries\n\n### Test What Matters\n\n**Focus on valuable tests**:\n- Test business logic and domain models\n- Test API contracts and error handling\n- Don't test framework code (FastAPI, React, etc.)\n- Integration tests for critical workflows\n\n### Database as Source of Truth\n\n**SurrealDB is our single source of truth**:\n- All state persisted in database\n- No business logic in database layer\n- Use SurrealDB features (record links, queries) appropriately\n- Schema migrations for all schema changes\n\n## 🚫 Anti-Patterns to Avoid\n\n### Feature Creep\n\n**What it looks like**:\n- Adding features because they're \"cool\" or \"easy\"\n- Building features for edge cases before common cases work well\n- Trying to be everything to everyone\n\n**Why we avoid it**:\n- Increases complexity and maintenance burden\n- Makes the tool harder to learn and use\n- Dilutes the core value proposition\n\n**Instead**:\n- Focus on core use cases\n- Say no to features that don't align with vision\n- Build extensibility points for edge cases\n\n### Premature Optimization\n\n**What it looks like**:\n- Optimizing code before knowing if it's slow\n- Complex caching strategies without measuring impact\n- Trading code clarity for marginal performance gains\n\n**Why we avoid it**:\n- Makes code harder to understand and maintain\n- Optimizes the wrong things\n- Wastes development time\n\n**Instead**:\n- Measure first, optimize second\n- Focus on algorithmic improvements\n- Profile before making performance changes\n\n### Over-Engineering\n\n**What it looks like**:\n- Building abstraction layers \"in case we need them later\"\n- Implementing design patterns for 3-line functions\n- Creating frameworks instead of solving problems\n\n**Why we avoid it**:\n- Increases cognitive load for contributors\n- Makes simple changes require touching many files\n- Hides the actual business logic\n\n**Instead**:\n- Start simple, refactor when patterns emerge\n- Optimize for readability and clarity\n- Use abstractions when they simplify, not complicate\n\n### Breaking Changes Without Migration Path\n\n**What it looks like**:\n- Changing database schema without migration scripts\n- Modifying API contracts without versioning\n- Removing features without deprecation warnings\n\n**Why we avoid it**:\n- Breaks existing installations\n- Frustrates users and contributors\n- Creates maintenance nightmares\n\n**Instead**:\n- Always provide migration scripts for schema changes\n- Deprecate before removing\n- Document breaking changes clearly\n\n## 🤝 Decision-Making Framework\n\nWhen evaluating new features or changes, ask:\n\n### 1. Does it align with our vision?\n- Does it help users own their research data?\n- Does it support privacy and self-hosting?\n- Does it fit our core use cases?\n\n### 2. Does it follow our principles?\n- Is it simple to use and understand?\n- Does it work via API?\n- Does it support multiple providers?\n- Can it be extended by users?\n\n### 3. Is the implementation sound?\n- Does it maintain separation of concerns?\n- Is it properly typed and validated?\n- Does it include tests?\n- Is it documented?\n\n### 4. What is the cost?\n- How much complexity does it add?\n- How much maintenance burden?\n- Does it introduce new dependencies?\n- Will it be used enough to justify the cost?\n\n### 5. Are there alternatives?\n- Can existing features solve this problem?\n- Can this be built as a plugin or extension?\n- Should this be a separate tool instead?\n\n## 📚 Examples of Principle-Driven Decisions\n\n### Why we migrated from Streamlit to Next.js\n\n**Principle**: API-First Architecture\n\n**Reasoning**:\n- Streamlit coupled UI and backend logic\n- Difficult to build external integrations\n- Limited control over API behavior\n- Next.js + FastAPI provides clear separation\n\n### Why we use Esperanto for AI providers\n\n**Principle**: Multi-Provider Flexibility\n\n**Reasoning**:\n- Abstracts provider-specific details\n- Easy to add new providers\n- Consistent interface across providers\n- No vendor lock-in\n\n### Why we have a Background Command System\n\n**Principle**: Async-First for Performance\n\n**Reasoning**:\n- Podcast generation takes minutes\n- Users shouldn't wait for long operations\n- Need status tracking and error handling\n- Supports future batch operations\n\n### Why we support Local Ollama\n\n**Principle**: Privacy First\n\n**Reasoning**:\n- Enables fully offline operation\n- No data sent to external services\n- Free for users after hardware cost\n- Aligns with self-hosted philosophy\n\n## 🔄 Evolution of Principles\n\nThese principles are not set in stone. As the project grows and we learn from users, some principles may evolve. However, changes to core principles should be:\n\n1. **Well-justified** - Clear reasoning for why the change is needed\n2. **Discussed openly** - Community input on major changes\n3. **Documented** - Updated in this document with explanation\n4. **Gradual** - Not implemented as breaking changes when possible\n\n---\n\n## For Contributors\n\nWhen proposing a feature or change:\n\n1. **Reference these principles** - Explain how your proposal aligns\n2. **Identify trade-offs** - Be honest about what you're trading for what\n3. **Suggest alternatives** - Show you've considered other approaches\n4. **Be open to feedback** - Maintainers may see concerns you don't\n\n**Remember**: A \"no\" to a feature isn't a judgment on you or your idea. It means we're staying focused on our core vision. We appreciate all contributions and ideas!\n\n---\n\n**Questions about these principles?** Open a discussion on GitHub or join our [Discord](https://discord.gg/37XJPXfz2w).\n" }, { "path": "docs/7-DEVELOPMENT/development-setup.md", "content": "# Local Development Setup\n\nThis guide walks you through setting up Open Notebook for local development. Follow these steps to get the full stack running on your machine.\n\n## Prerequisites\n\nBefore you start, ensure you have the following installed:\n\n- **Python 3.11+** - Check with: `python --version`\n- **uv** (recommended) or **pip** - Install from: https://github.com/astral-sh/uv\n- **SurrealDB** - Via Docker or binary (see below)\n- **Docker** (optional) - For containerized database\n- **Node.js 18+** (optional) - For frontend development\n- **Git** - For version control\n\n## Step 1: Clone and Initial Setup\n\n```bash\n# Clone the repository\ngit clone https://github.com/lfnovo/open-notebook.git\ncd open-notebook\n\n# Add upstream remote for keeping your fork updated\ngit remote add upstream https://github.com/lfnovo/open-notebook.git\n```\n\n## Step 2: Install Python Dependencies\n\n```bash\n# Using uv (recommended)\nuv sync\n\n# Or using pip\npip install -e .\n```\n\n## Step 3: Environment Variables\n\nCreate a `.env` file in the project root with your configuration:\n\n```bash\n# Copy from example\ncp .env.example .env\n```\n\nEdit `.env` with your settings:\n\n```bash\n# Database\nSURREAL_URL=ws://localhost:8000/rpc\nSURREAL_USER=root\nSURREAL_PASSWORD=password\nSURREAL_NAMESPACE=open_notebook\nSURREAL_DATABASE=development\n\n# Credential encryption (required for storing API keys)\nOPEN_NOTEBOOK_ENCRYPTION_KEY=my-dev-secret-key\n\n# Application\nAPP_PASSWORD= # Optional password protection\nDEBUG=true\nLOG_LEVEL=DEBUG\n```\n\n### AI Provider Configuration\n\nAfter starting the API and frontend, configure your AI provider via the Settings UI:\n\n1. Open **http://localhost:3000** → **Settings** → **API Keys**\n2. Click **Add Credential** → Select your provider\n3. Enter your API key (get from provider dashboard)\n4. Click **Save**, then **Test Connection**\n5. Click **Discover Models** → **Register Models**\n\nPopular providers:\n- **OpenAI** - https://platform.openai.com/api-keys\n- **Anthropic (Claude)** - https://console.anthropic.com/\n- **Google** - https://ai.google.dev/\n- **Groq** - https://console.groq.com/\n\nFor local development, you can also use:\n- **Ollama** - Run locally without API keys (see \"Local Ollama\" below)\n\n> **Note:** API key environment variables (e.g., `OPENAI_API_KEY`) are deprecated. Use the Settings UI to manage credentials instead.\n\n## Step 4: Start SurrealDB\n\n### Option A: Using Docker (Recommended)\n\n```bash\n# Start SurrealDB in memory\ndocker run -d --name surrealdb -p 8000:8000 \\\n surrealdb/surrealdb:v2 start \\\n --user root --pass password \\\n --bind 0.0.0.0:8000 memory\n\n# Or with persistent storage\ndocker run -d --name surrealdb -p 8000:8000 \\\n -v surrealdb_data:/data \\\n surrealdb/surrealdb:v2 start \\\n --user root --pass password \\\n --bind 0.0.0.0:8000 file:/data/surreal.db\n```\n\n### Option B: Using Make\n\n```bash\nmake database\n```\n\n### Option C: Using Docker Compose\n\n```bash\ndocker compose up -d surrealdb\n```\n\n### Verify SurrealDB is Running\n\n```bash\n# Should show server information\ncurl http://localhost:8000/\n```\n\n## Step 5: Run Database Migrations\n\nDatabase migrations run automatically when you start the API. The first startup will apply any pending migrations.\n\nTo verify migrations manually:\n\n```bash\n# API will run migrations on startup\nuv run python -m api.main\n```\n\nCheck the logs - you should see messages like:\n```\nRunning migration 001_initial_schema\nRunning migration 002_add_vectors\n...\nMigrations completed successfully\n```\n\n## Step 6: Start the API Server\n\nIn a new terminal window:\n\n```bash\n# Terminal 2: Start API (port 5055)\nuv run --env-file .env uvicorn api.main:app --host 0.0.0.0 --port 5055\n\n# Or using the shortcut\nmake api\n```\n\nYou should see:\n```\nINFO: Application startup complete\nINFO: Uvicorn running on http://0.0.0.0:5055\n```\n\n### Verify API is Running\n\n```bash\n# Check health endpoint\ncurl http://localhost:5055/health\n\n# View API documentation\nopen http://localhost:5055/docs\n```\n\n## Step 7: Start the Frontend (Optional)\n\nIf you want to work on the frontend, start Next.js in another terminal:\n\n```bash\n# Terminal 3: Start Next.js frontend (port 3000)\ncd frontend\nnpm install # First time only\nnpm run dev\n```\n\nYou should see:\n```\n> next dev\n ▲ Next.js 16.x\n - Local: http://localhost:3000\n```\n\n### Access the Frontend\n\nOpen your browser to: http://localhost:3000\n\n## Verification Checklist\n\nAfter setup, verify everything is working:\n\n- [ ] **SurrealDB**: `curl http://localhost:8000/` returns content\n- [ ] **API**: `curl http://localhost:5055/health` returns `{\"status\": \"ok\"}`\n- [ ] **API Docs**: `open http://localhost:5055/docs` works\n- [ ] **Database**: API logs show migrations completing\n- [ ] **Frontend** (optional): `http://localhost:3000` loads\n\n## Starting Services Together\n\n### Quick Start All Services\n\n```bash\nmake start-all\n```\n\nThis starts SurrealDB, API, and frontend in one command.\n\n### Individual Terminals (Recommended for Development)\n\n**Terminal 1 - Database:**\n```bash\nmake database\n```\n\n**Terminal 2 - API:**\n```bash\nmake api\n```\n\n**Terminal 3 - Frontend:**\n```bash\ncd frontend && npm run dev\n```\n\n## Development Tools Setup\n\n### Pre-commit Hooks (Optional but Recommended)\n\nInstall git hooks to automatically check code quality:\n\n```bash\nuv run pre-commit install\n```\n\nNow your commits will be checked before they're made.\n\n### Code Quality Commands\n\n```bash\n# Lint Python code (auto-fix)\nmake ruff\n# or: ruff check . --fix\n\n# Type check Python code\nmake lint\n# or: uv run python -m mypy .\n\n# Run tests\nuv run pytest\n\n# Run tests with coverage\nuv run pytest --cov=open_notebook\n```\n\n## Common Development Tasks\n\n### Running Tests\n\n```bash\n# Run all tests\nuv run pytest\n\n# Run specific test file\nuv run pytest tests/test_notebooks.py\n\n# Run with coverage report\nuv run pytest --cov=open_notebook --cov-report=html\n```\n\n### Creating a Feature Branch\n\n```bash\n# Create and switch to new branch\ngit checkout -b feature/my-feature\n\n# Make changes, then commit\ngit add .\ngit commit -m \"feat: add my feature\"\n\n# Push to your fork\ngit push origin feature/my-feature\n```\n\n### Updating from Upstream\n\n```bash\n# Fetch latest changes\ngit fetch upstream\n\n# Rebase your branch\ngit rebase upstream/main\n\n# Push updated branch\ngit push origin feature/my-feature -f\n```\n\n## Troubleshooting\n\n### \"Connection refused\" on SurrealDB\n\n**Problem**: API can't connect to SurrealDB\n\n**Solutions**:\n1. Check if SurrealDB is running: `docker ps | grep surrealdb`\n2. Verify URL in `.env`: Should be `ws://localhost:8000/rpc`\n3. Restart SurrealDB: `docker stop surrealdb && docker rm surrealdb`\n4. Then restart with: `docker run -d --name surrealdb -p 8000:8000 surrealdb/surrealdb:v2 start --user root --pass password --bind 0.0.0.0:8000 memory`\n\n### \"Address already in use\"\n\n**Problem**: Port 5055 or 3000 is already in use\n\n**Solutions**:\n```bash\n# Find process using port\nlsof -i :5055 # Check port 5055\n\n# Kill process (macOS/Linux)\nkill -9 \n\n# Or use different port\nuvicorn api.main:app --port 5056\n```\n\n### Module not found errors\n\n**Problem**: Import errors when running API\n\n**Solutions**:\n```bash\n# Reinstall dependencies\nuv sync\n\n# Or with pip\npip install -e .\n```\n\n### Database migration failures\n\n**Problem**: API fails to start with migration errors\n\n**Solutions**:\n1. Check SurrealDB is running: `curl http://localhost:8000/`\n2. Check credentials in `.env` match your SurrealDB setup\n3. Check logs for specific migration error: `make api 2>&1 | grep -i migration`\n4. Verify database exists: Check SurrealDB console at http://localhost:8000/\n\n### Migrations not applying\n\n**Problem**: Database schema seems outdated\n\n**Solutions**:\n1. Restart API - migrations run on startup: `make api`\n2. Check logs show \"Migrations completed successfully\"\n3. Verify `/migrations/` folder exists and has files\n4. Check SurrealDB is writable and not in read-only mode\n\n## Optional: Local Ollama Setup\n\nFor testing with local AI models:\n\n```bash\n# Install Ollama from https://ollama.ai\n\n# Pull a model (e.g., Mistral 7B)\nollama pull mistral\n```\n\nThen configure via the Settings UI:\n1. Go to **Settings** → **API Keys** → **Add Credential** → **Ollama**\n2. Enter base URL: `http://localhost:11434`\n3. Click **Save**, then **Test Connection**\n4. Click **Discover Models** → **Register Models**\n\n## Optional: Docker Development Environment\n\nRun entire stack in Docker:\n\n```bash\n# Start all services\ndocker compose --profile multi up\n\n# Logs\ndocker compose logs -f\n\n# Stop services\ndocker compose down\n```\n\n## Next Steps\n\nAfter setup is complete:\n\n1. **Read the Contributing Guide** - [contributing.md](contributing.md)\n2. **Explore the Architecture** - Check the documentation\n3. **Find an Issue** - Look for \"good first issue\" on GitHub\n4. **Set Up Pre-commit** - Install git hooks for code quality\n5. **Join Discord** - https://discord.gg/37XJPXfz2w\n\n## Getting Help\n\nIf you get stuck:\n\n- **Discord**: [Join our server](https://discord.gg/37XJPXfz2w) for real-time help\n- **GitHub Issues**: Check existing issues for similar problems\n- **GitHub Discussions**: Ask questions in discussions\n- **Documentation**: See [code-standards.md](code-standards.md) and [testing.md](testing.md)\n\n---\n\n**Ready to contribute?** Go to [contributing.md](contributing.md) for the contribution workflow.\n" }, { "path": "docs/7-DEVELOPMENT/index.md", "content": "# Development\n\nWelcome to the Open Notebook development documentation! Whether you're contributing code, understanding our architecture, or maintaining the project, you'll find guidance here.\n\n## 🎯 Pick Your Path\n\n### 👨‍💻 I Want to Contribute Code\n\nStart with **[Contributing Guide](contributing.md)** for the workflow, then check:\n- **[Quick Start](quick-start.md)** - Clone, install, verify in 5 minutes\n- **[Development Setup](development-setup.md)** - Complete local environment guide\n- **[Code Standards](code-standards.md)** - How to write code that fits our style\n- **[Testing](testing.md)** - How to write and run tests\n\n**First time?** Check out our [Contributing Guide](contributing.md) for the issue-first workflow.\n\n---\n\n### 🏗️ I Want to Understand the Architecture\n\n**[Architecture Overview](architecture.md)** covers:\n- 3-tier system design\n- Tech stack and rationale\n- Key components and workflows\n- Design patterns we use\n\nFor deeper dives, check `/open_notebook/` CLAUDE.md for component-specific guidance.\n\n---\n\n### 👨‍🔧 I'm a Maintainer\n\n**[Maintainer Guide](maintainer-guide.md)** covers:\n- Issue triage and management\n- Pull request review process\n- Communication templates\n- Best practices\n\n---\n\n## 📚 Quick Links\n\n| Document | For | Purpose |\n|---|---|---|\n| [Quick Start](quick-start.md) | New developers | Clone, install, and verify setup (5 min) |\n| [Development Setup](development-setup.md) | Local development | Complete environment setup guide |\n| [Contributing](contributing.md) | Code contributors | Workflow: issue → code → PR |\n| [Code Standards](code-standards.md) | Writing code | Style guides for Python, FastAPI, DB |\n| [Testing](testing.md) | Testing code | How to write and run tests |\n| [Architecture](architecture.md) | Understanding system | System design, tech stack, workflows |\n| [Design Principles](design-principles.md) | All developers | What guides our decisions |\n| [API Reference](api-reference.md) | Building integrations | Complete REST API documentation |\n| [Maintainer Guide](maintainer-guide.md) | Maintainers | Managing issues, PRs, releases |\n\n---\n\n## 🚀 Current Development Priorities\n\nWe're actively looking for help with:\n\n1. **Frontend Enhancement** - Improve Next.js/React UI with real-time updates\n2. **Performance** - Async processing and caching optimizations\n3. **Testing** - Expand test coverage across components\n4. **Documentation** - API examples and developer guides\n5. **Integrations** - New content sources and AI providers\n\nSee GitHub Issues labeled `good first issue` or `help wanted`.\n\n---\n\n## 💬 Getting Help\n\n- **Discord**: [Join our server](https://discord.gg/37XJPXfz2w) for real-time discussions\n- **GitHub Discussions**: For architecture questions\n- **GitHub Issues**: For bugs and features\n\nDon't be shy! We're here to help new contributors succeed.\n\n---\n\n## 📖 Additional Resources\n\n### External Documentation\n- [FastAPI Docs](https://fastapi.tiangolo.com/)\n- [SurrealDB Docs](https://surrealdb.com/docs)\n- [LangChain Docs](https://python.langchain.com/)\n- [Next.js Docs](https://nextjs.org/docs)\n\n### Our Libraries\n- [Esperanto](https://github.com/lfnovo/esperanto) - Multi-provider AI abstraction\n- [Content Core](https://github.com/lfnovo/content-core) - Content processing\n- [Podcast Creator](https://github.com/lfnovo/podcast-creator) - Podcast generation\n\n---\n\nReady to get started? Head over to **[Quick Start](quick-start.md)**! 🎉\n" }, { "path": "docs/7-DEVELOPMENT/maintainer-guide.md", "content": "# Maintainer Guide\n\nThis guide is for project maintainers to help manage contributions effectively while maintaining project quality and vision.\n\n## Table of Contents\n\n- [Issue Management](#issue-management)\n- [Pull Request Review](#pull-request-review)\n- [Common Scenarios](#common-scenarios)\n- [Communication Templates](#communication-templates)\n\n## Issue Management\n\n### When a New Issue is Created\n\n**1. Initial Triage** (within 24-48 hours)\n\n- Add appropriate labels:\n - `bug`, `enhancement`, `documentation`, etc.\n - `good first issue` for beginner-friendly tasks\n - `needs-triage` until reviewed\n - `help wanted` if you'd welcome community contributions\n\n- Quick assessment:\n - Is it clear and well-described?\n - Is it aligned with project vision? (See [design-principles.md](design-principles.md))\n - Does it duplicate an existing issue?\n\n**2. Initial Response**\n\n```markdown\nThanks for opening this issue! We'll review it and get back to you soon.\n\n[If it's a bug] In the meantime, have you checked our troubleshooting guide?\n\n[If it's a feature] You might find our [design principles](design-principles.md) helpful for understanding what we're building toward.\n```\n\n**3. Decision Making**\n\nAsk yourself:\n- Does this align with our [design principles](design-principles.md)?\n- Is this something we want in the core project, or better as a plugin/extension?\n- Do we have the capacity to support this feature long-term?\n- Will this benefit most users, or just a specific use case?\n\n**4. Issue Assignment**\n\nIf the contributor checked \"I am a developer and would like to work on this\":\n\n**For Accepted Issues:**\n```markdown\nGreat idea! This aligns well with our goals, particularly [specific design principle].\n\nI see you'd like to work on this. Before you start:\n\n1. Please share your proposed approach/solution\n2. Review our [Contributing Guide](contributing.md) and [Design Principles](design-principles.md)\n3. Once we agree on the approach, I'll assign this to you\n\nLooking forward to your thoughts!\n```\n\n**For Issues Needing Clarification:**\n```markdown\nThanks for offering to work on this! Before we proceed, we need to clarify a few things:\n\n1. [Question 1]\n2. [Question 2]\n\nOnce we have these details, we can discuss the best approach.\n```\n\n**For Issues Not Aligned with Vision:**\n```markdown\nThank you for the suggestion and for offering to work on this!\n\nAfter reviewing against our [design principles](design-principles.md), we've decided not to pursue this in the core project because [specific reason].\n\nHowever, you might be able to achieve this through [alternative approach, if applicable].\n\nWe appreciate your interest in contributing! Feel free to check out our [open issues](link) for other ways to contribute.\n```\n\n### Labels to Use\n\n**Priority:**\n- `priority: critical` - Security issues, data loss bugs\n- `priority: high` - Major functionality broken\n- `priority: medium` - Annoying bugs, useful features\n- `priority: low` - Nice to have, edge cases\n\n**Status:**\n- `needs-triage` - Not yet reviewed by maintainer\n- `needs-info` - Waiting for more information from reporter\n- `needs-discussion` - Requires community/team discussion\n- `ready` - Approved and ready to be worked on\n- `in-progress` - Someone is actively working on this\n- `blocked` - Cannot proceed due to external dependency\n\n**Type:**\n- `bug` - Something is broken\n- `enhancement` - New feature or improvement\n- `documentation` - Documentation improvements\n- `question` - General questions\n- `refactor` - Code cleanup/restructuring\n\n**Difficulty:**\n- `good first issue` - Good for newcomers\n- `help wanted` - Community contributions welcome\n- `advanced` - Requires deep codebase knowledge\n\n## Pull Request Review\n\n### Initial PR Review Checklist\n\n**Before diving into code:**\n\n- [ ] Is there an associated approved issue?\n- [ ] Does the PR reference the issue number?\n- [ ] Is the PR description clear about what changed and why?\n- [ ] Did the contributor check the relevant boxes in the PR template?\n- [ ] Are there tests? Screenshots (for UI changes)?\n\n**Red Flags** (may require closing PR):\n- No associated issue\n- Issue was not assigned to contributor\n- PR tries to solve multiple unrelated problems\n- Breaking changes without discussion\n- Conflicts with project vision\n\n### Code Review Process\n\n**1. High-Level Review**\n\n- Does the approach align with our architecture?\n- Is the solution appropriately scoped?\n- Are there simpler alternatives?\n- Does it follow our design principles?\n\n**2. Code Quality Review**\n\nPython:\n- [ ] Follows PEP 8\n- [ ] Has type hints\n- [ ] Has docstrings\n- [ ] Proper error handling\n- [ ] No security vulnerabilities\n\nTypeScript/Frontend:\n- [ ] Follows TypeScript best practices\n- [ ] Proper component structure\n- [ ] No console.logs left in production code\n- [ ] Accessible UI components\n\n**3. Testing Review**\n\n- [ ] Has appropriate test coverage\n- [ ] Tests are meaningful (not just for coverage percentage)\n- [ ] Tests pass locally and in CI\n- [ ] Edge cases are tested\n\n**4. Documentation Review**\n\n- [ ] Code is well-commented\n- [ ] Complex logic is explained\n- [ ] User-facing documentation updated (if applicable)\n- [ ] API documentation updated (if API changed)\n- [ ] Migration guide provided (if breaking change)\n\n### Providing Feedback\n\n**Positive Feedback** (important!):\n```markdown\nThanks for this PR! I really like [specific thing they did well].\n\n[Feedback on what needs to change]\n```\n\n**Requesting Changes:**\n```markdown\nThis is a great start! A few things to address:\n\n1. **[High-level concern]**: [Explanation and suggested approach]\n2. **[Code quality issue]**: [Specific example and fix]\n3. **[Testing gap]**: [What scenarios need coverage]\n\nLet me know if you have questions about any of this!\n```\n\n**Suggesting Alternative Approach:**\n```markdown\nI appreciate the effort you put into this! However, I'm concerned about [specific issue].\n\nHave you considered [alternative approach]? It might be better because [reasons].\n\nWhat do you think?\n```\n\n## Common Scenarios\n\n### Scenario 1: Good Code, Wrong Approach\n\n**Situation**: Contributor wrote quality code, but solved the problem in a way that doesn't fit our architecture.\n\n**Response:**\n```markdown\nThank you for this PR! The code quality is great, and I can see you put thought into this.\n\nHowever, I'm concerned that this approach [specific architectural concern]. In our architecture, we [explain the pattern we follow].\n\nWould you be open to refactoring this to [suggested approach]? I'm happy to provide guidance on the specifics.\n\nAlternatively, if you don't have time for a refactor, I can take over and finish this up (with credit to you, of course).\n\nLet me know what you prefer!\n```\n\n### Scenario 2: PR Without Assigned Issue\n\n**Situation**: Contributor submitted PR without going through issue approval process.\n\n**Response:**\n```markdown\nThanks for the PR! I appreciate you taking the time to contribute.\n\nHowever, to maintain project coherence, we require all PRs to be linked to an approved issue that was assigned to the contributor. This is explained in our [Contributing Guide](contributing.md).\n\nThis helps us:\n- Ensure work aligns with project vision\n- Prevent duplicate efforts\n- Discuss approach before implementation\n\nCould you please:\n1. Create an issue describing this change\n2. Wait for it to be reviewed and assigned to you\n3. We can then reopen this PR or you can create a new one\n\nSorry for the inconvenience - this process helps us manage the project effectively.\n```\n\n### Scenario 3: Feature Request Not Aligned with Vision\n\n**Situation**: Well-intentioned feature that doesn't fit project goals.\n\n**Response:**\n```markdown\nThank you for this suggestion! I can see how this would be useful for [specific use case].\n\nAfter reviewing against our [design principles](design-principles.md), we've decided not to include this in the core project because [specific reason - e.g., \"it conflicts with our 'Simplicity Over Features' principle\" or \"it would require dependencies that conflict with our privacy-first approach\"].\n\nSome alternatives:\n- [If applicable] This could be built as a plugin/extension\n- [If applicable] This functionality might be achievable through [existing feature]\n- [If applicable] You might be interested in [other tool] which is designed for this use case\n\nWe appreciate your contribution and hope you understand. Feel free to check our roadmap or open issues for other ways to contribute!\n```\n\n### Scenario 4: Contributor Ghosts After Feedback\n\n**Situation**: You requested changes, but contributor hasn't responded in 2+ weeks.\n\n**After 2 weeks:**\n```markdown\nHey there! Just checking in on this PR. Do you have time to address the feedback, or would you like someone else to take over?\n\nNo pressure either way - just want to make sure this doesn't fall through the cracks.\n```\n\n**After 1 month with no response:**\n```markdown\nThanks again for starting this work! Since we haven't heard back, I'm going to close this PR for now.\n\nIf you want to pick this up again in the future, feel free to reopen it or create a new PR. Alternatively, I'll mark the issue as available for someone else to work on.\n\nWe appreciate your contribution!\n```\n\nThen:\n- Close the PR\n- Unassign the issue\n- Add `help wanted` label to the issue\n\n### Scenario 5: Breaking Changes Without Discussion\n\n**Situation**: PR introduces breaking changes that weren't discussed.\n\n**Response:**\n```markdown\nThanks for this PR! However, I notice this introduces breaking changes that weren't discussed in the original issue.\n\nBreaking changes require:\n1. Prior discussion and approval\n2. Migration guide for users\n3. Deprecation period (when possible)\n4. Clear documentation of the change\n\nCould we discuss the breaking changes first? Specifically:\n- [What breaks and why]\n- [Who will be affected]\n- [Migration path]\n\nWe may need to adjust the approach to minimize impact on existing users.\n```\n\n## Communication Templates\n\n### Closing a PR (Misaligned with Vision)\n\n```markdown\nThank you for taking the time to contribute! We really appreciate it.\n\nAfter careful review, we've decided not to merge this PR because [specific reason related to design principles].\n\nThis isn't a reflection on your code quality - it's about maintaining focus on our core goals as outlined in [design-principles.md](design-principles.md).\n\nWe'd love to have you contribute in other ways! Check out:\n- Good first issues\n- Help wanted issues\n- Our roadmap\n\nThanks again for your interest in Open Notebook!\n```\n\n### Closing a Stale Issue\n\n```markdown\nWe're closing this issue due to inactivity. If this is still relevant, feel free to reopen it with updated information.\n\nThanks!\n```\n\n### Asking for More Information\n\n```markdown\nThanks for reporting this! To help us investigate, could you provide:\n\n1. [Specific information needed]\n2. [Logs, screenshots, etc.]\n3. [Steps to reproduce]\n\nThis will help us understand the issue better and find a solution.\n```\n\n### Thanking a Contributor\n\n```markdown\nMerged!\n\nThank you so much for this contribution, @username! [Specific thing they did well].\n\nThis will be included in the next release.\n```\n\n## Best Practices\n\n### Be Kind and Respectful\n\n- Thank contributors for their time and effort\n- Assume good intentions\n- Be patient with newcomers\n- Explain *why*, not just *what*\n\n### Be Clear and Direct\n\n- Don't leave ambiguity about next steps\n- Be specific about what needs to change\n- Explain architectural decisions\n- Set clear expectations\n\n### Be Consistent\n\n- Apply the same standards to all contributors\n- Follow the process you've defined\n- Document decisions for future reference\n\n### Be Protective of Project Vision\n\n- It's okay to say \"no\"\n- Prioritize long-term maintainability\n- Don't accept features you can't support\n- Keep the project focused\n\n### Be Responsive\n\n- Respond to issues within 48 hours (even just to acknowledge)\n- Review PRs within a week when possible\n- Keep contributors updated on status\n- Close stale issues/PRs to keep things tidy\n\n## When in Doubt\n\nAsk yourself:\n1. Does this align with our [design principles](design-principles.md)?\n2. Will we be able to maintain this feature long-term?\n3. Does this benefit most users, or just an edge case?\n4. Is there a simpler alternative?\n5. Would I want to support this in 2 years?\n\nIf you're unsure, it's perfectly fine to:\n- Ask for input from other maintainers\n- Start a discussion issue\n- Sleep on it before making a decision\n\n---\n\n**Remember**: Good maintainership is about balancing openness to contributions with protection of project vision. You're not being mean by saying \"no\" to things that don't fit - you're being a responsible steward of the project.\n" }, { "path": "docs/7-DEVELOPMENT/quick-start.md", "content": "# Quick Start - Development\n\nGet Open Notebook running locally in 5 minutes.\n\n## Prerequisites\n\n- **Python 3.11+**\n- **Git**\n- **uv** (package manager) - install with `curl -LsSf https://astral.sh/uv/install.sh | sh`\n- **Docker** (optional, for SurrealDB)\n\n## 1. Clone the Repository (2 min)\n\n```bash\n# Fork the repository on GitHub first, then clone your fork\ngit clone https://github.com/YOUR_USERNAME/open-notebook.git\ncd open-notebook\n\n# Add upstream remote for updates\ngit remote add upstream https://github.com/lfnovo/open-notebook.git\n```\n\n## 2. Install Dependencies (2 min)\n\n```bash\n# Install Python dependencies\nuv sync\n\n# Verify uv is working\nuv --version\n```\n\n## 3. Start Services (1 min)\n\nIn separate terminal windows:\n\n```bash\n# Terminal 1: Start SurrealDB (database)\nmake database\n# or: docker run -d --name surrealdb -p 8000:8000 surrealdb/surrealdb:v2 start --user root --pass password --bind 0.0.0.0:8000 memory\n\n# Terminal 2: Start API (backend on port 5055)\nmake api\n# or: uv run --env-file .env uvicorn api.main:app --host 0.0.0.0 --port 5055\n\n# Terminal 3: Start Frontend (UI on port 3000)\ncd frontend && npm run dev\n```\n\n## 4. Verify Everything Works (instant)\n\n- **API Health**: http://localhost:5055/health → should return `{\"status\": \"ok\"}`\n- **API Docs**: http://localhost:5055/docs → interactive API documentation\n- **Frontend**: http://localhost:3000 → Open Notebook UI\n\n**All three show up?** ✅ You're ready to develop!\n\n---\n\n## Next Steps\n\n- **First Issue?** Pick a [good first issue](https://github.com/lfnovo/open-notebook/issues?q=label%3A%22good+first+issue%22)\n- **Understand the code?** Read [Architecture Overview](architecture.md)\n- **Make changes?** Follow [Contributing Guide](contributing.md)\n- **Setup details?** See [Development Setup](development-setup.md)\n\n---\n\n## Troubleshooting\n\n### \"Port 5055 already in use\"\n```bash\n# Find what's using the port\nlsof -i :5055\n\n# Use a different port\nuv run uvicorn api.main:app --port 5056\n```\n\n### \"Can't connect to SurrealDB\"\n```bash\n# Check if SurrealDB is running\ndocker ps | grep surrealdb\n\n# Restart it\nmake database\n```\n\n### \"Python version is too old\"\n```bash\n# Check your Python version\npython --version # Should be 3.11+\n\n# Use Python 3.11 specifically\nuv sync --python 3.11\n```\n\n### \"npm: command not found\"\n```bash\n# Install Node.js from https://nodejs.org/\n# Then install frontend dependencies\ncd frontend && npm install\n```\n\n---\n\n## Common Development Commands\n\n```bash\n# Run tests\nuv run pytest\n\n# Format code\nmake ruff\n\n# Type checking\nmake lint\n\n# Run the full stack\nmake start-all\n\n# View API documentation\nopen http://localhost:5055/docs\n```\n\n---\n\nNeed more help? See [Development Setup](development-setup.md) for details or join our [Discord](https://discord.gg/37XJPXfz2w).\n" }, { "path": "docs/7-DEVELOPMENT/testing.md", "content": "# Testing Guide\n\nThis document provides guidelines for writing tests in Open Notebook. Testing is critical to maintaining code quality and preventing regressions.\n\n## Testing Philosophy\n\n### What to Test\n\nFocus on testing the things that matter most:\n\n- **Business Logic** - Core domain models and their operations\n- **API Contracts** - HTTP endpoint behavior and error handling\n- **Critical Workflows** - End-to-end flows that users depend on\n- **Data Persistence** - Database operations and data integrity\n- **Error Conditions** - How the system handles failures gracefully\n\n### What NOT to Test\n\nDon't waste time testing framework code:\n\n- Framework functionality (FastAPI, React, etc.)\n- Third-party library implementation\n- Simple getters/setters without logic\n- View/presentation layer rendering (unless it contains logic)\n\n## Test Structure\n\nWe use **pytest** with async support for all Python tests:\n\n```python\nimport pytest\nfrom httpx import AsyncClient\nfrom open_notebook.domain.notebook import Notebook\n\n@pytest.mark.asyncio\nasync def test_create_notebook():\n \"\"\"Test notebook creation.\"\"\"\n notebook = Notebook(name=\"Test Notebook\", description=\"Test description\")\n await notebook.save()\n\n assert notebook.id is not None\n assert notebook.name == \"Test Notebook\"\n assert notebook.created is not None\n\n@pytest.mark.asyncio\nasync def test_api_create_notebook():\n \"\"\"Test notebook creation via API.\"\"\"\n async with AsyncClient(app=app, base_url=\"http://test\") as client:\n response = await client.post(\n \"/api/notebooks\",\n json={\"name\": \"Test Notebook\", \"description\": \"Test description\"}\n )\n assert response.status_code == 200\n data = response.json()\n assert data[\"name\"] == \"Test Notebook\"\n```\n\n## Test Categories\n\n### 1. Unit Tests\n\nTest individual functions and methods in isolation:\n\n```python\n@pytest.mark.asyncio\nasync def test_notebook_validation():\n \"\"\"Test that notebook name validation works.\"\"\"\n with pytest.raises(InvalidInputError):\n Notebook(name=\"\", description=\"test\")\n\n@pytest.mark.asyncio\nasync def test_notebook_archive():\n \"\"\"Test notebook archiving.\"\"\"\n notebook = Notebook(name=\"Test\", description=\"\")\n notebook.archive()\n assert notebook.archived is True\n```\n\n**Location**: `tests/unit/`\n\n### 2. Integration Tests\n\nTest component interactions and database operations:\n\n```python\n@pytest.mark.asyncio\nasync def test_create_notebook_with_sources():\n \"\"\"Test creating a notebook and adding sources.\"\"\"\n notebook = await create_notebook(name=\"Research\", description=\"\")\n source = await add_source(notebook_id=notebook.id, url=\"https://example.com\")\n\n retrieved = await get_notebook_with_sources(notebook.id)\n assert len(retrieved.sources) == 1\n assert retrieved.sources[0].id == source.id\n```\n\n**Location**: `tests/integration/`\n\n### 3. API Tests\n\nTest HTTP endpoints and error responses:\n\n```python\n@pytest.mark.asyncio\nasync def test_get_notebooks_endpoint():\n \"\"\"Test GET /notebooks endpoint.\"\"\"\n async with AsyncClient(app=app, base_url=\"http://test\") as client:\n response = await client.get(\"/api/notebooks\")\n assert response.status_code == 200\n data = response.json()\n assert isinstance(data, list)\n\n@pytest.mark.asyncio\nasync def test_create_notebook_validation():\n \"\"\"Test that invalid input is rejected.\"\"\"\n async with AsyncClient(app=app, base_url=\"http://test\") as client:\n response = await client.post(\n \"/api/notebooks\",\n json={\"name\": \"\", \"description\": \"\"}\n )\n assert response.status_code == 400\n```\n\n**Location**: `tests/api/`\n\n### 4. Database Tests\n\nTest data persistence and query correctness:\n\n```python\n@pytest.mark.asyncio\nasync def test_save_and_retrieve_notebook():\n \"\"\"Test saving and retrieving a notebook from database.\"\"\"\n notebook = Notebook(name=\"Test\", description=\"desc\")\n await notebook.save()\n\n retrieved = await Notebook.get(notebook.id)\n assert retrieved.name == \"Test\"\n assert retrieved.description == \"desc\"\n\n@pytest.mark.asyncio\nasync def test_query_by_criteria():\n \"\"\"Test querying notebooks by criteria.\"\"\"\n await create_notebook(\"Active\", \"\")\n await create_notebook(\"Archived\", \"\")\n\n active = await repo_query(\n \"SELECT * FROM notebook WHERE archived = false\"\n )\n assert len(active) >= 1\n```\n\n**Location**: `tests/database/`\n\n## Running Tests\n\n### Run All Tests\n\n```bash\nuv run pytest\n```\n\n### Run Specific Test File\n\n```bash\nuv run pytest tests/test_notebooks.py\n```\n\n### Run Specific Test Function\n\n```bash\nuv run pytest tests/test_notebooks.py::test_create_notebook\n```\n\n### Run with Coverage Report\n\n```bash\nuv run pytest --cov=open_notebook\n```\n\n### Run Only Unit Tests\n\n```bash\nuv run pytest tests/unit/\n```\n\n### Run Only Integration Tests\n\n```bash\nuv run pytest tests/integration/\n```\n\n### Run Tests in Verbose Mode\n\n```bash\nuv run pytest -v\n```\n\n### Run Tests with Output\n\n```bash\nuv run pytest -s\n```\n\n## Test Fixtures\n\nUse pytest fixtures for common setup and teardown:\n\n```python\nimport pytest\n\n@pytest.fixture\nasync def test_notebook():\n \"\"\"Create a test notebook.\"\"\"\n notebook = Notebook(name=\"Test Notebook\", description=\"Test description\")\n await notebook.save()\n yield notebook\n await notebook.delete()\n\n@pytest.fixture\nasync def api_client():\n \"\"\"Create an API test client.\"\"\"\n async with AsyncClient(app=app, base_url=\"http://test\") as client:\n yield client\n\n@pytest.fixture\nasync def test_notebook_with_sources(test_notebook):\n \"\"\"Create a test notebook with sample sources.\"\"\"\n source1 = Source(notebook_id=test_notebook.id, url=\"https://example.com\")\n source2 = Source(notebook_id=test_notebook.id, url=\"https://example.org\")\n await source1.save()\n await source2.save()\n\n test_notebook.sources = [source1, source2]\n yield test_notebook\n\n # Cleanup\n await source1.delete()\n await source2.delete()\n```\n\n## Best Practices\n\n### 1. Write Descriptive Test Names\n\n```python\n# Good - clearly describes what is being tested\nasync def test_create_notebook_with_valid_name_succeeds():\n ...\n\n# Bad - vague about what's being tested\nasync def test_notebook():\n ...\n```\n\n### 2. Use Docstrings\n\n```python\n@pytest.mark.asyncio\nasync def test_vector_search_returns_sorted_results():\n \"\"\"Test that vector search results are sorted by relevance score.\"\"\"\n # Implementation\n```\n\n### 3. Test Edge Cases\n\n```python\n@pytest.mark.asyncio\nasync def test_search_with_empty_query():\n \"\"\"Test that empty query raises error.\"\"\"\n with pytest.raises(InvalidInputError):\n await vector_search(\"\")\n\n@pytest.mark.asyncio\nasync def test_search_with_very_long_query():\n \"\"\"Test that very long query is handled.\"\"\"\n long_query = \"x\" * 10000\n results = await vector_search(long_query)\n assert isinstance(results, list)\n\n@pytest.mark.asyncio\nasync def test_search_with_special_characters():\n \"\"\"Test that special characters are handled.\"\"\"\n results = await vector_search(\"@#$%^&*()\")\n assert isinstance(results, list)\n```\n\n### 4. Use Assertions Effectively\n\n```python\n# Good - specific assertions\nassert notebook.name == \"Test\"\nassert len(notebook.sources) == 3\nassert notebook.created is not None\n\n# Less good - too broad\nassert notebook is not None\nassert notebook # ambiguous what's being tested\n```\n\n### 5. Test Both Success and Failure Cases\n\n```python\n@pytest.mark.asyncio\nasync def test_create_notebook_success():\n \"\"\"Test successful notebook creation.\"\"\"\n notebook = await create_notebook(name=\"Research\", description=\"AI\")\n assert notebook.id is not None\n assert notebook.name == \"Research\"\n\n@pytest.mark.asyncio\nasync def test_create_notebook_empty_name_fails():\n \"\"\"Test that empty name raises error.\"\"\"\n with pytest.raises(InvalidInputError):\n await create_notebook(name=\"\", description=\"\")\n\n@pytest.mark.asyncio\nasync def test_create_notebook_duplicate_fails():\n \"\"\"Test that duplicate names are handled.\"\"\"\n await create_notebook(name=\"Research\", description=\"\")\n with pytest.raises(DuplicateError):\n await create_notebook(name=\"Research\", description=\"\")\n```\n\n### 6. Keep Tests Independent\n\n```python\n# Good - test is self-contained\n@pytest.mark.asyncio\nasync def test_archive_notebook():\n notebook = Notebook(name=\"Test\", description=\"\")\n await notebook.save()\n await notebook.archive()\n assert notebook.archived is True\n\n# Bad - depends on another test's state\n@pytest.mark.asyncio\nasync def test_archive_existing_notebook():\n # Assumes test_create_notebook ran first\n await notebook.archive() # notebook undefined\n```\n\n### 7. Use Fixtures for Reusable Setup\n\n```python\n# Instead of repeating setup:\n@pytest.fixture\nasync def client_with_auth(api_client, mock_auth):\n \"\"\"Client with authentication set up.\"\"\"\n api_client.headers.update({\"Authorization\": f\"Bearer {mock_auth.token}\"})\n yield api_client\n\n@pytest.mark.asyncio\nasync def test_protected_endpoint(client_with_auth):\n \"\"\"Test protected endpoint.\"\"\"\n response = await client_with_auth.get(\"/api/protected\")\n assert response.status_code == 200\n```\n\n## Coverage Goals\n\n- Aim for 70%+ overall coverage\n- 90%+ coverage for critical business logic\n- Don't obsess over 100% - focus on meaningful tests\n- Use `--cov` flag to check coverage: `uv run pytest --cov=open_notebook`\n\n## Async Test Patterns\n\n### Testing Async Functions\n\n```python\n@pytest.mark.asyncio\nasync def test_async_operation():\n \"\"\"Test async function.\"\"\"\n result = await some_async_function()\n assert result is not None\n```\n\n### Testing Concurrent Operations\n\n```python\n@pytest.mark.asyncio\nasync def test_concurrent_notebook_creation():\n \"\"\"Test creating multiple notebooks concurrently.\"\"\"\n tasks = [\n create_notebook(f\"Notebook {i}\", \"\")\n for i in range(10)\n ]\n notebooks = await asyncio.gather(*tasks)\n assert len(notebooks) == 10\n assert all(n.id for n in notebooks)\n```\n\n## Common Testing Errors\n\n### Error: \"event loop is closed\"\n\nSolution: Use the async fixture properly:\n```python\n@pytest.fixture\nasync def notebook(): # Use async fixture\n notebook = Notebook(name=\"Test\", description=\"\")\n await notebook.save()\n yield notebook\n await notebook.delete()\n```\n\n### Error: \"object is not awaitable\"\n\nSolution: Make sure you're using await:\n```python\n# Wrong\nresult = create_notebook(\"Test\", \"\")\n\n# Right\nresult = await create_notebook(\"Test\", \"\")\n```\n\n---\n\n**See also:**\n- [Code Standards](code-standards.md) - Code formatting and style\n- [Contributing Guide](contributing.md) - Overall contribution workflow\n" }, { "path": "docs/SECURITY_REVIEW.md", "content": "# Security Review - API Configuration UI\n\n## Date: 2026-01-27 (Updated: 2026-01-28)\n## Reviewer: Security Audit\n\n---\n\n## Summary\n\nSecurity review of the API key management implementation for Open Notebook. The implementation uses a database-first approach with environment variable fallback.\n\n---\n\n## Encryption\n\n| Item | Status | Notes |\n|------|--------|-------|\n| Fernet encryption implemented | PASS | `open_notebook/utils/encryption.py` uses AES-128-CBC + HMAC-SHA256 |\n| Keys encrypted before DB storage | PASS | `encrypt_value()` applied on save |\n| Keys decrypted only when needed | PASS | `decrypt_value()` called when reading |\n| Encryption key required | PASS | No default key; ValueError if not configured |\n| Docker secrets support | PASS | `_FILE` suffix pattern supported |\n| Documented in .env.example | PASS | Encryption key documented |\n\n---\n\n## API Security\n\n| Item | Status | Notes |\n|------|--------|-------|\n| Test endpoint implemented | PASS | `connection_tester.py` validates keys |\n| Test doesn't expose keys | PASS | Only returns success/failure |\n| Error messages don't leak info | PASS | Generic error messages |\n| URL validation for SSRF | PASS | Blocks private IPs (except Ollama) |\n| Rate limiting | NOT IMPL | Future enhancement |\n\n---\n\n## Frontend Security\n\n| Item | Status | Notes |\n|------|--------|-------|\n| No keys in localStorage | PASS | Keys only in React state during entry |\n| Keys masked in UI | PASS | Shows `************` placeholder |\n| No keys in console.log | PASS | No logging of sensitive data |\n| autocomplete attributes | PARTIAL | Some forms missing autocomplete=\"off\" |\n\n---\n\n## Authentication\n\n| Item | Status | Notes |\n|------|--------|-------|\n| Password protection | PASS | Bearer token authentication |\n| Default password | PASS | \"open-notebook-change-me\" when not set |\n| Docker secrets support | PASS | `_FILE` suffix for password |\n| Security warnings | PASS | Logged when using defaults |\n\n---\n\n## Files Reviewed\n\n| Component | Path | Status |\n|-----------|------|--------|\n| Encryption | `open_notebook/utils/encryption.py` | PASS |\n| Credential model | `open_notebook/domain/credential.py` | PASS |\n| Credentials router | `api/routers/credentials.py` | PASS |\n| Key provider | `open_notebook/ai/key_provider.py` | PASS |\n| Connection tester | `open_notebook/ai/connection_tester.py` | PASS |\n| Auth middleware | `api/auth.py` | PASS |\n| Frontend forms | `frontend/src/components/settings/*.tsx` | PASS |\n| Environment example | `.env.example` | PASS |\n\n---\n\n## Remaining Recommendations\n\n### Future Improvements\n\n1. **Rate limiting** - Add rate limiting on `/credentials/*` endpoints\n2. **Autocomplete attributes** - Add `autocomplete=\"new-password\"` to all password inputs\n3. **Show last 4 characters** - Display `********xxxx` format for key identification\n4. **Audit logging** - Log API key changes with timestamps\n\n---\n\n## Conclusion\n\nThe API Configuration UI implementation meets security requirements:\n\n- API keys encrypted at rest using Fernet (key must be explicitly configured)\n- Keys never returned to frontend\n- URL validation prevents SSRF attacks\n- Docker secrets supported for production deployments\n\n**Review Status: PASS**\n" }, { "path": "docs/index.md", "content": "# Open Notebook Documentation\n\nWelcome to Open Notebook - a privacy-focused AI research assistant. This documentation is organized for different needs.\n\n---\n\n## 🎯 Choose Your Path\n\n### I'm brand new\n→ Start here: **[0-START-HERE](0-START-HERE/index.md)**\n- Learn what Open Notebook is\n- Pick your setup path (OpenAI, cloud, local/Ollama)\n- 5-minute quick start\n\n### I need to install/deploy\n→ Go here: **[1-INSTALLATION](1-INSTALLATION/index.md)**\n- Multiple installation routes\n- Docker Compose (recommended)\n- From source (developers)\n- Single container (shared hosting)\n\n### I want to understand how it works\n→ Read this: **[2-CORE-CONCEPTS](2-CORE-CONCEPTS/index.md)**\n- Mental models and architecture\n- How RAG (retrieval-augmented generation) works\n- Notebooks, sources, and notes explained\n- Chat vs. transformations vs. podcasts\n\n### I want to use it (tutorials)\n→ Follow this: **[3-USER-GUIDE](3-USER-GUIDE/index.md)**\n- How to add sources (PDFs, URLs, audio, video)\n- Creating and organizing notes\n- Chat effectively with your research\n- Creating podcasts from research\n- Search techniques\n\n### I need to configure it\n→ Check this: **[5-CONFIGURATION](5-CONFIGURATION/index.md)**\n- Choose and setup AI provider\n- API configuration\n- Database setup\n- Advanced tuning\n\n### I need provider-specific help\n→ Go here: **[4-AI-PROVIDERS](4-AI-PROVIDERS/index.md)**\n- OpenAI, Anthropic, Google, Groq, Ollama, Azure\n- Model comparisons\n- Cost estimates\n- Setup paths\n\n### Something's not working\n→ Troubleshoot: **[6-TROUBLESHOOTING](6-TROUBLESHOOTING/index.md)**\n- Quick fixes (top 10 issues)\n- Installation problems\n- Connection issues\n- AI/chat problems\n- Content processing issues\n- Podcast problems\n\n### I want to contribute/develop\n→ Read this: **[7-DEVELOPMENT](7-DEVELOPMENT/index.md)**\n- Architecture and tech stack\n- Contributing guidelines\n- API reference\n- Testing\n\n---\n\n## 📊 Documentation Overview\n\n### By Section\n\n**[0-START-HERE](0-START-HERE/index.md)** — Entry point\n- What is Open Notebook?\n- Quick start guides (3 routes)\n- First 5 minutes\n\n**[1-INSTALLATION](1-INSTALLATION/index.md)** — Getting it running\n- Multiple installation routes\n- Docker, single-container, from-source\n- Requirements and setup\n\n**[2-CORE-CONCEPTS](2-CORE-CONCEPTS/index.md)** — Understanding the system\n- Notebooks, sources, notes hierarchy\n- RAG (retrieval-augmented generation)\n- Chat, transformations, podcasts\n- Context management\n\n**[3-USER-GUIDE](3-USER-GUIDE/index.md)** — Using features\n- Adding sources (all types)\n- Working with notes\n- Chat effectively\n- Creating podcasts\n- Searching (text and semantic)\n\n**[4-AI-PROVIDERS](4-AI-PROVIDERS/index.md)** — AI configuration\n- Provider comparison\n- Setup for each provider\n- Model recommendations\n- Cost estimates\n\n**[5-CONFIGURATION](5-CONFIGURATION/index.md)** — Complete reference\n- AI provider setup (detailed)\n- Database configuration\n- Server/API settings\n- Advanced tuning\n- Environment variables (complete reference)\n\n**[6-TROUBLESHOOTING](6-TROUBLESHOOTING/index.md)** — Problem solving\n- Quick fixes (top 10)\n- Installation issues\n- Connection problems\n- AI/chat issues\n- Content processing\n- Podcast generation\n- Getting help\n\n**[7-DEVELOPMENT](7-DEVELOPMENT/index.md)** — For contributors\n- Architecture\n- Contributing guidelines\n- API reference\n- Testing & development\n\n---\n\n## 🔍 Find What You Need\n\n### By Problem Type\n\n**Installation & Setup**\n- Fresh install? → [0-START-HERE](0-START-HERE/index.md)\n- Detailed installation routes? → [1-INSTALLATION](1-INSTALLATION/index.md)\n- Configuration reference? → [5-CONFIGURATION](5-CONFIGURATION/index.md)\n- Provider setup? → [4-AI-PROVIDERS](4-AI-PROVIDERS/index.md)\n\n**Using Open Notebook**\n- How to use features? → [3-USER-GUIDE](3-USER-GUIDE/index.md)\n- Understanding concepts? → [2-CORE-CONCEPTS](2-CORE-CONCEPTS/index.md)\n- Chat not working? → [6-TROUBLESHOOTING - AI Issues](6-TROUBLESHOOTING/ai-chat-issues.md)\n- Files won't upload? → [6-TROUBLESHOOTING - Quick Fixes](6-TROUBLESHOOTING/quick-fixes.md#4-cannot-process-file-or-unsupported-format)\n\n**Troubleshooting**\n- Quick fix? → [6-TROUBLESHOOTING - Quick Fixes](6-TROUBLESHOOTING/quick-fixes.md)\n- Can't connect? → [6-TROUBLESHOOTING - Connection](6-TROUBLESHOOTING/connection-issues.md)\n- Chat issues? → [6-TROUBLESHOOTING - AI Issues](6-TROUBLESHOOTING/ai-chat-issues.md)\n- Podcast problems? → [6-TROUBLESHOOTING - Quick Fixes](6-TROUBLESHOOTING/quick-fixes.md#8-podcast-generation-failed)\n\n**Development**\n- Architecture? → [7-DEVELOPMENT - Architecture](7-DEVELOPMENT/architecture.md)\n- Contributing? → [7-DEVELOPMENT - Contributing](7-DEVELOPMENT/contributing.md)\n- API reference? → [7-DEVELOPMENT - API Reference](7-DEVELOPMENT/api-reference.md)\n\n---\n\n## 📚 Reading Paths\n\n### Path 1: Complete Beginner (1-2 hours)\n1. [0-START-HERE/index.md](0-START-HERE/index.md) — Understand what it is\n2. [0-START-HERE Quick Start](0-START-HERE/index.md) — Set it up\n3. [2-CORE-CONCEPTS/index.md](2-CORE-CONCEPTS/index.md) — Understand concepts\n4. [3-USER-GUIDE/index.md](3-USER-GUIDE/index.md) — Learn features\n\n**Result:** Fully understand how to use Open Notebook\n\n### Path 2: Get Running Fast (15 minutes)\n1. [0-START-HERE](0-START-HERE/index.md) — Pick your path\n2. Follow quick-start guide for your setup\n3. Start using!\n\n**Result:** Running in 15 minutes, learn details later\n\n### Path 3: DevOps/Deployment (1-2 hours)\n1. [1-INSTALLATION](1-INSTALLATION/index.md) — Understand routes\n2. [5-CONFIGURATION](5-CONFIGURATION/index.md) — Reference setup\n3. [7-DEVELOPMENT - Architecture](../7-DEVELOPMENT/architecture.md) — Understand system\n\n**Result:** Ready to deploy to production\n\n### Path 4: Troubleshooting (5-30 minutes)\n1. [6-TROUBLESHOOTING/index.md](6-TROUBLESHOOTING/index.md) — Identify problem\n2. Find specific guide\n3. Follow solutions\n\n**Result:** Problem solved!\n\n---\n\n## ❓ Common Questions\n\n**Q: Where do I start?**\nA: → [0-START-HERE](0-START-HERE/index.md) — Choose your setup path\n\n**Q: How do I install it?**\nA: → [1-INSTALLATION](1-INSTALLATION/index.md) — Multiple routes available\n\n**Q: How do I use [feature]?**\nA: → [3-USER-GUIDE](3-USER-GUIDE/index.md) — Step-by-step tutorials\n\n**Q: Why does [feature] work like that?**\nA: → [2-CORE-CONCEPTS](2-CORE-CONCEPTS/index.md) — Understand the mental model\n\n**Q: How do I configure [provider]?**\nA: → [4-AI-PROVIDERS](4-AI-PROVIDERS/index.md) or [5-CONFIGURATION](5-CONFIGURATION/index.md)\n\n**Q: Something's broken, what do I do?**\nA: → [6-TROUBLESHOOTING](6-TROUBLESHOOTING/index.md) — Problem solver\n\n**Q: How does the system work?**\nA: → [2-CORE-CONCEPTS](2-CORE-CONCEPTS/index.md) — Architecture and concepts\n\n**Q: Can I contribute?**\nA: → [7-DEVELOPMENT](../7-DEVELOPMENT/index.md) — Contributing guide\n\n---\n\n## 📖 How This Documentation is Organized\n\n### Principles\n- **Progressive Disclosure**: Start simple, go deeper if needed\n- **Multiple Entry Routes**: Different paths for different users\n- **High Signal-to-Noise**: Focused content, no fluff\n- **Step-by-Step**: Clear instructions you can follow\n- **Decision Trees**: Help you pick the right path\n- **Symptom-Based**: Troubleshooting by what's broken\n\n### Structure\n- **0-START-HERE** — Entry point (everyone starts here)\n- **1-INSTALLATION** — Multiple setup routes\n- **2-CORE-CONCEPTS** — Mental models (understand why)\n- **3-USER-GUIDE** — How to use (step-by-step)\n- **4-AI-PROVIDERS** — Provider guides\n- **5-CONFIGURATION** — Reference material\n- **6-TROUBLESHOOTING** — Problem solving\n- **7-DEVELOPMENT** — For contributors\n\n---\n\n## 🚀 Quick Navigation\n\n### First Time?\n→ **[START HERE](0-START-HERE/index.md)**\n\n### Just Want to Use It?\n→ **[QUICK START](0-START-HERE/index.md)** (5 minutes)\n\n### Something Broken?\n→ **[TROUBLESHOOTING](6-TROUBLESHOOTING/index.md)**\n\n### Full Reference?\n→ **[CONFIGURATION](5-CONFIGURATION/index.md)**\n\n### Developer?\n→ **[DEVELOPMENT](7-DEVELOPMENT/index.md)**\n\n---\n\n## 📞 Getting Help\n\n- **Discord Community** — https://discord.gg/37XJPXfz2w\n- **GitHub Issues** — https://github.com/lfnovo/open-notebook/issues\n- **Documentation** — You're reading it!\n\n---\n\n## 📈 Documentation Stats\n\n- **8 major sections**\n- **35+ focused guides**\n- **~80,000 words**\n- **Covers all features**\n- **Multiple entry paths**\n- **Progressive difficulty**\n\n---\n\n## 🎯 Start Here\n\n**First time using Open Notebook?**\n→ Go to **[0-START-HERE](0-START-HERE/index.md)**\n\n**Experienced, looking for specific help?**\n→ Use the navigation above to find your section\n\n**Something not working?**\n→ Go to **[TROUBLESHOOTING](6-TROUBLESHOOTING/index.md)**\n\n---\n\nLast updated: January 2026 | Open Notebook v1.2.4+\n" }, { "path": "examples/README.md", "content": "# Docker Compose Examples\n\nThis folder contains different `docker-compose.yml` configurations for various use cases.\n\n## 📋 Available Examples\n\n### `docker-compose-full-local.yml` - 100% Local AI (No Cloud APIs) 🌟\n**Use this if:** You want complete privacy with zero external API dependencies\n\n**Features:**\n- **Ollama**: Local LLM and embeddings (mistral, llama, etc.)\n- **Speaches**: Local TTS (text-to-speech) and STT (speech-to-text)\n- Everything runs on your machine - nothing sent to cloud\n- Perfect for privacy, offline work, or air-gapped environments\n\n**Setup:**\n1. Copy to your project folder as `docker-compose.yml`\n2. Run: `docker compose up -d`\n3. Download models (see file comments for commands)\n4. Configure all providers in UI (detailed instructions in file)\n\n**Requirements:**\n- Minimum: 8GB RAM, 20GB disk, 4 CPU cores\n- Recommended: 16GB+ RAM, NVIDIA GPU (8GB+ VRAM), 50GB disk\n\n**Documentation:**\n- [Local TTS Guide](../docs/5-CONFIGURATION/local-tts.md)\n- [Local STT Guide](../docs/5-CONFIGURATION/local-stt.md)\n\n---\n\n### `docker-compose-speaches.yml` - Local Speech Processing\n**Use this if:** You want free TTS/STT but use cloud LLMs\n\n**Features:**\n- **Speaches**: Local text-to-speech and speech-to-text\n- Use with cloud LLM providers (OpenAI, Anthropic, etc.)\n- Great for podcast generation without TTS API costs\n- Private audio processing\n\n**Setup:**\n1. Copy to your project folder as `docker-compose.yml`\n2. Run: `docker compose up -d`\n3. Download speech models (see file for commands)\n4. Configure cloud LLM + local Speaches in UI\n\n**Documentation:**\n- [Local TTS Guide](../docs/5-CONFIGURATION/local-tts.md)\n- [Local STT Guide](../docs/5-CONFIGURATION/local-stt.md)\n\n---\n\n### `docker-compose-ollama.yml` - Free Local AI with Ollama\n**Use this if:** You want to run AI models locally without API costs\n\n**Features:**\n- Includes Ollama service for local AI models\n- No external API keys needed (for LLM and embeddings)\n- Full privacy - everything runs on your machine\n- Great for testing or privacy-focused deployments\n\n**Setup:**\n1. Copy to your project folder as `docker-compose.yml`\n2. Run: `docker compose up -d`\n3. Pull a model: `docker exec open_notebook-ollama-1 ollama pull mistral`\n4. Configure in UI: Settings → API Keys → Add Ollama (URL: `http://ollama:11434`)\n\n**Recommended models:**\n- **LLM**: `mistral`, `llama3.1`, `qwen2.5`\n- **Embeddings**: `nomic-embed-text`, `mxbai-embed-large`\n\n---\n\n### `docker-compose-single.yml` - Single Container (Deprecated)\n**Use this if:** You need all services in one container (not recommended)\n\n**⚠️ Deprecated:** We recommend using the standard multi-container setup (`docker-compose.yml` in root) for better reliability and easier troubleshooting.\n\n**Features:**\n- Single container includes SurrealDB, API, and Frontend\n- Simpler for very constrained environments\n- Less flexible for debugging and scaling\n\n---\n\n### `docker-compose-dev.yml` - Development Setup\n**Use this if:** You're contributing to Open Notebook or developing custom features\n\n**Features:**\n- Hot-reload for code changes\n- Separate backend and frontend services\n- Build from source instead of using pre-built images\n- Includes development tools and debugging\n\n**Prerequisites:**\n- Python 3.11+\n- Node.js 18+\n- uv (Python package manager)\n\n**Setup:**\nSee [Development Guide](../docs/7-DEVELOPMENT/index.md)\n\n---\n\n## 🔄 How to Use These Examples\n\n1. **Choose** the example that fits your use case\n2. **Copy** the file to your project folder:\n ```bash\n cp examples/docker-compose-ollama.yml docker-compose.yml\n ```\n3. **Edit** the `OPEN_NOTEBOOK_ENCRYPTION_KEY` value\n4. **Run** the services:\n ```bash\n docker compose up -d\n ```\n\n---\n\n## 💡 Need a Custom Setup?\n\nYou can combine features from multiple examples. Common customizations:\n\n### Add Ollama to Standard Setup\nAdd this to the main `docker-compose.yml`:\n\n```yaml\n ollama:\n image: ollama/ollama:latest\n ports:\n - \"11434:11434\"\n volumes:\n - ollama_models:/root/.ollama\n restart: always\n\nvolumes:\n ollama_models:\n```\n\n### Add Reverse Proxy\nSee [Reverse Proxy Guide](../docs/5-CONFIGURATION/reverse-proxy.md)\n\n### Add Basic Auth\nAdd to `open_notebook` service environment:\n```yaml\n- BASIC_AUTH_USERNAME=admin\n- BASIC_AUTH_PASSWORD=your-secure-password\n```\n\n---\n\n## 📚 Documentation\n\n- [Installation Guide](../docs/1-INSTALLATION/index.md)\n- [Configuration Reference](../docs/5-CONFIGURATION/environment-reference.md)\n- [Troubleshooting](../docs/6-TROUBLESHOOTING/index.md)\n\n---\n\n## 🆘 Need Help?\n\n- **Discord**: [Join our community](https://discord.gg/37XJPXfz2w)\n- **Issues**: [GitHub Issues](https://github.com/lfnovo/open-notebook/issues)\n" }, { "path": "examples/docker-compose-dev.yml", "content": "services:\n surrealdb:\n image: surrealdb/surrealdb:v2\n volumes:\n - ./surreal_data:/mydata\n environment:\n - SURREAL_EXPERIMENTAL_GRAPHQL=true\n ports:\n - \"8000:8000\"\n command: start --log info --user root --pass root rocksdb:/mydata/mydatabase.db\n pull_policy: always\n user: root\n restart: always\n open_notebook:\n build:\n context: .\n dockerfile: Dockerfile\n ports:\n - \"8502:8502\"\n - \"5055:5055\"\n env_file:\n - ./docker.env\n depends_on:\n - surrealdb\n volumes:\n - ./notebook_data:/app/data\n restart: always\n \n" }, { "path": "examples/docker-compose-full-local.yml", "content": "# Docker Compose - 100% Local AI Setup\n#\n# This is the complete privacy-focused setup with NO external APIs needed:\n# - Ollama: Local LLM and embeddings (mistral, llama, nomic-embed, etc.)\n# - Speaches: Local TTS (text-to-speech) and STT (speech-to-text)\n# - Open Notebook: Your research assistant\n# - SurrealDB: Local database\n#\n# Perfect for:\n# - Complete privacy (nothing leaves your machine)\n# - Offline work\n# - No API costs\n# - Air-gapped environments\n# - Testing and development\n#\n# Usage:\n# 1. Copy this file to your project folder as docker-compose.yml\n# 2. Change OPEN_NOTEBOOK_ENCRYPTION_KEY below\n# 3. Run: docker compose up -d\n# 4. Pull models (see instructions below)\n# 5. Configure providers in UI\n#\n# Full documentation:\n# - Ollama setup: https://github.com/lfnovo/open-notebook/blob/main/examples/README.md\n# - TTS setup: https://github.com/lfnovo/open-notebook/blob/main/docs/5-CONFIGURATION/local-tts.md\n# - STT setup: https://github.com/lfnovo/open-notebook/blob/main/docs/5-CONFIGURATION/local-stt.md\n\nservices:\n surrealdb:\n image: surrealdb/surrealdb:v2\n command: start --log info --user root --pass root rocksdb:/mydata/mydatabase.db\n user: root\n ports:\n - \"8000:8000\"\n volumes:\n - ./surreal_data:/mydata\n environment:\n - SURREAL_EXPERIMENTAL_GRAPHQL=true\n restart: always\n pull_policy: always\n\n ollama:\n image: ollama/ollama:latest\n ports:\n - \"11434:11434\"\n volumes:\n - ollama_models:/root/.ollama\n restart: always\n pull_policy: always\n # For GPU acceleration (NVIDIA), add:\n # deploy:\n # resources:\n # reservations:\n # devices:\n # - driver: nvidia\n # count: 1\n # capabilities: [gpu]\n\n speaches:\n image: ghcr.io/speaches-ai/speaches:latest-cpu\n container_name: speaches\n ports:\n - \"8969:8000\"\n volumes:\n - hf-hub-cache:/home/ubuntu/.cache/huggingface/hub\n restart: unless-stopped\n # For GPU acceleration, use: ghcr.io/speaches-ai/speaches:latest-cuda\n # and add GPU device mapping (see docs)\n\n open_notebook:\n image: lfnovo/open_notebook:v1-latest\n ports:\n - \"8502:8502\"\n - \"5055:5055\"\n environment:\n # REQUIRED: Change this to your own secret string\n - OPEN_NOTEBOOK_ENCRYPTION_KEY=change-me-to-a-secret-string\n\n # Database connection\n - SURREAL_URL=ws://surrealdb:8000/rpc\n - SURREAL_USER=root\n - SURREAL_PASSWORD=root\n - SURREAL_NAMESPACE=open_notebook\n - SURREAL_DATABASE=open_notebook\n\n # Ollama connection (optional, can also configure via UI)\n - OLLAMA_BASE_URL=http://ollama:11434\n volumes:\n - ./notebook_data:/app/data\n depends_on:\n - surrealdb\n - ollama\n - speaches\n restart: always\n pull_policy: always\n\nvolumes:\n ollama_models:\n hf-hub-cache:\n\n# ==========================================\n# AFTER STARTING: Download Models\n# ==========================================\n#\n# Ollama Models (LLM):\n# docker exec open_notebook-ollama-1 ollama pull mistral\n# docker exec open_notebook-ollama-1 ollama pull llama3.1\n# docker exec open_notebook-ollama-1 ollama pull qwen2.5\n#\n# Ollama Models (Embeddings):\n# docker exec open_notebook-ollama-1 ollama pull nomic-embed-text\n# docker exec open_notebook-ollama-1 ollama pull mxbai-embed-large\n#\n# Speaches (TTS):\n# docker compose exec speaches uv tool run speaches-cli model download speaches-ai/Kokoro-82M-v1.0-ONNX\n#\n# Speaches (STT):\n# docker compose exec speaches uv tool run speaches-cli model download Systran/faster-whisper-small\n#\n# ==========================================\n# CONFIGURATION IN OPEN NOTEBOOK\n# ==========================================\n#\n# 1. Configure Ollama:\n# - Go to Settings → API Keys\n# - Add Credential → Select \"Ollama\"\n# - Base URL: http://ollama:11434\n# - Save → Test Connection → Discover Models → Register Models\n#\n# 2. Configure Speaches (TTS/STT):\n# - Go to Settings → API Keys\n# - Add Credential → Select \"OpenAI-Compatible\"\n# - Name: \"Local Speaches\"\n# - Base URL for TTS: http://host.docker.internal:8969/v1 (macOS/Windows)\n# or: http://172.17.0.1:8969/v1 (Linux)\n# - Base URL for STT: (same as TTS)\n# - Save → Test Connection\n#\n# 3. Discover Speech Models:\n# - In the Speaches credential you just created, click Discover Models\n# - Select and register the models you need (e.g. TTS and STT)\n# - If models aren't discovered automatically, add them manually:\n# * TTS: speaches-ai/Kokoro-82M-v1.0-ONNX\n# * STT: Systran/faster-whisper-small\n#\n# ==========================================\n# RECOMMENDED MODELS\n# ==========================================\n#\n# For LLM (choose based on your hardware):\n# - Fast: mistral (7B), qwen2.5 (7B)\n# - Balanced: llama3.1 (8B)\n# - Best quality: qwen2.5 (14B+), llama3.1 (70B) - requires powerful GPU\n#\n# For Embeddings:\n# - nomic-embed-text (recommended, 137M params)\n# - mxbai-embed-large (334M params, better quality)\n#\n# For TTS:\n# - speaches-ai/Kokoro-82M-v1.0-ONNX (good quality, fast)\n#\n# For STT (Whisper):\n# - faster-whisper-small (balanced, ~500MB)\n# - faster-whisper-base (faster, less accurate)\n# - faster-whisper-large-v3 (best quality, slower, ~3GB)\n#\n# ==========================================\n# HARDWARE REQUIREMENTS\n# ==========================================\n#\n# Minimum (CPU only):\n# - 8 GB RAM\n# - 20 GB disk space\n# - 4 CPU cores\n#\n# Recommended (with GPU):\n# - 16+ GB RAM\n# - 8+ GB VRAM (NVIDIA GPU)\n# - 50 GB disk space\n# - 8+ CPU cores\n#\n# ==========================================\n# COST COMPARISON\n# ==========================================\n#\n# Local (this setup):\n# - Cost: $0 (after hardware)\n# - Privacy: 100% private\n# - Speed: Depends on hardware\n#\n# Cloud (OpenAI + ElevenLabs):\n# - LLM: ~$0.01-0.10 per 1K tokens\n# - Embeddings: ~$0.0001 per 1K tokens\n# - TTS: ~$0.015 per minute\n# - STT: ~$0.006 per minute\n# - Privacy: Data sent to providers\n# - Speed: Usually faster\n" }, { "path": "examples/docker-compose-ollama.yml", "content": "# Docker Compose with Ollama (Free Local AI)\n#\n# This setup includes Ollama for running local AI models without API costs.\n# Great for privacy-focused deployments or testing without cloud dependencies.\n#\n# Usage:\n# 1. Copy this file to your project folder as docker-compose.yml\n# 2. Change OPEN_NOTEBOOK_ENCRYPTION_KEY below\n# 3. Run: docker compose up -d\n# 4. Pull a model: docker exec open_notebook-ollama-1 ollama pull mistral\n# 5. Configure Ollama in UI: Settings → API Keys → Add Ollama (URL: http://ollama:11434)\n\nservices:\n surrealdb:\n image: surrealdb/surrealdb:v2\n command: start --log info --user root --pass root rocksdb:/mydata/mydatabase.db\n user: root\n ports:\n - \"8000:8000\"\n volumes:\n - ./surreal_data:/mydata\n environment:\n - SURREAL_EXPERIMENTAL_GRAPHQL=true\n restart: always\n pull_policy: always\n\n ollama:\n image: ollama/ollama:latest\n ports:\n - \"11434:11434\"\n volumes:\n - ollama_models:/root/.ollama\n restart: always\n pull_policy: always\n\n open_notebook:\n image: lfnovo/open_notebook:v1-latest\n ports:\n - \"8502:8502\"\n - \"5055:5055\"\n environment:\n # REQUIRED: Change this to your own secret string\n - OPEN_NOTEBOOK_ENCRYPTION_KEY=change-me-to-a-secret-string\n\n # Database connection\n - SURREAL_URL=ws://surrealdb:8000/rpc\n - SURREAL_USER=root\n - SURREAL_PASSWORD=root\n - SURREAL_NAMESPACE=open_notebook\n - SURREAL_DATABASE=open_notebook\n\n # Ollama connection\n - OLLAMA_BASE_URL=http://ollama:11434\n volumes:\n - ./notebook_data:/app/data\n depends_on:\n - surrealdb\n - ollama\n restart: always\n pull_policy: always\n\nvolumes:\n ollama_models:\n" }, { "path": "examples/docker-compose-single.yml", "content": "services:\n open_notebook_single:\n # image: lfnovo/open_notebook:v1-latest-single\n build:\n context: .\n dockerfile: Dockerfile.single\n ports:\n - \"8502:8502\" # Next.js Frontend\n - \"5055:5055\" # REST API\n env_file:\n - ./docker.env\n environment:\n # Override for single-container mode: SurrealDB runs on localhost inside the same container\n - SURREAL_URL=ws://localhost:8000/rpc\n volumes:\n - ./notebook_data:/app/data # Application data\n - ./surreal_single_data:/mydata # SurrealDB data\n restart: always\n # Single container includes all services: SurrealDB, API, Worker, and Next.js Frontend\n # Access:\n # - Next.js UI: http://localhost:8502\n # - REST API: http://localhost:5055\n # - API Documentation: http://localhost:5055/docs" }, { "path": "examples/docker-compose-speaches.yml", "content": "# Docker Compose with Speaches (Local TTS/STT)\n#\n# This setup includes Speaches for free, private speech processing:\n# - Text-to-Speech (TTS): Generate podcast audio locally\n# - Speech-to-Text (STT): Transcribe audio/video content locally\n#\n# Why Speaches?\n# - Free: No per-minute/per-character costs\n# - Private: Audio never leaves your machine\n# - Offline: Works without internet\n# - OpenAI-compatible: Drop-in replacement for OpenAI TTS/STT\n#\n# Usage:\n# 1. Copy this file to your project folder as docker-compose.yml\n# 2. Change OPEN_NOTEBOOK_ENCRYPTION_KEY below\n# 3. Run: docker compose up -d\n# 4. Download models (see instructions below)\n# 5. Configure in UI: Settings → API Keys → Add OpenAI-Compatible\n#\n# Full documentation:\n# - TTS setup: https://github.com/lfnovo/open-notebook/blob/main/docs/5-CONFIGURATION/local-tts.md\n# - STT setup: https://github.com/lfnovo/open-notebook/blob/main/docs/5-CONFIGURATION/local-stt.md\n\nservices:\n surrealdb:\n image: surrealdb/surrealdb:v2\n command: start --log info --user root --pass root rocksdb:/mydata/mydatabase.db\n user: root\n ports:\n - \"8000:8000\"\n volumes:\n - ./surreal_data:/mydata\n environment:\n - SURREAL_EXPERIMENTAL_GRAPHQL=true\n restart: always\n pull_policy: always\n\n speaches:\n image: ghcr.io/speaches-ai/speaches:latest-cpu\n container_name: speaches\n ports:\n - \"8969:8000\"\n volumes:\n - hf-hub-cache:/home/ubuntu/.cache/huggingface/hub\n restart: unless-stopped\n # For GPU acceleration, use: ghcr.io/speaches-ai/speaches:latest-cuda\n # and add GPU device mapping (see docs/5-CONFIGURATION/local-tts.md)\n\n open_notebook:\n image: lfnovo/open_notebook:v1-latest\n ports:\n - \"8502:8502\"\n - \"5055:5055\"\n environment:\n # REQUIRED: Change this to your own secret string\n - OPEN_NOTEBOOK_ENCRYPTION_KEY=change-me-to-a-secret-string\n\n # Database connection\n - SURREAL_URL=ws://surrealdb:8000/rpc\n - SURREAL_USER=root\n - SURREAL_PASSWORD=root\n - SURREAL_NAMESPACE=open_notebook\n - SURREAL_DATABASE=open_notebook\n volumes:\n - ./notebook_data:/app/data\n depends_on:\n - surrealdb\n - speaches\n restart: always\n pull_policy: always\n\nvolumes:\n hf-hub-cache:\n\n# ==========================================\n# AFTER STARTING: Download Speech Models\n# ==========================================\n#\n# For TTS (Text-to-Speech):\n# docker compose exec speaches uv tool run speaches-cli model download speaches-ai/Kokoro-82M-v1.0-ONNX\n#\n# For STT (Speech-to-Text):\n# docker compose exec speaches uv tool run speaches-cli model download Systran/faster-whisper-small\n#\n# ==========================================\n# CONFIGURATION IN OPEN NOTEBOOK\n# ==========================================\n#\n# 1. Go to Settings → API Keys\n# 2. Click \"Add Credential\" → Select \"OpenAI-Compatible\"\n# 3. Configure:\n# - Name: \"Local Speaches\"\n# - Base URL for TTS: http://host.docker.internal:8969/v1 (macOS/Windows)\n# or: http://172.17.0.1:8969/v1 (Linux)\n# - Base URL for STT: (same as TTS)\n# 4. Click Save → Test Connection\n#\n# 5. Go to Settings → Models\n# 6. Add TTS Model:\n# - Provider: openai_compatible\n# - Model Name: speaches-ai/Kokoro-82M-v1.0-ONNX\n# - Display Name: Local TTS\n#\n# 7. Add STT Model:\n# - Provider: openai_compatible\n# - Model Name: Systran/faster-whisper-small\n# - Display Name: Local Whisper\n#\n# ==========================================\n# TESTING\n# ==========================================\n#\n# Test TTS:\n# curl \"http://localhost:8969/v1/audio/speech\" -s \\\n# -H \"Content-Type: application/json\" \\\n# --output test.mp3 \\\n# --data '{\"input\": \"Hello, local TTS works!\", \"model\": \"speaches-ai/Kokoro-82M-v1.0-ONNX\", \"voice\": \"af_bella\"}'\n#\n# Test STT:\n# curl \"http://localhost:8969/v1/audio/transcriptions\" \\\n# -F \"file=@test.mp3\" \\\n# -F \"model=Systran/faster-whisper-small\"\n#\n# Available voices: af_bella, af_sarah, am_adam, am_michael, bf_emma, bm_george\n# Available models: See docs/5-CONFIGURATION/local-stt.md for model sizes\n" }, { "path": "frontend/.gitignore", "content": "# See https://help.github.com/articles/ignoring-files/ for more about ignoring files.\n\n# dependencies\n/node_modules\n/.pnp\n.pnp.*\n.yarn/*\n!.yarn/patches\n!.yarn/plugins\n!.yarn/releases\n!.yarn/versions\n\n# testing\n/coverage\n\n# next.js\n/.next/\n/out/\n\n# production\n/build\n\n# misc\n.DS_Store\n*.pem\n\n# debug\nnpm-debug.log*\nyarn-debug.log*\nyarn-error.log*\n.pnpm-debug.log*\n\n# env files (can opt-in for committing if needed)\n.env*\n\n# vercel\n.vercel\n\n# typescript\n*.tsbuildinfo\nnext-env.d.ts\n\ndoc_exports/" }, { "path": "frontend/components.json", "content": "{\n \"$schema\": \"https://ui.shadcn.com/schema.json\",\n \"style\": \"new-york\",\n \"rsc\": true,\n \"tsx\": true,\n \"tailwind\": {\n \"config\": \"\",\n \"css\": \"src/app/globals.css\",\n \"baseColor\": \"neutral\",\n \"cssVariables\": true,\n \"prefix\": \"\"\n },\n \"aliases\": {\n \"components\": \"@/components\",\n \"utils\": \"@/lib/utils\",\n \"ui\": \"@/components/ui\",\n \"lib\": \"@/lib\",\n \"hooks\": \"@/hooks\"\n },\n \"iconLibrary\": \"lucide\"\n}" }, { "path": "frontend/eslint.config.mjs", "content": "import { dirname } from \"path\";\nimport { fileURLToPath } from \"url\";\nimport { FlatCompat } from \"@eslint/eslintrc\";\n\nconst __filename = fileURLToPath(import.meta.url);\nconst __dirname = dirname(__filename);\n\nconst compat = new FlatCompat({\n baseDirectory: __dirname,\n});\n\nconst eslintConfig = [\n ...compat.extends(\"next/core-web-vitals\", \"next/typescript\"),\n];\n\nexport default eslintConfig;\n" }, { "path": "frontend/next.config.ts", "content": "import type { NextConfig } from \"next\";\n\nconst nextConfig: NextConfig = {\n // Enable standalone output for optimized Docker deployment\n output: \"standalone\",\n\n // Experimental features\n // Type assertion needed: proxyClientMaxBodySize is valid in Next.js 15 but types lag behind\n experimental: {\n // Increase proxy body size limit for file uploads (default is 10MB)\n // This allows larger files to be uploaded through the /api/* rewrite proxy to FastAPI\n proxyClientMaxBodySize: '100mb',\n } as NextConfig['experimental'],\n\n // API Rewrites: Proxy /api/* requests to FastAPI backend\n // This simplifies reverse proxy configuration - users only need to proxy to port 8502\n // Next.js handles internal routing to the API backend on port 5055\n async rewrites() {\n // INTERNAL_API_URL: Where Next.js server-side should proxy API requests\n // Default: http://localhost:5055 (single-container deployment)\n // Override for multi-container: INTERNAL_API_URL=http://api-service:5055\n const internalApiUrl = process.env.INTERNAL_API_URL || 'http://localhost:5055'\n\n console.log(`[Next.js Rewrites] Proxying /api/* to ${internalApiUrl}/api/*`)\n\n return [\n {\n source: '/api/:path*',\n destination: `${internalApiUrl}/api/:path*`,\n },\n ]\n },\n};\n\nexport default nextConfig;\n" }, { "path": "frontend/package.json", "content": "{\n \"name\": \"frontend\",\n \"version\": \"0.1.0\",\n \"private\": true,\n \"scripts\": {\n \"dev\": \"next dev\",\n \"build\": \"next build\",\n \"start\": \"node start-server.js\",\n \"lint\": \"eslint src/\",\n \"test\": \"vitest run\",\n \"test:watch\": \"vitest\",\n \"test:ui\": \"vitest --ui\"\n },\n \"dependencies\": {\n \"@hookform/resolvers\": \"^5.1.1\",\n \"@monaco-editor/react\": \"^4.7.0\",\n \"@radix-ui/react-accordion\": \"^1.2.12\",\n \"@radix-ui/react-alert-dialog\": \"^1.1.14\",\n \"@radix-ui/react-checkbox\": \"^1.3.2\",\n \"@radix-ui/react-collapsible\": \"^1.1.11\",\n \"@radix-ui/react-dialog\": \"^1.1.15\",\n \"@radix-ui/react-dropdown-menu\": \"^2.1.15\",\n \"@radix-ui/react-label\": \"^2.1.7\",\n \"@radix-ui/react-popover\": \"^1.1.15\",\n \"@radix-ui/react-progress\": \"^1.1.7\",\n \"@radix-ui/react-radio-group\": \"^1.3.8\",\n \"@radix-ui/react-scroll-area\": \"^1.2.9\",\n \"@radix-ui/react-select\": \"^2.2.5\",\n \"@radix-ui/react-separator\": \"^1.1.7\",\n \"@radix-ui/react-slot\": \"^1.2.3\",\n \"@radix-ui/react-tabs\": \"^1.1.12\",\n \"@radix-ui/react-tooltip\": \"^1.2.7\",\n \"@tailwindcss/typography\": \"^0.5.16\",\n \"@tanstack/react-query\": \"^5.83.0\",\n \"@uiw/react-md-editor\": \"^4.0.8\",\n \"axios\": \"^1.13.5\",\n \"class-variance-authority\": \"^0.7.1\",\n \"clsx\": \"^2.1.1\",\n \"cmdk\": \"^1.1.1\",\n \"date-fns\": \"^4.1.0\",\n \"i18next\": \"^25.7.3\",\n \"i18next-browser-languagedetector\": \"^8.2.0\",\n \"lucide-react\": \"^0.525.0\",\n \"next\": \"^16.1.5\",\n \"next-themes\": \"^0.4.6\",\n \"react\": \"^19.2.3\",\n \"react-dom\": \"^19.2.3\",\n \"react-hook-form\": \"^7.60.0\",\n \"react-i18next\": \"^16.5.0\",\n \"react-markdown\": \"^10.1.0\",\n \"remark-gfm\": \"^4.0.1\",\n \"sonner\": \"^2.0.6\",\n \"tailwind-merge\": \"^3.3.1\",\n \"use-debounce\": \"^10.0.6\",\n \"zod\": \"^4.0.5\",\n \"zustand\": \"^5.0.6\"\n },\n \"devDependencies\": {\n \"@eslint/eslintrc\": \"^3\",\n \"@tailwindcss/postcss\": \"^4\",\n \"@types/node\": \"^20\",\n \"@types/react\": \"^19\",\n \"@types/react-dom\": \"^19\",\n \"eslint\": \"^9\",\n \"eslint-config-next\": \"15.4.2\",\n \"jsdom\": \"^26.0.0\",\n \"tailwindcss\": \"^4\",\n \"tw-animate-css\": \"^1.3.5\",\n \"typescript\": \"^5\",\n \"vitest\": \"^3.0.0\",\n \"@vitest/ui\": \"^3.0.0\",\n \"@vitejs/plugin-react\": \"^4.3.4\",\n \"@testing-library/react\": \"^16.2.0\",\n \"@testing-library/jest-dom\": \"^6.6.3\"\n }\n}\n" }, { "path": "frontend/postcss.config.mjs", "content": "const config = {\n plugins: [\"@tailwindcss/postcss\"],\n};\n\nexport default config;\n" }, { "path": "frontend/src/CLAUDE.md", "content": "# Frontend Architecture\n\nNext.js React application providing UI for Open Notebook research assistant. Three-layer architecture: **pages** (Next.js App Router), **components** (feature-specific UI), and **lib** (data fetching, state management, utilities).\n\n## High-Level Data Flow\n\n```\nPages (Next.js) → Components (feature-specific) → Hooks (queries/mutations)\n ↓\n Stores (auth/modal state) → API module → Backend\n```\n\nUser interactions trigger mutations/queries via hooks, which communicate with the backend through the API module. Store state (auth, modals) flows back to components via hooks. Child CLAUDE.md files document specific modules in detail:\n\n- **`lib/api/CLAUDE.md`**: Axios client, FormData handling, interceptors\n- **`lib/hooks/CLAUDE.md`**: TanStack Query wrappers, SSE streaming, context building\n- **`lib/stores/CLAUDE.md`**: Zustand auth/modal state, localStorage persistence\n- **`lib/locales/CLAUDE.md`**: Internationalization (i18n) system, translation files\n- **`components/ui/CLAUDE.md`**: Radix UI primitives, CVA styling, accessibility\n\n## Architectural Layers\n\n### Pages (`src/app/`) — Next.js App Router\n- `(auth)/login`: Authentication entry point\n- `(dashboard)/`: Protected routes (notebooks, sources, search, models, etc.)\n- Directory-based routing; each `page.tsx` is a route endpoint\n- **Key pattern**: Pages call hooks to fetch data, render components with state\n- **Router groups** `(auth)`, `(dashboard)` organize routes by feature without affecting URL\n\n### Components (`src/components/`) — Feature-Specific UI\n- **layout**: `AppShell.tsx`, `AppSidebar.tsx` — main layout wrapper used by all pages\n- **providers**: `ThemeProvider`, `QueryProvider`, `ModalProvider` — app-wide context setup\n- **auth**: `LoginForm.tsx` — authentication UI\n- **common**: `CommandPalette`, `ErrorBoundary`, `ContextToggle`, `ModelSelector` — shared across pages\n- **ui**: Reusable Radix UI building blocks (see child CLAUDE.md)\n- **source**, **notebooks**, **search**, **podcasts**: Feature-specific components consuming hooks\n\n**Component composition pattern**: Pages → Feature components → UI components. Feature components handle page-level state (loading, error), UI components remain stateless and styled.\n\n### Lib (`src/lib/`) — Data & State Layer\n\n#### `lib/api/` — Backend Communication\n- **`client.ts`**: Central Axios instance with auth interceptor, FormData handling, 10-min timeout\n- **`query-client.ts`**: TanStack Query configuration\n- **Resource modules** (`sources.ts`, `chat.ts`, `notebooks.ts`, etc.): Endpoint-specific functions returning typed responses\n- **Pattern**: All requests go through `apiClient`; auth token auto-added from localStorage\n\n#### `lib/hooks/` — React Query + Custom Logic\n- **Query hooks**: `useNotebookSources`, `useSources`, `useSource` — TanStack Query wrappers with cache keys\n- **Mutation hooks**: `useCreateSource`, `useUpdateSource`, `useDeleteSource` — mutations with toast feedback + cache invalidation\n- **Complex hooks**: `useNotebookChat`, `useSourceChat` — session management, message streaming, context building\n- **SSE streaming**: `useAsk` — parses newline-delimited JSON from backend for multi-stage workflows\n- **Pattern**: Hooks return `{ data, isLoading, error, refetch }` + action functions; cache invalidation on mutations\n\n#### `lib/stores/` — Application State\n- **`auth-store.ts`**: Authentication state (token, isAuthenticated) with 30-second check caching\n- **Zustand + persist middleware**: Auto-syncs sensitive state to localStorage\n- **Pattern**: Store actions (`login()`, `logout()`, `checkAuth()`) update state; consumed via hooks in components\n\n#### `lib/types/` — TypeScript Definitions\n- API request/response shapes, domain models (Notebook, Source, Note, etc.)\n- Ensures type safety across API calls and store mutations\n\n#### `lib/locales/` — Internationalization (i18n)\n- **Locale files** (`en-US/`, `pt-BR/`, `zh-CN/`, `zh-TW/`, `ja-JP/`): Translation strings organized by feature\n- **`i18n.ts`**: i18next configuration with language detection\n- **`use-translation.ts`**: Custom hook with Proxy-based `t.section.key` access pattern\n- **Pattern**: Components call `useTranslation()` hook; access strings via `t.common.save`, `t.notebooks.title`\n\n## Data & Control Flow Walkthrough\n\n### Example: Notebook Chat\n1. **Page** (`notebooks/[id]/page.tsx`) fetches initial data, passes `notebookId` to `ChatColumn` component\n2. **Hook call** (`useNotebookChat()`):\n - Queries sessions for notebook via TanStack Query\n - Sets up message state + context building logic\n - Returns `{ messages, sendMessage(), setModelOverride() }`\n3. **Component renders**: `ChatColumn` displays messages, text input\n4. **User sends message**: Component calls `sendMessage()` hook\n5. **Hook execution**:\n - Builds context from selected sources/notes via `buildContext()` helper\n - Calls `chatApi.sendMessage()` (from API module)\n - Client-side optimistic update: adds message to local state before response\n6. **Backend response** arrives, TanStack Query updates cache\n7. **Cache invalidation** on other source/note mutations ensures stale UI refreshes\n\n### Example: File Upload with Source Creation\n1. **Component** (`SourceDialog`) renders form with file picker\n2. **Hook** (`useFileUpload`):\n - Converts file to FormData (JSON fields stringified)\n - Calls `sourcesApi.create()` with FormData\n - API client interceptor deletes Content-Type header (lets browser set multipart boundary)\n3. **Toast notifications** show progress\n4. **Cache invalidation** on success: `queryClient.invalidateQueries(['sources'])`\n5. **Related queries** auto-refetch: notebooks, sources list, etc.\n\n## Key Patterns & Cross-Layer Coordination\n\n### Caching & Invalidation\n- **Query keys**: `QUERY_KEYS.notebook(id)`, `QUERY_KEYS.sources(notebookId)` — hierarchical structure\n- **Broad invalidation**: `['sources']` invalidates all source queries; trade-off between accuracy + performance\n- **Auto-refetch**: `refetchOnWindowFocus: true` on frequently-changing data (sources, notebooks)\n\n### Auth & Protected Routes\n- **Proxy** (`src/proxy.ts`): Redirects root `/` to `/notebooks`\n- **Auth store**: Validates token via `/notebooks` API call (actual validation, not JWT decode)\n- **Interceptor**: Adds `Bearer {token}` to all requests; 401 response clears auth and redirects to login\n\n### Modal State Management\n- **Modal hooks**: Components query modal state from stores\n- **Context**: Modals pass data (e.g., notebook ID) to child components\n- **Pattern**: One store per modal type; triggered by button clicks + data passing via hook arguments\n\n### Error Handling\n- **API errors**: All request failures propagate to consuming code; components show toast notifications\n- **Error message resolution** (`lib/utils/error-handler.ts`): `getApiErrorMessage()` tries i18n mapping first via `ERROR_MAP`, then falls back to displaying the backend's descriptive error message directly. This ensures user-friendly error messages from the error classification system are shown as-is.\n- **Toast feedback**: Mutations show success/error toasts (from `sonner` library)\n- **Error boundary**: App-level error boundary catches React render errors; shows fallback UI\n\n### FormData Handling\n- **JSON fields**: Nested objects (arrays, objects) must be JSON stringified before FormData\n- **Content-Type header**: Removed by interceptor for FormData requests (lets browser set boundary)\n- **Example**: `sources` array converted to string via `JSON.stringify()` before appending to FormData\n\n## Component Organization Within Features\n\n- **Feature folders** (`source/`, `notebooks/`, `podcasts/`): Group related components\n- **Composition**: Larger components nest smaller ones; no deep prop drilling (state lifted to hooks)\n- **Dialog patterns**: Features define dialog components for inline actions (edit, create, delete)\n- **Props**: Components accept data + action callbacks from parent or hooks\n\n## Providers & Context Setup\n\n**Root layout** (`app/layout.tsx`) wraps app with (outermost → innermost):\n1. `ErrorBoundary` — React error boundary (catches all render errors)\n2. `ThemeProvider` — next-themes for light/dark mode\n3. `QueryProvider` — TanStack Query client\n4. `I18nProvider` — i18next initialization and language loading overlay\n5. `ConnectionGuard` — checks backend connectivity on startup\n6. `Toaster` — sonner toast notification system (inside ConnectionGuard)\n\n## Important Gotchas & Design Decisions\n\n- **Token storage**: Stored in localStorage under `auth-storage` key (Zustand persist); consumed by API interceptor\n- **Base URL discovery**: API client fetches base URL from runtime config on first request (async; can be slow on startup)\n- **Optimistic updates**: Chat messages added to state before server confirmation; removed on error\n- **Modal lifecycle**: Dialogs not auto-reset; parent must clear form state after submit\n- **Focus management**: Dialog auto-focuses first input; can cause layout shifts if inputs are conditional\n- **Cache invalidation breadth**: Trade-off between precision + simplicity; broad invalidation simpler but may over-fetch\n\n## How to Add a New Feature\n\n1. **Create page**: `app/(dashboard)/feature/page.tsx` — calls hooks, renders components\n2. **Create feature components**: `components/feature/` — compose UI + business logic\n3. **Add hooks** (if data needed): `lib/hooks/useFeature.ts` — TanStack Query wrapper\n4. **Add API module** (if backend call needed): `lib/api/feature.ts` — resource-specific functions\n5. **Add types**: `lib/types/api.ts` — request/response shapes\n6. **Use UI components**: Import from `components/ui/` for consistent styling\n7. **Handle auth**: Middleware redirects unauthenticated users; no special handling needed in component\n\n## Testing\n\n- **Hooks**: Mock API functions, wrap in `QueryClientProvider`, assert query/mutation behavior\n- **Components**: Mock hooks via `vi.fn()`, test rendering + user interactions\n- **API calls**: Mock `axios` interceptors; test request/response shapes\n- **Stores**: Mock store state, test mutations via `act()`, assert state changes\n\nSee child CLAUDE.md files for module-specific testing patterns.\n" }, { "path": "frontend/src/app/(auth)/login/page.tsx", "content": "import { LoginForm } from '@/components/auth/LoginForm'\nimport { ErrorBoundary } from '@/components/common/ErrorBoundary'\n\nexport default function LoginPage() {\n return (\n \n \n \n )\n}" }, { "path": "frontend/src/app/(dashboard)/advanced/components/RebuildEmbeddings.tsx", "content": "'use client'\n\nimport { useState, useEffect, useCallback } from 'react'\nimport { useMutation } from '@tanstack/react-query'\nimport { Card, CardContent, CardDescription, CardHeader, CardTitle } from '@/components/ui/card'\nimport { Button } from '@/components/ui/button'\nimport { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from '@/components/ui/select'\nimport { Checkbox } from '@/components/ui/checkbox'\nimport { Label } from '@/components/ui/label'\nimport { Alert, AlertDescription } from '@/components/ui/alert'\nimport { Progress } from '@/components/ui/progress'\nimport { Loader2, AlertCircle, CheckCircle2, XCircle, Clock } from 'lucide-react'\nimport {\n Accordion,\n AccordionContent,\n AccordionItem,\n AccordionTrigger,\n} from '@/components/ui/accordion'\nimport { embeddingApi } from '@/lib/api/embedding'\nimport type { RebuildEmbeddingsRequest, RebuildStatusResponse } from '@/lib/api/embedding'\nimport { useTranslation } from '@/lib/hooks/use-translation'\n\nexport function RebuildEmbeddings() {\n const { t } = useTranslation()\n const [mode, setMode] = useState<'existing' | 'all'>('existing')\n const [includeSources, setIncludeSources] = useState(true)\n const [includeNotes, setIncludeNotes] = useState(true)\n const [includeInsights, setIncludeInsights] = useState(true)\n const [commandId, setCommandId] = useState(null)\n const [status, setStatus] = useState(null)\n const [pollingInterval, setPollingInterval] = useState(null)\n\n // Rebuild mutation\n const rebuildMutation = useMutation({\n mutationFn: async (request: RebuildEmbeddingsRequest) => {\n return embeddingApi.rebuildEmbeddings(request)\n },\n onSuccess: (data) => {\n setCommandId(data.command_id)\n // Start polling for status\n startPolling(data.command_id)\n }\n })\n\n // Start polling for rebuild status\n const startPolling = (cmdId: string) => {\n if (pollingInterval) {\n clearInterval(pollingInterval)\n }\n\n const interval = setInterval(async () => {\n try {\n const statusData = await embeddingApi.getRebuildStatus(cmdId)\n setStatus(statusData)\n\n // Stop polling if completed or failed\n if (statusData.status === 'completed' || statusData.status === 'failed') {\n stopPolling()\n }\n } catch (error) {\n console.error('Failed to fetch rebuild status:', error)\n }\n }, 5000) // Poll every 5 seconds\n\n setPollingInterval(interval)\n }\n\n // Stop polling\n const stopPolling = useCallback(() => {\n if (pollingInterval) {\n clearInterval(pollingInterval)\n setPollingInterval(null)\n }\n }, [pollingInterval])\n\n // Cleanup on unmount\n useEffect(() => {\n return () => {\n stopPolling()\n }\n }, [stopPolling])\n\n const handleStartRebuild = () => {\n const request: RebuildEmbeddingsRequest = {\n mode,\n include_sources: includeSources,\n include_notes: includeNotes,\n include_insights: includeInsights\n }\n\n rebuildMutation.mutate(request)\n }\n\n const handleReset = () => {\n stopPolling()\n setCommandId(null)\n setStatus(null)\n rebuildMutation.reset()\n }\n\n const isAnyTypeSelected = includeSources || includeNotes || includeInsights\n const isRebuildActive = commandId && status && (status.status === 'queued' || status.status === 'running')\n\n const progressData = status?.progress\n const stats = status?.stats\n\n const totalItems = progressData?.total_items ?? progressData?.total ?? 0\n const processedItems = progressData?.processed_items ?? progressData?.processed ?? 0\n const derivedProgressPercent = progressData?.percentage ?? (totalItems > 0 ? (processedItems / totalItems) * 100 : 0)\n const progressPercent = Number.isFinite(derivedProgressPercent) ? derivedProgressPercent : 0\n\n const sourcesProcessed = stats?.sources_processed ?? stats?.sources ?? 0\n const notesProcessed = stats?.notes_processed ?? stats?.notes ?? 0\n const insightsProcessed = stats?.insights_processed ?? stats?.insights ?? 0\n const failedItems = stats?.failed_items ?? stats?.failed ?? 0\n\n const computedDuration = status?.started_at && status?.completed_at\n ? (new Date(status.completed_at).getTime() - new Date(status.started_at).getTime()) / 1000\n : undefined\n const processingTimeSeconds = stats?.processing_time ?? computedDuration\n\n return (\n \n \n \n {t.advanced.rebuildEmbeddings}\n \n \n {t.advanced.rebuildEmbeddingsDesc}\n \n \n \n {/* Configuration Form */}\n {!isRebuildActive && (\n
\n
\n \n \n

\n {mode === 'existing'\n ? t.advanced.rebuild.existingDesc\n : t.advanced.rebuild.allDesc}\n

\n
\n\n
\n {t.advanced.rebuild.include}\n
\n
\n setIncludeSources(checked === true)}\n />\n \n
\n
\n setIncludeNotes(checked === true)}\n />\n \n
\n
\n setIncludeInsights(checked === true)}\n />\n \n
\n
\n {!isAnyTypeSelected && (\n \n \n \n {t.advanced.rebuild.selectOneError}\n \n \n )}\n
\n\n \n {rebuildMutation.isPending ? (\n <>\n \n {t.advanced.rebuild.starting}\n \n ) : (\n t.advanced.rebuild.startBtn\n )}\n \n\n {rebuildMutation.isError && (\n \n \n \n {t.advanced.rebuild.failed}: {(rebuildMutation.error as Error)?.message || t.common.error}\n \n \n )}\n
\n )}\n\n {/* Status Display */}\n {status && (\n
\n
\n
\n {status.status === 'queued' && }\n {status.status === 'running' && }\n {status.status === 'completed' && }\n {status.status === 'failed' && }\n
\n \n {status.status === 'queued' && t.advanced.rebuild.queued}\n {status.status === 'running' && t.advanced.rebuild.running}\n {status.status === 'completed' && t.advanced.rebuild.completed}\n {status.status === 'failed' && t.advanced.rebuild.failed}\n \n {status.status === 'running' && (\n \n {t.advanced.rebuild.leavePageHint}\n \n )}\n
\n
\n {(status.status === 'completed' || status.status === 'failed') && (\n \n )}\n
\n\n {progressData && (\n
\n
\n {t.common.progress}\n \n {t.advanced.rebuild.itemsProcessed\n .replace('{processed}', processedItems.toString())\n .replace('{total}', totalItems.toString())\n .replace('{percent}', progressPercent.toFixed(1))}\n \n
\n \n {failedItems > 0 && (\n

\n ⚠️ {t.advanced.rebuild.failedItems.replace('{count}', failedItems.toString())}\n

\n )}\n
\n )}\n\n {stats && (\n
\n
\n

{t.navigation.sources}

\n

{sourcesProcessed}

\n
\n
\n

{t.common.notes}

\n

{notesProcessed}

\n
\n
\n

{t.common.insights}

\n

{insightsProcessed}

\n
\n
\n

{t.advanced.rebuild.time}

\n

\n {processingTimeSeconds !== undefined ? `${processingTimeSeconds.toFixed(1)}s` : '—'}\n

\n
\n
\n )}\n\n {status.error_message && (\n \n \n {status.error_message}\n \n )}\n\n {status.started_at && (\n
\n

{t.common.created.replace('{time}', new Date(status.started_at).toLocaleString())}

\n {status.completed_at && (\n

{t.notebooks.updated}: {new Date(status.completed_at).toLocaleString()}

\n )}\n
\n )}\n
\n )}\n\n {/* Help Section */}\n \n \n {t.advanced.rebuild.whenToRebuild}\n \n

{t.advanced.rebuild.whenToRebuildAns}

\n \n \n\n \n {t.advanced.rebuild.howLong}\n \n

{t.advanced.rebuild.howLongAns}

\n \n \n\n \n {t.advanced.rebuild.isSafe}\n \n

{t.advanced.rebuild.isSafeAns}

\n \n \n \n \n \n )\n}\n" }, { "path": "frontend/src/app/(dashboard)/advanced/components/SystemInfo.tsx", "content": "'use client'\n\nimport { useEffect, useState } from 'react'\nimport { Card } from '@/components/ui/card'\nimport { getConfig } from '@/lib/config'\nimport { Badge } from '@/components/ui/badge'\nimport { useTranslation } from '@/lib/hooks/use-translation'\n\nexport function SystemInfo() {\n const { t } = useTranslation()\n const [config, setConfig] = useState<{\n version: string\n latestVersion?: string | null\n hasUpdate?: boolean\n } | null>(null)\n const [isLoading, setIsLoading] = useState(true)\n\n useEffect(() => {\n const loadConfig = async () => {\n try {\n const cfg = await getConfig()\n setConfig(cfg)\n } catch (error) {\n console.error('Failed to load config:', error)\n } finally {\n setIsLoading(false)\n }\n }\n\n loadConfig()\n }, [])\n\n if (isLoading) {\n return (\n \n
\n

{t.advanced.systemInfo}

\n
{t.common.loading}
\n
\n \n )\n }\n\n return (\n \n
\n

{t.advanced.systemInfo}

\n\n
\n {/* Current Version */}\n
\n {t.advanced.currentVersion}\n {config?.version || t.advanced.unknown}\n
\n\n {/* Latest Version */}\n {config?.latestVersion && (\n
\n {t.advanced.latestVersion}\n {config.latestVersion}\n
\n )}\n\n {/* Update Status */}\n
\n {t.advanced.status}\n {config?.hasUpdate ? (\n \n {t.advanced.updateAvailable.replace('{version}', config.latestVersion || '')}\n \n ) : config?.latestVersion ? (\n \n {t.advanced.upToDate}\n \n ) : (\n \n {t.advanced.unknown}\n \n )}\n
\n\n {/* GitHub Repository Link */}\n {config?.hasUpdate && (\n
\n \n {t.advanced.viewOnGithub}\n \n \n \n \n
\n )}\n\n {/* Version Check Failed Message */}\n {!config?.latestVersion && config?.version && (\n
\n {t.advanced.updateCheckFailed}\n
\n )}\n
\n
\n \n )\n}\n" }, { "path": "frontend/src/app/(dashboard)/advanced/page.tsx", "content": "'use client'\n\nimport { AppShell } from '@/components/layout/AppShell'\nimport { RebuildEmbeddings } from './components/RebuildEmbeddings'\nimport { SystemInfo } from './components/SystemInfo'\nimport { useTranslation } from '@/lib/hooks/use-translation'\n\nexport default function AdvancedPage() {\n const { t } = useTranslation()\n return (\n \n
\n
\n
\n
\n

{t.advanced.title}

\n

\n {t.advanced.desc}\n

\n
\n\n \n \n
\n
\n
\n \n )\n}\n" }, { "path": "frontend/src/app/(dashboard)/layout.tsx", "content": "'use client'\n\nimport { useAuth } from '@/lib/hooks/use-auth'\nimport { useVersionCheck } from '@/lib/hooks/use-version-check'\nimport { useRouter } from 'next/navigation'\nimport { useEffect, useState } from 'react'\nimport { LoadingSpinner } from '@/components/common/LoadingSpinner'\nimport { ErrorBoundary } from '@/components/common/ErrorBoundary'\nimport { ModalProvider } from '@/components/providers/ModalProvider'\nimport { CreateDialogsProvider } from '@/lib/hooks/use-create-dialogs'\nimport { CommandPalette } from '@/components/common/CommandPalette'\n\nexport default function DashboardLayout({\n children,\n}: {\n children: React.ReactNode\n}) {\n const { isAuthenticated, isLoading } = useAuth()\n const router = useRouter()\n const [hasCheckedAuth, setHasCheckedAuth] = useState(false)\n\n // Check for version updates once per session\n useVersionCheck()\n\n useEffect(() => {\n // Mark that we've completed the initial auth check\n if (!isLoading) {\n setHasCheckedAuth(true)\n\n // Redirect to login if not authenticated\n if (!isAuthenticated) {\n // Store the current path to redirect back after login\n const currentPath = window.location.pathname + window.location.search\n sessionStorage.setItem('redirectAfterLogin', currentPath)\n router.push('/login')\n }\n }\n }, [isAuthenticated, isLoading, router])\n\n // Show loading spinner during initial auth check or while loading\n if (isLoading || !hasCheckedAuth) {\n return (\n
\n \n
\n )\n }\n\n // Don't render anything if not authenticated (during redirect)\n if (!isAuthenticated) {\n return null\n }\n\n return (\n \n \n {children}\n \n \n \n \n )\n}\n" }, { "path": "frontend/src/app/(dashboard)/notebooks/[id]/page.tsx", "content": "'use client'\n\nimport { useState, useEffect } from 'react'\nimport { useParams } from 'next/navigation'\nimport { AppShell } from '@/components/layout/AppShell'\nimport { NotebookHeader } from '../components/NotebookHeader'\nimport { SourcesColumn } from '../components/SourcesColumn'\nimport { NotesColumn } from '../components/NotesColumn'\nimport { ChatColumn } from '../components/ChatColumn'\nimport { useNotebook } from '@/lib/hooks/use-notebooks'\nimport { useNotebookSources } from '@/lib/hooks/use-sources'\nimport { useNotes } from '@/lib/hooks/use-notes'\nimport { LoadingSpinner } from '@/components/common/LoadingSpinner'\nimport { useNotebookColumnsStore } from '@/lib/stores/notebook-columns-store'\nimport { useIsDesktop } from '@/lib/hooks/use-media-query'\nimport { useTranslation } from '@/lib/hooks/use-translation'\nimport { cn } from '@/lib/utils'\nimport { Tabs, TabsList, TabsTrigger } from '@/components/ui/tabs'\nimport { FileText, StickyNote, MessageSquare } from 'lucide-react'\n\nexport type ContextMode = 'off' | 'insights' | 'full'\n\nexport interface ContextSelections {\n sources: Record\n notes: Record\n}\n\nexport default function NotebookPage() {\n const { t } = useTranslation()\n const params = useParams()\n\n // Ensure the notebook ID is properly decoded from URL\n const notebookId = params?.id ? decodeURIComponent(params.id as string) : ''\n\n const { data: notebook, isLoading: notebookLoading } = useNotebook(notebookId)\n const {\n sources,\n isLoading: sourcesLoading,\n refetch: refetchSources,\n hasNextPage,\n isFetchingNextPage,\n fetchNextPage,\n } = useNotebookSources(notebookId)\n const { data: notes, isLoading: notesLoading } = useNotes(notebookId)\n\n // Get collapse states for dynamic layout\n const { sourcesCollapsed, notesCollapsed } = useNotebookColumnsStore()\n\n // Detect desktop to avoid double-mounting ChatColumn\n const isDesktop = useIsDesktop()\n\n // Mobile tab state (Sources, Notes, or Chat)\n const [mobileActiveTab, setMobileActiveTab] = useState<'sources' | 'notes' | 'chat'>('chat')\n\n // Context selection state\n const [contextSelections, setContextSelections] = useState({\n sources: {},\n notes: {}\n })\n\n // Initialize and update selections when sources load or change\n useEffect(() => {\n if (sources && sources.length > 0) {\n setContextSelections(prev => {\n const newSourceSelections = { ...prev.sources }\n sources.forEach(source => {\n const currentMode = newSourceSelections[source.id]\n const hasInsights = source.insights_count > 0\n\n if (currentMode === undefined) {\n // Initial setup - default based on insights availability\n newSourceSelections[source.id] = hasInsights ? 'insights' : 'full'\n } else if (currentMode === 'full' && hasInsights) {\n // Source gained insights while in 'full' mode - auto-switch to 'insights'\n newSourceSelections[source.id] = 'insights'\n }\n })\n return { ...prev, sources: newSourceSelections }\n })\n }\n }, [sources])\n\n useEffect(() => {\n if (notes && notes.length > 0) {\n setContextSelections(prev => {\n const newNoteSelections = { ...prev.notes }\n notes.forEach(note => {\n // Only set default if not already set\n if (!(note.id in newNoteSelections)) {\n // Notes default to 'full'\n newNoteSelections[note.id] = 'full'\n }\n })\n return { ...prev, notes: newNoteSelections }\n })\n }\n }, [notes])\n\n // Handler to update context selection\n const handleContextModeChange = (itemId: string, mode: ContextMode, type: 'source' | 'note') => {\n setContextSelections(prev => ({\n ...prev,\n [type === 'source' ? 'sources' : 'notes']: {\n ...(type === 'source' ? prev.sources : prev.notes),\n [itemId]: mode\n }\n }))\n }\n\n if (notebookLoading) {\n return (\n
\n \n
\n )\n }\n\n if (!notebook) {\n return (\n \n
\n

{t.notebooks.notFound}

\n

{t.notebooks.notFoundDesc}

\n
\n \n )\n }\n\n return (\n \n
\n
\n \n
\n\n
\n {/* Mobile: Tabbed interface - only render on mobile to avoid double-mounting */}\n {!isDesktop && (\n <>\n
\n setMobileActiveTab(value as 'sources' | 'notes' | 'chat')}>\n \n \n \n {t.navigation.sources}\n \n \n \n {t.common.notes}\n \n \n \n {t.common.chat}\n \n \n \n
\n\n {/* Mobile: Show only active tab */}\n
\n {mobileActiveTab === 'sources' && (\n handleContextModeChange(sourceId, mode, 'source')}\n hasNextPage={hasNextPage}\n isFetchingNextPage={isFetchingNextPage}\n fetchNextPage={fetchNextPage}\n />\n )}\n {mobileActiveTab === 'notes' && (\n handleContextModeChange(noteId, mode, 'note')}\n />\n )}\n {mobileActiveTab === 'chat' && (\n \n )}\n
\n \n )}\n\n {/* Desktop: Collapsible columns layout */}\n
\n {/* Sources Column */}\n
\n handleContextModeChange(sourceId, mode, 'source')}\n hasNextPage={hasNextPage}\n isFetchingNextPage={isFetchingNextPage}\n fetchNextPage={fetchNextPage}\n />\n
\n\n {/* Notes Column */}\n
\n handleContextModeChange(noteId, mode, 'note')}\n />\n
\n\n {/* Chat Column - always expanded, takes remaining space */}\n
\n \n
\n
\n
\n
\n \n )\n}\n" }, { "path": "frontend/src/app/(dashboard)/notebooks/components/ChatColumn.test.tsx", "content": "import { render, screen } from '@testing-library/react'\nimport { describe, it, expect, vi } from 'vitest'\nimport { ChatColumn } from './ChatColumn'\nimport { useNotes } from '@/lib/hooks/use-notes'\nimport { useNotebookChat } from '@/lib/hooks/useNotebookChat'\n\n// Mock the hooks\nvi.mock('@/lib/hooks/use-notes')\nvi.mock('@/lib/hooks/useNotebookChat')\nvi.mock('@/components/source/ChatPanel', () => ({\n ChatPanel: () =>
\n}))\n\n// Type-safe mock factory for useNotes hook\nfunction createNotesMock(overrides: { isLoading?: boolean } = {}) {\n return {\n data: [],\n isLoading: overrides.isLoading ?? false,\n } as unknown as ReturnType\n}\n\n// Type-safe mock factory for useNotebookChat hook\nfunction createChatMock() {\n return {\n messages: [],\n isSending: false,\n tokenCount: 0,\n charCount: 0,\n sessions: [],\n currentSessionId: null,\n } as unknown as ReturnType\n}\n\ndescribe('ChatColumn', () => {\n const baseProps = {\n notebookId: 'test-notebook',\n contextSelections: {\n sources: {},\n notes: {}\n },\n sources: [],\n }\n\n it('shows loading spinner when fetching data', () => {\n vi.mocked(useNotes).mockReturnValue(createNotesMock({ isLoading: true }))\n vi.mocked(useNotebookChat).mockReturnValue(createChatMock())\n\n render()\n\n // Should show loading spinner\n expect(screen.getByTestId('loading-spinner')).toBeInTheDocument()\n })\n\n it('renders chat panel when data is loaded', () => {\n vi.mocked(useNotes).mockReturnValue(createNotesMock({ isLoading: false }))\n vi.mocked(useNotebookChat).mockReturnValue(createChatMock())\n\n render()\n\n // Should show chat panel\n expect(screen.getByTestId('chat-panel')).toBeInTheDocument()\n })\n})\n" }, { "path": "frontend/src/app/(dashboard)/notebooks/components/ChatColumn.tsx", "content": "'use client'\n\nimport { useMemo } from 'react'\nimport { useNotebookChat } from '@/lib/hooks/useNotebookChat'\nimport { useNotes } from '@/lib/hooks/use-notes'\nimport { ChatPanel } from '@/components/source/ChatPanel'\nimport { LoadingSpinner } from '@/components/common/LoadingSpinner'\nimport { Card, CardContent } from '@/components/ui/card'\nimport { AlertCircle } from 'lucide-react'\nimport { ContextSelections } from '../[id]/page'\nimport { useTranslation } from '@/lib/hooks/use-translation'\nimport { SourceListResponse } from '@/lib/types/api'\n\ninterface ChatColumnProps {\n notebookId: string\n contextSelections: ContextSelections\n sources: SourceListResponse[]\n sourcesLoading: boolean\n}\n\nexport function ChatColumn({ notebookId, contextSelections, sources, sourcesLoading }: ChatColumnProps) {\n const { t } = useTranslation()\n\n // Fetch notes for this notebook\n const { data: notes = [], isLoading: notesLoading } = useNotes(notebookId)\n\n // Initialize notebook chat hook\n const chat = useNotebookChat({\n notebookId,\n sources,\n notes,\n contextSelections\n })\n\n // Calculate context stats for indicator\n const contextStats = useMemo(() => {\n let sourcesInsights = 0\n let sourcesFull = 0\n let notesCount = 0\n\n // Count sources by mode\n sources.forEach(source => {\n const mode = contextSelections.sources[source.id]\n if (mode === 'insights') {\n sourcesInsights++\n } else if (mode === 'full') {\n sourcesFull++\n }\n })\n\n // Count notes that are included (not 'off')\n notes.forEach(note => {\n const mode = contextSelections.notes[note.id]\n if (mode === 'full') {\n notesCount++\n }\n })\n\n return {\n sourcesInsights,\n sourcesFull,\n notesCount,\n tokenCount: chat.tokenCount,\n charCount: chat.charCount\n }\n }, [sources, notes, contextSelections, chat.tokenCount, chat.charCount])\n\n // Show loading state while sources/notes are being fetched\n if (sourcesLoading || notesLoading) {\n return (\n \n \n \n \n \n )\n }\n\n // Show error state if data fetch failed (unlikely but good to handle)\n if (!sources && !notes) {\n return (\n \n \n
\n \n

{t.chat.unableToLoadChat}

\n

{t.common.refreshPage || 'Please try refreshing the page'}

\n
\n \n \n )\n }\n\n return (\n chat.sendMessage(message, modelOverride)}\n modelOverride={chat.currentSession?.model_override ?? chat.pendingModelOverride ?? undefined}\n onModelChange={(model) => chat.setModelOverride(model ?? null)}\n sessions={chat.sessions}\n currentSessionId={chat.currentSessionId}\n onCreateSession={(title) => chat.createSession(title)}\n onSelectSession={chat.switchSession}\n onUpdateSession={(sessionId, title) => chat.updateSession(sessionId, { title })}\n onDeleteSession={chat.deleteSession}\n loadingSessions={chat.loadingSessions}\n notebookContextStats={contextStats}\n notebookId={notebookId}\n />\n )\n}\n" }, { "path": "frontend/src/app/(dashboard)/notebooks/components/NoteEditorDialog.tsx", "content": "'use client'\n\nimport { Controller, useForm, useWatch } from 'react-hook-form'\nimport { useEffect, useState } from 'react'\nimport { useQueryClient } from '@tanstack/react-query'\nimport { zodResolver } from '@hookform/resolvers/zod'\nimport { z } from 'zod'\nimport { Dialog, DialogContent, DialogTitle } from '@/components/ui/dialog'\nimport { Button } from '@/components/ui/button'\nimport { useCreateNote, useUpdateNote, useNote } from '@/lib/hooks/use-notes'\nimport { QUERY_KEYS } from '@/lib/api/query-client'\nimport { MarkdownEditor } from '@/components/ui/markdown-editor'\nimport { InlineEdit } from '@/components/common/InlineEdit'\nimport { cn } from \"@/lib/utils\";\nimport { useTranslation } from '@/lib/hooks/use-translation'\n\nconst createNoteSchema = z.object({\n title: z.string().optional(),\n content: z.string().min(1, 'Content is required'),\n})\n\ntype CreateNoteFormData = z.infer\n\ninterface NoteEditorDialogProps {\n open: boolean\n onOpenChange: (open: boolean) => void\n notebookId: string\n note?: { id: string; title: string | null; content: string | null }\n}\n\nexport function NoteEditorDialog({ open, onOpenChange, notebookId, note }: NoteEditorDialogProps) {\n const { t } = useTranslation()\n const createNote = useCreateNote()\n const updateNote = useUpdateNote()\n const queryClient = useQueryClient()\n const isEditing = Boolean(note)\n\n // Ensure note ID has 'note:' prefix for API calls\n const noteIdWithPrefix = note?.id\n ? (note.id.includes(':') ? note.id : `note:${note.id}`)\n : ''\n\n const { data: fetchedNote, isLoading: noteLoading } = useNote(noteIdWithPrefix, { enabled: open && !!note?.id })\n const isSaving = isEditing ? updateNote.isPending : createNote.isPending\n const {\n handleSubmit,\n control,\n formState: { errors },\n reset,\n setValue,\n } = useForm({\n resolver: zodResolver(createNoteSchema),\n defaultValues: {\n title: '',\n content: '',\n },\n })\n const watchTitle = useWatch({ control, name: 'title' })\n const [isEditorFullscreen, setIsEditorFullscreen] = useState(false)\n\n useEffect(() => {\n if (!open) {\n reset({ title: '', content: '' })\n return\n }\n\n const source = fetchedNote ?? note\n const title = source?.title ?? ''\n const content = source?.content ?? ''\n\n reset({ title, content })\n }, [open, note, fetchedNote, reset])\n\n useEffect(() => {\n if (!open) return\n\n const observer = new MutationObserver(() => {\n setIsEditorFullscreen(!!document.querySelector('.w-md-editor-fullscreen'))\n })\n observer.observe(document.body, { subtree: true, attributes: true, attributeFilter: ['class'] })\n return () => observer.disconnect()\n }, [open])\n\n const onSubmit = async (data: CreateNoteFormData) => {\n if (note) {\n await updateNote.mutateAsync({\n id: noteIdWithPrefix,\n data: {\n title: data.title || undefined,\n content: data.content,\n },\n })\n // Only invalidate notebook-specific queries if we have a notebookId\n if (notebookId) {\n queryClient.invalidateQueries({ queryKey: QUERY_KEYS.notes(notebookId) })\n }\n } else {\n // Creating a note requires a notebookId\n if (!notebookId) {\n console.error('Cannot create note without notebook_id')\n return\n }\n await createNote.mutateAsync({\n title: data.title || undefined,\n content: data.content,\n note_type: 'human',\n notebook_id: notebookId,\n })\n }\n reset()\n onOpenChange(false)\n }\n\n const handleClose = () => {\n reset()\n setIsEditorFullscreen(false)\n onOpenChange(false)\n }\n\n return (\n \n \n \n {isEditing ? t.sources.editNote : t.sources.createNote}\n \n
\n {isEditing && noteLoading ? (\n
\n {t.common.loading}\n
\n ) : (\n <>\n
\n setValue('title', value || '')}\n placeholder={t.sources.addTitle}\n emptyText={t.sources.untitledNote}\n className=\"text-xl font-semibold\"\n inputClassName=\"text-xl font-semibold\"\n />\n
\n\n
\n (\n \n )}\n />\n {errors.content && (\n

{errors.content.message}

\n )}\n
\n \n )}\n\n
\n \n \n {isSaving\n ? isEditing ? `${t.common.saving}...` : `${t.common.creating}...`\n : isEditing\n ? t.sources.saveNote\n : t.sources.createNoteBtn}\n \n
\n
\n \n \n )\n}\n" }, { "path": "frontend/src/app/(dashboard)/notebooks/components/NotebookCard.tsx", "content": "'use client'\n\nimport { useRouter } from 'next/navigation'\nimport { NotebookResponse } from '@/lib/types/api'\nimport { Card, CardContent, CardDescription, CardHeader, CardTitle } from '@/components/ui/card'\nimport { Button } from '@/components/ui/button'\nimport { Badge } from '@/components/ui/badge'\nimport { MoreHorizontal, Archive, ArchiveRestore, Trash2, FileText, StickyNote } from 'lucide-react'\nimport { formatDistanceToNow } from 'date-fns'\nimport {\n DropdownMenu,\n DropdownMenuContent,\n DropdownMenuItem,\n DropdownMenuTrigger,\n} from '@/components/ui/dropdown-menu'\nimport { useUpdateNotebook } from '@/lib/hooks/use-notebooks'\nimport { NotebookDeleteDialog } from './NotebookDeleteDialog'\nimport { useState } from 'react'\nimport { useTranslation } from '@/lib/hooks/use-translation'\nimport { getDateLocale } from '@/lib/utils/date-locale'\ninterface NotebookCardProps {\n notebook: NotebookResponse\n}\n\nexport function NotebookCard({ notebook }: NotebookCardProps) {\n const { t, language } = useTranslation()\n const [showDeleteDialog, setShowDeleteDialog] = useState(false)\n const router = useRouter()\n const updateNotebook = useUpdateNotebook()\n\n const handleArchiveToggle = (e: React.MouseEvent) => {\n e.stopPropagation()\n updateNotebook.mutate({\n id: notebook.id,\n data: { archived: !notebook.archived }\n })\n }\n\n const handleCardClick = () => {\n router.push(`/notebooks/${encodeURIComponent(notebook.id)}`)\n }\n\n return (\n <>\n \n \n
\n
\n \n {notebook.name}\n \n {notebook.archived && (\n \n {t.notebooks.archived}\n \n )}\n
\n \n \n \n e.stopPropagation()}\n >\n \n \n \n e.stopPropagation()}>\n \n {notebook.archived ? (\n <>\n \n {t.notebooks.unarchive}\n \n ) : (\n <>\n \n {t.notebooks.archive}\n \n )}\n \n {\n e.stopPropagation()\n setShowDeleteDialog(true)\n }}\n className=\"text-red-600\"\n >\n \n {t.common.delete}\n \n \n \n
\n \n \n \n \n {notebook.description || t.chat.noDescription}\n \n\n
\n {t.common.updated.replace('{time}', formatDistanceToNow(new Date(notebook.updated), { \n addSuffix: true,\n locale: getDateLocale(language)\n }))}\n
\n\n {/* Item counts footer */}\n
\n \n \n {notebook.source_count}\n \n \n \n {notebook.note_count}\n \n
\n \n \n\n \n \n )\n}\n" }, { "path": "frontend/src/app/(dashboard)/notebooks/components/NotebookDeleteDialog.tsx", "content": "'use client'\n\nimport { useState, useEffect } from 'react'\nimport {\n AlertDialog,\n AlertDialogAction,\n AlertDialogCancel,\n AlertDialogContent,\n AlertDialogDescription,\n AlertDialogFooter,\n AlertDialogHeader,\n AlertDialogTitle,\n} from '@/components/ui/alert-dialog'\nimport { RadioGroup, RadioGroupItem } from '@/components/ui/radio-group'\nimport { Label } from '@/components/ui/label'\nimport { useTranslation } from '@/lib/hooks/use-translation'\nimport { LoadingSpinner } from '@/components/common/LoadingSpinner'\nimport { useNotebookDeletePreview, useDeleteNotebook } from '@/lib/hooks/use-notebooks'\nimport { useRouter } from 'next/navigation'\n\ninterface NotebookDeleteDialogProps {\n open: boolean\n onOpenChange: (open: boolean) => void\n notebookId: string\n notebookName: string\n redirectAfterDelete?: boolean\n}\n\nexport function NotebookDeleteDialog({\n open,\n onOpenChange,\n notebookId,\n notebookName,\n redirectAfterDelete = false,\n}: NotebookDeleteDialogProps) {\n const { t } = useTranslation()\n const router = useRouter()\n const [sourceAction, setSourceAction] = useState<'keep' | 'delete'>('keep')\n\n // Reset state when dialog opens\n useEffect(() => {\n if (open) {\n setSourceAction('keep')\n }\n }, [open, notebookId])\n\n // Fetch delete preview when dialog is open\n const { data: preview, isLoading: isLoadingPreview, error: previewError } = useNotebookDeletePreview(\n notebookId,\n open\n )\n\n const deleteNotebook = useDeleteNotebook()\n\n const handleConfirm = async () => {\n await deleteNotebook.mutateAsync({\n id: notebookId,\n deleteExclusiveSources: sourceAction === 'delete',\n })\n onOpenChange(false)\n if (redirectAfterDelete) {\n router.push('/notebooks')\n }\n }\n\n const isDeleting = deleteNotebook.isPending\n\n return (\n \n \n \n {t.notebooks.deleteNotebook}\n \n {t.notebooks.deleteNotebookDesc.replace('{name}', notebookName)}\n \n \n\n
\n {isLoadingPreview ? (\n
\n \n {t.notebooks.deleteNotebookLoading}\n
\n ) : previewError ? (\n
\n {t.common.error}: {previewError.message || 'Failed to load preview'}\n
\n ) : preview ? (\n <>\n {/* Notes section */}\n
\n {preview.note_count > 0 ? (\n

\n {t.notebooks.deleteNotebookNotes.replace(\n '{count}',\n String(preview.note_count)\n )}\n

\n ) : (\n

{t.notebooks.deleteNotebookNoNotes}

\n )}\n
\n\n {/* Shared sources - always above the line */}\n {preview.shared_source_count > 0 && (\n
\n

\n {t.notebooks.deleteNotebookSharedSources.replace(\n '{count}',\n String(preview.shared_source_count)\n )}\n

\n
\n )}\n\n {/* No sources message */}\n {preview.exclusive_source_count === 0 && preview.shared_source_count === 0 && (\n
\n

{t.notebooks.deleteNotebookNoSources}

\n
\n )}\n\n {/* Exclusive sources section - below the line with radio buttons */}\n {preview.exclusive_source_count > 0 && (\n
\n

\n {t.notebooks.deleteNotebookExclusiveSources.replace(\n '{count}',\n String(preview.exclusive_source_count)\n )}\n

\n setSourceAction(value as 'keep' | 'delete')}\n disabled={isDeleting}\n >\n
\n \n \n
\n
\n \n \n
\n \n
\n )}\n \n ) : null}\n
\n\n \n {t.common.cancel}\n \n {isDeleting ? (\n <>\n \n {t.common.deleting}\n \n ) : (\n t.common.delete\n )}\n \n \n \n \n )\n}\n" }, { "path": "frontend/src/app/(dashboard)/notebooks/components/NotebookHeader.tsx", "content": "'use client'\n\nimport { useState } from 'react'\nimport { NotebookResponse } from '@/lib/types/api'\nimport { Button } from '@/components/ui/button'\nimport { Badge } from '@/components/ui/badge'\nimport { Archive, ArchiveRestore, Trash2 } from 'lucide-react'\nimport { useUpdateNotebook } from '@/lib/hooks/use-notebooks'\nimport { NotebookDeleteDialog } from './NotebookDeleteDialog'\nimport { formatDistanceToNow } from 'date-fns'\nimport { getDateLocale } from '@/lib/utils/date-locale'\nimport { InlineEdit } from '@/components/common/InlineEdit'\nimport { useTranslation } from '@/lib/hooks/use-translation'\n\ninterface NotebookHeaderProps {\n notebook: NotebookResponse\n}\n\nexport function NotebookHeader({ notebook }: NotebookHeaderProps) {\n const { t, language } = useTranslation()\n const dfLocale = getDateLocale(language)\n const [showDeleteDialog, setShowDeleteDialog] = useState(false)\n \n const updateNotebook = useUpdateNotebook()\n\n const handleUpdateName = async (name: string) => {\n if (!name || name === notebook.name) return\n \n await updateNotebook.mutateAsync({\n id: notebook.id,\n data: { name }\n })\n }\n\n const handleUpdateDescription = async (description: string) => {\n if (description === notebook.description) return\n \n await updateNotebook.mutateAsync({\n id: notebook.id,\n data: { description: description || undefined }\n })\n }\n\n const handleArchiveToggle = () => {\n updateNotebook.mutate({\n id: notebook.id,\n data: { archived: !notebook.archived }\n })\n }\n\n return (\n <>\n
\n
\n
\n
\n \n {notebook.archived && (\n {t.notebooks.archived}\n )}\n
\n
\n \n {notebook.archived ? (\n <>\n \n {t.notebooks.unarchive}\n \n ) : (\n <>\n \n {t.notebooks.archive}\n \n )}\n \n setShowDeleteDialog(true)}\n className=\"text-red-600 hover:text-red-700\"\n >\n \n {t.common.delete}\n \n
\n
\n \n \n \n
\n {t.common.created.replace('{time}', formatDistanceToNow(new Date(notebook.created), { addSuffix: true, locale: dfLocale }))} • \n {t.common.updated.replace('{time}', formatDistanceToNow(new Date(notebook.updated), { addSuffix: true, locale: dfLocale }))}\n
\n
\n
\n\n \n \n )\n}" }, { "path": "frontend/src/app/(dashboard)/notebooks/components/NotebookList.tsx", "content": "'use client'\n\nimport { NotebookResponse } from '@/lib/types/api'\nimport { NotebookCard } from './NotebookCard'\nimport { LoadingSpinner } from '@/components/common/LoadingSpinner'\nimport { EmptyState } from '@/components/common/EmptyState'\nimport { Book, ChevronDown, ChevronRight, Plus } from 'lucide-react'\nimport { Button } from '@/components/ui/button'\nimport { useState } from 'react'\nimport { useTranslation } from '@/lib/hooks/use-translation'\n\ninterface NotebookListProps {\n notebooks?: NotebookResponse[]\n isLoading: boolean\n title: string\n collapsible?: boolean\n emptyTitle?: string\n emptyDescription?: string\n onAction?: () => void\n actionLabel?: string\n}\n\nexport function NotebookList({ \n notebooks, \n isLoading, \n title, \n collapsible = false,\n emptyTitle,\n emptyDescription,\n onAction,\n actionLabel,\n}: NotebookListProps) {\n const { t } = useTranslation()\n const [isExpanded, setIsExpanded] = useState(!collapsible)\n\n if (isLoading) {\n return (\n
\n \n
\n )\n }\n\n if (!notebooks || notebooks.length === 0) {\n return (\n \n \n {actionLabel}\n \n ) : undefined}\n />\n )\n }\n\n return (\n
\n
\n {collapsible && (\n setIsExpanded(!isExpanded)}\n >\n {isExpanded ? (\n \n ) : (\n \n )}\n \n )}\n

{title}

\n ({notebooks.length})\n
\n\n {isExpanded && (\n
\n {notebooks.map((notebook) => (\n \n ))}\n
\n )}\n
\n )\n}\n" }, { "path": "frontend/src/app/(dashboard)/notebooks/components/NotesColumn.tsx", "content": "'use client'\n\nimport { useState, useMemo } from 'react'\nimport { NoteResponse } from '@/lib/types/api'\nimport { Button } from '@/components/ui/button'\nimport { Card, CardContent, CardHeader, CardTitle } from '@/components/ui/card'\nimport {\n DropdownMenu,\n DropdownMenuContent,\n DropdownMenuItem,\n DropdownMenuTrigger,\n} from '@/components/ui/dropdown-menu'\nimport { Plus, StickyNote, Bot, User, MoreVertical, Trash2 } from 'lucide-react'\nimport { LoadingSpinner } from '@/components/common/LoadingSpinner'\nimport { EmptyState } from '@/components/common/EmptyState'\nimport { Badge } from '@/components/ui/badge'\nimport { NoteEditorDialog } from './NoteEditorDialog'\nimport { getDateLocale } from '@/lib/utils/date-locale'\nimport { formatDistanceToNow } from 'date-fns'\nimport { ContextToggle } from '@/components/common/ContextToggle'\nimport { ContextMode } from '../[id]/page'\nimport { useDeleteNote } from '@/lib/hooks/use-notes'\nimport { ConfirmDialog } from '@/components/common/ConfirmDialog'\nimport { CollapsibleColumn, createCollapseButton } from '@/components/notebooks/CollapsibleColumn'\nimport { useNotebookColumnsStore } from '@/lib/stores/notebook-columns-store'\nimport { useTranslation } from '@/lib/hooks/use-translation'\n\ninterface NotesColumnProps {\n notes?: NoteResponse[]\n isLoading: boolean\n notebookId: string\n contextSelections?: Record\n onContextModeChange?: (noteId: string, mode: ContextMode) => void\n}\n\nexport function NotesColumn({\n notes,\n isLoading,\n notebookId,\n contextSelections,\n onContextModeChange\n}: NotesColumnProps) {\n const { t, language } = useTranslation()\n const [showAddDialog, setShowAddDialog] = useState(false)\n const [editingNote, setEditingNote] = useState(null)\n const [deleteDialogOpen, setDeleteDialogOpen] = useState(false)\n const [noteToDelete, setNoteToDelete] = useState(null)\n\n const deleteNote = useDeleteNote()\n\n // Collapsible column state\n const { notesCollapsed, toggleNotes } = useNotebookColumnsStore()\n const collapseButton = useMemo(\n () => createCollapseButton(toggleNotes, t.common.notes),\n [toggleNotes, t.common.notes]\n )\n\n const handleDeleteClick = (noteId: string) => {\n setNoteToDelete(noteId)\n setDeleteDialogOpen(true)\n }\n\n const handleDeleteConfirm = async () => {\n if (!noteToDelete) return\n\n try {\n await deleteNote.mutateAsync(noteToDelete)\n setDeleteDialogOpen(false)\n setNoteToDelete(null)\n } catch (error) {\n console.error('Failed to delete note:', error)\n }\n }\n\n return (\n <>\n \n \n \n
\n {t.common.notes}\n
\n {\n setEditingNote(null)\n setShowAddDialog(true)\n }}\n >\n \n {t.common.writeNote}\n \n {collapseButton}\n
\n
\n \n\n \n {isLoading ? (\n
\n \n
\n ) : !notes || notes.length === 0 ? (\n \n ) : (\n
\n {notes.map((note) => (\n setEditingNote(note)}\n >\n
\n
\n {note.note_type === 'ai' ? (\n \n ) : (\n \n )}\n \n {note.note_type === 'ai' ? t.common.aiGenerated : t.common.human}\n \n
\n\n
\n \n {formatDistanceToNow(new Date(note.updated), { \n addSuffix: true,\n locale: getDateLocale(language)\n })}\n \n\n {/* Context toggle - only show if handler provided */}\n {onContextModeChange && contextSelections?.[note.id] && (\n
event.stopPropagation()}>\n onContextModeChange(note.id, mode)}\n />\n
\n )}\n\n {/* Ellipsis menu for delete action */}\n \n \n e.stopPropagation()}\n >\n \n \n \n \n {\n e.stopPropagation()\n handleDeleteClick(note.id)\n }}\n className=\"text-red-600 focus:text-red-600\"\n >\n \n {t.notebooks.deleteNote}\n \n \n \n
\n
\n\n {note.title && (\n

{note.title}

\n )}\n\n {note.content && (\n

\n {note.content}\n

\n )}\n
\n ))}\n
\n )}\n \n \n \n\n {\n if (!open) {\n setShowAddDialog(false)\n setEditingNote(null)\n } else {\n setShowAddDialog(true)\n }\n }}\n notebookId={notebookId}\n note={editingNote ?? undefined}\n />\n\n \n \n )\n}\n" }, { "path": "frontend/src/app/(dashboard)/notebooks/components/SourcesColumn.tsx", "content": "'use client'\n\nimport { useState, useMemo, useRef, useCallback, useEffect } from 'react'\nimport { SourceListResponse } from '@/lib/types/api'\nimport { Button } from '@/components/ui/button'\nimport { Card, CardContent, CardHeader, CardTitle } from '@/components/ui/card'\nimport {\n DropdownMenu,\n DropdownMenuContent,\n DropdownMenuItem,\n DropdownMenuTrigger,\n} from '@/components/ui/dropdown-menu'\nimport { Plus, FileText, Link2, ChevronDown, Loader2 } from 'lucide-react'\nimport { LoadingSpinner } from '@/components/common/LoadingSpinner'\nimport { EmptyState } from '@/components/common/EmptyState'\nimport { AddSourceDialog } from '@/components/sources/AddSourceDialog'\nimport { AddExistingSourceDialog } from '@/components/sources/AddExistingSourceDialog'\nimport { SourceCard } from '@/components/sources/SourceCard'\nimport { useDeleteSource, useRetrySource, useRemoveSourceFromNotebook } from '@/lib/hooks/use-sources'\nimport { ConfirmDialog } from '@/components/common/ConfirmDialog'\nimport { useModalManager } from '@/lib/hooks/use-modal-manager'\nimport { ContextMode } from '../[id]/page'\nimport { CollapsibleColumn, createCollapseButton } from '@/components/notebooks/CollapsibleColumn'\nimport { useNotebookColumnsStore } from '@/lib/stores/notebook-columns-store'\nimport { useTranslation } from '@/lib/hooks/use-translation'\n\ninterface SourcesColumnProps {\n sources?: SourceListResponse[]\n isLoading: boolean\n notebookId: string\n notebookName?: string\n onRefresh?: () => void\n contextSelections?: Record\n onContextModeChange?: (sourceId: string, mode: ContextMode) => void\n // Pagination props\n hasNextPage?: boolean\n isFetchingNextPage?: boolean\n fetchNextPage?: () => void\n}\n\nexport function SourcesColumn({\n sources,\n isLoading,\n notebookId,\n onRefresh,\n contextSelections,\n onContextModeChange,\n hasNextPage,\n isFetchingNextPage,\n fetchNextPage,\n}: SourcesColumnProps) {\n const { t } = useTranslation()\n const [dropdownOpen, setDropdownOpen] = useState(false)\n const [addDialogOpen, setAddDialogOpen] = useState(false)\n const [addExistingDialogOpen, setAddExistingDialogOpen] = useState(false)\n const [deleteDialogOpen, setDeleteDialogOpen] = useState(false)\n const [sourceToDelete, setSourceToDelete] = useState(null)\n const [removeDialogOpen, setRemoveDialogOpen] = useState(false)\n const [sourceToRemove, setSourceToRemove] = useState(null)\n\n const { openModal } = useModalManager()\n const deleteSource = useDeleteSource()\n const retrySource = useRetrySource()\n const removeFromNotebook = useRemoveSourceFromNotebook()\n\n // Collapsible column state\n const { sourcesCollapsed, toggleSources } = useNotebookColumnsStore()\n const collapseButton = useMemo(\n () => createCollapseButton(toggleSources, t.navigation.sources),\n [toggleSources, t.navigation.sources]\n )\n\n // Scroll container ref for infinite scroll\n const scrollContainerRef = useRef(null)\n\n // Handle scroll for infinite loading\n const handleScroll = useCallback(() => {\n const container = scrollContainerRef.current\n if (!container || !hasNextPage || isFetchingNextPage || !fetchNextPage) return\n\n const { scrollTop, scrollHeight, clientHeight } = container\n // Load more when user scrolls within 200px of the bottom\n if (scrollHeight - scrollTop - clientHeight < 200) {\n fetchNextPage()\n }\n }, [hasNextPage, isFetchingNextPage, fetchNextPage])\n\n // Attach scroll listener\n useEffect(() => {\n const container = scrollContainerRef.current\n if (!container) return\n\n container.addEventListener('scroll', handleScroll)\n return () => container.removeEventListener('scroll', handleScroll)\n }, [handleScroll])\n \n const handleDeleteClick = (sourceId: string) => {\n setSourceToDelete(sourceId)\n setDeleteDialogOpen(true)\n }\n\n const handleDeleteConfirm = async () => {\n if (!sourceToDelete) return\n\n try {\n await deleteSource.mutateAsync(sourceToDelete)\n setDeleteDialogOpen(false)\n setSourceToDelete(null)\n onRefresh?.()\n } catch (error) {\n console.error('Failed to delete source:', error)\n }\n }\n\n const handleRemoveFromNotebook = (sourceId: string) => {\n setSourceToRemove(sourceId)\n setRemoveDialogOpen(true)\n }\n\n const handleRemoveConfirm = async () => {\n if (!sourceToRemove) return\n\n try {\n await removeFromNotebook.mutateAsync({\n notebookId,\n sourceId: sourceToRemove\n })\n setRemoveDialogOpen(false)\n setSourceToRemove(null)\n } catch (error) {\n console.error('Failed to remove source from notebook:', error)\n // Error toast is handled by the hook\n }\n }\n\n const handleRetry = async (sourceId: string) => {\n try {\n await retrySource.mutateAsync(sourceId)\n } catch (error) {\n console.error('Failed to retry source:', error)\n }\n }\n\n const handleSourceClick = (sourceId: string) => {\n openModal('source', sourceId)\n }\n\n return (\n <>\n \n \n \n
\n {t.navigation.sources}\n
\n \n \n \n \n \n { setDropdownOpen(false); setAddDialogOpen(true); }}>\n \n {t.sources.addSource}\n \n { setDropdownOpen(false); setAddExistingDialogOpen(true); }}>\n \n {t.sources.addExistingTitle}\n \n \n \n {collapseButton}\n
\n
\n \n\n \n {isLoading ? (\n
\n \n
\n ) : !sources || sources.length === 0 ? (\n \n ) : (\n
\n {sources.map((source) => (\n onContextModeChange(source.id, mode)\n : undefined\n }\n />\n ))}\n {/* Loading indicator for infinite scroll */}\n {isFetchingNextPage && (\n
\n \n
\n )}\n
\n )}\n \n \n \n\n \n\n \n\n \n\n \n \n )\n}\n" }, { "path": "frontend/src/app/(dashboard)/notebooks/page.tsx", "content": "'use client'\n\nimport { useMemo, useState } from 'react'\n\nimport { AppShell } from '@/components/layout/AppShell'\nimport { NotebookList } from './components/NotebookList'\nimport { Button } from '@/components/ui/button'\nimport { Plus, RefreshCw } from 'lucide-react'\nimport { useNotebooks } from '@/lib/hooks/use-notebooks'\nimport { CreateNotebookDialog } from '@/components/notebooks/CreateNotebookDialog'\nimport { Input } from '@/components/ui/input'\nimport { useTranslation } from '@/lib/hooks/use-translation'\n\nexport default function NotebooksPage() {\n const { t } = useTranslation()\n const [createDialogOpen, setCreateDialogOpen] = useState(false)\n const [searchTerm, setSearchTerm] = useState('')\n const { data: notebooks, isLoading, refetch } = useNotebooks(false)\n const { data: archivedNotebooks } = useNotebooks(true)\n\n const normalizedQuery = searchTerm.trim().toLowerCase()\n\n const filteredActive = useMemo(() => {\n if (!notebooks) {\n return undefined\n }\n if (!normalizedQuery) {\n return notebooks\n }\n return notebooks.filter((notebook) =>\n notebook.name.toLowerCase().includes(normalizedQuery)\n )\n }, [notebooks, normalizedQuery])\n\n const filteredArchived = useMemo(() => {\n if (!archivedNotebooks) {\n return undefined\n }\n if (!normalizedQuery) {\n return archivedNotebooks\n }\n return archivedNotebooks.filter((notebook) =>\n notebook.name.toLowerCase().includes(normalizedQuery)\n )\n }, [archivedNotebooks, normalizedQuery])\n\n const hasArchived = (archivedNotebooks?.length ?? 0) > 0\n const isSearching = normalizedQuery.length > 0\n\n return (\n \n
\n
\n
\n
\n

{t.notebooks.title}

\n \n
\n
\n setSearchTerm(event.target.value)}\n placeholder={t.notebooks.searchPlaceholder}\n autoComplete=\"off\"\n aria-label={t.common.accessibility?.searchNotebooks || \"Search notebooks\"}\n className=\"w-full sm:w-64\"\n />\n \n
\n
\n \n
\n setCreateDialogOpen(true) : undefined}\n actionLabel={!isSearching ? t.notebooks.newNotebook : undefined}\n />\n \n {hasArchived && (\n \n )}\n
\n
\n
\n\n \n \n )\n}\n" }, { "path": "frontend/src/app/(dashboard)/page.tsx", "content": "import { redirect } from 'next/navigation'\n\nexport default function DashboardPage() {\n redirect('/notebooks')\n}" }, { "path": "frontend/src/app/(dashboard)/podcasts/page.tsx", "content": "'use client'\n\nimport { useMemo, useState } from 'react'\nimport { AlertTriangle } from 'lucide-react'\n\nimport { AppShell } from '@/components/layout/AppShell'\nimport { Tabs, TabsContent, TabsList, TabsTrigger } from '@/components/ui/tabs'\nimport { Alert, AlertDescription, AlertTitle } from '@/components/ui/alert'\nimport { EpisodesTab } from '@/components/podcasts/EpisodesTab'\nimport { TemplatesTab } from '@/components/podcasts/TemplatesTab'\nimport { Mic, LayoutTemplate } from 'lucide-react'\nimport { useTranslation } from '@/lib/hooks/use-translation'\nimport { useEpisodeProfiles, useSpeakerProfiles } from '@/lib/hooks/use-podcasts'\nimport { needsModelSetup } from '@/lib/types/podcasts'\n\nexport default function PodcastsPage() {\n const { t } = useTranslation()\n const [activeTab, setActiveTab] = useState<'episodes' | 'templates'>('episodes')\n\n const { episodeProfiles } = useEpisodeProfiles()\n const { speakerProfiles } = useSpeakerProfiles(episodeProfiles)\n\n const hasUnconfiguredProfiles = useMemo(() => {\n return episodeProfiles.some(needsModelSetup) || speakerProfiles.some(needsModelSetup)\n }, [episodeProfiles, speakerProfiles])\n\n return (\n \n
\n
\n
\n

{t.podcasts.listTitle}

\n

\n {t.podcasts.listDesc}\n

\n
\n\n {hasUnconfiguredProfiles ? (\n \n \n {t.podcasts.setupRequired}\n \n {t.podcasts.setupRequiredDesc}\n \n \n ) : null}\n\n setActiveTab(value as 'episodes' | 'templates')}\n className=\"space-y-6\"\n >\n
\n

{t.podcasts.chooseAView}

\n \n \n \n {t.podcasts.episodesTab}\n \n \n \n {t.podcasts.templatesTab}\n \n \n
\n\n \n \n \n\n \n \n \n \n
\n
\n \n )\n}\n" }, { "path": "frontend/src/app/(dashboard)/search/page.tsx", "content": "'use client'\n\nimport { useCallback, useEffect, useMemo, useRef, useState } from 'react'\nimport { useSearchParams } from 'next/navigation'\nimport { useTranslation } from '@/lib/hooks/use-translation'\nimport { AppShell } from '@/components/layout/AppShell'\nimport { Tabs, TabsContent, TabsList, TabsTrigger } from '@/components/ui/tabs'\nimport { Input } from '@/components/ui/input'\nimport { Textarea } from '@/components/ui/textarea'\nimport { Button } from '@/components/ui/button'\nimport { RadioGroup, RadioGroupItem } from '@/components/ui/radio-group'\nimport { Label } from '@/components/ui/label'\nimport { Checkbox } from '@/components/ui/checkbox'\nimport { Card, CardContent, CardHeader, CardTitle } from '@/components/ui/card'\nimport { Badge } from '@/components/ui/badge'\nimport { Collapsible, CollapsibleContent, CollapsibleTrigger } from '@/components/ui/collapsible'\nimport { Search, ChevronDown, AlertCircle, Settings, Save, MessageCircleQuestion } from 'lucide-react'\nimport { useSearch } from '@/lib/hooks/use-search'\nimport { useAsk } from '@/lib/hooks/use-ask'\nimport { useModelDefaults, useModels } from '@/lib/hooks/use-models'\nimport { useModalManager } from '@/lib/hooks/use-modal-manager'\nimport { LoadingSpinner } from '@/components/common/LoadingSpinner'\nimport { StreamingResponse } from '@/components/search/StreamingResponse'\nimport { AdvancedModelsDialog } from '@/components/search/AdvancedModelsDialog'\nimport { SaveToNotebooksDialog } from '@/components/search/SaveToNotebooksDialog'\n\nexport default function SearchPage() {\n const { t } = useTranslation()\n // URL params\n const searchParams = useSearchParams()\n const urlQuery = searchParams?.get('q') || ''\n const rawMode = searchParams?.get('mode')\n const urlMode = rawMode === 'search' ? 'search' : 'ask'\n\n // Tab state (controlled)\n const [activeTab, setActiveTab] = useState<'ask' | 'search'>(\n urlMode === 'search' ? 'search' : 'ask'\n )\n\n // Search state\n const [searchQuery, setSearchQuery] = useState(urlMode === 'search' ? urlQuery : '')\n const [searchType, setSearchType] = useState<'text' | 'vector'>('text')\n const [searchSources, setSearchSources] = useState(true)\n const [searchNotes, setSearchNotes] = useState(true)\n\n // Ask state\n const [askQuestion, setAskQuestion] = useState(urlMode === 'ask' ? urlQuery : '')\n\n // Advanced models dialog\n const [showAdvancedModels, setShowAdvancedModels] = useState(false)\n const [customModels, setCustomModels] = useState<{\n strategy: string\n answer: string\n finalAnswer: string\n } | null>(null)\n\n // Save to notebooks dialog\n const [showSaveDialog, setShowSaveDialog] = useState(false)\n\n // Hooks\n const searchMutation = useSearch()\n const ask = useAsk()\n const { data: modelDefaults, isLoading: modelsLoading } = useModelDefaults()\n const { data: availableModels } = useModels()\n const { openModal } = useModalManager()\n\n const modelNameById = useMemo(() => {\n if (!availableModels) {\n return new Map()\n }\n return new Map(availableModels.map((model) => [model.id, model.name]))\n }, [availableModels])\n\n const resolveModelName = (id?: string | null) => {\n if (!id) return t.searchPage.notSet\n return modelNameById.get(id) ?? id\n }\n\n const hasEmbeddingModel = !!modelDefaults?.default_embedding_model\n\n // Track if we've already auto-triggered from URL params\n const hasAutoTriggeredRef = useRef(false)\n const lastUrlParamsRef = useRef({ q: '', mode: '' })\n\n const handleSearch = useCallback(() => {\n if (!searchQuery.trim()) return\n\n searchMutation.mutate({\n query: searchQuery,\n type: searchType,\n limit: 100,\n search_sources: searchSources,\n search_notes: searchNotes,\n minimum_score: 0.2\n })\n }, [searchQuery, searchType, searchSources, searchNotes, searchMutation])\n\n const handleKeyPress = (e: React.KeyboardEvent) => {\n if (e.key === 'Enter') {\n handleSearch()\n }\n }\n\n const handleAsk = useCallback(() => {\n if (!askQuestion.trim() || !modelDefaults?.default_chat_model) return\n\n const models = customModels || {\n strategy: modelDefaults.default_chat_model,\n answer: modelDefaults.default_chat_model,\n finalAnswer: modelDefaults.default_chat_model\n }\n\n ask.sendAsk(askQuestion, models)\n }, [askQuestion, modelDefaults, customModels, ask])\n\n // Auto-trigger search/ask when arriving with URL params\n useEffect(() => {\n // Skip if already triggered or no query\n if (hasAutoTriggeredRef.current || !urlQuery) return\n\n // Wait for models to load before triggering ask\n if (urlMode === 'ask' && modelsLoading) return\n\n if (urlMode === 'search') {\n handleSearch()\n hasAutoTriggeredRef.current = true\n } else if (urlMode === 'ask' && modelDefaults?.default_chat_model) {\n handleAsk()\n hasAutoTriggeredRef.current = true\n }\n }, [urlQuery, urlMode, modelsLoading, modelDefaults, handleSearch, handleAsk])\n\n // Handle URL param changes while on page (e.g., from command palette again)\n useEffect(() => {\n const currentQ = searchParams?.get('q') || ''\n const rawCurrentMode = searchParams?.get('mode')\n const currentMode = rawCurrentMode === 'search' ? 'search' : 'ask'\n\n // Check if URL params have changed\n if (currentQ !== lastUrlParamsRef.current.q || currentMode !== lastUrlParamsRef.current.mode) {\n lastUrlParamsRef.current = { q: currentQ, mode: currentMode }\n\n if (currentQ) {\n // Update state based on mode\n if (currentMode === 'search') {\n setSearchQuery(currentQ)\n setActiveTab('search')\n // Reset trigger flag so we auto-trigger with new params\n hasAutoTriggeredRef.current = false\n } else {\n setAskQuestion(currentQ)\n setActiveTab('ask')\n hasAutoTriggeredRef.current = false\n }\n }\n }\n }, [searchParams])\n\n return (\n \n
\n

{t.searchPage.askAndSearch}

\n\n setActiveTab(v as 'ask' | 'search')} className=\"w-full space-y-6\">\n
\n

{t.searchPage.chooseAMode}

\n \n \n \n {t.searchPage.askBeta}\n \n \n \n {t.searchPage.search}\n \n \n
\n\n \n \n \n {t.searchPage.askYourKb}\n

\n {t.searchPage.askYourKbDesc}\n

\n \n \n {/* Question Input */}\n
\n \n setAskQuestion(e.target.value)}\n onKeyDown={(e) => {\n // Submit on Cmd/Ctrl+Enter\n if ((e.metaKey || e.ctrlKey) && e.key === 'Enter' && !ask.isStreaming && askQuestion.trim()) {\n e.preventDefault()\n handleAsk()\n }\n }}\n disabled={ask.isStreaming}\n rows={3}\n aria-label={t.common.accessibility.enterQuestion}\n />\n

{t.searchPage.pressToSubmit}

\n
\n\n {/* Models Display */}\n {!hasEmbeddingModel ? (\n
\n \n {t.searchPage.noEmbeddingModel}\n
\n ) : (\n <>\n
\n
\n \n setShowAdvancedModels(true)}\n disabled={ask.isStreaming}\n className=\"h-auto py-1 px-2\"\n >\n \n {t.searchPage.advanced}\n \n
\n
\n \n {t.searchPage.strategy}: {resolveModelName(customModels?.strategy || modelDefaults?.default_chat_model)}\n \n \n {t.searchPage.answer}: {resolveModelName(customModels?.answer || modelDefaults?.default_chat_model)}\n \n \n {t.searchPage.final}: {resolveModelName(customModels?.finalAnswer || modelDefaults?.default_chat_model)}\n \n
\n
\n\n
\n \n {ask.isStreaming ? (\n <>\n \n {t.searchPage.processing}\n \n ) : (\n t.searchPage.ask\n )}\n \n\n {ask.finalAnswer && (\n setShowSaveDialog(true)}\n className=\"w-full\"\n >\n \n {t.searchPage.saveToNotebooks}\n \n )}\n
\n \n )}\n\n {/* Streaming Response */}\n \n\n {/* Advanced Models Dialog */}\n \n\n {/* Save to Notebooks Dialog */}\n {ask.finalAnswer && (\n \n )}\n \n \n \n\n \n \n \n {t.searchPage.search}\n

\n {t.searchPage.searchDesc}\n

\n \n \n {/* Search Input */}\n
\n \n
\n setSearchQuery(e.target.value)}\n onKeyPress={handleKeyPress}\n disabled={searchMutation.isPending}\n className=\"flex-1\"\n aria-label={t.common.accessibility.enterSearch}\n autoComplete=\"off\"\n />\n \n {searchMutation.isPending ? (\n \n ) : (\n \n )}\n {t.searchPage.search}\n \n
\n

{t.searchPage.pressToSearch}

\n
\n\n {/* Search Options */}\n
\n {/* Search Type */}\n
\n {t.searchPage.searchType}\n {!hasEmbeddingModel && (\n
\n \n {t.searchPage.vectorSearchWarning}\n
\n )}\n setSearchType(value)}\n disabled={modelsLoading || searchMutation.isPending}\n >\n
\n \n \n
\n
\n \n \n {t.searchPage.vectorSearch}\n \n
\n \n
\n\n {/* Search Locations */}\n
\n {t.searchPage.searchIn}\n
\n
\n setSearchSources(checked as boolean)}\n disabled={searchMutation.isPending}\n />\n \n
\n
\n setSearchNotes(checked as boolean)}\n disabled={searchMutation.isPending}\n />\n \n
\n
\n
\n
\n\n {/* Search Results */}\n {searchMutation.data && (\n
\n
\n

\n {t.searchPage.resultsFound.replace('{count}', searchMutation.data.total_count.toString())}\n

\n {searchMutation.data.search_type === 'text' ? t.searchPage.textSearch : t.searchPage.vectorSearch}\n
\n\n {searchMutation.data.results.length === 0 ? (\n \n \n {t.searchPage.noResultsFor.replace('{query}', searchQuery)}\n \n \n ) : (\n
\n {searchMutation.data.results.map((result, index) => {\n // Parse type from parent_id (format: \"source:id\" or \"note:id\" or \"source_insight:id\")\n // Handle null parent_id gracefully (orphaned records)\n if (!result.parent_id) {\n console.warn('Search result with null parent_id:', result)\n return null\n }\n const [type, id] = result.parent_id.split(':')\n const modalType = type === 'source_insight' ? 'insight' : type as 'source' | 'note' | 'insight'\n\n return (\n \n \n
\n
\n openModal(modalType, id)}\n className=\"text-primary hover:underline font-medium\"\n >\n {result.title}\n \n \n {result.final_score.toFixed(2)}\n \n
\n
\n\n {result.matches && result.matches.length > 0 && (\n \n \n \n {t.searchPage.matches.replace('{count}', result.matches.length.toString())}\n \n \n {result.matches.map((match, i) => (\n
\n {match}\n
\n ))}\n \n \n )}\n \n \n )})}\n
\n )}\n
\n )}\n \n \n \n \n
\n \n )\n}\n" }, { "path": "frontend/src/app/(dashboard)/settings/api-keys/page.tsx", "content": "'use client'\n\nimport { useMemo, useState, useEffect, useId } from 'react'\nimport { useForm } from 'react-hook-form'\nimport { AppShell } from '@/components/layout/AppShell'\nimport { LoadingSpinner } from '@/components/common/LoadingSpinner'\nimport { Alert, AlertTitle, AlertDescription } from '@/components/ui/alert'\nimport { Button } from '@/components/ui/button'\nimport { Card, CardContent, CardDescription, CardHeader, CardTitle } from '@/components/ui/card'\nimport { Badge } from '@/components/ui/badge'\nimport { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from '@/components/ui/select'\nimport { Label } from '@/components/ui/label'\nimport {\n Dialog,\n DialogContent,\n DialogDescription,\n DialogFooter,\n DialogHeader,\n DialogTitle,\n} from '@/components/ui/dialog'\nimport {\n RefreshCw,\n Key,\n ShieldAlert,\n Plus,\n Edit,\n Trash2,\n Plug,\n Loader2,\n Check,\n X,\n AlertCircle,\n Wand2,\n MessageSquare,\n Code,\n Mic,\n Volume2,\n Bot,\n} from 'lucide-react'\nimport { useTranslation } from '@/lib/hooks/use-translation'\nimport { useModels, useDeleteModel, useModelDefaults, useUpdateModelDefaults, useAutoAssignDefaults, useTestModel } from '@/lib/hooks/use-models'\nimport {\n useCredentials,\n useCredential,\n useCredentialStatus,\n useEnvStatus,\n useCreateCredential,\n useUpdateCredential,\n useDeleteCredential,\n useTestCredential,\n useDiscoverModels,\n useRegisterModels,\n useMigrateFromEnv,\n} from '@/lib/hooks/use-credentials'\nimport { Credential, CreateCredentialRequest, UpdateCredentialRequest, DiscoveredModel } from '@/lib/api/credentials'\nimport { Model, ModelDefaults } from '@/lib/types/models'\nimport { MigrationBanner, ModelTestResultDialog } from '@/components/settings'\nimport { EmbeddingModelChangeDialog } from '@/components/settings/EmbeddingModelChangeDialog'\n\ntype ModelType = 'language' | 'embedding' | 'text_to_speech' | 'speech_to_text'\n\n// Provider display names\nconst PROVIDER_DISPLAY_NAMES: Record = {\n openai: 'OpenAI',\n anthropic: 'Anthropic',\n google: 'Google AI',\n groq: 'Groq',\n mistral: 'Mistral AI',\n deepseek: 'DeepSeek',\n xai: 'xAI (Grok)',\n openrouter: 'OpenRouter',\n voyage: 'Voyage AI',\n elevenlabs: 'ElevenLabs',\n ollama: 'Ollama',\n azure: 'Azure OpenAI',\n vertex: 'Google Vertex AI',\n openai_compatible: 'OpenAI Compatible',\n}\n\n// All providers in display order\nconst ALL_PROVIDERS = [\n 'openai', 'anthropic', 'google', 'groq', 'mistral', 'deepseek',\n 'xai', 'openrouter', 'voyage', 'elevenlabs', 'ollama',\n 'azure', 'vertex', 'openai_compatible',\n]\n\n// Default modalities per provider\nconst PROVIDER_MODALITIES: Record = {\n openai: ['language', 'embedding', 'text_to_speech', 'speech_to_text'],\n anthropic: ['language'],\n google: ['language', 'embedding', 'text_to_speech', 'speech_to_text'],\n groq: ['language', 'speech_to_text'],\n mistral: ['language', 'embedding'],\n deepseek: ['language'],\n xai: ['language'],\n openrouter: ['language', 'embedding'],\n voyage: ['embedding'],\n elevenlabs: ['text_to_speech', 'speech_to_text'],\n ollama: ['language', 'embedding'],\n azure: ['language', 'embedding', 'text_to_speech', 'speech_to_text'],\n vertex: ['language', 'embedding', 'text_to_speech'],\n openai_compatible: ['language', 'embedding', 'text_to_speech', 'speech_to_text'],\n}\n\n// Documentation links\nconst PROVIDER_DOCS: Record = {\n openai: 'https://platform.openai.com/api-keys',\n anthropic: 'https://console.anthropic.com/settings/keys',\n google: 'https://aistudio.google.com/app/apikey',\n groq: 'https://console.groq.com/keys',\n mistral: 'https://console.mistral.ai/api-keys/',\n deepseek: 'https://platform.deepseek.com/api_keys',\n xai: 'https://console.x.ai/',\n openrouter: 'https://openrouter.ai/keys',\n voyage: 'https://dash.voyageai.com/api-keys',\n elevenlabs: 'https://elevenlabs.io/app/settings/api-keys',\n azure: 'https://portal.azure.com/#view/Microsoft_Azure_ProjectOxford/CognitiveServicesHub/~/OpenAI',\n vertex: 'https://cloud.google.com/vertex-ai/docs/start/cloud-environment',\n openai_compatible: 'https://github.com/lfnovo/open-notebook/blob/main/docs/5-CONFIGURATION/openai-compatible.md',\n}\n\nconst TYPE_ICONS: Record = {\n language: ,\n embedding: ,\n text_to_speech: ,\n speech_to_text: ,\n}\n\nconst TYPE_COLORS: Record = {\n language: 'bg-blue-100 text-blue-700 dark:bg-blue-900/30 dark:text-blue-300',\n embedding: 'bg-purple-100 text-purple-700 dark:bg-purple-900/30 dark:text-purple-300',\n text_to_speech: 'bg-amber-100 text-amber-700 dark:bg-amber-900/30 dark:text-amber-300',\n speech_to_text: 'bg-teal-100 text-teal-700 dark:bg-teal-900/30 dark:text-teal-300',\n}\n\nconst TYPE_COLOR_INACTIVE = 'bg-muted text-muted-foreground opacity-50'\n\nconst TYPE_LABELS: Record = {\n language: 'Language',\n embedding: 'Embedding',\n text_to_speech: 'TTS',\n speech_to_text: 'STT',\n}\n\n// =============================================================================\n// Credential Form Dialog\n// =============================================================================\n\nfunction CredentialFormDialog({\n open,\n onOpenChange,\n provider,\n credential,\n}: {\n open: boolean\n onOpenChange: (open: boolean) => void\n provider: string\n credential?: Credential | null\n}) {\n const { t } = useTranslation()\n const createCredential = useCreateCredential()\n const updateCredential = useUpdateCredential()\n const isEditing = !!credential\n const isSubmitting = createCredential.isPending || updateCredential.isPending\n\n const isVertex = provider === 'vertex'\n const isOllama = provider === 'ollama'\n const isOpenAICompatible = provider === 'openai_compatible'\n const requiresApiKey = !isVertex && !isOllama && !isOpenAICompatible\n\n const [name, setName] = useState('')\n const [apiKey, setApiKey] = useState('')\n const [baseUrl, setBaseUrl] = useState('')\n const [showApiKey, setShowApiKey] = useState(false)\n const [project, setProject] = useState('')\n const [location, setLocation] = useState('')\n const [credentialsPath, setCredentialsPath] = useState('')\n // Modalities\n const [modalities, setModalities] = useState([])\n\n useEffect(() => {\n if (credential) {\n setName(credential.name || '')\n setBaseUrl(credential.base_url || '')\n setApiKey('')\n setProject(credential.project || '')\n setLocation(credential.location || '')\n setCredentialsPath(credential.credentials_path || '')\n setModalities(credential.modalities || [])\n } else {\n setName('')\n setBaseUrl('')\n setApiKey('')\n setProject('')\n setLocation('')\n setCredentialsPath('')\n setModalities(PROVIDER_MODALITIES[provider] || ['language'])\n }\n }, [credential, provider])\n\n const handleSubmit = (e: React.FormEvent) => {\n e.preventDefault()\n\n const onSuccess = () => {\n onOpenChange(false)\n }\n\n if (isEditing && credential) {\n const data: UpdateCredentialRequest = {}\n if (name !== credential.name) data.name = name\n if (apiKey.trim()) data.api_key = apiKey.trim()\n if (baseUrl !== (credential.base_url || '')) data.base_url = baseUrl || undefined\n if (JSON.stringify(modalities) !== JSON.stringify(credential.modalities)) data.modalities = modalities\n if (isVertex) {\n if (project !== (credential.project || '')) data.project = project.trim() || undefined\n if (location !== (credential.location || '')) data.location = location.trim() || undefined\n if (credentialsPath !== (credential.credentials_path || '')) data.credentials_path = credentialsPath.trim() || undefined\n }\n updateCredential.mutate({ credentialId: credential.id, data }, { onSuccess })\n } else {\n const data: CreateCredentialRequest = {\n name: name || `${PROVIDER_DISPLAY_NAMES[provider] || provider} Config`,\n provider,\n modalities,\n api_key: apiKey.trim() || undefined,\n base_url: baseUrl || undefined,\n }\n if (isVertex) {\n data.project = project.trim() || undefined\n data.location = location.trim() || undefined\n data.credentials_path = credentialsPath.trim() || undefined\n }\n createCredential.mutate(data, { onSuccess })\n }\n }\n\n const isValid = isEditing\n ? true\n : isVertex\n ? name.trim() !== '' && project.trim() !== '' && location.trim() !== ''\n : name.trim() !== '' && (!requiresApiKey || apiKey.trim() !== '')\n\n const docsUrl = PROVIDER_DOCS[provider]\n\n return (\n \n \n \n \n {isEditing\n ? t.apiKeys.editConfig.replace('{provider}', PROVIDER_DISPLAY_NAMES[provider] || provider)\n : t.apiKeys.addConfig.replace('{provider}', PROVIDER_DISPLAY_NAMES[provider] || provider)}\n \n \n
\n {/* Name */}\n
\n \n setName(e.target.value)}\n placeholder={`${PROVIDER_DISPLAY_NAMES[provider] || provider} Production`}\n disabled={isSubmitting}\n />\n

{t.apiKeys.configNameHint}

\n
\n\n {/* Vertex fields */}\n {isVertex ? (\n <>\n
\n \n setProject(e.target.value)}\n placeholder=\"my-gcp-project\"\n disabled={isSubmitting}\n />\n
\n
\n \n setLocation(e.target.value)}\n placeholder=\"us-central1\"\n disabled={isSubmitting}\n />\n
\n
\n \n setCredentialsPath(e.target.value)}\n placeholder=\"/path/to/service-account.json\"\n disabled={isSubmitting}\n />\n
\n \n ) : (\n /* API Key */\n
\n \n
\n setApiKey(e.target.value)}\n placeholder={isEditing ? '••••••••••••' : 'sk-...'}\n disabled={isSubmitting}\n autoComplete=\"off\"\n />\n setShowApiKey(!showApiKey)}\n className=\"absolute right-3 top-1/2 -translate-y-1/2 text-muted-foreground hover:text-foreground text-xs\"\n tabIndex={-1}\n >\n {showApiKey ? 'Hide' : 'Show'}\n \n
\n {isEditing &&

{t.apiKeys.apiKeyEditHint}

}\n {docsUrl && (\n \n {t.apiKeys.getApiKey} →\n \n )}\n
\n )}\n\n {/* Base URL (non-Vertex) */}\n {!isVertex && (\n
\n \n setBaseUrl(e.target.value)}\n placeholder={isOllama ? 'http://localhost:11434' : 'https://api.example.com/v1'}\n disabled={isSubmitting}\n />\n

{t.apiKeys.baseUrlOverrideHint}

\n
\n )}\n\n {/* Actions */}\n
\n \n \n
\n
\n \n \n )\n}\n\n// =============================================================================\n// Model Discovery Dialog\n// =============================================================================\n\nfunction DiscoverModelsDialog({\n open,\n onOpenChange,\n credential,\n}: {\n open: boolean\n onOpenChange: (open: boolean) => void\n credential: Credential\n}) {\n const { t } = useTranslation()\n const discoverModels = useDiscoverModels()\n const registerModels = useRegisterModels()\n const [discoveredModels, setDiscoveredModels] = useState([])\n const [selectedModels, setSelectedModels] = useState>(new Set())\n const [hasDiscovered, setHasDiscovered] = useState(false)\n const [discoveryError, setDiscoveryError] = useState(null)\n const [searchQuery, setSearchQuery] = useState('')\n const [customModelSelected, setCustomModelSelected] = useState(false)\n // Model type selector - default to credential's first modality\n const [selectedType, setSelectedType] = useState(\n (credential.modalities[0] as ModelType) || 'language'\n )\n\n useEffect(() => {\n if (open && !hasDiscovered) {\n setDiscoveryError(null)\n discoverModels.mutate(credential.id, {\n onSuccess: (result) => {\n const seen = new Set()\n const unique = result.discovered.filter(m => {\n if (seen.has(m.name)) return false\n seen.add(m.name)\n return true\n })\n setDiscoveredModels(unique)\n setSelectedModels(new Set())\n setHasDiscovered(true)\n },\n onError: (error: unknown) => {\n setHasDiscovered(true)\n const msg = error instanceof Error ? error.message : String(error)\n setDiscoveryError(msg)\n },\n })\n }\n if (!open) {\n setHasDiscovered(false)\n setDiscoveredModels([])\n setSelectedModels(new Set())\n setDiscoveryError(null)\n setSearchQuery('')\n setCustomModelSelected(false)\n setSelectedType((credential.modalities[0] as ModelType) || 'language')\n }\n // eslint-disable-next-line react-hooks/exhaustive-deps -- intentionally only fires on open/close\n }, [open])\n\n // Reset custom selection when search changes\n useEffect(() => {\n setCustomModelSelected(false)\n }, [searchQuery])\n\n // Filter discovered models by search query\n const filteredModels = useMemo(() => {\n if (!searchQuery.trim()) return discoveredModels\n const q = searchQuery.toLowerCase()\n return discoveredModels.filter(m => m.name.toLowerCase().includes(q))\n }, [discoveredModels, searchQuery])\n\n // Show custom model option when search doesn't exactly match any discovered model\n const showCustomOption = useMemo(() => {\n if (!searchQuery.trim()) return false\n const q = searchQuery.trim().toLowerCase()\n return !discoveredModels.some(m => m.name.toLowerCase() === q)\n }, [discoveredModels, searchQuery])\n\n const handleRegister = () => {\n const selected = discoveredModels\n .filter(m => selectedModels.has(m.name))\n .map(m => ({\n name: m.name,\n provider: m.provider,\n model_type: selectedType,\n }))\n if (customModelSelected && showCustomOption) {\n selected.push({\n name: searchQuery.trim(),\n provider: credential.provider,\n model_type: selectedType,\n })\n }\n registerModels.mutate(\n { credentialId: credential.id, models: selected },\n { onSuccess: () => onOpenChange(false) }\n )\n }\n\n const totalSelected = selectedModels.size + (customModelSelected && showCustomOption ? 1 : 0)\n\n const toggleModel = (name: string) => {\n setSelectedModels(prev => {\n const next = new Set(prev)\n if (next.has(name)) next.delete(name)\n else next.add(name)\n return next\n })\n }\n\n const toggleAll = () => {\n const filteredNames = filteredModels.map(m => m.name)\n const allFilteredSelected = filteredNames.every(n => selectedModels.has(n))\n if (allFilteredSelected) {\n setSelectedModels(prev => {\n const next = new Set(prev)\n filteredNames.forEach(n => next.delete(n))\n return next\n })\n } else {\n setSelectedModels(prev => {\n const next = new Set(prev)\n filteredNames.forEach(n => next.add(n))\n return next\n })\n }\n }\n\n return (\n \n \n \n \n {t.models.discoverModels} - {PROVIDER_DISPLAY_NAMES[credential.provider] || credential.provider}\n \n \n {credential.name}\n \n \n\n {discoverModels.isPending ? (\n
\n \n
\n ) : discoveryError ? (\n \n \n {discoveryError}\n \n ) : (\n
\n {/* Model type selector */}\n
\n \n \n

{t.models.modelTypeHint}

\n
\n\n {/* Search input */}\n setSearchQuery(e.target.value)}\n />\n\n {/* Select all / count (only when there are discovered models to select) */}\n {filteredModels.length > 0 && (\n
\n \n
\n )}\n\n {/* Model list */}\n
\n {filteredModels.map((model) => (\n \n toggleModel(model.name)}\n className=\"rounded\"\n />\n {model.name}\n {model.description && model.description !== model.name && (\n ({model.description})\n )}\n \n ))}\n\n {/* Custom model option */}\n {showCustomOption && (\n \n )}\n\n {filteredModels.length === 0 && !showCustomOption && (\n

{t.models.noModelsFound}

\n )}\n
\n
\n )}\n\n \n \n \n {registerModels.isPending && }\n {t.common.add} ({totalSelected})\n \n \n \n \n )\n}\n\n// =============================================================================\n// Delete Credential Dialog\n// =============================================================================\n\nfunction DeleteCredentialDialog({\n open,\n onOpenChange,\n credential,\n allCredentials,\n}: {\n open: boolean\n onOpenChange: (open: boolean) => void\n credential: Credential\n allCredentials: Credential[]\n}) {\n const { t } = useTranslation()\n const deleteCredential = useDeleteCredential()\n const [migrateToId, setMigrateToId] = useState('')\n\n const otherCredentials = allCredentials.filter(\n c => c.id !== credential.id && c.provider === credential.provider\n )\n\n const handleDeleteWithModels = () => {\n deleteCredential.mutate(\n { credentialId: credential.id, options: { delete_models: true } },\n { onSuccess: () => onOpenChange(false) }\n )\n }\n\n const handleMigrate = () => {\n if (!migrateToId) return\n deleteCredential.mutate(\n { credentialId: credential.id, options: { migrate_to: migrateToId } },\n { onSuccess: () => onOpenChange(false) }\n )\n }\n\n const handleDeleteOnly = () => {\n deleteCredential.mutate(\n { credentialId: credential.id },\n { onSuccess: () => onOpenChange(false) }\n )\n }\n\n return (\n \n \n \n {t.apiKeys.deleteConfig}\n \n {t.apiKeys.deleteConfigConfirm.replace('{name}', credential.name)}\n \n \n\n {credential.model_count > 0 && (\n \n \n \n This credential has {credential.model_count} linked model(s).\n {otherCredentials.length > 0 && (\n
\n \n \n
\n )}\n \n \n )}\n\n \n \n {credential.model_count > 0 && migrateToId && (\n \n )}\n 0 ? handleDeleteWithModels : handleDeleteOnly}\n disabled={deleteCredential.isPending}\n >\n {deleteCredential.isPending && }\n {credential.model_count > 0 ? 'Delete with Models' : t.common.delete}\n \n \n \n \n )\n}\n\n// =============================================================================\n// Credential Card (shows credential + its models)\n// =============================================================================\n\nfunction CredentialItem({\n credential,\n models,\n defaults,\n allCredentials,\n}: {\n credential: Credential\n models: Model[]\n defaults: ModelDefaults | null\n allCredentials: Credential[]\n}) {\n const { t } = useTranslation()\n const { testCredential, isPending: isTestPending, testResults } = useTestCredential()\n const { testModel, isPending: isModelTestPending, testingModelId, testResult: modelTestResult, testedModelName, clearResult: clearModelTestResult } = useTestModel()\n const deleteModel = useDeleteModel()\n const [editOpen, setEditOpen] = useState(false)\n const [deleteOpen, setDeleteOpen] = useState(false)\n const [discoverOpen, setDiscoverOpen] = useState(false)\n // Full credential data needed for edit form\n const { data: fullCredential } = useCredential(editOpen ? credential.id : '')\n\n const linkedModels = models.filter(m => m.credential === credential.id)\n const activeTypes = new Set(linkedModels.map(m => m.type))\n const testResult = testResults[credential.id]\n\n // Extract translations used in model badge loops to avoid excessive Proxy accesses\n const testModelLabel = t.models.testModel\n const deleteModelLabel = t.models.deleteModel\n\n // Check which models are defaults\n const defaultSlots: Record = {}\n if (defaults) {\n const slotMap: Record = {\n 'Chat': defaults.default_chat_model,\n 'Transform': defaults.default_transformation_model,\n 'Tools': defaults.default_tools_model,\n 'Large Ctx': defaults.large_context_model,\n 'Embedding': defaults.default_embedding_model,\n 'TTS': defaults.default_text_to_speech_model,\n 'STT': defaults.default_speech_to_text_model,\n }\n for (const [slot, modelId] of Object.entries(slotMap)) {\n if (modelId) defaultSlots[modelId] = slot\n }\n }\n\n return (\n <>\n
\n
\n
\n {credential.name}\n
\n {credential.modalities.map(mod => (\n \n {TYPE_ICONS[mod as ModelType]}\n {TYPE_LABELS[mod as ModelType] || mod}\n \n ))}\n
\n {credential.has_api_key && (\n \n \n Key\n \n )}\n
\n
\n {testResult && (\n testResult.success\n ? \n : \n )}\n testCredential(credential.id)}\n disabled={isTestPending}\n title={t.apiKeys.testConnection}\n >\n {isTestPending ? : }\n Test\n \n setDiscoverOpen(true)}\n title={t.apiKeys.syncModels}\n >\n \n Models\n \n \n setDeleteOpen(true)}\n className=\"text-destructive hover:text-destructive hover:bg-destructive/10\"\n title={t.common.delete}\n >\n \n \n
\n
\n\n {/* Linked models grouped by type */}\n {linkedModels.length > 0 && (\n
\n {(['language', 'embedding', 'text_to_speech', 'speech_to_text'] as ModelType[])\n .filter(type => linkedModels.some(m => m.type === type))\n .map(type => (\n
\n \n {TYPE_ICONS[type]}\n {TYPE_LABELS[type]}\n \n
\n {linkedModels.filter(m => m.type === type).map(model => {\n const defaultSlot = defaultSlots[model.id]\n return (\n \n {model.name}\n {defaultSlot && ({defaultSlot})}\n testModel(model.id, model.name)}\n disabled={isModelTestPending && testingModelId === model.id}\n title={testModelLabel}\n >\n {isModelTestPending && testingModelId === model.id\n ? \n : \n }\n \n deleteModel.mutate(model.id)}\n title={deleteModelLabel}\n >\n \n \n \n )\n })}\n
\n
\n ))}\n
\n )}\n\n\n
\n\n {/* Edit dialog */}\n {editOpen && (\n \n )}\n\n {/* Delete dialog */}\n {deleteOpen && (\n \n )}\n\n {/* Discover models dialog */}\n {discoverOpen && (\n \n )}\n\n {/* Model test result dialog */}\n { if (!open) clearModelTestResult() }}\n result={modelTestResult}\n modelName={testedModelName}\n />\n \n )\n}\n\n// =============================================================================\n// Provider Section (shows all credentials for a provider)\n// =============================================================================\n\nfunction ProviderSection({\n provider,\n credentials,\n models,\n defaults,\n allCredentials,\n encryptionReady,\n}: {\n provider: string\n credentials: Credential[]\n models: Model[]\n defaults: ModelDefaults | null\n allCredentials: Credential[]\n encryptionReady: boolean\n}) {\n const { t } = useTranslation()\n const [addOpen, setAddOpen] = useState(false)\n\n const displayName = PROVIDER_DISPLAY_NAMES[provider] || provider\n const modalities = PROVIDER_MODALITIES[provider] || ['language']\n const hasCredentials = credentials.length > 0\n\n // Models linked to any credential of this provider\n const providerModels = models.filter(m =>\n credentials.some(c => c.id === m.credential)\n )\n const activeTypes = new Set(providerModels.map(m => m.type))\n\n return (\n \n \n
\n
\n {displayName}\n
\n {modalities.map((type) => (\n \n {TYPE_ICONS[type]}\n {TYPE_LABELS[type]}\n \n ))}\n
\n
\n
\n {hasCredentials ? (\n \n \n {t.apiKeys.configured}\n \n ) : (\n \n \n {t.apiKeys.notConfigured}\n \n )}\n
\n
\n \n \n {credentials.map(cred => (\n \n ))}\n\n setAddOpen(true)}\n className=\"w-full gap-2\"\n disabled={!encryptionReady}\n >\n \n {t.apiKeys.addConfig}\n \n \n\n {addOpen && (\n \n )}\n \n )\n}\n\n// =============================================================================\n// Default Models Section\n// =============================================================================\n\nfunction DefaultModelSelectors({\n models,\n defaults,\n}: {\n models: Model[]\n defaults: ModelDefaults\n}) {\n const { t } = useTranslation()\n const updateDefaults = useUpdateModelDefaults()\n const autoAssign = useAutoAssignDefaults()\n const { setValue, watch } = useForm({ defaultValues: defaults })\n const generatedId = useId()\n\n const [showEmbeddingDialog, setShowEmbeddingDialog] = useState(false)\n const [pendingEmbeddingChange, setPendingEmbeddingChange] = useState<{\n key: keyof ModelDefaults; value: string; oldModelId?: string; newModelId?: string\n } | null>(null)\n\n useEffect(() => {\n if (defaults) {\n Object.entries(defaults).forEach(([key, value]) => {\n setValue(key as keyof ModelDefaults, value)\n })\n }\n }, [defaults, setValue])\n\n interface DefaultConfig {\n key: keyof ModelDefaults\n label: string\n description: string\n modelType: ModelType\n required?: boolean\n id: string\n }\n\n const primaryConfigs: DefaultConfig[] = [\n { key: 'default_chat_model', label: t.models.chatModelLabel, description: t.models.chatModelDesc, modelType: 'language', required: true, id: `${generatedId}-chat` },\n { key: 'default_embedding_model', label: t.models.embeddingModelLabel, description: t.models.embeddingModelDesc, modelType: 'embedding', required: true, id: `${generatedId}-embed` },\n { key: 'default_text_to_speech_model', label: t.models.ttsModelLabel, description: t.models.ttsModelDesc, modelType: 'text_to_speech', id: `${generatedId}-tts` },\n { key: 'default_speech_to_text_model', label: t.models.sttModelLabel, description: t.models.sttModelDesc, modelType: 'speech_to_text', id: `${generatedId}-stt` },\n ]\n\n const advancedConfigs: DefaultConfig[] = [\n { key: 'default_transformation_model', label: t.models.transformationModelLabel, description: t.models.transformationModelDesc, modelType: 'language', required: true, id: `${generatedId}-transform` },\n { key: 'default_tools_model', label: t.models.toolsModelLabel, description: t.models.toolsModelDesc, modelType: 'language', id: `${generatedId}-tools` },\n { key: 'large_context_model', label: t.models.largeContextModelLabel, description: t.models.largeContextModelDesc, modelType: 'language', id: `${generatedId}-large` },\n ]\n\n const defaultConfigs = [...primaryConfigs, ...advancedConfigs]\n\n const handleChange = (key: keyof ModelDefaults, value: string) => {\n if (key === 'default_embedding_model') {\n const current = defaults[key]\n if (current && current !== value) {\n setPendingEmbeddingChange({ key, value, oldModelId: current, newModelId: value })\n setShowEmbeddingDialog(true)\n return\n }\n }\n updateDefaults.mutate({ [key]: value || null })\n }\n\n const handleConfirmEmbeddingChange = () => {\n if (pendingEmbeddingChange) {\n updateDefaults.mutate({ [pendingEmbeddingChange.key]: pendingEmbeddingChange.value || null })\n setPendingEmbeddingChange(null)\n }\n }\n\n const getModelsForType = (type: ModelType) => models.filter(m => m.type === type)\n\n const missingRequired = defaultConfigs\n .filter(c => {\n if (!c.required) return false\n const value = defaults[c.key]\n if (!value) return true\n return !models.filter(m => m.type === c.modelType).some(m => m.id === value)\n })\n .map(c => c.label)\n\n return (\n \n \n {t.models.defaultAssignments}\n {t.models.defaultAssignmentsDesc}\n \n \n {missingRequired.length > 0 && (\n \n \n \n {t.models.missingRequiredModels.replace('{models}', missingRequired.join(', '))}\n autoAssign.mutate()}\n disabled={autoAssign.isPending}\n className=\"shrink-0 gap-1.5\"\n >\n {autoAssign.isPending ? : }\n {autoAssign.isPending ? t.models.autoAssigning : t.models.autoAssign}\n \n \n \n )}\n\n {/* Primary models: Chat, Embedding, TTS, STT */}\n
\n {primaryConfigs.map(config => {\n const available = getModelsForType(config.modelType)\n const currentValue = watch(config.key) || undefined\n const isValid = currentValue && available.some(m => m.id === currentValue)\n\n return (\n
\n \n
\n handleChange(config.key, v)}\n >\n 0 ? 'border-destructive' : ''}`}\n >\n 0\n ? t.models.requiredModelPlaceholder\n : t.models.selectModelPlaceholder\n } />\n \n \n {available.sort((a, b) => a.name.localeCompare(b.name)).map(model => (\n \n
\n {model.name}\n {model.provider}\n
\n \n ))}\n \n \n {!config.required && currentValue && (\n \n )}\n
\n
\n )\n })}\n
\n\n {/* Advanced models: Transformation, Tools, Large Context */}\n
\n

{t.navigation.advanced}

\n
\n {advancedConfigs.map(config => {\n const available = getModelsForType(config.modelType)\n const currentValue = watch(config.key) || undefined\n const isValid = currentValue && available.some(m => m.id === currentValue)\n\n return (\n
\n \n
\n handleChange(config.key, v)}\n >\n 0 ? 'border-destructive' : ''}`}\n >\n 0\n ? t.models.requiredModelPlaceholder\n : t.models.selectModelPlaceholder\n } />\n \n \n {available.sort((a, b) => a.name.localeCompare(b.name)).map(model => (\n \n
\n {model.name}\n {model.provider}\n
\n \n ))}\n \n \n {!config.required && currentValue && (\n \n )}\n
\n

{config.description}

\n
\n )\n })}\n
\n
\n \n\n { if (!open) { setPendingEmbeddingChange(null); setShowEmbeddingDialog(false) } }}\n onConfirm={handleConfirmEmbeddingChange}\n oldModelName={pendingEmbeddingChange?.oldModelId ? models.find(m => m.id === pendingEmbeddingChange.oldModelId)?.name : undefined}\n newModelName={pendingEmbeddingChange?.newModelId ? models.find(m => m.id === pendingEmbeddingChange.newModelId)?.name : undefined}\n />\n \n )\n}\n\n// =============================================================================\n// Main Page\n// =============================================================================\n\nexport default function ApiKeysPage() {\n const { t } = useTranslation()\n\n // Data\n const { data: credentials, isLoading: credentialsLoading } = useCredentials()\n const { data: models, isLoading: modelsLoading } = useModels()\n const { data: defaults, isLoading: defaultsLoading } = useModelDefaults()\n const { data: credentialStatus } = useCredentialStatus()\n const { data: envStatus } = useEnvStatus()\n\n const encryptionReady = credentialStatus?.encryption_configured ?? true\n\n // Group credentials by provider\n const credentialsByProvider = useMemo(() => {\n const grouped: Record = {}\n for (const provider of ALL_PROVIDERS) {\n grouped[provider] = []\n }\n if (credentials) {\n for (const cred of credentials) {\n if (!grouped[cred.provider]) grouped[cred.provider] = []\n grouped[cred.provider].push(cred)\n }\n }\n return grouped\n }, [credentials])\n\n // Providers needing migration\n const providersToMigrate = useMemo(() => {\n if (!envStatus || !credentialStatus) return []\n const providers: string[] = []\n for (const provider in envStatus) {\n if (envStatus[provider] && credentialStatus.source[provider] === 'environment') {\n providers.push(provider)\n }\n }\n return providers\n }, [envStatus, credentialStatus])\n\n // Sort: configured providers first\n const sortedProviders = useMemo(() => {\n return [...ALL_PROVIDERS].sort((a, b) => {\n const aHas = (credentialsByProvider[a]?.length || 0) > 0 ? 1 : 0\n const bHas = (credentialsByProvider[b]?.length || 0) > 0 ? 1 : 0\n return bHas - aHas\n })\n }, [credentialsByProvider])\n\n const isLoading = credentialsLoading || modelsLoading || defaultsLoading\n\n if (isLoading) {\n return (\n \n
\n \n
\n \n )\n }\n\n return (\n \n
\n
\n {/* Header */}\n
\n

\n \n {t.apiKeys.title}\n

\n

{t.apiKeys.description}

\n
\n\n {/* Encryption warning */}\n {!encryptionReady && (\n \n \n {t.apiKeys.encryptionRequired}\n \n \n {t.apiKeys.encryptionRequiredDescription}\n \n \n \n )}\n\n {/* Migration banner */}\n {encryptionReady && }\n\n {/* Default Model Selectors */}\n {models && defaults && (\n \n )}\n\n {/* Provider Cards */}\n
\n {sortedProviders.map(provider => (\n \n ))}\n
\n\n {/* Help link */}\n
\n \n {t.apiKeys.learnMore}\n \n
\n
\n
\n \n )\n}\n" }, { "path": "frontend/src/app/(dashboard)/settings/components/SettingsForm.tsx", "content": "'use client'\n\nimport { useForm, Controller } from 'react-hook-form'\nimport { zodResolver } from '@hookform/resolvers/zod'\nimport { z } from 'zod'\nimport { Card, CardContent, CardHeader, CardTitle, CardDescription } from '@/components/ui/card'\nimport { Button } from '@/components/ui/button'\nimport { Label } from '@/components/ui/label'\nimport { LoadingSpinner } from '@/components/common/LoadingSpinner'\nimport { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from '@/components/ui/select'\nimport { Collapsible, CollapsibleContent, CollapsibleTrigger } from '@/components/ui/collapsible'\nimport { Alert, AlertTitle, AlertDescription } from '@/components/ui/alert'\nimport { useSettings, useUpdateSettings } from '@/lib/hooks/use-settings'\nimport { useEffect, useState } from 'react'\nimport { ChevronDownIcon } from 'lucide-react'\nimport { useTranslation } from '@/lib/hooks/use-translation'\n\nconst settingsSchema = z.object({\n default_content_processing_engine_doc: z.enum(['auto', 'docling', 'simple']).optional(),\n default_content_processing_engine_url: z.enum(['auto', 'firecrawl', 'jina', 'simple']).optional(),\n default_embedding_option: z.enum(['ask', 'always', 'never']).optional(),\n auto_delete_files: z.enum(['yes', 'no']).optional(),\n})\n\ntype SettingsFormData = z.infer\n\nexport function SettingsForm() {\n const { t } = useTranslation()\n const { data: settings, isLoading, error } = useSettings()\n const updateSettings = useUpdateSettings()\n const [expandedSections, setExpandedSections] = useState>({\n doc: false,\n url: false,\n embedding: false,\n files: false\n })\n const [hasResetForm, setHasResetForm] = useState(false)\n \n \n const {\n control,\n handleSubmit,\n reset,\n formState: { isDirty }\n } = useForm({\n resolver: zodResolver(settingsSchema),\n defaultValues: {\n default_content_processing_engine_doc: undefined,\n default_content_processing_engine_url: undefined,\n default_embedding_option: undefined,\n auto_delete_files: undefined,\n }\n })\n\n\n const toggleSection = (section: string) => {\n setExpandedSections(prev => ({ ...prev, [section]: !prev[section] }))\n }\n\n useEffect(() => {\n if (settings && settings.default_content_processing_engine_doc && !hasResetForm) {\n const formData = {\n default_content_processing_engine_doc: settings.default_content_processing_engine_doc as 'auto' | 'docling' | 'simple',\n default_content_processing_engine_url: settings.default_content_processing_engine_url as 'auto' | 'firecrawl' | 'jina' | 'simple',\n default_embedding_option: settings.default_embedding_option as 'ask' | 'always' | 'never',\n auto_delete_files: settings.auto_delete_files as 'yes' | 'no',\n }\n reset(formData)\n setHasResetForm(true)\n }\n }, [hasResetForm, reset, settings])\n\n const onSubmit = async (data: SettingsFormData) => {\n await updateSettings.mutateAsync(data)\n }\n\n if (isLoading) {\n return (\n
\n \n
\n )\n }\n\n if (error) {\n return (\n \n {t.settings.loadFailed}\n \n {error instanceof Error ? error.message : t.common.error}\n \n \n )\n }\n\n return (\n
\n \n \n {t.settings.contentProcessing}\n \n {t.settings.contentProcessingDesc}\n \n \n \n
\n \n (\n \n \n \n \n \n {t.settings.autoRecommended}\n {t.settings.docling}\n {t.settings.simple}\n \n \n )}\n />\n toggleSection('doc')}>\n \n \n {t.settings.helpMeChoose}\n \n \n

{t.settings.docHelp}

\n \n \n
\n \n
\n \n (\n \n \n \n \n \n {t.settings.autoRecommended}\n {t.settings.firecrawl}\n {t.settings.jina}\n {t.settings.simple}\n \n \n )}\n />\n toggleSection('url')}>\n \n \n {t.settings.helpMeChoose}\n \n \n

{t.settings.urlHelp}

\n \n \n
\n \n \n\n \n \n {t.settings.embeddingAndSearch}\n \n {t.settings.embeddingAndSearchDesc}\n \n \n \n
\n \n (\n \n \n \n \n \n {t.settings.ask}\n {t.settings.always}\n {t.settings.never}\n \n \n )}\n />\n toggleSection('embedding')}>\n \n \n {t.settings.helpMeChoose}\n \n \n

{t.settings.embeddingHelp}

\n \n \n
\n \n \n\n \n \n {t.settings.fileManagement}\n \n {t.settings.fileManagementDesc}\n \n \n \n
\n \n (\n \n \n \n \n \n {t.common.yes}\n {t.common.no}\n \n \n )}\n />\n toggleSection('files')}>\n \n \n {t.settings.helpMeChoose}\n \n \n

{t.settings.filesHelp}

\n \n \n
\n \n \n\n
\n \n
\n
\n )\n}\n" }, { "path": "frontend/src/app/(dashboard)/settings/page.tsx", "content": "'use client'\n\nimport { AppShell } from '@/components/layout/AppShell'\nimport { SettingsForm } from './components/SettingsForm'\nimport { useSettings } from '@/lib/hooks/use-settings'\nimport { Button } from '@/components/ui/button'\nimport { RefreshCw } from 'lucide-react'\nimport { useTranslation } from '@/lib/hooks/use-translation'\n\nexport default function SettingsPage() {\n const { t } = useTranslation()\n const { refetch } = useSettings()\n\n return (\n \n
\n
\n
\n
\n

{t.navigation.settings}

\n \n
\n\n \n
\n
\n
\n \n )\n}\n" }, { "path": "frontend/src/app/(dashboard)/sources/[id]/page.tsx", "content": "'use client'\n\nimport { useRouter, useParams } from 'next/navigation'\nimport { useCallback } from 'react'\nimport { Button } from '@/components/ui/button'\nimport { ArrowLeft } from 'lucide-react'\nimport { useSourceChat } from '@/lib/hooks/useSourceChat'\nimport { ChatPanel } from '@/components/source/ChatPanel'\nimport { useNavigation } from '@/lib/hooks/use-navigation'\nimport { SourceDetailContent } from '@/components/source/SourceDetailContent'\n\nexport default function SourceDetailPage() {\n const router = useRouter()\n const params = useParams()\n const sourceId = params?.id ? decodeURIComponent(params.id as string) : ''\n const navigation = useNavigation()\n\n // Initialize source chat\n const chat = useSourceChat(sourceId)\n\n const handleBack = useCallback(() => {\n const returnPath = navigation.getReturnPath()\n router.push(returnPath)\n navigation.clearReturnTo()\n }, [navigation, router])\n\n return (\n
\n {/* Back button */}\n
\n \n \n {navigation.getReturnLabel()}\n \n
\n\n {/* Main content: Source detail + Chat */}\n
\n {/* Left column - Source detail */}\n
\n \n
\n\n {/* Right column - Chat */}\n
\n chat.sendMessage(message, model)}\n modelOverride={chat.currentSession?.model_override}\n onModelChange={(model) => {\n if (chat.currentSessionId) {\n chat.updateSession(chat.currentSessionId, { model_override: model })\n }\n }}\n sessions={chat.sessions}\n currentSessionId={chat.currentSessionId}\n onCreateSession={(title) => chat.createSession({ title })}\n onSelectSession={chat.switchSession}\n onUpdateSession={(sessionId, title) => chat.updateSession(sessionId, { title })}\n onDeleteSession={chat.deleteSession}\n loadingSessions={chat.loadingSessions}\n />\n
\n
\n
\n )\n}\n" }, { "path": "frontend/src/app/(dashboard)/sources/page.tsx", "content": "'use client'\n\nimport { useState, useEffect, useCallback, useRef } from 'react'\nimport { useRouter } from 'next/navigation'\nimport { sourcesApi } from '@/lib/api/sources'\nimport { SourceListResponse } from '@/lib/types/api'\nimport { LoadingSpinner } from '@/components/common/LoadingSpinner'\nimport { EmptyState } from '@/components/common/EmptyState'\nimport { AppShell } from '@/components/layout/AppShell'\nimport { ConfirmDialog } from '@/components/common/ConfirmDialog'\nimport { FileText, Link as LinkIcon, Upload, AlignLeft, Trash2, ArrowUpDown } from 'lucide-react'\nimport { formatDistanceToNow } from 'date-fns'\nimport { Badge } from '@/components/ui/badge'\nimport { Button } from '@/components/ui/button'\nimport { useTranslation } from '@/lib/hooks/use-translation'\nimport { getDateLocale } from '@/lib/utils/date-locale'\nimport { cn } from '@/lib/utils'\nimport { toast } from 'sonner'\nimport { getApiErrorKey } from '@/lib/utils/error-handler'\n\nexport default function SourcesPage() {\n const { t, language } = useTranslation()\n const [sources, setSources] = useState([])\n const [loading, setLoading] = useState(true)\n const [loadingMore, setLoadingMore] = useState(false)\n const [error, setError] = useState(null)\n const [selectedIndex, setSelectedIndex] = useState(0)\n const [sortBy, setSortBy] = useState<'created' | 'updated'>('updated')\n const [sortOrder, setSortOrder] = useState<'asc' | 'desc'>('desc')\n const [deleteDialog, setDeleteDialog] = useState<{ open: boolean; source: SourceListResponse | null }>({\n open: false,\n source: null\n })\n const router = useRouter()\n const tableRef = useRef(null)\n const scrollContainerRef = useRef(null)\n const offsetRef = useRef(0)\n const loadingMoreRef = useRef(false)\n const hasMoreRef = useRef(true)\n const PAGE_SIZE = 30\n\n const fetchSources = useCallback(async (reset = false) => {\n try {\n // Check flags before proceeding\n if (!reset && (loadingMoreRef.current || !hasMoreRef.current)) {\n return\n }\n\n if (reset) {\n setLoading(true)\n offsetRef.current = 0\n setSources([])\n hasMoreRef.current = true\n } else {\n loadingMoreRef.current = true\n setLoadingMore(true)\n }\n\n const data = await sourcesApi.list({\n limit: PAGE_SIZE,\n offset: offsetRef.current,\n sort_by: sortBy,\n sort_order: sortOrder,\n })\n\n if (reset) {\n setSources(data)\n } else {\n setSources(prev => [...prev, ...data])\n }\n\n // Check if we have more data\n const hasMoreData = data.length === PAGE_SIZE\n hasMoreRef.current = hasMoreData\n offsetRef.current += data.length\n } catch (err) {\n console.error('Failed to fetch sources:', err)\n setError(t.sources.failedToLoad)\n toast.error(t.sources.failedToLoad)\n } finally {\n setLoading(false)\n setLoadingMore(false)\n loadingMoreRef.current = false\n }\n }, [sortBy, sortOrder, t.sources.failedToLoad])\n\n // Initial load and when sort changes\n useEffect(() => {\n fetchSources(true)\n // eslint-disable-next-line react-hooks/exhaustive-deps\n }, [sortBy, sortOrder])\n\n useEffect(() => {\n // Focus the table when component mounts or sources change\n if (sources.length > 0 && tableRef.current) {\n tableRef.current.focus()\n }\n }, [sources])\n\n useEffect(() => {\n const handleKeyDown = (e: KeyboardEvent) => {\n if (sources.length === 0) return\n\n switch (e.key) {\n case 'ArrowDown':\n e.preventDefault()\n setSelectedIndex((prev) => {\n const newIndex = Math.min(prev + 1, sources.length - 1)\n // Scroll to keep selected row visible\n setTimeout(() => scrollToSelectedRow(newIndex), 0)\n return newIndex\n })\n break\n case 'ArrowUp':\n e.preventDefault()\n setSelectedIndex((prev) => {\n const newIndex = Math.max(prev - 1, 0)\n // Scroll to keep selected row visible\n setTimeout(() => scrollToSelectedRow(newIndex), 0)\n return newIndex\n })\n break\n case 'Enter':\n e.preventDefault()\n if (sources[selectedIndex]) {\n router.push(`/sources/${sources[selectedIndex].id}`)\n }\n break\n case 'Home':\n e.preventDefault()\n setSelectedIndex(0)\n setTimeout(() => scrollToSelectedRow(0), 0)\n break\n case 'End':\n e.preventDefault()\n const lastIndex = sources.length - 1\n setSelectedIndex(lastIndex)\n setTimeout(() => scrollToSelectedRow(lastIndex), 0)\n break\n }\n }\n\n window.addEventListener('keydown', handleKeyDown)\n return () => window.removeEventListener('keydown', handleKeyDown)\n }, [sources, selectedIndex, router])\n\n const scrollToSelectedRow = (index: number) => {\n const scrollContainer = scrollContainerRef.current\n if (!scrollContainer) return\n\n // Find the selected row element\n const rows = scrollContainer.querySelectorAll('tbody tr')\n const selectedRow = rows[index] as HTMLElement\n if (!selectedRow) return\n\n const containerRect = scrollContainer.getBoundingClientRect()\n const rowRect = selectedRow.getBoundingClientRect()\n\n // Check if row is above visible area\n if (rowRect.top < containerRect.top) {\n selectedRow.scrollIntoView({ behavior: 'smooth', block: 'start' })\n }\n // Check if row is below visible area\n else if (rowRect.bottom > containerRect.bottom) {\n selectedRow.scrollIntoView({ behavior: 'smooth', block: 'end' })\n }\n }\n\n // Set up scroll listener after sources are loaded\n useEffect(() => {\n const scrollContainer = scrollContainerRef.current\n if (!scrollContainer) return\n\n let scrollTimeout: NodeJS.Timeout | null = null\n\n const handleScroll = () => {\n if (scrollTimeout) {\n clearTimeout(scrollTimeout)\n }\n\n scrollTimeout = setTimeout(() => {\n if (!scrollContainerRef.current) return\n\n const { scrollTop, scrollHeight, clientHeight } = scrollContainerRef.current\n const distanceFromBottom = scrollHeight - scrollTop - clientHeight\n\n // Load more when within 200px of the bottom\n if (distanceFromBottom < 200 && !loadingMoreRef.current && hasMoreRef.current) {\n fetchSources(false)\n }\n }, 100)\n }\n\n scrollContainer.addEventListener('scroll', handleScroll)\n handleScroll() // Check on mount\n\n return () => {\n scrollContainer.removeEventListener('scroll', handleScroll)\n if (scrollTimeout) {\n clearTimeout(scrollTimeout)\n }\n }\n }, [fetchSources, sources.length])\n\n const toggleSort = (field: 'created' | 'updated') => {\n if (sortBy === field) {\n // Toggle order if clicking the same field\n setSortOrder(prev => prev === 'asc' ? 'desc' : 'asc')\n } else {\n // Switch to new field with default desc order\n setSortBy(field)\n setSortOrder('desc')\n }\n }\n\n const getSourceIcon = (source: SourceListResponse) => {\n if (source.asset?.url) return \n if (source.asset?.file_path) return \n return \n }\n\n const getSourceType = (source: SourceListResponse) => {\n if (source.asset?.url) return t.sources.type.link\n if (source.asset?.file_path) return t.sources.type.file\n return t.sources.type.text\n }\n\n const handleRowClick = useCallback((index: number, sourceId: string) => {\n setSelectedIndex(index)\n router.push(`/sources/${sourceId}`)\n }, [router])\n\n const handleDeleteClick = useCallback((e: React.MouseEvent, source: SourceListResponse) => {\n e.stopPropagation() // Prevent row click\n setDeleteDialog({ open: true, source })\n }, [])\n\n const handleDeleteConfirm = async () => {\n if (!deleteDialog.source) return\n\n try {\n await sourcesApi.delete(deleteDialog.source.id)\n toast.success(t.sources.deleteSuccess)\n // Remove the deleted source from the list\n setSources(prev => prev.filter(s => s.id !== deleteDialog.source?.id))\n setDeleteDialog({ open: false, source: null })\n } catch (err: unknown) {\n const error = err as { response?: { data?: { detail?: string } }, message?: string };\n console.error('Failed to delete source:', error)\n toast.error(t(getApiErrorKey(error.response?.data?.detail || error.message)))\n }\n }\n\n if (loading) {\n return (\n \n
\n \n
\n \n )\n }\n\n if (error) {\n return (\n \n
\n

{error}

\n
\n \n )\n }\n\n if (sources.length === 0) {\n return (\n \n \n \n )\n }\n\n return (\n \n
\n
\n

{t.sources.allSources}

\n

\n {t.sources.allSourcesDesc}\n

\n
\n\n
\n \n \n \n \n \n \n \n \n \n \n \n \n {t.common.type}\n \n \n {t.common.title}\n \n \n toggleSort('created')}\n className=\"h-8 px-2 hover:bg-muted\"\n >\n {t.common.created_label}\n \n {sortBy === 'created' && (\n \n {sortOrder === 'asc' ? '↑' : '↓'}\n \n )}\n \n \n \n {t.sources.insights}\n \n \n {t.sources.embedded}\n \n \n {t.common.actions}\n \n \n \n \n {sources.map((source, index) => (\n handleRowClick(index, source.id)}\n onMouseEnter={() => setSelectedIndex(index)}\n className={cn(\n \"border-b transition-colors cursor-pointer\",\n selectedIndex === index\n ? \"bg-accent\"\n : \"hover:bg-muted/50\"\n )}\n >\n \n
\n {getSourceIcon(source)}\n \n {getSourceType(source)}\n \n
\n \n \n
\n \n {source.title || t.sources.untitledSource}\n \n {source.asset?.url && (\n \n {source.asset.url}\n \n )}\n
\n \n \n {formatDistanceToNow(new Date(source.created), { \n addSuffix: true,\n locale: getDateLocale(language)\n })}\n \n \n {source.insights_count || 0}\n \n \n \n {source.embedded ? t.sources.yes : t.sources.no}\n \n \n \n handleDeleteClick(e, source)}\n className=\"text-destructive hover:text-destructive\"\n >\n \n \n \n \n ))}\n {loadingMore && (\n \n \n
\n \n {t.sources.loadingMore}\n
\n \n \n )}\n \n \n
\n
\n\n setDeleteDialog({ open, source: deleteDialog.source })}\n title={t.sources.delete}\n description={t.sources.deleteConfirmWithTitle.replace('{title}', deleteDialog.source?.title || t.sources.untitledSource)}\n confirmText={t.common.delete}\n confirmVariant=\"destructive\"\n onConfirm={handleDeleteConfirm}\n />\n \n )\n}" }, { "path": "frontend/src/app/(dashboard)/transformations/components/DefaultPromptEditor.tsx", "content": "'use client'\n\nimport { useState, useEffect, useId } from 'react'\nimport { Card, CardContent, CardDescription, CardHeader, CardTitle } from '@/components/ui/card'\nimport { Button } from '@/components/ui/button'\nimport { Textarea } from '@/components/ui/textarea'\nimport { Label } from '@/components/ui/label'\nimport { Collapsible, CollapsibleContent, CollapsibleTrigger } from '@/components/ui/collapsible'\nimport { ChevronDown, ChevronRight, Settings } from 'lucide-react'\nimport { useDefaultPrompt, useUpdateDefaultPrompt } from '@/lib/hooks/use-transformations'\nimport { useTranslation } from '@/lib/hooks/use-translation'\n\nexport function DefaultPromptEditor() {\n const [isOpen, setIsOpen] = useState(false)\n const [prompt, setPrompt] = useState('')\n const { data: defaultPrompt, isLoading } = useDefaultPrompt()\n const updateDefaultPrompt = useUpdateDefaultPrompt()\n const { t } = useTranslation()\n const textareaId = useId()\n\n useEffect(() => {\n if (defaultPrompt) {\n setPrompt(defaultPrompt.transformation_instructions || '')\n }\n }, [defaultPrompt])\n\n const handleSave = () => {\n updateDefaultPrompt.mutate({ transformation_instructions: prompt })\n }\n\n return (\n \n \n \n \n
\n
\n \n
\n {t.transformations.defaultPrompt}\n \n {t.transformations.defaultPromptDesc}\n \n
\n
\n {isOpen ? (\n \n ) : (\n \n )}\n
\n \n \n \n \n
\n \n setPrompt(e.target.value)}\n placeholder={t.transformations.defaultPromptPlaceholder}\n className=\"min-h-[200px] font-mono text-sm\"\n disabled={isLoading}\n />\n
\n
\n \n
\n \n \n \n \n )\n}" }, { "path": "frontend/src/app/(dashboard)/transformations/components/TransformationCard.tsx", "content": "'use client'\n\nimport { useState } from 'react'\nimport { Card, CardContent, CardHeader } from '@/components/ui/card'\nimport { Button } from '@/components/ui/button'\nimport { ConfirmDialog } from '@/components/common/ConfirmDialog'\nimport { Badge } from '@/components/ui/badge'\nimport { Collapsible, CollapsibleContent, CollapsibleTrigger } from '@/components/ui/collapsible'\nimport { ChevronDown, ChevronRight, Trash2, Wand2, Edit } from 'lucide-react'\nimport { Transformation } from '@/lib/types/transformations'\nimport { useDeleteTransformation } from '@/lib/hooks/use-transformations'\nimport { useTranslation } from '@/lib/hooks/use-translation'\nimport { cn } from '@/lib/utils'\n\ninterface TransformationCardProps {\n transformation: Transformation\n onPlayground?: () => void\n onEdit?: () => void\n}\n\nexport function TransformationCard({ transformation, onPlayground, onEdit }: TransformationCardProps) {\n const { t } = useTranslation()\n const [isExpanded, setIsExpanded] = useState(false)\n const [showDeleteDialog, setShowDeleteDialog] = useState(false)\n const deleteTransformation = useDeleteTransformation()\n\n const handleDelete = () => {\n deleteTransformation.mutate(transformation.id)\n setShowDeleteDialog(false)\n }\n\n return (\n <>\n \n \n \n
\n \n
\n {isExpanded ? (\n \n ) : (\n \n )}\n
\n {transformation.name}\n {!isExpanded && transformation.description && (\n {transformation.description}\n )}\n
\n {transformation.apply_default && (\n {t.common.default}\n )}\n
\n \n\n
\n {onPlayground && (\n \n )}\n {onEdit && (\n \n )}\n setShowDeleteDialog(true)}\n >\n \n \n
\n
\n \n\n \n \n
\n

{t.common.title}

\n

{transformation.title || t.sources.untitledSource}

\n
\n\n {transformation.description && (\n
\n

{t.common.description}

\n

{transformation.description}

\n
\n )}\n\n
\n

{t.transformations.systemPrompt}

\n 
\n                  {transformation.prompt}\n
\n
\n \n \n \n \n\n \n \n )\n}\n" }, { "path": "frontend/src/app/(dashboard)/transformations/components/TransformationEditorDialog.tsx", "content": "'use client'\n\nimport { useEffect, useId } from 'react'\nimport { Controller, useForm } from 'react-hook-form'\nimport { zodResolver } from '@hookform/resolvers/zod'\nimport { z } from 'zod'\nimport { Dialog, DialogContent, DialogTitle, DialogDescription } from '@/components/ui/dialog'\nimport { Button } from '@/components/ui/button'\nimport { Input } from '@/components/ui/input'\nimport { Textarea } from '@/components/ui/textarea'\nimport { Checkbox } from '@/components/ui/checkbox'\nimport { Label } from '@/components/ui/label'\nimport { MarkdownEditor } from '@/components/ui/markdown-editor'\nimport { useCreateTransformation, useUpdateTransformation, useTransformation } from '@/lib/hooks/use-transformations'\nimport { Transformation } from '@/lib/types/transformations'\nimport { useQueryClient } from '@tanstack/react-query'\nimport { TRANSFORMATION_QUERY_KEYS } from '@/lib/hooks/use-transformations'\nimport { useTranslation } from '@/lib/hooks/use-translation'\n\nconst transformationSchema = z.object({\n name: z.string().min(1),\n title: z.string().min(1),\n description: z.string().optional(),\n prompt: z.string().min(1),\n apply_default: z.boolean().optional(),\n})\n\ntype TransformationFormData = z.infer\n\ninterface TransformationEditorDialogProps {\n open: boolean\n onOpenChange: (open: boolean) => void\n transformation?: Transformation\n}\n\nexport function TransformationEditorDialog({ open, onOpenChange, transformation }: TransformationEditorDialogProps) {\n const { t } = useTranslation()\n const nameId = useId()\n const titleId = useId()\n const defaultId = useId()\n const descriptionId = useId()\n const promptId = useId()\n const isEditing = Boolean(transformation)\n const { data: fetchedTransformation, isLoading } = useTransformation(transformation?.id ?? '', {\n enabled: open && Boolean(transformation?.id),\n })\n const createTransformation = useCreateTransformation()\n const updateTransformation = useUpdateTransformation()\n const queryClient = useQueryClient()\n\n const {\n control,\n handleSubmit,\n formState: { errors },\n reset,\n } = useForm({\n resolver: zodResolver(transformationSchema),\n defaultValues: {\n name: '',\n title: '',\n description: '',\n prompt: '',\n apply_default: false,\n },\n })\n\n useEffect(() => {\n if (!open) {\n reset({ name: '', title: '', description: '', prompt: '', apply_default: false })\n return\n }\n\n const source = fetchedTransformation ?? transformation\n reset({\n name: source?.name ?? '',\n title: source?.title ?? '',\n description: source?.description ?? '',\n prompt: source?.prompt ?? '',\n apply_default: source?.apply_default ?? false,\n })\n }, [open, transformation, fetchedTransformation, reset])\n\n const onSubmit = async (data: TransformationFormData) => {\n if (transformation) {\n await updateTransformation.mutateAsync({\n id: transformation.id,\n data: {\n name: data.name,\n title: data.title || undefined,\n description: data.description || undefined,\n prompt: data.prompt,\n apply_default: Boolean(data.apply_default),\n },\n })\n queryClient.invalidateQueries({ queryKey: TRANSFORMATION_QUERY_KEYS.transformation(transformation.id) })\n } else {\n await createTransformation.mutateAsync({\n name: data.name,\n title: data.title || data.name,\n description: data.description || '',\n prompt: data.prompt,\n apply_default: Boolean(data.apply_default),\n })\n }\n\n reset()\n onOpenChange(false)\n }\n\n const handleClose = () => {\n reset()\n onOpenChange(false)\n }\n\n const isSaving = transformation ? updateTransformation.isPending : createTransformation.isPending\n\n return (\n \n \n \n {isEditing ? t.common.edit : t.transformations.createNew}\n \n \n {isEditing ? t.common.editTransformation : t.transformations.createNew}\n \n
\n {isEditing && isLoading ? (\n
\n {t.common.loading}\n
\n ) : (\n <>\n
\n
\n \n (\n \n )}\n />\n {errors.name && (\n

{errors.name.message}

\n )}\n
\n\n
\n
\n \n (\n \n )}\n />\n
\n
\n (\n field.onChange(Boolean(checked))}\n />\n )}\n />\n \n
\n
\n\n
\n \n (\n \n )}\n />\n
\n
\n\n
\n \n (\n \n )}\n />\n {errors.prompt && (\n

{errors.prompt.message}

\n )}\n

\n {t.transformations.promptHint}\n

\n
\n \n )}\n\n
\n \n \n
\n
\n \n \n )\n}\n" }, { "path": "frontend/src/app/(dashboard)/transformations/components/TransformationPlayground.tsx", "content": "'use client'\n\nimport { useState } from 'react'\nimport { Card, CardContent, CardDescription, CardHeader, CardTitle } from '@/components/ui/card'\nimport { Button } from '@/components/ui/button'\nimport { Textarea } from '@/components/ui/textarea'\nimport { Label } from '@/components/ui/label'\nimport { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from '@/components/ui/select'\nimport { ScrollArea } from '@/components/ui/scroll-area'\nimport { Play, Loader2 } from 'lucide-react'\nimport { Transformation } from '@/lib/types/transformations'\nimport { useExecuteTransformation } from '@/lib/hooks/use-transformations'\nimport { ModelSelector } from '@/components/common/ModelSelector'\nimport { useTranslation } from '@/lib/hooks/use-translation'\nimport ReactMarkdown from 'react-markdown'\nimport remarkGfm from 'remark-gfm'\n\ninterface TransformationPlaygroundProps {\n transformations: Transformation[] | undefined\n selectedTransformation?: Transformation\n}\n\nexport function TransformationPlayground({ transformations, selectedTransformation }: TransformationPlaygroundProps) {\n const { t } = useTranslation()\n const [selectedId, setSelectedId] = useState(selectedTransformation?.id || '')\n const [inputText, setInputText] = useState('')\n const [modelId, setModelId] = useState('')\n const [output, setOutput] = useState('')\n \n const executeTransformation = useExecuteTransformation()\n\n const handleExecute = async () => {\n if (!selectedId || !modelId || !inputText.trim()) {\n return\n }\n\n const result = await executeTransformation.mutateAsync({\n transformation_id: selectedId,\n input_text: inputText,\n model_id: modelId\n })\n\n setOutput(result.output)\n }\n\n const canExecute = selectedId && modelId && inputText.trim() && !executeTransformation.isPending\n\n return (\n
\n \n \n {t.transformations.playground}\n \n {t.transformations.desc}\n \n \n \n
\n
\n \n \n
\n\n
\n \n
\n
\n\n
\n \n setInputText(e.target.value)}\n placeholder={t.transformations.inputPlaceholder}\n rows={8}\n className=\"font-mono text-sm\"\n />\n
\n\n
\n \n
\n\n {output && (\n
\n {t.transformations.outputLabel}\n \n \n \n
\n (\n
\n {children}
\n
\n ),\n thead: ({ children }) => {children},\n tbody: ({ children }) => {children},\n tr: ({ children }) => {children},\n th: ({ children }) => {children},\n td: ({ children }) => {children},\n }}\n >\n {output}\n \n
\n \n \n \n
\n )}\n \n \n
\n )\n}" }, { "path": "frontend/src/app/(dashboard)/transformations/components/TransformationsList.tsx", "content": "'use client'\n\nimport { useState } from 'react'\nimport { Button } from '@/components/ui/button'\nimport { Plus } from 'lucide-react'\nimport { TransformationCard } from './TransformationCard'\nimport { EmptyState } from '@/components/common/EmptyState'\nimport { LoadingSpinner } from '@/components/common/LoadingSpinner'\nimport { Wand2 } from 'lucide-react'\nimport { Transformation } from '@/lib/types/transformations'\nimport { TransformationEditorDialog } from './TransformationEditorDialog'\nimport { useTranslation } from '@/lib/hooks/use-translation'\n\ninterface TransformationsListProps {\n transformations: Transformation[] | undefined\n isLoading: boolean\n onPlayground?: (transformation: Transformation) => void\n}\n\nexport function TransformationsList({ transformations, isLoading, onPlayground }: TransformationsListProps) {\n const { t } = useTranslation()\n const [editorOpen, setEditorOpen] = useState(false)\n const [editingTransformation, setEditingTransformation] = useState()\n\n const handleOpenEditor = (trans?: Transformation) => {\n setEditingTransformation(trans)\n setEditorOpen(true)\n }\n\n if (isLoading) {\n return (\n
\n \n
\n )\n }\n\n if (!transformations || transformations.length === 0) {\n return (\n handleOpenEditor()}>\n \n {t.transformations.createNew}\n \n }\n />\n )\n }\n\n return (\n <>\n
\n
\n

{t.transformations.listTitle}

\n \n
\n\n
\n {transformations.map((transformation) => (\n onPlayground(transformation) : undefined}\n onEdit={() => handleOpenEditor(transformation)}\n />\n ))}\n
\n
\n\n {\n setEditorOpen(open)\n if (!open) {\n setEditingTransformation(undefined)\n }\n }}\n transformation={editingTransformation}\n />\n \n )\n}\n" }, { "path": "frontend/src/app/(dashboard)/transformations/page.tsx", "content": "'use client'\n\nimport { useState } from 'react'\nimport { AppShell } from '@/components/layout/AppShell'\nimport { Button } from '@/components/ui/button'\nimport { Tabs, TabsContent, TabsList, TabsTrigger } from '@/components/ui/tabs'\nimport { DefaultPromptEditor } from './components/DefaultPromptEditor'\nimport { TransformationsList } from './components/TransformationsList'\nimport { TransformationPlayground } from './components/TransformationPlayground'\nimport { useTransformations } from '@/lib/hooks/use-transformations'\nimport { Transformation } from '@/lib/types/transformations'\nimport { Wand2, Play, RefreshCw } from 'lucide-react'\nimport { useTranslation } from '@/lib/hooks/use-translation'\n\nexport default function TransformationsPage() {\n const { t } = useTranslation()\n const [activeTab, setActiveTab] = useState('transformations')\n const [selectedTransformation, setSelectedTransformation] = useState()\n const { data: transformations, isLoading, refetch } = useTransformations()\n\n const handlePlayground = (transformation: Transformation) => {\n setSelectedTransformation(transformation)\n setActiveTab('playground')\n }\n\n return (\n \n
\n
\n
\n
\n

{t.transformations.title}

\n \n
\n
\n\n
\n

\n {t.transformations.desc}\n

\n
\n\n \n
\n

{t.transformations.workspace}

\n \n \n \n {t.transformations.title}\n \n \n \n {t.transformations.playground}\n \n \n
\n \n \n \n \n \n \n \n \n \n \n
\n
\n \n )\n}\n" }, { "path": "frontend/src/app/config/route.ts", "content": "import { NextRequest, NextResponse } from 'next/server'\n\n/**\n * Runtime Configuration Endpoint\n *\n * This endpoint provides server-side environment variables to the client at runtime.\n * This solves the NEXT_PUBLIC_* limitation where variables are baked into the build.\n *\n * Environment Variables:\n * - API_URL: Where the browser/client should make API requests (public/external URL)\n * - INTERNAL_API_URL: Where Next.js server-side should proxy API requests (internal URL)\n * Default: http://localhost:5055 (used by Next.js rewrites in next.config.ts)\n *\n * Why two different variables?\n * - API_URL: Used by browser clients, can be https://your-domain.com or http://server-ip:5055\n * - INTERNAL_API_URL: Used by Next.js rewrites for server-side proxying, typically http://localhost:5055\n *\n * Auto-detection logic for API_URL:\n * 1. If API_URL env var is set, use it (explicit override)\n * 2. Otherwise, detect from incoming HTTP request headers (zero-config)\n * 3. Fallback to localhost:5055 if detection fails\n *\n * This allows the same Docker image to work in different deployment scenarios.\n */\nexport async function GET(request: NextRequest) {\n // Priority 1: Check if API_URL is explicitly set\n const envApiUrl = process.env.API_URL || process.env.NEXT_PUBLIC_API_URL\n\n if (envApiUrl) {\n return NextResponse.json({\n apiUrl: envApiUrl,\n })\n }\n\n // Priority 2: Auto-detect from request headers\n try {\n // Get the protocol (http or https)\n // Check X-Forwarded-Proto first (for reverse proxies), then fallback to request scheme\n const proto = request.headers.get('x-forwarded-proto') ||\n request.nextUrl.protocol.replace(':', '') ||\n 'http'\n\n // Get the host header (includes port if non-standard)\n const hostHeader = request.headers.get('host')\n\n if (hostHeader) {\n // Extract just the hostname (remove port if present)\n const hostname = hostHeader.split(':')[0]\n\n // Construct the API URL with port 5055\n const apiUrl = `${proto}://${hostname}:5055`\n\n console.log(`[runtime-config] Auto-detected API URL: ${apiUrl} (proto=${proto}, host=${hostHeader})`)\n\n return NextResponse.json({\n apiUrl,\n })\n }\n } catch (error) {\n console.error('[runtime-config] Auto-detection failed:', error)\n }\n\n // Priority 3: Fallback to localhost\n console.log('[runtime-config] Using fallback: http://localhost:5055')\n return NextResponse.json({\n apiUrl: 'http://localhost:5055',\n })\n}\n" }, { "path": "frontend/src/app/globals.css", "content": "@import \"tailwindcss\";\n@import \"tw-animate-css\";\n\n@theme inline {\n --color-background: var(--background);\n --color-foreground: var(--foreground);\n --font-sans: var(--font-geist-sans);\n --font-mono: var(--font-geist-mono);\n --color-sidebar-ring: var(--sidebar-ring);\n --color-sidebar-border: var(--sidebar-border);\n --color-sidebar-accent-foreground: var(--sidebar-accent-foreground);\n --color-sidebar-accent: var(--sidebar-accent);\n --color-sidebar-primary-foreground: var(--sidebar-primary-foreground);\n --color-sidebar-primary: var(--sidebar-primary);\n --color-sidebar-foreground: var(--sidebar-foreground);\n --color-sidebar: var(--sidebar);\n --color-chart-5: var(--chart-5);\n --color-chart-4: var(--chart-4);\n --color-chart-3: var(--chart-3);\n --color-chart-2: var(--chart-2);\n --color-chart-1: var(--chart-1);\n --color-ring: var(--ring);\n --color-input: var(--input);\n --color-border: var(--border);\n --color-destructive: var(--destructive);\n --color-accent-foreground: var(--accent-foreground);\n --color-accent: var(--accent);\n --color-muted-foreground: var(--muted-foreground);\n --color-muted: var(--muted);\n --color-secondary-foreground: var(--secondary-foreground);\n --color-secondary: var(--secondary);\n --color-primary-foreground: var(--primary-foreground);\n --color-primary: var(--primary);\n --color-popover-foreground: var(--popover-foreground);\n --color-popover: var(--popover);\n --color-card-foreground: var(--card-foreground);\n --color-card: var(--card);\n --radius-sm: calc(var(--radius) - 4px);\n --radius-md: calc(var(--radius) - 2px);\n --radius-lg: var(--radius);\n --radius-xl: calc(var(--radius) + 4px);\n}\n\n:root {\n --radius: 0.65rem;\n --background: oklch(1 0 0);\n --foreground: oklch(0.141 0.005 285.823);\n --card: oklch(1 0 0);\n --card-foreground: oklch(0.141 0.005 285.823);\n --popover: oklch(1 0 0);\n --popover-foreground: oklch(0.141 0.005 285.823);\n --primary: oklch(0.623 0.214 259.815);\n --primary-foreground: oklch(0.97 0.014 254.604);\n --secondary: oklch(0.967 0.001 286.375);\n --secondary-foreground: oklch(0.21 0.006 285.885);\n --muted: oklch(0.967 0.001 286.375);\n --muted-foreground: oklch(0.552 0.016 285.938);\n --accent: oklch(0.967 0.001 286.375);\n --accent-foreground: oklch(0.21 0.006 285.885);\n --destructive: oklch(0.577 0.245 27.325);\n --border: oklch(0.92 0.004 286.32);\n --input: oklch(0.92 0.004 286.32);\n --ring: oklch(0.623 0.214 259.815);\n --chart-1: oklch(0.646 0.222 41.116);\n --chart-2: oklch(0.6 0.118 184.704);\n --chart-3: oklch(0.398 0.07 227.392);\n --chart-4: oklch(0.828 0.189 84.429);\n --chart-5: oklch(0.769 0.188 70.08);\n --sidebar: oklch(0.985 0 0);\n --sidebar-foreground: oklch(0.141 0.005 285.823);\n --sidebar-primary: oklch(0.623 0.214 259.815);\n --sidebar-primary-foreground: oklch(0.97 0.014 254.604);\n --sidebar-accent: oklch(0.92 0.01 286.375);\n --sidebar-accent-foreground: oklch(0.21 0.006 285.885);\n --sidebar-border: oklch(0.92 0.004 286.32);\n --sidebar-ring: oklch(0.623 0.214 259.815);\n}\n\n.dark {\n --background: oklch(0.141 0.005 285.823);\n --foreground: oklch(0.985 0 0);\n --card: oklch(0.21 0.006 285.885);\n --card-foreground: oklch(0.985 0 0);\n --popover: oklch(0.21 0.006 285.885);\n --popover-foreground: oklch(0.985 0 0);\n --primary: oklch(0.546 0.245 262.881);\n --primary-foreground: oklch(0.985 0 0);\n --secondary: oklch(0.274 0.006 286.033);\n --secondary-foreground: oklch(0.985 0 0);\n --muted: oklch(0.274 0.006 286.033);\n --muted-foreground: oklch(0.705 0.015 286.067);\n --accent: oklch(0.274 0.006 286.033);\n --accent-foreground: oklch(0.985 0 0);\n --destructive: oklch(0.704 0.191 22.216);\n --border: oklch(1 0 0 / 10%);\n --input: oklch(1 0 0 / 15%);\n --ring: oklch(0.488 0.243 264.376);\n --chart-1: oklch(0.488 0.243 264.376);\n --chart-2: oklch(0.696 0.17 162.48);\n --chart-3: oklch(0.769 0.188 70.08);\n --chart-4: oklch(0.627 0.265 303.9);\n --chart-5: oklch(0.645 0.246 16.439);\n --sidebar: oklch(0.21 0.006 285.885);\n --sidebar-foreground: oklch(0.985 0 0);\n --sidebar-primary: oklch(0.546 0.245 262.881);\n --sidebar-primary-foreground: oklch(0.985 0 0);\n --sidebar-accent: oklch(0.35 0.01 286.033);\n --sidebar-accent-foreground: oklch(0.985 0 0);\n --sidebar-border: oklch(1 0 0 / 10%);\n --sidebar-ring: oklch(0.488 0.243 264.376);\n}\n\n@layer base {\n * {\n @apply border-border outline-ring/50;\n }\n\n html {\n @apply antialiased;\n }\n\n body {\n @apply bg-background text-foreground transition-colors;\n }\n\n /* Ensure proper theme inheritance for popovers and dropdowns */\n .dark {\n color-scheme: dark;\n }\n\n :root {\n color-scheme: light;\n }\n\n /* Ensure Radix UI components inherit theme properly */\n [data-radix-popper-content-wrapper] {\n @apply z-50;\n }\n\n /* Force theme inheritance for portaled content */\n .dark [data-radix-popper-content-wrapper],\n .dark [data-overlay-container] {\n color-scheme: dark;\n }\n\n /* Ensure sidebar gets proper theme */\n .app-sidebar {\n background-color: var(--sidebar);\n color: var(--sidebar-foreground);\n border-color: var(--sidebar-border);\n }\n\n /* Enhanced sidebar menu item hover effects */\n .sidebar-menu-item {\n @apply transition-all duration-200 ease-out;\n }\n\n .sidebar-menu-item:hover {\n @apply scale-[1.02];\n background-color: var(--sidebar-accent) !important;\n box-shadow: 0 2px 8px rgba(0, 0, 0, 0.08);\n }\n\n .dark .sidebar-menu-item:hover {\n box-shadow: 0 2px 8px rgba(0, 0, 0, 0.25);\n }\n\n /* Enhanced hover effects for cards */\n .card-hover {\n @apply transition-all duration-200 cursor-pointer;\n }\n\n .card-hover:hover {\n background-color: var(--muted) !important;\n border-color: var(--border);\n transform: translateY(-1px);\n box-shadow: 0 4px 12px rgba(0, 0, 0, 0.1);\n }\n\n .dark .card-hover:hover {\n box-shadow: 0 4px 12px rgba(0, 0, 0, 0.3);\n }\n\n /* Ensure clickable cards show pointer cursor */\n .clickable-card {\n cursor: pointer !important;\n }\n\n .clickable-card * {\n cursor: pointer !important;\n }\n}\n" }, { "path": "frontend/src/app/layout.tsx", "content": "import type { Metadata } from \"next\";\nimport { Inter } from \"next/font/google\";\nimport \"./globals.css\";\nimport { Toaster } from \"@/components/ui/sonner\";\nimport { QueryProvider } from \"@/components/providers/QueryProvider\";\nimport { ThemeProvider } from \"@/components/providers/ThemeProvider\";\nimport { ErrorBoundary } from \"@/components/common/ErrorBoundary\";\nimport { ConnectionGuard } from \"@/components/common/ConnectionGuard\";\nimport { themeScript } from \"@/lib/theme-script\";\nimport { I18nProvider } from \"@/components/providers/I18nProvider\";\n\nconst inter = Inter({ subsets: [\"latin\"] });\n\nexport const metadata: Metadata = {\n title: \"Open Notebook\",\n description: \"Privacy-focused research and knowledge management\",\n};\n\nexport default function RootLayout({\n children,\n}: Readonly<{\n children: React.ReactNode;\n}>) {\n return (\n \n \n