Repository: lfnovo/open-notebook Branch: main Commit: a42e2a347ee2 Files: 446 Total size: 2.7 MB Directory structure: gitextract_5wx36h6b/ ├── .dockerignore ├── .github/ │ ├── ISSUE_TEMPLATE/ │ │ ├── bug_report.yml │ │ ├── config.yml │ │ ├── feature_request.yml │ │ └── installation_issue.yml │ ├── pull_request_template.md │ └── workflows/ │ ├── build-and-release.yml │ ├── build-dev.yml │ ├── claude-code-review.yml │ ├── claude.yml │ └── test.yml ├── .gitignore ├── .python-version ├── .worktreeinclude ├── CHANGELOG.md ├── CLAUDE.md ├── CONFIGURATION.md ├── CONTRIBUTING.md ├── Dockerfile ├── Dockerfile.single ├── LICENSE ├── MAINTAINER_GUIDE.md ├── Makefile ├── README.dev.md ├── README.md ├── api/ │ ├── CLAUDE.md │ ├── __init__.py │ ├── auth.py │ ├── chat_service.py │ ├── client.py │ ├── command_service.py │ ├── context_service.py │ ├── credentials_service.py │ ├── embedding_service.py │ ├── episode_profiles_service.py │ ├── insights_service.py │ ├── main.py │ ├── models.py │ ├── models_service.py │ ├── notebook_service.py │ ├── notes_service.py │ ├── podcast_api_service.py │ ├── podcast_service.py │ ├── routers/ │ │ ├── __init__.py │ │ ├── auth.py │ │ ├── chat.py │ │ ├── commands.py │ │ ├── config.py │ │ ├── context.py │ │ ├── credentials.py │ │ ├── embedding.py │ │ ├── embedding_rebuild.py │ │ ├── episode_profiles.py │ │ ├── insights.py │ │ ├── languages.py │ │ ├── models.py │ │ ├── notebooks.py │ │ ├── notes.py │ │ ├── podcasts.py │ │ ├── search.py │ │ ├── settings.py │ │ ├── source_chat.py │ │ ├── sources.py │ │ ├── speaker_profiles.py │ │ └── transformations.py │ ├── search_service.py │ ├── settings_service.py │ ├── sources_service.py │ └── transformations_service.py ├── commands/ │ ├── CLAUDE.md │ ├── __init__.py │ ├── embedding_commands.py │ ├── example_commands.py │ ├── podcast_commands.py │ └── source_commands.py ├── docker-compose.yml ├── docs/ │ ├── 0-START-HERE/ │ │ ├── index.md │ │ ├── quick-start-cloud.md │ │ ├── quick-start-local.md │ │ └── quick-start-openai.md │ ├── 1-INSTALLATION/ │ │ ├── docker-compose.md │ │ ├── from-source.md │ │ ├── index.md │ │ └── single-container.md │ ├── 2-CORE-CONCEPTS/ │ │ ├── ai-context-rag.md │ │ ├── chat-vs-transformations.md │ │ ├── index.md │ │ ├── notebooks-sources-notes.md │ │ └── podcasts-explained.md │ ├── 3-USER-GUIDE/ │ │ ├── adding-sources.md │ │ ├── api-configuration.md │ │ ├── chat-effectively.md │ │ ├── citations.md │ │ ├── creating-podcasts.md │ │ ├── index.md │ │ ├── interface-overview.md │ │ ├── search.md │ │ ├── transformations.md │ │ └── working-with-notes.md │ ├── 4-AI-PROVIDERS/ │ │ └── index.md │ ├── 5-CONFIGURATION/ │ │ ├── advanced.md │ │ ├── ai-providers.md │ │ ├── database.md │ │ ├── environment-reference.md │ │ ├── index.md │ │ ├── local-stt.md │ │ ├── local-tts.md │ │ ├── mcp-integration.md │ │ ├── ollama.md │ │ ├── openai-compatible.md │ │ ├── reverse-proxy.md │ │ └── security.md │ ├── 6-TROUBLESHOOTING/ │ │ ├── ai-chat-issues.md │ │ ├── connection-issues.md │ │ ├── faq.md │ │ ├── index.md │ │ └── quick-fixes.md │ ├── 7-DEVELOPMENT/ │ │ ├── api-reference.md │ │ ├── architecture.md │ │ ├── code-standards.md │ │ ├── contributing.md │ │ ├── design-principles.md │ │ ├── development-setup.md │ │ ├── index.md │ │ ├── maintainer-guide.md │ │ ├── quick-start.md │ │ └── testing.md │ ├── SECURITY_REVIEW.md │ └── index.md ├── examples/ │ ├── README.md │ ├── docker-compose-dev.yml │ ├── docker-compose-full-local.yml │ ├── docker-compose-ollama.yml │ ├── docker-compose-single.yml │ └── docker-compose-speaches.yml ├── frontend/ │ ├── .gitignore │ ├── components.json │ ├── eslint.config.mjs │ ├── next.config.ts │ ├── package.json │ ├── postcss.config.mjs │ ├── src/ │ │ ├── CLAUDE.md │ │ ├── app/ │ │ │ ├── (auth)/ │ │ │ │ └── login/ │ │ │ │ └── page.tsx │ │ │ ├── (dashboard)/ │ │ │ │ ├── advanced/ │ │ │ │ │ ├── components/ │ │ │ │ │ │ ├── RebuildEmbeddings.tsx │ │ │ │ │ │ └── SystemInfo.tsx │ │ │ │ │ └── page.tsx │ │ │ │ ├── layout.tsx │ │ │ │ ├── notebooks/ │ │ │ │ │ ├── [id]/ │ │ │ │ │ │ └── page.tsx │ │ │ │ │ ├── components/ │ │ │ │ │ │ ├── ChatColumn.test.tsx │ │ │ │ │ │ ├── ChatColumn.tsx │ │ │ │ │ │ ├── NoteEditorDialog.tsx │ │ │ │ │ │ ├── NotebookCard.tsx │ │ │ │ │ │ ├── NotebookDeleteDialog.tsx │ │ │ │ │ │ ├── NotebookHeader.tsx │ │ │ │ │ │ ├── NotebookList.tsx │ │ │ │ │ │ ├── NotesColumn.tsx │ │ │ │ │ │ └── SourcesColumn.tsx │ │ │ │ │ └── page.tsx │ │ │ │ ├── page.tsx │ │ │ │ ├── podcasts/ │ │ │ │ │ └── page.tsx │ │ │ │ ├── search/ │ │ │ │ │ └── page.tsx │ │ │ │ ├── settings/ │ │ │ │ │ ├── api-keys/ │ │ │ │ │ │ └── page.tsx │ │ │ │ │ ├── components/ │ │ │ │ │ │ └── SettingsForm.tsx │ │ │ │ │ └── page.tsx │ │ │ │ ├── sources/ │ │ │ │ │ ├── [id]/ │ │ │ │ │ │ └── page.tsx │ │ │ │ │ └── page.tsx │ │ │ │ └── transformations/ │ │ │ │ ├── components/ │ │ │ │ │ ├── DefaultPromptEditor.tsx │ │ │ │ │ ├── TransformationCard.tsx │ │ │ │ │ ├── TransformationEditorDialog.tsx │ │ │ │ │ ├── TransformationPlayground.tsx │ │ │ │ │ └── TransformationsList.tsx │ │ │ │ └── page.tsx │ │ │ ├── config/ │ │ │ │ └── route.ts │ │ │ ├── globals.css │ │ │ ├── layout.tsx │ │ │ └── page.tsx │ │ ├── components/ │ │ │ ├── auth/ │ │ │ │ └── LoginForm.tsx │ │ │ ├── common/ │ │ │ │ ├── CommandPalette.tsx │ │ │ │ ├── ConfirmDialog.test.tsx │ │ │ │ ├── ConfirmDialog.tsx │ │ │ │ ├── ConnectionGuard.tsx │ │ │ │ ├── ContextIndicator.tsx │ │ │ │ ├── ContextToggle.tsx │ │ │ │ ├── EmptyState.tsx │ │ │ │ ├── ErrorBoundary.tsx │ │ │ │ ├── InlineEdit.tsx │ │ │ │ ├── LanguageLoadingOverlay.tsx │ │ │ │ ├── LanguageToggle.tsx │ │ │ │ ├── LoadingSpinner.tsx │ │ │ │ ├── ModelSelector.tsx │ │ │ │ └── ThemeToggle.tsx │ │ │ ├── errors/ │ │ │ │ └── ConnectionErrorOverlay.tsx │ │ │ ├── layout/ │ │ │ │ ├── AppShell.tsx │ │ │ │ ├── AppSidebar.test.tsx │ │ │ │ ├── AppSidebar.tsx │ │ │ │ └── SetupBanner.tsx │ │ │ ├── notebooks/ │ │ │ │ ├── CollapsibleColumn.tsx │ │ │ │ └── CreateNotebookDialog.tsx │ │ │ ├── podcasts/ │ │ │ │ ├── EpisodeCard.tsx │ │ │ │ ├── EpisodeProfilesPanel.tsx │ │ │ │ ├── EpisodesTab.tsx │ │ │ │ ├── GeneratePodcastDialog.tsx │ │ │ │ ├── SpeakerProfilesPanel.tsx │ │ │ │ ├── TemplatesTab.tsx │ │ │ │ └── forms/ │ │ │ │ ├── EpisodeProfileFormDialog.tsx │ │ │ │ └── SpeakerProfileFormDialog.tsx │ │ │ ├── providers/ │ │ │ │ ├── I18nProvider.tsx │ │ │ │ ├── ModalProvider.tsx │ │ │ │ ├── QueryProvider.tsx │ │ │ │ └── ThemeProvider.tsx │ │ │ ├── search/ │ │ │ │ ├── AdvancedModelsDialog.tsx │ │ │ │ ├── SaveToNotebooksDialog.tsx │ │ │ │ └── StreamingResponse.tsx │ │ │ ├── settings/ │ │ │ │ ├── EmbeddingModelChangeDialog.tsx │ │ │ │ ├── MigrationBanner.tsx │ │ │ │ ├── ModelTestResultDialog.tsx │ │ │ │ └── index.ts │ │ │ ├── source/ │ │ │ │ ├── ChatPanel.tsx │ │ │ │ ├── MessageActions.tsx │ │ │ │ ├── ModelSelector.tsx │ │ │ │ ├── NotebookAssociations.tsx │ │ │ │ ├── SessionManager.tsx │ │ │ │ ├── SourceDetailContent.tsx │ │ │ │ ├── SourceDialog.tsx │ │ │ │ └── SourceInsightDialog.tsx │ │ │ ├── sources/ │ │ │ │ ├── AddExistingSourceDialog.tsx │ │ │ │ ├── AddSourceButton.tsx │ │ │ │ ├── AddSourceDialog.tsx │ │ │ │ ├── README.md │ │ │ │ ├── SourceCard.tsx │ │ │ │ ├── index.ts │ │ │ │ └── steps/ │ │ │ │ ├── NotebooksStep.tsx │ │ │ │ ├── ProcessingStep.tsx │ │ │ │ └── SourceTypeStep.tsx │ │ │ └── ui/ │ │ │ ├── CLAUDE.md │ │ │ ├── accordion.tsx │ │ │ ├── alert-dialog.tsx │ │ │ ├── alert.tsx │ │ │ ├── badge.tsx │ │ │ ├── button.tsx │ │ │ ├── card.tsx │ │ │ ├── checkbox-list.tsx │ │ │ ├── checkbox.tsx │ │ │ ├── collapsible.tsx │ │ │ ├── command.tsx │ │ │ ├── dialog.tsx │ │ │ ├── dropdown-menu.tsx │ │ │ ├── form-section.tsx │ │ │ ├── input.tsx │ │ │ ├── label.tsx │ │ │ ├── markdown-editor.tsx │ │ │ ├── popover.tsx │ │ │ ├── progress.tsx │ │ │ ├── radio-group.tsx │ │ │ ├── scroll-area.tsx │ │ │ ├── select.tsx │ │ │ ├── separator.tsx │ │ │ ├── sonner.tsx │ │ │ ├── tabs.tsx │ │ │ ├── textarea.tsx │ │ │ ├── tooltip.tsx │ │ │ └── wizard-container.tsx │ │ ├── lib/ │ │ │ ├── api/ │ │ │ │ ├── CLAUDE.md │ │ │ │ ├── chat.ts │ │ │ │ ├── client.ts │ │ │ │ ├── credentials.ts │ │ │ │ ├── embedding.ts │ │ │ │ ├── insights.ts │ │ │ │ ├── models.ts │ │ │ │ ├── notebooks.ts │ │ │ │ ├── notes.ts │ │ │ │ ├── podcasts.ts │ │ │ │ ├── query-client.ts │ │ │ │ ├── search.ts │ │ │ │ ├── settings.ts │ │ │ │ ├── source-chat.ts │ │ │ │ ├── sources.ts │ │ │ │ └── transformations.ts │ │ │ ├── config.test.ts │ │ │ ├── config.ts │ │ │ ├── hooks/ │ │ │ │ ├── CLAUDE.md │ │ │ │ ├── use-ask.ts │ │ │ │ ├── use-auth.ts │ │ │ │ ├── use-create-dialogs.tsx │ │ │ │ ├── use-credentials.ts │ │ │ │ ├── use-insights.ts │ │ │ │ ├── use-media-query.ts │ │ │ │ ├── use-modal-manager.test.ts │ │ │ │ ├── use-modal-manager.ts │ │ │ │ ├── use-models.ts │ │ │ │ ├── use-navigation.ts │ │ │ │ ├── use-notebooks.ts │ │ │ │ ├── use-notes.ts │ │ │ │ ├── use-podcasts.ts │ │ │ │ ├── use-search.ts │ │ │ │ ├── use-settings.ts │ │ │ │ ├── use-sources.ts │ │ │ │ ├── use-toast.ts │ │ │ │ ├── use-transformations.ts │ │ │ │ ├── use-translation.test.ts │ │ │ │ ├── use-translation.ts │ │ │ │ ├── use-version-check.ts │ │ │ │ ├── useNotebookChat.ts │ │ │ │ └── useSourceChat.ts │ │ │ ├── i18n-events.ts │ │ │ ├── i18n.ts │ │ │ ├── locales/ │ │ │ │ ├── CLAUDE.md │ │ │ │ ├── bn-IN/ │ │ │ │ │ └── index.ts │ │ │ │ ├── en-US/ │ │ │ │ │ └── index.ts │ │ │ │ ├── fr-FR/ │ │ │ │ │ └── index.ts │ │ │ │ ├── index.test.ts │ │ │ │ ├── index.ts │ │ │ │ ├── it-IT/ │ │ │ │ │ └── index.ts │ │ │ │ ├── ja-JP/ │ │ │ │ │ └── index.ts │ │ │ │ ├── pt-BR/ │ │ │ │ │ └── index.ts │ │ │ │ ├── ru-RU/ │ │ │ │ │ └── index.ts │ │ │ │ ├── zh-CN/ │ │ │ │ │ └── index.ts │ │ │ │ └── zh-TW/ │ │ │ │ └── index.ts │ │ │ ├── stores/ │ │ │ │ ├── CLAUDE.md │ │ │ │ ├── auth-store.ts │ │ │ │ ├── navigation-store.ts │ │ │ │ ├── notebook-columns-store.ts │ │ │ │ ├── sidebar-store.ts │ │ │ │ └── theme-store.ts │ │ │ ├── theme-script.ts │ │ │ ├── types/ │ │ │ │ ├── api.ts │ │ │ │ ├── auth.ts │ │ │ │ ├── common.ts │ │ │ │ ├── config.ts │ │ │ │ ├── models.ts │ │ │ │ ├── podcasts.ts │ │ │ │ ├── search.ts │ │ │ │ └── transformations.ts │ │ │ ├── utils/ │ │ │ │ ├── date-locale.ts │ │ │ │ ├── error-handler.ts │ │ │ │ └── source-references.tsx │ │ │ └── utils.ts │ │ ├── proxy.ts │ │ └── test/ │ │ ├── jest-dom.d.ts │ │ └── setup.ts │ ├── start-server.js │ ├── tailwind.config.ts │ ├── tsconfig.json │ └── vitest.config.ts ├── mypy.ini ├── open_notebook/ │ ├── CLAUDE.md │ ├── __init__.py │ ├── ai/ │ │ ├── CLAUDE.md │ │ ├── __init__.py │ │ ├── connection_tester.py │ │ ├── key_provider.py │ │ ├── model_discovery.py │ │ ├── models.py │ │ └── provision.py │ ├── config.py │ ├── database/ │ │ ├── CLAUDE.md │ │ ├── async_migrate.py │ │ ├── migrate.py │ │ ├── migrations/ │ │ │ ├── 1.surrealql │ │ │ ├── 10.surrealql │ │ │ ├── 10_down.surrealql │ │ │ ├── 11.surrealql │ │ │ ├── 11_down.surrealql │ │ │ ├── 12.surrealql │ │ │ ├── 12_down.surrealql │ │ │ ├── 13.surrealql │ │ │ ├── 13_down.surrealql │ │ │ ├── 14.surrealql │ │ │ ├── 14_down.surrealql │ │ │ ├── 1_down.surrealql │ │ │ ├── 2.surrealql │ │ │ ├── 2_down.surrealql │ │ │ ├── 3.surrealql │ │ │ ├── 3_down.surrealql │ │ │ ├── 4.surrealql │ │ │ ├── 4_down.surrealql │ │ │ ├── 5.surrealql │ │ │ ├── 5_down.surrealql │ │ │ ├── 6.surrealql │ │ │ ├── 6_down.surrealql │ │ │ ├── 7.surrealql │ │ │ ├── 7_down.surrealql │ │ │ ├── 8.surrealql │ │ │ ├── 8_down.surrealql │ │ │ ├── 9.surrealql │ │ │ └── 9_down.surrealql │ │ └── repository.py │ ├── domain/ │ │ ├── CLAUDE.md │ │ ├── __init__.py │ │ ├── base.py │ │ ├── content_settings.py │ │ ├── credential.py │ │ ├── notebook.py │ │ ├── provider_config.py │ │ └── transformation.py │ ├── exceptions.py │ ├── graphs/ │ │ ├── CLAUDE.md │ │ ├── ask.py │ │ ├── chat.py │ │ ├── prompt.py │ │ ├── source.py │ │ ├── source_chat.py │ │ ├── tools.py │ │ └── transformation.py │ ├── podcasts/ │ │ ├── CLAUDE.md │ │ ├── __init__.py │ │ ├── migration.py │ │ └── models.py │ └── utils/ │ ├── CLAUDE.md │ ├── README.md │ ├── __init__.py │ ├── chunking.py │ ├── context_builder.py │ ├── embedding.py │ ├── encryption.py │ ├── error_classifier.py │ ├── graph_utils.py │ ├── text_utils.py │ ├── token_utils.py │ └── version_utils.py ├── prompts/ │ ├── CLAUDE.md │ ├── ask/ │ │ ├── entry.jinja │ │ ├── final_answer.jinja │ │ └── query_process.jinja │ ├── chat/ │ │ └── system.jinja │ ├── podcast/ │ │ ├── outline.jinja │ │ └── transcript.jinja │ └── source_chat/ │ └── system.jinja ├── pyproject.toml ├── run_api.py ├── scripts/ │ ├── README.md │ ├── export_docs.py │ └── wait-for-api.sh ├── supervisord.conf ├── supervisord.single.conf └── tests/ ├── README.md ├── conftest.py ├── test_chunking.py ├── test_domain.py ├── test_embedding.py ├── test_graphs.py ├── test_models_api.py ├── test_notes_api.py ├── test_podcast_path.py ├── test_url_validation.py └── test_utils.py ================================================ FILE CONTENTS ================================================ ================================================ FILE: .dockerignore ================================================ # Git .git .gitignore # Python __pycache__ *.pyc *.pyo *.pyd .venv venv ENV env .pytest_cache .mypy_cache .ruff_cache # Frontend frontend/node_modules frontend/.next frontend/dist frontend/out frontend/.env* frontend/*.log # Project data .antigravity .gemini tmp data mydata notebook_data surreal_data surreal-data surreal_single_data *.db *.log docker.env .env docker-compose* # Documentation & CI (not needed in image) docs .github # IDE and OS files .vscode .idea *.swp *.swo *~ .DS_Store ================================================ FILE: .github/ISSUE_TEMPLATE/bug_report.yml ================================================ name: 🐛 Bug Report description: Report a bug or unexpected behavior (app is running but misbehaving) title: "[Bug]: " labels: ["bug", "needs-triage"] body: - type: markdown attributes: value: | Thanks for reporting a bug! Please fill out the information below to help us understand and fix the issue. **Note**: If you're having installation or setup issues, please use the "Installation Issue" template instead. - type: textarea id: what-happened attributes: label: What did you do when it broke? description: Describe the steps you took that led to the bug placeholder: | 1. I went to the Notebooks page 2. I clicked on "Create New Notebook" 3. I filled in the form and clicked "Save" 4. Then the error occurred... validations: required: true - type: textarea id: how-broke attributes: label: How did it break? description: What happened that was unexpected? What did you expect to happen instead? placeholder: | Expected: The notebook should be created and I should see it in the list Actual: I got an error message saying "Failed to create notebook" validations: required: true - type: textarea id: logs-screenshots attributes: label: Logs or Screenshots description: | Please provide any error messages, logs, or screenshots that might help us understand the issue. **How to get logs:** - Docker: `docker compose logs -f open_notebook` - Check browser console (F12 → Console tab) placeholder: | Paste logs here or drag and drop screenshots. Error messages, stack traces, or browser console errors are very helpful! validations: required: false - type: dropdown id: version attributes: label: Open Notebook Version description: Which version are you using? options: - v1-latest (Docker) - v1-latest-single (Docker) - Latest from main branch - Other (please specify in additional context) validations: required: true - type: textarea id: environment attributes: label: Environment description: What environment are you running in? placeholder: | - OS: Ubuntu 22.04 / Windows 11 / macOS 14 - Browser: Chrome 120 validations: required: false - type: textarea id: additional-context attributes: label: Additional Context description: Any other information that might be helpful placeholder: "This started happening after I upgraded to v1.5.0..." validations: required: false - type: checkboxes id: willing-to-contribute attributes: label: Contribution description: Would you like to work on fixing this bug? options: - label: I am a developer and would like to work on fixing this issue (pending maintainer approval) required: false - type: markdown attributes: value: | --- **Next Steps:** 1. A maintainer will review your bug report 2. If you checked the box above and want to fix it, please propose your solution approach 3. Wait for assignment before starting development 4. See our [Contributing Guide](https://github.com/lfnovo/open-notebook/blob/main/CONTRIBUTING.md) for more details ================================================ FILE: .github/ISSUE_TEMPLATE/config.yml ================================================ blank_issues_enabled: false contact_links: - name: 💬 Discord Community url: https://discord.gg/37XJPXfz2w about: Get help from the community and share ideas - name: 🤖 Installation Assistant (ChatGPT) url: https://chatgpt.com/g/g-68776e2765b48191bd1bae3f30212631-open-notebook-installation-assistant about: CustomGPT that knows all our docs. Really useful. Try it. - name: 📚 Documentation url: https://github.com/lfnovo/open-notebook/tree/main/docs about: Browse our comprehensive documentation ================================================ FILE: .github/ISSUE_TEMPLATE/feature_request.yml ================================================ name: ✨ Feature Suggestion description: Suggest a new feature or improvement for Open Notebook title: "[Feature]: " labels: ["enhancement", "needs-triage"] body: - type: markdown attributes: value: | Thanks for taking the time to suggest a feature! Your ideas help make Open Notebook better for everyone. - type: textarea id: feature-description attributes: label: Feature Description description: What feature would you like to see added or improved? placeholder: "I would like to be able to..." validations: required: true - type: textarea id: why-helpful attributes: label: Why would this be helpful? description: Explain how this feature would benefit you and other users placeholder: "This would help because..." validations: required: true - type: textarea id: proposed-solution attributes: label: Proposed Solution (Optional) description: If you have ideas on how to implement this feature, please share them placeholder: "This could be implemented by..." validations: required: false - type: textarea id: additional-context attributes: label: Additional Context description: Any other context, screenshots, or examples that might be helpful placeholder: "For example, other tools do this by..." validations: required: false - type: checkboxes id: willing-to-contribute attributes: label: Contribution description: Would you like to work on implementing this feature? options: - label: I am a developer and would like to work on implementing this feature (pending maintainer approval) required: false - type: markdown attributes: value: | --- **Next Steps:** 1. A maintainer will review your feature request 2. If approved and you checked the box above, the issue will be assigned to you 3. Please wait for assignment before starting development 4. See our [Contributing Guide](https://github.com/lfnovo/open-notebook/blob/main/CONTRIBUTING.md) for more details ================================================ FILE: .github/ISSUE_TEMPLATE/installation_issue.yml ================================================ name: 🔧 Installation Issue description: Report problems with installation, setup, or connectivity title: "[Install]: " labels: ["installation", "needs-triage"] body: - type: markdown attributes: value: | ## ⚠️ Before You Continue **Please try these resources first:** 1. 🤖 **[Installation Assistant ChatGPT](https://chatgpt.com/g/g-68776e2765b48191bd1bae3f30212631-open-notebook-installation-assistant)** - Our AI assistant can help you troubleshoot most installation issues instantly! 2. 📚 **[Installation Guide](https://github.com/lfnovo/open-notebook/blob/main/docs/getting-started/installation.md)** - Comprehensive setup instructions 3. 🐋 **[Docker Deployment Guide](https://github.com/lfnovo/open-notebook/blob/main/docs/deployment/docker.md)** - Detailed Docker setup 4. 🦙 **Ollama Issues?** Read our [Ollama Guide](https://github.com/lfnovo/open-notebook/blob/main/docs/features/ollama.md) first 5. 💬 **[Discord Community](https://discord.gg/37XJPXfz2w)** - Get real-time help from the community --- If you've tried the above and still need help, please fill out the form below with as much detail as possible. - type: dropdown id: installation-method attributes: label: Installation Method description: How are you trying to install Open Notebook? options: - Docker (single container - v1-latest-single) - Docker (multi-container - docker-compose) - Local development (make start-all) - Other (please specify below) validations: required: true - type: textarea id: issue-description attributes: label: What is the issue? description: Describe the installation or setup problem you're experiencing placeholder: | Example: "I can't connect to the database" or "The container won't start" or "Getting 404 errors when accessing the UI" validations: required: true - type: textarea id: logs attributes: label: Logs description: | Please provide relevant logs. **This is very important for diagnosing issues!** **How to get logs:** - Docker single container: `docker logs open-notebook` - Docker Compose: `docker compose logs -f` - Specific service: `docker compose logs -f open_notebook` placeholder: | Paste your logs here. Include the full error message and stack trace if available. render: shell validations: required: false - type: textarea id: docker-compose attributes: label: Docker Compose Configuration description: | If using Docker Compose, please paste your `docker-compose.yml` file here. **⚠️ IMPORTANT: Redact any sensitive information (API keys, passwords, etc.)** placeholder: | services: open_notebook: image: lfnovo/open_notebook:v1-latest-single ports: - "8502:8502" - "5055:5055" environment: - OPENAI_API_KEY=sk-***REDACTED*** ... render: yaml validations: required: false - type: textarea id: env-file attributes: label: Environment File description: | If using an `.env` or `docker.env` file, please paste it here. **⚠️ IMPORTANT: REDACT ALL API KEYS AND PASSWORDS!** placeholder: | SURREAL_URL=ws://surrealdb:8000/rpc SURREAL_USER=root SURREAL_PASSWORD=***REDACTED*** OPENAI_API_KEY=sk-***REDACTED*** ANTHROPIC_API_KEY=sk-ant-***REDACTED*** render: shell validations: required: false - type: textarea id: system-info attributes: label: System Information description: Tell us about your setup placeholder: | - Operating System: Ubuntu 22.04 / Windows 11 / macOS 14 - Docker version: `docker --version` - Docker Compose version: `docker compose version` - Architecture: amd64 / arm64 (Apple Silicon) - Available disk space: `df -h` - Available memory: `free -h` (Linux) or Activity Monitor (Mac) validations: required: false - type: textarea id: additional-context attributes: label: Additional Context description: Any other information that might be helpful placeholder: | - Are you behind a corporate proxy or firewall? - Are you using a VPN? - Have you made any custom modifications? - Did this work before and suddenly break? validations: required: false - type: checkboxes id: checklist attributes: label: Pre-submission Checklist description: Please confirm you've tried these steps options: - label: I tried the [Installation Assistant ChatGPT](https://chatgpt.com/g/g-68776e2765b48191bd1bae3f30212631-open-notebook-installation-assistant) required: false - label: I read the relevant documentation ([Installation Guide](https://github.com/lfnovo/open-notebook/blob/main/docs/getting-started/installation.md) or [Ollama Guide](https://github.com/lfnovo/open-notebook/blob/main/docs/features/ollama.md)) required: false - label: I searched existing issues to see if this was already reported required: true - label: I redacted all sensitive information (API keys, passwords, etc.) required: true ================================================ FILE: .github/pull_request_template.md ================================================ ## Description ## Related Issue Fixes # ## Type of Change - [ ] Bug fix (non-breaking change that fixes an issue) - [ ] New feature (non-breaking change that adds functionality) - [ ] Breaking change (fix or feature that would cause existing functionality to not work as expected) - [ ] Documentation update - [ ] Code refactoring (no functional changes) - [ ] Performance improvement - [ ] Test coverage improvement ## How Has This Been Tested? - [ ] Tested locally with Docker - [ ] Tested locally with development setup - [ ] Added new unit tests - [ ] Existing tests pass (`uv run pytest`) - [ ] Manual testing performed (describe below) **Test Details:** ## Design Alignment **Which design principles does this PR support?** (See [DESIGN_PRINCIPLES.md](../DESIGN_PRINCIPLES.md)) - [ ] Privacy First - [ ] Simplicity Over Features - [ ] API-First Architecture - [ ] Multi-Provider Flexibility - [ ] Extensibility Through Standards - [ ] Async-First for Performance **Explanation:** ## Checklist ### Code Quality - [ ] My code follows PEP 8 style guidelines (Python) - [ ] My code follows TypeScript best practices (Frontend) - [ ] I have added type hints to my code (Python) - [ ] I have added JSDoc comments where appropriate (TypeScript) - [ ] I have performed a self-review of my code - [ ] I have commented my code, particularly in hard-to-understand areas - [ ] My changes generate no new warnings or errors ### Testing - [ ] I have added tests that prove my fix is effective or that my feature works - [ ] New and existing unit tests pass locally with my changes - [ ] I ran linting: `make ruff` or `ruff check . --fix` - [ ] I ran type checking: `make lint` or `uv run python -m mypy .` ### Documentation - [ ] I have updated the relevant documentation in `/docs` (if applicable) - [ ] I have added/updated docstrings for new/modified functions - [ ] I have updated the API documentation (if API changes were made) - [ ] I have added comments to complex logic ### Database Changes - [ ] I have created migration scripts for any database schema changes (in `/migrations`) - [ ] Migration includes both up and down scripts - [ ] Migration has been tested locally ### Breaking Changes - [ ] This PR includes breaking changes - [ ] I have documented the migration path for users - [ ] I have updated MIGRATION.md (if applicable) ## Screenshots (if applicable) ## Additional Context ## Pre-Submission Verification Before submitting, please verify: - [ ] I have read [CONTRIBUTING.md](../CONTRIBUTING.md) - [ ] I have read [DESIGN_PRINCIPLES.md](../DESIGN_PRINCIPLES.md) - [ ] This PR addresses an approved issue that was assigned to me - [ ] I have not included unrelated changes in this PR - [ ] My PR title follows conventional commits format (e.g., "feat: add user authentication") --- **Thank you for contributing to Open Notebook!** 🎉 ================================================ FILE: .github/workflows/build-and-release.yml ================================================ name: Build and Release on: workflow_dispatch: inputs: push_latest: description: 'Also push v1-latest tags' required: true default: false type: boolean release: types: [published] permissions: contents: read packages: write env: GHCR_IMAGE: ghcr.io/lfnovo/open-notebook DOCKERHUB_IMAGE: lfnovo/open_notebook jobs: extract-version: runs-on: ubuntu-latest outputs: version: ${{ steps.version.outputs.version }} has_dockerhub_secrets: ${{ steps.check.outputs.has_dockerhub_secrets }} steps: - name: Checkout uses: actions/checkout@v4 - name: Extract version from pyproject.toml id: version run: | VERSION=$(grep -m1 '^version = ' pyproject.toml | cut -d'"' -f2) echo "version=$VERSION" >> $GITHUB_OUTPUT echo "Extracted version: $VERSION" - name: Check for Docker Hub credentials id: check env: SECRET_DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} SECRET_DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} run: | if [[ -n ""$SECRET_DOCKER_USERNAME"" && -n ""$SECRET_DOCKER_PASSWORD"" ]]; then echo "has_dockerhub_secrets=true" >> $GITHUB_OUTPUT echo "Docker Hub credentials available" else echo "has_dockerhub_secrets=false" >> $GITHUB_OUTPUT echo "Docker Hub credentials not available - will only push to GHCR" fi build-regular: needs: extract-version runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 - name: Free up disk space run: | sudo rm -rf /usr/share/dotnet sudo rm -rf /usr/local/lib/android sudo rm -rf /opt/ghc sudo rm -rf /opt/hostedtoolcache/CodeQL sudo docker image prune --all --force df -h - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - name: Login to GitHub Container Registry uses: docker/login-action@v3 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Login to Docker Hub if: needs.extract-version.outputs.has_dockerhub_secrets == 'true' uses: docker/login-action@v3 with: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} - name: Cache Docker layers uses: actions/cache@v3 with: path: /tmp/.buildx-cache key: ${{ runner.os }}-buildx-regular-${{ github.sha }} restore-keys: | ${{ runner.os }}-buildx-regular- - name: Prepare Docker tags for regular build id: tags-regular env: ENV_GHCR_IMAGE: ${{ env.GHCR_IMAGE }} GITHUB_EVENT_INPUTS_PUSH_LATEST: ${{ github.event.inputs.push_latest }} GITHUB_EVENT_NAME: ${{ github.event_name }} GITHUB_EVENT_RELEASE_PRERELEASE: ${{ github.event.release.prerelease }} ENV_DOCKERHUB_IMAGE: ${{ env.DOCKERHUB_IMAGE }} run: | TAGS=""$ENV_GHCR_IMAGE":${{ needs.extract-version.outputs.version }}" # Determine if we should push latest tags PUSH_LATEST=""$GITHUB_EVENT_INPUTS_PUSH_LATEST"" if [[ -z "$PUSH_LATEST" ]]; then PUSH_LATEST="false" fi # Add GHCR latest tag if requested or for non-prerelease releases if [[ "$PUSH_LATEST" == "true" ]] || [[ ""$GITHUB_EVENT_NAME"" == "release" && ""$GITHUB_EVENT_RELEASE_PRERELEASE"" != "true" ]]; then TAGS="${TAGS},"$ENV_GHCR_IMAGE":v1-latest" fi # Add Docker Hub tags if credentials available if [[ "${{ needs.extract-version.outputs.has_dockerhub_secrets }}" == "true" ]]; then TAGS="${TAGS},"$ENV_DOCKERHUB_IMAGE":${{ needs.extract-version.outputs.version }}" if [[ "$PUSH_LATEST" == "true" ]] || [[ ""$GITHUB_EVENT_NAME"" == "release" && ""$GITHUB_EVENT_RELEASE_PRERELEASE"" != "true" ]]; then TAGS="${TAGS},"$ENV_DOCKERHUB_IMAGE":v1-latest" fi fi echo "tags=${TAGS}" >> $GITHUB_OUTPUT echo "Generated tags: ${TAGS}" - name: Build and push regular image uses: docker/build-push-action@v5 with: context: . file: ./Dockerfile platforms: linux/amd64,linux/arm64 push: true tags: ${{ steps.tags-regular.outputs.tags }} cache-from: type=local,src=/tmp/.buildx-cache cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max - name: Move cache run: | rm -rf /tmp/.buildx-cache mv /tmp/.buildx-cache-new /tmp/.buildx-cache build-single: needs: extract-version runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 - name: Free up disk space run: | sudo rm -rf /usr/share/dotnet sudo rm -rf /usr/local/lib/android sudo rm -rf /opt/ghc sudo rm -rf /opt/hostedtoolcache/CodeQL sudo docker image prune --all --force df -h - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - name: Login to GitHub Container Registry uses: docker/login-action@v3 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Login to Docker Hub if: needs.extract-version.outputs.has_dockerhub_secrets == 'true' uses: docker/login-action@v3 with: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} - name: Cache Docker layers uses: actions/cache@v3 with: path: /tmp/.buildx-cache-single key: ${{ runner.os }}-buildx-single-${{ github.sha }} restore-keys: | ${{ runner.os }}-buildx-single- - name: Prepare Docker tags for single build id: tags-single env: ENV_GHCR_IMAGE: ${{ env.GHCR_IMAGE }} GITHUB_EVENT_INPUTS_PUSH_LATEST: ${{ github.event.inputs.push_latest }} GITHUB_EVENT_NAME: ${{ github.event_name }} GITHUB_EVENT_RELEASE_PRERELEASE: ${{ github.event.release.prerelease }} ENV_DOCKERHUB_IMAGE: ${{ env.DOCKERHUB_IMAGE }} run: | TAGS=""$ENV_GHCR_IMAGE":${{ needs.extract-version.outputs.version }}-single" # Determine if we should push latest tags PUSH_LATEST=""$GITHUB_EVENT_INPUTS_PUSH_LATEST"" if [[ -z "$PUSH_LATEST" ]]; then PUSH_LATEST="false" fi # Add GHCR latest tag if requested or for non-prerelease releases if [[ "$PUSH_LATEST" == "true" ]] || [[ ""$GITHUB_EVENT_NAME"" == "release" && ""$GITHUB_EVENT_RELEASE_PRERELEASE"" != "true" ]]; then TAGS="${TAGS},"$ENV_GHCR_IMAGE":v1-latest-single" fi # Add Docker Hub tags if credentials available if [[ "${{ needs.extract-version.outputs.has_dockerhub_secrets }}" == "true" ]]; then TAGS="${TAGS},"$ENV_DOCKERHUB_IMAGE":${{ needs.extract-version.outputs.version }}-single" if [[ "$PUSH_LATEST" == "true" ]] || [[ ""$GITHUB_EVENT_NAME"" == "release" && ""$GITHUB_EVENT_RELEASE_PRERELEASE"" != "true" ]]; then TAGS="${TAGS},"$ENV_DOCKERHUB_IMAGE":v1-latest-single" fi fi echo "tags=${TAGS}" >> $GITHUB_OUTPUT echo "Generated tags: ${TAGS}" - name: Build and push single-container image uses: docker/build-push-action@v5 with: context: . file: ./Dockerfile.single platforms: linux/amd64,linux/arm64 push: true tags: ${{ steps.tags-single.outputs.tags }} cache-from: type=local,src=/tmp/.buildx-cache-single cache-to: type=local,dest=/tmp/.buildx-cache-single-new,mode=max - name: Move cache run: | rm -rf /tmp/.buildx-cache-single mv /tmp/.buildx-cache-single-new /tmp/.buildx-cache-single summary: needs: [extract-version, build-regular, build-single] runs-on: ubuntu-latest if: always() steps: - name: Build Summary env: GITHUB_EVENT_INPUTS_PUSH_LATEST_____FALSE_: ${{ github.event.inputs.push_latest || 'false' }} ENV_GHCR_IMAGE: ${{ env.GHCR_IMAGE }} ENV_DOCKERHUB_IMAGE: ${{ env.DOCKERHUB_IMAGE }} GITHUB_EVENT_INPUTS_PUSH_LATEST: ${{ github.event.inputs.push_latest }} run: | echo "## Build Summary" >> $GITHUB_STEP_SUMMARY echo "**Version:** ${{ needs.extract-version.outputs.version }}" >> $GITHUB_STEP_SUMMARY echo "**Push v1-Latest:** "$GITHUB_EVENT_INPUTS_PUSH_LATEST_____FALSE_"" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY echo "### Registries:" >> $GITHUB_STEP_SUMMARY echo "✅ **GHCR:** \`"$ENV_GHCR_IMAGE"\`" >> $GITHUB_STEP_SUMMARY if [[ "${{ needs.extract-version.outputs.has_dockerhub_secrets }}" == "true" ]]; then echo "✅ **Docker Hub:** \`"$ENV_DOCKERHUB_IMAGE"\`" >> $GITHUB_STEP_SUMMARY else echo "⏭️ **Docker Hub:** Skipped (credentials not configured)" >> $GITHUB_STEP_SUMMARY fi echo "" >> $GITHUB_STEP_SUMMARY echo "### Images Built:" >> $GITHUB_STEP_SUMMARY if [[ "${{ needs.build-regular.result }}" == "success" ]]; then echo "✅ **Regular (GHCR):** \`"$ENV_GHCR_IMAGE":${{ needs.extract-version.outputs.version }}\`" >> $GITHUB_STEP_SUMMARY if [[ ""$GITHUB_EVENT_INPUTS_PUSH_LATEST"" == "true" ]]; then echo "✅ **Regular v1-Latest (GHCR):** \`"$ENV_GHCR_IMAGE":v1-latest\`" >> $GITHUB_STEP_SUMMARY fi if [[ "${{ needs.extract-version.outputs.has_dockerhub_secrets }}" == "true" ]]; then echo "✅ **Regular (Docker Hub):** \`"$ENV_DOCKERHUB_IMAGE":${{ needs.extract-version.outputs.version }}\`" >> $GITHUB_STEP_SUMMARY if [[ ""$GITHUB_EVENT_INPUTS_PUSH_LATEST"" == "true" ]]; then echo "✅ **Regular v1-Latest (Docker Hub):** \`"$ENV_DOCKERHUB_IMAGE":v1-latest\`" >> $GITHUB_STEP_SUMMARY fi fi elif [[ "${{ needs.build-regular.result }}" == "skipped" ]]; then echo "⏭️ **Regular:** Skipped" >> $GITHUB_STEP_SUMMARY else echo "❌ **Regular:** Failed" >> $GITHUB_STEP_SUMMARY fi if [[ "${{ needs.build-single.result }}" == "success" ]]; then echo "✅ **Single (GHCR):** \`"$ENV_GHCR_IMAGE":${{ needs.extract-version.outputs.version }}-single\`" >> $GITHUB_STEP_SUMMARY if [[ ""$GITHUB_EVENT_INPUTS_PUSH_LATEST"" == "true" ]]; then echo "✅ **Single v1-Latest (GHCR):** \`"$ENV_GHCR_IMAGE":v1-latest-single\`" >> $GITHUB_STEP_SUMMARY fi if [[ "${{ needs.extract-version.outputs.has_dockerhub_secrets }}" == "true" ]]; then echo "✅ **Single (Docker Hub):** \`"$ENV_DOCKERHUB_IMAGE":${{ needs.extract-version.outputs.version }}-single\`" >> $GITHUB_STEP_SUMMARY if [[ ""$GITHUB_EVENT_INPUTS_PUSH_LATEST"" == "true" ]]; then echo "✅ **Single v1-Latest (Docker Hub):** \`"$ENV_DOCKERHUB_IMAGE":v1-latest-single\`" >> $GITHUB_STEP_SUMMARY fi fi elif [[ "${{ needs.build-single.result }}" == "skipped" ]]; then echo "⏭️ **Single:** Skipped" >> $GITHUB_STEP_SUMMARY else echo "❌ **Single:** Failed" >> $GITHUB_STEP_SUMMARY fi echo "" >> $GITHUB_STEP_SUMMARY echo "### Platforms:" >> $GITHUB_STEP_SUMMARY echo "- linux/amd64" >> $GITHUB_STEP_SUMMARY echo "- linux/arm64" >> $GITHUB_STEP_SUMMARY ================================================ FILE: .github/workflows/build-dev.yml ================================================ name: Development Build on: pull_request: branches: [ main ] push: branches: [ main ] paths-ignore: - '**.md' - 'docs/**' - 'notebooks/**' - '.github/workflows/claude*.yml' workflow_dispatch: inputs: platform: description: 'Platform to build' required: true default: 'linux/amd64' type: choice options: - linux/amd64 - linux/arm64 - linux/amd64,linux/arm64 permissions: contents: read packages: write env: GHCR_IMAGE: ghcr.io/lfnovo/open-notebook DOCKERHUB_IMAGE: lfnovo/open_notebook jobs: extract-version: runs-on: ubuntu-latest outputs: version: ${{ steps.version.outputs.version }} has_dockerhub_secrets: ${{ steps.check.outputs.has_dockerhub_secrets }} is_push_to_main: ${{ steps.check.outputs.is_push_to_main }} steps: - name: Checkout uses: actions/checkout@v4 - name: Extract version from pyproject.toml id: version run: | VERSION=$(grep -m1 '^version = ' pyproject.toml | cut -d'"' -f2) echo "version=$VERSION" >> $GITHUB_OUTPUT echo "Extracted version: $VERSION" - name: Check environment id: check env: SECRET_DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} SECRET_DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} run: | # Check for Docker Hub credentials if [[ -n "$SECRET_DOCKER_USERNAME" && -n "$SECRET_DOCKER_PASSWORD" ]]; then echo "has_dockerhub_secrets=true" >> $GITHUB_OUTPUT echo "Docker Hub credentials available" else echo "has_dockerhub_secrets=false" >> $GITHUB_OUTPUT echo "Docker Hub credentials not available" fi # Check if this is a push to main (not a PR) if [[ "${{ github.event_name }}" == "push" && "${{ github.ref }}" == "refs/heads/main" ]]; then echo "is_push_to_main=true" >> $GITHUB_OUTPUT echo "This is a push to main - will publish v1-dev tags" else echo "is_push_to_main=false" >> $GITHUB_OUTPUT echo "This is a PR or manual run - test build only" fi build-regular: needs: extract-version runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 - name: Free up disk space if: needs.extract-version.outputs.is_push_to_main == 'true' run: | sudo rm -rf /usr/share/dotnet sudo rm -rf /usr/local/lib/android sudo rm -rf /opt/ghc sudo rm -rf /opt/hostedtoolcache/CodeQL sudo docker image prune --all --force df -h - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - name: Login to GitHub Container Registry if: needs.extract-version.outputs.is_push_to_main == 'true' uses: docker/login-action@v3 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Login to Docker Hub if: needs.extract-version.outputs.is_push_to_main == 'true' && needs.extract-version.outputs.has_dockerhub_secrets == 'true' uses: docker/login-action@v3 with: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} - name: Cache Docker layers uses: actions/cache@v3 with: path: /tmp/.buildx-cache-dev key: ${{ runner.os }}-buildx-dev-${{ github.sha }} restore-keys: | ${{ runner.os }}-buildx-dev- - name: Prepare Docker tags id: tags run: | if [[ "${{ needs.extract-version.outputs.is_push_to_main }}" == "true" ]]; then # Push to main: build and push v1-dev tags TAGS="${{ env.GHCR_IMAGE }}:v1-dev" if [[ "${{ needs.extract-version.outputs.has_dockerhub_secrets }}" == "true" ]]; then TAGS="${TAGS},${{ env.DOCKERHUB_IMAGE }}:v1-dev" fi echo "tags=${TAGS}" >> $GITHUB_OUTPUT echo "push=true" >> $GITHUB_OUTPUT echo "platforms=linux/amd64,linux/arm64" >> $GITHUB_OUTPUT else # PR or manual: test build only echo "tags=${{ env.DOCKERHUB_IMAGE }}:${{ needs.extract-version.outputs.version }}-dev" >> $GITHUB_OUTPUT echo "push=false" >> $GITHUB_OUTPUT echo "platforms=${{ github.event.inputs.platform || 'linux/amd64' }}" >> $GITHUB_OUTPUT fi - name: Build and push regular image uses: docker/build-push-action@v5 with: context: . file: ./Dockerfile platforms: ${{ steps.tags.outputs.platforms }} push: ${{ steps.tags.outputs.push }} tags: ${{ steps.tags.outputs.tags }} cache-from: type=local,src=/tmp/.buildx-cache-dev cache-to: type=local,dest=/tmp/.buildx-cache-dev-new,mode=max - name: Move cache run: | rm -rf /tmp/.buildx-cache-dev mv /tmp/.buildx-cache-dev-new /tmp/.buildx-cache-dev build-single: needs: extract-version # Only build single image on push to main if: needs.extract-version.outputs.is_push_to_main == 'true' runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 - name: Free up disk space run: | sudo rm -rf /usr/share/dotnet sudo rm -rf /usr/local/lib/android sudo rm -rf /opt/ghc sudo rm -rf /opt/hostedtoolcache/CodeQL sudo docker image prune --all --force df -h - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - name: Login to GitHub Container Registry uses: docker/login-action@v3 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Login to Docker Hub if: needs.extract-version.outputs.has_dockerhub_secrets == 'true' uses: docker/login-action@v3 with: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} - name: Cache Docker layers uses: actions/cache@v3 with: path: /tmp/.buildx-cache-dev-single key: ${{ runner.os }}-buildx-dev-single-${{ github.sha }} restore-keys: | ${{ runner.os }}-buildx-dev-single- - name: Prepare Docker tags id: tags run: | TAGS="${{ env.GHCR_IMAGE }}:v1-dev-single" if [[ "${{ needs.extract-version.outputs.has_dockerhub_secrets }}" == "true" ]]; then TAGS="${TAGS},${{ env.DOCKERHUB_IMAGE }}:v1-dev-single" fi echo "tags=${TAGS}" >> $GITHUB_OUTPUT - name: Build and push single-container image uses: docker/build-push-action@v5 with: context: . file: ./Dockerfile.single platforms: linux/amd64,linux/arm64 push: true tags: ${{ steps.tags.outputs.tags }} cache-from: type=local,src=/tmp/.buildx-cache-dev-single cache-to: type=local,dest=/tmp/.buildx-cache-dev-single-new,mode=max - name: Move cache run: | rm -rf /tmp/.buildx-cache-dev-single mv /tmp/.buildx-cache-dev-single-new /tmp/.buildx-cache-dev-single summary: needs: [extract-version, build-regular, build-single] runs-on: ubuntu-latest if: always() steps: - name: Development Build Summary run: | echo "## Development Build Summary" >> $GITHUB_STEP_SUMMARY echo "**Version:** ${{ needs.extract-version.outputs.version }}" >> $GITHUB_STEP_SUMMARY echo "**Event:** ${{ github.event_name }}" >> $GITHUB_STEP_SUMMARY echo "**Push to Main:** ${{ needs.extract-version.outputs.is_push_to_main }}" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY if [[ "${{ needs.extract-version.outputs.is_push_to_main }}" == "true" ]]; then echo "### Published Tags:" >> $GITHUB_STEP_SUMMARY if [[ "${{ needs.build-regular.result }}" == "success" ]]; then echo "✅ **Regular:** \`${{ env.GHCR_IMAGE }}:v1-dev\`" >> $GITHUB_STEP_SUMMARY if [[ "${{ needs.extract-version.outputs.has_dockerhub_secrets }}" == "true" ]]; then echo "✅ **Regular (Docker Hub):** \`${{ env.DOCKERHUB_IMAGE }}:v1-dev\`" >> $GITHUB_STEP_SUMMARY fi else echo "❌ **Regular:** Build failed" >> $GITHUB_STEP_SUMMARY fi if [[ "${{ needs.build-single.result }}" == "success" ]]; then echo "✅ **Single:** \`${{ env.GHCR_IMAGE }}:v1-dev-single\`" >> $GITHUB_STEP_SUMMARY if [[ "${{ needs.extract-version.outputs.has_dockerhub_secrets }}" == "true" ]]; then echo "✅ **Single (Docker Hub):** \`${{ env.DOCKERHUB_IMAGE }}:v1-dev-single\`" >> $GITHUB_STEP_SUMMARY fi elif [[ "${{ needs.build-single.result }}" == "skipped" ]]; then echo "⏭️ **Single:** Skipped" >> $GITHUB_STEP_SUMMARY else echo "❌ **Single:** Build failed" >> $GITHUB_STEP_SUMMARY fi echo "" >> $GITHUB_STEP_SUMMARY echo "### Platforms:" >> $GITHUB_STEP_SUMMARY echo "- linux/amd64" >> $GITHUB_STEP_SUMMARY echo "- linux/arm64" >> $GITHUB_STEP_SUMMARY else echo "### Test Build Results:" >> $GITHUB_STEP_SUMMARY if [[ "${{ needs.build-regular.result }}" == "success" ]]; then echo "✅ **Dockerfile:** Build successful" >> $GITHUB_STEP_SUMMARY else echo "❌ **Dockerfile:** Build failed" >> $GITHUB_STEP_SUMMARY fi echo "" >> $GITHUB_STEP_SUMMARY echo "### Notes:" >> $GITHUB_STEP_SUMMARY echo "- This is a test build (no images pushed to registry)" >> $GITHUB_STEP_SUMMARY echo "- Merge to main to publish \`v1-dev\` tags" >> $GITHUB_STEP_SUMMARY echo "- For stable releases, use the 'Build and Release' workflow" >> $GITHUB_STEP_SUMMARY fi ================================================ FILE: .github/workflows/claude-code-review.yml ================================================ name: Claude Code Review on: pull_request: types: [opened, synchronize, ready_for_review, reopened] pull_request_target: types: [opened, synchronize, ready_for_review, reopened] # Optional: Only run on specific file changes # paths: # - "src/**/*.ts" # - "src/**/*.tsx" # - "src/**/*.js" # - "src/**/*.jsx" jobs: claude-review: # Run for fork PRs (via pull_request_target) OR same-repo PRs (via pull_request), but not both if: | (github.event_name == 'pull_request_target' && github.event.pull_request.head.repo.full_name != github.repository) || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository) runs-on: ubuntu-latest permissions: contents: read pull-requests: write issues: write id-token: write steps: - name: Checkout repository uses: actions/checkout@v4 with: ref: ${{ github.event.pull_request.head.sha }} fetch-depth: 1 persist-credentials: false - name: Run Claude Code Review id: claude-review uses: anthropics/claude-code-action@v1 with: github_token: ${{ secrets.GITHUB_TOKEN }} claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} plugin_marketplaces: 'https://github.com/anthropics/claude-code.git' plugins: 'code-review@claude-code-plugins' prompt: '/code-review:code-review ${{ github.repository }}/pull/${{ github.event.pull_request.number }}' # See https://github.com/anthropics/claude-code-action/blob/main/docs/usage.md # or https://code.claude.com/docs/en/cli-reference for available options ================================================ FILE: .github/workflows/claude.yml ================================================ name: Claude Code on: issue_comment: types: [created] pull_request_review_comment: types: [created] issues: types: [opened, assigned] pull_request_review: types: [submitted] jobs: claude: if: | (github.event_name == 'issue_comment' && contains(github.event.comment.body, '@claude')) || (github.event_name == 'pull_request_review_comment' && contains(github.event.comment.body, '@claude')) || (github.event_name == 'pull_request_review' && contains(github.event.review.body, '@claude')) || (github.event_name == 'issues' && (contains(github.event.issue.body, '@claude') || contains(github.event.issue.title, '@claude'))) runs-on: ubuntu-latest permissions: contents: read pull-requests: write issues: write id-token: write actions: read # Required for Claude to read CI results on PRs steps: - name: Checkout repository uses: actions/checkout@v4 with: fetch-depth: 1 - name: Run Claude Code id: claude uses: anthropics/claude-code-action@v1 with: github_token: ${{ secrets.GITHUB_TOKEN }} claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} # This is an optional setting that allows Claude to read CI results on PRs additional_permissions: | actions: read # Optional: Give a custom prompt to Claude. If this is not specified, Claude will perform the instructions specified in the comment that tagged it. # prompt: 'Update the pull request description to include a summary of changes.' # Optional: Add claude_args to customize behavior and configuration # See https://github.com/anthropics/claude-code-action/blob/main/docs/usage.md # or https://code.claude.com/docs/en/cli-reference for available options # claude_args: '--allowed-tools Bash(gh pr:*)' ================================================ FILE: .github/workflows/test.yml ================================================ name: Tests on: pull_request: branches: [main] push: branches: [main] paths-ignore: - '**.md' - 'docs/**' - '.github/workflows/claude*.yml' permissions: contents: read jobs: backend: name: Backend Tests runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 - name: Set up uv uses: astral-sh/setup-uv@v4 with: enable-cache: true - name: Set up Python run: uv python install - name: Install dependencies run: uv sync - name: Run tests run: uv run pytest tests/ -v frontend: name: Frontend Tests runs-on: ubuntu-latest defaults: run: working-directory: frontend steps: - name: Checkout uses: actions/checkout@v4 - name: Set up Node.js uses: actions/setup-node@v4 with: node-version: 22 cache: npm cache-dependency-path: frontend/package-lock.json - name: Install dependencies run: npm ci - name: Run tests run: npm test ================================================ FILE: .gitignore ================================================ .env prompts/patterns/user/ /notebooks/ data/ .uploads/ sqlite-db/ surreal-data/ docker.env notebook_data/ # Python-specific *.py[cod] __pycache__/ *.so todo.md temp/ google-credentials.json # Distribution / packaging .Python build/ develop-eggs/ dist/ downloads/ eggs/ .eggs/ /lib/ /lib64/ parts/ sdist/ var/ wheels/ share/python-wheels/ *.egg-info/ .installed.cfg *.egg # PyInstaller *.manifest *.spec # Installer logs pip-log.txt pip-delete-this-directory.txt # Unit test / coverage reports htmlcov/ .tox/ .nox/ .coverage .coverage.* .cache nosetests.xml coverage.xml *.cover *.py,cover .hypothesis/ .pytest_cache/ # Jupyter Notebook .ipynb_checkpoints # IPython profile_default/ ipython_config.py # Environments .env .venv env/ venv/ ENV/ env.bak/ venv.bak/ # PyCharm .idea/ # VS Code .vscode/ # Spyder project settings .spyderproject .spyproject # Rope project settings .ropeproject # mkdocs documentation /site # mypy .mypy_cache/ .dmypy.json dmypy.json # Pyre type checker .pyre/ # pytype static type analyzer .pytype/ # Cython debug symbols cython_debug/ # macOS .DS_Store # Windows Thumbs.db ehthumbs.db desktop.ini # Linux *~ # Log files *.log # Database files *.db *.sqlite3 .quarentena claude-logs/ .claude/sessions **/claude-logs docs/custom_gpt doc_exports/ specs/ .claude .sisyphus .playwright-mcp/ *.local.yml **/*.local.md ================================================ FILE: .python-version ================================================ 3.12 ================================================ FILE: .worktreeinclude ================================================ .env .env.local .env.* **/.claude/settings.local.json CLAUDE.local.md ================================================ FILE: CHANGELOG.md ================================================ # Changelog All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] ## [1.8.1] - 2026-03-10 ### Added - i18n support for Bengali (bn-IN) (#643) - Podcast language support via podcast-creator 0.12.0 (#645) - Upgrade default Azure API version for model testing and fetching (#638) ### Fixed - Tiktoken network errors in offline/air-gapped Docker deployments — pre-downloads encoding at build time (#264, #622) - SurrealDB getting stuck (#656) ### Dependencies - Bump esperanto to 2.19.5 (#657) - Bump langgraph from 1.0.6 to 1.0.10rc1 (#658) - Bump authlib from 1.6.6 to 1.6.7 (#649) - Bump lxml-html-clean from 0.4.3 to 0.4.4 (#646) - Bump rollup from 4.55.1 to 4.59.0 (#635) - Bump minimatch in frontend (#634) - Bump tar from 7.5.9 to 7.5.11 (#650, #659) ## [1.7.4] - 2026-02-18 ### Fixed - Embedding large documents (3MB+) fails with 413 Payload Too Large (#594) - `generate_embeddings()` now batches texts in groups of 50 with per-batch retry, preventing provider payload limits from being exceeded - 413 errors now classified with user-friendly message in error classifier - Misleading "Created 0 embedded chunks" log in `process_source_command` — embedding is fire-and-forget, so the count was always 0; now logs "embedding submitted" instead ## [1.7.3] - 2026-02-17 ### Added - Retry button for failed podcast episodes in the UI (#211, #218) - Error details displayed on failed podcast episodes (#185, #355) - `POST /podcasts/episodes/{id}/retry` API endpoint for re-submitting failed episodes - `error_message` field in podcast episode API responses ### Fixed - Podcast generation failures now correctly marked as "failed" instead of "completed" (#300, #335) - Disabled automatic retries for podcast generation to prevent duplicate episode records (#302) ### Dependencies - Bump podcast-creator to >= 0.11.2 - Bump esperanto to >= 2.19.4 ## [1.7.2] - 2026-02-16 ### Added - Error classification utility that maps LLM provider errors to user-friendly messages (#506) - Global exception handlers in FastAPI for all custom exception types with proper HTTP status codes - `getApiErrorMessage()` frontend helper that falls back to backend messages when no i18n mapping exists ### Fixed - LLM errors (invalid API key, wrong model, rate limits) now show descriptive messages instead of "An unexpected error occurred" (#590) - SSE streaming error events in source chat and ask hooks were swallowed by inner JSON parse catch blocks - Transformation execution errors were caught and re-wrapped as generic 500s instead of using proper status codes - Fail fast when source content extraction returns empty instead of retrying (#589) - Chat input and message overflow with long unbroken strings (#588) - Word-wrap overflow in source cards, note editor, inline edit, note titles, and dialog content (#588) - Translation proxy shadowing `name` keys (#588) - OpenAI-compatible provider name handling via Esperanto update (#583) ### Changed - `ValueError` replaced with `ConfigurationError` in model provisioning for proper error classification - `ConfigurationError` added to command retry `stop_on` lists to avoid retrying permanent config failures ### Dependencies - Bump esperanto to 2.19.3 (#583) - Bump podcast-creator to 0.9.1 ## [1.7.1] - 2026-02-14 ### Added - French (fr-FR) language support (#581) - CI test workflow and improved i18n validation (#580) - Expose embed `command_id` in note API responses (#545) ### Fixed - ElevenLabs TTS credential passthrough via Esperanto update (#578) - Handle empty/whitespace source content without retry loop (#576) - Increase transformation `max_tokens` and update Esperanto dep (#568) - Turn the embedding field into optional (#557) ### Docs - Fix docker container names in local setup guides (#577) ### Dependencies - Bump langchain-core from 1.2.7 to 1.2.11 (#564) - Bump cryptography from 46.0.3 to 46.0.5 (#563) ## [1.7.0] - 2026-02-10 ### Added - **Credential-Based Provider Management** (#477) - New Settings → API Keys page for managing AI provider credentials via the UI - Support for 14 providers: OpenAI, Anthropic, Google, Groq, Mistral, DeepSeek, xAI, OpenRouter, Voyage AI, ElevenLabs, Ollama, Azure OpenAI, OpenAI-Compatible, and Vertex AI - Secure storage of API keys in SurrealDB with field-level encryption (Fernet AES-128-CBC + HMAC-SHA256) - One-click connection testing, model discovery, and model registration per credential - Migration tool to import existing environment variable keys into the credential system - Azure OpenAI support with service-specific endpoints (LLM, Embedding, STT, TTS) - OpenAI-Compatible support with per-service URL configurations - Vertex AI support with project, location, and credentials path - Environment variable API keys deprecated in favor of Settings UI - **Security Enhancements** - Docker secrets support via `_FILE` suffix pattern (e.g., `OPEN_NOTEBOOK_PASSWORD_FILE`) - Default encryption key derived from "0p3n-N0t3b0ok" for easy setup (change in production!) - Default password "open-notebook-change-me" for out-of-box experience (change in production!) - URL validation for SSRF protection - blocks private IPs and localhost (except for Ollama which runs locally) - Security warnings logged when using default credentials - HTML clipboard detection for text sources (#426) - When pasting content, automatically detects HTML format (e.g., from Word, web pages) - Shows info message when HTML is detected, informing user it will be converted to Markdown - Preserves formatting that would be lost with plain text paste - Bump content-core to 0.11.0 for HTML to Markdown conversion support - **Improved Getting Started Experience** - Simplified docker-compose.yml in repository root (single official file) - Added examples/ folder with ready-made configurations: - `docker-compose-ollama.yml` - Local AI with Ollama - `docker-compose-speaches.yml` - Local TTS/STT with Speaches - `docker-compose-full-local.yml` - 100% local setup (Ollama + Speaches) - Inline quick start in README (no need to navigate to docs) - Cross-references between docker-compose examples and documentation - .env.example template with all configuration options ### Fixed - Azure form race condition: all configuration now saved in single atomic request - Migration API "error error" display: added proper MigrationResult model with message field - Connection tester for Ollama providers: improved error handling and URL validation - SqliteSaver async compatibility issues in chat system (#509, #525, #538) - Re-embedding failures with empty content (#513, #515) - Deletion cascade for notes and sources (#77) - YouTube content availability issues (#494) - Large document embedding errors (#489) ### Security - API keys are encrypted at rest using Fernet symmetric encryption - Keys are never returned to the frontend, only configuration status - SSRF protection prevents internal network access via URL validation ### Docs - Complete documentation update for credential-based system across 25 files - All quick-start, installation, and configuration guides now use Settings UI workflow - Environment variable API key instructions moved to deprecated/legacy sections - Fixed broken links in installation docs - Added comprehensive examples/ folder with documented docker-compose configurations - Updated local-tts.md and local-stt.md with links to ready-made examples ### Internationalization - Added Russian (ru-RU) language support (#524) - Added Italian (it-IT) language support (#508) ## [1.6.2] - 2026-01-24 ### Fixed - Connection error with llama.cpp and OpenAI-compatible providers (#465) - Bump Esperanto to 2.17.2 which fixes LangChain connection errors caused by garbage collection ## [1.6.1] - 2026-01-22 ### Fixed - "Failed to send message" error with unhelpful logs when chat model is not configured (#358) - Added detailed error logging with model selection context and full traceback - Improved error messages to guide users to Settings → Models - Added warnings when default models are not configured ### Docs - Ollama troubleshooting: Added "Model Name Configuration" section emphasizing exact model names from `ollama list` - Added troubleshooting entry for "Failed to send message" error with step-by-step solutions - Updated AI Chat Issues documentation with model configuration guidance ## [1.6.0] - 2026-01-21 ### Added - Content-type aware text chunking with automatic HTML, Markdown, and plain text detection (#350, #142) - Unified embedding generation with mean pooling for large content that exceeds model context limits - Dedicated embedding commands: `embed_note`, `embed_insight`, `embed_source` - New utility modules: `chunking.py` and `embedding.py` in `open_notebook/utils/` - Japanese (ja-JP) language support (#450) ### Changed - Embedding is now fire-and-forget: domain models submit embedding commands asynchronously after save - `rebuild_embeddings_command` now delegates to individual embed_* commands instead of inline processing - Chunk size reduced to 1500 characters for better compatibility with Ollama embedding models - Bump Esperanto to 2.16 for increased Ollama context window support ### Removed - Legacy embedding commands: `embed_single_item_command`, `embed_chunk_command`, `vectorize_source_command` - `needs_embedding()` and `get_embedding_content()` methods from domain models - `split_text()` function from text_utils (replaced by `chunk_text()` in chunking module) ### Fixed - Embedding failures when content exceeds model context limits (#350, #142) - Empty note titles when saving from chat (clean thinking tags from prompt graph output) - Orphaned embedding/insight records when deleting sources (cascade delete) - Search results crash with null parent_id (defensive frontend check) - Database migration 10 cleans up existing orphaned records ## [1.5.2] - 2026-01-15 ### Performance - Improved source listing speed by 20-30x (#436, closes #351) - Added database indexes on `source` field for `source_insight` and `source_embedding` tables - Use SurrealDB `FETCH` clause for command status instead of N async calls ## [1.5.1] - 2026-01-15 ### Fixed - Podcast dialog infinite loop error caused by excessive translation Proxy accesses in loops - Podcast dialog UI freezing when typing episode name or additional instructions - Removed incorrect translation keys for user-defined episode profiles (user content should not be translated) ## [1.5.0] - 2026-01-15 ### Added - Internationalization (i18n) support with Chinese (Simplified and Traditional) translations (#371, closes #344, #349, #360) - Frontend test infrastructure with Vitest (#371) - Language toggle component for switching UI language (#371) - Date localization using date-fns locales (#371) - Error message translation system (#371) ### Fixed - Accessibility improvements: added missing `id`, `name`, and `autoComplete` attributes to form inputs (#371) - Added `DialogDescription` to dialogs for Radix UI accessibility compliance (#371) - Fixed "Collapsible is changing from uncontrolled to controlled" warning in SettingsForm (#371) - Fixed lint command for Next.js 16 compatibility (`eslint` instead of `next lint`) ### Changed - Dockerfile optimizations: better layer caching, `--no-install-recommends` for smaller images (#371) - Dockerfile.single refactored into 3 separate build stages for better caching (#371) ## [1.4.0] - 2026-01-14 ### Added - CTA button to empty state notebook list for better onboarding (#408) - Offline deployment support for Docker containers (#414) ### Fixed - Large file uploads (>10MB) by upgrading to Next.js 16 (#423) - Orphaned uploaded files when sources are removed (#421) - Broken documentation links to ai-providers.md (#419) - ZIP support indication removed from UI (#418) - Duplicate Claude Code workflow runs on PRs (#417) - Claude Code review workflow now runs on PRs from forks (#416) ### Changed - Upgraded Next.js from 15.4.10 to 16.1.1 (#423) - Upgraded React from 19.1.0 to 19.2.3 (#423) - Renamed `middleware.ts` to `proxy.ts` for Next.js 16 compatibility (#423) ### Dependencies - next: 15.4.10 → 16.1.1 - react: 19.1.0 → 19.2.3 - react-dom: 19.1.0 → 19.2.3 ## [1.2.4] - 2025-12-14 ### Added - Infinite scroll for notebook sources - no more 50 source limit (#325) - Markdown table rendering in chat responses, search results, and insights (#325) ### Fixed - Timeout errors with Ollama and local LLMs - increased to 10 minutes (#325) - "Unable to Connect to API Server" on Docker startup - frontend now waits for API health check (#325, #315) - SSL issues with langchain (#274) - Query key consistency for source mutations to properly refresh infinite scroll (#325) - Docker compose start-all flow (#323) ### Changed - Timeout configuration now uses granular httpx.Timeout (short connect, long read) (#325) ### Dependencies - Updated next.js to 15.4.10 - Updated httpx to >=0.27.0 for SSL fix ================================================ FILE: CLAUDE.md ================================================ # Open Notebook - Root CLAUDE.md This file provides architectural guidance for contributors working on Open Notebook at the project level. ## Project Overview **Open Notebook** is an open-source, privacy-focused alternative to Google's Notebook LM. It's an AI-powered research assistant enabling users to upload multi-modal content (PDFs, audio, video, web pages), generate intelligent notes, search semantically, chat with AI models, and produce professional podcasts—all with complete control over data and choice of AI providers. **Key Values**: Privacy-first, multi-provider AI support, fully self-hosted option, open-source transparency. --- ## Three-Tier Architecture ``` ┌─────────────────────────────────────────────────────────┐ │ Frontend (React/Next.js) │ │ frontend/ @ port 3000 │ ├─────────────────────────────────────────────────────────┤ │ - Notebooks, sources, notes, chat, podcasts, search UI │ │ - Zustand state management, TanStack Query (React Query)│ │ - Shadcn/ui component library with Tailwind CSS │ └────────────────────────┬────────────────────────────────┘ │ HTTP REST ┌────────────────────────▼────────────────────────────────┐ │ API (FastAPI) │ │ api/ @ port 5055 │ ├─────────────────────────────────────────────────────────┤ │ - REST endpoints for notebooks, sources, notes, chat │ │ - LangGraph workflow orchestration │ │ - Job queue for async operations (podcasts) │ │ - Multi-provider AI provisioning via Esperanto │ └────────────────────────┬────────────────────────────────┘ │ SurrealQL ┌────────────────────────▼────────────────────────────────┐ │ Database (SurrealDB) │ │ Graph database @ port 8000 │ ├─────────────────────────────────────────────────────────┤ │ - Records: Notebook, Source, Note, ChatSession, Credential│ │ - Relationships: source-to-notebook, note-to-source │ │ - Vector embeddings for semantic search │ └─────────────────────────────────────────────────────────┘ ``` --- ## Useful sources User documentation is at @docs/ ## Tech Stack ### Frontend (`frontend/`) - **Framework**: Next.js 16 (React 19) - **Language**: TypeScript - **State Management**: Zustand - **Data Fetching**: TanStack Query (React Query) - **Styling**: Tailwind CSS + Shadcn/ui - **Build Tool**: Webpack (via Next.js) - **i18n compatible**: All front-end changes must also consider the translation keys ### API Backend (`api/` + `open_notebook/`) - **Framework**: FastAPI 0.104+ - **Language**: Python 3.11+ - **Workflows**: LangGraph state machines - **Database**: SurrealDB async driver - **AI Providers**: Esperanto library (8+ providers: OpenAI, Anthropic, Google, Groq, Ollama, Mistral, DeepSeek, xAI) - **Job Queue**: Surreal-Commands for async jobs (podcasts) - **Logging**: Loguru - **Validation**: Pydantic v2 - **Testing**: Pytest ### Database - **SurrealDB**: Graph database with built-in embedding storage and vector search - **Schema Migrations**: Automatic on API startup via AsyncMigrationManager ### Additional Services - **Content Processing**: content-core library (file/URL extraction) - **Prompts**: AI-Prompter with Jinja2 templating - **Podcast Generation**: podcast-creator library - **Embeddings**: Multi-provider via Esperanto --- ## Architecture Highlights ### 1. Async-First Design - All database queries, graph invocations, and API calls are async (await) - SurrealDB async driver with connection pooling - FastAPI handles concurrent requests efficiently ### 2. LangGraph Workflows - **source.py**: Content ingestion (extract → embed → save) - **chat.py**: Conversational agent with message history - **ask.py**: Search + synthesis (retrieve relevant sources → LLM) - **transformation.py**: Custom transformations on sources - All use `provision_langchain_model()` for smart model selection ### 3. Multi-Provider AI - **Esperanto library**: Unified interface to 8+ AI providers - **Credential system**: Individual encrypted credential records per provider; models link to credentials for direct config - **ModelManager**: Factory pattern with fallback logic; uses credential config when available, env vars as fallback - **Smart selection**: Detects large contexts, prefers long-context models - **Override support**: Per-request model configuration ### 4. Database Schema - **Automatic migrations**: AsyncMigrationManager runs on API startup - **SurrealDB graph model**: Records with relationships and embeddings - **Vector search**: Built-in semantic search across all content - **Transactions**: Repo functions handle ACID operations ### 5. Authentication - **Current**: Simple password middleware (insecure, dev-only) - **Production**: Replace with OAuth/JWT (see CONFIGURATION.md) --- ## Important Quirks & Gotchas ### API Startup - **Migrations run automatically** on startup; check logs for errors - **Must start API before UI**: UI depends on API for all data - **SurrealDB must be running**: API fails without database connection ### Frontend-Backend Communication - **Base API URL**: Configured in `.env.local` (default: http://localhost:5055) - **CORS enabled**: Configured in `api/main.py` (allow all origins in dev) - **Rate limiting**: Not built-in; add at proxy layer for production ### LangGraph Workflows - **Blocking operations**: Chat/podcast workflows may take minutes; no timeout - **State persistence**: Uses SQLite checkpoint storage in `/data/sqlite-db/` - **Model fallback**: If primary model fails, falls back to cheaper/smaller model ### Podcast Generation - **Async job queue**: `podcast_service.py` submits jobs but doesn't wait - **Track status**: Use `/commands/{command_id}` endpoint to poll status - **TTS failures**: Fall back to silent audio if speech synthesis fails ### Content Processing - **File extraction**: Uses content-core library; supports 50+ file types - **URL handling**: Extracts text + metadata from web pages - **Large files**: Content processing is sync; may block API briefly --- ## Component References See dedicated CLAUDE.md files for detailed guidance: - **[frontend/CLAUDE.md](frontend/CLAUDE.md)**: React/Next.js architecture, state management, API integration - **[api/CLAUDE.md](api/CLAUDE.md)**: FastAPI structure, service pattern, endpoint development - **[open_notebook/CLAUDE.md](open_notebook/CLAUDE.md)**: Backend core, domain models, LangGraph workflows, AI provisioning - **[open_notebook/domain/CLAUDE.md](open_notebook/domain/CLAUDE.md)**: Data models, repository pattern, search functions - **[open_notebook/ai/CLAUDE.md](open_notebook/ai/CLAUDE.md)**: ModelManager, AI provider integration, Esperanto usage - **[open_notebook/graphs/CLAUDE.md](open_notebook/graphs/CLAUDE.md)**: LangGraph workflow design, state machines - **[open_notebook/database/CLAUDE.md](open_notebook/database/CLAUDE.md)**: SurrealDB operations, migrations, async patterns --- ## Documentation Map - **[README.md](README.md)**: Project overview, features, quick start - **[docs/index.md](docs/index.md)**: Complete user & deployment documentation - **[CONFIGURATION.md](CONFIGURATION.md)**: Environment variables, model configuration - **[CONTRIBUTING.md](CONTRIBUTING.md)**: Contribution guidelines - **[MAINTAINER_GUIDE.md](MAINTAINER_GUIDE.md)**: Release & maintenance procedures --- ## Testing Strategy - **Unit tests**: `tests/test_domain.py`, `test_models_api.py` - **Graph tests**: `tests/test_graphs.py` (workflow integration) - **Utils tests**: `tests/test_utils.py`, `tests/test_chunking.py`, `tests/test_embedding.py` - **Run all**: `uv run pytest tests/` - **Coverage**: Check with `pytest --cov` --- ## Common Tasks ### Add a New API Endpoint 1. Create router in `api/routers/feature.py` 2. Create service in `api/feature_service.py` 3. Define schemas in `api/models.py` 4. Register router in `api/main.py` 5. Test via http://localhost:5055/docs ### Add a New LangGraph Workflow 1. Create `open_notebook/graphs/workflow_name.py` 2. Define StateDict and node functions 3. Build graph with `.add_node()` / `.add_edge()` 4. Invoke in service: `graph.ainvoke({"input": ...}, config={"..."})` 5. Test with sample data in `tests/` ### Add Database Migration 1. Create `migrations/XXX_description.surql` 2. Write SurrealQL schema changes 3. Create `migrations/XXX_description_down.surql` (optional rollback) 4. API auto-detects on startup; migration runs if newer than recorded version ### Deploy to Production 1. Review [CONFIGURATION.md](CONFIGURATION.md) for security settings 2. Use `make docker-release` for multi-platform image 3. Push to Docker Hub / GitHub Container Registry 4. Deploy `docker compose --profile multi up` 5. Verify migrations via API logs --- ## Support & Community - **Documentation**: https://open-notebook.ai - **Discord**: https://discord.gg/37XJPXfz2w - **Issues**: https://github.com/lfnovo/open-notebook/issues - **License**: MIT (see LICENSE) ================================================ FILE: CONFIGURATION.md ================================================ # Configuration Guide **📍 This file has moved!** All configuration documentation has been consolidated into the new documentation structure. 👉 **[Read the Configuration Guide](docs/5-CONFIGURATION/index.md)** --- ## Quick Links - **AI Provider Setup** → [AI Providers](docs/5-CONFIGURATION/ai-providers.md) - **Environment Variables Reference** → [Environment Reference](docs/5-CONFIGURATION/environment-reference.md) - **Database Configuration** → [Database Setup](docs/5-CONFIGURATION/database.md) - **Server Configuration** → [Server Settings](docs/5-CONFIGURATION/server.md) - **Security Setup** → [Security Configuration](docs/5-CONFIGURATION/security.md) - **Reverse Proxy** → [Reverse Proxy Setup](docs/5-CONFIGURATION/reverse-proxy.md) - **Advanced Tuning** → [Advanced Configuration](docs/5-CONFIGURATION/advanced.md) --- ## What You'll Find The new configuration documentation includes: - **Complete environment variable reference** with examples - **Provider-specific setup guides** for OpenAI, Anthropic, Google, Groq, Ollama, and more - **Production deployment configurations** with security best practices - **Reverse proxy examples** for Nginx, Caddy, Traefik - **Database tuning** for performance optimization - **Troubleshooting guides** for common configuration issues --- For all configuration details, see **[docs/5-CONFIGURATION/](docs/5-CONFIGURATION/index.md)**. ================================================ FILE: CONTRIBUTING.md ================================================ # Contributing to Open Notebook **📍 This file has moved!** All contribution guidelines have been consolidated into the new development documentation structure. 👉 **[Read the Contributing Guide](docs/7-DEVELOPMENT/contributing.md)** --- ## Quick Links - **Want to contribute code?** → [Contributing Guide](docs/7-DEVELOPMENT/contributing.md) - **Want to understand the architecture?** → [Architecture Overview](docs/7-DEVELOPMENT/architecture.md) - **Want to understand our design philosophy?** → [Design Principles](docs/7-DEVELOPMENT/design-principles.md) - **Are you a maintainer?** → [Maintainer Guide](docs/7-DEVELOPMENT/maintainer-guide.md) - **New developer?** → [Quick Start](docs/7-DEVELOPMENT/quick-start.md) --- ## The Issue-First Workflow **TL;DR**: Create an issue first, get it assigned, THEN code. This prevents wasted effort and ensures your work aligns with the project. [See details →](docs/7-DEVELOPMENT/contributing.md) --- For all contribution details, see **[docs/7-DEVELOPMENT/contributing.md](docs/7-DEVELOPMENT/contributing.md)**. ================================================ FILE: Dockerfile ================================================ # Build stage FROM python:3.12-slim-bookworm AS builder # Install uv using the official method COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/ # Install system dependencies required for building certain Python packages # Add Node.js 20.x LTS for building frontend # NOTE: gcc/g++/make removed - uv should download pre-built wheels. Add back if build fails. # NOTE: gcc/g++/make required for some python dependencies RUN apt-get update && apt-get install -y --no-install-recommends \ curl \ build-essential \ && curl -fsSL https://deb.nodesource.com/setup_20.x | bash - \ && apt-get install -y nodejs \ && rm -rf /var/lib/apt/lists/* # Set build optimization environment variables ENV MAKEFLAGS="-j$(nproc)" ENV PYTHONDONTWRITEBYTECODE=1 ENV PYTHONUNBUFFERED=1 ENV UV_COMPILE_BYTECODE=1 ENV UV_LINK_MODE=copy # Set the working directory in the container to /app WORKDIR /app # Copy dependency files and minimal package structure first for better layer caching COPY pyproject.toml uv.lock ./ COPY open_notebook/__init__.py ./open_notebook/__init__.py # Install dependencies with optimizations (this layer will be cached unless dependencies change) RUN uv sync --frozen --no-dev # Pre-download tiktoken encoding so the app works offline (issue #264). # /app/tiktoken-cache is intentionally outside /app/data/ so that volume mounts # of /app/data (for user data persistence) do not hide the pre-baked encoding. # config.py reads TIKTOKEN_CACHE_DIR from the environment to pick up this path. ENV TIKTOKEN_CACHE_DIR=/app/tiktoken-cache RUN mkdir -p /app/tiktoken-cache && \ .venv/bin/python -c "import tiktoken; tiktoken.get_encoding('o200k_base')" # Copy the rest of the application code COPY . /app # Install frontend dependencies and build WORKDIR /app/frontend ARG NPM_REGISTRY=https://registry.npmjs.org/ COPY frontend/package.json frontend/package-lock.json ./ RUN npm config set registry ${NPM_REGISTRY} RUN npm ci COPY frontend/ ./ RUN npm run build # Return to app root WORKDIR /app # Runtime stage FROM python:3.12-slim-bookworm AS runtime # Install only runtime system dependencies (no build tools) # Add Node.js 20.x LTS for running frontend RUN apt-get update && apt-get upgrade -y && apt-get install -y --no-install-recommends \ ffmpeg \ supervisor \ curl \ && curl -fsSL https://deb.nodesource.com/setup_20.x | bash - \ && apt-get install -y nodejs \ && rm -rf /var/lib/apt/lists/* # Install uv using the official method COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/ # Set the working directory in the container to /app WORKDIR /app # Copy the virtual environment from builder stage COPY --from=builder /app/.venv /app/.venv # Copy the source code (the rest) COPY . /app # Copy pre-downloaded tiktoken encoding from builder (outside /data/ — volume-mount safe) COPY --from=builder /app/tiktoken-cache /app/tiktoken-cache # Ensure uv uses the existing venv without attempting network operations ENV UV_NO_SYNC=1 ENV VIRTUAL_ENV=/app/.venv # Point the app at the pre-baked tiktoken encoding (see open_notebook/config.py) ENV TIKTOKEN_CACHE_DIR=/app/tiktoken-cache # Bind Next.js to all interfaces (required for Docker networking and reverse proxies) ENV HOSTNAME=0.0.0.0 # Copy built frontend from builder stage COPY --from=builder /app/frontend/.next/standalone /app/frontend/ COPY --from=builder /app/frontend/.next/static /app/frontend/.next/static COPY --from=builder /app/frontend/public /app/frontend/public COPY --from=builder /app/frontend/start-server.js /app/frontend/start-server.js # Expose ports for Frontend and API EXPOSE 8502 5055 RUN mkdir -p /app/data # Copy and make executable the wait-for-api script COPY scripts/wait-for-api.sh /app/scripts/wait-for-api.sh RUN chmod +x /app/scripts/wait-for-api.sh # Copy supervisord configuration COPY supervisord.conf /etc/supervisor/conf.d/supervisord.conf # Create log directories RUN mkdir -p /var/log/supervisor # Runtime API URL Configuration # The API_URL environment variable can be set at container runtime to configure # where the frontend should connect to the API. This allows the same Docker image # to work in different deployment scenarios without rebuilding. # # If not set, the system will auto-detect based on incoming requests. # Set API_URL when using reverse proxies or custom domains. # # Example: docker run -e API_URL=https://your-domain.com/api ... CMD ["/usr/bin/supervisord", "-c", "/etc/supervisor/conf.d/supervisord.conf"] ================================================ FILE: Dockerfile.single ================================================ # Stage 1: Frontend Builder FROM node:20-slim AS frontend-builder WORKDIR /app/frontend # Copy dependency files first to leverage cache COPY frontend/package.json frontend/package-lock.json ./ ARG NPM_REGISTRY=https://registry.npmjs.org/ RUN npm config set registry ${NPM_REGISTRY} RUN npm ci # Copy the rest of the frontend source COPY frontend/ ./ # Build the frontend RUN npm run build # Stage 2: SurrealDB binary (pinned to v2 to match docker-compose.yml) FROM surrealdb/surrealdb:v2 AS surreal-binary # Stage 4: Backend Builder FROM python:3.12-slim-bookworm AS backend-builder # Install build dependencies RUN apt-get update && apt-get upgrade -y && apt-get install -y --no-install-recommends build-essential && rm -rf /var/lib/apt/lists/* # Install uv COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/ WORKDIR /app # Set build optimization environment variables ENV UV_HTTP_TIMEOUT=120 # Copy dependency files first COPY pyproject.toml uv.lock ./ COPY open_notebook/__init__.py ./open_notebook/__init__.py # Install dependencies RUN uv sync --frozen --no-dev # Pre-download tiktoken encoding so the app works offline (issue #264). # /app/tiktoken-cache is intentionally outside /app/data/ so that volume mounts # of /app/data (for user data persistence) do not hide the pre-baked encoding. # config.py reads TIKTOKEN_CACHE_DIR from the environment to pick up this path. ENV TIKTOKEN_CACHE_DIR=/app/tiktoken-cache RUN mkdir -p /app/tiktoken-cache && \ .venv/bin/python -c "import tiktoken; tiktoken.get_encoding('o200k_base')" # Stage 5: Runtime FROM python:3.12-slim-bookworm AS runtime # Install runtime dependencies RUN apt-get update && apt-get upgrade -y && apt-get install -y \ ffmpeg \ supervisor \ curl \ && curl -fsSL https://deb.nodesource.com/setup_20.x | bash - \ && apt-get install -y nodejs \ && rm -rf /var/lib/apt/lists/* # Install SurrealDB (copied from pinned v2 image to match docker-compose.yml) COPY --from=surreal-binary /surreal /usr/local/bin/surreal # Install uv (optional but helpful for some scripts) COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/ WORKDIR /app # Copy backend virtualenv and source code COPY --from=backend-builder /app/.venv /app/.venv COPY . /app/ # Copy pre-downloaded tiktoken encoding from builder (outside /data/ — volume-mount safe) COPY --from=backend-builder /app/tiktoken-cache /app/tiktoken-cache # Copy built frontend from standalone output COPY --from=frontend-builder /app/frontend/.next/standalone /app/frontend/ COPY --from=frontend-builder /app/frontend/.next/static /app/frontend/.next/static COPY --from=frontend-builder /app/frontend/public /app/frontend/public # Bind Next.js to all interfaces (required for Docker networking and reverse proxies) ENV HOSTNAME=0.0.0.0 # Point the app at the pre-baked tiktoken encoding (see open_notebook/config.py) ENV TIKTOKEN_CACHE_DIR=/app/tiktoken-cache # Setup directories and permissions RUN mkdir -p /app/data /mydata # Ensure wait-for-api script is executable RUN chmod +x /app/scripts/wait-for-api.sh # Copy supervisord configuration COPY supervisord.single.conf /etc/supervisor/conf.d/supervisord.conf # Create log directories RUN mkdir -p /var/log/supervisor # Expose ports EXPOSE 8502 5055 # Set startup command CMD ["/usr/bin/supervisord", "-c", "/etc/supervisor/conf.d/supervisord.conf"] ================================================ FILE: LICENSE ================================================ MIT License Copyright (c) 2024 Luis Novo Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ================================================ FILE: MAINTAINER_GUIDE.md ================================================ # Maintainer Guide **📍 This file has moved!** All maintainer guidelines have been consolidated into the new development documentation structure. 👉 **[Read the Maintainer Guide](docs/7-DEVELOPMENT/maintainer-guide.md)** --- ## Quick Links - **Maintainer Guide** → [docs/7-DEVELOPMENT/maintainer-guide.md](docs/7-DEVELOPMENT/maintainer-guide.md) - **Contributing Guide** → [docs/7-DEVELOPMENT/contributing.md](docs/7-DEVELOPMENT/contributing.md) - **Design Principles** → [docs/7-DEVELOPMENT/design-principles.md](docs/7-DEVELOPMENT/design-principles.md) --- For all maintainer details, see **[docs/7-DEVELOPMENT/maintainer-guide.md](docs/7-DEVELOPMENT/maintainer-guide.md)**. ================================================ FILE: Makefile ================================================ .PHONY: run frontend check ruff database lint api start-all stop-all status clean-cache worker worker-start worker-stop worker-restart .PHONY: docker-buildx-prepare docker-buildx-clean docker-buildx-reset .PHONY: docker-push docker-push-latest docker-release docker-build-local tag export-docs # Get version from pyproject.toml VERSION := $(shell grep -m1 version pyproject.toml | cut -d'"' -f2) # Image names for both registries DOCKERHUB_IMAGE := lfnovo/open_notebook GHCR_IMAGE := ghcr.io/lfnovo/open-notebook # Build platforms PLATFORMS := linux/amd64,linux/arm64 database: docker compose up -d surrealdb run: @echo "⚠️ Warning: Starting frontend only. For full functionality, use 'make start-all'" cd frontend && npm run dev frontend: cd frontend && npm run dev lint: uv run python -m mypy . ruff: ruff check . --fix # === Docker Build Setup === docker-buildx-prepare: @docker buildx inspect multi-platform-builder >/dev/null 2>&1 || \ docker buildx create --use --name multi-platform-builder --driver docker-container @docker buildx use multi-platform-builder docker-buildx-clean: @echo "🧹 Cleaning up buildx builders..." @docker buildx rm multi-platform-builder 2>/dev/null || true @docker ps -a | grep buildx_buildkit | awk '{print $$1}' | xargs -r docker rm -f 2>/dev/null || true @echo "✅ Buildx cleanup complete!" docker-buildx-reset: docker-buildx-clean docker-buildx-prepare @echo "✅ Buildx reset complete!" # === Docker Build Targets === # Build production image for local platform only (no push) docker-build-local: @echo "🔨 Building production image locally ($(shell uname -m))..." docker build \ -t $(DOCKERHUB_IMAGE):$(VERSION) \ -t $(DOCKERHUB_IMAGE):local \ . @echo "✅ Built $(DOCKERHUB_IMAGE):$(VERSION) and $(DOCKERHUB_IMAGE):local" @echo "Run with: docker run -p 5055:5055 -p 3000:3000 $(DOCKERHUB_IMAGE):local" # Build and push version tags ONLY (no latest) for both regular and single images docker-push: docker-buildx-prepare @echo "📤 Building and pushing version $(VERSION) to both registries..." @echo "🔨 Building regular image..." docker buildx build --pull \ --platform $(PLATFORMS) \ --progress=plain \ -t $(DOCKERHUB_IMAGE):$(VERSION) \ -t $(GHCR_IMAGE):$(VERSION) \ --push \ . @echo "🔨 Building single-container image..." docker buildx build --pull \ --platform $(PLATFORMS) \ --progress=plain \ -f Dockerfile.single \ -t $(DOCKERHUB_IMAGE):$(VERSION)-single \ -t $(GHCR_IMAGE):$(VERSION)-single \ --push \ . @echo "✅ Pushed version $(VERSION) to both registries (latest NOT updated)" @echo " 📦 Docker Hub:" @echo " - $(DOCKERHUB_IMAGE):$(VERSION)" @echo " - $(DOCKERHUB_IMAGE):$(VERSION)-single" @echo " 📦 GHCR:" @echo " - $(GHCR_IMAGE):$(VERSION)" @echo " - $(GHCR_IMAGE):$(VERSION)-single" # Update v1-latest tags to current version (both regular and single images) docker-push-latest: docker-buildx-prepare @echo "📤 Updating v1-latest tags to version $(VERSION)..." @echo "🔨 Building regular image with latest tag..." docker buildx build --pull \ --platform $(PLATFORMS) \ --progress=plain \ -t $(DOCKERHUB_IMAGE):$(VERSION) \ -t $(DOCKERHUB_IMAGE):v1-latest \ -t $(GHCR_IMAGE):$(VERSION) \ -t $(GHCR_IMAGE):v1-latest \ --push \ . @echo "🔨 Building single-container image with latest tag..." docker buildx build --pull \ --platform $(PLATFORMS) \ --progress=plain \ -f Dockerfile.single \ -t $(DOCKERHUB_IMAGE):$(VERSION)-single \ -t $(DOCKERHUB_IMAGE):v1-latest-single \ -t $(GHCR_IMAGE):$(VERSION)-single \ -t $(GHCR_IMAGE):v1-latest-single \ --push \ . @echo "✅ Updated v1-latest to version $(VERSION)" @echo " 📦 Docker Hub:" @echo " - $(DOCKERHUB_IMAGE):$(VERSION) → v1-latest" @echo " - $(DOCKERHUB_IMAGE):$(VERSION)-single → v1-latest-single" @echo " 📦 GHCR:" @echo " - $(GHCR_IMAGE):$(VERSION) → v1-latest" @echo " - $(GHCR_IMAGE):$(VERSION)-single → v1-latest-single" # Full release: push version AND update latest tags docker-release: docker-push-latest @echo "✅ Full release complete for version $(VERSION)" tag: @version=$$(grep '^version = ' pyproject.toml | sed 's/version = "\(.*\)"/\1/'); \ echo "Creating tag v$$version"; \ git tag "v$$version"; \ git push origin "v$$version" dev: docker compose -f docker-compose.dev.yml up --build full: docker compose -f docker-compose.full.yml up --build api: uv run --env-file .env run_api.py .PHONY: worker worker-start worker-stop worker-restart worker: worker-start worker-start: @echo "Starting surreal-commands worker..." uv run --env-file .env surreal-commands-worker --import-modules commands worker-stop: @echo "Stopping surreal-commands worker..." pkill -f "surreal-commands-worker" || true worker-restart: worker-stop @sleep 2 @$(MAKE) worker-start # === Service Management === start-all: @echo "🚀 Starting Open Notebook (Database + API + Worker + Frontend)..." @echo "📊 Starting SurrealDB..." @docker compose -f docker-compose.dev.yml up -d surrealdb @sleep 3 @echo "🔧 Starting API backend..." @uv run run_api.py & @sleep 3 @echo "⚙️ Starting background worker..." @uv run --env-file .env surreal-commands-worker --import-modules commands & @sleep 2 @echo "🌐 Starting Next.js frontend..." @echo "✅ All services started!" @echo "📱 Frontend: http://localhost:3000" @echo "🔗 API: http://localhost:5055" @echo "📚 API Docs: http://localhost:5055/docs" cd frontend && npm run dev stop-all: @echo "🛑 Stopping all Open Notebook services..." @pkill -f "next dev" || true @pkill -f "surreal-commands-worker" || true @pkill -f "run_api.py" || true @pkill -f "uvicorn api.main:app" || true @docker compose down @echo "✅ All services stopped!" status: @echo "📊 Open Notebook Service Status:" @echo "Database (SurrealDB):" @docker compose ps surrealdb 2>/dev/null || echo " ❌ Not running" @echo "API Backend:" @pgrep -f "run_api.py\|uvicorn api.main:app" >/dev/null && echo " ✅ Running" || echo " ❌ Not running" @echo "Background Worker:" @pgrep -f "surreal-commands-worker" >/dev/null && echo " ✅ Running" || echo " ❌ Not running" @echo "Next.js Frontend:" @pgrep -f "next dev" >/dev/null && echo " ✅ Running" || echo " ❌ Not running" # === Documentation Export === export-docs: @echo "📚 Exporting documentation..." @uv run python scripts/export_docs.py @echo "✅ Documentation export complete!" # === Cleanup === clean-cache: @echo "🧹 Cleaning cache directories..." @find . -name "__pycache__" -type d -exec rm -rf {} + 2>/dev/null || true @find . -name ".mypy_cache" -type d -exec rm -rf {} + 2>/dev/null || true @find . -name ".ruff_cache" -type d -exec rm -rf {} + 2>/dev/null || true @find . -name ".pytest_cache" -type d -exec rm -rf {} + 2>/dev/null || true @find . -name "*.pyc" -type f -delete 2>/dev/null || true @find . -name "*.pyo" -type f -delete 2>/dev/null || true @find . -name "*.pyd" -type f -delete 2>/dev/null || true @echo "✅ Cache directories cleaned!" ================================================ FILE: README.dev.md ================================================ # Developer Guide This guide is for developers working on Open Notebook. For end-user documentation, see [README.md](README.md) and [docs/](docs/). ## Quick Start for Development ```bash # 1. Clone and setup git clone https://github.com/lfnovo/open-notebook.git cd open-notebook # 2. Copy environment files cp .env.example .env cp .env.example docker.env # 3. Install dependencies uv sync # 4. Start all services (recommended for development) make start-all ``` ## Development Workflows ### When to Use What? | Workflow | Use Case | Speed | Production Parity | |----------|----------|-------|-------------------| | **Local Services** (`make start-all`) | Day-to-day development, fastest iteration | ⚡⚡⚡ Fast | Medium | | **Docker Compose** (`make dev`) | Testing containerized setup | ⚡⚡ Medium | High | | **Local Docker Build** (`make docker-build-local`) | Testing Dockerfile changes | ⚡ Slow | Very High | | **Multi-platform Build** (`make docker-push`) | Publishing releases | 🐌 Very Slow | Exact | --- ## 1. Local Development (Recommended) **Best for:** Daily development, hot reload, debugging ### Setup ```bash # Start database make database # Start all services (DB + API + Worker + Frontend) make start-all ``` ### What This Does 1. Starts SurrealDB in Docker (port 8000) 2. Starts FastAPI backend (port 5055) 3. Starts background worker (surreal-commands) 4. Starts Next.js frontend (port 3000) ### Individual Services ```bash # Just the database make database # Just the API make api # Just the frontend make frontend # Just the worker make worker ``` ### Checking Status ```bash # See what's running make status # Stop everything make stop-all ``` ### Advantages - ✅ Fastest iteration (hot reload) - ✅ Easy debugging (direct process access) - ✅ Low resource usage - ✅ Direct log access ### Disadvantages - ❌ Doesn't test Docker build - ❌ Environment may differ from production - ❌ Requires local Python/Node setup --- ## 2. Docker Compose Development **Best for:** Testing containerized setup, CI/CD verification ```bash # Start with dev profile make dev # Or full stack make full ``` ### Configuration Files - `docker-compose.dev.yml` - Development setup - `docker-compose.full.yml` - Full stack setup - `docker-compose.yml` - Base configuration ### Advantages - ✅ Closer to production environment - ✅ Isolated dependencies - ✅ Easy to share exact environment ### Disadvantages - ❌ Slower rebuilds - ❌ More complex debugging - ❌ Higher resource usage --- ## 3. Testing Production Docker Images **Best for:** Verifying Dockerfile changes before publishing ### Build Locally ```bash # Build production image for your platform only make docker-build-local ``` This creates two tags: - `lfnovo/open_notebook:` (from pyproject.toml) - `lfnovo/open_notebook:local` ### Run Locally ```bash docker run -p 5055:5055 -p 3000:3000 lfnovo/open_notebook:local ``` ### When to Use - ✅ Before pushing to registry - ✅ Testing Dockerfile changes - ✅ Debugging production-specific issues - ✅ Verifying build process --- ## 4. Publishing Docker Images ### Workflow ```bash # 1. Test locally first make docker-build-local # 2. If successful, push version tag (no latest update) make docker-push # 3. Test the pushed version in staging/production # 4. When ready, promote to latest make docker-push-latest ``` ### Available Commands | Command | What It Does | Updates Latest? | |---------|--------------|-----------------| | `make docker-build-local` | Build for current platform only | No registry push | | `make docker-push` | Push version tags to registries | ❌ No | | `make docker-push-latest` | Push version + update v1-latest | ✅ Yes | | `make docker-release` | Full release (same as docker-push-latest) | ✅ Yes | ### Publishing Details - **Platforms:** `linux/amd64`, `linux/arm64` - **Registries:** Docker Hub + GitHub Container Registry - **Image Variants:** Regular + Single-container (`-single`) - **Version Source:** `pyproject.toml` ### Creating Git Tags ```bash # Create and push git tag matching pyproject.toml version make tag ``` --- ## Code Quality ```bash # Run linter with auto-fix make ruff # Run type checking make lint # Run tests uv run pytest tests/ # Clean cache directories make clean-cache ``` --- ## Common Development Tasks ### Adding a New Feature 1. Create feature branch 2. Develop using `make start-all` 3. Write tests 4. Run `make ruff` and `make lint` 5. Test with `make docker-build-local` 6. Create PR ### Fixing a Bug 1. Reproduce locally with `make start-all` 2. Add test case demonstrating bug 3. Fix the bug 4. Verify test passes 5. Check with `make docker-build-local` ### Updating Dependencies ```bash # Add Python dependency uv add package-name # Update dependencies uv sync # Frontend dependencies cd frontend && npm install package-name ``` ### Adding a New Language (i18n) Open Notebook supports internationalization. To add a new language: 1. **Create locale file**: Copy an existing locale as template ```bash cp frontend/src/lib/locales/en-US/index.ts frontend/src/lib/locales/pt-BR/index.ts ``` 2. **Translate all strings** in the new file. The structure includes: - `common`: Shared UI elements (buttons, labels) - `notebooks`, `sources`, `notes`: Feature-specific strings - `chat`, `search`, `podcasts`: Module-specific strings - `apiErrors`: Error message translations 3. **Register the locale** in `frontend/src/lib/locales/index.ts`: ```typescript import { ptBR } from './pt-BR' export const locales = { 'en-US': enUS, 'zh-CN': zhCN, 'zh-TW': zhTW, 'pt-BR': ptBR, // Add your locale } ``` 4. **Add date-fns locale** in `frontend/src/lib/utils/date-locale.ts`: ```typescript import { zhCN, enUS, zhTW, ptBR } from 'date-fns/locale' const LOCALE_MAP: Record = { 'zh-CN': zhCN, 'zh-TW': zhTW, 'en-US': enUS, 'pt-BR': ptBR, // Add your locale } ``` 5. **Test**: Switch languages using the language toggle in the UI header. ### Database Migrations Database migrations run **automatically** when the API starts. 1. Create migration file: `migrations/XXX_description.surql` 2. Write SurrealQL schema changes 3. (Optional) Create rollback: `migrations/XXX_description_down.surql` 4. Restart API - migration runs on startup --- ## Troubleshooting ### Services Won't Start ```bash # Check status make status # Check database docker compose ps surrealdb # View logs docker compose logs surrealdb # Restart everything make stop-all make start-all ``` ### Port Already in Use ```bash # Find process using port lsof -i :5055 lsof -i :3000 lsof -i :8000 # Kill stuck processes make stop-all ``` ### Database Connection Issues ```bash # Verify SurrealDB is running docker compose ps surrealdb # Check connection settings in .env cat .env | grep SURREAL ``` ### Docker Build Fails ```bash # Clean Docker cache docker builder prune # Reset buildx make docker-buildx-reset # Try local build first make docker-build-local ``` --- ## Project Structure ``` open-notebook/ ├── api/ # FastAPI backend ├── frontend/ # Next.js React frontend ├── open_notebook/ # Python core library │ ├── domain/ # Domain models │ ├── graphs/ # LangGraph workflows │ ├── ai/ # AI provider integration │ └── database/ # SurrealDB operations ├── migrations/ # Database migrations ├── tests/ # Test suite ├── docs/ # User documentation └── Makefile # Development commands ``` See component-specific CLAUDE.md files for detailed architecture: - [frontend/CLAUDE.md](frontend/CLAUDE.md) - [api/CLAUDE.md](api/CLAUDE.md) - [open_notebook/CLAUDE.md](open_notebook/CLAUDE.md) --- ## Environment Variables ### Required for Local Development ```bash # .env file SURREAL_URL=ws://localhost:8000 SURREAL_USER=root SURREAL_PASS=root SURREAL_DB=open_notebook SURREAL_NS=production # AI Provider (at least one required) OPENAI_API_KEY=sk-... # OR ANTHROPIC_API_KEY=sk-ant-... # OR configure other providers (see docs/5-CONFIGURATION/) ``` See [docs/5-CONFIGURATION/](docs/5-CONFIGURATION/) for complete configuration guide. --- ## Performance Tips ### Speed Up Local Development 1. **Use `make start-all`** instead of Docker for daily work 2. **Keep SurrealDB running** between sessions (`make database`) 3. **Use `make docker-build-local`** only when testing Dockerfile changes 4. **Skip multi-platform builds** until ready to publish ### Reduce Resource Usage ```bash # Stop unused services make stop-all # Clean up Docker docker system prune -a # Clean Python cache make clean-cache ``` --- ## TODO: Sections to Add - [ ] Frontend development guide (hot reload, component structure) - [ ] API development guide (adding endpoints, services) - [ ] LangGraph workflow development - [ ] Testing strategy and coverage - [ ] Debugging tips (VSCode/PyCharm setup) - [ ] CI/CD pipeline overview - [ ] Release process checklist - [ ] Common error messages and solutions --- ## Resources - **Documentation:** https://open-notebook.ai - **Discord:** https://discord.gg/37XJPXfz2w - **Issues:** https://github.com/lfnovo/open-notebook/issues - **Contributing:** [CONTRIBUTING.md](CONTRIBUTING.md) - **Maintainer Guide:** [MAINTAINER_GUIDE.md](MAINTAINER_GUIDE.md) --- **Last Updated:** January 2025 ================================================ FILE: README.md ================================================ [![Forks][forks-shield]][forks-url] [![Stargazers][stars-shield]][stars-url] [![Issues][issues-shield]][issues-url] [![MIT License][license-shield]][license-url]
Logo

Open Notebook

An open source, privacy-focused alternative to Google's Notebook LM!
Join our Discord server for help, to share workflow ideas, and suggest features!
Checkout our website »

📚 Get Started · 📖 User Guide · ✨ Features · 🚀 Deploy

lfnovo%2Fopen-notebook | Trendshift

Deutsch | Español | français | 日本語 | 한국어 | Português | Русский | 中文
## A private, multi-model, 100% local, full-featured alternative to Notebook LM ![New Notebook](docs/assets/asset_list.png) In a world dominated by Artificial Intelligence, having the ability to think 🧠 and acquire new knowledge 💡, is a skill that should not be a privilege for a few, nor restricted to a single provider. **Open Notebook empowers you to:** - 🔒 **Control your data** - Keep your research private and secure - 🤖 **Choose your AI models** - Support for 16+ providers including OpenAI, Anthropic, Ollama, LM Studio, and more - 📚 **Organize multi-modal content** - PDFs, videos, audio, web pages, and more - 🎙️ **Generate professional podcasts** - Advanced multi-speaker podcast generation - 🔍 **Search intelligently** - Full-text and vector search across all your content - 💬 **Chat with context** - AI conversations powered by your research - 🌐 **Multi-language UI** - English, Portuguese, Chinese (Simplified & Traditional), Japanese, Russian, and Bengali support Learn more about our project at [https://www.open-notebook.ai](https://www.open-notebook.ai) --- ## 🆚 Open Notebook vs Google Notebook LM | Feature | Open Notebook | Google Notebook LM | Advantage | |---------|---------------|--------------------|-----------| | **Privacy & Control** | Self-hosted, your data | Google cloud only | Complete data sovereignty | | **AI Provider Choice** | 16+ providers (OpenAI, Anthropic, Ollama, LM Studio, etc.) | Google models only | Flexibility and cost optimization | | **Podcast Speakers** | 1-4 speakers with custom profiles | 2 speakers only | Extreme flexibility | | **Content Transformations** | Custom and built-in | Limited options | Unlimited processing power | | **API Access** | Full REST API | No API | Complete automation | | **Deployment** | Docker, cloud, or local | Google hosted only | Deploy anywhere | | **Citations** | Basic references (will improve) | Comprehensive with sources | Research integrity | | **Customization** | Open source, fully customizable | Closed system | Unlimited extensibility | | **Cost** | Pay only for AI usage | Free tier + Monthly subscription | Transparent and controllable | **Why Choose Open Notebook?** - 🔒 **Privacy First**: Your sensitive research stays completely private - 💰 **Cost Control**: Choose cheaper AI providers or run locally with Ollama - 🎙️ **Better Podcasts**: Full script control and multi-speaker flexibility vs limited 2-speaker deep-dive format - 🔧 **Unlimited Customization**: Modify, extend, and integrate as needed - 🌐 **No Vendor Lock-in**: Switch providers, deploy anywhere, own your data ### Built With [![Python][Python]][Python-url] [![Next.js][Next.js]][Next-url] [![React][React]][React-url] [![SurrealDB][SurrealDB]][SurrealDB-url] [![LangChain][LangChain]][LangChain-url] ## 🚀 Quick Start (2 Minutes) ### Prerequisites - [Docker Desktop](https://www.docker.com/products/docker-desktop/) installed - That's it! (API keys configured later in the UI) ### Step 1: Get docker-compose.yml **Option A:** Download directly ```bash curl -o docker-compose.yml https://raw.githubusercontent.com/lfnovo/open-notebook/main/docker-compose.yml ``` **Option B:** Create the file manually Copy this into a new file called `docker-compose.yml`: ```yaml services: surrealdb: image: surrealdb/surrealdb:v2 command: start --log info --user root --pass root rocksdb:/mydata/mydatabase.db user: root ports: - "8000:8000" volumes: - ./surreal_data:/mydata restart: always open_notebook: image: lfnovo/open_notebook:v1-latest ports: - "8502:8502" - "5055:5055" environment: - OPEN_NOTEBOOK_ENCRYPTION_KEY=change-me-to-a-secret-string - SURREAL_URL=ws://surrealdb:8000/rpc - SURREAL_USER=root - SURREAL_PASSWORD=root - SURREAL_NAMESPACE=open_notebook - SURREAL_DATABASE=open_notebook volumes: - ./notebook_data:/app/data depends_on: - surrealdb restart: always ``` ### Step 2: Set Your Encryption Key Edit `docker-compose.yml` and change this line: ```yaml - OPEN_NOTEBOOK_ENCRYPTION_KEY=change-me-to-a-secret-string ``` to any secret value (e.g., `my-super-secret-key-123`) ### Step 3: Start Services ```bash docker compose up -d ``` Wait 15-20 seconds, then open: **http://localhost:8502** ### Step 4: Configure AI Provider 1. Go to **Settings** → **API Keys** 2. Click **Add Credential** 3. Choose your provider (OpenAI, Anthropic, Google, etc.) 4. Paste your API key and click **Save** 5. Click **Test Connection** → **Discover Models** → **Register Models** Done! You're ready to create your first notebook. > **Need an API key?** Get one from: > [OpenAI](https://platform.openai.com/api-keys) · [Anthropic](https://console.anthropic.com/) · [Google](https://aistudio.google.com/) · [Groq](https://console.groq.com/) (free tier) > **Want free local AI?** See [examples/docker-compose-ollama.yml](examples/) for Ollama setup --- ### 📚 More Installation Options - **[With Ollama (Free Local AI)](examples/docker-compose-ollama.yml)** - Run models locally without API costs - **[From Source (Developers)](docs/1-INSTALLATION/from-source.md)** - For development and contributions - **[Complete Installation Guide](docs/1-INSTALLATION/index.md)** - All deployment scenarios --- ### 📖 Need Help? - **🤖 AI Installation Assistant**: [CustomGPT to help you install](https://chatgpt.com/g/g-68776e2765b48191bd1bae3f30212631-open-notebook-installation-assistant) - **🆘 Troubleshooting**: [5-minute troubleshooting guide](docs/6-TROUBLESHOOTING/quick-fixes.md) - **💬 Community Support**: [Discord Server](https://discord.gg/37XJPXfz2w) - **🐛 Report Issues**: [GitHub Issues](https://github.com/lfnovo/open-notebook/issues) --- ## Star History [![Star History Chart](https://api.star-history.com/svg?repos=lfnovo/open-notebook&type=date&legend=top-left)](https://www.star-history.com/#lfnovo/open-notebook&type=date&legend=top-left) ## Provider Support Matrix Thanks to the [Esperanto](https://github.com/lfnovo/esperanto) library, we support this providers out of the box! | Provider | LLM Support | Embedding Support | Speech-to-Text | Text-to-Speech | |--------------|-------------|------------------|----------------|----------------| | OpenAI | ✅ | ✅ | ✅ | ✅ | | Anthropic | ✅ | ❌ | ❌ | ❌ | | Groq | ✅ | ❌ | ✅ | ❌ | | Google (GenAI) | ✅ | ✅ | ❌ | ✅ | | Vertex AI | ✅ | ✅ | ❌ | ✅ | | Ollama | ✅ | ✅ | ❌ | ❌ | | Perplexity | ✅ | ❌ | ❌ | ❌ | | ElevenLabs | ❌ | ❌ | ✅ | ✅ | | Azure OpenAI | ✅ | ✅ | ❌ | ❌ | | Mistral | ✅ | ✅ | ❌ | ❌ | | DeepSeek | ✅ | ❌ | ❌ | ❌ | | Voyage | ❌ | ✅ | ❌ | ❌ | | xAI | ✅ | ❌ | ❌ | ❌ | | OpenRouter | ✅ | ❌ | ❌ | ❌ | | OpenAI Compatible* | ✅ | ❌ | ❌ | ❌ | *Supports LM Studio and any OpenAI-compatible endpoint ## ✨ Key Features ### Core Capabilities - **🔒 Privacy-First**: Your data stays under your control - no cloud dependencies - **🎯 Multi-Notebook Organization**: Manage multiple research projects seamlessly - **📚 Universal Content Support**: PDFs, videos, audio, web pages, Office docs, and more - **🤖 Multi-Model AI Support**: 16+ providers including OpenAI, Anthropic, Ollama, Google, LM Studio, and more - **🎙️ Professional Podcast Generation**: Advanced multi-speaker podcasts with Episode Profiles - **🔍 Intelligent Search**: Full-text and vector search across all your content - **💬 Context-Aware Chat**: AI conversations powered by your research materials - **📝 AI-Assisted Notes**: Generate insights or write notes manually ### Advanced Features - **⚡ Reasoning Model Support**: Full support for thinking models like DeepSeek-R1 and Qwen3 - **🔧 Content Transformations**: Powerful customizable actions to summarize and extract insights - **🌐 Comprehensive REST API**: Full programmatic access for custom integrations [![API Docs](https://img.shields.io/badge/API-Documentation-blue?style=flat-square)](http://localhost:5055/docs) - **🔐 Optional Password Protection**: Secure public deployments with authentication - **📊 Fine-Grained Context Control**: Choose exactly what to share with AI models - **📎 Citations**: Get answers with proper source citations ## Podcast Feature [![Check out our podcast sample](https://img.youtube.com/vi/D-760MlGwaI/0.jpg)](https://www.youtube.com/watch?v=D-760MlGwaI) ## 📚 Documentation ### Getting Started - **[📖 Introduction](docs/0-START-HERE/index.md)** - Learn what Open Notebook offers - **[⚡ Quick Start](docs/0-START-HERE/quick-start.md)** - Get up and running in 5 minutes - **[🔧 Installation](docs/1-INSTALLATION/index.md)** - Comprehensive setup guide - **[🎯 Your First Notebook](docs/0-START-HERE/first-notebook.md)** - Step-by-step tutorial ### User Guide - **[📱 Interface Overview](docs/3-USER-GUIDE/interface-overview.md)** - Understanding the layout - **[📚 Notebooks](docs/3-USER-GUIDE/notebooks.md)** - Organizing your research - **[📄 Sources](docs/3-USER-GUIDE/sources.md)** - Managing content types - **[📝 Notes](docs/3-USER-GUIDE/notes.md)** - Creating and managing notes - **[💬 Chat](docs/3-USER-GUIDE/chat.md)** - AI conversations - **[🔍 Search](docs/3-USER-GUIDE/search.md)** - Finding information ### Advanced Topics - **[🎙️ Podcast Generation](docs/2-CORE-CONCEPTS/podcasts.md)** - Create professional podcasts - **[🔧 Content Transformations](docs/2-CORE-CONCEPTS/transformations.md)** - Customize content processing - **[🤖 AI Models](docs/4-AI-PROVIDERS/index.md)** - AI model configuration - **[🔌 MCP Integration](docs/5-CONFIGURATION/mcp-integration.md)** - Connect with Claude Desktop, VS Code and other MCP clients - **[🔧 REST API Reference](docs/7-DEVELOPMENT/api-reference.md)** - Complete API documentation - **[🔐 Security](docs/5-CONFIGURATION/security.md)** - Password protection and privacy - **[🚀 Deployment](docs/1-INSTALLATION/index.md)** - Complete deployment guides for all scenarios

(back to top)

## 🗺️ Roadmap ### Upcoming Features - **Live Front-End Updates**: Real-time UI updates for smoother experience - **Async Processing**: Faster UI through asynchronous content processing - **Cross-Notebook Sources**: Reuse research materials across projects - **Bookmark Integration**: Connect with your favorite bookmarking apps ### Recently Completed ✅ - **Next.js Frontend**: Modern React-based frontend with improved performance - **Comprehensive REST API**: Full programmatic access to all functionality - **Multi-Model Support**: 16+ AI providers including OpenAI, Anthropic, Ollama, LM Studio - **Advanced Podcast Generator**: Professional multi-speaker podcasts with Episode Profiles - **Content Transformations**: Powerful customizable actions for content processing - **Enhanced Citations**: Improved layout and finer control for source citations - **Multiple Chat Sessions**: Manage different conversations within notebooks See the [open issues](https://github.com/lfnovo/open-notebook/issues) for a full list of proposed features and known issues.

(back to top)

## 📖 Need Help? - **🤖 AI Installation Assistant**: We have a [CustomGPT built to help you install Open Notebook](https://chatgpt.com/g/g-68776e2765b48191bd1bae3f30212631-open-notebook-installation-assistant) - it will guide you through each step! - **New to Open Notebook?** Start with our [Getting Started Guide](docs/0-START-HERE/index.md) - **Need installation help?** Check our [Installation Guide](docs/1-INSTALLATION/index.md) - **Want to see it in action?** Try our [Quick Start Tutorial](docs/0-START-HERE/quick-start.md) ## 🤝 Community & Contributing ### Join the Community - 💬 **[Discord Server](https://discord.gg/37XJPXfz2w)** - Get help, share ideas, and connect with other users - 🐛 **[GitHub Issues](https://github.com/lfnovo/open-notebook/issues)** - Report bugs and request features - ⭐ **Star this repo** - Show your support and help others discover Open Notebook ### Contributing We welcome contributions! We're especially looking for help with: - **Frontend Development**: Help improve our modern Next.js/React UI - **Testing & Bug Fixes**: Make Open Notebook more robust - **Feature Development**: Build the coolest research tool together - **Documentation**: Improve guides and tutorials **Current Tech Stack**: Python, FastAPI, Next.js, React, SurrealDB **Future Roadmap**: Real-time updates, enhanced async processing See our [Contributing Guide](CONTRIBUTING.md) for detailed information on how to get started.

(back to top)

## 📄 License Open Notebook is MIT licensed. See the [LICENSE](LICENSE) file for details. **Community Support**: - 💬 [Discord Server](https://discord.gg/37XJPXfz2w) - Get help, share ideas, and connect with users - 🐛 [GitHub Issues](https://github.com/lfnovo/open-notebook/issues) - Report bugs and request features - 🌐 [Website](https://www.open-notebook.ai) - Learn more about the project

(back to top)

[contributors-shield]: https://img.shields.io/github/contributors/lfnovo/open-notebook.svg?style=for-the-badge [contributors-url]: https://github.com/lfnovo/open-notebook/graphs/contributors [forks-shield]: https://img.shields.io/github/forks/lfnovo/open-notebook.svg?style=for-the-badge [forks-url]: https://github.com/lfnovo/open-notebook/network/members [stars-shield]: https://img.shields.io/github/stars/lfnovo/open-notebook.svg?style=for-the-badge [stars-url]: https://github.com/lfnovo/open-notebook/stargazers [issues-shield]: https://img.shields.io/github/issues/lfnovo/open-notebook.svg?style=for-the-badge [issues-url]: https://github.com/lfnovo/open-notebook/issues [license-shield]: https://img.shields.io/github/license/lfnovo/open-notebook.svg?style=for-the-badge [license-url]: https://github.com/lfnovo/open-notebook/blob/master/LICENSE.txt [linkedin-shield]: https://img.shields.io/badge/-LinkedIn-black.svg?style=for-the-badge&logo=linkedin&colorB=555 [linkedin-url]: https://linkedin.com/in/lfnovo [product-screenshot]: images/screenshot.png [Next.js]: https://img.shields.io/badge/Next.js-000000?style=for-the-badge&logo=next.js&logoColor=white [Next-url]: https://nextjs.org/ [React]: https://img.shields.io/badge/React-61DAFB?style=for-the-badge&logo=react&logoColor=black [React-url]: https://reactjs.org/ [Python]: https://img.shields.io/badge/Python-3776AB?style=for-the-badge&logo=python&logoColor=white [Python-url]: https://www.python.org/ [LangChain]: https://img.shields.io/badge/LangChain-3A3A3A?style=for-the-badge&logo=chainlink&logoColor=white [LangChain-url]: https://www.langchain.com/ [SurrealDB]: https://img.shields.io/badge/SurrealDB-FF5E00?style=for-the-badge&logo=databricks&logoColor=white [SurrealDB-url]: https://surrealdb.com/ ================================================ FILE: api/CLAUDE.md ================================================ # API Module FastAPI-based REST backend exposing services for notebooks, sources, notes, chat, podcasts, and AI model management. ## Purpose FastAPI application serving three architectural layers: routes (HTTP endpoints), services (business logic), and models (request/response schemas). Integrates LangGraph workflows (chat, ask, source_chat), SurrealDB persistence, and AI providers via Esperanto. ## Architecture Overview **Three layers**: 1. **Routes** (`routers/*`): HTTP endpoints mapping to services 2. **Services** (`*_service.py`): Business logic orchestrating domain models, database, graphs, AI providers 3. **Models** (`models.py`): Pydantic request/response schemas with validation **Startup flow**: - Load .env environment variables - Initialize CORS middleware + password auth middleware - Run database migrations via AsyncMigrationManager on lifespan startup - Run podcast profile data migration (legacy string to model registry conversion) - Register all routers **Key services**: - `chat_service.py`: Invokes chat graph with messages, context - `podcast_service.py`: Orchestrates outline + transcript generation - `sources_service.py`: Content ingestion, vectorization, metadata - `notes_service.py`: Note creation, linking to sources/insights - `transformations_service.py`: Applies transformations to content - `models_service.py`: Manages AI provider/model configuration - `episode_profiles_service.py`: Manages podcast speaker/episode profiles ## Component Catalog ### Main Application - **main.py**: FastAPI app initialization, CORS setup, auth middleware, lifespan event, router registration - **Lifespan handler**: Runs AsyncMigrationManager on startup (database schema migration) - **Auth middleware**: PasswordAuthMiddleware protects endpoints (password-based access control) ### Services (Business Logic) - **chat_service.py**: Invokes chat.py graph; handles message history via SqliteSaver - **podcast_service.py**: Generates outline (outline.jinja), then transcript (transcript.jinja) for episodes - **sources_service.py**: Ingests files/URLs (content_core), extracts text, vectorizes, saves to SurrealDB - **transformations_service.py**: Applies transformations via transformation.py graph - **models_service.py**: Manages ModelManager config (AI provider overrides) - **episode_profiles_service.py**: CRUD for EpisodeProfile and SpeakerProfile models - **insights_service.py**: Generates and retrieves source insights - **notes_service.py**: Creates notes linked to sources/insights ### Models (Schemas) - **models.py**: Pydantic schemas for request/response validation - Request bodies: ChatRequest, CreateNoteRequest, PodcastGenerationRequest, etc. - Response bodies: ChatResponse, NoteResponse, PodcastResponse, etc. - Custom validators for enum fields, file paths, model references ### Routers - **routers/chat.py**: POST /chat - **routers/source_chat.py**: POST /source/{source_id}/chat - **routers/podcasts.py**: POST /podcasts, GET /podcasts/{id}, POST /podcasts/episodes/{id}/retry, etc. - **routers/notes.py**: POST /notes, GET /notes/{id} - **routers/sources.py**: POST /sources, GET /sources/{id}, DELETE /sources/{id} - **routers/models.py**: GET /models, POST /models/config - **routers/credentials.py**: CRUD + test + discover + migrate for credential management - **routers/transformations.py**: POST /transformations - **routers/insights.py**: GET /sources/{source_id}/insights - **routers/auth.py**: POST /auth/password (password-based auth) - **routers/languages.py**: GET /languages (available podcast languages via pycountry+babel) - **routers/commands.py**: GET /commands/{command_id} (job status tracking) ## Common Patterns - **Service injection via FastAPI**: Routers import services directly; no DI framework - **Async/await throughout**: All DB queries, graph invocations, AI calls are async - **SurrealDB transactions**: Services use repo_query, repo_create, repo_upsert from database layer - **Config override pattern**: Models/config override via models_service passed to graph.ainvoke(config=...) - **Error handling**: Custom exception hierarchy (`open_notebook.exceptions`) with global FastAPI exception handlers mapping to HTTP status codes (see Error Handling section below). LangGraph nodes use `classify_error()` to convert raw LLM provider errors into typed exceptions with user-friendly messages. - **Logging**: loguru logger in main.py; services expected to log key operations - **Response normalization**: All responses follow standard schema (data + metadata structure) ## Key Dependencies - `fastapi`: FastAPI app, routers, HTTPException - `pydantic`: Validation models with Field, field_validator - `open_notebook.graphs`: chat, ask, source_chat, source, transformation graphs - `open_notebook.database`: SurrealDB repository functions (repo_query, repo_create, repo_upsert) - `open_notebook.domain`: Notebook, Source, Note, SourceInsight models - `open_notebook.ai.provision`: provision_langchain_model() factory - `ai_prompter`: Prompter for template rendering - `content_core`: extract_content() for file/URL processing - `esperanto`: AI provider client library (LLM, embeddings, TTS) - `surreal_commands`: Job queue for async operations (podcast generation) - `loguru`: Structured logging ## Important Quirks & Gotchas - **Migration auto-run**: Database schema migrations run on every API startup (via lifespan); no manual migration steps - **PasswordAuthMiddleware is basic**: Uses simple password check; production deployments should replace with OAuth/JWT - **No request rate limiting**: No built-in rate limiting; deployment must add via proxy/middleware - **Service state is stateless**: Services don't cache results; each request re-queries database/AI models - **Graph invocation is blocking**: chat/podcast workflows may take minutes; no timeout handling in services - **Command job fire-and-forget**: podcast_service.py submits jobs but doesn't wait (async job queue pattern) - **Model override scoping**: Model config override via RunnableConfig is per-request only (not persistent) - **CORS open by default**: main.py CORS settings allow all origins (restrict before production) - **No OpenAPI security scheme**: API docs available without auth (disable before production) - **Services don't validate user permission**: All endpoints trust authentication layer; no per-notebook permission checks ## Error Handling ### Global Exception Handlers (`main.py`) FastAPI exception handlers map custom exception types from `open_notebook.exceptions` to HTTP status codes. All error responses include CORS headers. | Exception Class | HTTP Status | Use Case | |----------------|-------------|----------| | `NotFoundError` | 404 | Resource not found | | `InvalidInputError` | 400 | Bad request data | | `AuthenticationError` | 401 | Invalid/missing API key | | `RateLimitError` | 429 | Provider rate limit exceeded | | `ConfigurationError` | 422 | Wrong model name, missing config | | `NetworkError` | 502 | Cannot reach AI provider | | `ExternalServiceError` | 502 | Provider returned error (500/503, context length) | | `OpenNotebookError` (base) | 500 | Any other application error | ### Error Classification (`open_notebook.utils.error_classifier`) The `classify_error()` function maps raw exceptions from LLM providers/Esperanto/LangChain into the typed exceptions above with user-friendly messages. Used in all LangGraph graph nodes and SSE streaming handlers. **Flow**: Raw exception → keyword matching → `(ExceptionClass, user_message)` → raised → caught by global handler → HTTP response with descriptive message. ### Frontend Integration The frontend `getApiErrorMessage()` helper (`lib/utils/error-handler.ts`) tries i18n mapping first, then falls back to displaying the backend's descriptive error message directly. --- ## How to Add New Endpoint 1. Create router file in `routers/` (e.g., `routers/new_feature.py`) 2. Import router into `main.py` and register: `app.include_router(new_feature.router, tags=["new_feature"])` 3. Create service in `new_feature_service.py` with business logic 4. Define request/response schemas in `models.py` (or create `new_feature_models.py`) 5. Implement router functions calling service methods 6. Test with `uv run uvicorn api.main:app --host 0.0.0.0 --port 5055` ## Testing Patterns - **Interactive docs**: http://localhost:5055/docs (Swagger UI) - **Direct service tests**: Import service, call methods directly with test data - **Mock graphs**: Replace graph.ainvoke() with mock for testing service logic - **Database: Use test database** (separate SurrealDB instance or mock repo_query) --- ## Credential Management (API Configuration UI) The Credential Management system enables users to configure AI provider credentials through the UI instead of environment variables. Keys are stored securely in SurrealDB (encrypted via Fernet) with database-first fallback to environment variables. ### Router: `routers/credentials.py` **Endpoints**: | Method | Endpoint | Description | |--------|----------|-------------| | GET | `/credentials` | List all credentials (optional `?provider=` filter) | | GET | `/credentials/by-provider/{provider}` | List credentials for a provider | | POST | `/credentials` | Create a new credential | | GET | `/credentials/{credential_id}` | Get a specific credential | | PUT | `/credentials/{credential_id}` | Update a credential | | DELETE | `/credentials/{credential_id}` | Delete a credential | | POST | `/credentials/{credential_id}/test` | Test connection using credential | | POST | `/credentials/{credential_id}/discover` | Discover available models | | POST | `/credentials/{credential_id}/register-models` | Register discovered models | | POST | `/credentials/migrate-from-provider-config` | Migrate from legacy ProviderConfig | **Supported Providers** (13 total): - Simple API key: `openai`, `anthropic`, `google`, `groq`, `mistral`, `deepseek`, `xai`, `openrouter`, `voyage`, `elevenlabs` - URL-based: `ollama` - Multi-field: `azure`, `vertex`, `openai_compatible` **Security Features**: - NEVER returns actual API key values (only metadata) - URL validation (SSRF protection) on all URL fields via `_validate_url()` - Allows private IPs and localhost for self-hosted services (Ollama, LM Studio) - Requires `OPEN_NOTEBOOK_ENCRYPTION_KEY` to be set for storing credentials ### Domain Model: `Credential` (`open_notebook/domain/credential.py`) Individual credential records replacing the old `ProviderConfig` singleton. Each credential stores: - Provider name, display name, modalities - Encrypted API key (via Fernet) - Provider-specific config (base_url, endpoint, api_version, etc.) ### Integration with Key Provider (`open_notebook/ai/key_provider.py`) The `key_provider` module provisions DB-stored credentials into environment variables for Esperanto compatibility: **Database-first Pattern**: 1. API endpoint saves keys to `Credential` records (encrypted in SurrealDB) 2. Before model provisioning, `provision_provider_keys(provider)` checks DB, then env vars 3. Keys from DB are set as environment variables for Esperanto compatibility 4. Existing env vars remain unchanged if no DB config exists **Key Functions**: - `get_api_key(provider)`: Get API key (DB first, env fallback) - `provision_provider_keys(provider)`: Set env vars from DB for a provider - `provision_all_keys()`: Load all provider keys from DB into env vars ### Authentication No changes to authentication. The `credentials` router uses the same `PasswordAuthMiddleware` as all other endpoints. Keys are protected by the same password-based auth. **Auth Flow** (unchanged from `api/auth.py`): - `PasswordAuthMiddleware`: Global middleware checking `Authorization: Bearer {password}` header - Default password: `open-notebook-change-me` (set `OPEN_NOTEBOOK_PASSWORD` in production) - Docker secrets support via `OPEN_NOTEBOOK_PASSWORD_FILE` ### Connection Testing (`open_notebook/ai/connection_tester.py`) The `/credentials/{credential_id}/test` endpoint uses minimal API calls to verify credentials: - Loads Credential via `Credential.get(config_id)`, uses `credential.to_esperanto_config()` - Uses cheapest/smallest models per provider (TEST_MODELS map) - Returns success status and descriptive message - Special handlers for ollama, openai_compatible, and azure providers ### Migration Workflows Two migration endpoints help users transition to the credential system: **From environment variables** (`POST /credentials/migrate-from-env`): 1. Checks each provider for env var presence 2. Creates Credential records from env var values 3. Returns summary: migrated, skipped, errors **From legacy ProviderConfig** (`POST /credentials/migrate-from-provider-config`): 1. Reads old ProviderConfig records from database 2. Converts each to individual Credential records 3. Returns summary: migrated, skipped, errors ### Example Usage ```python # Check status GET /credentials/status # Response: {"configured": {"openai": true, "anthropic": false}, "source": {"openai": "database", "anthropic": "none"}, "encryption_configured": true} # Create credential POST /credentials {"name": "My OpenAI Key", "provider": "openai", "modalities": ["language", "embedding"], "api_key": "sk-proj-..."} # Test connection POST /credentials/{credential_id}/test # Response: {"provider": "openai", "success": true, "message": "Connection successful"} # Discover models POST /credentials/{credential_id}/discover # Response: {"provider": "openai", "models": [{"model_id": "gpt-4", "name": "gpt-4", ...}], "credential_id": "..."} # Migrate from env POST /credentials/migrate-from-env # Response: {"message": "Migration complete. Migrated 3 providers.", "migrated": ["openai", "anthropic", "groq"], "skipped": [], "errors": []} ``` ================================================ FILE: api/__init__.py ================================================ ================================================ FILE: api/auth.py ================================================ from typing import Optional from fastapi import Depends, HTTPException, Request from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer from loguru import logger from starlette.middleware.base import BaseHTTPMiddleware from starlette.responses import JSONResponse from open_notebook.utils.encryption import get_secret_from_env class PasswordAuthMiddleware(BaseHTTPMiddleware): """ Middleware to check password authentication for all API requests. Always active with default password if OPEN_NOTEBOOK_PASSWORD is not set. Supports Docker secrets via OPEN_NOTEBOOK_PASSWORD_FILE. """ def __init__(self, app, excluded_paths: Optional[list] = None): super().__init__(app) self.password = get_secret_from_env("OPEN_NOTEBOOK_PASSWORD") self.excluded_paths = excluded_paths or [ "/", "/health", "/docs", "/openapi.json", "/redoc", ] async def dispatch(self, request: Request, call_next): # Skip authentication if no password is set if not self.password: return await call_next(request) # Skip authentication for excluded paths if request.url.path in self.excluded_paths: return await call_next(request) # Skip authentication for CORS preflight requests (OPTIONS) if request.method == "OPTIONS": return await call_next(request) # Check authorization header auth_header = request.headers.get("Authorization") if not auth_header: return JSONResponse( status_code=401, content={"detail": "Missing authorization header"}, headers={"WWW-Authenticate": "Bearer"}, ) # Expected format: "Bearer {password}" try: scheme, credentials = auth_header.split(" ", 1) if scheme.lower() != "bearer": raise ValueError("Invalid authentication scheme") except ValueError: return JSONResponse( status_code=401, content={"detail": "Invalid authorization header format"}, headers={"WWW-Authenticate": "Bearer"}, ) # Check password if credentials != self.password: return JSONResponse( status_code=401, content={"detail": "Invalid password"}, headers={"WWW-Authenticate": "Bearer"}, ) # Password is correct, proceed with the request response = await call_next(request) return response # Optional: HTTPBearer security scheme for OpenAPI documentation security = HTTPBearer(auto_error=False) def check_api_password( credentials: Optional[HTTPAuthorizationCredentials] = Depends(security), ) -> bool: """ Utility function to check API password. Can be used as a dependency in individual routes if needed. Supports Docker secrets via OPEN_NOTEBOOK_PASSWORD_FILE. Returns True without checking credentials if OPEN_NOTEBOOK_PASSWORD is not configured. Raises 401 if credentials are missing or don't match the configured password. """ password = get_secret_from_env("OPEN_NOTEBOOK_PASSWORD") # No password configured - skip authentication if not password: return True # No credentials provided if not credentials: raise HTTPException( status_code=401, detail="Missing authorization", headers={"WWW-Authenticate": "Bearer"}, ) # Check password if credentials.credentials != password: raise HTTPException( status_code=401, detail="Invalid password", headers={"WWW-Authenticate": "Bearer"}, ) return True ================================================ FILE: api/chat_service.py ================================================ """ Chat service for API operations. Provides async interface for chat functionality. """ import os from typing import Any, Dict, List, Optional import httpx from loguru import logger class ChatService: """Service for chat-related API operations""" def __init__(self): self.base_url = os.getenv("API_BASE_URL", "http://127.0.0.1:5055") # Add authentication header if password is set self.headers = {} password = os.getenv("OPEN_NOTEBOOK_PASSWORD") if password: self.headers["Authorization"] = f"Bearer {password}" async def get_sessions(self, notebook_id: str) -> List[Dict[str, Any]]: """Get all chat sessions for a notebook""" try: async with httpx.AsyncClient() as client: response = await client.get( f"{self.base_url}/api/chat/sessions", params={"notebook_id": notebook_id}, headers=self.headers, ) response.raise_for_status() return response.json() except Exception as e: logger.error(f"Error fetching chat sessions: {str(e)}") raise async def create_session( self, notebook_id: str, title: Optional[str] = None, model_override: Optional[str] = None, ) -> Dict[str, Any]: """Create a new chat session""" try: data: Dict[str, Any] = {"notebook_id": notebook_id} if title is not None: data["title"] = title if model_override is not None: data["model_override"] = model_override async with httpx.AsyncClient() as client: response = await client.post( f"{self.base_url}/api/chat/sessions", json=data, headers=self.headers, ) response.raise_for_status() return response.json() except Exception as e: logger.error(f"Error creating chat session: {str(e)}") raise async def get_session(self, session_id: str) -> Dict[str, Any]: """Get a specific session with messages""" try: async with httpx.AsyncClient() as client: response = await client.get( f"{self.base_url}/api/chat/sessions/{session_id}", headers=self.headers, ) response.raise_for_status() return response.json() except Exception as e: logger.error(f"Error fetching session: {str(e)}") raise async def update_session( self, session_id: str, title: Optional[str] = None, model_override: Optional[str] = None, ) -> Dict[str, Any]: """Update session properties""" try: data: Dict[str, Any] = {} if title is not None: data["title"] = title if model_override is not None: data["model_override"] = model_override if not data: raise ValueError( "At least one field must be provided to update a session" ) async with httpx.AsyncClient() as client: response = await client.put( f"{self.base_url}/api/chat/sessions/{session_id}", json=data, headers=self.headers, ) response.raise_for_status() return response.json() except Exception as e: logger.error(f"Error updating session: {str(e)}") raise async def delete_session(self, session_id: str) -> Dict[str, Any]: """Delete a chat session""" try: async with httpx.AsyncClient() as client: response = await client.delete( f"{self.base_url}/api/chat/sessions/{session_id}", headers=self.headers, ) response.raise_for_status() return response.json() except Exception as e: logger.error(f"Error deleting session: {str(e)}") raise async def execute_chat( self, session_id: str, message: str, context: Dict[str, Any], model_override: Optional[str] = None, ) -> Dict[str, Any]: """Execute a chat request""" try: data = {"session_id": session_id, "message": message, "context": context} if model_override is not None: data["model_override"] = model_override # Short connect timeout (10s), long read timeout (10 min) for Ollama/local LLMs timeout = httpx.Timeout(connect=10.0, read=600.0, write=30.0, pool=10.0) async with httpx.AsyncClient(timeout=timeout) as client: response = await client.post( f"{self.base_url}/api/chat/execute", json=data, headers=self.headers ) response.raise_for_status() return response.json() except Exception as e: logger.error(f"Error executing chat: {str(e)}") raise async def build_context( self, notebook_id: str, context_config: Dict[str, Any] ) -> Dict[str, Any]: """Build context for a notebook""" try: data = {"notebook_id": notebook_id, "context_config": context_config} async with httpx.AsyncClient() as client: response = await client.post( f"{self.base_url}/api/chat/context", json=data, headers=self.headers ) response.raise_for_status() return response.json() except Exception as e: logger.error(f"Error building context: {str(e)}") raise # Global instance chat_service = ChatService() ================================================ FILE: api/client.py ================================================ """ API client for Open Notebook API. This module provides a client interface to interact with the Open Notebook API. """ import os from typing import Any, Dict, List, Optional, Union import httpx from loguru import logger class APIClient: """Client for Open Notebook API.""" def __init__(self, base_url: Optional[str] = None): self.base_url = base_url or os.getenv("API_BASE_URL", "http://127.0.0.1:5055") # Timeout increased to 5 minutes (300s) to accommodate slow LLM operations # (transformations, insights) on slower hardware (Ollama, LM Studio, remote APIs) # Configurable via API_CLIENT_TIMEOUT environment variable (in seconds) timeout_str = os.getenv("API_CLIENT_TIMEOUT", "300.0") try: timeout_value = float(timeout_str) # Validate timeout is within reasonable bounds (30s - 3600s / 1 hour) if timeout_value < 30: logger.warning( f"API_CLIENT_TIMEOUT={timeout_value}s is too low, using minimum of 30s" ) timeout_value = 30.0 elif timeout_value > 3600: logger.warning( f"API_CLIENT_TIMEOUT={timeout_value}s is too high, using maximum of 3600s" ) timeout_value = 3600.0 self.timeout = timeout_value except ValueError: logger.error( f"Invalid API_CLIENT_TIMEOUT value '{timeout_str}', using default 300s" ) self.timeout = 300.0 # Add authentication header if password is set self.headers = {} password = os.getenv("OPEN_NOTEBOOK_PASSWORD") if password: self.headers["Authorization"] = f"Bearer {password}" def _make_request( self, method: str, endpoint: str, timeout: Optional[float] = None, **kwargs ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]: """Make HTTP request to the API.""" url = f"{self.base_url}{endpoint}" request_timeout = timeout if timeout is not None else self.timeout # Merge headers headers = kwargs.get("headers", {}) headers.update(self.headers) kwargs["headers"] = headers try: with httpx.Client(timeout=request_timeout) as client: response = client.request(method, url, **kwargs) response.raise_for_status() return response.json() except httpx.RequestError as e: logger.error(f"Request error for {method} {url}: {str(e)}") raise ConnectionError(f"Failed to connect to API: {str(e)}") except httpx.HTTPStatusError as e: logger.error( f"HTTP error {e.response.status_code} for {method} {url}: {e.response.text}" ) raise RuntimeError( f"API request failed: {e.response.status_code} - {e.response.text}" ) except Exception as e: logger.error(f"Unexpected error for {method} {url}: {str(e)}") raise # Notebooks API methods def get_notebooks( self, archived: Optional[bool] = None, order_by: str = "updated desc" ) -> List[Dict[Any, Any]]: """Get all notebooks.""" params: Dict[str, Any] = {"order_by": order_by} if archived is not None: params["archived"] = str(archived).lower() result = self._make_request("GET", "/api/notebooks", params=params) return result if isinstance(result, list) else [result] def create_notebook( self, name: str, description: str = "" ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]: """Create a new notebook.""" data = {"name": name, "description": description} return self._make_request("POST", "/api/notebooks", json=data) def get_notebook( self, notebook_id: str ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]: """Get a specific notebook.""" return self._make_request("GET", f"/api/notebooks/{notebook_id}") def update_notebook( self, notebook_id: str, **updates ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]: """Update a notebook.""" return self._make_request("PUT", f"/api/notebooks/{notebook_id}", json=updates) def delete_notebook( self, notebook_id: str ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]: """Delete a notebook.""" return self._make_request("DELETE", f"/api/notebooks/{notebook_id}") # Search API methods def search( self, query: str, search_type: str = "text", limit: int = 100, search_sources: bool = True, search_notes: bool = True, minimum_score: float = 0.2, ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]: """Search the knowledge base.""" data = { "query": query, "type": search_type, "limit": limit, "search_sources": search_sources, "search_notes": search_notes, "minimum_score": minimum_score, } return self._make_request("POST", "/api/search", json=data) def ask_simple( self, question: str, strategy_model: str, answer_model: str, final_answer_model: str, ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]: """Ask the knowledge base a question (simple, non-streaming).""" data = { "question": question, "strategy_model": strategy_model, "answer_model": answer_model, "final_answer_model": final_answer_model, } # Use configured timeout for long-running ask operations return self._make_request( "POST", "/api/search/ask/simple", json=data, timeout=self.timeout ) # Models API methods def get_models(self, model_type: Optional[str] = None) -> List[Dict[Any, Any]]: """Get all models with optional type filtering.""" params = {} if model_type: params["type"] = model_type result = self._make_request("GET", "/api/models", params=params) return result if isinstance(result, list) else [result] def create_model( self, name: str, provider: str, model_type: str ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]: """Create a new model.""" data = { "name": name, "provider": provider, "type": model_type, } return self._make_request("POST", "/api/models", json=data) def delete_model( self, model_id: str ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]: """Delete a model.""" return self._make_request("DELETE", f"/api/models/{model_id}") def get_default_models(self) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]: """Get default model assignments.""" return self._make_request("GET", "/api/models/defaults") def update_default_models( self, **defaults ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]: """Update default model assignments.""" return self._make_request("PUT", "/api/models/defaults", json=defaults) # Transformations API methods def get_transformations(self) -> List[Dict[Any, Any]]: """Get all transformations.""" result = self._make_request("GET", "/api/transformations") return result if isinstance(result, list) else [result] def create_transformation( self, name: str, title: str, description: str, prompt: str, apply_default: bool = False, ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]: """Create a new transformation.""" data = { "name": name, "title": title, "description": description, "prompt": prompt, "apply_default": apply_default, } return self._make_request("POST", "/api/transformations", json=data) def get_transformation( self, transformation_id: str ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]: """Get a specific transformation.""" return self._make_request("GET", f"/api/transformations/{transformation_id}") def update_transformation( self, transformation_id: str, **updates ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]: """Update a transformation.""" return self._make_request( "PUT", f"/api/transformations/{transformation_id}", json=updates ) def delete_transformation( self, transformation_id: str ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]: """Delete a transformation.""" return self._make_request("DELETE", f"/api/transformations/{transformation_id}") def execute_transformation( self, transformation_id: str, input_text: str, model_id: str ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]: """Execute a transformation on input text.""" data = { "transformation_id": transformation_id, "input_text": input_text, "model_id": model_id, } # Use configured timeout for transformation operations return self._make_request( "POST", "/api/transformations/execute", json=data, timeout=self.timeout ) # Notes API methods def get_notes(self, notebook_id: Optional[str] = None) -> List[Dict[Any, Any]]: """Get all notes with optional notebook filtering.""" params = {} if notebook_id: params["notebook_id"] = notebook_id result = self._make_request("GET", "/api/notes", params=params) return result if isinstance(result, list) else [result] def create_note( self, content: str, title: Optional[str] = None, note_type: str = "human", notebook_id: Optional[str] = None, ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]: """Create a new note.""" data = { "content": content, "note_type": note_type, } if title: data["title"] = title if notebook_id: data["notebook_id"] = notebook_id return self._make_request("POST", "/api/notes", json=data) def get_note(self, note_id: str) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]: """Get a specific note.""" return self._make_request("GET", f"/api/notes/{note_id}") def update_note( self, note_id: str, **updates ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]: """Update a note.""" return self._make_request("PUT", f"/api/notes/{note_id}", json=updates) def delete_note(self, note_id: str) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]: """Delete a note.""" return self._make_request("DELETE", f"/api/notes/{note_id}") # Embedding API methods def embed_content( self, item_id: str, item_type: str, async_processing: bool = False ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]: """Embed content for vector search.""" data = { "item_id": item_id, "item_type": item_type, "async_processing": async_processing, } # Use configured timeout for embedding operations return self._make_request("POST", "/api/embed", json=data, timeout=self.timeout) def rebuild_embeddings( self, mode: str = "existing", include_sources: bool = True, include_notes: bool = True, include_insights: bool = True, ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]: """Rebuild embeddings in bulk. Note: This operation can take a long time for large databases. Consider increasing API_CLIENT_TIMEOUT to 600-900s for bulk rebuilds. """ data = { "mode": mode, "include_sources": include_sources, "include_notes": include_notes, "include_insights": include_insights, } # Use double the configured timeout for bulk rebuild operations (or configured value if already high) rebuild_timeout = max(self.timeout, min(self.timeout * 2, 3600.0)) return self._make_request( "POST", "/api/embeddings/rebuild", json=data, timeout=rebuild_timeout ) def get_rebuild_status( self, command_id: str ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]: """Get status of a rebuild operation.""" return self._make_request("GET", f"/api/embeddings/rebuild/{command_id}/status") # Settings API methods def get_settings(self) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]: """Get all application settings.""" return self._make_request("GET", "/api/settings") def update_settings( self, **settings ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]: """Update application settings.""" return self._make_request("PUT", "/api/settings", json=settings) # Context API methods def get_notebook_context( self, notebook_id: str, context_config: Optional[Dict] = None ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]: """Get context for a notebook.""" data: Dict[str, Any] = {"notebook_id": notebook_id} if context_config: data["context_config"] = context_config result = self._make_request( "POST", f"/api/notebooks/{notebook_id}/context", json=data ) return result if isinstance(result, dict) else {} # Sources API methods def get_sources(self, notebook_id: Optional[str] = None) -> List[Dict[Any, Any]]: """Get all sources with optional notebook filtering.""" params = {} if notebook_id: params["notebook_id"] = notebook_id result = self._make_request("GET", "/api/sources", params=params) return result if isinstance(result, list) else [result] def create_source( self, notebook_id: Optional[str] = None, notebooks: Optional[List[str]] = None, source_type: str = "text", url: Optional[str] = None, file_path: Optional[str] = None, content: Optional[str] = None, title: Optional[str] = None, transformations: Optional[List[str]] = None, embed: bool = False, delete_source: bool = False, async_processing: bool = False, ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]: """Create a new source.""" data = { "type": source_type, "embed": embed, "delete_source": delete_source, "async_processing": async_processing, } # Handle backward compatibility for notebook_id vs notebooks if notebooks: data["notebooks"] = notebooks elif notebook_id: data["notebook_id"] = notebook_id else: raise ValueError("Either notebook_id or notebooks must be provided") if url: data["url"] = url if file_path: data["file_path"] = file_path if content: data["content"] = content if title: data["title"] = title if transformations: data["transformations"] = transformations # Use configured timeout for source creation (especially PDF processing with OCR) return self._make_request( "POST", "/api/sources/json", json=data, timeout=self.timeout ) def get_source(self, source_id: str) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]: """Get a specific source.""" return self._make_request("GET", f"/api/sources/{source_id}") def get_source_status( self, source_id: str ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]: """Get processing status for a source.""" return self._make_request("GET", f"/api/sources/{source_id}/status") def update_source( self, source_id: str, **updates ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]: """Update a source.""" return self._make_request("PUT", f"/api/sources/{source_id}", json=updates) def delete_source( self, source_id: str ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]: """Delete a source.""" return self._make_request("DELETE", f"/api/sources/{source_id}") # Insights API methods def get_source_insights(self, source_id: str) -> List[Dict[Any, Any]]: """Get all insights for a specific source.""" result = self._make_request("GET", f"/api/sources/{source_id}/insights") return result if isinstance(result, list) else [result] def get_insight( self, insight_id: str ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]: """Get a specific insight.""" return self._make_request("GET", f"/api/insights/{insight_id}") def delete_insight( self, insight_id: str ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]: """Delete a specific insight.""" return self._make_request("DELETE", f"/api/insights/{insight_id}") def save_insight_as_note( self, insight_id: str, notebook_id: Optional[str] = None ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]: """Convert an insight to a note.""" data = {} if notebook_id: data["notebook_id"] = notebook_id return self._make_request( "POST", f"/api/insights/{insight_id}/save-as-note", json=data ) def create_source_insight( self, source_id: str, transformation_id: str, model_id: Optional[str] = None ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]: """Create a new insight for a source by running a transformation.""" data = {"transformation_id": transformation_id} if model_id: data["model_id"] = model_id return self._make_request( "POST", f"/api/sources/{source_id}/insights", json=data ) # Episode Profiles API methods def get_episode_profiles(self) -> List[Dict[Any, Any]]: """Get all episode profiles.""" result = self._make_request("GET", "/api/episode-profiles") return result if isinstance(result, list) else [result] def get_episode_profile( self, profile_name: str ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]: """Get a specific episode profile by name.""" return self._make_request("GET", f"/api/episode-profiles/{profile_name}") def create_episode_profile( self, name: str, description: str = "", speaker_config: str = "", outline_provider: str = "", outline_model: str = "", transcript_provider: str = "", transcript_model: str = "", default_briefing: str = "", num_segments: int = 5, ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]: """Create a new episode profile.""" data = { "name": name, "description": description, "speaker_config": speaker_config, "outline_provider": outline_provider, "outline_model": outline_model, "transcript_provider": transcript_provider, "transcript_model": transcript_model, "default_briefing": default_briefing, "num_segments": num_segments, } return self._make_request("POST", "/api/episode-profiles", json=data) def update_episode_profile( self, profile_id: str, **updates ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]: """Update an episode profile.""" return self._make_request( "PUT", f"/api/episode-profiles/{profile_id}", json=updates ) def delete_episode_profile( self, profile_id: str ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]: """Delete an episode profile.""" return self._make_request("DELETE", f"/api/episode-profiles/{profile_id}") # Global client instance api_client = APIClient() ================================================ FILE: api/command_service.py ================================================ from typing import Any, Dict, List, Optional from loguru import logger from surreal_commands import get_command_status, submit_command class CommandService: """Generic service layer for command operations""" @staticmethod async def submit_command_job( module_name: str, # Actually app_name for surreal-commands command_name: str, command_args: Dict[str, Any], context: Optional[Dict[str, Any]] = None, ) -> str: """Submit a generic command job for background processing""" try: # Ensure command modules are imported before submitting # This is needed because submit_command validates against local registry try: import commands.podcast_commands # noqa: F401 except ImportError as import_err: logger.error(f"Failed to import command modules: {import_err}") raise ValueError("Command modules not available") # surreal-commands expects: submit_command(app_name, command_name, args) cmd_id = submit_command( module_name, # This is actually the app name (e.g., "open_notebook") command_name, # Command name (e.g., "process_text") command_args, # Input data ) # Convert RecordID to string if needed if not cmd_id: raise ValueError("Failed to get cmd_id from submit_command") cmd_id_str = str(cmd_id) logger.info( f"Submitted command job: {cmd_id_str} for {module_name}.{command_name}" ) return cmd_id_str except Exception as e: logger.error(f"Failed to submit command job: {e}") raise @staticmethod async def get_command_status(job_id: str) -> Dict[str, Any]: """Get status of any command job""" try: status = await get_command_status(job_id) return { "job_id": job_id, "status": status.status if status else "unknown", "result": status.result if status else None, "error_message": getattr(status, "error_message", None) if status else None, "created": str(status.created) if status and hasattr(status, "created") and status.created else None, "updated": str(status.updated) if status and hasattr(status, "updated") and status.updated else None, "progress": getattr(status, "progress", None) if status else None, } except Exception as e: logger.error(f"Failed to get command status: {e}") raise @staticmethod async def list_command_jobs( module_filter: Optional[str] = None, command_filter: Optional[str] = None, status_filter: Optional[str] = None, limit: int = 50, ) -> List[Dict[str, Any]]: """List command jobs with optional filtering""" # This will be implemented with proper SurrealDB queries # For now, return empty list as this is foundation phase return [] @staticmethod async def cancel_command_job(job_id: str) -> bool: """Cancel a running command job""" try: # Implementation depends on surreal-commands cancellation support # For now, just log the attempt logger.info(f"Attempting to cancel job: {job_id}") return True except Exception as e: logger.error(f"Failed to cancel command job: {e}") raise ================================================ FILE: api/context_service.py ================================================ """ Context service layer using API. """ from typing import Any, Dict, List, Optional, Union from loguru import logger from api.client import api_client class ContextService: """Service layer for context operations using API.""" def __init__(self): logger.info("Using API for context operations") def get_notebook_context( self, notebook_id: str, context_config: Optional[Dict] = None ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]: """Get context for a notebook.""" result = api_client.get_notebook_context( notebook_id=notebook_id, context_config=context_config ) return result # Global service instance context_service = ContextService() ================================================ FILE: api/credentials_service.py ================================================ """ Credentials Service Business logic for managing AI provider credentials. Extracted from the credentials router to follow the service layer pattern. All functions raise ValueError for business errors (router converts to HTTPException). """ import ipaddress import os import socket from typing import Dict, List, Optional from urllib.parse import urlparse import httpx from loguru import logger from pydantic import SecretStr from api.models import CredentialResponse from open_notebook.domain.credential import Credential from open_notebook.utils.encryption import get_secret_from_env # ============================================================================= # Constants # ============================================================================= # Provider environment variable configuration. # - "required": ALL listed env vars must be set for the provider to be considered configured. # - "required_any": at least ONE of the listed env vars must be set. # - "optional": additional env vars used during migration but not required. PROVIDER_ENV_CONFIG: Dict[str, dict] = { "openai": {"required": ["OPENAI_API_KEY"]}, "anthropic": {"required": ["ANTHROPIC_API_KEY"]}, "google": {"required_any": ["GOOGLE_API_KEY", "GEMINI_API_KEY"]}, "groq": {"required": ["GROQ_API_KEY"]}, "mistral": {"required": ["MISTRAL_API_KEY"]}, "deepseek": {"required": ["DEEPSEEK_API_KEY"]}, "xai": {"required": ["XAI_API_KEY"]}, "openrouter": {"required": ["OPENROUTER_API_KEY"]}, "voyage": {"required": ["VOYAGE_API_KEY"]}, "elevenlabs": {"required": ["ELEVENLABS_API_KEY"]}, "ollama": {"required": ["OLLAMA_API_BASE"]}, "vertex": { "required": ["VERTEX_PROJECT", "VERTEX_LOCATION"], "optional": ["GOOGLE_APPLICATION_CREDENTIALS"], }, "azure": { "required": ["AZURE_OPENAI_API_KEY", "AZURE_OPENAI_ENDPOINT", "AZURE_OPENAI_API_VERSION"], "optional": [ "AZURE_OPENAI_ENDPOINT_LLM", "AZURE_OPENAI_ENDPOINT_EMBEDDING", "AZURE_OPENAI_ENDPOINT_STT", "AZURE_OPENAI_ENDPOINT_TTS", ], }, "openai_compatible": { "required_any": ["OPENAI_COMPATIBLE_BASE_URL", "OPENAI_COMPATIBLE_API_KEY"], }, } PROVIDER_MODALITIES: Dict[str, List[str]] = { "openai": ["language", "embedding", "speech_to_text", "text_to_speech"], "anthropic": ["language"], "google": ["language", "embedding"], "groq": ["language", "speech_to_text"], "mistral": ["language", "embedding"], "deepseek": ["language"], "xai": ["language"], "openrouter": ["language"], "voyage": ["embedding"], "elevenlabs": ["text_to_speech"], "ollama": ["language", "embedding"], "vertex": ["language", "embedding"], "azure": ["language", "embedding", "speech_to_text", "text_to_speech"], "openai_compatible": ["language", "embedding", "speech_to_text", "text_to_speech"], } # ============================================================================= # URL Validation (SSRF protection) # ============================================================================= def validate_url(url: str, provider: str) -> None: """ Validate URL format for API endpoints. This is a self-hosted application, so we allow: - Private IPs (10.x, 172.16-31.x, 192.168.x) for self-hosted services - Localhost for local services (Ollama, LM Studio, etc.) We only block: - Invalid schemes (must be http or https) - Malformed URLs - Link-local addresses (169.254.x.x) - used for cloud metadata endpoints - Hostnames that resolve to link-local addresses Args: url: The URL to validate provider: The provider name (for logging/context) Raises: ValueError: If the URL is invalid """ if not url or not url.strip(): return # Empty URLs handled elsewhere try: parsed = urlparse(url.strip()) # Validate scheme - only http/https allowed if parsed.scheme not in ("http", "https"): raise ValueError( f"Invalid URL scheme: '{parsed.scheme}'. Only http and https are allowed." ) # Extract hostname hostname = parsed.hostname if not hostname: raise ValueError("Invalid URL: hostname could not be determined.") # Try to parse as IP address to check for dangerous addresses try: ip = ipaddress.ip_address(hostname) # Block link-local addresses (169.254.x.x) - used for cloud metadata # These are dangerous as they can expose cloud instance credentials if ip.is_link_local: raise ValueError( "Link-local addresses (169.254.x.x) are not allowed for security reasons. " "These addresses are used for cloud metadata endpoints." ) # Block IPv4-mapped IPv6 addresses pointing to link-local # e.g. ::ffff:169.254.169.254 bypasses IPv6 is_link_local check if hasattr(ip, "ipv4_mapped") and ip.ipv4_mapped and ip.ipv4_mapped.is_link_local: raise ValueError( "Link-local addresses (169.254.x.x) are not allowed for security reasons. " "These addresses are used for cloud metadata endpoints." ) except ValueError as ve: # Re-raise our own ValueErrors if "Link-local" in str(ve) or "Invalid URL" in str(ve): raise # Not an IP address, it's a hostname - need to resolve and check try: # Resolve hostname to IP address resolved_ips = socket.getaddrinfo(hostname, None) for family, _, _, _, sockaddr in resolved_ips: ip_addr = sockaddr[0] try: parsed_ip = ipaddress.ip_address(ip_addr) if parsed_ip.is_link_local: raise ValueError( f"Hostname '{hostname}' resolves to a link-local address (169.254.x.x) which is not allowed for security reasons. " "These addresses are used for cloud metadata endpoints." ) # Block IPv4-mapped IPv6 addresses pointing to link-local if ( hasattr(parsed_ip, "ipv4_mapped") and parsed_ip.ipv4_mapped and parsed_ip.ipv4_mapped.is_link_local ): raise ValueError( f"Hostname '{hostname}' resolves to a link-local address (169.254.x.x) which is not allowed for security reasons. " "These addresses are used for cloud metadata endpoints." ) except ValueError as inner_ve: if "link-local" in str(inner_ve).lower() or "Link-local" in str(inner_ve): raise # Skip non-IP addresses (e.g., IPv6 zones) continue except socket.gaierror: # Could not resolve hostname - allow it since the URL may be # valid in the deployment environment (e.g., Azure endpoints, # internal DNS names). We only block link-local addresses. pass except ValueError: raise except Exception: raise ValueError("Invalid URL format. Check server logs for details.") # ============================================================================= # Helpers # ============================================================================= def require_encryption_key() -> None: """Raise ValueError if encryption key is not configured.""" if not get_secret_from_env("OPEN_NOTEBOOK_ENCRYPTION_KEY"): raise ValueError( "Encryption key not configured. " "Set OPEN_NOTEBOOK_ENCRYPTION_KEY to enable storing API keys." ) def credential_to_response(cred: Credential, model_count: int = 0) -> CredentialResponse: """Convert a Credential domain object to API response.""" return CredentialResponse( id=cred.id or "", name=cred.name, provider=cred.provider, modalities=cred.modalities, base_url=cred.base_url, endpoint=cred.endpoint, api_version=cred.api_version, endpoint_llm=cred.endpoint_llm, endpoint_embedding=cred.endpoint_embedding, endpoint_stt=cred.endpoint_stt, endpoint_tts=cred.endpoint_tts, project=cred.project, location=cred.location, credentials_path=cred.credentials_path, has_api_key=cred.api_key is not None, created=str(cred.created) if cred.created else "", updated=str(cred.updated) if cred.updated else "", model_count=model_count, ) def check_env_configured(provider: str) -> bool: """Check if a provider has sufficient env vars configured for migration.""" config = PROVIDER_ENV_CONFIG.get(provider) if not config: return False if "required_any" in config: return any(bool(os.environ.get(v, "").strip()) for v in config["required_any"]) elif "required" in config: return all(bool(os.environ.get(v, "").strip()) for v in config["required"]) return False def get_default_modalities(provider: str) -> List[str]: """Get default modalities for a provider.""" return PROVIDER_MODALITIES.get(provider.lower(), ["language"]) def create_credential_from_env(provider: str) -> Credential: """Create a Credential from environment variables for a given provider.""" modalities = get_default_modalities(provider) name = "Default (Migrated from env)" if provider == "ollama": return Credential( name=name, provider=provider, modalities=modalities, base_url=os.environ.get("OLLAMA_API_BASE"), ) elif provider == "vertex": return Credential( name=name, provider=provider, modalities=modalities, project=os.environ.get("VERTEX_PROJECT"), location=os.environ.get("VERTEX_LOCATION"), credentials_path=os.environ.get("GOOGLE_APPLICATION_CREDENTIALS"), ) elif provider == "azure": return Credential( name=name, provider=provider, modalities=modalities, api_key=SecretStr(os.environ["AZURE_OPENAI_API_KEY"]), endpoint=os.environ.get("AZURE_OPENAI_ENDPOINT"), api_version=os.environ.get("AZURE_OPENAI_API_VERSION"), endpoint_llm=os.environ.get("AZURE_OPENAI_ENDPOINT_LLM"), endpoint_embedding=os.environ.get("AZURE_OPENAI_ENDPOINT_EMBEDDING"), endpoint_stt=os.environ.get("AZURE_OPENAI_ENDPOINT_STT"), endpoint_tts=os.environ.get("AZURE_OPENAI_ENDPOINT_TTS"), ) elif provider == "openai_compatible": api_key = os.environ.get("OPENAI_COMPATIBLE_API_KEY") return Credential( name=name, provider=provider, modalities=modalities, api_key=SecretStr(api_key) if api_key else None, base_url=os.environ.get("OPENAI_COMPATIBLE_BASE_URL"), ) elif provider == "google": # Support both GOOGLE_API_KEY and GEMINI_API_KEY (fallback) api_key = os.environ.get("GOOGLE_API_KEY") or os.environ.get("GEMINI_API_KEY") return Credential( name=name, provider=provider, modalities=modalities, api_key=SecretStr(api_key) if api_key else None, ) else: # Simple API key providers config = PROVIDER_ENV_CONFIG.get(provider, {}) required = config.get("required", []) env_var = required[0] if required else None api_key = os.environ.get(env_var) if env_var else None return Credential( name=name, provider=provider, modalities=modalities, api_key=SecretStr(api_key) if api_key else None, ) # ============================================================================= # Service Functions # ============================================================================= async def get_provider_status() -> dict: """ Get configuration status: encryption key status, and per-provider configured/source information. """ encryption_configured = bool(get_secret_from_env("OPEN_NOTEBOOK_ENCRYPTION_KEY")) configured: Dict[str, bool] = {} source: Dict[str, str] = {} for provider in PROVIDER_ENV_CONFIG: env_configured = check_env_configured(provider) try: db_credentials = await Credential.get_by_provider(provider) db_configured = len(db_credentials) > 0 except Exception: db_configured = False configured[provider] = db_configured or env_configured if db_configured: source[provider] = "database" elif env_configured: source[provider] = "environment" else: source[provider] = "none" return { "configured": configured, "source": source, "encryption_configured": encryption_configured, } async def get_env_status() -> Dict[str, bool]: """Check what's configured via environment variables.""" env_status: Dict[str, bool] = {} for provider in PROVIDER_ENV_CONFIG: env_status[provider] = check_env_configured(provider) return env_status async def test_credential(credential_id: str) -> dict: """ Test connection using a credential's configuration. Returns dict with provider, success, message keys. """ provider = "unknown" try: cred = await Credential.get(credential_id) config = cred.to_esperanto_config() from open_notebook.ai.connection_tester import ( _test_azure_connection, _test_ollama_connection, _test_openai_compatible_connection, ) provider = cred.provider.lower() # Handle special providers if provider == "ollama": base_url = config.get("base_url", "http://localhost:11434") success, message = await _test_ollama_connection(base_url) return {"provider": provider, "success": success, "message": message} if provider == "openai_compatible": base_url = config.get("base_url") api_key = config.get("api_key") if not base_url: return { "provider": provider, "success": False, "message": "No base URL configured", } success, message = await _test_openai_compatible_connection( base_url, api_key ) return {"provider": provider, "success": success, "message": message} if provider == "azure": success, message = await _test_azure_connection( endpoint=config.get("endpoint"), api_key=config.get("api_key"), api_version=config.get("api_version"), ) return {"provider": provider, "success": success, "message": message} # Standard provider: use Esperanto to create and test from esperanto.factory import AIFactory from open_notebook.ai.connection_tester import TEST_MODELS if provider not in TEST_MODELS: return { "provider": provider, "success": False, "message": f"Unknown provider: {provider}", } test_model, test_type = TEST_MODELS[provider] if not test_model: return { "provider": provider, "success": False, "message": f"No test model configured for {provider}", } if test_type == "language": model = AIFactory.create_language( model_name=test_model, provider=provider, config=config ) lc_model = model.to_langchain() await lc_model.ainvoke("Hi") return {"provider": provider, "success": True, "message": "Connection successful"} elif test_type == "embedding": model = AIFactory.create_embedding( model_name=test_model, provider=provider, config=config ) await model.aembed(["test"]) return {"provider": provider, "success": True, "message": "Connection successful"} elif test_type == "text_to_speech": AIFactory.create_text_to_speech(model_name=test_model, provider=provider, config=config) return { "provider": provider, "success": True, "message": "Connection successful (key format valid)", } return { "provider": provider, "success": False, "message": f"Unsupported test type: {test_type}", } except Exception as e: error_msg = str(e) if "401" in error_msg or "unauthorized" in error_msg.lower(): return {"provider": provider, "success": False, "message": "Invalid API key"} elif "403" in error_msg or "forbidden" in error_msg.lower(): return {"provider": provider, "success": False, "message": "API key lacks required permissions"} elif "rate" in error_msg.lower() and "limit" in error_msg.lower(): return {"provider": provider, "success": True, "message": "Rate limited - but connection works"} elif "not found" in error_msg.lower() and "model" in error_msg.lower(): return {"provider": provider, "success": True, "message": "API key valid (test model not available)"} else: logger.debug(f"Test connection error for credential {credential_id}: {e}") truncated = error_msg[:100] + "..." if len(error_msg) > 100 else error_msg return {"provider": provider, "success": False, "message": f"Error: {truncated}"} async def discover_with_config(provider: str, config: dict) -> List[dict]: """ Discover models using explicit config instead of env vars. Returns model names only — no type classification. The user chooses the model type when registering. """ api_key = config.get("api_key") base_url = config.get("base_url") # Static model lists for providers without a listing API STATIC_MODELS: Dict[str, List[str]] = { "anthropic": [ "claude-opus-4-20250514", "claude-sonnet-4-20250514", "claude-3-5-sonnet-20241022", "claude-3-5-haiku-20241022", "claude-3-opus-20240229", "claude-3-sonnet-20240229", "claude-3-haiku-20240307", ], "voyage": [ "voyage-3", "voyage-3-lite", "voyage-code-3", "voyage-finance-2", "voyage-law-2", "voyage-multilingual-2", ], "elevenlabs": [ "eleven_multilingual_v2", "eleven_turbo_v2_5", "eleven_turbo_v2", "eleven_monolingual_v1", ], } if provider in STATIC_MODELS: if not api_key and provider != "ollama": return [] return [ {"name": m, "provider": provider} for m in STATIC_MODELS[provider] ] # API-based discovery URLs (OpenAI-style /models endpoints) url_map = { "openai": "https://api.openai.com/v1/models", "groq": "https://api.groq.com/openai/v1/models", "mistral": "https://api.mistral.ai/v1/models", "deepseek": "https://api.deepseek.com/models", "xai": "https://api.x.ai/v1/models", "openrouter": "https://openrouter.ai/api/v1/models", } if provider == "ollama": ollama_url = base_url or "http://localhost:11434" try: async with httpx.AsyncClient() as client: response = await client.get(f"{ollama_url}/api/tags", timeout=10.0) response.raise_for_status() data = response.json() return [ {"name": m.get("name", ""), "provider": "ollama"} for m in data.get("models", []) if m.get("name") ] except Exception as e: logger.warning(f"Failed to discover Ollama models: {e}") return [] if provider == "openai_compatible": if not base_url: return [] try: headers = {} if api_key: headers["Authorization"] = f"Bearer {api_key}" async with httpx.AsyncClient() as client: response = await client.get( f"{base_url.rstrip('/')}/models", headers=headers, timeout=30.0, ) response.raise_for_status() data = response.json() return [ {"name": m.get("id", ""), "provider": "openai_compatible"} for m in data.get("data", []) if m.get("id") ] except Exception as e: logger.warning(f"Failed to discover openai_compatible models: {e}") return [] if provider == "azure": endpoint = config.get("endpoint") api_version = config.get("api_version", "2024-10-21") if not endpoint or not api_key: return [] try: url = f"{endpoint.rstrip('/')}/openai/models?api-version={api_version}" headers = {"api-key": api_key} async with httpx.AsyncClient() as client: response = await client.get(url, headers=headers, timeout=30.0) response.raise_for_status() data = response.json() return [ {"name": m.get("id", ""), "provider": "azure"} for m in data.get("data", []) if m.get("id") ] except Exception as e: logger.warning(f"Failed to discover Azure models: {e}") return [] if provider == "vertex": # Vertex AI requires service-account OAuth2 for model listing. # Return a curated static list of well-known Vertex models instead. VERTEX_MODELS = [ "gemini-2.0-flash", "gemini-2.0-flash-lite", "gemini-1.5-pro", "gemini-1.5-flash", "text-embedding-005", ] return [{"name": m, "provider": "vertex"} for m in VERTEX_MODELS] if provider == "google": try: headers = {"X-Goog-Api-Key": api_key} if api_key else {} async with httpx.AsyncClient() as client: response = await client.get( "https://generativelanguage.googleapis.com/v1/models", headers=headers, timeout=30.0, ) response.raise_for_status() data = response.json() return [ { "name": model.get("name", "").replace("models/", ""), "provider": "google", "description": model.get("displayName"), } for model in data.get("models", []) if model.get("name") ] except Exception as e: logger.warning(f"Failed to discover Google models: {e}") return [] # Standard OpenAI-style API discovery discovery_url = url_map.get(provider) if not discovery_url or not api_key: return [] try: async with httpx.AsyncClient() as client: response = await client.get( discovery_url, headers={"Authorization": f"Bearer {api_key}"}, timeout=30.0, ) response.raise_for_status() data = response.json() return [ { "name": m.get("id", ""), "provider": provider, "description": m.get("name"), } for m in data.get("data", []) if m.get("id") ] except Exception as e: logger.warning(f"Failed to discover {provider} models: {e}") return [] async def register_models(credential_id: str, models_data: list) -> dict: """ Register discovered models and link them to a credential. Args: credential_id: The credential ID to link models to models_data: List of dicts with name, provider, model_type Returns: dict with created and existing counts """ cred = await Credential.get(credential_id) from open_notebook.ai.models import Model from open_notebook.database.repository import repo_query # Batch fetch existing models for this provider existing_models = await repo_query( "SELECT string::lowercase(name) as name, string::lowercase(type) as type FROM model " "WHERE string::lowercase(provider) = $provider", {"provider": cred.provider.lower()}, ) existing_keys = {(m["name"], m["type"]) for m in existing_models} created = 0 existing = 0 for model_data in models_data: key = (model_data.name.lower(), model_data.model_type.lower()) if key in existing_keys: existing += 1 continue new_model = Model( name=model_data.name, provider=model_data.provider or cred.provider, type=model_data.model_type, credential=cred.id, ) await new_model.save() created += 1 return {"created": created, "existing": existing} async def migrate_from_provider_config() -> dict: """ Migrate existing ProviderConfig data to individual credential records. Returns dict with message, migrated, skipped, errors. """ logger.info("=== Starting ProviderConfig migration ===") require_encryption_key() logger.info("Encryption key verified") from open_notebook.domain.provider_config import ProviderConfig config = await ProviderConfig.get_instance() logger.info( f"Found ProviderConfig with {len(config.credentials)} provider(s): " f"{', '.join(config.credentials.keys())}" ) migrated = [] skipped = [] errors = [] for provider, credentials_list in config.credentials.items(): for old_cred in credentials_list: try: # Check if a credential already exists for this provider with same name existing = await Credential.get_by_provider(provider) names = [c.name for c in existing] if old_cred.name in names: logger.info( f"[{provider}/{old_cred.name}] Already exists in DB, skipping" ) skipped.append(f"{provider}/{old_cred.name}") continue # Determine modalities from the provider type modalities = get_default_modalities(provider) logger.info(f"[{provider}/{old_cred.name}] Creating credential") new_cred = Credential( name=old_cred.name, provider=provider, modalities=modalities, api_key=old_cred.api_key, base_url=old_cred.base_url, endpoint=old_cred.endpoint, api_version=old_cred.api_version, endpoint_llm=old_cred.endpoint_llm, endpoint_embedding=old_cred.endpoint_embedding, endpoint_stt=old_cred.endpoint_stt, endpoint_tts=old_cred.endpoint_tts, project=old_cred.project, location=old_cred.location, credentials_path=old_cred.credentials_path, ) await new_cred.save() logger.info( f"[{provider}/{old_cred.name}] Credential saved (id={new_cred.id})" ) # Link existing models for this provider to the new credential from open_notebook.ai.models import Model from open_notebook.database.repository import repo_query provider_models = await repo_query( "SELECT * FROM model WHERE string::lowercase(provider) = $provider AND credential IS NONE", {"provider": provider.lower()}, ) if provider_models: logger.info( f"[{provider}/{old_cred.name}] Linking {len(provider_models)} " f"unassigned model(s)" ) for model_data in provider_models: model = Model(**model_data) model.credential = new_cred.id await model.save() migrated.append(f"{provider}/{old_cred.name}") except Exception as e: logger.error( f"[{provider}/{old_cred.name}] Migration FAILED: " f"{type(e).__name__}: {e}", exc_info=True, ) errors.append(f"{provider}/{old_cred.name}: {e}") logger.info( f"=== ProviderConfig migration complete === " f"migrated={len(migrated)} skipped={len(skipped)} errors={len(errors)}" ) if migrated: logger.info(f" Migrated: {', '.join(migrated)}") if skipped: logger.info(f" Skipped: {', '.join(skipped)}") if errors: logger.error(f" Errors: {'; '.join(errors)}") return { "message": f"Migration complete. Migrated {len(migrated)} credentials.", "migrated": migrated, "skipped": skipped, "errors": errors, } async def migrate_from_env() -> dict: """ Migrate API keys from environment variables to credential records. Returns dict with message, migrated, skipped, not_configured, errors. """ logger.info("=== Starting environment variable migration ===") logger.info( f"Checking {len(PROVIDER_ENV_CONFIG)} providers: " f"{', '.join(PROVIDER_ENV_CONFIG.keys())}" ) require_encryption_key() logger.info("Encryption key verified") from open_notebook.ai.models import Model from open_notebook.database.repository import repo_query migrated = [] skipped = [] not_configured = [] errors = [] for provider in PROVIDER_ENV_CONFIG: try: if not check_env_configured(provider): logger.debug(f"[{provider}] No env vars configured, skipping") not_configured.append(provider) continue logger.info(f"[{provider}] Env vars detected, checking for existing credentials") existing = await Credential.get_by_provider(provider) if existing: logger.info( f"[{provider}] Already has {len(existing)} credential(s) in DB, skipping" ) skipped.append(provider) continue logger.info(f"[{provider}] Creating credential from env vars") cred = create_credential_from_env(provider) await cred.save() logger.info(f"[{provider}] Credential saved successfully (id={cred.id})") # Link unassigned models to this credential provider_models = await repo_query( "SELECT * FROM model WHERE string::lowercase(provider) = $provider AND credential IS NONE", {"provider": provider.lower()}, ) if provider_models: logger.info( f"[{provider}] Linking {len(provider_models)} unassigned model(s) " f"to credential {cred.id}" ) for model_data in provider_models: model = Model(**model_data) model.credential = cred.id await model.save() else: logger.info(f"[{provider}] No unassigned models to link") migrated.append(provider) except Exception as e: logger.error( f"[{provider}] Migration FAILED: {type(e).__name__}: {e}", exc_info=True, ) errors.append(f"{provider}: {e}") logger.info( f"=== Environment variable migration complete === " f"migrated={len(migrated)} skipped={len(skipped)} " f"not_configured={len(not_configured)} errors={len(errors)}" ) if migrated: logger.info(f" Migrated: {', '.join(migrated)}") if skipped: logger.info(f" Skipped (already in DB): {', '.join(skipped)}") if errors: logger.error(f" Errors: {'; '.join(errors)}") return { "message": f"Migration complete. Migrated {len(migrated)} providers.", "migrated": migrated, "skipped": skipped, "not_configured": not_configured, "errors": errors, } ================================================ FILE: api/embedding_service.py ================================================ """ Embedding service layer using API. """ from typing import Any, Dict, List, Union from loguru import logger from api.client import api_client class EmbeddingService: """Service layer for embedding operations using API.""" def __init__(self): logger.info("Using API for embedding operations") def embed_content( self, item_id: str, item_type: str ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]: """Embed content for vector search.""" result = api_client.embed_content(item_id=item_id, item_type=item_type) return result # Global service instance embedding_service = EmbeddingService() ================================================ FILE: api/episode_profiles_service.py ================================================ """ Episode profiles service layer using API. """ from typing import List from loguru import logger from api.client import api_client from open_notebook.podcasts.models import EpisodeProfile class EpisodeProfilesService: """Service layer for episode profiles operations using API.""" def __init__(self): logger.info("Using API for episode profiles operations") def get_all_episode_profiles(self) -> List[EpisodeProfile]: """Get all episode profiles.""" profiles_data = api_client.get_episode_profiles() # Convert API response to EpisodeProfile objects profiles = [] for profile_data in profiles_data: profile = EpisodeProfile( name=profile_data["name"], description=profile_data.get("description", ""), speaker_config=profile_data["speaker_config"], outline_provider=profile_data["outline_provider"], outline_model=profile_data["outline_model"], transcript_provider=profile_data["transcript_provider"], transcript_model=profile_data["transcript_model"], default_briefing=profile_data["default_briefing"], num_segments=profile_data["num_segments"], ) profile.id = profile_data["id"] profiles.append(profile) return profiles def get_episode_profile(self, profile_name: str) -> EpisodeProfile: """Get a specific episode profile by name.""" profile_response = api_client.get_episode_profile(profile_name) profile_data = ( profile_response if isinstance(profile_response, dict) else profile_response[0] ) profile = EpisodeProfile( name=profile_data["name"], description=profile_data.get("description", ""), speaker_config=profile_data["speaker_config"], outline_provider=profile_data["outline_provider"], outline_model=profile_data["outline_model"], transcript_provider=profile_data["transcript_provider"], transcript_model=profile_data["transcript_model"], default_briefing=profile_data["default_briefing"], num_segments=profile_data["num_segments"], ) profile.id = profile_data["id"] return profile def create_episode_profile( self, name: str, description: str = "", speaker_config: str = "", outline_provider: str = "", outline_model: str = "", transcript_provider: str = "", transcript_model: str = "", default_briefing: str = "", num_segments: int = 5, ) -> EpisodeProfile: """Create a new episode profile.""" profile_response = api_client.create_episode_profile( name=name, description=description, speaker_config=speaker_config, outline_provider=outline_provider, outline_model=outline_model, transcript_provider=transcript_provider, transcript_model=transcript_model, default_briefing=default_briefing, num_segments=num_segments, ) profile_data = ( profile_response if isinstance(profile_response, dict) else profile_response[0] ) profile = EpisodeProfile( name=profile_data["name"], description=profile_data.get("description", ""), speaker_config=profile_data["speaker_config"], outline_provider=profile_data["outline_provider"], outline_model=profile_data["outline_model"], transcript_provider=profile_data["transcript_provider"], transcript_model=profile_data["transcript_model"], default_briefing=profile_data["default_briefing"], num_segments=profile_data["num_segments"], ) profile.id = profile_data["id"] return profile def delete_episode_profile(self, profile_id: str) -> bool: """Delete an episode profile.""" api_client.delete_episode_profile(profile_id) return True # Global service instance episode_profiles_service = EpisodeProfilesService() ================================================ FILE: api/insights_service.py ================================================ """ Insights service layer using API. """ from typing import List, Optional from loguru import logger from api.client import api_client from open_notebook.domain.notebook import Note, SourceInsight class InsightsService: """Service layer for insights operations using API.""" def __init__(self): logger.info("Using API for insights operations") def get_source_insights(self, source_id: str) -> List[SourceInsight]: """Get all insights for a specific source.""" insights_data = api_client.get_source_insights(source_id) # Convert API response to SourceInsight objects insights = [] for insight_data in insights_data: insight = SourceInsight( insight_type=insight_data["insight_type"], content=insight_data["content"], ) insight.id = insight_data["id"] insight.created = insight_data["created"] insight.updated = insight_data["updated"] insights.append(insight) return insights def get_insight(self, insight_id: str) -> SourceInsight: """Get a specific insight.""" insight_response = api_client.get_insight(insight_id) insight_data = ( insight_response if isinstance(insight_response, dict) else insight_response[0] ) insight = SourceInsight( insight_type=insight_data["insight_type"], content=insight_data["content"], ) insight.id = insight_data["id"] insight.created = insight_data["created"] insight.updated = insight_data["updated"] # Note: source_id from API response is not stored; use await insight.get_source() if needed return insight def delete_insight(self, insight_id: str) -> bool: """Delete a specific insight.""" api_client.delete_insight(insight_id) return True def save_insight_as_note( self, insight_id: str, notebook_id: Optional[str] = None ) -> Note: """Convert an insight to a note.""" note_response = api_client.save_insight_as_note(insight_id, notebook_id) note_data = ( note_response if isinstance(note_response, dict) else note_response[0] ) note = Note( title=note_data["title"], content=note_data["content"], note_type=note_data["note_type"], ) note.id = note_data["id"] note.created = note_data["created"] note.updated = note_data["updated"] return note def create_source_insight( self, source_id: str, transformation_id: str, model_id: Optional[str] = None ) -> SourceInsight: """Create a new insight for a source by running a transformation.""" insight_response = api_client.create_source_insight( source_id, transformation_id, model_id ) insight_data = ( insight_response if isinstance(insight_response, dict) else insight_response[0] ) insight = SourceInsight( insight_type=insight_data["insight_type"], content=insight_data["content"], ) insight.id = insight_data["id"] insight.created = insight_data["created"] insight.updated = insight_data["updated"] # Note: source_id from API response is not stored; use await insight.get_source() if needed return insight # Global service instance insights_service = InsightsService() ================================================ FILE: api/main.py ================================================ # Load environment variables from dotenv import load_dotenv load_dotenv() from contextlib import asynccontextmanager from fastapi import FastAPI, Request from fastapi.middleware.cors import CORSMiddleware from fastapi.responses import JSONResponse from loguru import logger from starlette.exceptions import HTTPException as StarletteHTTPException from api.auth import PasswordAuthMiddleware from open_notebook.exceptions import ( AuthenticationError, ConfigurationError, ExternalServiceError, InvalidInputError, NetworkError, NotFoundError, OpenNotebookError, RateLimitError, ) from api.routers import ( auth, chat, config, context, credentials, embedding, embedding_rebuild, episode_profiles, insights, languages, models, notebooks, notes, podcasts, search, settings, source_chat, sources, speaker_profiles, transformations, ) from api.routers import commands as commands_router from open_notebook.database.async_migrate import AsyncMigrationManager from open_notebook.utils.encryption import get_secret_from_env # Import commands to register them in the API process try: logger.info("Commands imported in API process") except Exception as e: logger.error(f"Failed to import commands in API process: {e}") @asynccontextmanager async def lifespan(app: FastAPI): """ Lifespan event handler for the FastAPI application. Runs database migrations automatically on startup. """ import os # Startup: Security checks logger.info("Starting API initialization...") # Security check: Encryption key if not get_secret_from_env("OPEN_NOTEBOOK_ENCRYPTION_KEY"): logger.warning( "OPEN_NOTEBOOK_ENCRYPTION_KEY not set. " "API key encryption will fail until this is configured. " "Set OPEN_NOTEBOOK_ENCRYPTION_KEY to any secret string." ) # Run database migrations try: migration_manager = AsyncMigrationManager() current_version = await migration_manager.get_current_version() logger.info(f"Current database version: {current_version}") if await migration_manager.needs_migration(): logger.warning("Database migrations are pending. Running migrations...") await migration_manager.run_migration_up() new_version = await migration_manager.get_current_version() logger.success( f"Migrations completed successfully. Database is now at version {new_version}" ) else: logger.info( "Database is already at the latest version. No migrations needed." ) except Exception as e: logger.error(f"CRITICAL: Database migration failed: {str(e)}") logger.exception(e) # Fail fast - don't start the API with an outdated database schema raise RuntimeError(f"Failed to run database migrations: {str(e)}") from e # Run podcast profile data migration (legacy strings -> Model registry) try: from open_notebook.podcasts.migration import migrate_podcast_profiles await migrate_podcast_profiles() except Exception as e: logger.warning(f"Podcast profile migration encountered errors: {e}") # Non-fatal: profiles can be migrated manually via UI logger.success("API initialization completed successfully") # Yield control to the application yield # Shutdown: cleanup if needed logger.info("API shutdown complete") app = FastAPI( title="Open Notebook API", description="API for Open Notebook - Research Assistant", lifespan=lifespan, ) # Add password authentication middleware first # Exclude /api/auth/status and /api/config from authentication app.add_middleware( PasswordAuthMiddleware, excluded_paths=[ "/", "/health", "/docs", "/openapi.json", "/redoc", "/api/auth/status", "/api/config", ], ) # Add CORS middleware last (so it processes first) app.add_middleware( CORSMiddleware, allow_origins=["*"], # In production, replace with specific origins allow_credentials=True, allow_methods=["*"], allow_headers=["*"], ) # Custom exception handler to ensure CORS headers are included in error responses # This helps when errors occur before the CORS middleware can process them @app.exception_handler(StarletteHTTPException) async def custom_http_exception_handler(request: Request, exc: StarletteHTTPException): """ Custom exception handler that ensures CORS headers are included in error responses. This is particularly important for 413 (Payload Too Large) errors during file uploads. Note: If a reverse proxy (nginx, traefik) returns 413 before the request reaches FastAPI, this handler won't be called. In that case, configure your reverse proxy to add CORS headers to error responses. """ # Get the origin from the request origin = request.headers.get("origin", "*") return JSONResponse( status_code=exc.status_code, content={"detail": exc.detail}, headers={ **(exc.headers or {}), "Access-Control-Allow-Origin": origin, "Access-Control-Allow-Credentials": "true", "Access-Control-Allow-Methods": "*", "Access-Control-Allow-Headers": "*", }, ) def _cors_headers(request: Request) -> dict[str, str]: origin = request.headers.get("origin", "*") return { "Access-Control-Allow-Origin": origin, "Access-Control-Allow-Credentials": "true", "Access-Control-Allow-Methods": "*", "Access-Control-Allow-Headers": "*", } @app.exception_handler(NotFoundError) async def not_found_error_handler(request: Request, exc: NotFoundError): return JSONResponse( status_code=404, content={"detail": str(exc)}, headers=_cors_headers(request), ) @app.exception_handler(InvalidInputError) async def invalid_input_error_handler(request: Request, exc: InvalidInputError): return JSONResponse( status_code=400, content={"detail": str(exc)}, headers=_cors_headers(request), ) @app.exception_handler(AuthenticationError) async def authentication_error_handler(request: Request, exc: AuthenticationError): return JSONResponse( status_code=401, content={"detail": str(exc)}, headers=_cors_headers(request), ) @app.exception_handler(RateLimitError) async def rate_limit_error_handler(request: Request, exc: RateLimitError): return JSONResponse( status_code=429, content={"detail": str(exc)}, headers=_cors_headers(request), ) @app.exception_handler(ConfigurationError) async def configuration_error_handler(request: Request, exc: ConfigurationError): return JSONResponse( status_code=422, content={"detail": str(exc)}, headers=_cors_headers(request), ) @app.exception_handler(NetworkError) async def network_error_handler(request: Request, exc: NetworkError): return JSONResponse( status_code=502, content={"detail": str(exc)}, headers=_cors_headers(request), ) @app.exception_handler(ExternalServiceError) async def external_service_error_handler(request: Request, exc: ExternalServiceError): return JSONResponse( status_code=502, content={"detail": str(exc)}, headers=_cors_headers(request), ) @app.exception_handler(OpenNotebookError) async def open_notebook_error_handler(request: Request, exc: OpenNotebookError): return JSONResponse( status_code=500, content={"detail": str(exc)}, headers=_cors_headers(request), ) # Include routers app.include_router(auth.router, prefix="/api", tags=["auth"]) app.include_router(config.router, prefix="/api", tags=["config"]) app.include_router(notebooks.router, prefix="/api", tags=["notebooks"]) app.include_router(search.router, prefix="/api", tags=["search"]) app.include_router(models.router, prefix="/api", tags=["models"]) app.include_router(transformations.router, prefix="/api", tags=["transformations"]) app.include_router(notes.router, prefix="/api", tags=["notes"]) app.include_router(embedding.router, prefix="/api", tags=["embedding"]) app.include_router( embedding_rebuild.router, prefix="/api/embeddings", tags=["embeddings"] ) app.include_router(settings.router, prefix="/api", tags=["settings"]) app.include_router(context.router, prefix="/api", tags=["context"]) app.include_router(sources.router, prefix="/api", tags=["sources"]) app.include_router(insights.router, prefix="/api", tags=["insights"]) app.include_router(commands_router.router, prefix="/api", tags=["commands"]) app.include_router(podcasts.router, prefix="/api", tags=["podcasts"]) app.include_router(episode_profiles.router, prefix="/api", tags=["episode-profiles"]) app.include_router(speaker_profiles.router, prefix="/api", tags=["speaker-profiles"]) app.include_router(chat.router, prefix="/api", tags=["chat"]) app.include_router(source_chat.router, prefix="/api", tags=["source-chat"]) app.include_router(credentials.router, prefix="/api", tags=["credentials"]) app.include_router(languages.router, prefix="/api", tags=["languages"]) @app.get("/") async def root(): return {"message": "Open Notebook API is running"} @app.get("/health") async def health(): return {"status": "healthy"} ================================================ FILE: api/models.py ================================================ from typing import Any, Dict, List, Literal, Optional from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator # Notebook models class NotebookCreate(BaseModel): name: str = Field(..., description="Name of the notebook") description: str = Field(default="", description="Description of the notebook") class NotebookUpdate(BaseModel): name: Optional[str] = Field(None, description="Name of the notebook") description: Optional[str] = Field(None, description="Description of the notebook") archived: Optional[bool] = Field( None, description="Whether the notebook is archived" ) class NotebookResponse(BaseModel): id: str name: str description: str archived: bool created: str updated: str source_count: int note_count: int # Search models class SearchRequest(BaseModel): query: str = Field(..., description="Search query") type: Literal["text", "vector"] = Field("text", description="Search type") limit: int = Field(100, description="Maximum number of results", le=1000) search_sources: bool = Field(True, description="Include sources in search") search_notes: bool = Field(True, description="Include notes in search") minimum_score: float = Field( 0.2, description="Minimum score for vector search", ge=0, le=1 ) class SearchResponse(BaseModel): results: List[Dict[str, Any]] = Field(..., description="Search results") total_count: int = Field(..., description="Total number of results") search_type: str = Field(..., description="Type of search performed") class AskRequest(BaseModel): question: str = Field(..., description="Question to ask the knowledge base") strategy_model: str = Field(..., description="Model ID for query strategy") answer_model: str = Field(..., description="Model ID for individual answers") final_answer_model: str = Field(..., description="Model ID for final answer") class AskResponse(BaseModel): answer: str = Field(..., description="Final answer from the knowledge base") question: str = Field(..., description="Original question") # Models API models class ModelCreate(BaseModel): name: str = Field(..., description="Model name (e.g., gpt-5-mini, claude, gemini)") provider: str = Field( ..., description="Provider name (e.g., openai, anthropic, gemini)" ) type: str = Field( ..., description="Model type (language, embedding, text_to_speech, speech_to_text)", ) credential: Optional[str] = Field( None, description="Credential ID to link this model to" ) class ModelResponse(BaseModel): id: str name: str provider: str type: str credential: Optional[str] = None created: str updated: str class DefaultModelsResponse(BaseModel): default_chat_model: Optional[str] = None default_transformation_model: Optional[str] = None large_context_model: Optional[str] = None default_text_to_speech_model: Optional[str] = None default_speech_to_text_model: Optional[str] = None default_embedding_model: Optional[str] = None default_tools_model: Optional[str] = None class ProviderAvailabilityResponse(BaseModel): available: List[str] = Field(..., description="List of available providers") unavailable: List[str] = Field(..., description="List of unavailable providers") supported_types: Dict[str, List[str]] = Field( ..., description="Provider to supported model types mapping" ) # Transformations API models class TransformationCreate(BaseModel): name: str = Field(..., description="Transformation name") title: str = Field(..., description="Display title for the transformation") description: str = Field( ..., description="Description of what this transformation does" ) prompt: str = Field(..., description="The transformation prompt") apply_default: bool = Field( False, description="Whether to apply this transformation by default" ) class TransformationUpdate(BaseModel): name: Optional[str] = Field(None, description="Transformation name") title: Optional[str] = Field( None, description="Display title for the transformation" ) description: Optional[str] = Field( None, description="Description of what this transformation does" ) prompt: Optional[str] = Field(None, description="The transformation prompt") apply_default: Optional[bool] = Field( None, description="Whether to apply this transformation by default" ) class TransformationResponse(BaseModel): id: str name: str title: str description: str prompt: str apply_default: bool created: str updated: str class TransformationExecuteRequest(BaseModel): model_config = ConfigDict(protected_namespaces=()) transformation_id: str = Field( ..., description="ID of the transformation to execute" ) input_text: str = Field(..., description="Text to transform") model_id: str = Field(..., description="Model ID to use for the transformation") class TransformationExecuteResponse(BaseModel): model_config = ConfigDict(protected_namespaces=()) output: str = Field(..., description="Transformed text") transformation_id: str = Field(..., description="ID of the transformation used") model_id: str = Field(..., description="Model ID used") # Default Prompt API models class DefaultPromptResponse(BaseModel): transformation_instructions: str = Field( ..., description="Default transformation instructions" ) class DefaultPromptUpdate(BaseModel): transformation_instructions: str = Field( ..., description="Default transformation instructions" ) # Notes API models class NoteCreate(BaseModel): title: Optional[str] = Field(None, description="Note title") content: str = Field(..., description="Note content") note_type: Optional[str] = Field("human", description="Type of note (human, ai)") notebook_id: Optional[str] = Field( None, description="Notebook ID to add the note to" ) class NoteUpdate(BaseModel): title: Optional[str] = Field(None, description="Note title") content: Optional[str] = Field(None, description="Note content") note_type: Optional[str] = Field(None, description="Type of note (human, ai)") class NoteResponse(BaseModel): id: str title: Optional[str] content: Optional[str] note_type: Optional[str] created: str updated: str command_id: Optional[str] = None # Embedding API models class EmbedRequest(BaseModel): item_id: str = Field(..., description="ID of the item to embed") item_type: str = Field(..., description="Type of item (source, note)") async_processing: bool = Field( False, description="Process asynchronously in background" ) class EmbedResponse(BaseModel): success: bool = Field(..., description="Whether embedding was successful") message: str = Field(..., description="Result message") item_id: str = Field(..., description="ID of the item that was embedded") item_type: str = Field(..., description="Type of item that was embedded") command_id: Optional[str] = Field( None, description="Command ID for async processing" ) # Rebuild request/response models class RebuildRequest(BaseModel): mode: Literal["existing", "all"] = Field( ..., description="Rebuild mode: 'existing' only re-embeds items with embeddings, 'all' embeds everything", ) include_sources: bool = Field(True, description="Include sources in rebuild") include_notes: bool = Field(True, description="Include notes in rebuild") include_insights: bool = Field(True, description="Include insights in rebuild") class RebuildResponse(BaseModel): command_id: str = Field(..., description="Command ID to track progress") total_items: int = Field(..., description="Estimated number of items to process") message: str = Field(..., description="Status message") class RebuildProgress(BaseModel): processed: int = Field(..., description="Number of items processed") total: int = Field(..., description="Total items to process") percentage: float = Field(..., description="Progress percentage") class RebuildStats(BaseModel): sources: int = Field(0, description="Sources processed") notes: int = Field(0, description="Notes processed") insights: int = Field(0, description="Insights processed") failed: int = Field(0, description="Failed items") class RebuildStatusResponse(BaseModel): command_id: str = Field(..., description="Command ID") status: str = Field(..., description="Status: queued, running, completed, failed") progress: Optional[RebuildProgress] = None stats: Optional[RebuildStats] = None started_at: Optional[str] = None completed_at: Optional[str] = None error_message: Optional[str] = None # Settings API models class SettingsResponse(BaseModel): default_content_processing_engine_doc: Optional[str] = None default_content_processing_engine_url: Optional[str] = None default_embedding_option: Optional[str] = None auto_delete_files: Optional[str] = None youtube_preferred_languages: Optional[List[str]] = None class SettingsUpdate(BaseModel): default_content_processing_engine_doc: Optional[str] = None default_content_processing_engine_url: Optional[str] = None default_embedding_option: Optional[str] = None auto_delete_files: Optional[str] = None youtube_preferred_languages: Optional[List[str]] = None # Sources API models class AssetModel(BaseModel): file_path: Optional[str] = None url: Optional[str] = None class SourceCreate(BaseModel): # Backward compatibility: support old single notebook_id notebook_id: Optional[str] = Field( None, description="Notebook ID to add the source to (deprecated, use notebooks)" ) # New multi-notebook support notebooks: Optional[List[str]] = Field( None, description="List of notebook IDs to add the source to" ) # Required fields type: str = Field(..., description="Source type: link, upload, or text") url: Optional[str] = Field(None, description="URL for link type") file_path: Optional[str] = Field(None, description="File path for upload type") content: Optional[str] = Field(None, description="Text content for text type") title: Optional[str] = Field(None, description="Source title") transformations: Optional[List[str]] = Field( default_factory=list, description="Transformation IDs to apply" ) embed: bool = Field(False, description="Whether to embed content for vector search") delete_source: bool = Field( False, description="Whether to delete uploaded file after processing" ) # New async processing support async_processing: bool = Field( False, description="Whether to process source asynchronously" ) @model_validator(mode="after") def validate_notebook_fields(self): # Ensure only one of notebook_id or notebooks is provided if self.notebook_id is not None and self.notebooks is not None: raise ValueError( "Cannot specify both 'notebook_id' and 'notebooks'. Use 'notebooks' for multi-notebook support." ) # Convert single notebook_id to notebooks array for internal processing if self.notebook_id is not None: self.notebooks = [self.notebook_id] # Keep notebook_id for backward compatibility in response # Set empty array if no notebooks specified (allow sources without notebooks) if self.notebooks is None: self.notebooks = [] return self class SourceUpdate(BaseModel): title: Optional[str] = Field(None, description="Source title") topics: Optional[List[str]] = Field(None, description="Source topics") class SourceResponse(BaseModel): id: str title: Optional[str] topics: Optional[List[str]] asset: Optional[AssetModel] full_text: Optional[str] embedded: bool embedded_chunks: int file_available: Optional[bool] = None created: str updated: str # New fields for async processing command_id: Optional[str] = None status: Optional[str] = None processing_info: Optional[Dict] = None # Notebook associations notebooks: Optional[List[str]] = None class SourceListResponse(BaseModel): id: str title: Optional[str] topics: Optional[List[str]] asset: Optional[AssetModel] embedded: bool # Boolean flag indicating if source has embeddings embedded_chunks: int # Number of embedded chunks insights_count: int created: str updated: str file_available: Optional[bool] = None # Status fields for async processing command_id: Optional[str] = None status: Optional[str] = None processing_info: Optional[Dict[str, Any]] = None # Context API models class ContextConfig(BaseModel): sources: Dict[str, str] = Field( default_factory=dict, description="Source inclusion config {source_id: level}" ) notes: Dict[str, str] = Field( default_factory=dict, description="Note inclusion config {note_id: level}" ) class ContextRequest(BaseModel): notebook_id: str = Field(..., description="Notebook ID to get context for") context_config: Optional[ContextConfig] = Field( None, description="Context configuration" ) class ContextResponse(BaseModel): notebook_id: str sources: List[Dict[str, Any]] = Field(..., description="Source context data") notes: List[Dict[str, Any]] = Field(..., description="Note context data") total_tokens: Optional[int] = Field(None, description="Estimated token count") # Insights API models class SourceInsightResponse(BaseModel): id: str source_id: str insight_type: str content: str created: str updated: str class InsightCreationResponse(BaseModel): """Response for async insight creation.""" status: Literal["pending"] = "pending" message: str = "Insight generation started" source_id: str transformation_id: str command_id: Optional[str] = None class SaveAsNoteRequest(BaseModel): notebook_id: Optional[str] = Field(None, description="Notebook ID to add note to") class CreateSourceInsightRequest(BaseModel): model_config = ConfigDict(protected_namespaces=()) transformation_id: str = Field(..., description="ID of transformation to apply") model_id: Optional[str] = Field( None, description="Model ID (uses default if not provided)" ) # Source status response class SourceStatusResponse(BaseModel): status: Optional[str] = Field(None, description="Processing status") message: str = Field(..., description="Descriptive message about the status") processing_info: Optional[Dict[str, Any]] = Field( None, description="Detailed processing information" ) command_id: Optional[str] = Field(None, description="Command ID if available") # Error response class ErrorResponse(BaseModel): error: str message: str # API Key Configuration models class SetApiKeyRequest(BaseModel): """Request to set an API key for a provider.""" api_key: Optional[str] = Field(None, description="API key for the provider") base_url: Optional[str] = Field( None, description="Base URL for URL-based providers (Ollama, OpenAI-compatible)" ) endpoint: Optional[str] = Field( None, description="Endpoint URL for Azure OpenAI" ) api_version: Optional[str] = Field( None, description="API version for Azure OpenAI" ) endpoint_llm: Optional[str] = Field( None, description="Service-specific endpoint for LLM (Azure)" ) endpoint_embedding: Optional[str] = Field( None, description="Service-specific endpoint for embedding (Azure)" ) endpoint_stt: Optional[str] = Field( None, description="Service-specific endpoint for STT (Azure)" ) endpoint_tts: Optional[str] = Field( None, description="Service-specific endpoint for TTS (Azure)" ) service_type: Optional[Literal["llm", "embedding", "stt", "tts"]] = Field( None, description="Service type for OpenAI-compatible providers (llm, embedding, stt, tts)", ) # Vertex AI specific fields vertex_project: Optional[str] = Field( None, description="Google Cloud Project ID for Vertex AI" ) vertex_location: Optional[str] = Field( None, description="Google Cloud Region for Vertex AI (e.g., us-central1)" ) vertex_credentials_path: Optional[str] = Field( None, description="Path to Google Cloud service account JSON file" ) @field_validator( "api_key", "base_url", "endpoint", "api_version", "endpoint_llm", "endpoint_embedding", "endpoint_stt", "endpoint_tts", "vertex_project", "vertex_location", "vertex_credentials_path", mode="before", ) @classmethod def validate_not_empty_string(cls, v: Optional[str]) -> Optional[str]: """Reject empty strings - convert to None or raise error.""" if v is not None: stripped = v.strip() if not stripped: return None # Treat empty/whitespace-only as None return stripped return v class ApiKeyStatusResponse(BaseModel): """Response showing which providers are configured and their source.""" configured: Dict[str, bool] = Field( ..., description="Map of provider name to whether it is configured" ) source: Dict[str, Literal["database", "environment", "none"]] = Field( ..., description="Map of provider name to configuration source (database, environment, or none)", ) encryption_configured: bool = Field( ..., description="Whether OPEN_NOTEBOOK_ENCRYPTION_KEY is set (required to store keys in database)", ) class TestConnectionResponse(BaseModel): """Response from testing a provider connection.""" provider: str = Field(..., description="Provider name that was tested") success: bool = Field(..., description="Whether connection test succeeded") message: str = Field(..., description="Result message with details") class MigrateFromEnvRequest(BaseModel): """Request to migrate API keys from environment variables to database.""" force: bool = Field( False, description="Force overwrite existing database configurations" ) class MigrationResult(BaseModel): """Response from migrating API keys from environment to database.""" message: str = Field(..., description="Summary message") migrated: List[str] = Field( default_factory=list, description="Providers successfully migrated" ) skipped: List[str] = Field( default_factory=list, description="Providers skipped (already in DB)" ) errors: List[str] = Field( default_factory=list, description="Migration errors by provider" ) # Notebook delete cascade models # Credential models class CreateCredentialRequest(BaseModel): """Request to create a new credential.""" name: str = Field(..., description="Credential name") provider: str = Field(..., description="Provider name (openai, anthropic, etc.)") modalities: List[str] = Field( default_factory=list, description="Supported modalities (language, embedding, text_to_speech, speech_to_text)", ) api_key: Optional[str] = Field(None, description="API key (stored encrypted)") base_url: Optional[str] = Field(None, description="Base URL") endpoint: Optional[str] = Field(None, description="Endpoint URL (Azure)") api_version: Optional[str] = Field(None, description="API version (Azure)") endpoint_llm: Optional[str] = Field(None, description="LLM endpoint") endpoint_embedding: Optional[str] = Field(None, description="Embedding endpoint") endpoint_stt: Optional[str] = Field(None, description="STT endpoint") endpoint_tts: Optional[str] = Field(None, description="TTS endpoint") project: Optional[str] = Field(None, description="Project ID (Vertex)") location: Optional[str] = Field(None, description="Location (Vertex)") credentials_path: Optional[str] = Field( None, description="Credentials file path (Vertex)" ) class UpdateCredentialRequest(BaseModel): """Request to update an existing credential.""" name: Optional[str] = Field(None, description="Credential name") modalities: Optional[List[str]] = Field(None, description="Supported modalities") api_key: Optional[str] = Field(None, description="API key (stored encrypted)") base_url: Optional[str] = Field(None, description="Base URL") endpoint: Optional[str] = Field(None, description="Endpoint URL") api_version: Optional[str] = Field(None, description="API version") endpoint_llm: Optional[str] = Field(None, description="LLM endpoint") endpoint_embedding: Optional[str] = Field(None, description="Embedding endpoint") endpoint_stt: Optional[str] = Field(None, description="STT endpoint") endpoint_tts: Optional[str] = Field(None, description="TTS endpoint") project: Optional[str] = Field(None, description="Project ID") location: Optional[str] = Field(None, description="Location") credentials_path: Optional[str] = Field(None, description="Credentials path") class CredentialResponse(BaseModel): """Response for a credential (never includes api_key).""" id: str name: str provider: str modalities: List[str] base_url: Optional[str] = None endpoint: Optional[str] = None api_version: Optional[str] = None endpoint_llm: Optional[str] = None endpoint_embedding: Optional[str] = None endpoint_stt: Optional[str] = None endpoint_tts: Optional[str] = None project: Optional[str] = None location: Optional[str] = None credentials_path: Optional[str] = None has_api_key: bool = False created: str updated: str model_count: int = 0 class CredentialDeleteResponse(BaseModel): """Response for credential deletion.""" message: str deleted_models: int = 0 class DiscoveredModelResponse(BaseModel): """A model discovered from a provider.""" name: str provider: str model_type: Optional[str] = None description: Optional[str] = None class DiscoverModelsResponse(BaseModel): """Response from model discovery.""" credential_id: str provider: str discovered: List[DiscoveredModelResponse] class RegisterModelData(BaseModel): """A model to register with user-specified type.""" name: str provider: str model_type: str # Required: user specifies the type class RegisterModelsRequest(BaseModel): """Request to register discovered models.""" models: List[RegisterModelData] class RegisterModelsResponse(BaseModel): """Response from model registration.""" created: int existing: int class NotebookDeletePreview(BaseModel): notebook_id: str = Field(..., description="ID of the notebook") notebook_name: str = Field(..., description="Name of the notebook") note_count: int = Field(..., description="Number of notes that will be deleted") exclusive_source_count: int = Field( ..., description="Number of sources only in this notebook" ) shared_source_count: int = Field( ..., description="Number of sources shared with other notebooks" ) class NotebookDeleteResponse(BaseModel): message: str = Field(..., description="Success message") deleted_notes: int = Field(..., description="Number of notes deleted") deleted_sources: int = Field(..., description="Number of exclusive sources deleted") unlinked_sources: int = Field( ..., description="Number of sources unlinked from notebook" ) ================================================ FILE: api/models_service.py ================================================ """ Models service layer using API. """ from typing import List, Optional from loguru import logger from api.client import api_client from open_notebook.ai.models import DefaultModels, Model class ModelsService: """Service layer for models operations using API.""" def __init__(self): logger.info("Using API for models operations") def get_all_models(self, model_type: Optional[str] = None) -> List[Model]: """Get all models with optional type filtering.""" models_data = api_client.get_models(model_type=model_type) # Convert API response to Model objects models = [] for model_data in models_data: model = Model( name=model_data["name"], provider=model_data["provider"], type=model_data["type"], ) model.id = model_data["id"] model.created = model_data["created"] model.updated = model_data["updated"] models.append(model) return models def create_model(self, name: str, provider: str, model_type: str) -> Model: """Create a new model.""" response = api_client.create_model(name, provider, model_type) model_data = response if isinstance(response, dict) else response[0] model = Model( name=model_data["name"], provider=model_data["provider"], type=model_data["type"], ) model.id = model_data["id"] model.created = model_data["created"] model.updated = model_data["updated"] return model def delete_model(self, model_id: str) -> bool: """Delete a model.""" api_client.delete_model(model_id) return True def get_default_models(self) -> DefaultModels: """Get default model assignments.""" response = api_client.get_default_models() defaults_data = response if isinstance(response, dict) else response[0] defaults = DefaultModels() # Set the values from API response defaults.default_chat_model = defaults_data.get("default_chat_model") defaults.default_transformation_model = defaults_data.get( "default_transformation_model" ) defaults.large_context_model = defaults_data.get("large_context_model") defaults.default_text_to_speech_model = defaults_data.get( "default_text_to_speech_model" ) defaults.default_speech_to_text_model = defaults_data.get( "default_speech_to_text_model" ) defaults.default_embedding_model = defaults_data.get("default_embedding_model") defaults.default_tools_model = defaults_data.get("default_tools_model") return defaults def update_default_models(self, defaults: DefaultModels) -> DefaultModels: """Update default model assignments.""" updates = { "default_chat_model": defaults.default_chat_model, "default_transformation_model": defaults.default_transformation_model, "large_context_model": defaults.large_context_model, "default_text_to_speech_model": defaults.default_text_to_speech_model, "default_speech_to_text_model": defaults.default_speech_to_text_model, "default_embedding_model": defaults.default_embedding_model, "default_tools_model": defaults.default_tools_model, } response = api_client.update_default_models(**updates) defaults_data = response if isinstance(response, dict) else response[0] # Update the defaults object with the response defaults.default_chat_model = defaults_data.get("default_chat_model") defaults.default_transformation_model = defaults_data.get( "default_transformation_model" ) defaults.large_context_model = defaults_data.get("large_context_model") defaults.default_text_to_speech_model = defaults_data.get( "default_text_to_speech_model" ) defaults.default_speech_to_text_model = defaults_data.get( "default_speech_to_text_model" ) defaults.default_embedding_model = defaults_data.get("default_embedding_model") defaults.default_tools_model = defaults_data.get("default_tools_model") return defaults # Global service instance models_service = ModelsService() ================================================ FILE: api/notebook_service.py ================================================ """ Notebook service layer using API. """ from typing import List, Optional from loguru import logger from api.client import api_client from open_notebook.domain.notebook import Notebook class NotebookService: """Service layer for notebook operations using API.""" def __init__(self): logger.info("Using API for notebook operations") def get_all_notebooks(self, order_by: str = "updated desc") -> List[Notebook]: """Get all notebooks.""" notebooks_data = api_client.get_notebooks(order_by=order_by) # Convert API response to Notebook objects notebooks = [] for nb_data in notebooks_data: nb = Notebook( name=nb_data["name"], description=nb_data["description"], archived=nb_data["archived"], ) nb.id = nb_data["id"] nb.created = nb_data["created"] nb.updated = nb_data["updated"] notebooks.append(nb) return notebooks def get_notebook(self, notebook_id: str) -> Optional[Notebook]: """Get a specific notebook.""" response = api_client.get_notebook(notebook_id) nb_data = response if isinstance(response, dict) else response[0] nb = Notebook( name=nb_data["name"], description=nb_data["description"], archived=nb_data["archived"], ) nb.id = nb_data["id"] nb.created = nb_data["created"] nb.updated = nb_data["updated"] return nb def create_notebook(self, name: str, description: str = "") -> Notebook: """Create a new notebook.""" response = api_client.create_notebook(name, description) nb_data = response if isinstance(response, dict) else response[0] nb = Notebook( name=nb_data["name"], description=nb_data["description"], archived=nb_data["archived"], ) nb.id = nb_data["id"] nb.created = nb_data["created"] nb.updated = nb_data["updated"] return nb def update_notebook(self, notebook: Notebook) -> Notebook: """Update a notebook.""" updates = { "name": notebook.name, "description": notebook.description, "archived": notebook.archived, } response = api_client.update_notebook(notebook.id or "", **updates) nb_data = response if isinstance(response, dict) else response[0] # Update the notebook object with the response notebook.name = nb_data["name"] notebook.description = nb_data["description"] notebook.archived = nb_data["archived"] notebook.updated = nb_data["updated"] return notebook def delete_notebook(self, notebook: Notebook) -> bool: """Delete a notebook.""" api_client.delete_notebook(notebook.id or "") return True # Global service instance notebook_service = NotebookService() ================================================ FILE: api/notes_service.py ================================================ """ Notes service layer using API. """ from typing import List, Optional from loguru import logger from api.client import api_client from open_notebook.domain.notebook import Note class NotesService: """Service layer for notes operations using API.""" def __init__(self): logger.info("Using API for notes operations") def get_all_notes(self, notebook_id: Optional[str] = None) -> List[Note]: """Get all notes with optional notebook filtering.""" notes_data = api_client.get_notes(notebook_id=notebook_id) # Convert API response to Note objects notes = [] for note_data in notes_data: note = Note( title=note_data["title"], content=note_data["content"], note_type=note_data["note_type"], ) note.id = note_data["id"] note.created = note_data["created"] note.updated = note_data["updated"] notes.append(note) return notes def get_note(self, note_id: str) -> Note: """Get a specific note.""" note_response = api_client.get_note(note_id) note_data = ( note_response if isinstance(note_response, dict) else note_response[0] ) note = Note( title=note_data["title"], content=note_data["content"], note_type=note_data["note_type"], ) note.id = note_data["id"] note.created = note_data["created"] note.updated = note_data["updated"] return note def create_note( self, content: str, title: Optional[str] = None, note_type: str = "human", notebook_id: Optional[str] = None, ) -> Note: """Create a new note.""" note_response = api_client.create_note( content=content, title=title, note_type=note_type, notebook_id=notebook_id ) note_data = ( note_response if isinstance(note_response, dict) else note_response[0] ) note = Note( title=note_data["title"], content=note_data["content"], note_type=note_data["note_type"], ) note.id = note_data["id"] note.created = note_data["created"] note.updated = note_data["updated"] return note def update_note(self, note: Note) -> Note: """Update a note.""" updates = { "title": note.title, "content": note.content, "note_type": note.note_type, } note_response = api_client.update_note(note.id or "", **updates) note_data = ( note_response if isinstance(note_response, dict) else note_response[0] ) # Update the note object with the response note.title = note_data["title"] note.content = note_data["content"] note.note_type = note_data["note_type"] note.updated = note_data["updated"] return note def delete_note(self, note_id: str) -> bool: """Delete a note.""" api_client.delete_note(note_id) return True # Global service instance notes_service = NotesService() ================================================ FILE: api/podcast_api_service.py ================================================ """ Podcast service layer using API client. This replaces direct httpx calls in the Streamlit pages. """ from typing import Any, Dict, List from loguru import logger from api.client import api_client class PodcastAPIService: """Service layer for podcast operations using API client.""" def __init__(self): logger.info("Using API client for podcast operations") # Episode methods def get_episodes(self) -> List[Dict[Any, Any]]: """Get all podcast episodes.""" result = api_client._make_request("GET", "/api/podcasts/episodes") return result if isinstance(result, list) else [result] def delete_episode(self, episode_id: str) -> bool: """Delete a podcast episode.""" try: api_client._make_request("DELETE", f"/api/podcasts/episodes/{episode_id}") return True except Exception as e: logger.error(f"Failed to delete episode: {e}") return False # Episode Profile methods def get_episode_profiles(self) -> List[Dict]: """Get all episode profiles.""" return api_client.get_episode_profiles() def create_episode_profile(self, profile_data: Dict) -> bool: """Create a new episode profile.""" try: api_client.create_episode_profile(**profile_data) return True except Exception as e: logger.error(f"Failed to create episode profile: {e}") return False def update_episode_profile(self, profile_id: str, profile_data: Dict) -> bool: """Update an episode profile.""" try: api_client.update_episode_profile(profile_id, **profile_data) return True except Exception as e: logger.error(f"Failed to update episode profile: {e}") return False def delete_episode_profile(self, profile_id: str) -> bool: """Delete an episode profile.""" try: api_client.delete_episode_profile(profile_id) return True except Exception as e: logger.error(f"Failed to delete episode profile: {e}") return False def duplicate_episode_profile(self, profile_id: str) -> bool: """Duplicate an episode profile.""" try: api_client._make_request( "POST", f"/api/episode-profiles/{profile_id}/duplicate" ) return True except Exception as e: logger.error(f"Failed to duplicate episode profile: {e}") return False # Speaker Profile methods def get_speaker_profiles(self) -> List[Dict[Any, Any]]: """Get all speaker profiles.""" result = api_client._make_request("GET", "/api/speaker-profiles") return result if isinstance(result, list) else [result] def create_speaker_profile(self, profile_data: Dict) -> bool: """Create a new speaker profile.""" try: api_client._make_request("POST", "/api/speaker-profiles", json=profile_data) return True except Exception as e: logger.error(f"Failed to create speaker profile: {e}") return False def update_speaker_profile(self, profile_id: str, profile_data: Dict) -> bool: """Update a speaker profile.""" try: api_client._make_request( "PUT", f"/api/speaker-profiles/{profile_id}", json=profile_data ) return True except Exception as e: logger.error(f"Failed to update speaker profile: {e}") return False def delete_speaker_profile(self, profile_id: str) -> bool: """Delete a speaker profile.""" try: api_client._make_request("DELETE", f"/api/speaker-profiles/{profile_id}") return True except Exception as e: logger.error(f"Failed to delete speaker profile: {e}") return False def duplicate_speaker_profile(self, profile_id: str) -> bool: """Duplicate a speaker profile.""" try: api_client._make_request( "POST", f"/api/speaker-profiles/{profile_id}/duplicate" ) return True except Exception as e: logger.error(f"Failed to duplicate speaker profile: {e}") return False # Global service instance podcast_api_service = PodcastAPIService() ================================================ FILE: api/podcast_service.py ================================================ from typing import Any, Dict, Optional from fastapi import HTTPException from loguru import logger from pydantic import BaseModel from surreal_commands import get_command_status, submit_command from open_notebook.domain.notebook import Notebook from open_notebook.podcasts.models import EpisodeProfile, PodcastEpisode, SpeakerProfile class PodcastGenerationRequest(BaseModel): """Request model for podcast generation""" episode_profile: str speaker_profile: str episode_name: str content: Optional[str] = None notebook_id: Optional[str] = None briefing_suffix: Optional[str] = None class PodcastGenerationResponse(BaseModel): """Response model for podcast generation""" job_id: str status: str message: str episode_profile: str episode_name: str class PodcastService: """Service layer for podcast operations""" @staticmethod async def submit_generation_job( episode_profile_name: str, speaker_profile_name: str, episode_name: str, notebook_id: Optional[str] = None, content: Optional[str] = None, briefing_suffix: Optional[str] = None, ) -> str: """Submit a podcast generation job for background processing""" try: # Validate episode profile exists episode_profile = await EpisodeProfile.get_by_name(episode_profile_name) if not episode_profile: raise ValueError(f"Episode profile '{episode_profile_name}' not found") # Validate speaker profile exists speaker_profile = await SpeakerProfile.get_by_name(speaker_profile_name) if not speaker_profile: raise ValueError(f"Speaker profile '{speaker_profile_name}' not found") # Get content from notebook if not provided directly if not content and notebook_id: try: notebook = await Notebook.get(notebook_id) # Get notebook context (this may need to be adjusted based on actual Notebook implementation) content = ( await notebook.get_context() if hasattr(notebook, "get_context") else str(notebook) ) except Exception as e: logger.warning( f"Failed to get notebook content, using notebook_id as content: {e}" ) content = f"Notebook ID: {notebook_id}" if not content: raise ValueError( "Content is required - provide either content or notebook_id" ) # Prepare command arguments command_args = { "episode_profile": episode_profile_name, "speaker_profile": speaker_profile_name, "episode_name": episode_name, "content": str(content), "briefing_suffix": briefing_suffix, } # Ensure command modules are imported before submitting # This is needed because submit_command validates against local registry try: import commands.podcast_commands # noqa: F401 except ImportError as import_err: logger.error(f"Failed to import podcast commands: {import_err}") raise ValueError("Podcast commands not available") # Submit command to surreal-commands job_id = submit_command("open_notebook", "generate_podcast", command_args) # Convert RecordID to string if needed if not job_id: raise ValueError("Failed to get job_id from submit_command") job_id_str = str(job_id) logger.info( f"Submitted podcast generation job: {job_id_str} for episode '{episode_name}'" ) return job_id_str except Exception as e: logger.error(f"Failed to submit podcast generation job: {e}") raise HTTPException( status_code=500, detail=f"Failed to submit podcast generation job: {str(e)}", ) @staticmethod async def get_job_status(job_id: str) -> Dict[str, Any]: """Get status of a podcast generation job""" try: status = await get_command_status(job_id) return { "job_id": job_id, "status": status.status if status else "unknown", "result": status.result if status else None, "error_message": getattr(status, "error_message", None) if status else None, "created": str(status.created) if status and hasattr(status, "created") and status.created else None, "updated": str(status.updated) if status and hasattr(status, "updated") and status.updated else None, "progress": getattr(status, "progress", None) if status else None, } except Exception as e: logger.error(f"Failed to get podcast job status: {e}") raise HTTPException( status_code=500, detail=f"Failed to get job status: {str(e)}" ) @staticmethod async def list_episodes() -> list: """List all podcast episodes""" try: episodes = await PodcastEpisode.get_all(order_by="created desc") return episodes except Exception as e: logger.error(f"Failed to list podcast episodes: {e}") raise HTTPException( status_code=500, detail=f"Failed to list episodes: {str(e)}" ) @staticmethod async def get_episode(episode_id: str) -> PodcastEpisode: """Get a specific podcast episode""" try: episode = await PodcastEpisode.get(episode_id) return episode except Exception as e: logger.error(f"Failed to get podcast episode {episode_id}: {e}") raise HTTPException(status_code=404, detail=f"Episode not found: {str(e)}") class DefaultProfiles: """Utility class for creating default profiles (if needed beyond migration data)""" @staticmethod async def create_default_episode_profiles(): """Create default episode profiles if they don't exist""" try: # Check if profiles already exist existing = await EpisodeProfile.get_all() if existing: logger.info(f"Episode profiles already exist: {len(existing)} found") return existing # This would create profiles, but since we have migration data, # this is mainly for future extensibility logger.info( "Default episode profiles should be created via database migration" ) return [] except Exception as e: logger.error(f"Failed to create default episode profiles: {e}") raise @staticmethod async def create_default_speaker_profiles(): """Create default speaker profiles if they don't exist""" try: # Check if profiles already exist existing = await SpeakerProfile.get_all() if existing: logger.info(f"Speaker profiles already exist: {len(existing)} found") return existing # This would create profiles, but since we have migration data, # this is mainly for future extensibility logger.info( "Default speaker profiles should be created via database migration" ) return [] except Exception as e: logger.error(f"Failed to create default speaker profiles: {e}") raise ================================================ FILE: api/routers/__init__.py ================================================ ================================================ FILE: api/routers/auth.py ================================================ """ Authentication router for Open Notebook API. Provides endpoints to check authentication status. """ from fastapi import APIRouter from open_notebook.utils.encryption import get_secret_from_env router = APIRouter(prefix="/auth", tags=["auth"]) @router.get("/status") async def get_auth_status(): """ Check if authentication is enabled. Returns whether a password is required to access the API. Supports Docker secrets via OPEN_NOTEBOOK_PASSWORD_FILE. """ auth_enabled = bool(get_secret_from_env("OPEN_NOTEBOOK_PASSWORD")) return { "auth_enabled": auth_enabled, "message": "Authentication is required" if auth_enabled else "Authentication is disabled", } ================================================ FILE: api/routers/chat.py ================================================ import asyncio import traceback from typing import Any, Dict, List, Optional from fastapi import APIRouter, HTTPException, Query from langchain_core.runnables import RunnableConfig from loguru import logger from pydantic import BaseModel, Field from open_notebook.database.repository import ensure_record_id, repo_query from open_notebook.domain.notebook import ChatSession, Note, Notebook, Source from open_notebook.exceptions import ( NotFoundError, ) from open_notebook.graphs.chat import graph as chat_graph from open_notebook.utils.graph_utils import get_session_message_count router = APIRouter() # Request/Response models class CreateSessionRequest(BaseModel): notebook_id: str = Field(..., description="Notebook ID to create session for") title: Optional[str] = Field(None, description="Optional session title") model_override: Optional[str] = Field( None, description="Optional model override for this session" ) class UpdateSessionRequest(BaseModel): title: Optional[str] = Field(None, description="New session title") model_override: Optional[str] = Field( None, description="Model override for this session" ) class ChatMessage(BaseModel): id: str = Field(..., description="Message ID") type: str = Field(..., description="Message type (human|ai)") content: str = Field(..., description="Message content") timestamp: Optional[str] = Field(None, description="Message timestamp") class ChatSessionResponse(BaseModel): id: str = Field(..., description="Session ID") title: str = Field(..., description="Session title") notebook_id: Optional[str] = Field(None, description="Notebook ID") created: str = Field(..., description="Creation timestamp") updated: str = Field(..., description="Last update timestamp") message_count: Optional[int] = Field( None, description="Number of messages in session" ) model_override: Optional[str] = Field( None, description="Model override for this session" ) class ChatSessionWithMessagesResponse(ChatSessionResponse): messages: List[ChatMessage] = Field( default_factory=list, description="Session messages" ) class ExecuteChatRequest(BaseModel): session_id: str = Field(..., description="Chat session ID") message: str = Field(..., description="User message content") context: Dict[str, Any] = Field( ..., description="Chat context with sources and notes" ) model_override: Optional[str] = Field( None, description="Optional model override for this message" ) class ExecuteChatResponse(BaseModel): session_id: str = Field(..., description="Session ID") messages: List[ChatMessage] = Field(..., description="Updated message list") class BuildContextRequest(BaseModel): notebook_id: str = Field(..., description="Notebook ID") context_config: Dict[str, Any] = Field(..., description="Context configuration") class BuildContextResponse(BaseModel): context: Dict[str, Any] = Field(..., description="Built context data") token_count: int = Field(..., description="Estimated token count") char_count: int = Field(..., description="Character count") class SuccessResponse(BaseModel): success: bool = Field(True, description="Operation success status") message: str = Field(..., description="Success message") @router.get("/chat/sessions", response_model=List[ChatSessionResponse]) async def get_sessions(notebook_id: str = Query(..., description="Notebook ID")): """Get all chat sessions for a notebook.""" try: # Get notebook to verify it exists notebook = await Notebook.get(notebook_id) if not notebook: raise HTTPException(status_code=404, detail="Notebook not found") # Get sessions for this notebook sessions_list = await notebook.get_chat_sessions() results = [] for session in sessions_list: session_id = str(session.id) # Get message count from LangGraph state msg_count = await get_session_message_count(chat_graph, session_id) results.append( ChatSessionResponse( id=session.id or "", title=session.title or "Untitled Session", notebook_id=notebook_id, created=str(session.created), updated=str(session.updated), message_count=msg_count, model_override=getattr(session, "model_override", None), ) ) return results except NotFoundError: raise HTTPException(status_code=404, detail="Notebook not found") except Exception as e: logger.error(f"Error fetching chat sessions: {str(e)}") raise HTTPException( status_code=500, detail=f"Error fetching chat sessions: {str(e)}" ) @router.post("/chat/sessions", response_model=ChatSessionResponse) async def create_session(request: CreateSessionRequest): """Create a new chat session.""" try: # Verify notebook exists notebook = await Notebook.get(request.notebook_id) if not notebook: raise HTTPException(status_code=404, detail="Notebook not found") # Create new session session = ChatSession( title=request.title or f"Chat Session {asyncio.get_event_loop().time():.0f}", model_override=request.model_override, ) await session.save() # Relate session to notebook await session.relate_to_notebook(request.notebook_id) return ChatSessionResponse( id=session.id or "", title=session.title or "", notebook_id=request.notebook_id, created=str(session.created), updated=str(session.updated), message_count=0, model_override=session.model_override, ) except NotFoundError: raise HTTPException(status_code=404, detail="Notebook not found") except Exception as e: logger.error(f"Error creating chat session: {str(e)}") raise HTTPException( status_code=500, detail=f"Error creating chat session: {str(e)}" ) @router.get( "/chat/sessions/{session_id}", response_model=ChatSessionWithMessagesResponse ) async def get_session(session_id: str): """Get a specific session with its messages.""" try: # Get session # Ensure session_id has proper table prefix full_session_id = ( session_id if session_id.startswith("chat_session:") else f"chat_session:{session_id}" ) session = await ChatSession.get(full_session_id) if not session: raise HTTPException(status_code=404, detail="Session not found") # Get session state from LangGraph to retrieve messages # Use sync get_state() in a thread since SqliteSaver doesn't support async thread_state = await asyncio.to_thread( chat_graph.get_state, config=RunnableConfig(configurable={"thread_id": full_session_id}), ) # Extract messages from state messages: list[ChatMessage] = [] if thread_state and thread_state.values and "messages" in thread_state.values: for msg in thread_state.values["messages"]: messages.append( ChatMessage( id=getattr(msg, "id", f"msg_{len(messages)}"), type=msg.type if hasattr(msg, "type") else "unknown", content=msg.content if hasattr(msg, "content") else str(msg), timestamp=None, # LangChain messages don't have timestamps by default ) ) # Find notebook_id (we need to query the relationship) # Ensure session_id has proper table prefix full_session_id = ( session_id if session_id.startswith("chat_session:") else f"chat_session:{session_id}" ) notebook_query = await repo_query( "SELECT out FROM refers_to WHERE in = $session_id", {"session_id": ensure_record_id(full_session_id)}, ) notebook_id = notebook_query[0]["out"] if notebook_query else None if not notebook_id: # This might be an old session created before API migration logger.warning( f"No notebook relationship found for session {session_id} - may be an orphaned session" ) return ChatSessionWithMessagesResponse( id=session.id or "", title=session.title or "Untitled Session", notebook_id=notebook_id, created=str(session.created), updated=str(session.updated), message_count=len(messages), messages=messages, model_override=getattr(session, "model_override", None), ) except NotFoundError: raise HTTPException(status_code=404, detail="Session not found") except Exception as e: logger.error(f"Error fetching session: {str(e)}") raise HTTPException(status_code=500, detail=f"Error fetching session: {str(e)}") @router.put("/chat/sessions/{session_id}", response_model=ChatSessionResponse) async def update_session(session_id: str, request: UpdateSessionRequest): """Update session title.""" try: # Ensure session_id has proper table prefix full_session_id = ( session_id if session_id.startswith("chat_session:") else f"chat_session:{session_id}" ) session = await ChatSession.get(full_session_id) if not session: raise HTTPException(status_code=404, detail="Session not found") update_data = request.model_dump(exclude_unset=True) if "title" in update_data: session.title = update_data["title"] if "model_override" in update_data: session.model_override = update_data["model_override"] await session.save() # Find notebook_id # Ensure session_id has proper table prefix full_session_id = ( session_id if session_id.startswith("chat_session:") else f"chat_session:{session_id}" ) notebook_query = await repo_query( "SELECT out FROM refers_to WHERE in = $session_id", {"session_id": ensure_record_id(full_session_id)}, ) notebook_id = notebook_query[0]["out"] if notebook_query else None # Get message count from LangGraph state msg_count = await get_session_message_count(chat_graph, full_session_id) return ChatSessionResponse( id=session.id or "", title=session.title or "", notebook_id=notebook_id, created=str(session.created), updated=str(session.updated), message_count=msg_count, model_override=session.model_override, ) except NotFoundError: raise HTTPException(status_code=404, detail="Session not found") except Exception as e: logger.error(f"Error updating session: {str(e)}") raise HTTPException(status_code=500, detail=f"Error updating session: {str(e)}") @router.delete("/chat/sessions/{session_id}", response_model=SuccessResponse) async def delete_session(session_id: str): """Delete a chat session.""" try: # Ensure session_id has proper table prefix full_session_id = ( session_id if session_id.startswith("chat_session:") else f"chat_session:{session_id}" ) session = await ChatSession.get(full_session_id) if not session: raise HTTPException(status_code=404, detail="Session not found") await session.delete() return SuccessResponse(success=True, message="Session deleted successfully") except NotFoundError: raise HTTPException(status_code=404, detail="Session not found") except Exception as e: logger.error(f"Error deleting session: {str(e)}") raise HTTPException(status_code=500, detail=f"Error deleting session: {str(e)}") @router.post("/chat/execute", response_model=ExecuteChatResponse) async def execute_chat(request: ExecuteChatRequest): """Execute a chat request and get AI response.""" try: # Verify session exists # Ensure session_id has proper table prefix full_session_id = ( request.session_id if request.session_id.startswith("chat_session:") else f"chat_session:{request.session_id}" ) session = await ChatSession.get(full_session_id) if not session: raise HTTPException(status_code=404, detail="Session not found") # Determine model override (per-request override takes precedence over session-level) model_override = ( request.model_override if request.model_override is not None else getattr(session, "model_override", None) ) # Get current state # Use sync get_state() in a thread since SqliteSaver doesn't support async current_state = await asyncio.to_thread( chat_graph.get_state, config=RunnableConfig(configurable={"thread_id": full_session_id}), ) # Prepare state for execution state_values = current_state.values if current_state else {} state_values["messages"] = state_values.get("messages", []) state_values["context"] = request.context state_values["model_override"] = model_override # Add user message to state from langchain_core.messages import HumanMessage user_message = HumanMessage(content=request.message) state_values["messages"].append(user_message) # Execute chat graph result = chat_graph.invoke( input=state_values, # type: ignore[arg-type] config=RunnableConfig( configurable={ "thread_id": full_session_id, "model_id": model_override, } ), ) # Update session timestamp await session.save() # Convert messages to response format messages: list[ChatMessage] = [] for msg in result.get("messages", []): messages.append( ChatMessage( id=getattr(msg, "id", f"msg_{len(messages)}"), type=msg.type if hasattr(msg, "type") else "unknown", content=msg.content if hasattr(msg, "content") else str(msg), timestamp=None, ) ) return ExecuteChatResponse(session_id=request.session_id, messages=messages) except NotFoundError: raise HTTPException(status_code=404, detail="Session not found") except Exception as e: # Log detailed error with context for debugging logger.error( f"Error executing chat: {str(e)}\n" f" Session ID: {request.session_id}\n" f" Model override: {request.model_override}\n" f" Traceback:\n{traceback.format_exc()}" ) raise HTTPException(status_code=500, detail=f"Error executing chat: {str(e)}") @router.post("/chat/context", response_model=BuildContextResponse) async def build_context(request: BuildContextRequest): """Build context for a notebook based on context configuration.""" try: # Verify notebook exists notebook = await Notebook.get(request.notebook_id) if not notebook: raise HTTPException(status_code=404, detail="Notebook not found") context_data: dict[str, list[dict[str, str]]] = {"sources": [], "notes": []} total_content = "" # Process context configuration if provided if request.context_config: # Process sources for source_id, status in request.context_config.get("sources", {}).items(): if "not in" in status: continue try: # Add table prefix if not present full_source_id = ( source_id if source_id.startswith("source:") else f"source:{source_id}" ) try: source = await Source.get(full_source_id) except Exception: continue if "insights" in status: source_context = await source.get_context(context_size="short") context_data["sources"].append(source_context) total_content += str(source_context) elif "full content" in status: source_context = await source.get_context(context_size="long") context_data["sources"].append(source_context) total_content += str(source_context) except Exception as e: logger.warning(f"Error processing source {source_id}: {str(e)}") continue # Process notes for note_id, status in request.context_config.get("notes", {}).items(): if "not in" in status: continue try: # Add table prefix if not present full_note_id = ( note_id if note_id.startswith("note:") else f"note:{note_id}" ) note = await Note.get(full_note_id) if not note: continue if "full content" in status: note_context = note.get_context(context_size="long") context_data["notes"].append(note_context) total_content += str(note_context) except Exception as e: logger.warning(f"Error processing note {note_id}: {str(e)}") continue else: # Default behavior - include all sources and notes with short context sources = await notebook.get_sources() for source in sources: try: source_context = await source.get_context(context_size="short") context_data["sources"].append(source_context) total_content += str(source_context) except Exception as e: logger.warning(f"Error processing source {source.id}: {str(e)}") continue notes = await notebook.get_notes() for note in notes: try: note_context = note.get_context(context_size="short") context_data["notes"].append(note_context) total_content += str(note_context) except Exception as e: logger.warning(f"Error processing note {note.id}: {str(e)}") continue # Calculate character and token counts char_count = len(total_content) # Use token count utility if available try: from open_notebook.utils import token_count estimated_tokens = token_count(total_content) if total_content else 0 except ImportError: # Fallback to simple estimation estimated_tokens = char_count // 4 return BuildContextResponse( context=context_data, token_count=estimated_tokens, char_count=char_count ) except HTTPException: raise except Exception as e: logger.error(f"Error building context: {str(e)}") raise HTTPException(status_code=500, detail=f"Error building context: {str(e)}") ================================================ FILE: api/routers/commands.py ================================================ from typing import Any, Dict, List, Optional from fastapi import APIRouter, HTTPException, Query from loguru import logger from pydantic import BaseModel, Field from surreal_commands import registry from api.command_service import CommandService router = APIRouter() class CommandExecutionRequest(BaseModel): command: str = Field( ..., description="Command function name (e.g., 'process_text')" ) app: str = Field(..., description="Application name (e.g., 'open_notebook')") input: Dict[str, Any] = Field(..., description="Arguments to pass to the command") class CommandJobResponse(BaseModel): job_id: str status: str message: str class CommandJobStatusResponse(BaseModel): job_id: str status: str result: Optional[Dict[str, Any]] = None error_message: Optional[str] = None created: Optional[str] = None updated: Optional[str] = None progress: Optional[Dict[str, Any]] = None @router.post("/commands/jobs", response_model=CommandJobResponse) async def execute_command(request: CommandExecutionRequest): """ Submit a command for background processing. Returns immediately with job ID for status tracking. Example request: { "command": "process_text", "app": "open_notebook", "input": { "text": "Hello world", "operation": "uppercase" } } """ try: # Submit command using app name (not module name) job_id = await CommandService.submit_command_job( module_name=request.app, # This should be "open_notebook" command_name=request.command, command_args=request.input, ) return CommandJobResponse( job_id=job_id, status="submitted", message=f"Command '{request.command}' submitted successfully", ) except Exception as e: logger.error(f"Error submitting command: {str(e)}") raise HTTPException( status_code=500, detail="Failed to submit command" ) @router.get("/commands/jobs/{job_id}", response_model=CommandJobStatusResponse) async def get_command_job_status(job_id: str): """Get the status of a specific command job""" try: status_data = await CommandService.get_command_status(job_id) return CommandJobStatusResponse(**status_data) except Exception as e: logger.error(f"Error fetching job status: {str(e)}") raise HTTPException( status_code=500, detail="Failed to fetch job status" ) @router.get("/commands/jobs", response_model=List[Dict[str, Any]]) async def list_command_jobs( command_filter: Optional[str] = Query(None, description="Filter by command name"), status_filter: Optional[str] = Query(None, description="Filter by status"), limit: int = Query(50, description="Maximum number of jobs to return"), ): """List command jobs with optional filtering""" try: jobs = await CommandService.list_command_jobs( command_filter=command_filter, status_filter=status_filter, limit=limit ) return jobs except Exception as e: logger.error(f"Error listing command jobs: {str(e)}") raise HTTPException( status_code=500, detail="Failed to list command jobs" ) @router.delete("/commands/jobs/{job_id}") async def cancel_command_job(job_id: str): """Cancel a running command job""" try: success = await CommandService.cancel_command_job(job_id) return {"job_id": job_id, "cancelled": success} except Exception as e: logger.error(f"Error cancelling command job: {str(e)}") raise HTTPException( status_code=500, detail="Failed to cancel command job" ) @router.get("/commands/registry/debug") async def debug_registry(): """Debug endpoint to see what commands are registered""" try: # Get all registered commands all_items = registry.get_all_commands() # Create JSON-serializable data command_items = [] for item in all_items: try: command_items.append( { "app_id": item.app_id, "name": item.name, "full_id": f"{item.app_id}.{item.name}", } ) except Exception as item_error: logger.error(f"Error processing item: {item_error}") # Get the basic command structure try: commands_dict: dict[str, list[str]] = {} for item in all_items: if item.app_id not in commands_dict: commands_dict[item.app_id] = [] commands_dict[item.app_id].append(item.name) except Exception: commands_dict = {} return { "total_commands": len(all_items), "commands_by_app": commands_dict, "command_items": command_items, } except Exception as e: logger.error(f"Error debugging registry: {str(e)}") return { "error": str(e), "total_commands": 0, "commands_by_app": {}, "command_items": [], } ================================================ FILE: api/routers/config.py ================================================ import asyncio import os import time import tomllib from pathlib import Path from typing import Optional from fastapi import APIRouter, Request from loguru import logger from open_notebook.database.repository import repo_query from open_notebook.utils.version_utils import ( compare_versions, get_version_from_github_async, ) router = APIRouter() # In-memory cache for version check results _version_cache: dict = { "latest_version": None, "has_update": False, "timestamp": 0, "check_failed": False, } # Cache TTL in seconds (24 hours) VERSION_CACHE_TTL = 24 * 60 * 60 def get_version() -> str: """Read version from pyproject.toml""" try: pyproject_path = Path(__file__).parent.parent.parent / "pyproject.toml" with open(pyproject_path, "rb") as f: pyproject = tomllib.load(f) return pyproject.get("project", {}).get("version", "unknown") except Exception as e: logger.warning(f"Could not read version from pyproject.toml: {e}") return "unknown" async def get_latest_version_cached(current_version: str) -> tuple[Optional[str], bool]: """ Check for the latest version from GitHub with caching. Returns: tuple: (latest_version, has_update) - latest_version: str or None if check failed - has_update: bool indicating if update is available """ global _version_cache # Check if cache is still valid (within TTL) cache_age = time.time() - _version_cache["timestamp"] if _version_cache["timestamp"] > 0 and cache_age < VERSION_CACHE_TTL: logger.debug(f"Using cached version check result (age: {cache_age:.0f}s)") return _version_cache["latest_version"], _version_cache["has_update"] # Cache expired or not yet set if _version_cache["timestamp"] > 0: logger.info(f"Version cache expired (age: {cache_age:.0f}s), refreshing...") # Perform version check with strict error handling try: logger.info("Checking for latest version from GitHub...") # Fetch latest version from GitHub with 10-second timeout latest_version = await get_version_from_github_async( "https://github.com/lfnovo/open-notebook", "main" ) logger.info( f"Latest version from GitHub: {latest_version}, Current version: {current_version}" ) # Compare versions has_update = compare_versions(current_version, latest_version) < 0 # Cache the result _version_cache["latest_version"] = latest_version _version_cache["has_update"] = has_update _version_cache["timestamp"] = time.time() _version_cache["check_failed"] = False logger.info(f"Version check complete. Update available: {has_update}") return latest_version, has_update except Exception as e: logger.warning(f"Version check failed: {e}") # Cache the failure to avoid repeated attempts _version_cache["latest_version"] = None _version_cache["has_update"] = False _version_cache["timestamp"] = time.time() _version_cache["check_failed"] = True return None, False async def check_database_health() -> dict: """ Check if database is reachable using a lightweight query. Returns: dict with 'status' ("online" | "offline") and optional 'error' """ try: # 2-second timeout for database health check result = await asyncio.wait_for(repo_query("RETURN 1"), timeout=2.0) if result: return {"status": "online"} return {"status": "offline", "error": "Empty result"} except asyncio.TimeoutError: logger.warning("Database health check timed out after 2 seconds") return {"status": "offline", "error": "Health check timeout"} except Exception as e: logger.warning(f"Database health check failed: {e}") return {"status": "offline", "error": str(e)} @router.get("/config") async def get_config(request: Request): """ Get frontend configuration. Returns version information and health status. Note: The frontend determines the API URL via its own runtime-config endpoint, so this endpoint no longer returns apiUrl. Also checks for version updates from GitHub (with caching and error handling). """ # Get current version current_version = get_version() # Check for updates (with caching and error handling) # This MUST NOT break the endpoint - wrapped in try-except as extra safety latest_version = None has_update = False try: latest_version, has_update = await get_latest_version_cached(current_version) except Exception as e: # Extra safety: ensure version check never breaks the config endpoint logger.error(f"Unexpected error during version check: {e}") # Check database health db_health = await check_database_health() db_status = db_health["status"] if db_status == "offline": logger.warning(f"Database offline: {db_health.get('error', 'Unknown error')}") return { "version": current_version, "latestVersion": latest_version, "hasUpdate": has_update, "dbStatus": db_status, } ================================================ FILE: api/routers/context.py ================================================ from fastapi import APIRouter, HTTPException from loguru import logger from api.models import ContextRequest, ContextResponse from open_notebook.domain.notebook import Note, Notebook, Source from open_notebook.exceptions import InvalidInputError from open_notebook.utils import token_count router = APIRouter() @router.post("/notebooks/{notebook_id}/context", response_model=ContextResponse) async def get_notebook_context(notebook_id: str, context_request: ContextRequest): """Get context for a notebook based on configuration.""" try: # Verify notebook exists notebook = await Notebook.get(notebook_id) if not notebook: raise HTTPException(status_code=404, detail="Notebook not found") context_data: dict[str, list[dict[str, str]]] = {"note": [], "source": []} total_content = "" # Process context configuration if provided if context_request.context_config: # Process sources for source_id, status in context_request.context_config.sources.items(): if "not in" in status: continue try: # Add table prefix if not present full_source_id = ( source_id if source_id.startswith("source:") else f"source:{source_id}" ) try: source = await Source.get(full_source_id) except Exception: continue if "insights" in status: source_context = await source.get_context(context_size="short") context_data["source"].append(source_context) total_content += str(source_context) elif "full content" in status: source_context = await source.get_context(context_size="long") context_data["source"].append(source_context) total_content += str(source_context) except Exception as e: logger.warning(f"Error processing source {source_id}: {str(e)}") continue # Process notes for note_id, status in context_request.context_config.notes.items(): if "not in" in status: continue try: # Add table prefix if not present full_note_id = ( note_id if note_id.startswith("note:") else f"note:{note_id}" ) note = await Note.get(full_note_id) if not note: continue if "full content" in status: note_context = note.get_context(context_size="long") context_data["note"].append(note_context) total_content += str(note_context) except Exception as e: logger.warning(f"Error processing note {note_id}: {str(e)}") continue else: # Default behavior - include all sources and notes with short context sources = await notebook.get_sources() for source in sources: try: source_context = await source.get_context(context_size="short") context_data["source"].append(source_context) total_content += str(source_context) except Exception as e: logger.warning(f"Error processing source {source.id}: {str(e)}") continue notes = await notebook.get_notes() for note in notes: try: note_context = note.get_context(context_size="short") context_data["note"].append(note_context) total_content += str(note_context) except Exception as e: logger.warning(f"Error processing note {note.id}: {str(e)}") continue # Calculate estimated token count estimated_tokens = token_count(total_content) if total_content else 0 return ContextResponse( notebook_id=notebook_id, sources=context_data["source"], notes=context_data["note"], total_tokens=estimated_tokens, ) except HTTPException: raise except InvalidInputError as e: raise HTTPException(status_code=400, detail=str(e)) except Exception as e: logger.error(f"Error getting context for notebook {notebook_id}: {str(e)}") raise HTTPException(status_code=500, detail=f"Error getting context: {str(e)}") ================================================ FILE: api/routers/credentials.py ================================================ """ Credentials Router Thin HTTP layer for managing individual AI provider credentials. Business logic lives in api.credentials_service. Endpoints: - GET /credentials - List all credentials - GET /credentials/by-provider/{provider} - List credentials for a provider - POST /credentials - Create a new credential - GET /credentials/{credential_id} - Get a specific credential - PUT /credentials/{credential_id} - Update a credential - DELETE /credentials/{credential_id} - Delete a credential - POST /credentials/{credential_id}/test - Test connection - POST /credentials/{credential_id}/discover - Discover models - POST /credentials/{credential_id}/register-models - Register models NEVER returns actual API key values - only metadata. """ from typing import List, Optional from fastapi import APIRouter, HTTPException, Query from loguru import logger from pydantic import SecretStr from api.credentials_service import ( credential_to_response, discover_with_config, migrate_from_env as svc_migrate_from_env, migrate_from_provider_config as svc_migrate_from_provider_config, register_models, require_encryption_key, test_credential as svc_test_credential, validate_url, ) from api.credentials_service import ( get_env_status as svc_get_env_status, get_provider_status, ) from api.models import ( CreateCredentialRequest, CredentialDeleteResponse, CredentialResponse, DiscoveredModelResponse, DiscoverModelsResponse, RegisterModelsRequest, RegisterModelsResponse, UpdateCredentialRequest, ) from open_notebook.domain.credential import Credential router = APIRouter(prefix="/credentials", tags=["credentials"]) def _handle_value_error(e: ValueError, status_code: int = 400) -> HTTPException: """Convert a ValueError from the service layer to an HTTPException.""" return HTTPException(status_code=status_code, detail=str(e)) # ============================================================================= # Status endpoints # ============================================================================= @router.get("/status") async def get_status(): """ Get configuration status: encryption key status, and per-provider configured/source information. """ try: return await get_provider_status() except Exception as e: logger.error(f"Error fetching status: {e}") raise HTTPException(status_code=500, detail="Failed to fetch credential status") @router.get("/env-status") async def get_env_status(): """Check what's configured via environment variables.""" try: return await svc_get_env_status() except Exception as e: logger.error(f"Error checking env status: {e}") raise HTTPException(status_code=500, detail="Failed to check environment status") # ============================================================================= # CRUD endpoints # ============================================================================= @router.get("", response_model=List[CredentialResponse]) async def list_credentials( provider: Optional[str] = Query(None, description="Filter by provider"), ): """List all credentials, optionally filtered by provider.""" try: if provider: credentials = await Credential.get_by_provider(provider) else: credentials = await Credential.get_all(order_by="provider, created") result = [] for cred in credentials: models = await cred.get_linked_models() result.append(credential_to_response(cred, len(models))) return result except Exception as e: logger.error(f"Error listing credentials: {e}") raise HTTPException(status_code=500, detail="Failed to list credentials") @router.get("/by-provider/{provider}", response_model=List[CredentialResponse]) async def list_credentials_by_provider(provider: str): """List all credentials for a specific provider.""" try: credentials = await Credential.get_by_provider(provider.lower()) result = [] for cred in credentials: models = await cred.get_linked_models() result.append(credential_to_response(cred, len(models))) return result except Exception as e: logger.error(f"Error listing credentials for {provider}: {e}") raise HTTPException(status_code=500, detail="Failed to list credentials for provider") @router.post("", response_model=CredentialResponse, status_code=201) async def create_credential(request: CreateCredentialRequest): """Create a new credential.""" try: require_encryption_key() except ValueError as e: raise _handle_value_error(e) # Validate all URL fields for url_field in [ request.base_url, request.endpoint, request.endpoint_llm, request.endpoint_embedding, request.endpoint_stt, request.endpoint_tts, ]: if url_field: try: validate_url(url_field, request.provider) except ValueError as e: raise _handle_value_error(e) try: cred = Credential( name=request.name, provider=request.provider.lower(), modalities=request.modalities, api_key=SecretStr(request.api_key) if request.api_key else None, base_url=request.base_url, endpoint=request.endpoint, api_version=request.api_version, endpoint_llm=request.endpoint_llm, endpoint_embedding=request.endpoint_embedding, endpoint_stt=request.endpoint_stt, endpoint_tts=request.endpoint_tts, project=request.project, location=request.location, credentials_path=request.credentials_path, ) await cred.save() return credential_to_response(cred, 0) except Exception as e: logger.error(f"Error creating credential: {e}") raise HTTPException(status_code=500, detail="Failed to create credential") @router.get("/{credential_id}", response_model=CredentialResponse) async def get_credential(credential_id: str): """Get a specific credential by ID. Never returns api_key.""" try: cred = await Credential.get(credential_id) models = await cred.get_linked_models() return credential_to_response(cred, len(models)) except Exception as e: logger.error(f"Error fetching credential {credential_id}: {e}") raise HTTPException(status_code=404, detail="Credential not found") @router.put("/{credential_id}", response_model=CredentialResponse) async def update_credential(credential_id: str, request: UpdateCredentialRequest): """Update an existing credential.""" try: require_encryption_key() except ValueError as e: raise _handle_value_error(e) # Validate all URL fields being updated for url_field in [ request.base_url, request.endpoint, request.endpoint_llm, request.endpoint_embedding, request.endpoint_stt, request.endpoint_tts, ]: if url_field: try: validate_url(url_field, "update") except ValueError as e: raise _handle_value_error(e) try: cred = await Credential.get(credential_id) if request.name is not None: cred.name = request.name if request.modalities is not None: cred.modalities = request.modalities if request.api_key is not None: cred.api_key = SecretStr(request.api_key) if request.base_url is not None: cred.base_url = request.base_url or None if request.endpoint is not None: cred.endpoint = request.endpoint or None if request.api_version is not None: cred.api_version = request.api_version or None if request.endpoint_llm is not None: cred.endpoint_llm = request.endpoint_llm or None if request.endpoint_embedding is not None: cred.endpoint_embedding = request.endpoint_embedding or None if request.endpoint_stt is not None: cred.endpoint_stt = request.endpoint_stt or None if request.endpoint_tts is not None: cred.endpoint_tts = request.endpoint_tts or None if request.project is not None: cred.project = request.project or None if request.location is not None: cred.location = request.location or None if request.credentials_path is not None: cred.credentials_path = request.credentials_path or None await cred.save() models = await cred.get_linked_models() return credential_to_response(cred, len(models)) except HTTPException: raise except Exception as e: logger.error(f"Error updating credential {credential_id}: {e}") raise HTTPException(status_code=500, detail="Failed to update credential") @router.delete("/{credential_id}", response_model=CredentialDeleteResponse) async def delete_credential( credential_id: str, delete_models: bool = Query(False, description="Also delete linked models"), migrate_to: Optional[str] = Query( None, description="Migrate linked models to this credential ID" ), ): """ Delete a credential. If the credential has linked models: - Pass delete_models=true to delete them - Pass migrate_to= to reassign them - Without either, returns 409 with linked model info """ try: cred = await Credential.get(credential_id) linked_models = await cred.get_linked_models() if linked_models and not delete_models and not migrate_to: raise HTTPException( status_code=409, detail={ "message": f"Credential has {len(linked_models)} linked model(s)", "model_ids": [m.id for m in linked_models], "model_names": [f"{m.provider}/{m.name}" for m in linked_models], }, ) deleted_models = 0 if linked_models and migrate_to: # Migrate models to another credential target_cred = await Credential.get(migrate_to) for model in linked_models: model.credential = target_cred.id await model.save() elif linked_models and delete_models: # Delete linked models for model in linked_models: await model.delete() deleted_models += 1 # Delete the credential await cred.delete() return CredentialDeleteResponse( message="Credential deleted successfully", deleted_models=deleted_models, ) except HTTPException: raise except Exception as e: logger.error(f"Error deleting credential {credential_id}: {e}") raise HTTPException(status_code=500, detail="Failed to delete credential") # ============================================================================= # Test / Discover / Register endpoints # ============================================================================= @router.post("/{credential_id}/test") async def test_credential(credential_id: str): """Test connection using this credential's configuration.""" return await svc_test_credential(credential_id) @router.post("/{credential_id}/discover", response_model=DiscoverModelsResponse) async def discover_models_for_credential(credential_id: str): """Discover available models using this credential's API key.""" try: cred = await Credential.get(credential_id) config = cred.to_esperanto_config() provider = cred.provider.lower() discovered = await discover_with_config(provider, config) return DiscoverModelsResponse( credential_id=cred.id or "", provider=provider, discovered=[ DiscoveredModelResponse( name=d["name"], provider=d["provider"], description=d.get("description"), ) for d in discovered ], ) except Exception as e: logger.error(f"Error discovering models for credential {credential_id}: {e}") raise HTTPException(status_code=500, detail="Failed to discover models") @router.post("/{credential_id}/register-models", response_model=RegisterModelsResponse) async def register_models_for_credential( credential_id: str, request: RegisterModelsRequest ): """Register discovered models and link them to this credential.""" try: result = await register_models(credential_id, request.models) return RegisterModelsResponse(**result) except Exception as e: logger.error(f"Error registering models for credential {credential_id}: {e}") raise HTTPException(status_code=500, detail="Failed to register models") # ============================================================================= # Migration endpoints # ============================================================================= @router.post("/migrate-from-provider-config") async def migrate_from_provider_config(): """Migrate existing ProviderConfig data to individual credential records.""" try: return await svc_migrate_from_provider_config() except ValueError as e: raise _handle_value_error(e) except Exception as e: logger.error(f"ProviderConfig migration FAILED: {type(e).__name__}: {e}", exc_info=True) raise HTTPException(status_code=500, detail="Migration from provider config failed") @router.post("/migrate-from-env") async def migrate_from_env(): """Migrate API keys from environment variables to credential records.""" try: return await svc_migrate_from_env() except ValueError as e: raise _handle_value_error(e) except Exception as e: logger.error(f"Env migration FAILED: {type(e).__name__}: {e}", exc_info=True) raise HTTPException(status_code=500, detail="Migration from environment variables failed") ================================================ FILE: api/routers/embedding.py ================================================ from fastapi import APIRouter, HTTPException from loguru import logger from api.command_service import CommandService from api.models import EmbedRequest, EmbedResponse from open_notebook.ai.models import model_manager from open_notebook.domain.notebook import Note, Source router = APIRouter() @router.post("/embed", response_model=EmbedResponse) async def embed_content(embed_request: EmbedRequest): """Embed content for vector search.""" try: # Check if embedding model is available if not await model_manager.get_embedding_model(): raise HTTPException( status_code=400, detail="No embedding model configured. Please configure one in the Models section.", ) item_id = embed_request.item_id item_type = embed_request.item_type.lower() # Validate item type if item_type not in ["source", "note"]: raise HTTPException( status_code=400, detail="Item type must be either 'source' or 'note'" ) # Branch based on processing mode if embed_request.async_processing: # ASYNC PATH: Submit command for background processing logger.info(f"Using async processing for {item_type} {item_id}") try: # Import commands to ensure they're registered import commands.embedding_commands # noqa: F401 # Submit type-specific command if item_type == "source": command_name = "embed_source" command_input = {"source_id": item_id} else: # note command_name = "embed_note" command_input = {"note_id": item_id} command_id = await CommandService.submit_command_job( "open_notebook", command_name, command_input, ) logger.info(f"Submitted async {command_name} command: {command_id}") return EmbedResponse( success=True, message="Embedding queued for background processing", item_id=item_id, item_type=item_type, command_id=command_id, ) except Exception as e: logger.error(f"Failed to submit async embedding command: {e}") raise HTTPException( status_code=500, detail=f"Failed to queue embedding: {str(e)}" ) else: # DOMAIN MODEL PATH: Submit job via domain model convenience methods # These methods internally call submit_command() - still fire-and-forget logger.info(f"Using domain model path for {item_type} {item_id}") command_id = None # Get the item and submit embedding job if item_type == "source": source_item = await Source.get(item_id) if not source_item: raise HTTPException(status_code=404, detail="Source not found") # Submit embed_source job (returns command_id for tracking) command_id = await source_item.vectorize() message = "Source embedding job submitted" elif item_type == "note": note_item = await Note.get(item_id) if not note_item: raise HTTPException(status_code=404, detail="Note not found") # Note.save() internally submits embed_note command and returns command_id command_id = await note_item.save() message = "Note embedding job submitted" return EmbedResponse( success=True, message=message, item_id=item_id, item_type=item_type, command_id=command_id, ) except HTTPException: raise except Exception as e: logger.error( f"Error embedding {embed_request.item_type} {embed_request.item_id}: {str(e)}" ) raise HTTPException( status_code=500, detail=f"Error embedding content: {str(e)}" ) ================================================ FILE: api/routers/embedding_rebuild.py ================================================ from fastapi import APIRouter, HTTPException from loguru import logger from surreal_commands import get_command_status from api.command_service import CommandService from api.models import ( RebuildProgress, RebuildRequest, RebuildResponse, RebuildStats, RebuildStatusResponse, ) from open_notebook.database.repository import repo_query router = APIRouter() @router.post("/rebuild", response_model=RebuildResponse) async def start_rebuild(request: RebuildRequest): """ Start a background job to rebuild embeddings. - **mode**: "existing" (re-embed items with embeddings) or "all" (embed everything) - **include_sources**: Include sources in rebuild (default: true) - **include_notes**: Include notes in rebuild (default: true) - **include_insights**: Include insights in rebuild (default: true) Returns command ID to track progress and estimated item count. """ try: logger.info(f"Starting rebuild request: mode={request.mode}") # Import commands to ensure they're registered import commands.embedding_commands # noqa: F401 # Estimate total items (quick count query) # This is a rough estimate before the command runs total_estimate = 0 if request.include_sources: if request.mode == "existing": # Count sources with embeddings result = await repo_query( """ SELECT VALUE count(array::distinct( SELECT VALUE source.id FROM source_embedding WHERE embedding != none AND array::len(embedding) > 0 )) as count FROM {} """ ) else: # Count all sources with content result = await repo_query( "SELECT VALUE count() as count FROM source WHERE full_text != none GROUP ALL" ) if result and isinstance(result[0], dict): total_estimate += result[0].get("count", 0) elif result: total_estimate += result[0] if isinstance(result[0], int) else 0 if request.include_notes: if request.mode == "existing": result = await repo_query( "SELECT VALUE count() as count FROM note WHERE embedding != none AND array::len(embedding) > 0 GROUP ALL" ) else: result = await repo_query( "SELECT VALUE count() as count FROM note WHERE content != none GROUP ALL" ) if result and isinstance(result[0], dict): total_estimate += result[0].get("count", 0) elif result: total_estimate += result[0] if isinstance(result[0], int) else 0 if request.include_insights: if request.mode == "existing": result = await repo_query( "SELECT VALUE count() as count FROM source_insight WHERE embedding != none AND array::len(embedding) > 0 GROUP ALL" ) else: result = await repo_query( "SELECT VALUE count() as count FROM source_insight GROUP ALL" ) if result and isinstance(result[0], dict): total_estimate += result[0].get("count", 0) elif result: total_estimate += result[0] if isinstance(result[0], int) else 0 logger.info(f"Estimated {total_estimate} items to process") # Submit command command_id = await CommandService.submit_command_job( "open_notebook", "rebuild_embeddings", { "mode": request.mode, "include_sources": request.include_sources, "include_notes": request.include_notes, "include_insights": request.include_insights, }, ) logger.info(f"Submitted rebuild command: {command_id}") return RebuildResponse( command_id=command_id, total_items=total_estimate, message=f"Rebuild operation started. Estimated {total_estimate} items to process.", ) except Exception as e: logger.error(f"Failed to start rebuild: {e}") logger.exception(e) raise HTTPException( status_code=500, detail=f"Failed to start rebuild operation: {str(e)}" ) @router.get("/rebuild/{command_id}/status", response_model=RebuildStatusResponse) async def get_rebuild_status(command_id: str): """ Get the status of a rebuild operation. Returns: - **status**: queued, running, completed, failed - **progress**: processed count, total count, percentage - **stats**: breakdown by type (sources, notes, insights, failed) - **timestamps**: started_at, completed_at """ try: # Get command status from surreal_commands status = await get_command_status(command_id) if not status: raise HTTPException(status_code=404, detail="Rebuild command not found") # Build response based on status response = RebuildStatusResponse( command_id=command_id, status=status.status, ) # Extract metadata from command result if status.result and isinstance(status.result, dict): result = status.result # Build progress info if "total_items" in result and "jobs_submitted" in result: total = result["total_items"] submitted = result["jobs_submitted"] response.progress = RebuildProgress( processed=submitted, total=total, percentage=round((submitted / total * 100) if total > 0 else 0, 2), ) # Build stats response.stats = RebuildStats( sources=result.get("sources_submitted", 0), notes=result.get("notes_submitted", 0), insights=result.get("insights_submitted", 0), failed=result.get("failed_submissions", 0), ) # Add timestamps if hasattr(status, "created") and status.created: response.started_at = str(status.created) if hasattr(status, "updated") and status.updated: response.completed_at = str(status.updated) # Add error message if failed if ( status.status == "failed" and status.result and isinstance(status.result, dict) ): response.error_message = status.result.get("error_message", "Unknown error") return response except HTTPException: raise except Exception as e: logger.error(f"Failed to get rebuild status: {e}") logger.exception(e) raise HTTPException( status_code=500, detail=f"Failed to get rebuild status: {str(e)}" ) ================================================ FILE: api/routers/episode_profiles.py ================================================ from typing import List, Optional from fastapi import APIRouter, HTTPException from loguru import logger from pydantic import BaseModel, Field from open_notebook.podcasts.models import EpisodeProfile router = APIRouter() class EpisodeProfileResponse(BaseModel): id: str name: str description: str speaker_config: str outline_llm: Optional[str] = None transcript_llm: Optional[str] = None language: Optional[str] = None default_briefing: str num_segments: int # Legacy fields (for display/migration awareness) outline_provider: Optional[str] = None outline_model: Optional[str] = None transcript_provider: Optional[str] = None transcript_model: Optional[str] = None def _profile_to_response(profile: EpisodeProfile) -> EpisodeProfileResponse: return EpisodeProfileResponse( id=str(profile.id), name=profile.name, description=profile.description or "", speaker_config=profile.speaker_config, outline_llm=profile.outline_llm, transcript_llm=profile.transcript_llm, language=profile.language, default_briefing=profile.default_briefing, num_segments=profile.num_segments, outline_provider=profile.outline_provider, outline_model=profile.outline_model, transcript_provider=profile.transcript_provider, transcript_model=profile.transcript_model, ) @router.get("/episode-profiles", response_model=List[EpisodeProfileResponse]) async def list_episode_profiles(): """List all available episode profiles""" try: profiles = await EpisodeProfile.get_all(order_by="name asc") return [_profile_to_response(p) for p in profiles] except Exception as e: logger.error(f"Failed to fetch episode profiles: {e}") raise HTTPException( status_code=500, detail="Failed to fetch episode profiles" ) @router.get("/episode-profiles/{profile_name}", response_model=EpisodeProfileResponse) async def get_episode_profile(profile_name: str): """Get a specific episode profile by name""" try: profile = await EpisodeProfile.get_by_name(profile_name) if not profile: raise HTTPException( status_code=404, detail=f"Episode profile '{profile_name}' not found" ) return _profile_to_response(profile) except HTTPException: raise except Exception as e: logger.error(f"Failed to fetch episode profile '{profile_name}': {e}") raise HTTPException( status_code=500, detail="Failed to fetch episode profile" ) class EpisodeProfileCreate(BaseModel): name: str = Field(..., description="Unique profile name") description: str = Field("", description="Profile description") speaker_config: str = Field(..., description="Reference to speaker profile name") outline_llm: Optional[str] = Field(None, description="Model record ID for outline") transcript_llm: Optional[str] = Field( None, description="Model record ID for transcript" ) language: Optional[str] = Field(None, description="Podcast language code") default_briefing: str = Field(..., description="Default briefing template") num_segments: int = Field(default=5, description="Number of podcast segments") # Legacy fields (accepted but not required) outline_provider: Optional[str] = None outline_model: Optional[str] = None transcript_provider: Optional[str] = None transcript_model: Optional[str] = None @router.post("/episode-profiles", response_model=EpisodeProfileResponse) async def create_episode_profile(profile_data: EpisodeProfileCreate): """Create a new episode profile""" try: profile = EpisodeProfile( name=profile_data.name, description=profile_data.description, speaker_config=profile_data.speaker_config, outline_llm=profile_data.outline_llm, transcript_llm=profile_data.transcript_llm, language=profile_data.language, default_briefing=profile_data.default_briefing, num_segments=profile_data.num_segments, outline_provider=profile_data.outline_provider, outline_model=profile_data.outline_model, transcript_provider=profile_data.transcript_provider, transcript_model=profile_data.transcript_model, ) await profile.save() return _profile_to_response(profile) except Exception as e: logger.error(f"Failed to create episode profile: {e}") raise HTTPException( status_code=500, detail="Failed to create episode profile" ) @router.put("/episode-profiles/{profile_id}", response_model=EpisodeProfileResponse) async def update_episode_profile(profile_id: str, profile_data: EpisodeProfileCreate): """Update an existing episode profile""" try: profile = await EpisodeProfile.get(profile_id) if not profile: raise HTTPException( status_code=404, detail=f"Episode profile '{profile_id}' not found" ) profile.name = profile_data.name profile.description = profile_data.description profile.speaker_config = profile_data.speaker_config profile.outline_llm = profile_data.outline_llm profile.transcript_llm = profile_data.transcript_llm profile.language = profile_data.language profile.default_briefing = profile_data.default_briefing profile.num_segments = profile_data.num_segments profile.outline_provider = profile_data.outline_provider profile.outline_model = profile_data.outline_model profile.transcript_provider = profile_data.transcript_provider profile.transcript_model = profile_data.transcript_model await profile.save() return _profile_to_response(profile) except HTTPException: raise except Exception as e: logger.error(f"Failed to update episode profile: {e}") raise HTTPException( status_code=500, detail="Failed to update episode profile" ) @router.delete("/episode-profiles/{profile_id}") async def delete_episode_profile(profile_id: str): """Delete an episode profile""" try: profile = await EpisodeProfile.get(profile_id) if not profile: raise HTTPException( status_code=404, detail=f"Episode profile '{profile_id}' not found" ) await profile.delete() return {"message": "Episode profile deleted successfully"} except HTTPException: raise except Exception as e: logger.error(f"Failed to delete episode profile: {e}") raise HTTPException( status_code=500, detail="Failed to delete episode profile" ) @router.post( "/episode-profiles/{profile_id}/duplicate", response_model=EpisodeProfileResponse ) async def duplicate_episode_profile(profile_id: str): """Duplicate an episode profile""" try: original = await EpisodeProfile.get(profile_id) if not original: raise HTTPException( status_code=404, detail=f"Episode profile '{profile_id}' not found" ) duplicate = EpisodeProfile( name=f"{original.name} - Copy", description=original.description, speaker_config=original.speaker_config, outline_llm=original.outline_llm, transcript_llm=original.transcript_llm, language=original.language, default_briefing=original.default_briefing, num_segments=original.num_segments, outline_provider=original.outline_provider, outline_model=original.outline_model, transcript_provider=original.transcript_provider, transcript_model=original.transcript_model, ) await duplicate.save() return _profile_to_response(duplicate) except HTTPException: raise except Exception as e: logger.error(f"Failed to duplicate episode profile: {e}") raise HTTPException( status_code=500, detail="Failed to duplicate episode profile" ) ================================================ FILE: api/routers/insights.py ================================================ from fastapi import APIRouter, HTTPException from loguru import logger from api.models import NoteResponse, SaveAsNoteRequest, SourceInsightResponse from open_notebook.domain.notebook import SourceInsight from open_notebook.exceptions import InvalidInputError router = APIRouter() @router.get("/insights/{insight_id}", response_model=SourceInsightResponse) async def get_insight(insight_id: str): """Get a specific insight by ID.""" try: insight = await SourceInsight.get(insight_id) if not insight: raise HTTPException(status_code=404, detail="Insight not found") # Get source ID from the insight relationship source = await insight.get_source() return SourceInsightResponse( id=insight.id or "", source_id=source.id or "", insight_type=insight.insight_type, content=insight.content, created=str(insight.created), updated=str(insight.updated), ) except HTTPException: raise except Exception as e: logger.error(f"Error fetching insight {insight_id}: {str(e)}") raise HTTPException(status_code=500, detail="Error fetching insight") @router.delete("/insights/{insight_id}") async def delete_insight(insight_id: str): """Delete a specific insight.""" try: insight = await SourceInsight.get(insight_id) if not insight: raise HTTPException(status_code=404, detail="Insight not found") await insight.delete() return {"message": "Insight deleted successfully"} except HTTPException: raise except Exception as e: logger.error(f"Error deleting insight {insight_id}: {str(e)}") raise HTTPException(status_code=500, detail="Error deleting insight") @router.post("/insights/{insight_id}/save-as-note", response_model=NoteResponse) async def save_insight_as_note(insight_id: str, request: SaveAsNoteRequest): """Convert an insight to a note.""" try: insight = await SourceInsight.get(insight_id) if not insight: raise HTTPException(status_code=404, detail="Insight not found") # Use the existing save_as_note method from the domain model note = await insight.save_as_note(request.notebook_id) return NoteResponse( id=note.id or "", title=note.title, content=note.content, note_type=note.note_type, created=str(note.created), updated=str(note.updated), ) except HTTPException: raise except InvalidInputError as e: raise HTTPException(status_code=400, detail=str(e)) except Exception as e: logger.error(f"Error saving insight {insight_id} as note: {str(e)}") raise HTTPException( status_code=500, detail="Error saving insight as note" ) ================================================ FILE: api/routers/languages.py ================================================ from typing import List import pycountry from babel import Locale from babel.core import get_global from fastapi import APIRouter from pydantic import BaseModel router = APIRouter() # Additional regional variants for languages where the distinction matters # (TTS accent, vocabulary, spelling differences) _EXTRA_VARIANTS = [ "pt_PT", "en_GB", "en_AU", "en_IN", "es_MX", "es_AR", "es_CO", "fr_CA", "fr_CH", "zh_TW", "zh_HK", "de_AT", "de_CH", "ar_SA", "nl_BE", ] class LanguageResponse(BaseModel): code: str name: str @router.get("/languages", response_model=List[LanguageResponse]) async def list_languages(): """List available languages as BCP 47 locale codes (e.g. pt-BR, en-US).""" likely_subtags = get_global("likely_subtags") languages = [] seen = set() # 1. For each language, resolve its default locale via CLDR likely subtags for lang in pycountry.languages: if not hasattr(lang, "alpha_2"): continue code = lang.alpha_2 likely = likely_subtags.get(code) if likely: try: loc = Locale.parse(likely) if loc.territory: bcp47 = f"{loc.language}-{loc.territory}" display = loc.get_display_name("en") if bcp47 not in seen: seen.add(bcp47) languages.append(LanguageResponse(code=bcp47, name=display)) continue except Exception: pass # Fallback: bare language code if code not in seen: seen.add(code) languages.append(LanguageResponse(code=code, name=lang.name)) # 2. Add important regional variants for locale_str in _EXTRA_VARIANTS: try: loc = Locale.parse(locale_str) bcp47 = f"{loc.language}-{loc.territory}" if bcp47 not in seen: seen.add(bcp47) display = loc.get_display_name("en") languages.append(LanguageResponse(code=bcp47, name=display)) except Exception: pass languages.sort(key=lambda x: x.name) return languages ================================================ FILE: api/routers/models.py ================================================ import os import traceback from typing import Dict, List, Optional from esperanto import AIFactory from fastapi import APIRouter, HTTPException, Query from loguru import logger from pydantic import BaseModel from api.models import ( DefaultModelsResponse, ModelCreate, ModelResponse, ProviderAvailabilityResponse, ) from open_notebook.domain.credential import Credential from open_notebook.ai.connection_tester import test_individual_model from open_notebook.ai.key_provider import provision_provider_keys from open_notebook.ai.model_discovery import ( discover_provider_models, get_provider_model_count, sync_all_providers, sync_provider_models, ) from open_notebook.ai.models import DefaultModels, Model from open_notebook.exceptions import InvalidInputError router = APIRouter() # ============================================================================= # Model Discovery Response Models # ============================================================================= class DiscoveredModelResponse(BaseModel): """Response model for a discovered model.""" name: str provider: str model_type: str description: Optional[str] = None class ProviderSyncResponse(BaseModel): """Response model for provider sync operation.""" provider: str discovered: int new: int existing: int class AllProvidersSyncResponse(BaseModel): """Response model for syncing all providers.""" results: Dict[str, ProviderSyncResponse] total_discovered: int total_new: int class ProviderModelCountResponse(BaseModel): """Response model for provider model counts.""" provider: str counts: Dict[str, int] total: int class AutoAssignResult(BaseModel): """Response model for auto-assign operation.""" assigned: Dict[str, str] # slot_name -> model_id skipped: List[str] # slots already assigned missing: List[str] # slots with no available models class ModelTestResponse(BaseModel): """Response model for individual model test.""" success: bool message: str details: Optional[str] = None # Provider priority for auto-assignment (higher priority first) PROVIDER_PRIORITY = [ "openai", "anthropic", "google", "mistral", "groq", "deepseek", "xai", "openrouter", "ollama", "azure", "openai_compatible", ] # Model preference patterns (preferred models within each provider) MODEL_PREFERENCES = { "openai": ["gpt-4o", "gpt-4", "gpt-3.5-turbo"], "anthropic": ["claude-3-5-sonnet", "claude-3-opus", "claude-3-sonnet"], "google": ["gemini-2.0", "gemini-1.5-pro", "gemini-pro"], "mistral": ["mistral-large", "mixtral"], "groq": ["llama-3.3", "llama-3.1", "mixtral"], } async def _check_provider_has_credential(provider: str) -> bool: """Check if a provider has any credentials configured in the database.""" try: credentials = await Credential.get_by_provider(provider) return len(credentials) > 0 except Exception: pass return False def _check_azure_support(mode: str) -> bool: """ Check if Azure OpenAI provider is available for a specific mode. Args: mode: One of 'LLM', 'EMBEDDING', 'STT', 'TTS' Returns: bool: True if either generic or mode-specific env vars are set """ # Check generic configuration (applies to all modes) generic = ( os.environ.get("AZURE_OPENAI_API_KEY") is not None and os.environ.get("AZURE_OPENAI_ENDPOINT") is not None and os.environ.get("AZURE_OPENAI_API_VERSION") is not None ) # Check mode-specific configuration (takes precedence) specific = ( os.environ.get(f"AZURE_OPENAI_API_KEY_{mode}") is not None and os.environ.get(f"AZURE_OPENAI_ENDPOINT_{mode}") is not None and os.environ.get(f"AZURE_OPENAI_API_VERSION_{mode}") is not None ) return generic or specific def _check_openai_compatible_support(mode: str) -> bool: """ Check if OpenAI-compatible provider is available for a specific mode. Args: mode: One of 'LLM', 'EMBEDDING', 'STT', 'TTS' Returns: bool: True if either generic or mode-specific env var is set """ generic = os.environ.get("OPENAI_COMPATIBLE_BASE_URL") is not None specific = os.environ.get(f"OPENAI_COMPATIBLE_BASE_URL_{mode}") is not None generic_key = os.environ.get("OPENAI_COMPATIBLE_API_KEY") is not None specific_key = os.environ.get(f"OPENAI_COMPATIBLE_API_KEY_{mode}") is not None return generic or specific or generic_key or specific_key @router.get("/models", response_model=List[ModelResponse]) async def get_models( type: Optional[str] = Query(None, description="Filter by model type"), ): """Get all configured models with optional type filtering.""" try: if type: models = await Model.get_models_by_type(type) else: models = await Model.get_all() return [ ModelResponse( id=model.id, name=model.name, provider=model.provider, type=model.type, credential=model.credential, created=str(model.created), updated=str(model.updated), ) for model in models ] except Exception as e: logger.error(f"Error fetching models: {str(e)}") raise HTTPException(status_code=500, detail=f"Error fetching models: {str(e)}") @router.post("/models", response_model=ModelResponse) async def create_model(model_data: ModelCreate): """Create a new model configuration.""" try: # Validate model type valid_types = ["language", "embedding", "text_to_speech", "speech_to_text"] if model_data.type not in valid_types: raise HTTPException( status_code=400, detail=f"Invalid model type. Must be one of: {valid_types}", ) # Check for duplicate model name under the same provider and type (case-insensitive) from open_notebook.database.repository import repo_query existing = await repo_query( "SELECT * FROM model WHERE string::lowercase(provider) = $provider AND string::lowercase(name) = $name AND string::lowercase(type) = $type LIMIT 1", { "provider": model_data.provider.lower(), "name": model_data.name.lower(), "type": model_data.type.lower(), }, ) if existing: raise HTTPException( status_code=400, detail=f"Model '{model_data.name}' already exists for provider '{model_data.provider}' with type '{model_data.type}'", ) new_model = Model( name=model_data.name, provider=model_data.provider, type=model_data.type, credential=model_data.credential, ) await new_model.save() return ModelResponse( id=new_model.id or "", name=new_model.name, provider=new_model.provider, type=new_model.type, credential=new_model.credential, created=str(new_model.created), updated=str(new_model.updated), ) except HTTPException: raise except InvalidInputError as e: raise HTTPException(status_code=400, detail=str(e)) except Exception as e: logger.error(f"Error creating model: {str(e)}") raise HTTPException(status_code=500, detail=f"Error creating model: {str(e)}") @router.delete("/models/{model_id}") async def delete_model(model_id: str): """Delete a model configuration.""" try: model = await Model.get(model_id) if not model: raise HTTPException(status_code=404, detail="Model not found") await model.delete() return {"message": "Model deleted successfully"} except HTTPException: raise except Exception as e: logger.error(f"Error deleting model {model_id}: {str(e)}") raise HTTPException(status_code=500, detail=f"Error deleting model: {str(e)}") @router.post("/models/{model_id}/test", response_model=ModelTestResponse) async def test_model(model_id: str): """Test if a specific model is correctly configured and functional.""" try: model = await Model.get(model_id) if not model: raise HTTPException(status_code=404, detail="Model not found") except HTTPException: raise except Exception: raise HTTPException(status_code=404, detail="Model not found") try: success, message = await test_individual_model(model) return ModelTestResponse(success=success, message=message) except Exception as e: logger.error(f"Error testing model {model_id}: {traceback.format_exc()}") return ModelTestResponse( success=False, message=str(e)[:200], ) @router.get("/models/defaults", response_model=DefaultModelsResponse) async def get_default_models(): """Get default model assignments.""" try: defaults = await DefaultModels.get_instance() return DefaultModelsResponse( default_chat_model=defaults.default_chat_model, # type: ignore[attr-defined] default_transformation_model=defaults.default_transformation_model, # type: ignore[attr-defined] large_context_model=defaults.large_context_model, # type: ignore[attr-defined] default_text_to_speech_model=defaults.default_text_to_speech_model, # type: ignore[attr-defined] default_speech_to_text_model=defaults.default_speech_to_text_model, # type: ignore[attr-defined] default_embedding_model=defaults.default_embedding_model, # type: ignore[attr-defined] default_tools_model=defaults.default_tools_model, # type: ignore[attr-defined] ) except Exception as e: logger.error(f"Error fetching default models: {str(e)}") raise HTTPException( status_code=500, detail=f"Error fetching default models: {str(e)}" ) @router.put("/models/defaults", response_model=DefaultModelsResponse) async def update_default_models(defaults_data: DefaultModelsResponse): """Update default model assignments.""" try: defaults = await DefaultModels.get_instance() # Update only provided fields if defaults_data.default_chat_model is not None: defaults.default_chat_model = defaults_data.default_chat_model # type: ignore[attr-defined] if defaults_data.default_transformation_model is not None: defaults.default_transformation_model = ( defaults_data.default_transformation_model ) # type: ignore[attr-defined] if defaults_data.large_context_model is not None: defaults.large_context_model = defaults_data.large_context_model # type: ignore[attr-defined] if defaults_data.default_text_to_speech_model is not None: defaults.default_text_to_speech_model = ( defaults_data.default_text_to_speech_model ) # type: ignore[attr-defined] if defaults_data.default_speech_to_text_model is not None: defaults.default_speech_to_text_model = ( defaults_data.default_speech_to_text_model ) # type: ignore[attr-defined] if defaults_data.default_embedding_model is not None: defaults.default_embedding_model = defaults_data.default_embedding_model # type: ignore[attr-defined] if defaults_data.default_tools_model is not None: defaults.default_tools_model = defaults_data.default_tools_model # type: ignore[attr-defined] await defaults.update() # No cache refresh needed - next access will fetch fresh data from DB return DefaultModelsResponse( default_chat_model=defaults.default_chat_model, # type: ignore[attr-defined] default_transformation_model=defaults.default_transformation_model, # type: ignore[attr-defined] large_context_model=defaults.large_context_model, # type: ignore[attr-defined] default_text_to_speech_model=defaults.default_text_to_speech_model, # type: ignore[attr-defined] default_speech_to_text_model=defaults.default_speech_to_text_model, # type: ignore[attr-defined] default_embedding_model=defaults.default_embedding_model, # type: ignore[attr-defined] default_tools_model=defaults.default_tools_model, # type: ignore[attr-defined] ) except HTTPException: raise except Exception as e: logger.error(f"Error updating default models: {str(e)}") raise HTTPException( status_code=500, detail=f"Error updating default models: {str(e)}" ) @router.get("/models/providers", response_model=ProviderAvailabilityResponse) async def get_provider_availability(): """Get provider availability based on database config and environment variables.""" try: # Check which providers have credentials in the database or env vars # For each provider, check DB credentials first, then env vars as fallback # Simple env var mapping for backward compatibility env_var_map = { "openai": "OPENAI_API_KEY", "anthropic": "ANTHROPIC_API_KEY", "google": "GOOGLE_API_KEY", "groq": "GROQ_API_KEY", "mistral": "MISTRAL_API_KEY", "deepseek": "DEEPSEEK_API_KEY", "xai": "XAI_API_KEY", "openrouter": "OPENROUTER_API_KEY", "voyage": "VOYAGE_API_KEY", "elevenlabs": "ELEVENLABS_API_KEY", "ollama": "OLLAMA_API_BASE", } provider_status = {} # Check simple providers: credential in DB or env var for provider, env_var in env_var_map.items(): has_cred = await _check_provider_has_credential(provider) has_env = os.environ.get(env_var) is not None provider_status[provider] = has_cred or has_env # Google also supports GEMINI_API_KEY if not provider_status.get("google"): provider_status["google"] = os.environ.get("GEMINI_API_KEY") is not None # Vertex: DB credential or env vars provider_status["vertex"] = ( await _check_provider_has_credential("vertex") or os.environ.get("VERTEX_PROJECT") is not None ) # Azure: DB credential or env vars provider_status["azure"] = ( await _check_provider_has_credential("azure") or _check_azure_support("LLM") or _check_azure_support("EMBEDDING") or _check_azure_support("STT") or _check_azure_support("TTS") ) # OpenAI-compatible: DB credential or env vars provider_status["openai-compatible"] = ( await _check_provider_has_credential("openai_compatible") or _check_openai_compatible_support("LLM") or _check_openai_compatible_support("EMBEDDING") or _check_openai_compatible_support("STT") or _check_openai_compatible_support("TTS") ) available_providers = [k for k, v in provider_status.items() if v] unavailable_providers = [k for k, v in provider_status.items() if not v] # Get supported model types from Esperanto esperanto_available = AIFactory.get_available_providers() # Build supported types mapping only for available providers supported_types: dict[str, list[str]] = {} for provider in available_providers: supported_types[provider] = [] # Map Esperanto model types to our environment variable modes mode_mapping = { "language": "LLM", "embedding": "EMBEDDING", "speech_to_text": "STT", "text_to_speech": "TTS", } # Special handling for openai-compatible to check mode-specific availability if provider == "openai-compatible": has_db_cred = await _check_provider_has_credential("openai_compatible") for model_type, mode in mode_mapping.items(): if ( model_type in esperanto_available and provider in esperanto_available[model_type] ): if has_db_cred or _check_openai_compatible_support(mode): supported_types[provider].append(model_type) # Special handling for azure to check mode-specific availability elif provider == "azure": has_db_cred = await _check_provider_has_credential("azure") for model_type, mode in mode_mapping.items(): if ( model_type in esperanto_available and provider in esperanto_available[model_type] ): if has_db_cred or _check_azure_support(mode): supported_types[provider].append(model_type) else: # Standard provider detection for model_type, providers in esperanto_available.items(): if provider in providers: supported_types[provider].append(model_type) return ProviderAvailabilityResponse( available=available_providers, unavailable=unavailable_providers, supported_types=supported_types, ) except Exception as e: logger.error(f"Error checking provider availability: {str(e)}") raise HTTPException( status_code=500, detail=f"Error checking provider availability: {str(e)}" ) # ============================================================================= # Model Discovery Endpoints # ============================================================================= @router.get( "/models/discover/{provider}", response_model=List[DiscoveredModelResponse] ) async def discover_models(provider: str): """ Discover available models from a provider without registering them. This endpoint queries the provider's API to list available models but does not save them to the database. Use the sync endpoint to both discover and register models. """ try: # Provision DB-stored credentials into env vars before discovery await provision_provider_keys(provider) discovered = await discover_provider_models(provider) return [ DiscoveredModelResponse( name=m.name, provider=m.provider, model_type=m.model_type, description=m.description, ) for m in discovered ] except Exception as e: logger.error(f"Error discovering models for {provider}: {str(e)}") raise HTTPException( status_code=500, detail="Error discovering models. Check server logs for details." ) @router.post("/models/sync/{provider}", response_model=ProviderSyncResponse) async def sync_models(provider: str): """ Sync models for a specific provider. Discovers available models from the provider's API and registers any new models in the database. Existing models are skipped. Returns counts of discovered, new, and existing models. """ try: # Provision DB-stored credentials into env vars before discovery await provision_provider_keys(provider) discovered, new, existing = await sync_provider_models( provider, auto_register=True ) return ProviderSyncResponse( provider=provider, discovered=discovered, new=new, existing=existing, ) except Exception as e: logger.error(f"Error syncing models for {provider}: {str(e)}") raise HTTPException(status_code=500, detail="Error syncing models. Check server logs for details.") @router.post("/models/sync", response_model=AllProvidersSyncResponse) async def sync_all_models(): """ Sync models for all configured providers. Discovers and registers models from all providers that have valid API keys configured. This is useful for initial setup or periodic refresh of available models. """ try: results = await sync_all_providers() response_results = {} total_discovered = 0 total_new = 0 for provider, (discovered, new, existing) in results.items(): response_results[provider] = ProviderSyncResponse( provider=provider, discovered=discovered, new=new, existing=existing, ) total_discovered += discovered total_new += new return AllProvidersSyncResponse( results=response_results, total_discovered=total_discovered, total_new=total_new, ) except Exception as e: logger.error(f"Error syncing all models: {str(e)}") raise HTTPException( status_code=500, detail=f"Error syncing all models: {str(e)}" ) @router.get("/models/count/{provider}", response_model=ProviderModelCountResponse) async def get_model_count(provider: str): """ Get count of registered models for a provider, grouped by type. Returns counts for each model type (language, embedding, speech_to_text, text_to_speech) as well as total count. """ try: counts = await get_provider_model_count(provider) total = sum(counts.values()) return ProviderModelCountResponse( provider=provider, counts=counts, total=total, ) except Exception as e: logger.error(f"Error getting model count for {provider}: {str(e)}") raise HTTPException( status_code=500, detail=f"Error getting model count: {str(e)}" ) @router.get("/models/by-provider/{provider}", response_model=List[ModelResponse]) async def get_models_by_provider(provider: str): """ Get all registered models for a specific provider. Returns models from the database that belong to the specified provider. """ try: from open_notebook.database.repository import repo_query models = await repo_query( "SELECT * FROM model WHERE provider = $provider ORDER BY type, name", {"provider": provider}, ) return [ ModelResponse( id=model.get("id", ""), name=model.get("name", ""), provider=model.get("provider", ""), type=model.get("type", ""), credential=model.get("credential"), created=str(model.get("created", "")), updated=str(model.get("updated", "")), ) for model in models ] except Exception as e: logger.error(f"Error fetching models for {provider}: {str(e)}") raise HTTPException( status_code=500, detail=f"Error fetching models: {str(e)}" ) def _get_preferred_model( models: List[Dict], provider_priority: List[str], model_preferences: Dict ) -> Optional[Dict]: """ Select the best model from a list based on provider priority and model preferences. Args: models: List of model dictionaries with 'provider', 'name', 'id' keys provider_priority: List of providers in preference order model_preferences: Dict mapping provider to list of preferred model name patterns Returns: The best model dict, or None if no models available """ if not models: return None # Group models by provider by_provider: Dict[str, List[Dict]] = {} for model in models: provider = model.get("provider", "") if provider not in by_provider: by_provider[provider] = [] by_provider[provider].append(model) # Find first provider with models (in priority order) for provider in provider_priority: if provider in by_provider: provider_models = by_provider[provider] # Check for preferred models within this provider if provider in model_preferences: for preference in model_preferences[provider]: for model in provider_models: if preference.lower() in model.get("name", "").lower(): return model # Fall back to first model from this provider return provider_models[0] # Fall back to first model from any provider return models[0] if models else None @router.post("/models/auto-assign", response_model=AutoAssignResult) async def auto_assign_defaults(): """ Auto-assign default models based on available models. This endpoint intelligently assigns the first available model of each required type to the corresponding default slot. It uses provider priority (preferring premium providers like OpenAI, Anthropic) and model preferences within each provider. Returns: - assigned: Dict of slot names to assigned model IDs - skipped: List of slots that already have models assigned - missing: List of slots with no available models """ try: from open_notebook.database.repository import repo_query # Get current defaults defaults = await DefaultModels.get_instance() # Get all models grouped by type all_models = await repo_query( "SELECT * FROM model ORDER BY provider, name", {}, ) # Group models by type models_by_type: Dict[str, List[Dict]] = { "language": [], "embedding": [], "text_to_speech": [], "speech_to_text": [], } for model in all_models: model_type = model.get("type", "") if model_type in models_by_type: models_by_type[model_type].append(model) # Define slot configuration: (slot_name, model_type, current_value) slot_configs = [ ("default_chat_model", "language", defaults.default_chat_model), # type: ignore[attr-defined] ("default_transformation_model", "language", defaults.default_transformation_model), # type: ignore[attr-defined] ("default_tools_model", "language", defaults.default_tools_model), # type: ignore[attr-defined] ("large_context_model", "language", defaults.large_context_model), # type: ignore[attr-defined] ("default_embedding_model", "embedding", defaults.default_embedding_model), # type: ignore[attr-defined] ("default_text_to_speech_model", "text_to_speech", defaults.default_text_to_speech_model), # type: ignore[attr-defined] ("default_speech_to_text_model", "speech_to_text", defaults.default_speech_to_text_model), # type: ignore[attr-defined] ] assigned: Dict[str, str] = {} skipped: List[str] = [] missing: List[str] = [] for slot_name, model_type, current_value in slot_configs: if current_value: # Slot already has a value skipped.append(slot_name) continue available_models = models_by_type.get(model_type, []) if not available_models: # No models of this type available missing.append(slot_name) continue # Select best model for this slot best_model = _get_preferred_model( available_models, PROVIDER_PRIORITY, MODEL_PREFERENCES ) if best_model: model_id = best_model.get("id", "") assigned[slot_name] = model_id # Update the defaults object setattr(defaults, slot_name, model_id) # Save updated defaults if any assignments were made if assigned: await defaults.update() return AutoAssignResult( assigned=assigned, skipped=skipped, missing=missing, ) except Exception as e: logger.error(f"Error auto-assigning defaults: {str(e)}") raise HTTPException( status_code=500, detail=f"Error auto-assigning defaults: {str(e)}" ) ================================================ FILE: api/routers/notebooks.py ================================================ from typing import List, Optional from fastapi import APIRouter, HTTPException, Query from loguru import logger from api.models import ( NotebookCreate, NotebookDeletePreview, NotebookDeleteResponse, NotebookResponse, NotebookUpdate, ) from open_notebook.database.repository import ensure_record_id, repo_query from open_notebook.domain.notebook import Notebook, Source from open_notebook.exceptions import InvalidInputError router = APIRouter() @router.get("/notebooks", response_model=List[NotebookResponse]) async def get_notebooks( archived: Optional[bool] = Query(None, description="Filter by archived status"), order_by: str = Query("updated desc", description="Order by field and direction"), ): """Get all notebooks with optional filtering and ordering.""" try: # Build the query with counts query = f""" SELECT *, count(<-reference.in) as source_count, count(<-artifact.in) as note_count FROM notebook ORDER BY {order_by} """ result = await repo_query(query) # Filter by archived status if specified if archived is not None: result = [nb for nb in result if nb.get("archived") == archived] return [ NotebookResponse( id=str(nb.get("id", "")), name=nb.get("name", ""), description=nb.get("description", ""), archived=nb.get("archived", False), created=str(nb.get("created", "")), updated=str(nb.get("updated", "")), source_count=nb.get("source_count", 0), note_count=nb.get("note_count", 0), ) for nb in result ] except Exception as e: logger.error(f"Error fetching notebooks: {str(e)}") raise HTTPException( status_code=500, detail=f"Error fetching notebooks: {str(e)}" ) @router.post("/notebooks", response_model=NotebookResponse) async def create_notebook(notebook: NotebookCreate): """Create a new notebook.""" try: new_notebook = Notebook( name=notebook.name, description=notebook.description, ) await new_notebook.save() return NotebookResponse( id=new_notebook.id or "", name=new_notebook.name, description=new_notebook.description, archived=new_notebook.archived or False, created=str(new_notebook.created), updated=str(new_notebook.updated), source_count=0, # New notebook has no sources note_count=0, # New notebook has no notes ) except InvalidInputError as e: raise HTTPException(status_code=400, detail=str(e)) except Exception as e: logger.error(f"Error creating notebook: {str(e)}") raise HTTPException( status_code=500, detail=f"Error creating notebook: {str(e)}" ) @router.get( "/notebooks/{notebook_id}/delete-preview", response_model=NotebookDeletePreview ) async def get_notebook_delete_preview(notebook_id: str): """Get a preview of what will be deleted when this notebook is deleted.""" try: notebook = await Notebook.get(notebook_id) if not notebook: raise HTTPException(status_code=404, detail="Notebook not found") preview = await notebook.get_delete_preview() return NotebookDeletePreview( notebook_id=str(notebook.id), notebook_name=notebook.name, note_count=preview["note_count"], exclusive_source_count=preview["exclusive_source_count"], shared_source_count=preview["shared_source_count"], ) except HTTPException: raise except Exception as e: logger.error(f"Error getting delete preview for notebook {notebook_id}: {e}") raise HTTPException( status_code=500, detail=f"Error fetching notebook deletion preview: {str(e)}", ) @router.get("/notebooks/{notebook_id}", response_model=NotebookResponse) async def get_notebook(notebook_id: str): """Get a specific notebook by ID.""" try: # Query with counts for single notebook query = """ SELECT *, count(<-reference.in) as source_count, count(<-artifact.in) as note_count FROM $notebook_id """ result = await repo_query(query, {"notebook_id": ensure_record_id(notebook_id)}) if not result: raise HTTPException(status_code=404, detail="Notebook not found") nb = result[0] return NotebookResponse( id=str(nb.get("id", "")), name=nb.get("name", ""), description=nb.get("description", ""), archived=nb.get("archived", False), created=str(nb.get("created", "")), updated=str(nb.get("updated", "")), source_count=nb.get("source_count", 0), note_count=nb.get("note_count", 0), ) except HTTPException: raise except Exception as e: logger.error(f"Error fetching notebook {notebook_id}: {str(e)}") raise HTTPException( status_code=500, detail=f"Error fetching notebook: {str(e)}" ) @router.put("/notebooks/{notebook_id}", response_model=NotebookResponse) async def update_notebook(notebook_id: str, notebook_update: NotebookUpdate): """Update a notebook.""" try: notebook = await Notebook.get(notebook_id) if not notebook: raise HTTPException(status_code=404, detail="Notebook not found") # Update only provided fields if notebook_update.name is not None: notebook.name = notebook_update.name if notebook_update.description is not None: notebook.description = notebook_update.description if notebook_update.archived is not None: notebook.archived = notebook_update.archived await notebook.save() # Query with counts after update query = """ SELECT *, count(<-reference.in) as source_count, count(<-artifact.in) as note_count FROM $notebook_id """ result = await repo_query(query, {"notebook_id": ensure_record_id(notebook_id)}) if result: nb = result[0] return NotebookResponse( id=str(nb.get("id", "")), name=nb.get("name", ""), description=nb.get("description", ""), archived=nb.get("archived", False), created=str(nb.get("created", "")), updated=str(nb.get("updated", "")), source_count=nb.get("source_count", 0), note_count=nb.get("note_count", 0), ) # Fallback if query fails return NotebookResponse( id=notebook.id or "", name=notebook.name, description=notebook.description, archived=notebook.archived or False, created=str(notebook.created), updated=str(notebook.updated), source_count=0, note_count=0, ) except HTTPException: raise except InvalidInputError as e: raise HTTPException(status_code=400, detail=str(e)) except Exception as e: logger.error(f"Error updating notebook {notebook_id}: {str(e)}") raise HTTPException( status_code=500, detail=f"Error updating notebook: {str(e)}" ) @router.post("/notebooks/{notebook_id}/sources/{source_id}") async def add_source_to_notebook(notebook_id: str, source_id: str): """Add an existing source to a notebook (create the reference).""" try: # Check if notebook exists notebook = await Notebook.get(notebook_id) if not notebook: raise HTTPException(status_code=404, detail="Notebook not found") # Check if source exists source = await Source.get(source_id) if not source: raise HTTPException(status_code=404, detail="Source not found") # Check if reference already exists (idempotency) existing_ref = await repo_query( "SELECT * FROM reference WHERE out = $source_id AND in = $notebook_id", { "notebook_id": ensure_record_id(notebook_id), "source_id": ensure_record_id(source_id), }, ) # If reference doesn't exist, create it if not existing_ref: await repo_query( "RELATE $source_id->reference->$notebook_id", { "notebook_id": ensure_record_id(notebook_id), "source_id": ensure_record_id(source_id), }, ) return {"message": "Source linked to notebook successfully"} except HTTPException: raise except Exception as e: logger.error( f"Error linking source {source_id} to notebook {notebook_id}: {str(e)}" ) raise HTTPException( status_code=500, detail=f"Error linking source to notebook: {str(e)}" ) @router.delete("/notebooks/{notebook_id}/sources/{source_id}") async def remove_source_from_notebook(notebook_id: str, source_id: str): """Remove a source from a notebook (delete the reference).""" try: # Check if notebook exists notebook = await Notebook.get(notebook_id) if not notebook: raise HTTPException(status_code=404, detail="Notebook not found") # Delete the reference record linking source to notebook await repo_query( "DELETE FROM reference WHERE out = $notebook_id AND in = $source_id", { "notebook_id": ensure_record_id(notebook_id), "source_id": ensure_record_id(source_id), }, ) return {"message": "Source removed from notebook successfully"} except HTTPException: raise except Exception as e: logger.error( f"Error removing source {source_id} from notebook {notebook_id}: {str(e)}" ) raise HTTPException( status_code=500, detail=f"Error removing source from notebook: {str(e)}" ) @router.delete("/notebooks/{notebook_id}", response_model=NotebookDeleteResponse) async def delete_notebook( notebook_id: str, delete_exclusive_sources: bool = Query( False, description="Whether to delete sources that belong only to this notebook", ), ): """ Delete a notebook with cascade deletion. Always deletes all notes associated with the notebook. If delete_exclusive_sources is True, also deletes sources that belong only to this notebook (not linked to any other notebooks). """ try: notebook = await Notebook.get(notebook_id) if not notebook: raise HTTPException(status_code=404, detail="Notebook not found") result = await notebook.delete(delete_exclusive_sources=delete_exclusive_sources) return NotebookDeleteResponse( message="Notebook deleted successfully", deleted_notes=result["deleted_notes"], deleted_sources=result["deleted_sources"], unlinked_sources=result["unlinked_sources"], ) except HTTPException: raise except Exception as e: logger.error(f"Error deleting notebook {notebook_id}: {str(e)}") raise HTTPException( status_code=500, detail=f"Error deleting notebook: {str(e)}" ) ================================================ FILE: api/routers/notes.py ================================================ from typing import List, Literal, Optional from fastapi import APIRouter, HTTPException, Query from loguru import logger from api.models import NoteCreate, NoteResponse, NoteUpdate from open_notebook.domain.notebook import Note from open_notebook.exceptions import InvalidInputError router = APIRouter() @router.get("/notes", response_model=List[NoteResponse]) async def get_notes( notebook_id: Optional[str] = Query(None, description="Filter by notebook ID"), ): """Get all notes with optional notebook filtering.""" try: if notebook_id: # Get notes for a specific notebook from open_notebook.domain.notebook import Notebook notebook = await Notebook.get(notebook_id) if not notebook: raise HTTPException(status_code=404, detail="Notebook not found") notes = await notebook.get_notes() else: # Get all notes notes = await Note.get_all(order_by="updated desc") return [ NoteResponse( id=note.id or "", title=note.title, content=note.content, note_type=note.note_type, created=str(note.created), updated=str(note.updated), ) for note in notes ] except HTTPException: raise except Exception as e: logger.error(f"Error fetching notes: {str(e)}") raise HTTPException(status_code=500, detail=f"Error fetching notes: {str(e)}") @router.post("/notes", response_model=NoteResponse) async def create_note(note_data: NoteCreate): """Create a new note.""" try: # Auto-generate title if not provided and it's an AI note title = note_data.title if not title and note_data.note_type == "ai" and note_data.content: from open_notebook.graphs.prompt import graph as prompt_graph prompt = "Based on the Note below, please provide a Title for this content, with max 15 words" result = await prompt_graph.ainvoke( { # type: ignore[arg-type] "input_text": note_data.content, "prompt": prompt, } ) title = result.get("output", "Untitled Note") # Validate note_type note_type: Optional[Literal["human", "ai"]] = None if note_data.note_type in ("human", "ai"): note_type = note_data.note_type # type: ignore[assignment] elif note_data.note_type is not None: raise HTTPException( status_code=400, detail="note_type must be 'human' or 'ai'" ) new_note = Note( title=title, content=note_data.content, note_type=note_type, ) command_id = await new_note.save() # Add to notebook if specified if note_data.notebook_id: from open_notebook.domain.notebook import Notebook notebook = await Notebook.get(note_data.notebook_id) if not notebook: raise HTTPException(status_code=404, detail="Notebook not found") await new_note.add_to_notebook(note_data.notebook_id) return NoteResponse( id=new_note.id or "", title=new_note.title, content=new_note.content, note_type=new_note.note_type, created=str(new_note.created), updated=str(new_note.updated), command_id=str(command_id) if command_id else None, ) except HTTPException: raise except InvalidInputError as e: raise HTTPException(status_code=400, detail=str(e)) except Exception as e: logger.error(f"Error creating note: {str(e)}") raise HTTPException(status_code=500, detail=f"Error creating note: {str(e)}") @router.get("/notes/{note_id}", response_model=NoteResponse) async def get_note(note_id: str): """Get a specific note by ID.""" try: note = await Note.get(note_id) if not note: raise HTTPException(status_code=404, detail="Note not found") return NoteResponse( id=note.id or "", title=note.title, content=note.content, note_type=note.note_type, created=str(note.created), updated=str(note.updated), ) except HTTPException: raise except Exception as e: logger.error(f"Error fetching note {note_id}: {str(e)}") raise HTTPException(status_code=500, detail=f"Error fetching note: {str(e)}") @router.put("/notes/{note_id}", response_model=NoteResponse) async def update_note(note_id: str, note_update: NoteUpdate): """Update a note.""" try: note = await Note.get(note_id) if not note: raise HTTPException(status_code=404, detail="Note not found") # Update only provided fields if note_update.title is not None: note.title = note_update.title if note_update.content is not None: note.content = note_update.content if note_update.note_type is not None: if note_update.note_type in ("human", "ai"): note.note_type = note_update.note_type # type: ignore[assignment] else: raise HTTPException( status_code=400, detail="note_type must be 'human' or 'ai'" ) command_id = await note.save() return NoteResponse( id=note.id or "", title=note.title, content=note.content, note_type=note.note_type, created=str(note.created), updated=str(note.updated), command_id=str(command_id) if command_id else None, ) except HTTPException: raise except InvalidInputError as e: raise HTTPException(status_code=400, detail=str(e)) except Exception as e: logger.error(f"Error updating note {note_id}: {str(e)}") raise HTTPException(status_code=500, detail=f"Error updating note: {str(e)}") @router.delete("/notes/{note_id}") async def delete_note(note_id: str): """Delete a note.""" try: note = await Note.get(note_id) if not note: raise HTTPException(status_code=404, detail="Note not found") await note.delete() return {"message": "Note deleted successfully"} except HTTPException: raise except Exception as e: logger.error(f"Error deleting note {note_id}: {str(e)}") raise HTTPException(status_code=500, detail=f"Error deleting note: {str(e)}") ================================================ FILE: api/routers/podcasts.py ================================================ from pathlib import Path from typing import List, Optional from urllib.parse import unquote, urlparse from fastapi import APIRouter, HTTPException from fastapi.responses import FileResponse from loguru import logger from pydantic import BaseModel from api.podcast_service import ( PodcastGenerationRequest, PodcastGenerationResponse, PodcastService, ) router = APIRouter() class PodcastEpisodeResponse(BaseModel): id: str name: str episode_profile: dict speaker_profile: dict briefing: str audio_file: Optional[str] = None audio_url: Optional[str] = None transcript: Optional[dict] = None outline: Optional[dict] = None created: Optional[str] = None job_status: Optional[str] = None error_message: Optional[str] = None def _resolve_audio_path(audio_file: str) -> Path: if audio_file.startswith("file://"): parsed = urlparse(audio_file) return Path(unquote(parsed.path)) return Path(audio_file) @router.post("/podcasts/generate", response_model=PodcastGenerationResponse) async def generate_podcast(request: PodcastGenerationRequest): """ Generate a podcast episode using Episode Profiles. Returns immediately with job ID for status tracking. """ try: job_id = await PodcastService.submit_generation_job( episode_profile_name=request.episode_profile, speaker_profile_name=request.speaker_profile, episode_name=request.episode_name, notebook_id=request.notebook_id, content=request.content, briefing_suffix=request.briefing_suffix, ) return PodcastGenerationResponse( job_id=job_id, status="submitted", message=f"Podcast generation started for episode '{request.episode_name}'", episode_profile=request.episode_profile, episode_name=request.episode_name, ) except Exception as e: logger.error(f"Error generating podcast: {str(e)}") raise HTTPException( status_code=500, detail="Failed to generate podcast" ) @router.get("/podcasts/jobs/{job_id}") async def get_podcast_job_status(job_id: str): """Get the status of a podcast generation job""" try: status_data = await PodcastService.get_job_status(job_id) return status_data except Exception as e: logger.error(f"Error fetching podcast job status: {str(e)}") raise HTTPException( status_code=500, detail="Failed to fetch job status" ) @router.get("/podcasts/episodes", response_model=List[PodcastEpisodeResponse]) async def list_podcast_episodes(): """List all podcast episodes""" try: episodes = await PodcastService.list_episodes() response_episodes = [] for episode in episodes: # Skip incomplete episodes without command or audio if not episode.command and not episode.audio_file: continue # Get job status and error message if available job_status = None error_message = None if episode.command: try: detail = await episode.get_job_detail() job_status = detail["status"] error_message = detail["error_message"] except Exception: job_status = "unknown" else: # No command but has audio file = completed import job_status = "completed" audio_url = None if episode.audio_file: audio_path = _resolve_audio_path(episode.audio_file) if audio_path.exists(): audio_url = f"/api/podcasts/episodes/{episode.id}/audio" response_episodes.append( PodcastEpisodeResponse( id=str(episode.id), name=episode.name, episode_profile=episode.episode_profile, speaker_profile=episode.speaker_profile, briefing=episode.briefing, audio_file=episode.audio_file, audio_url=audio_url, transcript=episode.transcript, outline=episode.outline, created=str(episode.created) if episode.created else None, job_status=job_status, error_message=error_message, ) ) return response_episodes except Exception as e: logger.error(f"Error listing podcast episodes: {str(e)}") raise HTTPException( status_code=500, detail="Failed to list podcast episodes" ) @router.get("/podcasts/episodes/{episode_id}", response_model=PodcastEpisodeResponse) async def get_podcast_episode(episode_id: str): """Get a specific podcast episode""" try: episode = await PodcastService.get_episode(episode_id) # Get job status and error message if available job_status = None error_message = None if episode.command: try: detail = await episode.get_job_detail() job_status = detail["status"] error_message = detail["error_message"] except Exception: job_status = "unknown" else: # No command but has audio file = completed import job_status = "completed" if episode.audio_file else "unknown" audio_url = None if episode.audio_file: audio_path = _resolve_audio_path(episode.audio_file) if audio_path.exists(): audio_url = f"/api/podcasts/episodes/{episode.id}/audio" return PodcastEpisodeResponse( id=str(episode.id), name=episode.name, episode_profile=episode.episode_profile, speaker_profile=episode.speaker_profile, briefing=episode.briefing, audio_file=episode.audio_file, audio_url=audio_url, transcript=episode.transcript, outline=episode.outline, created=str(episode.created) if episode.created else None, job_status=job_status, error_message=error_message, ) except Exception as e: logger.error(f"Error fetching podcast episode: {str(e)}") raise HTTPException(status_code=404, detail="Episode not found") @router.get("/podcasts/episodes/{episode_id}/audio") async def stream_podcast_episode_audio(episode_id: str): """Stream the audio file associated with a podcast episode""" try: episode = await PodcastService.get_episode(episode_id) except HTTPException: raise except Exception as e: logger.error(f"Error fetching podcast episode for audio: {str(e)}") raise HTTPException(status_code=404, detail="Episode not found") if not episode.audio_file: raise HTTPException(status_code=404, detail="Episode has no audio file") audio_path = _resolve_audio_path(episode.audio_file) if not audio_path.exists(): raise HTTPException(status_code=404, detail="Audio file not found on disk") return FileResponse( audio_path, media_type="audio/mpeg", filename=audio_path.name, ) @router.post("/podcasts/episodes/{episode_id}/retry") async def retry_podcast_episode(episode_id: str): """Retry a failed podcast episode by deleting it and submitting a new job""" try: episode = await PodcastService.get_episode(episode_id) # Validate episode is in a failed state detail = await episode.get_job_detail() if detail["status"] not in ("failed", "error"): raise HTTPException( status_code=400, detail=f"Episode is not in a failed state (current: {detail['status']})", ) # Extract params for re-submission ep_profile_name = episode.episode_profile.get("name") sp_profile_name = episode.speaker_profile.get("name") episode_name = episode.name content = episode.content if not ep_profile_name or not sp_profile_name: raise HTTPException( status_code=400, detail="Cannot retry: episode or speaker profile name missing from stored data", ) # Delete audio file if any if episode.audio_file: audio_path = _resolve_audio_path(episode.audio_file) if audio_path.exists(): try: audio_path.unlink() except Exception as e: logger.warning(f"Failed to delete audio file {audio_path}: {e}") # Delete the failed episode await episode.delete() # Submit a new job job_id = await PodcastService.submit_generation_job( episode_profile_name=ep_profile_name, speaker_profile_name=sp_profile_name, episode_name=episode_name, content=content, ) return {"job_id": job_id, "message": "Retry submitted successfully"} except HTTPException: raise except Exception as e: logger.error(f"Error retrying podcast episode: {str(e)}") raise HTTPException( status_code=500, detail="Failed to retry episode" ) @router.delete("/podcasts/episodes/{episode_id}") async def delete_podcast_episode(episode_id: str): """Delete a podcast episode and its associated audio file""" try: # Get the episode first to check if it exists and get the audio file path episode = await PodcastService.get_episode(episode_id) # Delete the physical audio file if it exists if episode.audio_file: audio_path = _resolve_audio_path(episode.audio_file) if audio_path.exists(): try: audio_path.unlink() logger.info(f"Deleted audio file: {audio_path}") except Exception as e: logger.warning(f"Failed to delete audio file {audio_path}: {e}") # Delete the episode from the database await episode.delete() logger.info(f"Deleted podcast episode: {episode_id}") return {"message": "Episode deleted successfully", "episode_id": episode_id} except Exception as e: logger.error(f"Error deleting podcast episode: {str(e)}") raise HTTPException( status_code=500, detail="Failed to delete episode" ) ================================================ FILE: api/routers/search.py ================================================ import json from typing import AsyncGenerator from fastapi import APIRouter, HTTPException from fastapi.responses import StreamingResponse from loguru import logger from api.models import AskRequest, AskResponse, SearchRequest, SearchResponse from open_notebook.ai.models import Model, model_manager from open_notebook.domain.notebook import text_search, vector_search from open_notebook.exceptions import DatabaseOperationError, InvalidInputError from open_notebook.graphs.ask import graph as ask_graph router = APIRouter() @router.post("/search", response_model=SearchResponse) async def search_knowledge_base(search_request: SearchRequest): """Search the knowledge base using text or vector search.""" try: if search_request.type == "vector": # Check if embedding model is available for vector search if not await model_manager.get_embedding_model(): raise HTTPException( status_code=400, detail="Vector search requires an embedding model. Please configure one in the Models section.", ) results = await vector_search( keyword=search_request.query, results=search_request.limit, source=search_request.search_sources, note=search_request.search_notes, minimum_score=search_request.minimum_score, ) else: # Text search results = await text_search( keyword=search_request.query, results=search_request.limit, source=search_request.search_sources, note=search_request.search_notes, ) return SearchResponse( results=results or [], total_count=len(results) if results else 0, search_type=search_request.type, ) except InvalidInputError as e: raise HTTPException(status_code=400, detail=str(e)) except DatabaseOperationError as e: logger.error(f"Database error during search: {str(e)}") raise HTTPException(status_code=500, detail=f"Search failed: {str(e)}") except Exception as e: logger.error(f"Unexpected error during search: {str(e)}") raise HTTPException(status_code=500, detail=f"Search failed: {str(e)}") async def stream_ask_response( question: str, strategy_model: Model, answer_model: Model, final_answer_model: Model ) -> AsyncGenerator[str, None]: """Stream the ask response as Server-Sent Events.""" try: final_answer = None async for chunk in ask_graph.astream( input=dict(question=question), # type: ignore[arg-type] config=dict( configurable=dict( strategy_model=strategy_model.id, answer_model=answer_model.id, final_answer_model=final_answer_model.id, ) ), stream_mode="updates", ): if "agent" in chunk: strategy_data = { "type": "strategy", "reasoning": chunk["agent"]["strategy"].reasoning, "searches": [ {"term": search.term, "instructions": search.instructions} for search in chunk["agent"]["strategy"].searches ], } yield f"data: {json.dumps(strategy_data)}\n\n" elif "provide_answer" in chunk: for answer in chunk["provide_answer"]["answers"]: answer_data = {"type": "answer", "content": answer} yield f"data: {json.dumps(answer_data)}\n\n" elif "write_final_answer" in chunk: final_answer = chunk["write_final_answer"]["final_answer"] final_data = {"type": "final_answer", "content": final_answer} yield f"data: {json.dumps(final_data)}\n\n" # Send completion signal completion_data = {"type": "complete", "final_answer": final_answer} yield f"data: {json.dumps(completion_data)}\n\n" except Exception as e: from open_notebook.utils.error_classifier import classify_error _, user_message = classify_error(e) logger.error(f"Error in ask streaming: {str(e)}") error_data = {"type": "error", "message": user_message} yield f"data: {json.dumps(error_data)}\n\n" @router.post("/search/ask") async def ask_knowledge_base(ask_request: AskRequest): """Ask the knowledge base a question using AI models.""" try: # Validate models exist strategy_model = await Model.get(ask_request.strategy_model) answer_model = await Model.get(ask_request.answer_model) final_answer_model = await Model.get(ask_request.final_answer_model) if not strategy_model: raise HTTPException( status_code=400, detail=f"Strategy model {ask_request.strategy_model} not found", ) if not answer_model: raise HTTPException( status_code=400, detail=f"Answer model {ask_request.answer_model} not found", ) if not final_answer_model: raise HTTPException( status_code=400, detail=f"Final answer model {ask_request.final_answer_model} not found", ) # Check if embedding model is available if not await model_manager.get_embedding_model(): raise HTTPException( status_code=400, detail="Ask feature requires an embedding model. Please configure one in the Models section.", ) # For streaming response return StreamingResponse( stream_ask_response( ask_request.question, strategy_model, answer_model, final_answer_model ), media_type="text/plain", ) except HTTPException: raise except Exception as e: logger.error(f"Error in ask endpoint: {str(e)}") raise HTTPException(status_code=500, detail=f"Ask operation failed: {str(e)}") @router.post("/search/ask/simple", response_model=AskResponse) async def ask_knowledge_base_simple(ask_request: AskRequest): """Ask the knowledge base a question and return a simple response (non-streaming).""" try: # Validate models exist strategy_model = await Model.get(ask_request.strategy_model) answer_model = await Model.get(ask_request.answer_model) final_answer_model = await Model.get(ask_request.final_answer_model) if not strategy_model: raise HTTPException( status_code=400, detail=f"Strategy model {ask_request.strategy_model} not found", ) if not answer_model: raise HTTPException( status_code=400, detail=f"Answer model {ask_request.answer_model} not found", ) if not final_answer_model: raise HTTPException( status_code=400, detail=f"Final answer model {ask_request.final_answer_model} not found", ) # Check if embedding model is available if not await model_manager.get_embedding_model(): raise HTTPException( status_code=400, detail="Ask feature requires an embedding model. Please configure one in the Models section.", ) # Run the ask graph and get final result final_answer = None async for chunk in ask_graph.astream( input=dict(question=ask_request.question), # type: ignore[arg-type] config=dict( configurable=dict( strategy_model=strategy_model.id, answer_model=answer_model.id, final_answer_model=final_answer_model.id, ) ), stream_mode="updates", ): if "write_final_answer" in chunk: final_answer = chunk["write_final_answer"]["final_answer"] if not final_answer: raise HTTPException(status_code=500, detail="No answer generated") return AskResponse(answer=final_answer, question=ask_request.question) except HTTPException: raise except Exception as e: logger.error(f"Error in ask simple endpoint: {str(e)}") raise HTTPException(status_code=500, detail=f"Ask operation failed: {str(e)}") ================================================ FILE: api/routers/settings.py ================================================ from fastapi import APIRouter, HTTPException from loguru import logger from api.models import SettingsResponse, SettingsUpdate from open_notebook.domain.content_settings import ContentSettings from open_notebook.exceptions import InvalidInputError router = APIRouter() @router.get("/settings", response_model=SettingsResponse) async def get_settings(): """Get all application settings.""" try: settings: ContentSettings = await ContentSettings.get_instance() # type: ignore[assignment] return SettingsResponse( default_content_processing_engine_doc=settings.default_content_processing_engine_doc, default_content_processing_engine_url=settings.default_content_processing_engine_url, default_embedding_option=settings.default_embedding_option, auto_delete_files=settings.auto_delete_files, youtube_preferred_languages=settings.youtube_preferred_languages, ) except Exception as e: logger.error(f"Error fetching settings: {str(e)}") raise HTTPException( status_code=500, detail="Error fetching settings" ) @router.put("/settings", response_model=SettingsResponse) async def update_settings(settings_update: SettingsUpdate): """Update application settings.""" try: settings: ContentSettings = await ContentSettings.get_instance() # type: ignore[assignment] # Update only provided fields if settings_update.default_content_processing_engine_doc is not None: # Cast to proper literal type from typing import Literal, cast settings.default_content_processing_engine_doc = cast( Literal["auto", "docling", "simple"], settings_update.default_content_processing_engine_doc, ) if settings_update.default_content_processing_engine_url is not None: from typing import Literal, cast settings.default_content_processing_engine_url = cast( Literal["auto", "firecrawl", "jina", "simple"], settings_update.default_content_processing_engine_url, ) if settings_update.default_embedding_option is not None: from typing import Literal, cast settings.default_embedding_option = cast( Literal["ask", "always", "never"], settings_update.default_embedding_option, ) if settings_update.auto_delete_files is not None: from typing import Literal, cast settings.auto_delete_files = cast( Literal["yes", "no"], settings_update.auto_delete_files ) if settings_update.youtube_preferred_languages is not None: settings.youtube_preferred_languages = ( settings_update.youtube_preferred_languages ) await settings.update() return SettingsResponse( default_content_processing_engine_doc=settings.default_content_processing_engine_doc, default_content_processing_engine_url=settings.default_content_processing_engine_url, default_embedding_option=settings.default_embedding_option, auto_delete_files=settings.auto_delete_files, youtube_preferred_languages=settings.youtube_preferred_languages, ) except HTTPException: raise except InvalidInputError as e: raise HTTPException(status_code=400, detail=str(e)) except Exception as e: logger.error(f"Error updating settings: {str(e)}") raise HTTPException( status_code=500, detail="Error updating settings" ) ================================================ FILE: api/routers/source_chat.py ================================================ import asyncio import json from typing import AsyncGenerator, List, Optional from fastapi import APIRouter, HTTPException, Path from fastapi.responses import StreamingResponse from langchain_core.messages import HumanMessage from langchain_core.runnables import RunnableConfig from loguru import logger from pydantic import BaseModel, Field from open_notebook.database.repository import ensure_record_id, repo_query from open_notebook.domain.notebook import ChatSession, Source from open_notebook.exceptions import ( NotFoundError, ) from open_notebook.graphs.source_chat import source_chat_graph as source_chat_graph from open_notebook.utils.graph_utils import get_session_message_count router = APIRouter() # Request/Response models class CreateSourceChatSessionRequest(BaseModel): source_id: str = Field(..., description="Source ID to create chat session for") title: Optional[str] = Field(None, description="Optional session title") model_override: Optional[str] = Field( None, description="Optional model override for this session" ) class UpdateSourceChatSessionRequest(BaseModel): title: Optional[str] = Field(None, description="New session title") model_override: Optional[str] = Field( None, description="Model override for this session" ) class ChatMessage(BaseModel): id: str = Field(..., description="Message ID") type: str = Field(..., description="Message type (human|ai)") content: str = Field(..., description="Message content") timestamp: Optional[str] = Field(None, description="Message timestamp") class ContextIndicator(BaseModel): sources: List[str] = Field( default_factory=list, description="Source IDs used in context" ) insights: List[str] = Field( default_factory=list, description="Insight IDs used in context" ) notes: List[str] = Field( default_factory=list, description="Note IDs used in context" ) class SourceChatSessionResponse(BaseModel): id: str = Field(..., description="Session ID") title: str = Field(..., description="Session title") source_id: str = Field(..., description="Source ID") model_override: Optional[str] = Field( None, description="Model override for this session" ) created: str = Field(..., description="Creation timestamp") updated: str = Field(..., description="Last update timestamp") message_count: Optional[int] = Field( None, description="Number of messages in session" ) class SourceChatSessionWithMessagesResponse(SourceChatSessionResponse): messages: List[ChatMessage] = Field( default_factory=list, description="Session messages" ) context_indicators: Optional[ContextIndicator] = Field( None, description="Context indicators from last response" ) class SendMessageRequest(BaseModel): message: str = Field(..., description="User message content") model_override: Optional[str] = Field( None, description="Optional model override for this message" ) class SuccessResponse(BaseModel): success: bool = Field(True, description="Operation success status") message: str = Field(..., description="Success message") @router.post( "/sources/{source_id}/chat/sessions", response_model=SourceChatSessionResponse ) async def create_source_chat_session( request: CreateSourceChatSessionRequest, source_id: str = Path(..., description="Source ID"), ): """Create a new chat session for a source.""" try: # Verify source exists full_source_id = ( source_id if source_id.startswith("source:") else f"source:{source_id}" ) source = await Source.get(full_source_id) if not source: raise HTTPException(status_code=404, detail="Source not found") # Create new session with model_override support session = ChatSession( title=request.title or f"Source Chat {asyncio.get_event_loop().time():.0f}", model_override=request.model_override, ) await session.save() # Relate session to source using "refers_to" relation await session.relate("refers_to", full_source_id) return SourceChatSessionResponse( id=session.id or "", title=session.title or "Untitled Session", source_id=source_id, model_override=session.model_override, created=str(session.created), updated=str(session.updated), message_count=0, ) except NotFoundError: raise HTTPException(status_code=404, detail="Source not found") except Exception as e: logger.error(f"Error creating source chat session: {str(e)}") raise HTTPException( status_code=500, detail=f"Error creating source chat session: {str(e)}" ) @router.get( "/sources/{source_id}/chat/sessions", response_model=List[SourceChatSessionResponse] ) async def get_source_chat_sessions(source_id: str = Path(..., description="Source ID")): """Get all chat sessions for a source.""" try: # Verify source exists full_source_id = ( source_id if source_id.startswith("source:") else f"source:{source_id}" ) source = await Source.get(full_source_id) if not source: raise HTTPException(status_code=404, detail="Source not found") # Get sessions that refer to this source - first get relations, then sessions relations = await repo_query( "SELECT in FROM refers_to WHERE out = $source_id", {"source_id": ensure_record_id(full_source_id)}, ) sessions = [] for relation in relations: session_id_raw = relation.get("in") if session_id_raw: session_id = str(session_id_raw) session_result = await repo_query(f"SELECT * FROM {session_id_raw}") if session_result and len(session_result) > 0: session_data = session_result[0] # Get message count from LangGraph state msg_count = await get_session_message_count( source_chat_graph, session_id ) sessions.append( SourceChatSessionResponse( id=session_data.get("id") or "", title=session_data.get("title") or "Untitled Session", source_id=source_id, model_override=session_data.get("model_override"), created=str(session_data.get("created")), updated=str(session_data.get("updated")), message_count=msg_count, ) ) # Sort sessions by created date (newest first) sessions.sort(key=lambda x: x.created, reverse=True) return sessions except NotFoundError: raise HTTPException(status_code=404, detail="Source not found") except Exception as e: logger.error(f"Error fetching source chat sessions: {str(e)}") raise HTTPException( status_code=500, detail=f"Error fetching source chat sessions: {str(e)}" ) @router.get( "/sources/{source_id}/chat/sessions/{session_id}", response_model=SourceChatSessionWithMessagesResponse, ) async def get_source_chat_session( source_id: str = Path(..., description="Source ID"), session_id: str = Path(..., description="Session ID"), ): """Get a specific source chat session with its messages.""" try: # Verify source exists full_source_id = ( source_id if source_id.startswith("source:") else f"source:{source_id}" ) source = await Source.get(full_source_id) if not source: raise HTTPException(status_code=404, detail="Source not found") # Get session full_session_id = ( session_id if session_id.startswith("chat_session:") else f"chat_session:{session_id}" ) session = await ChatSession.get(full_session_id) if not session: raise HTTPException(status_code=404, detail="Session not found") # Verify session is related to this source relation_query = await repo_query( "SELECT * FROM refers_to WHERE in = $session_id AND out = $source_id", { "session_id": ensure_record_id(full_session_id), "source_id": ensure_record_id(full_source_id), }, ) if not relation_query: raise HTTPException( status_code=404, detail="Session not found for this source" ) # Get session state from LangGraph to retrieve messages # Use sync get_state() in a thread since SqliteSaver doesn't support async thread_state = await asyncio.to_thread( source_chat_graph.get_state, config=RunnableConfig(configurable={"thread_id": full_session_id}), ) # Extract messages from state messages: list[ChatMessage] = [] context_indicators = None if thread_state and thread_state.values: # Extract messages if "messages" in thread_state.values: for msg in thread_state.values["messages"]: messages.append( ChatMessage( id=getattr(msg, "id", f"msg_{len(messages)}"), type=msg.type if hasattr(msg, "type") else "unknown", content=msg.content if hasattr(msg, "content") else str(msg), timestamp=None, # LangChain messages don't have timestamps by default ) ) # Extract context indicators from the last state if "context_indicators" in thread_state.values: context_data = thread_state.values["context_indicators"] context_indicators = ContextIndicator( sources=context_data.get("sources", []), insights=context_data.get("insights", []), notes=context_data.get("notes", []), ) return SourceChatSessionWithMessagesResponse( id=session.id or "", title=session.title or "Untitled Session", source_id=source_id, model_override=getattr(session, "model_override", None), created=str(session.created), updated=str(session.updated), message_count=len(messages), messages=messages, context_indicators=context_indicators, ) except NotFoundError: raise HTTPException(status_code=404, detail="Source or session not found") except Exception as e: logger.error(f"Error fetching source chat session: {str(e)}") raise HTTPException( status_code=500, detail=f"Error fetching source chat session: {str(e)}" ) @router.put( "/sources/{source_id}/chat/sessions/{session_id}", response_model=SourceChatSessionResponse, ) async def update_source_chat_session( request: UpdateSourceChatSessionRequest, source_id: str = Path(..., description="Source ID"), session_id: str = Path(..., description="Session ID"), ): """Update source chat session title and/or model override.""" try: # Verify source exists full_source_id = ( source_id if source_id.startswith("source:") else f"source:{source_id}" ) source = await Source.get(full_source_id) if not source: raise HTTPException(status_code=404, detail="Source not found") # Get session full_session_id = ( session_id if session_id.startswith("chat_session:") else f"chat_session:{session_id}" ) session = await ChatSession.get(full_session_id) if not session: raise HTTPException(status_code=404, detail="Session not found") # Verify session is related to this source relation_query = await repo_query( "SELECT * FROM refers_to WHERE in = $session_id AND out = $source_id", { "session_id": ensure_record_id(full_session_id), "source_id": ensure_record_id(full_source_id), }, ) if not relation_query: raise HTTPException( status_code=404, detail="Session not found for this source" ) # Update session fields if request.title is not None: session.title = request.title if request.model_override is not None: session.model_override = request.model_override await session.save() # Get message count from LangGraph state msg_count = await get_session_message_count(source_chat_graph, full_session_id) return SourceChatSessionResponse( id=session.id or "", title=session.title or "Untitled Session", source_id=source_id, model_override=getattr(session, "model_override", None), created=str(session.created), updated=str(session.updated), message_count=msg_count, ) except NotFoundError: raise HTTPException(status_code=404, detail="Source or session not found") except Exception as e: logger.error(f"Error updating source chat session: {str(e)}") raise HTTPException( status_code=500, detail=f"Error updating source chat session: {str(e)}" ) @router.delete( "/sources/{source_id}/chat/sessions/{session_id}", response_model=SuccessResponse ) async def delete_source_chat_session( source_id: str = Path(..., description="Source ID"), session_id: str = Path(..., description="Session ID"), ): """Delete a source chat session.""" try: # Verify source exists full_source_id = ( source_id if source_id.startswith("source:") else f"source:{source_id}" ) source = await Source.get(full_source_id) if not source: raise HTTPException(status_code=404, detail="Source not found") # Get session full_session_id = ( session_id if session_id.startswith("chat_session:") else f"chat_session:{session_id}" ) session = await ChatSession.get(full_session_id) if not session: raise HTTPException(status_code=404, detail="Session not found") # Verify session is related to this source relation_query = await repo_query( "SELECT * FROM refers_to WHERE in = $session_id AND out = $source_id", { "session_id": ensure_record_id(full_session_id), "source_id": ensure_record_id(full_source_id), }, ) if not relation_query: raise HTTPException( status_code=404, detail="Session not found for this source" ) await session.delete() return SuccessResponse( success=True, message="Source chat session deleted successfully" ) except NotFoundError: raise HTTPException(status_code=404, detail="Source or session not found") except Exception as e: logger.error(f"Error deleting source chat session: {str(e)}") raise HTTPException( status_code=500, detail=f"Error deleting source chat session: {str(e)}" ) async def stream_source_chat_response( session_id: str, source_id: str, message: str, model_override: Optional[str] = None ) -> AsyncGenerator[str, None]: """Stream the source chat response as Server-Sent Events.""" try: # Get current state # Use sync get_state() in a thread since SqliteSaver doesn't support async current_state = await asyncio.to_thread( source_chat_graph.get_state, config=RunnableConfig(configurable={"thread_id": session_id}), ) # Prepare state for execution state_values = current_state.values if current_state else {} state_values["messages"] = state_values.get("messages", []) state_values["source_id"] = source_id state_values["model_override"] = model_override # Add user message to state user_message = HumanMessage(content=message) state_values["messages"].append(user_message) # Send user message event user_event = {"type": "user_message", "content": message, "timestamp": None} yield f"data: {json.dumps(user_event)}\n\n" # Execute source chat graph synchronously (like notebook chat does) result = source_chat_graph.invoke( input=state_values, # type: ignore[arg-type] config=RunnableConfig( configurable={"thread_id": session_id, "model_id": model_override} ), ) # Stream the complete AI response if "messages" in result: for msg in result["messages"]: if hasattr(msg, "type") and msg.type == "ai": ai_event = { "type": "ai_message", "content": msg.content if hasattr(msg, "content") else str(msg), "timestamp": None, } yield f"data: {json.dumps(ai_event)}\n\n" # Stream context indicators if "context_indicators" in result: context_event = { "type": "context_indicators", "data": result["context_indicators"], } yield f"data: {json.dumps(context_event)}\n\n" # Send completion signal completion_event = {"type": "complete"} yield f"data: {json.dumps(completion_event)}\n\n" except Exception as e: from open_notebook.utils.error_classifier import classify_error _, user_message = classify_error(e) logger.error(f"Error in source chat streaming: {str(e)}") error_event = {"type": "error", "message": user_message} yield f"data: {json.dumps(error_event)}\n\n" @router.post("/sources/{source_id}/chat/sessions/{session_id}/messages") async def send_message_to_source_chat( request: SendMessageRequest, source_id: str = Path(..., description="Source ID"), session_id: str = Path(..., description="Session ID"), ): """Send a message to source chat session with SSE streaming response.""" try: # Verify source exists full_source_id = ( source_id if source_id.startswith("source:") else f"source:{source_id}" ) source = await Source.get(full_source_id) if not source: raise HTTPException(status_code=404, detail="Source not found") # Verify session exists and is related to source full_session_id = ( session_id if session_id.startswith("chat_session:") else f"chat_session:{session_id}" ) session = await ChatSession.get(full_session_id) if not session: raise HTTPException(status_code=404, detail="Session not found") # Verify session is related to this source relation_query = await repo_query( "SELECT * FROM refers_to WHERE in = $session_id AND out = $source_id", { "session_id": ensure_record_id(full_session_id), "source_id": ensure_record_id(full_source_id), }, ) if not relation_query: raise HTTPException( status_code=404, detail="Session not found for this source" ) if not request.message: raise HTTPException(status_code=400, detail="Message content is required") # Determine model override (request override takes precedence over session override) model_override = request.model_override or getattr( session, "model_override", None ) # Update session timestamp await session.save() # Return streaming response return StreamingResponse( stream_source_chat_response( session_id=full_session_id, source_id=full_source_id, message=request.message, model_override=model_override, ), media_type="text/plain", headers={ "Cache-Control": "no-cache", "Connection": "keep-alive", "Content-Type": "text/plain; charset=utf-8", }, ) except HTTPException: raise except Exception as e: logger.error(f"Error sending message to source chat: {str(e)}") raise HTTPException(status_code=500, detail=f"Error sending message: {str(e)}") ================================================ FILE: api/routers/sources.py ================================================ import asyncio import os from pathlib import Path from typing import Any, List, Optional from fastapi import ( APIRouter, Depends, File, Form, HTTPException, Query, UploadFile, ) from fastapi.responses import FileResponse, Response from loguru import logger from surreal_commands import execute_command_sync, submit_command from api.command_service import CommandService from api.models import ( AssetModel, CreateSourceInsightRequest, InsightCreationResponse, SourceCreate, SourceInsightResponse, SourceListResponse, SourceResponse, SourceStatusResponse, SourceUpdate, ) from commands.source_commands import SourceProcessingInput from open_notebook.config import UPLOADS_FOLDER from open_notebook.database.repository import ensure_record_id, repo_query from open_notebook.domain.notebook import Notebook, Source from open_notebook.domain.transformation import Transformation from open_notebook.exceptions import InvalidInputError router = APIRouter() def generate_unique_filename(original_filename: str, upload_folder: str) -> str: """Generate unique filename like Streamlit app (append counter if file exists).""" file_path = Path(upload_folder) file_path.mkdir(parents=True, exist_ok=True) # Split filename and extension stem = Path(original_filename).stem suffix = Path(original_filename).suffix # Check if file exists and generate unique name counter = 0 while True: if counter == 0: new_filename = original_filename else: new_filename = f"{stem} ({counter}){suffix}" full_path = file_path / new_filename if not full_path.exists(): return str(full_path) counter += 1 async def save_uploaded_file(upload_file: UploadFile) -> str: """Save uploaded file to uploads folder and return file path.""" if not upload_file.filename: raise ValueError("No filename provided") # Generate unique filename file_path = generate_unique_filename(upload_file.filename, UPLOADS_FOLDER) try: # Save file with open(file_path, "wb") as f: content = await upload_file.read() f.write(content) logger.info(f"Saved uploaded file to: {file_path}") return file_path except Exception as e: logger.error(f"Failed to save uploaded file: {e}") # Clean up partial file if it exists if os.path.exists(file_path): os.unlink(file_path) raise def parse_source_form_data( type: str = Form(...), notebook_id: Optional[str] = Form(None), notebooks: Optional[str] = Form(None), # JSON string of notebook IDs url: Optional[str] = Form(None), content: Optional[str] = Form(None), title: Optional[str] = Form(None), transformations: Optional[str] = Form(None), # JSON string of transformation IDs embed: str = Form("false"), # Accept as string, convert to bool delete_source: str = Form("false"), # Accept as string, convert to bool async_processing: str = Form("false"), # Accept as string, convert to bool file: Optional[UploadFile] = File(None), ) -> tuple[SourceCreate, Optional[UploadFile]]: """Parse form data into SourceCreate model and return upload file separately.""" import json # Convert string booleans to actual booleans def str_to_bool(value: str) -> bool: return value.lower() in ("true", "1", "yes", "on") embed_bool = str_to_bool(embed) delete_source_bool = str_to_bool(delete_source) async_processing_bool = str_to_bool(async_processing) # Parse JSON strings notebooks_list = None if notebooks: try: notebooks_list = json.loads(notebooks) except json.JSONDecodeError: logger.error(f"Invalid JSON in notebooks field: {notebooks}") raise ValueError("Invalid JSON in notebooks field") transformations_list = [] if transformations: try: transformations_list = json.loads(transformations) except json.JSONDecodeError: logger.error(f"Invalid JSON in transformations field: {transformations}") raise ValueError("Invalid JSON in transformations field") # Create SourceCreate instance try: source_data = SourceCreate( type=type, notebook_id=notebook_id, notebooks=notebooks_list, url=url, content=content, title=title, file_path=None, # Will be set later if file is uploaded transformations=transformations_list, embed=embed_bool, delete_source=delete_source_bool, async_processing=async_processing_bool, ) pass # SourceCreate instance created successfully except Exception as e: logger.error(f"Failed to create SourceCreate instance: {e}") raise return source_data, file @router.get("/sources", response_model=List[SourceListResponse]) async def get_sources( notebook_id: Optional[str] = Query(None, description="Filter by notebook ID"), limit: int = Query( 50, ge=1, le=100, description="Number of sources to return (1-100)" ), offset: int = Query(0, ge=0, description="Number of sources to skip"), sort_by: str = Query( "updated", description="Field to sort by (created or updated)" ), sort_order: str = Query("desc", description="Sort order (asc or desc)"), ): """Get sources with pagination and sorting support.""" try: # Validate sort parameters if sort_by not in ["created", "updated"]: raise HTTPException( status_code=400, detail="sort_by must be 'created' or 'updated'" ) if sort_order.lower() not in ["asc", "desc"]: raise HTTPException( status_code=400, detail="sort_order must be 'asc' or 'desc'" ) # Build ORDER BY clause order_clause = f"ORDER BY {sort_by} {sort_order.upper()}" # Build the query if notebook_id: # Verify notebook exists first notebook = await Notebook.get(notebook_id) if not notebook: raise HTTPException(status_code=404, detail="Notebook not found") # Query sources for specific notebook - include command field with FETCH query = f""" SELECT id, asset, created, title, updated, topics, command, (SELECT VALUE count() FROM source_insight WHERE source = $parent.id GROUP ALL)[0].count OR 0 AS insights_count, (SELECT VALUE id FROM source_embedding WHERE source = $parent.id LIMIT 1) != [] AS embedded FROM (select value in from reference where out=$notebook_id) {order_clause} LIMIT $limit START $offset FETCH command """ result = await repo_query( query, { "notebook_id": ensure_record_id(notebook_id), "limit": limit, "offset": offset, }, ) else: # Query all sources - include command field with FETCH query = f""" SELECT id, asset, created, title, updated, topics, command, (SELECT VALUE count() FROM source_insight WHERE source = $parent.id GROUP ALL)[0].count OR 0 AS insights_count, (SELECT VALUE id FROM source_embedding WHERE source = $parent.id LIMIT 1) != [] AS embedded FROM source {order_clause} LIMIT $limit START $offset FETCH command """ result = await repo_query(query, {"limit": limit, "offset": offset}) # Convert result to response model # Command data is already fetched via FETCH command clause response_list = [] for row in result: command = row.get("command") command_id = None status = None processing_info = None # Extract status from fetched command object (already resolved by FETCH) if command and isinstance(command, dict): command_id = str(command.get("id")) if command.get("id") else None status = command.get("status") # Extract execution metadata from nested result structure result_data = command.get("result") execution_metadata = ( result_data.get("execution_metadata", {}) if isinstance(result_data, dict) else {} ) processing_info = { "started_at": execution_metadata.get("started_at"), "completed_at": execution_metadata.get("completed_at"), "error": command.get("error_message"), } elif command: # Command exists but FETCH failed to resolve it (broken reference) command_id = str(command) status = "unknown" response_list.append( SourceListResponse( id=row["id"], title=row.get("title"), topics=row.get("topics") or [], asset=AssetModel( file_path=row["asset"].get("file_path") if row.get("asset") else None, url=row["asset"].get("url") if row.get("asset") else None, ) if row.get("asset") else None, embedded=row.get("embedded", False), embedded_chunks=0, # Not needed in list view insights_count=row.get("insights_count", 0), created=str(row["created"]), updated=str(row["updated"]), # Status fields from fetched command command_id=command_id, status=status, processing_info=processing_info, ) ) return response_list except HTTPException: raise except Exception as e: logger.error(f"Error fetching sources: {str(e)}") raise HTTPException(status_code=500, detail=f"Error fetching sources: {str(e)}") @router.post("/sources", response_model=SourceResponse) async def create_source( form_data: tuple[SourceCreate, Optional[UploadFile]] = Depends( parse_source_form_data ), ): """Create a new source with support for both JSON and multipart form data.""" source_data, upload_file = form_data # Initialize file_path before try block so exception handlers can reference it file_path = None try: # Verify all specified notebooks exist (backward compatibility support) for notebook_id in source_data.notebooks or []: notebook = await Notebook.get(notebook_id) if not notebook: raise HTTPException( status_code=404, detail=f"Notebook {notebook_id} not found" ) # Handle file upload if provided if upload_file and source_data.type == "upload": try: file_path = await save_uploaded_file(upload_file) except Exception as e: logger.error(f"File upload failed: {e}") raise HTTPException( status_code=400, detail=f"File upload failed: {str(e)}" ) # Prepare content_state for processing content_state: dict[str, Any] = {} if source_data.type == "link": if not source_data.url: raise HTTPException( status_code=400, detail="URL is required for link type" ) content_state["url"] = source_data.url elif source_data.type == "upload": # Use uploaded file path or provided file_path (backward compatibility) final_file_path = file_path or source_data.file_path if not final_file_path: raise HTTPException( status_code=400, detail="File upload or file_path is required for upload type", ) content_state["file_path"] = final_file_path content_state["delete_source"] = source_data.delete_source elif source_data.type == "text": if not source_data.content: raise HTTPException( status_code=400, detail="Content is required for text type" ) content_state["content"] = source_data.content else: raise HTTPException( status_code=400, detail="Invalid source type. Must be link, upload, or text", ) # Validate transformations exist transformation_ids = source_data.transformations or [] for trans_id in transformation_ids: transformation = await Transformation.get(trans_id) if not transformation: raise HTTPException( status_code=404, detail=f"Transformation {trans_id} not found" ) # Branch based on processing mode if source_data.async_processing: # ASYNC PATH: Create source record first, then queue command logger.info("Using async processing path") # Create minimal source record - let SurrealDB generate the ID source = Source( title=source_data.title or "Processing...", topics=[], ) await source.save() # Add source to notebooks immediately so it appears in the UI # The source_graph will skip adding duplicates for notebook_id in source_data.notebooks or []: await source.add_to_notebook(notebook_id) try: # Import command modules to ensure they're registered import commands.source_commands # noqa: F401 # Submit command for background processing command_input = SourceProcessingInput( source_id=str(source.id), content_state=content_state, notebook_ids=source_data.notebooks, transformations=transformation_ids, embed=source_data.embed, ) command_id = await CommandService.submit_command_job( "open_notebook", # app name "process_source", # command name command_input.model_dump(), ) logger.info(f"Submitted async processing command: {command_id}") # Update source with command reference immediately # command_id already includes 'command:' prefix source.command = ensure_record_id(command_id) await source.save() # Return source with command info return SourceResponse( id=source.id or "", title=source.title, topics=source.topics or [], asset=None, # Will be populated after processing full_text=None, # Will be populated after processing embedded=False, # Will be updated after processing embedded_chunks=0, created=str(source.created), updated=str(source.updated), command_id=command_id, status="new", processing_info={"async": True, "queued": True}, ) except Exception as e: logger.error(f"Failed to submit async processing command: {e}") # Clean up source record on command submission failure try: await source.delete() except Exception: pass # Clean up uploaded file if we created it if file_path and upload_file: try: os.unlink(file_path) except Exception: pass raise HTTPException( status_code=500, detail=f"Failed to queue processing: {str(e)}" ) else: # SYNC PATH: Execute synchronously using execute_command_sync logger.info("Using sync processing path") try: # Import command modules to ensure they're registered import commands.source_commands # noqa: F401 # Create source record - let SurrealDB generate the ID source = Source( title=source_data.title or "Processing...", topics=[], ) await source.save() # Add source to notebooks immediately so it appears in the UI # The source_graph will skip adding duplicates for notebook_id in source_data.notebooks or []: await source.add_to_notebook(notebook_id) # Execute command synchronously command_input = SourceProcessingInput( source_id=str(source.id), content_state=content_state, notebook_ids=source_data.notebooks, transformations=transformation_ids, embed=source_data.embed, ) # Run in thread pool to avoid blocking the event loop # execute_command_sync uses asyncio.run() internally which can't # be called from an already-running event loop (FastAPI) result = await asyncio.to_thread( execute_command_sync, "open_notebook", # app name "process_source", # command name command_input.model_dump(), timeout=300, # 5 minute timeout for sync processing ) if not result.is_success(): logger.error(f"Sync processing failed: {result.error_message}") # Clean up source record try: await source.delete() except Exception: pass # Clean up uploaded file if we created it if file_path and upload_file: try: os.unlink(file_path) except Exception: pass raise HTTPException( status_code=500, detail=f"Processing failed: {result.error_message}", ) # Get the processed source if not source.id: raise HTTPException(status_code=500, detail="Source ID is missing") processed_source = await Source.get(source.id) if not processed_source: raise HTTPException( status_code=500, detail="Processed source not found" ) embedded_chunks = await processed_source.get_embedded_chunks() return SourceResponse( id=processed_source.id or "", title=processed_source.title, topics=processed_source.topics or [], asset=AssetModel( file_path=processed_source.asset.file_path if processed_source.asset else None, url=processed_source.asset.url if processed_source.asset else None, ) if processed_source.asset else None, full_text=processed_source.full_text, embedded=embedded_chunks > 0, embedded_chunks=embedded_chunks, created=str(processed_source.created), updated=str(processed_source.updated), # No command_id or status for sync processing (legacy behavior) ) except Exception as e: logger.error(f"Sync processing failed: {e}") # Clean up uploaded file if we created it if file_path and upload_file: try: os.unlink(file_path) except Exception: pass raise except HTTPException: # Clean up uploaded file on HTTP exceptions if we created it if file_path and upload_file: try: os.unlink(file_path) except Exception: pass raise except InvalidInputError as e: # Clean up uploaded file on validation errors if we created it if file_path and upload_file: try: os.unlink(file_path) except Exception: pass raise HTTPException(status_code=400, detail=str(e)) except Exception as e: logger.error(f"Error creating source: {str(e)}") # Clean up uploaded file on unexpected errors if we created it if file_path and upload_file: try: os.unlink(file_path) except Exception: pass raise HTTPException(status_code=500, detail=f"Error creating source: {str(e)}") @router.post("/sources/json", response_model=SourceResponse) async def create_source_json(source_data: SourceCreate): """Create a new source using JSON payload (legacy endpoint for backward compatibility).""" # Convert to form data format and call main endpoint form_data = (source_data, None) return await create_source(form_data) async def _resolve_source_file(source_id: str) -> tuple[str, str]: source = await Source.get(source_id) if not source: raise HTTPException(status_code=404, detail="Source not found") file_path = source.asset.file_path if source.asset else None if not file_path: raise HTTPException(status_code=404, detail="Source has no file to download") safe_root = os.path.realpath(UPLOADS_FOLDER) resolved_path = os.path.realpath(file_path) if not resolved_path.startswith(safe_root): logger.warning( f"Blocked download outside uploads directory for source {source_id}: {resolved_path}" ) raise HTTPException(status_code=403, detail="Access to file denied") if not os.path.exists(resolved_path): raise HTTPException(status_code=404, detail="File not found on server") filename = os.path.basename(resolved_path) return resolved_path, filename def _is_source_file_available(source: Source) -> Optional[bool]: if not source or not source.asset or not source.asset.file_path: return None file_path = source.asset.file_path safe_root = os.path.realpath(UPLOADS_FOLDER) resolved_path = os.path.realpath(file_path) if not resolved_path.startswith(safe_root): return False return os.path.exists(resolved_path) @router.get("/sources/{source_id}", response_model=SourceResponse) async def get_source(source_id: str): """Get a specific source by ID.""" try: source = await Source.get(source_id) if not source: raise HTTPException(status_code=404, detail="Source not found") # Get status information if command exists status = None processing_info = None if source.command: try: status = await source.get_status() processing_info = await source.get_processing_progress() except Exception as e: logger.warning(f"Failed to get status for source {source_id}: {e}") status = "unknown" embedded_chunks = await source.get_embedded_chunks() # Get associated notebooks notebooks_query = await repo_query( "SELECT VALUE out FROM reference WHERE in = $source_id", {"source_id": ensure_record_id(source.id or source_id)}, ) notebook_ids = ( [str(nb_id) for nb_id in notebooks_query] if notebooks_query else [] ) return SourceResponse( id=source.id or "", title=source.title, topics=source.topics or [], asset=AssetModel( file_path=source.asset.file_path if source.asset else None, url=source.asset.url if source.asset else None, ) if source.asset else None, full_text=source.full_text, embedded=embedded_chunks > 0, embedded_chunks=embedded_chunks, file_available=_is_source_file_available(source), created=str(source.created), updated=str(source.updated), # Status fields command_id=str(source.command) if source.command else None, status=status, processing_info=processing_info, # Notebook associations notebooks=notebook_ids, ) except HTTPException: raise except Exception as e: logger.error(f"Error fetching source {source_id}: {str(e)}") raise HTTPException(status_code=500, detail=f"Error fetching source: {str(e)}") @router.head("/sources/{source_id}/download") async def check_source_file(source_id: str): """Check if a source has a downloadable file.""" try: await _resolve_source_file(source_id) return Response(status_code=200) except HTTPException: raise except Exception as e: logger.error(f"Error checking file for source {source_id}: {str(e)}") raise HTTPException(status_code=500, detail="Failed to verify file") @router.get("/sources/{source_id}/download") async def download_source_file(source_id: str): """Download the original file associated with an uploaded source.""" try: resolved_path, filename = await _resolve_source_file(source_id) return FileResponse( path=resolved_path, filename=filename, media_type="application/octet-stream", ) except HTTPException: raise except Exception as e: logger.error(f"Error downloading file for source {source_id}: {str(e)}") raise HTTPException(status_code=500, detail="Failed to download source file") @router.get("/sources/{source_id}/status", response_model=SourceStatusResponse) async def get_source_status(source_id: str): """Get processing status for a source.""" try: # First, verify source exists source = await Source.get(source_id) if not source: raise HTTPException(status_code=404, detail="Source not found") # Check if this is a legacy source (no command) if not source.command: return SourceStatusResponse( status=None, message="Legacy source (completed before async processing)", processing_info=None, command_id=None, ) # Get command status and processing info try: status = await source.get_status() processing_info = await source.get_processing_progress() # Generate descriptive message based on status if status == "completed": message = "Source processing completed successfully" elif status == "failed": message = "Source processing failed" elif status == "running": message = "Source processing in progress" elif status == "queued": message = "Source processing queued" elif status == "unknown": message = "Source processing status unknown" else: message = f"Source processing status: {status}" return SourceStatusResponse( status=status, message=message, processing_info=processing_info, command_id=str(source.command) if source.command else None, ) except Exception as e: logger.warning(f"Failed to get status for source {source_id}: {e}") return SourceStatusResponse( status="unknown", message="Failed to retrieve processing status", processing_info=None, command_id=str(source.command) if source.command else None, ) except HTTPException: raise except Exception as e: logger.error(f"Error fetching status for source {source_id}: {str(e)}") raise HTTPException( status_code=500, detail=f"Error fetching source status: {str(e)}" ) @router.put("/sources/{source_id}", response_model=SourceResponse) async def update_source(source_id: str, source_update: SourceUpdate): """Update a source.""" try: source = await Source.get(source_id) if not source: raise HTTPException(status_code=404, detail="Source not found") # Update only provided fields if source_update.title is not None: source.title = source_update.title if source_update.topics is not None: source.topics = source_update.topics await source.save() embedded_chunks = await source.get_embedded_chunks() return SourceResponse( id=source.id or "", title=source.title, topics=source.topics or [], asset=AssetModel( file_path=source.asset.file_path if source.asset else None, url=source.asset.url if source.asset else None, ) if source.asset else None, full_text=source.full_text, embedded=embedded_chunks > 0, embedded_chunks=embedded_chunks, created=str(source.created), updated=str(source.updated), ) except HTTPException: raise except InvalidInputError as e: raise HTTPException(status_code=400, detail=str(e)) except Exception as e: logger.error(f"Error updating source {source_id}: {str(e)}") raise HTTPException(status_code=500, detail=f"Error updating source: {str(e)}") @router.post("/sources/{source_id}/retry", response_model=SourceResponse) async def retry_source_processing(source_id: str): """Retry processing for a failed or stuck source.""" try: # First, verify source exists source = await Source.get(source_id) if not source: raise HTTPException(status_code=404, detail="Source not found") # Check if source already has a running command if source.command: try: status = await source.get_status() if status in ["running", "queued"]: raise HTTPException( status_code=400, detail="Source is already processing. Cannot retry while processing is active.", ) except Exception as e: logger.warning( f"Failed to check current status for source {source_id}: {e}" ) # Continue with retry if we can't check status # Get notebooks that this source belongs to query = "SELECT notebook FROM reference WHERE source = $source_id" references = await repo_query(query, {"source_id": source_id}) notebook_ids = [str(ref["notebook"]) for ref in references] if not notebook_ids: raise HTTPException( status_code=400, detail="Source is not associated with any notebooks" ) # Prepare content_state based on source asset content_state = {} if source.asset: if source.asset.file_path: content_state = { "file_path": source.asset.file_path, "delete_source": False, # Don't delete on retry } elif source.asset.url: content_state = {"url": source.asset.url} else: raise HTTPException( status_code=400, detail="Source asset has no file_path or url" ) else: # Check if it's a text source by trying to get full_text if source.full_text: content_state = {"content": source.full_text} else: raise HTTPException( status_code=400, detail="Cannot determine source content for retry" ) try: # Import command modules to ensure they're registered import commands.source_commands # noqa: F401 # Submit new command for background processing command_input = SourceProcessingInput( source_id=str(source.id), content_state=content_state, notebook_ids=notebook_ids, transformations=[], # Use default transformations on retry embed=True, # Always embed on retry ) command_id = await CommandService.submit_command_job( "open_notebook", # app name "process_source", # command name command_input.model_dump(), ) logger.info( f"Submitted retry processing command: {command_id} for source {source_id}" ) # Update source with new command ID source.command = ensure_record_id(f"command:{command_id}") await source.save() # Get current embedded chunks count embedded_chunks = await source.get_embedded_chunks() # Return updated source response return SourceResponse( id=source.id or "", title=source.title, topics=source.topics or [], asset=AssetModel( file_path=source.asset.file_path if source.asset else None, url=source.asset.url if source.asset else None, ) if source.asset else None, full_text=source.full_text, embedded=embedded_chunks > 0, embedded_chunks=embedded_chunks, created=str(source.created), updated=str(source.updated), command_id=command_id, status="queued", processing_info={"retry": True, "queued": True}, ) except Exception as e: logger.error( f"Failed to submit retry processing command for source {source_id}: {e}" ) raise HTTPException( status_code=500, detail=f"Failed to queue retry processing: {str(e)}" ) except HTTPException: raise except Exception as e: logger.error(f"Error retrying source processing for {source_id}: {str(e)}") raise HTTPException( status_code=500, detail=f"Error retrying source processing: {str(e)}" ) @router.delete("/sources/{source_id}") async def delete_source(source_id: str): """Delete a source.""" try: source = await Source.get(source_id) if not source: raise HTTPException(status_code=404, detail="Source not found") await source.delete() return {"message": "Source deleted successfully"} except HTTPException: raise except Exception as e: logger.error(f"Error deleting source {source_id}: {str(e)}") raise HTTPException(status_code=500, detail=f"Error deleting source: {str(e)}") @router.get("/sources/{source_id}/insights", response_model=List[SourceInsightResponse]) async def get_source_insights(source_id: str): """Get all insights for a specific source.""" try: source = await Source.get(source_id) if not source: raise HTTPException(status_code=404, detail="Source not found") insights = await source.get_insights() return [ SourceInsightResponse( id=insight.id or "", source_id=source_id, insight_type=insight.insight_type, content=insight.content, created=str(insight.created), updated=str(insight.updated), ) for insight in insights ] except HTTPException: raise except Exception as e: logger.error(f"Error fetching insights for source {source_id}: {str(e)}") raise HTTPException( status_code=500, detail=f"Error fetching insights: {str(e)}" ) @router.post( "/sources/{source_id}/insights", response_model=InsightCreationResponse, status_code=202, ) async def create_source_insight(source_id: str, request: CreateSourceInsightRequest): """ Start insight generation for a source by running a transformation. This endpoint returns immediately with a 202 Accepted status. The transformation runs asynchronously in the background via the job queue. Poll GET /sources/{source_id}/insights to see when the insight is ready. """ try: # Validate source exists source = await Source.get(source_id) if not source: raise HTTPException(status_code=404, detail="Source not found") # Validate transformation exists transformation = await Transformation.get(request.transformation_id) if not transformation: raise HTTPException(status_code=404, detail="Transformation not found") # Submit transformation as background job (fire-and-forget) command_id = submit_command( "open_notebook", "run_transformation", { "source_id": source_id, "transformation_id": request.transformation_id, }, ) logger.info( f"Submitted run_transformation command {command_id} for source {source_id}" ) # Return immediately with command_id for status tracking return InsightCreationResponse( status="pending", message="Insight generation started", source_id=source_id, transformation_id=request.transformation_id, command_id=str(command_id), ) except HTTPException: raise except Exception as e: logger.error(f"Error starting insight generation for source {source_id}: {e}") raise HTTPException( status_code=500, detail=f"Error starting insight generation: {str(e)}" ) ================================================ FILE: api/routers/speaker_profiles.py ================================================ from typing import Any, Dict, List, Optional from fastapi import APIRouter, HTTPException from loguru import logger from pydantic import BaseModel, Field from open_notebook.podcasts.models import SpeakerProfile router = APIRouter() class SpeakerProfileResponse(BaseModel): id: str name: str description: str voice_model: Optional[str] = None speakers: List[Dict[str, Any]] # Legacy fields (for display/migration awareness) tts_provider: Optional[str] = None tts_model: Optional[str] = None def _profile_to_response(profile: SpeakerProfile) -> SpeakerProfileResponse: return SpeakerProfileResponse( id=str(profile.id), name=profile.name, description=profile.description or "", voice_model=profile.voice_model, speakers=profile.speakers, tts_provider=profile.tts_provider, tts_model=profile.tts_model, ) @router.get("/speaker-profiles", response_model=List[SpeakerProfileResponse]) async def list_speaker_profiles(): """List all available speaker profiles""" try: profiles = await SpeakerProfile.get_all(order_by="name asc") return [_profile_to_response(p) for p in profiles] except Exception as e: logger.error(f"Failed to fetch speaker profiles: {e}") raise HTTPException( status_code=500, detail="Failed to fetch speaker profiles" ) @router.get("/speaker-profiles/{profile_name}", response_model=SpeakerProfileResponse) async def get_speaker_profile(profile_name: str): """Get a specific speaker profile by name""" try: profile = await SpeakerProfile.get_by_name(profile_name) if not profile: raise HTTPException( status_code=404, detail=f"Speaker profile '{profile_name}' not found" ) return _profile_to_response(profile) except HTTPException: raise except Exception as e: logger.error(f"Failed to fetch speaker profile '{profile_name}': {e}") raise HTTPException( status_code=500, detail="Failed to fetch speaker profile" ) class SpeakerProfileCreate(BaseModel): name: str = Field(..., description="Unique profile name") description: str = Field("", description="Profile description") voice_model: Optional[str] = Field(None, description="Model record ID for TTS") speakers: List[Dict[str, Any]] = Field( ..., description="Array of speaker configurations" ) # Legacy fields (accepted but not required) tts_provider: Optional[str] = None tts_model: Optional[str] = None @router.post("/speaker-profiles", response_model=SpeakerProfileResponse) async def create_speaker_profile(profile_data: SpeakerProfileCreate): """Create a new speaker profile""" try: profile = SpeakerProfile( name=profile_data.name, description=profile_data.description, voice_model=profile_data.voice_model, speakers=profile_data.speakers, tts_provider=profile_data.tts_provider, tts_model=profile_data.tts_model, ) await profile.save() return _profile_to_response(profile) except Exception as e: logger.error(f"Failed to create speaker profile: {e}") raise HTTPException( status_code=500, detail="Failed to create speaker profile" ) @router.put("/speaker-profiles/{profile_id}", response_model=SpeakerProfileResponse) async def update_speaker_profile(profile_id: str, profile_data: SpeakerProfileCreate): """Update an existing speaker profile""" try: profile = await SpeakerProfile.get(profile_id) if not profile: raise HTTPException( status_code=404, detail=f"Speaker profile '{profile_id}' not found" ) profile.name = profile_data.name profile.description = profile_data.description profile.voice_model = profile_data.voice_model profile.speakers = profile_data.speakers profile.tts_provider = profile_data.tts_provider profile.tts_model = profile_data.tts_model await profile.save() return _profile_to_response(profile) except HTTPException: raise except Exception as e: logger.error(f"Failed to update speaker profile: {e}") raise HTTPException( status_code=500, detail="Failed to update speaker profile" ) @router.delete("/speaker-profiles/{profile_id}") async def delete_speaker_profile(profile_id: str): """Delete a speaker profile""" try: profile = await SpeakerProfile.get(profile_id) if not profile: raise HTTPException( status_code=404, detail=f"Speaker profile '{profile_id}' not found" ) await profile.delete() return {"message": "Speaker profile deleted successfully"} except HTTPException: raise except Exception as e: logger.error(f"Failed to delete speaker profile: {e}") raise HTTPException( status_code=500, detail="Failed to delete speaker profile" ) @router.post( "/speaker-profiles/{profile_id}/duplicate", response_model=SpeakerProfileResponse ) async def duplicate_speaker_profile(profile_id: str): """Duplicate a speaker profile""" try: original = await SpeakerProfile.get(profile_id) if not original: raise HTTPException( status_code=404, detail=f"Speaker profile '{profile_id}' not found" ) duplicate = SpeakerProfile( name=f"{original.name} - Copy", description=original.description, voice_model=original.voice_model, speakers=original.speakers, tts_provider=original.tts_provider, tts_model=original.tts_model, ) await duplicate.save() return _profile_to_response(duplicate) except HTTPException: raise except Exception as e: logger.error(f"Failed to duplicate speaker profile: {e}") raise HTTPException( status_code=500, detail="Failed to duplicate speaker profile" ) ================================================ FILE: api/routers/transformations.py ================================================ from typing import List from fastapi import APIRouter, HTTPException from loguru import logger from api.models import ( DefaultPromptResponse, DefaultPromptUpdate, TransformationCreate, TransformationExecuteRequest, TransformationExecuteResponse, TransformationResponse, TransformationUpdate, ) from open_notebook.ai.models import Model from open_notebook.domain.transformation import DefaultPrompts, Transformation from open_notebook.exceptions import InvalidInputError, OpenNotebookError from open_notebook.graphs.transformation import graph as transformation_graph router = APIRouter() @router.get("/transformations", response_model=List[TransformationResponse]) async def get_transformations(): """Get all transformations.""" try: transformations = await Transformation.get_all(order_by="name asc") return [ TransformationResponse( id=transformation.id or "", name=transformation.name, title=transformation.title, description=transformation.description, prompt=transformation.prompt, apply_default=transformation.apply_default, created=str(transformation.created), updated=str(transformation.updated), ) for transformation in transformations ] except Exception as e: logger.error(f"Error fetching transformations: {str(e)}") raise HTTPException( status_code=500, detail=f"Error fetching transformations: {str(e)}" ) @router.post("/transformations", response_model=TransformationResponse) async def create_transformation(transformation_data: TransformationCreate): """Create a new transformation.""" try: new_transformation = Transformation( name=transformation_data.name, title=transformation_data.title, description=transformation_data.description, prompt=transformation_data.prompt, apply_default=transformation_data.apply_default, ) await new_transformation.save() return TransformationResponse( id=new_transformation.id or "", name=new_transformation.name, title=new_transformation.title, description=new_transformation.description, prompt=new_transformation.prompt, apply_default=new_transformation.apply_default, created=str(new_transformation.created), updated=str(new_transformation.updated), ) except InvalidInputError as e: raise HTTPException(status_code=400, detail=str(e)) except Exception as e: logger.error(f"Error creating transformation: {str(e)}") raise HTTPException( status_code=500, detail=f"Error creating transformation: {str(e)}" ) @router.post("/transformations/execute", response_model=TransformationExecuteResponse) async def execute_transformation(execute_request: TransformationExecuteRequest): """Execute a transformation on input text.""" try: # Validate transformation exists transformation = await Transformation.get(execute_request.transformation_id) if not transformation: raise HTTPException(status_code=404, detail="Transformation not found") # Validate model exists model = await Model.get(execute_request.model_id) if not model: raise HTTPException(status_code=404, detail="Model not found") # Execute the transformation result = await transformation_graph.ainvoke( dict( # type: ignore[arg-type] input_text=execute_request.input_text, transformation=transformation, ), config=dict(configurable={"model_id": execute_request.model_id}), ) return TransformationExecuteResponse( output=result["output"], transformation_id=execute_request.transformation_id, model_id=execute_request.model_id, ) except HTTPException: raise except OpenNotebookError: raise # Let global exception handlers return proper status codes except Exception as e: logger.error(f"Error executing transformation: {str(e)}") raise HTTPException( status_code=500, detail=f"Error executing transformation: {str(e)}" ) @router.get("/transformations/default-prompt", response_model=DefaultPromptResponse) async def get_default_prompt(): """Get the default transformation prompt.""" try: default_prompts: DefaultPrompts = await DefaultPrompts.get_instance() # type: ignore[assignment] return DefaultPromptResponse( transformation_instructions=default_prompts.transformation_instructions or "" ) except Exception as e: logger.error(f"Error fetching default prompt: {str(e)}") raise HTTPException( status_code=500, detail=f"Error fetching default prompt: {str(e)}" ) @router.put("/transformations/default-prompt", response_model=DefaultPromptResponse) async def update_default_prompt(prompt_update: DefaultPromptUpdate): """Update the default transformation prompt.""" try: default_prompts: DefaultPrompts = await DefaultPrompts.get_instance() # type: ignore[assignment] default_prompts.transformation_instructions = ( prompt_update.transformation_instructions ) await default_prompts.update() return DefaultPromptResponse( transformation_instructions=default_prompts.transformation_instructions ) except Exception as e: logger.error(f"Error updating default prompt: {str(e)}") raise HTTPException( status_code=500, detail=f"Error updating default prompt: {str(e)}" ) @router.get( "/transformations/{transformation_id}", response_model=TransformationResponse ) async def get_transformation(transformation_id: str): """Get a specific transformation by ID.""" try: transformation = await Transformation.get(transformation_id) if not transformation: raise HTTPException(status_code=404, detail="Transformation not found") return TransformationResponse( id=transformation.id or "", name=transformation.name, title=transformation.title, description=transformation.description, prompt=transformation.prompt, apply_default=transformation.apply_default, created=str(transformation.created), updated=str(transformation.updated), ) except HTTPException: raise except Exception as e: logger.error(f"Error fetching transformation {transformation_id}: {str(e)}") raise HTTPException( status_code=500, detail=f"Error fetching transformation: {str(e)}" ) @router.put( "/transformations/{transformation_id}", response_model=TransformationResponse ) async def update_transformation( transformation_id: str, transformation_update: TransformationUpdate ): """Update a transformation.""" try: transformation = await Transformation.get(transformation_id) if not transformation: raise HTTPException(status_code=404, detail="Transformation not found") # Update only provided fields if transformation_update.name is not None: transformation.name = transformation_update.name if transformation_update.title is not None: transformation.title = transformation_update.title if transformation_update.description is not None: transformation.description = transformation_update.description if transformation_update.prompt is not None: transformation.prompt = transformation_update.prompt if transformation_update.apply_default is not None: transformation.apply_default = transformation_update.apply_default await transformation.save() return TransformationResponse( id=transformation.id or "", name=transformation.name, title=transformation.title, description=transformation.description, prompt=transformation.prompt, apply_default=transformation.apply_default, created=str(transformation.created), updated=str(transformation.updated), ) except HTTPException: raise except InvalidInputError as e: raise HTTPException(status_code=400, detail=str(e)) except Exception as e: logger.error(f"Error updating transformation {transformation_id}: {str(e)}") raise HTTPException( status_code=500, detail=f"Error updating transformation: {str(e)}" ) @router.delete("/transformations/{transformation_id}") async def delete_transformation(transformation_id: str): """Delete a transformation.""" try: transformation = await Transformation.get(transformation_id) if not transformation: raise HTTPException(status_code=404, detail="Transformation not found") await transformation.delete() return {"message": "Transformation deleted successfully"} except HTTPException: raise except Exception as e: logger.error(f"Error deleting transformation {transformation_id}: {str(e)}") raise HTTPException( status_code=500, detail=f"Error deleting transformation: {str(e)}" ) ================================================ FILE: api/search_service.py ================================================ """ Search service layer using API. """ from typing import Any, Dict, List, Union from loguru import logger from api.client import api_client class SearchService: """Service layer for search operations using API.""" def __init__(self): logger.info("Using API for search operations") def search( self, query: str, search_type: str = "text", limit: int = 100, search_sources: bool = True, search_notes: bool = True, minimum_score: float = 0.2, ) -> List[Dict[str, Any]]: """Search the knowledge base.""" response = api_client.search( query=query, search_type=search_type, limit=limit, search_sources=search_sources, search_notes=search_notes, minimum_score=minimum_score, ) if isinstance(response, dict): return response.get("results", []) return [] def ask_knowledge_base( self, question: str, strategy_model: str, answer_model: str, final_answer_model: str, ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]: """Ask the knowledge base a question.""" response = api_client.ask_simple( question=question, strategy_model=strategy_model, answer_model=answer_model, final_answer_model=final_answer_model, ) return response # Global service instance search_service = SearchService() ================================================ FILE: api/settings_service.py ================================================ """ Settings service layer using API. """ from loguru import logger from api.client import api_client from open_notebook.domain.content_settings import ContentSettings class SettingsService: """Service layer for settings operations using API.""" def __init__(self): logger.info("Using API for settings operations") def get_settings(self) -> ContentSettings: """Get application settings.""" settings_response = api_client.get_settings() settings_data = ( settings_response if isinstance(settings_response, dict) else settings_response[0] ) # Create ContentSettings object from API response settings = ContentSettings( default_content_processing_engine_doc=settings_data.get( "default_content_processing_engine_doc" ), default_content_processing_engine_url=settings_data.get( "default_content_processing_engine_url" ), default_embedding_option=settings_data.get("default_embedding_option"), auto_delete_files=settings_data.get("auto_delete_files"), youtube_preferred_languages=settings_data.get( "youtube_preferred_languages" ), ) return settings def update_settings(self, settings: ContentSettings) -> ContentSettings: """Update application settings.""" updates = { "default_content_processing_engine_doc": settings.default_content_processing_engine_doc, "default_content_processing_engine_url": settings.default_content_processing_engine_url, "default_embedding_option": settings.default_embedding_option, "auto_delete_files": settings.auto_delete_files, "youtube_preferred_languages": settings.youtube_preferred_languages, } settings_response = api_client.update_settings(**updates) settings_data = ( settings_response if isinstance(settings_response, dict) else settings_response[0] ) # Update the settings object with the response settings.default_content_processing_engine_doc = settings_data.get( "default_content_processing_engine_doc" ) settings.default_content_processing_engine_url = settings_data.get( "default_content_processing_engine_url" ) settings.default_embedding_option = settings_data.get( "default_embedding_option" ) settings.auto_delete_files = settings_data.get("auto_delete_files") settings.youtube_preferred_languages = settings_data.get( "youtube_preferred_languages" ) return settings # Global service instance settings_service = SettingsService() ================================================ FILE: api/sources_service.py ================================================ """ Sources service layer using API. """ from dataclasses import dataclass from typing import Dict, List, Optional, Union from loguru import logger from api.client import api_client from open_notebook.domain.notebook import Asset, Source @dataclass class SourceProcessingResult: """Result of source creation with optional async processing info.""" source: Source is_async: bool = False command_id: Optional[str] = None status: Optional[str] = None processing_info: Optional[Dict] = None @dataclass class SourceWithMetadata: """Source object with additional metadata from API.""" source: Source embedded_chunks: int # Expose common source properties for easy access @property def id(self): return self.source.id @property def title(self): return self.source.title @title.setter def title(self, value): self.source.title = value @property def topics(self): return self.source.topics @property def asset(self): return self.source.asset @property def full_text(self): return self.source.full_text @property def created(self): return self.source.created @property def updated(self): return self.source.updated class SourcesService: """Service layer for sources operations using API.""" def __init__(self): logger.info("Using API for sources operations") def get_all_sources( self, notebook_id: Optional[str] = None ) -> List[SourceWithMetadata]: """Get all sources with optional notebook filtering.""" sources_data = api_client.get_sources(notebook_id=notebook_id) # Convert API response to SourceWithMetadata objects sources = [] for source_data in sources_data: source = Source( title=source_data["title"], topics=source_data["topics"], asset=Asset( file_path=source_data["asset"]["file_path"] if source_data["asset"] else None, url=source_data["asset"]["url"] if source_data["asset"] else None, ) if source_data["asset"] else None, ) source.id = source_data["id"] source.created = source_data["created"] source.updated = source_data["updated"] # Wrap in SourceWithMetadata source_with_metadata = SourceWithMetadata( source=source, embedded_chunks=source_data.get("embedded_chunks", 0) ) sources.append(source_with_metadata) return sources def get_source(self, source_id: str) -> SourceWithMetadata: """Get a specific source.""" response = api_client.get_source(source_id) source_data = response if isinstance(response, dict) else response[0] source = Source( title=source_data["title"], topics=source_data["topics"], full_text=source_data["full_text"], asset=Asset( file_path=source_data["asset"]["file_path"] if source_data["asset"] else None, url=source_data["asset"]["url"] if source_data["asset"] else None, ) if source_data["asset"] else None, ) source.id = source_data["id"] source.created = source_data["created"] source.updated = source_data["updated"] return SourceWithMetadata( source=source, embedded_chunks=source_data.get("embedded_chunks", 0) ) def create_source( self, notebook_id: Optional[str] = None, source_type: str = "text", url: Optional[str] = None, file_path: Optional[str] = None, content: Optional[str] = None, title: Optional[str] = None, transformations: Optional[List[str]] = None, embed: bool = False, delete_source: bool = False, notebooks: Optional[List[str]] = None, async_processing: bool = False, ) -> Union[Source, SourceProcessingResult]: """ Create a new source with support for async processing. Args: notebook_id: Single notebook ID (deprecated, use notebooks parameter) source_type: Type of source (link, upload, text) url: URL for link sources file_path: File path for upload sources content: Text content for text sources title: Optional source title transformations: List of transformation IDs to apply embed: Whether to embed content for vector search delete_source: Whether to delete uploaded file after processing notebooks: List of notebook IDs to add source to (preferred over notebook_id) async_processing: Whether to process source asynchronously Returns: Source object for sync processing (backward compatibility) SourceProcessingResult for async processing (contains additional metadata) """ source_data = api_client.create_source( notebook_id=notebook_id, notebooks=notebooks, source_type=source_type, url=url, file_path=file_path, content=content, title=title, transformations=transformations, embed=embed, delete_source=delete_source, async_processing=async_processing, ) # Create Source object from response response_data = source_data if isinstance(source_data, dict) else source_data[0] source = Source( title=response_data["title"], topics=response_data.get("topics") or [], full_text=response_data.get("full_text"), asset=Asset( file_path=response_data["asset"]["file_path"] if response_data.get("asset") else None, url=response_data["asset"]["url"] if response_data.get("asset") else None, ) if response_data.get("asset") else None, ) source.id = response_data["id"] source.created = response_data["created"] source.updated = response_data["updated"] # Check if this is an async processing response if ( response_data.get("command_id") or response_data.get("status") or response_data.get("processing_info") ): # Ensure source_data is a dict for accessing attributes source_data_dict = ( source_data if isinstance(source_data, dict) else source_data[0] ) # Return enhanced result for async processing return SourceProcessingResult( source=source, is_async=True, command_id=source_data_dict.get("command_id"), status=source_data_dict.get("status"), processing_info=source_data_dict.get("processing_info"), ) else: # Return simple Source for backward compatibility return source def get_source_status(self, source_id: str) -> Dict: """Get processing status for a source.""" response = api_client.get_source_status(source_id) return response if isinstance(response, dict) else response[0] def create_source_async( self, notebook_id: Optional[str] = None, source_type: str = "text", url: Optional[str] = None, file_path: Optional[str] = None, content: Optional[str] = None, title: Optional[str] = None, transformations: Optional[List[str]] = None, embed: bool = False, delete_source: bool = False, notebooks: Optional[List[str]] = None, ) -> SourceProcessingResult: """ Create a new source with async processing enabled. This is a convenience method that always uses async processing. Returns a SourceProcessingResult with processing status information. """ result = self.create_source( notebook_id=notebook_id, notebooks=notebooks, source_type=source_type, url=url, file_path=file_path, content=content, title=title, transformations=transformations, embed=embed, delete_source=delete_source, async_processing=True, ) # Since we forced async_processing=True, this should always be a SourceProcessingResult if isinstance(result, SourceProcessingResult): return result else: # Fallback: wrap Source in SourceProcessingResult return SourceProcessingResult( source=result, is_async=False, # This shouldn't happen, but handle it gracefully ) def is_source_processing_complete(self, source_id: str) -> bool: """ Check if a source's async processing is complete. Returns True if processing is complete (success or failure), False if still processing or queued. """ try: status_data = self.get_source_status(source_id) status = status_data.get("status") return status in [ "completed", "failed", None, ] # None indicates legacy/sync source except Exception as e: logger.error(f"Error checking source processing status: {e}") return True # Assume complete on error def update_source(self, source: Source) -> Source: """Update a source.""" if not source.id: raise ValueError("Source ID is required for update") updates = { "title": source.title, "topics": source.topics, } source_data = api_client.update_source(source.id, **updates) # Ensure source_data is a dict source_data_dict = ( source_data if isinstance(source_data, dict) else source_data[0] ) # Update the source object with the response source.title = source_data_dict["title"] source.topics = source_data_dict["topics"] source.updated = source_data_dict["updated"] return source def delete_source(self, source_id: str) -> bool: """Delete a source.""" api_client.delete_source(source_id) return True # Global service instance sources_service = SourcesService() # Export important classes for easy importing __all__ = [ "SourcesService", "SourceWithMetadata", "SourceProcessingResult", "sources_service", ] ================================================ FILE: api/transformations_service.py ================================================ """ Transformations service layer using API. """ from datetime import datetime from typing import Any, Dict, List, Union from loguru import logger from api.client import api_client from open_notebook.domain.transformation import Transformation class TransformationsService: """Service layer for transformations operations using API.""" def __init__(self): logger.info("Using API for transformations operations") def get_all_transformations(self) -> List[Transformation]: """Get all transformations.""" transformations_data = api_client.get_transformations() # Convert API response to Transformation objects transformations = [] for trans_data in transformations_data: transformation = Transformation( name=trans_data["name"], title=trans_data["title"], description=trans_data["description"], prompt=trans_data["prompt"], apply_default=trans_data["apply_default"], ) transformation.id = trans_data["id"] transformation.created = datetime.fromisoformat( trans_data["created"].replace("Z", "+00:00") ) transformation.updated = datetime.fromisoformat( trans_data["updated"].replace("Z", "+00:00") ) transformations.append(transformation) return transformations def get_transformation(self, transformation_id: str) -> Transformation: """Get a specific transformation.""" response = api_client.get_transformation(transformation_id) trans_data = response if isinstance(response, dict) else response[0] transformation = Transformation( name=trans_data["name"], title=trans_data["title"], description=trans_data["description"], prompt=trans_data["prompt"], apply_default=trans_data["apply_default"], ) transformation.id = trans_data["id"] transformation.created = datetime.fromisoformat( trans_data["created"].replace("Z", "+00:00") ) transformation.updated = datetime.fromisoformat( trans_data["updated"].replace("Z", "+00:00") ) return transformation def create_transformation( self, name: str, title: str, description: str, prompt: str, apply_default: bool = False, ) -> Transformation: """Create a new transformation.""" response = api_client.create_transformation( name=name, title=title, description=description, prompt=prompt, apply_default=apply_default, ) trans_data = response if isinstance(response, dict) else response[0] transformation = Transformation( name=trans_data["name"], title=trans_data["title"], description=trans_data["description"], prompt=trans_data["prompt"], apply_default=trans_data["apply_default"], ) transformation.id = trans_data["id"] transformation.created = datetime.fromisoformat( trans_data["created"].replace("Z", "+00:00") ) transformation.updated = datetime.fromisoformat( trans_data["updated"].replace("Z", "+00:00") ) return transformation def update_transformation(self, transformation: Transformation) -> Transformation: """Update a transformation.""" if not transformation.id: raise ValueError("Transformation ID is required for update") updates = { "name": transformation.name, "title": transformation.title, "description": transformation.description, "prompt": transformation.prompt, "apply_default": transformation.apply_default, } response = api_client.update_transformation(transformation.id, **updates) trans_data = response if isinstance(response, dict) else response[0] # Update the transformation object with the response transformation.name = trans_data["name"] transformation.title = trans_data["title"] transformation.description = trans_data["description"] transformation.prompt = trans_data["prompt"] transformation.apply_default = trans_data["apply_default"] transformation.updated = datetime.fromisoformat( trans_data["updated"].replace("Z", "+00:00") ) return transformation def delete_transformation(self, transformation_id: str) -> bool: """Delete a transformation.""" api_client.delete_transformation(transformation_id) return True def execute_transformation( self, transformation_id: str, input_text: str, model_id: str ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]: """Execute a transformation on input text.""" result = api_client.execute_transformation( transformation_id=transformation_id, input_text=input_text, model_id=model_id, ) return result # Global service instance transformations_service = TransformationsService() ================================================ FILE: commands/CLAUDE.md ================================================ # Commands Module **Purpose**: Defines async command handlers for long-running operations via `surreal-commands` job queue system. ## Key Components ### Embedding Commands - **`embed_note_command`**: Embeds a single note using unified embedding pipeline with content-type aware processing. Uses MARKDOWN content type detection. Retry: 5 attempts, exponential jitter 1-60s. - **`embed_insight_command`**: Embeds a single source insight. Uses MARKDOWN content type. Retry: 5 attempts, exponential jitter 1-60s. - **`embed_source_command`**: Embeds a source by chunking full_text with content-type aware splitters (HTML, Markdown, plain), then batch embedding all chunks (batches of 50 with per-batch retry). Retry: 5 attempts, exponential jitter 1-60s. - **`create_insight_command`**: Creates a source insight with automatic retry on transaction conflicts. Creates the DB record, then submits `embed_insight` command (fire-and-forget). Retry: 5 attempts, exponential jitter 1-60s. Used by `Source.add_insight()`. - **`rebuild_embeddings_command`**: Submits individual embed_* commands for all sources/notes/insights. Returns immediately; actual embedding happens async. No retry (coordinator only). ### Other Commands - **`process_source_command`**: Ingests content through `source_graph`, creates embeddings (optional), and generates insights. Retries on transaction conflicts (exp. jitter, max 15×, 1-120s). - **`run_transformation_command`**: Runs a transformation on an existing source to generate an insight. Executes the transformation graph (LLM call) then creates insight via `create_insight_command`. Used by `POST /sources/{id}/insights` API endpoint. Retry: 5 attempts, exponential jitter 1-60s. - **`generate_podcast_command`**: Creates podcasts via podcast-creator library. Resolves model registry references and credentials for all profiles before invoking podcast-creator. Validates that outline_llm, transcript_llm, and voice_model are configured. - **`process_text_command`** (example): Test fixture for text operations (uppercase, lowercase, reverse, word_count). - **`analyze_data_command`** (example): Test fixture for numeric aggregations. ## Important Patterns - **Pydantic I/O**: All commands use `CommandInput`/`CommandOutput` subclasses for type safety and serialization. - **Error handling**: Permanent errors (ValueError) return failure output; all other exceptions auto-retry via surreal-commands. - **Retry configuration**: Uses `stop_on: [ValueError]` (blocklist approach) - retries all exceptions EXCEPT ValueError. This is more resilient than allowlist as new exception types auto-retry. - **Fire-and-forget embedding**: Domain models submit embed_* commands via `submit_command()` without waiting. Commands process asynchronously. - **Content-type aware chunking**: `embed_source_command` uses `chunk_text()` with automatic content type detection (HTML, Markdown, plain text) for optimal text splitting. Default: 1500 char chunks with 225 char overlap. - **Batch embedding**: `embed_source_command` uses `generate_embeddings()` which automatically batches texts (default 50) with per-batch retry to avoid exceeding provider payload limits. - **Mean pooling for large content**: `embed_note_command` and `embed_insight_command` use `generate_embedding()` which handles content larger than chunk size via mean pooling. - **Model dumping**: Recursive `full_model_dump()` utility converts Pydantic models → dicts for DB/API responses. - **Logging**: Uses `loguru.logger` throughout; logs execution start/end and key metrics (processing time, counts). - **Time tracking**: All commands measure `start_time` → `processing_time` for monitoring. ## Dependencies **External**: `surreal_commands` (command decorator, job queue, submit_command), `loguru`, `pydantic`, `podcast_creator` **Internal**: `open_notebook.domain.notebook` (Source, Note, SourceInsight), `open_notebook.utils.chunking` (chunk_text, detect_content_type), `open_notebook.utils.embedding` (generate_embedding, generate_embeddings), `open_notebook.database.repository` (repo_query, repo_insert) ## Quirks & Edge Cases - **source_commands**: `ensure_record_id()` wraps command IDs for DB storage; transaction conflicts trigger exponential backoff retry. ValueError exceptions are permanent (not retried). - **embedding_commands**: Content type detection uses file extension as primary source, heuristics as fallback. Chunks >1800 chars trigger secondary splitting. Empty/whitespace-only content returns ValueError (not retried). - **rebuild_embeddings_command**: Returns "jobs_submitted" not "processed_items" - embedding is async. Individual commands handle failures with their own retries. - **podcast_commands**: Profiles loaded from SurrealDB by name; model configs (credentials) resolved for ALL profiles before podcast-creator validation. Validates outline_llm/transcript_llm/voice_model are set. Episode records created mid-execution. - **Example commands**: Accept optional `delay_seconds` for testing async behavior; not for production. ## Code Example ```python @command("process_source", app="open_notebook", retry={ "max_attempts": 5, "wait_strategy": "exponential_jitter", "stop_on": [ValueError], # Don't retry validation errors }) async def process_source_command(input_data: SourceProcessingInput) -> SourceProcessingOutput: start_time = time.time() try: transformations = [await Transformation.get(id) for id in input_data.transformations] source = await Source.get(input_data.source_id) result = await source_graph.ainvoke({...}) return SourceProcessingOutput(success=True, ...) except ValueError as e: return SourceProcessingOutput(success=False, error_message=str(e)) # No retry except Exception as e: raise # Retry all other exceptions ``` ================================================ FILE: commands/__init__.py ================================================ """Surreal-commands integration for Open Notebook""" from .embedding_commands import ( embed_insight_command, embed_note_command, embed_source_command, rebuild_embeddings_command, ) from .example_commands import analyze_data_command, process_text_command from .podcast_commands import generate_podcast_command from .source_commands import process_source_command __all__ = [ # Embedding commands "embed_note_command", "embed_insight_command", "embed_source_command", "rebuild_embeddings_command", # Other commands "generate_podcast_command", "process_source_command", "process_text_command", "analyze_data_command", ] ================================================ FILE: commands/embedding_commands.py ================================================ import time from typing import Dict, List, Literal, Optional from loguru import logger from pydantic import BaseModel from surreal_commands import CommandInput, CommandOutput, command, submit_command from open_notebook.ai.models import model_manager from open_notebook.database.repository import ensure_record_id, repo_insert, repo_query from open_notebook.exceptions import ConfigurationError from open_notebook.domain.notebook import Note, Source, SourceInsight from open_notebook.utils.chunking import ContentType, chunk_text, detect_content_type from open_notebook.utils.embedding import generate_embedding, generate_embeddings def full_model_dump(model): if isinstance(model, BaseModel): return model.model_dump() elif isinstance(model, dict): return {k: full_model_dump(v) for k, v in model.items()} elif isinstance(model, list): return [full_model_dump(item) for item in model] else: return model def get_command_id(input_data: CommandInput) -> str: """Extract command_id from input_data's execution context, or return 'unknown'.""" if input_data.execution_context: return str(input_data.execution_context.command_id) return "unknown" class RebuildEmbeddingsInput(CommandInput): mode: Literal["existing", "all"] include_sources: bool = True include_notes: bool = True include_insights: bool = True class RebuildEmbeddingsOutput(CommandOutput): success: bool total_items: int jobs_submitted: int # Count of embedding commands submitted failed_submissions: int # Count of items that failed to submit sources_submitted: int = 0 notes_submitted: int = 0 insights_submitted: int = 0 processing_time: float error_message: Optional[str] = None # ============================================================================= # NEW EMBEDDING COMMANDS (Phase 3) # ============================================================================= class CreateInsightInput(CommandInput): """Input for creating a source insight with automatic retry on conflicts.""" source_id: str insight_type: str content: str class CreateInsightOutput(CommandOutput): """Output from insight creation command.""" success: bool insight_id: Optional[str] = None processing_time: float error_message: Optional[str] = None class EmbedNoteInput(CommandInput): """Input for embedding a single note.""" note_id: str class EmbedNoteOutput(CommandOutput): """Output from note embedding command.""" success: bool note_id: str processing_time: float error_message: Optional[str] = None class EmbedInsightInput(CommandInput): """Input for embedding a single source insight.""" insight_id: str class EmbedInsightOutput(CommandOutput): """Output from insight embedding command.""" success: bool insight_id: str processing_time: float error_message: Optional[str] = None class EmbedSourceInput(CommandInput): """Input for embedding a source (creates multiple chunk embeddings).""" source_id: str class EmbedSourceOutput(CommandOutput): """Output from source embedding command.""" success: bool source_id: str chunks_created: int processing_time: float error_message: Optional[str] = None @command( "embed_note", app="open_notebook", retry={ "max_attempts": 5, "wait_strategy": "exponential_jitter", "wait_min": 1, "wait_max": 60, "stop_on": [ValueError, ConfigurationError], # Don't retry validation/config errors "retry_log_level": "debug", }, ) async def embed_note_command(input_data: EmbedNoteInput) -> EmbedNoteOutput: """ Generate and store embedding for a single note. Uses the unified embedding pipeline with automatic chunking and mean pooling for notes that exceed the chunk size limit. Flow: 1. Load Note by ID 2. Generate embedding via generate_embedding() (auto-chunks + mean pools if needed) 3. UPSERT note embedding in database Retry Strategy: - Retries up to 5 times for transient failures (network, timeout, etc.) - Uses exponential-jitter backoff (1-60s) - Does NOT retry permanent failures (ValueError for validation errors) """ start_time = time.time() try: logger.info(f"Starting embedding for note: {input_data.note_id}") # 1. Load note note = await Note.get(input_data.note_id) if not note: raise ValueError(f"Note '{input_data.note_id}' not found") if not note.content or not note.content.strip(): raise ValueError(f"Note '{input_data.note_id}' has no content to embed") # 2. Generate embedding (auto-chunks + mean pools if needed) # Notes are typically markdown content cmd_id = get_command_id(input_data) embedding = await generate_embedding( note.content, content_type=ContentType.MARKDOWN, command_id=cmd_id ) # 3. UPSERT embedding into note record await repo_query( "UPDATE $note_id SET embedding = $embedding", { "note_id": ensure_record_id(input_data.note_id), "embedding": embedding, }, ) processing_time = time.time() - start_time logger.info( f"Successfully embedded note {input_data.note_id} in {processing_time:.2f}s" ) return EmbedNoteOutput( success=True, note_id=input_data.note_id, processing_time=processing_time, ) except ValueError as e: # Permanent failure - don't retry processing_time = time.time() - start_time cmd_id = get_command_id(input_data) logger.error( f"Failed to embed note {input_data.note_id} (command: {cmd_id}): {e}" ) return EmbedNoteOutput( success=False, note_id=input_data.note_id, processing_time=processing_time, error_message=str(e), ) except Exception as e: # Transient failure - will be retried (surreal-commands logs final failure) cmd_id = get_command_id(input_data) logger.debug( f"Transient error embedding note {input_data.note_id} " f"(command: {cmd_id}): {e}" ) raise @command( "embed_insight", app="open_notebook", retry={ "max_attempts": 5, "wait_strategy": "exponential_jitter", "wait_min": 1, "wait_max": 60, "stop_on": [ValueError, ConfigurationError], # Don't retry validation/config errors "retry_log_level": "debug", }, ) async def embed_insight_command(input_data: EmbedInsightInput) -> EmbedInsightOutput: """ Generate and store embedding for a single source insight. Uses the unified embedding pipeline with automatic chunking and mean pooling for insights that exceed the chunk size limit. Flow: 1. Load SourceInsight by ID 2. Generate embedding via generate_embedding() (auto-chunks + mean pools if needed) 3. UPSERT insight embedding in database Retry Strategy: - Retries up to 5 times for transient failures (network, timeout, etc.) - Uses exponential-jitter backoff (1-60s) - Does NOT retry permanent failures (ValueError for validation errors) """ start_time = time.time() try: logger.info(f"Starting embedding for insight: {input_data.insight_id}") # 1. Load insight insight = await SourceInsight.get(input_data.insight_id) if not insight: raise ValueError(f"Insight '{input_data.insight_id}' not found") if not insight.content or not insight.content.strip(): raise ValueError( f"Insight '{input_data.insight_id}' has no content to embed" ) # 2. Generate embedding (auto-chunks + mean pools if needed) # Insights are typically markdown content (generated by LLM) cmd_id = get_command_id(input_data) embedding = await generate_embedding( insight.content, content_type=ContentType.MARKDOWN, command_id=cmd_id ) # 3. UPSERT embedding into insight record await repo_query( "UPDATE $insight_id SET embedding = $embedding", { "insight_id": ensure_record_id(input_data.insight_id), "embedding": embedding, }, ) processing_time = time.time() - start_time logger.info( f"Successfully embedded insight {input_data.insight_id} in {processing_time:.2f}s" ) return EmbedInsightOutput( success=True, insight_id=input_data.insight_id, processing_time=processing_time, ) except ValueError as e: # Permanent failure - don't retry processing_time = time.time() - start_time cmd_id = get_command_id(input_data) logger.error( f"Failed to embed insight {input_data.insight_id} (command: {cmd_id}): {e}" ) return EmbedInsightOutput( success=False, insight_id=input_data.insight_id, processing_time=processing_time, error_message=str(e), ) except Exception as e: # Transient failure - will be retried (surreal-commands logs final failure) cmd_id = get_command_id(input_data) logger.debug( f"Transient error embedding insight {input_data.insight_id} " f"(command: {cmd_id}): {e}" ) raise @command( "embed_source", app="open_notebook", retry={ "max_attempts": 5, "wait_strategy": "exponential_jitter", "wait_min": 1, "wait_max": 60, "stop_on": [ValueError, ConfigurationError], # Don't retry validation/config errors "retry_log_level": "debug", }, ) async def embed_source_command(input_data: EmbedSourceInput) -> EmbedSourceOutput: """ Generate and store embeddings for a source document. Creates multiple chunk embeddings stored in the source_embedding table. Uses content-type aware chunking based on file extension or content heuristics. Flow: 1. Load Source by ID 2. DELETE existing source_embedding records for this source 3. Detect content type from file path or content 4. Chunk text using appropriate splitter 5. Generate embeddings for all chunks in batches 6. Bulk INSERT source_embedding records Retry Strategy: - Retries up to 5 times for transient failures (network, timeout, etc.) - Uses exponential-jitter backoff (1-60s) - Does NOT retry permanent failures (ValueError for validation errors) """ start_time = time.time() try: logger.info(f"Starting embedding for source: {input_data.source_id}") # 1. Load source source = await Source.get(input_data.source_id) if not source: raise ValueError(f"Source '{input_data.source_id}' not found") if not source.full_text or not source.full_text.strip(): raise ValueError(f"Source '{input_data.source_id}' has no text to embed") # 2. DELETE existing embeddings (idempotency) logger.debug(f"Deleting existing embeddings for source {input_data.source_id}") await repo_query( "DELETE source_embedding WHERE source = $source_id", {"source_id": ensure_record_id(input_data.source_id)}, ) # 3. Detect content type from file path if available file_path = source.asset.file_path if source.asset else None content_type = detect_content_type(source.full_text, file_path) logger.debug(f"Detected content type: {content_type.value}") # 4. Chunk text using appropriate splitter chunks = chunk_text(source.full_text, content_type=content_type) total_chunks = len(chunks) # Log chunk statistics for debugging chunk_sizes = [len(c) for c in chunks] logger.info( f"Created {total_chunks} chunks for source {input_data.source_id} " f"(sizes: min={min(chunk_sizes) if chunk_sizes else 0}, " f"max={max(chunk_sizes) if chunk_sizes else 0}, " f"avg={sum(chunk_sizes)//len(chunk_sizes) if chunk_sizes else 0} chars)" ) if total_chunks == 0: raise ValueError("No chunks created after splitting text") # 5. Generate embeddings for all chunks in batches cmd_id = get_command_id(input_data) logger.debug(f"Generating embeddings for {total_chunks} chunks") embeddings = await generate_embeddings(chunks, command_id=cmd_id) # Verify we got embeddings for all chunks if len(embeddings) != len(chunks): raise ValueError( f"Embedding count mismatch: got {len(embeddings)} embeddings " f"for {len(chunks)} chunks" ) # 6. Bulk INSERT source_embedding records records = [ { "source": ensure_record_id(input_data.source_id), "order": idx, "content": chunk, "embedding": embedding, } for idx, (chunk, embedding) in enumerate(zip(chunks, embeddings)) ] logger.debug(f"Inserting {len(records)} source_embedding records") await repo_insert("source_embedding", records) processing_time = time.time() - start_time logger.info( f"Successfully embedded source {input_data.source_id}: " f"{total_chunks} chunks in {processing_time:.2f}s" ) return EmbedSourceOutput( success=True, source_id=input_data.source_id, chunks_created=total_chunks, processing_time=processing_time, ) except ValueError as e: # Permanent failure - don't retry processing_time = time.time() - start_time cmd_id = get_command_id(input_data) logger.error( f"Failed to embed source {input_data.source_id} (command: {cmd_id}): {e}" ) return EmbedSourceOutput( success=False, source_id=input_data.source_id, chunks_created=0, processing_time=processing_time, error_message=str(e), ) except Exception as e: # Transient failure - will be retried (surreal-commands logs final failure) cmd_id = get_command_id(input_data) logger.debug( f"Transient error embedding source {input_data.source_id} " f"(command: {cmd_id}): {e}" ) raise @command( "create_insight", app="open_notebook", retry={ "max_attempts": 5, "wait_strategy": "exponential_jitter", "wait_min": 1, "wait_max": 60, "stop_on": [ValueError, ConfigurationError], # Don't retry validation/config errors "retry_log_level": "debug", }, ) async def create_insight_command( input_data: CreateInsightInput, ) -> CreateInsightOutput: """ Create a source insight with automatic retry on transaction conflicts. This command wraps the CREATE source_insight operation with retry logic to handle SurrealDB transaction conflicts that occur during batch imports when multiple parallel transformations try to create insights concurrently. Flow: 1. CREATE source_insight record in database 2. Submit embed_insight command (fire-and-forget) for async embedding 3. Return the insight_id Retry Strategy: - Retries up to 5 times for transient failures (network, timeout, etc.) - Uses exponential-jitter backoff (1-60s) - Does NOT retry permanent failures (ValueError for validation errors) """ start_time = time.time() try: logger.info( f"Creating insight for source {input_data.source_id}: " f"type={input_data.insight_type}" ) # 1. Create insight record in database result = await repo_query( """ CREATE source_insight CONTENT { "source": $source_id, "insight_type": $insight_type, "content": $content }; """, { "source_id": ensure_record_id(input_data.source_id), "insight_type": input_data.insight_type, "content": input_data.content, }, ) if not result or len(result) == 0: raise ValueError("Failed to create insight - no result returned") insight_id = str(result[0].get("id", "")) if not insight_id: raise ValueError("Failed to create insight - no ID in result") # 2. Submit embedding command (fire-and-forget) submit_command( "open_notebook", "embed_insight", {"insight_id": insight_id}, ) logger.debug(f"Submitted embed_insight command for {insight_id}") processing_time = time.time() - start_time logger.info( f"Successfully created insight {insight_id} for source " f"{input_data.source_id} in {processing_time:.2f}s" ) return CreateInsightOutput( success=True, insight_id=insight_id, processing_time=processing_time, ) except ValueError as e: # Permanent failure - don't retry processing_time = time.time() - start_time cmd_id = get_command_id(input_data) logger.error( f"Failed to create insight for source {input_data.source_id} " f"(command: {cmd_id}): {e}" ) return CreateInsightOutput( success=False, processing_time=processing_time, error_message=str(e), ) except Exception as e: # Transient failure - will be retried (surreal-commands logs final failure) cmd_id = get_command_id(input_data) logger.debug( f"Transient error creating insight for source {input_data.source_id} " f"(command: {cmd_id}): {e}" ) raise async def collect_items_for_rebuild( mode: str, include_sources: bool, include_notes: bool, include_insights: bool, ) -> Dict[str, List[str]]: """ Collect items to rebuild based on mode and include flags. Returns: Dict with keys: 'sources', 'notes', 'insights' containing lists of item IDs """ items: Dict[str, List[str]] = {"sources": [], "notes": [], "insights": []} if include_sources: if mode == "existing": # Query sources with embeddings (via source_embedding table) result = await repo_query( """ RETURN array::distinct( SELECT VALUE source.id FROM source_embedding WHERE embedding != none AND array::len(embedding) > 0 ) """ ) # RETURN returns the array directly as the result (not nested) if result: items["sources"] = [str(item) for item in result] else: items["sources"] = [] else: # mode == "all" # Query all sources with non-empty content result = await repo_query( "SELECT id FROM source WHERE full_text != none AND string::trim(full_text) != ''" ) items["sources"] = [str(item["id"]) for item in result] if result else [] logger.info(f"Collected {len(items['sources'])} sources for rebuild") if include_notes: if mode == "existing": # Query notes with embeddings result = await repo_query( "SELECT id FROM note WHERE embedding != none AND array::len(embedding) > 0" ) else: # mode == "all" # Query all notes with non-empty content result = await repo_query( "SELECT id FROM note WHERE content != none AND string::trim(content) != ''" ) items["notes"] = [str(item["id"]) for item in result] if result else [] logger.info(f"Collected {len(items['notes'])} notes for rebuild") if include_insights: if mode == "existing": # Query insights with embeddings result = await repo_query( "SELECT id FROM source_insight WHERE embedding != none AND array::len(embedding) > 0" ) else: # mode == "all" # Query all insights with non-empty content result = await repo_query( "SELECT id FROM source_insight WHERE content != none AND string::trim(content) != ''" ) items["insights"] = [str(item["id"]) for item in result] if result else [] logger.info(f"Collected {len(items['insights'])} insights for rebuild") return items @command("rebuild_embeddings", app="open_notebook", retry=None) async def rebuild_embeddings_command( input_data: RebuildEmbeddingsInput, ) -> RebuildEmbeddingsOutput: """ Rebuild embeddings for sources, notes, and/or insights. This command submits individual embedding jobs for each item: - embed_source for sources - embed_note for notes - embed_insight for insights The command returns after submitting all jobs. Actual embedding happens asynchronously via the individual commands (which have their own retry strategies). Retry Strategy: - Retries disabled (retry=None) for this coordinator command - Individual embed_* commands handle their own retries """ start_time = time.time() try: logger.info("=" * 60) logger.info(f"Starting embedding rebuild with mode={input_data.mode}") logger.info( f"Include: sources={input_data.include_sources}, notes={input_data.include_notes}, insights={input_data.include_insights}" ) logger.info("=" * 60) # Check embedding model availability (fail fast) EMBEDDING_MODEL = await model_manager.get_embedding_model() if not EMBEDDING_MODEL: raise ValueError( "No embedding model configured. Please configure one in the Models section." ) logger.info(f"Embedding model configured: {EMBEDDING_MODEL}") # Collect items to process (returns IDs only) items = await collect_items_for_rebuild( input_data.mode, input_data.include_sources, input_data.include_notes, input_data.include_insights, ) total_items = ( len(items["sources"]) + len(items["notes"]) + len(items["insights"]) ) logger.info(f"Total items to rebuild: {total_items}") if total_items == 0: logger.warning("No items found to rebuild") return RebuildEmbeddingsOutput( success=True, total_items=0, jobs_submitted=0, failed_submissions=0, processing_time=time.time() - start_time, ) # Initialize counters sources_submitted = 0 notes_submitted = 0 insights_submitted = 0 failed_submissions = 0 # Submit embed_source commands for sources logger.info(f"\nSubmitting {len(items['sources'])} source embedding jobs...") for idx, source_id in enumerate(items["sources"], 1): try: submit_command( "open_notebook", "embed_source", {"source_id": source_id}, ) sources_submitted += 1 if idx % 50 == 0 or idx == len(items["sources"]): logger.info( f" Progress: {idx}/{len(items['sources'])} source jobs submitted" ) except Exception as e: logger.error(f"Failed to submit embed_source for {source_id}: {e}") failed_submissions += 1 # Submit embed_note commands for notes logger.info(f"\nSubmitting {len(items['notes'])} note embedding jobs...") for idx, note_id in enumerate(items["notes"], 1): try: submit_command( "open_notebook", "embed_note", {"note_id": note_id}, ) notes_submitted += 1 if idx % 50 == 0 or idx == len(items["notes"]): logger.info( f" Progress: {idx}/{len(items['notes'])} note jobs submitted" ) except Exception as e: logger.error(f"Failed to submit embed_note for {note_id}: {e}") failed_submissions += 1 # Submit embed_insight commands for insights logger.info(f"\nSubmitting {len(items['insights'])} insight embedding jobs...") for idx, insight_id in enumerate(items["insights"], 1): try: submit_command( "open_notebook", "embed_insight", {"insight_id": insight_id}, ) insights_submitted += 1 if idx % 50 == 0 or idx == len(items["insights"]): logger.info( f" Progress: {idx}/{len(items['insights'])} insight jobs submitted" ) except Exception as e: logger.error(f"Failed to submit embed_insight for {insight_id}: {e}") failed_submissions += 1 processing_time = time.time() - start_time jobs_submitted = sources_submitted + notes_submitted + insights_submitted logger.info("=" * 60) logger.info("REBUILD JOBS SUBMITTED") logger.info(f" Total jobs submitted: {jobs_submitted}/{total_items}") logger.info(f" Sources: {sources_submitted}") logger.info(f" Notes: {notes_submitted}") logger.info(f" Insights: {insights_submitted}") logger.info(f" Failed submissions: {failed_submissions}") logger.info(f" Submission time: {processing_time:.2f}s") logger.info(" Note: Actual embedding happens asynchronously") logger.info("=" * 60) return RebuildEmbeddingsOutput( success=True, total_items=total_items, jobs_submitted=jobs_submitted, failed_submissions=failed_submissions, sources_submitted=sources_submitted, notes_submitted=notes_submitted, insights_submitted=insights_submitted, processing_time=processing_time, ) except Exception as e: processing_time = time.time() - start_time logger.error(f"Rebuild embeddings failed: {e}") logger.exception(e) return RebuildEmbeddingsOutput( success=False, total_items=0, jobs_submitted=0, failed_submissions=0, processing_time=processing_time, error_message=str(e), ) ================================================ FILE: commands/example_commands.py ================================================ import asyncio import time from typing import List, Optional from loguru import logger from pydantic import BaseModel from surreal_commands import command class TextProcessingInput(BaseModel): text: str operation: str = "uppercase" # uppercase, lowercase, word_count, reverse delay_seconds: Optional[int] = None # For testing async behavior class TextProcessingOutput(BaseModel): success: bool original_text: str processed_text: Optional[str] = None word_count: Optional[int] = None processing_time: float error_message: Optional[str] = None class DataAnalysisInput(BaseModel): numbers: List[float] analysis_type: str = "basic" # basic, detailed delay_seconds: Optional[int] = None class DataAnalysisOutput(BaseModel): success: bool analysis_type: str count: int sum: Optional[float] = None average: Optional[float] = None min_value: Optional[float] = None max_value: Optional[float] = None processing_time: float error_message: Optional[str] = None @command("process_text", app="open_notebook") async def process_text_command(input_data: TextProcessingInput) -> TextProcessingOutput: """ Example command for text processing. Tests basic command functionality and demonstrates different processing types. """ start_time = time.time() try: logger.info(f"Processing text with operation: {input_data.operation}") # Simulate processing delay if specified if input_data.delay_seconds: await asyncio.sleep(input_data.delay_seconds) processed_text = None word_count = None if input_data.operation == "uppercase": processed_text = input_data.text.upper() elif input_data.operation == "lowercase": processed_text = input_data.text.lower() elif input_data.operation == "reverse": processed_text = input_data.text[::-1] elif input_data.operation == "word_count": word_count = len(input_data.text.split()) processed_text = f"Word count: {word_count}" else: raise ValueError(f"Unknown operation: {input_data.operation}") processing_time = time.time() - start_time return TextProcessingOutput( success=True, original_text=input_data.text, processed_text=processed_text, word_count=word_count, processing_time=processing_time, ) except Exception as e: processing_time = time.time() - start_time logger.error(f"Text processing failed: {e}") return TextProcessingOutput( success=False, original_text=input_data.text, processing_time=processing_time, error_message=str(e), ) @command("analyze_data", app="open_notebook") async def analyze_data_command(input_data: DataAnalysisInput) -> DataAnalysisOutput: """ Example command for data analysis. Tests command with complex input/output and demonstrates error handling. """ start_time = time.time() try: logger.info( f"Analyzing {len(input_data.numbers)} numbers with {input_data.analysis_type} analysis" ) # Simulate processing delay if specified if input_data.delay_seconds: await asyncio.sleep(input_data.delay_seconds) if not input_data.numbers: raise ValueError("No numbers provided for analysis") count = len(input_data.numbers) sum_value = sum(input_data.numbers) average = sum_value / count min_value = min(input_data.numbers) max_value = max(input_data.numbers) processing_time = time.time() - start_time return DataAnalysisOutput( success=True, analysis_type=input_data.analysis_type, count=count, sum=sum_value, average=average, min_value=min_value, max_value=max_value, processing_time=processing_time, ) except Exception as e: processing_time = time.time() - start_time logger.error(f"Data analysis failed: {e}") return DataAnalysisOutput( success=False, analysis_type=input_data.analysis_type, count=0, processing_time=processing_time, error_message=str(e), ) ================================================ FILE: commands/podcast_commands.py ================================================ import time import uuid from pathlib import Path from typing import Optional from loguru import logger from pydantic import BaseModel from surreal_commands import CommandInput, CommandOutput, command from open_notebook.config import DATA_FOLDER from open_notebook.database.repository import ensure_record_id, repo_query from open_notebook.podcasts.models import ( EpisodeProfile, PodcastEpisode, SpeakerProfile, _resolve_model_config, ) try: from podcast_creator import configure, create_podcast except ImportError as e: logger.error(f"Failed to import podcast_creator: {e}") raise ValueError("podcast_creator library not available") def build_episode_output_dir(data_folder: str) -> tuple[str, Path]: """Build a filesystem-safe output directory path for a podcast episode. Uses a UUID as the directory name so the path is safe regardless of what the user typed as episode name (spaces, special chars, etc.). Returns: A tuple of (episode_dir_name, output_dir_path). """ episode_dir_name = str(uuid.uuid4()) output_dir = Path(f"{data_folder}/podcasts/episodes/{episode_dir_name}") return episode_dir_name, output_dir def full_model_dump(model): if isinstance(model, BaseModel): return model.model_dump() elif isinstance(model, dict): return {k: full_model_dump(v) for k, v in model.items()} elif isinstance(model, list): return [full_model_dump(item) for item in model] else: return model class PodcastGenerationInput(CommandInput): episode_profile: str speaker_profile: str episode_name: str content: str briefing_suffix: Optional[str] = None class PodcastGenerationOutput(CommandOutput): success: bool episode_id: Optional[str] = None audio_file_path: Optional[str] = None transcript: Optional[dict] = None outline: Optional[dict] = None processing_time: float error_message: Optional[str] = None @command("generate_podcast", app="open_notebook", retry={"max_attempts": 1}) async def generate_podcast_command( input_data: PodcastGenerationInput, ) -> PodcastGenerationOutput: """ Real podcast generation using podcast-creator library with Episode Profiles """ start_time = time.time() try: logger.info( f"Starting podcast generation for episode: {input_data.episode_name}" ) logger.info(f"Using episode profile: {input_data.episode_profile}") # 1. Load Episode and Speaker profiles from SurrealDB episode_profile = await EpisodeProfile.get_by_name(input_data.episode_profile) if not episode_profile: raise ValueError( f"Episode profile '{input_data.episode_profile}' not found" ) speaker_profile = await SpeakerProfile.get_by_name( episode_profile.speaker_config ) if not speaker_profile: raise ValueError( f"Speaker profile '{episode_profile.speaker_config}' not found" ) logger.info(f"Loaded episode profile: {episode_profile.name}") logger.info(f"Loaded speaker profile: {speaker_profile.name}") # 2. Validate that model registry fields are populated if not episode_profile.outline_llm: raise ValueError( f"Episode profile '{episode_profile.name}' has no outline model configured. " "Please update the profile to select an outline model." ) if not episode_profile.transcript_llm: raise ValueError( f"Episode profile '{episode_profile.name}' has no transcript model configured. " "Please update the profile to select a transcript model." ) if not speaker_profile.voice_model: raise ValueError( f"Speaker profile '{speaker_profile.name}' has no voice model configured. " "Please update the profile to select a voice model." ) # 3. Resolve model configs with credentials outline_provider, outline_model_name, outline_config = ( await episode_profile.resolve_outline_config() ) transcript_provider, transcript_model_name, transcript_config = ( await episode_profile.resolve_transcript_config() ) tts_provider, tts_model_name, tts_config = ( await speaker_profile.resolve_tts_config() ) logger.info( f"Resolved models - outline: {outline_provider}/{outline_model_name}, " f"transcript: {transcript_provider}/{transcript_model_name}, " f"tts: {tts_provider}/{tts_model_name}" ) # 4. Load all profiles and configure podcast-creator episode_profiles = await repo_query("SELECT * FROM episode_profile") speaker_profiles = await repo_query("SELECT * FROM speaker_profile") # Transform the surrealdb array into a dictionary for podcast-creator episode_profiles_dict = { profile["name"]: profile for profile in episode_profiles } speaker_profiles_dict = { profile["name"]: profile for profile in speaker_profiles } # 5. Inject resolved model configs into profile dicts # Resolve ALL episode profiles (podcast-creator validates all). # Remove profiles that fail resolution to prevent validation errors. for ep_name in list(episode_profiles_dict.keys()): ep_dict = episode_profiles_dict[ep_name] try: if ep_dict.get("outline_llm"): prov, model, conf = await _resolve_model_config( str(ep_dict["outline_llm"]) ) ep_dict["outline_provider"] = prov ep_dict["outline_model"] = model ep_dict["outline_config"] = conf if ep_dict.get("transcript_llm"): prov, model, conf = await _resolve_model_config( str(ep_dict["transcript_llm"]) ) ep_dict["transcript_provider"] = prov ep_dict["transcript_model"] = model ep_dict["transcript_config"] = conf except Exception as e: logger.warning( f"Failed to resolve models for episode profile '{ep_name}', " f"removing from config to prevent validation errors: {e}" ) del episode_profiles_dict[ep_name] # Resolve TTS for ALL speaker profiles (podcast-creator validates all). # Remove profiles that fail resolution to prevent validation errors. for sp_name in list(speaker_profiles_dict.keys()): sp_dict = speaker_profiles_dict[sp_name] if sp_dict.get("voice_model"): try: prov, model, conf = await _resolve_model_config( str(sp_dict["voice_model"]) ) sp_dict["tts_provider"] = prov sp_dict["tts_model"] = model sp_dict["tts_config"] = conf except Exception as e: logger.warning( f"Failed to resolve TTS for speaker profile '{sp_name}', " f"removing from config to prevent validation errors: {e}" ) del speaker_profiles_dict[sp_name] continue # Per-speaker TTS overrides for speaker in sp_dict.get("speakers", []): if speaker.get("voice_model"): try: prov, model, conf = await _resolve_model_config( str(speaker["voice_model"]) ) speaker["tts_provider"] = prov speaker["tts_model"] = model speaker["tts_config"] = conf except Exception as e: logger.warning( f"Failed to resolve per-speaker TTS for '{speaker.get('name')}': {e}" ) # 6. Generate briefing briefing = episode_profile.default_briefing if input_data.briefing_suffix: briefing += f"\n\nAdditional instructions: {input_data.briefing_suffix}" # Create the record for the episode and associate with the ongoing command episode = PodcastEpisode( name=input_data.episode_name, episode_profile=full_model_dump(episode_profile.model_dump()), speaker_profile=full_model_dump(speaker_profile.model_dump()), command=ensure_record_id(input_data.execution_context.command_id) if input_data.execution_context else None, briefing=briefing, content=input_data.content, audio_file=None, transcript=None, outline=None, ) await episode.save() configure("speakers_config", {"profiles": speaker_profiles_dict}) configure("episode_config", {"profiles": episode_profiles_dict}) logger.info("Configured podcast-creator with episode and speaker profiles") logger.info(f"Generated briefing (length: {len(briefing)} chars)") # 7. Create output directory using UUID for filesystem-safe paths episode_dir_name, output_dir = build_episode_output_dir(DATA_FOLDER) output_dir.mkdir(parents=True, exist_ok=True) logger.info(f"Created output directory: {output_dir}") # 8. Generate podcast using podcast-creator logger.info("Starting podcast generation with podcast-creator...") result = await create_podcast( content=input_data.content, briefing=briefing, episode_name=episode_dir_name, output_dir=str(output_dir), speaker_config=speaker_profile.name, episode_profile=episode_profile.name, ) episode.audio_file = ( str(result.get("final_output_file_path")) if result else None ) episode.transcript = { "transcript": full_model_dump(result["transcript"]) if result else None } episode.outline = full_model_dump(result["outline"]) if result else None await episode.save() processing_time = time.time() - start_time logger.info( f"Successfully generated podcast episode: {episode.id} in {processing_time:.2f}s" ) return PodcastGenerationOutput( success=True, episode_id=str(episode.id), audio_file_path=str(result.get("final_output_file_path")) if result else None, transcript={"transcript": full_model_dump(result["transcript"])} if result.get("transcript") else None, outline=full_model_dump(result["outline"]) if result.get("outline") else None, processing_time=processing_time, ) except ValueError: raise except Exception as e: logger.error(f"Podcast generation failed: {e}") logger.exception(e) error_msg = str(e) if "Invalid json output" in error_msg or "Expecting value" in error_msg: error_msg += ( "\n\nNOTE: This error commonly occurs with GPT-5 models that use extended thinking. " "The model may be putting all output inside tags, leaving nothing to parse. " "Try using gpt-4o, gpt-4o-mini, or gpt-4-turbo instead in your episode profile." ) raise RuntimeError(error_msg) from e ================================================ FILE: commands/source_commands.py ================================================ import time from typing import Any, Dict, List, Optional from loguru import logger from pydantic import BaseModel from surreal_commands import CommandInput, CommandOutput, command from open_notebook.database.repository import ensure_record_id from open_notebook.domain.notebook import Source from open_notebook.domain.transformation import Transformation from open_notebook.exceptions import ConfigurationError try: from open_notebook.graphs.source import source_graph from open_notebook.graphs.transformation import graph as transform_graph except ImportError as e: logger.error(f"Failed to import graphs: {e}") raise ValueError("graphs not available") def full_model_dump(model): if isinstance(model, BaseModel): return model.model_dump() elif isinstance(model, dict): return {k: full_model_dump(v) for k, v in model.items()} elif isinstance(model, list): return [full_model_dump(item) for item in model] else: return model class SourceProcessingInput(CommandInput): source_id: str content_state: Dict[str, Any] notebook_ids: List[str] transformations: List[str] embed: bool class SourceProcessingOutput(CommandOutput): success: bool source_id: str embedded_chunks: int = 0 insights_created: int = 0 processing_time: float error_message: Optional[str] = None @command( "process_source", app="open_notebook", retry={ "max_attempts": 15, # Handle deep queues (workaround for SurrealDB v2 transaction conflicts) "wait_strategy": "exponential_jitter", "wait_min": 1, "wait_max": 120, # Allow queue to drain "stop_on": [ValueError, ConfigurationError], # Don't retry validation/config errors "retry_log_level": "debug", # Avoid log noise during transaction conflicts }, ) async def process_source_command( input_data: SourceProcessingInput, ) -> SourceProcessingOutput: """ Process source content using the source_graph workflow """ start_time = time.time() try: logger.info(f"Starting source processing for source: {input_data.source_id}") logger.info(f"Notebook IDs: {input_data.notebook_ids}") logger.info(f"Transformations: {input_data.transformations}") logger.info(f"Embed: {input_data.embed}") # 1. Load transformation objects from IDs transformations = [] for trans_id in input_data.transformations: logger.info(f"Loading transformation: {trans_id}") transformation = await Transformation.get(trans_id) if not transformation: raise ValueError(f"Transformation '{trans_id}' not found") transformations.append(transformation) logger.info(f"Loaded {len(transformations)} transformations") # 2. Get existing source record to update its command field source = await Source.get(input_data.source_id) if not source: raise ValueError(f"Source '{input_data.source_id}' not found") # Update source with command reference source.command = ( ensure_record_id(input_data.execution_context.command_id) if input_data.execution_context else None ) await source.save() logger.info(f"Updated source {source.id} with command reference") # 3. Process source with all notebooks logger.info(f"Processing source with {len(input_data.notebook_ids)} notebooks") # Execute source_graph with all notebooks result = await source_graph.ainvoke( { # type: ignore[arg-type] "content_state": input_data.content_state, "notebook_ids": input_data.notebook_ids, # Use notebook_ids (plural) as expected by SourceState "apply_transformations": transformations, "embed": input_data.embed, "source_id": input_data.source_id, # Add the source_id to the state } ) processed_source = result["source"] # 4. Gather processing results (notebook associations handled by source_graph) # Note: embedding is fire-and-forget (async job), so we can't query the # count here — it hasn't completed yet. The embed_source_command logs # the actual count when it finishes. insights_list = await processed_source.get_insights() insights_created = len(insights_list) processing_time = time.time() - start_time embed_status = "submitted" if input_data.embed else "skipped" logger.info( f"Successfully processed source: {processed_source.id} in {processing_time:.2f}s" ) logger.info( f"Created {insights_created} insights, embedding {embed_status}" ) return SourceProcessingOutput( success=True, source_id=str(processed_source.id), embedded_chunks=0, insights_created=insights_created, processing_time=processing_time, ) except ValueError as e: # Validation errors are permanent failures - don't retry processing_time = time.time() - start_time logger.error(f"Source processing failed: {e}") return SourceProcessingOutput( success=False, source_id=input_data.source_id, processing_time=processing_time, error_message=str(e), ) except Exception as e: # Transient failure - will be retried (surreal-commands logs final failure) logger.debug( f"Transient error processing source {input_data.source_id}: {e}" ) raise # ============================================================================= # RUN TRANSFORMATION COMMAND # ============================================================================= class RunTransformationInput(CommandInput): """Input for running a transformation on an existing source.""" source_id: str transformation_id: str class RunTransformationOutput(CommandOutput): """Output from transformation command.""" success: bool source_id: str transformation_id: str processing_time: float error_message: Optional[str] = None @command( "run_transformation", app="open_notebook", retry={ "max_attempts": 5, "wait_strategy": "exponential_jitter", "wait_min": 1, "wait_max": 60, "stop_on": [ValueError, ConfigurationError], # Don't retry validation/config errors "retry_log_level": "debug", }, ) async def run_transformation_command( input_data: RunTransformationInput, ) -> RunTransformationOutput: """ Run a transformation on an existing source to generate an insight. This command runs the transformation graph which: 1. Loads the source and transformation 2. Calls the LLM to generate insight content 3. Creates the insight via create_insight command (fire-and-forget) Use this command for UI-triggered insight generation to avoid blocking the HTTP request while the LLM processes. Retry Strategy: - Retries up to 5 times for transient failures (network, timeout, etc.) - Uses exponential-jitter backoff (1-60s) - Does NOT retry permanent failures (ValueError for validation errors) """ start_time = time.time() try: logger.info( f"Running transformation {input_data.transformation_id} " f"on source {input_data.source_id}" ) # Load source source = await Source.get(input_data.source_id) if not source: raise ValueError(f"Source '{input_data.source_id}' not found") # Load transformation transformation = await Transformation.get(input_data.transformation_id) if not transformation: raise ValueError( f"Transformation '{input_data.transformation_id}' not found" ) # Run transformation graph (includes LLM call + insight creation) await transform_graph.ainvoke( input=dict(source=source, transformation=transformation) ) processing_time = time.time() - start_time logger.info( f"Successfully ran transformation {input_data.transformation_id} " f"on source {input_data.source_id} in {processing_time:.2f}s" ) return RunTransformationOutput( success=True, source_id=input_data.source_id, transformation_id=input_data.transformation_id, processing_time=processing_time, ) except ValueError as e: # Validation errors are permanent failures - don't retry processing_time = time.time() - start_time logger.error( f"Failed to run transformation {input_data.transformation_id} " f"on source {input_data.source_id}: {e}" ) return RunTransformationOutput( success=False, source_id=input_data.source_id, transformation_id=input_data.transformation_id, processing_time=processing_time, error_message=str(e), ) except Exception as e: # Transient failure - will be retried (surreal-commands logs final failure) logger.debug( f"Transient error running transformation {input_data.transformation_id} " f"on source {input_data.source_id}: {e}" ) raise ================================================ FILE: docker-compose.yml ================================================ services: surrealdb: image: surrealdb/surrealdb:v2 command: start --log info --user root --pass root rocksdb:/mydata/mydatabase.db user: root # Required for bind mounts on Linux ports: - "8000:8000" volumes: - ./surreal_data:/mydata environment: - SURREAL_EXPERIMENTAL_GRAPHQL=true restart: always pull_policy: always open_notebook: image: lfnovo/open_notebook:v1-latest ports: - "8502:8502" # Web UI - "5055:5055" # REST API environment: # REQUIRED: Change this to your own secret string # This encrypts your API keys in the database - OPEN_NOTEBOOK_ENCRYPTION_KEY=change-me-to-a-secret-string # Database connection (default values - no need to change) - SURREAL_URL=ws://surrealdb:8000/rpc - SURREAL_USER=root - SURREAL_PASSWORD=root - SURREAL_NAMESPACE=open_notebook - SURREAL_DATABASE=open_notebook volumes: - ./notebook_data:/app/data depends_on: - surrealdb restart: always pull_policy: always ================================================ FILE: docs/0-START-HERE/index.md ================================================ # Open Notebook - Start Here **Open Notebook** is a privacy-focused AI research assistant. Upload documents, chat with AI, generate notes, and create podcasts—all with complete control over your data. ## Choose Your Path ### 🚀 I want to use OpenAI (Fastest) **5 minutes to running.** GPT, simple setup, powerful results. → [OpenAI Quick Start](quick-start-openai.md) --- ### ☁️ I want to use other cloud AI (Anthropic, Google, OpenRouter, etc.) **5 minutes to running.** Choose from 15+ AI providers. → [Cloud Providers Quick Start](quick-start-cloud.md) --- ### 🏠 I want to run locally (Ollama or LMStudio, completely private) **5 minutes to running.** Keep everything private, on your machine. No costs. → [Local Quick Start](quick-start-local.md) --- ## What Can You Do? - 📄 **Upload Content**: PDFs, web links, audio, video, text - 🤖 **Chat with AI**: Ask questions about your documents with citations - 📝 **Generate Notes**: AI creates summaries and insights - 🎙️ **Create Podcasts**: Turn research into professional audio content - 🔍 **Search**: Full-text and semantic search across all content - ⚙️ **Transform**: Extract insights, analyze themes, create summaries ## Why Open Notebook? | Feature | Open Notebook | Notebook LM | |---------|---|---| | **Privacy** | Self-hosted, your control | Cloud, Google's servers | | **AI Choice** | 15+ providers | Google's models only | | **Podcast Speakers** | 1-4 customizable | 2 only | | **Cost** | Completely free | Free (but your data) | | **Offline** | Yes | No | ## Prerequisites - **Docker**: All paths use Docker (free) - **AI Provider**: Either a cloud API key OR use free local models (Ollama) --- ## Next Steps 1. Pick your path above ⬆️ 2. Follow the 5-minute quick start 3. Create your first notebook 4. Start uploading documents! --- **Need Help?** Join our [Discord community](https://discord.gg/37XJPXfz2w) or see [Full Documentation](../index.md). ================================================ FILE: docs/0-START-HERE/quick-start-cloud.md ================================================ # Quick Start - Cloud AI Providers (5 minutes) Get Open Notebook running with **Anthropic, Google, Groq, or other cloud providers**. Same simplicity as OpenAI, with more choices. ## Prerequisites 1. **Docker Desktop** installed - [Download here](https://www.docker.com/products/docker-desktop/) - Already have it? Skip to step 2 2. **API Key** from your chosen provider: - **OpenRouter** (100+ models, one key): https://openrouter.ai/keys - **Anthropic (Claude)**: https://console.anthropic.com/ - **Google (Gemini)**: https://aistudio.google.com/ - **Groq** (fast, free tier): https://console.groq.com/ - **Mistral**: https://console.mistral.ai/ - **DeepSeek**: https://platform.deepseek.com/ - **xAI (Grok)**: https://console.x.ai/ ## Step 1: Create Configuration (1 min) Create a new folder `open-notebook` and add this file: **docker-compose.yml**: ```yaml services: surrealdb: image: surrealdb/surrealdb:v2 command: start --user root --pass password --bind 0.0.0.0:8000 rocksdb:/mydata/mydatabase.db ports: - "8000:8000" volumes: - ./surreal_data:/mydata # Removed the healthcheck because the v2 image is too minimal to run wget/curl restart: always open_notebook: image: lfnovo/open_notebook:v1-latest pull_policy: always ports: - "8502:8502" # Web UI - "5055:5055" # API environment: - OPEN_NOTEBOOK_ENCRYPTION_KEY=change-me-to-a-secret-string - SURREAL_URL=ws://surrealdb:8000/rpc - SURREAL_USER=root - SURREAL_PASSWORD=password - SURREAL_NAMESPACE=open_notebook - SURREAL_DATABASE=open_notebook volumes: - ./notebook_data:/app/data depends_on: - surrealdb restart: always ``` **Edit the file:** - Replace `change-me-to-a-secret-string` with your own secret (any string works) --- ## Step 2: Start Services (1 min) Open terminal in your `open-notebook` folder: ```bash docker compose up -d ``` Wait 15-20 seconds for services to start. --- ## Step 3: Access Open Notebook (instant) Open your browser: ``` http://localhost:8502 ``` You should see the Open Notebook interface! --- ## Step 4: Configure Your AI Provider (1 min) 1. Go to **Settings** → **API Keys** 2. Click **Add Credential** 3. Select your provider (e.g., Anthropic, Google, Groq, OpenRouter) 4. Give it a name, paste your API key 5. Click **Save** 6. Click **Test Connection** — should show success 7. Click **Discover Models** → **Register Models** Your provider's models are now available! > **Multiple providers**: You can add credentials for as many providers as you want. Just repeat this step for each provider. --- ## Step 5: Configure Your Model (1 min) 1. Go to **Settings** (gear icon) 2. Navigate to **Models** 3. Select your provider's model: | Provider | Recommended Model | Notes | |----------|-------------------|-------| | **OpenRouter** | `anthropic/claude-3.5-sonnet` | Access 100+ models | | **Anthropic** | `claude-3-5-sonnet-latest` | Best reasoning | | **Google** | `gemini-2.0-flash` | Large context, fast | | **Groq** | `llama-3.3-70b-versatile` | Ultra-fast | | **Mistral** | `mistral-large-latest` | Strong European option | 4. Click **Save** --- ## Step 6: Create Your First Notebook (1 min) 1. Click **New Notebook** 2. Name: "My Research" 3. Click **Create** --- ## Step 7: Add Content & Chat (2 min) 1. Click **Add Source** 2. Choose **Web Link** 3. Paste any article URL 4. Wait for processing 5. Go to **Chat** and ask questions! --- ## Verification Checklist - [ ] Docker is running - [ ] You can access `http://localhost:8502` - [ ] Provider credential is configured and tested - [ ] Models are registered - [ ] You created a notebook - [ ] Chat works **All checked?** You're ready to research! --- ## Provider Comparison | Provider | Speed | Quality | Context | Cost | |----------|-------|---------|---------|------| | **OpenRouter** | Varies | Varies | Varies | Varies (100+ models) | | **Anthropic** | Medium | Excellent | 200K | $$$ | | **Google** | Fast | Very Good | 1M+ | $$ | | **Groq** | Ultra-fast | Good | 128K | $ (free tier) | | **Mistral** | Fast | Good | 128K | $$ | | **DeepSeek** | Medium | Very Good | 64K | $ | --- ## Troubleshooting ### "Model not found" Error 1. Go to **Settings** → **API Keys** 2. Click **Test Connection** on your credential 3. If valid, click **Discover Models** → **Register Models** 4. Check you have credits/access for the model ### "Cannot connect to server" ```bash docker ps # Check all services running docker compose logs # View logs docker compose restart # Restart everything ``` ### Provider-Specific Issues **Anthropic**: Ensure key starts with `sk-ant-` **Google**: Use AI Studio key, not Cloud Console **Groq**: Free tier has rate limits; upgrade if needed --- ## Cost Estimates Approximate costs per 1K tokens: | Provider | Input | Output | |----------|-------|--------| | Anthropic (Sonnet) | $0.003 | $0.015 | | Google (Flash) | $0.0001 | $0.0004 | | Groq (Llama 70B) | Free tier available | - | | Mistral (Large) | $0.002 | $0.006 | Check provider websites for current pricing. --- ## Next Steps 1. **Add Your Content**: PDFs, web links, documents 2. **Explore Features**: Podcasts, transformations, search 3. **Full Documentation**: [See all features](../3-USER-GUIDE/index.md) --- **Need help?** Join our [Discord community](https://discord.gg/37XJPXfz2w)! ================================================ FILE: docs/0-START-HERE/quick-start-local.md ================================================ # Quick Start - Local & Private (5 minutes) Get Open Notebook running with **100% local AI** using Ollama. No cloud API keys needed, completely private. ## Prerequisites 1. **Docker Desktop** installed - [Download here](https://www.docker.com/products/docker-desktop/) - Already have it? Skip to step 2 2. **Local LLM** - Choose one: - **Ollama** (recommended): [Download here](https://ollama.ai/) - **LM Studio** (GUI alternative): [Download here](https://lmstudio.ai) ## Step 1: Choose Your Setup (1 min) ### Local Machine (Same Computer) Everything runs on your machine. Recommended for testing/learning. ### Remote Server (Raspberry Pi, NAS, Cloud VM) Run on a different computer, access from another. Needs network configuration. --- ## Step 2: Create Configuration (1 min) Create a new folder `open-notebook-local` and add this file: **docker-compose.yml**: ```yaml services: surrealdb: image: surrealdb/surrealdb:v2 command: start --user root --pass password --bind 0.0.0.0:8000 rocksdb:/mydata/mydatabase.db ports: - "8000:8000" volumes: - ./surreal_data:/mydata open_notebook: image: lfnovo/open_notebook:v1-latest-single pull_policy: always ports: - "8502:8502" # Web UI (React frontend) - "5055:5055" # API (required!) environment: # Encryption key for credential storage (required) - OPEN_NOTEBOOK_ENCRYPTION_KEY=change-me-to-a-secret-string # Database (required) - SURREAL_URL=ws://surrealdb:8000/rpc - SURREAL_USER=root - SURREAL_PASSWORD=password - SURREAL_NAMESPACE=open_notebook - SURREAL_DATABASE=open_notebook volumes: - ./notebook_data:/app/data - ./surreal_data:/mydata depends_on: - surrealdb restart: always ollama: image: ollama/ollama:latest ports: - "11434:11434" volumes: - ./ollama_models:/root/.ollama environment: # Optional: set GPU support if available - OLLAMA_NUM_GPU=0 restart: always ``` **Edit the file:** - Replace `change-me-to-a-secret-string` with your own secret (any string works) --- ## Step 3: Start Services (1 min) Open terminal in your `open-notebook-local` folder: ```bash docker compose up -d ``` Wait 10-15 seconds for all services to start. --- ## Step 4: Download a Model (2-3 min) Ollama needs at least one language model. Pick one: ```bash # Fastest & smallest (recommended for testing) docker exec open-notebook-local-ollama-1 ollama pull mistral # OR: Better quality but slower docker exec open-notebook-local-ollama-1 ollama pull neural-chat # OR: Even better quality, more VRAM needed docker exec open-notebook-local-ollama-1 ollama pull llama2 ``` This downloads the model (will take 1-5 minutes depending on your internet). --- ## Step 5: Access Open Notebook (instant) Open your browser: ``` http://localhost:8502 ``` You should see the Open Notebook interface. --- ## Step 6: Configure Ollama Provider (1 min) 1. Go to **Settings** → **API Keys** 2. Click **Add Credential** 3. Select provider: **Ollama** 4. Give it a name (e.g., "Local Ollama") 5. Enter the base URL: `http://ollama:11434` 6. Click **Save** 7. Click **Test Connection** — should show success 8. Click **Discover Models** → **Register Models** --- ## Step 7: Configure Local Model (1 min) 1. Go to **Settings** → **Models** 2. Set: - **Language Model**: `ollama/mistral` (or whichever model you downloaded) - **Embedding Model**: `ollama/nomic-embed-text` (auto-downloads if missing) 3. Click **Save** --- ## Step 8: Create Your First Notebook (1 min) 1. Click **New Notebook** 2. Name: "My Private Research" 3. Click **Create** --- ## Step 9: Add Local Content (1 min) 1. Click **Add Source** 2. Choose **Text** 3. Paste some text or a local document 4. Click **Add** --- ## Step 10: Chat With Your Content (1 min) 1. Go to **Chat** 2. Type: "What did you learn from this?" 3. Click **Send** 4. Watch as the local Ollama model responds! --- ## Verification Checklist - [ ] Docker is running - [ ] You can access `http://localhost:8502` - [ ] Ollama credential is configured and tested - [ ] Models are registered - [ ] You created a notebook - [ ] Chat works with local model **All checked?** You have a completely **private, offline** research assistant! --- ## Advantages of Local Setup - **No API costs** - Free forever - **No internet required** - True offline capability - **Privacy first** - Your data never leaves your machine - **No subscriptions** - No monthly bills **Trade-off:** Slower than cloud models (depends on your CPU/GPU) --- ## Troubleshooting ### "ollama: command not found" Docker image name might be different: ```bash docker ps # Find the Ollama container name docker exec ollama pull mistral ``` ### Model Download Stuck Check internet connection and restart: ```bash docker compose restart ollama ``` Then retry the model pull command. ### "Address already in use" Error ```bash docker compose down docker compose up -d ``` ### Low Performance Check if GPU is available: ```bash # Show available GPUs docker exec open-notebook-local-ollama-1 ollama ps # Enable GPU in docker-compose.yml: # - OLLAMA_NUM_GPU=1 ``` Then restart: `docker compose restart ollama` ### Adding More Models ```bash # List available models docker exec open-notebook-local-ollama-1 ollama list # Pull additional model docker exec open-notebook-local-ollama-1 ollama pull neural-chat ``` --- ## Next Steps **Now that it's running:** 1. **Add Your Own Content**: PDFs, documents, articles (see 3-USER-GUIDE) 2. **Explore Features**: Podcasts, transformations, search 3. **Full Documentation**: [See all features](../3-USER-GUIDE/index.md) 4. **Scale Up**: Deploy to a server with better hardware for faster responses 5. **Benchmark Models**: Try different models to find the speed/quality tradeoff you prefer --- ## Alternative: Using LM Studio Instead of Ollama **Prefer a GUI?** LM Studio is easier for non-technical users: 1. Download LM Studio: https://lmstudio.ai 2. Open the app, download a model from the library 3. Go to "Local Server" tab, start server (port 1234) 4. In Open Notebook, go to **Settings** → **API Keys** 5. Click **Add Credential** → Select **OpenAI-Compatible** 6. Enter base URL: `http://host.docker.internal:1234/v1` 7. Enter API key: `lm-studio` (placeholder) 8. Click **Save**, then **Test Connection** 9. Configure in Settings → Models → Select your LM Studio model **Note**: LM Studio runs outside Docker, use `host.docker.internal` to connect. --- ## Going Further - **Switch models**: Change in Settings → Models anytime - **Add more models**: - Ollama: Run `ollama pull `, then re-discover models from the credential - LM Studio: Download from the app library - **Deploy to server**: Same docker-compose.yml works anywhere - **Use cloud hybrid**: Keep some local models, add cloud provider credentials for complex tasks --- ## Common Model Choices | Model | Speed | Quality | VRAM | Best For | |-------|-------|---------|------|----------| | **mistral** | Fast | Good | 4GB | Testing, general use | | **neural-chat** | Medium | Better | 6GB | Balanced, recommended | | **llama2** | Slow | Best | 8GB+ | Complex reasoning | | **phi** | Very Fast | Fair | 2GB | Minimal hardware | --- **Need Help?** Join our [Discord community](https://discord.gg/37XJPXfz2w) - many users run local setups! ================================================ FILE: docs/0-START-HERE/quick-start-openai.md ================================================ # Quick Start - OpenAI (5 minutes) Get Open Notebook running with OpenAI's GPT models. Fast, powerful, and simple. ## Prerequisites 1. **Docker Desktop** installed - [Download here](https://www.docker.com/products/docker-desktop/) - Already have it? Skip to step 2 2. **OpenAI API Key** (required) - Go to https://platform.openai.com/api-keys - Create account → Create new secret key - Add at least $5 in credits to your account - Copy the key (starts with `sk-`) ## Step 1: Create Configuration (1 min) Create a new folder `open-notebook` and add this file: **docker-compose.yml**: ```yaml services: surrealdb: image: surrealdb/surrealdb:v2 command: start --user root --pass password --bind 0.0.0.0:8000 rocksdb:/mydata/mydatabase.db ports: - "8000:8000" volumes: - ./surreal_data:/mydata open_notebook: image: lfnovo/open_notebook:v1-latest pull_policy: always ports: - "8502:8502" # Web UI - "5055:5055" # API environment: # Encryption key for credential storage (required) - OPEN_NOTEBOOK_ENCRYPTION_KEY=change-me-to-a-secret-string # Database (required) - SURREAL_URL=ws://surrealdb:8000/rpc - SURREAL_USER=root - SURREAL_PASSWORD=password - SURREAL_NAMESPACE=open_notebook - SURREAL_DATABASE=open_notebook volumes: - ./notebook_data:/app/data depends_on: - surrealdb restart: always ``` **Edit the file:** - Replace `change-me-to-a-secret-string` with your own secret (any string works) --- ## Step 2: Start Services (1 min) Open terminal in your `open-notebook` folder: ```bash docker compose up -d ``` Wait 15-20 seconds for services to start. --- ## Step 3: Access Open Notebook (instant) Open your browser: ``` http://localhost:8502 ``` You should see the Open Notebook interface! --- ## Step 4: Configure Your OpenAI Provider (1 min) 1. Go to **Settings** → **API Keys** 2. Click **Add Credential** 3. Select provider: **OpenAI** 4. Give it a name (e.g., "My OpenAI Key") 5. Paste your OpenAI API key 6. Click **Save** 7. Click **Test Connection** — should show success 8. Click **Discover Models** → **Register Models** Your OpenAI models are now available! --- ## Step 5: Create Your First Notebook (1 min) 1. Click **New Notebook** 2. Name: "My Research" 3. Click **Create** --- ## Step 6: Add a Source (1 min) 1. Click **Add Source** 2. Choose **Web Link** 3. Paste: `https://en.wikipedia.org/wiki/Artificial_intelligence` 4. Click **Add** 5. Wait for processing (30-60 seconds) --- ## Step 7: Chat With Your Content (1 min) 1. Go to **Chat** 2. Type: "What is artificial intelligence?" 3. Click **Send** 4. Watch as GPT responds with information from your source! --- ## Verification Checklist - [ ] Docker is running - [ ] You can access `http://localhost:8502` - [ ] OpenAI credential is configured and tested - [ ] You created a notebook - [ ] You added a source - [ ] Chat works **All checked?** You have a fully working AI research assistant! --- ## Using Different Models In your notebook, go to **Settings** → **Models** to choose: - `gpt-4o` - Best quality (recommended) - `gpt-4o-mini` - Fast and cheap (good for testing) --- ## Troubleshooting ### "Port 8502 already in use" Change the port in docker-compose.yml: ```yaml ports: - "8503:8502" # Use 8503 instead ``` Then access at `http://localhost:8503` ### "API key not working" 1. Go to **Settings** → **API Keys** 2. Click **Test Connection** on your OpenAI credential 3. If it fails, verify your key at https://platform.openai.com 4. Delete the credential and create a new one with the correct key ### "Cannot connect to server" ```bash docker ps # Check all services running docker compose logs # View logs docker compose restart # Restart everything ``` --- ## Next Steps 1. **Add Your Own Content**: PDFs, web links, documents 2. **Explore Features**: Podcasts, transformations, search 3. **Full Documentation**: [See all features](../3-USER-GUIDE/index.md) --- ## Cost Estimate OpenAI pricing (approximate): - **Conversation**: $0.01-0.10 per 1K tokens - **Embeddings**: $0.02 per 1M tokens - **Typical usage**: $1-5/month for light use, $20-50/month for heavy use Check https://openai.com/pricing for current rates. --- **Need help?** Join our [Discord community](https://discord.gg/37XJPXfz2w)! ================================================ FILE: docs/1-INSTALLATION/docker-compose.md ================================================ # Docker Compose Installation (Recommended) Multi-container setup with separate services. **Best for most users.** > **Alternative Registry:** All images are available on both Docker Hub (`lfnovo/open_notebook`) and GitHub Container Registry (`ghcr.io/lfnovo/open-notebook`). Use GHCR if Docker Hub is blocked or you prefer GitHub-native workflows. ## Prerequisites - **Docker Desktop** installed ([Download](https://www.docker.com/products/docker-desktop/)) - **5-10 minutes** of your time - **API key** for at least one AI provider (OpenAI recommended for beginners) ## Step 1: Get docker-compose.yml (1 min) **Option A: Download from repository** ```bash curl -o docker-compose.yml https://raw.githubusercontent.com/lfnovo/open-notebook/main/docker-compose.yml ``` **Option B: Use the official file from the repo** The official `docker-compose.yml` is in the root of our repository: [View on GitHub](https://github.com/lfnovo/open-notebook/blob/main/docker-compose.yml) Copy that file to your project folder. **Option C: Create manually** Create a file called `docker-compose.yml` with this content: ```yaml services: surrealdb: image: surrealdb/surrealdb:v2 command: start --log info --user root --pass root rocksdb:/mydata/mydatabase.db user: root # Required for bind mounts on Linux ports: - "8000:8000" volumes: - ./surreal_data:/mydata environment: - SURREAL_EXPERIMENTAL_GRAPHQL=true restart: always pull_policy: always open_notebook: image: lfnovo/open_notebook:v1-latest ports: - "8502:8502" # Web UI - "5055:5055" # REST API environment: # REQUIRED: Change this to your own secret string - OPEN_NOTEBOOK_ENCRYPTION_KEY=change-me-to-a-secret-string # Database connection (default values - no need to change) - SURREAL_URL=ws://surrealdb:8000/rpc - SURREAL_USER=root - SURREAL_PASSWORD=root - SURREAL_NAMESPACE=open_notebook - SURREAL_DATABASE=open_notebook volumes: - ./notebook_data:/app/data depends_on: - surrealdb restart: always pull_policy: always ``` **Edit the file:** - Replace `change-me-to-a-secret-string` with your own secret (any string works, e.g., `my-super-secret-key-123`) --- ## Step 2: Start Services (2 min) Open terminal in the `open-notebook` folder: ```bash docker compose up -d ``` Wait 15-20 seconds for all services to start: ``` ✅ surrealdb running on :8000 ✅ open_notebook running on :8502 (UI) and :5055 (API) ``` Check status: ```bash docker compose ps ``` --- ## Step 3: Verify Installation (1 min) **API Health:** ```bash curl http://localhost:5055/health # Should return: {"status": "healthy"} ``` **Frontend Access:** Open browser to: ``` http://localhost:8502 ``` You should see the Open Notebook interface! --- ## Step 4: Configure AI Provider (2 min) 1. Go to **Settings** → **API Keys** 2. Click **Add Credential** 3. Select your provider (e.g., OpenAI, Anthropic, Google) 4. Give it a name, paste your API key 5. Click **Save** 6. Click **Test Connection** — should show success 7. Click **Discover Models** → **Register Models** Your models are now available! > **Need an API key?** Get one from your chosen provider: > - **OpenAI**: https://platform.openai.com/api-keys > - **Anthropic**: https://console.anthropic.com/ > - **Google**: https://aistudio.google.com/ > - **Groq**: https://console.groq.com/ --- ## Step 5: First Notebook (2 min) 1. Click **New Notebook** 2. Name: "My Research" 3. Description: "Getting started" 4. Click **Create** Done! You now have a fully working Open Notebook instance. --- ## Configuration ### Adding Ollama (Free Local Models) Instead of manually editing, use our ready-made example: ```bash # Download the Ollama example curl -o docker-compose.yml https://raw.githubusercontent.com/lfnovo/open-notebook/main/examples/docker-compose-ollama.yml # Or copy from repo cp examples/docker-compose-ollama.yml docker-compose.yml ``` See [examples/docker-compose-ollama.yml](../../examples/docker-compose-ollama.yml) for the complete setup. **Manual setup:** Add this to your existing `docker-compose.yml`: ```yaml ollama: image: ollama/ollama:latest ports: - "11434:11434" volumes: - ollama_models:/root/.ollama restart: always volumes: ollama_models: ``` Then restart and pull a model: ```bash docker compose restart docker exec open-notebook-local-ollama-1 ollama pull mistral ``` Configure Ollama in the Settings UI: 1. Go to **Settings** → **API Keys** 2. Click **Add Credential** → Select **Ollama** 3. Enter base URL: `http://ollama:11434` 4. Click **Save**, then **Test Connection** 5. Click **Discover Models** → **Register Models** --- ## Environment Variables Reference | Variable | Purpose | Example | |----------|---------|---------| | `OPEN_NOTEBOOK_ENCRYPTION_KEY` | Encryption key for credentials | `my-secret-key` | | `SURREAL_URL` | Database connection | `ws://surrealdb:8000/rpc` | | `SURREAL_USER` | Database user | `root` | | `SURREAL_PASSWORD` | Database password | `root` | | `SURREAL_NAMESPACE` | Database namespace | `open_notebook` | | `SURREAL_DATABASE` | Database name | `open_notebook` | | `API_URL` | API external URL | `http://localhost:5055` | See [Environment Reference](../5-CONFIGURATION/environment-reference.md) for complete list. --- ## Common Tasks ### Stop Services ```bash docker compose down ``` ### View Logs ```bash # All services docker compose logs -f # Specific service docker compose logs -f api ``` ### Restart Services ```bash docker compose restart ``` ### Update to Latest Version ```bash docker compose down docker compose pull docker compose up -d ``` ### Remove All Data ```bash docker compose down -v ``` --- ## Troubleshooting ### "Cannot connect to API" Error 1. Check if Docker is running: ```bash docker ps ``` 2. Check if services are running: ```bash docker compose ps ``` 3. Check API logs: ```bash docker compose logs api ``` 4. Wait longer - services can take 20-30 seconds to start on first run --- ### Port Already in Use If you get "Port 8502 already in use", change the port: ```yaml ports: - "8503:8502" # Use 8503 instead - "5055:5055" # Keep API port same ``` Then access at `http://localhost:8503` --- ### Credential Issues 1. Go to **Settings** → **API Keys** 2. Click **Test Connection** on the credential 3. If it fails, verify key at provider's website 4. Check you have credits in your account 5. Delete and re-create the credential if needed --- ### Database Connection Issues Check SurrealDB is running: ```bash docker compose logs surrealdb ``` Reset database: ```bash docker compose down -v docker compose up -d ``` ### Database Permission Denied (Linux) If you see `Permission denied` or `Failed to create RocksDB directory` in SurrealDB logs: ```bash docker compose logs surrealdb | grep -i permission ``` This happens because SurrealDB runs as a non-root user but Docker creates bind mount directories as root. Add `user: root` to the surrealdb service: ```yaml surrealdb: image: surrealdb/surrealdb:v2 user: root # Fix for Linux bind mount permissions # ... rest of config ``` Then restart: ```bash docker compose down -v docker compose up -d ``` --- ## Alternative Setups Looking for different configurations? Check out our [examples/](../../examples/) folder: - **[Ollama Setup](../../examples/docker-compose-ollama.yml)** - Run local AI models (free, private) - **[Single Container](../../examples/docker-compose-single.yml)** - All-in-one container (deprecated, not recommended) - **[Development](../../examples/docker-compose-dev.yml)** - For contributors and developers Each example includes detailed comments and usage instructions. --- ## Next Steps 1. **Add Content**: Sources, notebooks, documents 2. **Configure Models**: Settings → Models (choose your preferences) 3. **Explore Features**: Chat, search, transformations 4. **Read Guide**: [User Guide](../3-USER-GUIDE/index.md) --- ## Production Deployment For production use, see: - [Security Hardening](../5-CONFIGURATION/security.md) - [Reverse Proxy](../5-CONFIGURATION/reverse-proxy.md) --- ## Getting Help - **Discord**: [Community support](https://discord.gg/37XJPXfz2w) - **Issues**: [GitHub Issues](https://github.com/lfnovo/open-notebook/issues) - **Docs**: [Full documentation](../index.md) ================================================ FILE: docs/1-INSTALLATION/from-source.md ================================================ # From Source Installation Clone the repository and run locally. **For developers and contributors.** ## Prerequisites - **Python 3.11+** - [Download](https://www.python.org/) - **Node.js 18+** - [Download](https://nodejs.org/) - **Git** - [Download](https://git-scm.com/) - **Docker** (for SurrealDB) - [Download](https://docker.com/) - **uv** (Python package manager) - `curl -LsSf https://astral.sh/uv/install.sh | sh` - API key from OpenAI or similar (or use Ollama for free) ## Quick Setup (10 minutes) ### 1. Clone Repository ```bash git clone https://github.com/lfnovo/open-notebook.git cd open-notebook # If you forked it: git clone https://github.com/YOUR_USERNAME/open-notebook.git cd open-notebook git remote add upstream https://github.com/lfnovo/open-notebook.git ``` ### 2. Install Python Dependencies ```bash uv sync uv pip install python-magic ``` #### 2.1 Alternative: Conda Setup (Optional) If you prefer using **Conda** to manage your environments, follow these steps instead of the standard `uv sync`: ```bash # Create and activate the environment conda create -n open-notebook python=3.11 -y conda activate open-notebook # Install uv inside conda to maintain compatibility with the Makefile conda install -c conda-forge uv nodejs -y # Sync dependencies uv sync ``` > **Note**: Installing `uv` inside your Conda environment ensures that commands like `make start-all` and `make api` continue to work seamlessly. ### 3. Start SurrealDB ```bash # Terminal 1 make database # or: docker compose up surrealdb ``` ### 4. Set Environment Variables ```bash cp .env.example .env # Edit .env and set: # OPEN_NOTEBOOK_ENCRYPTION_KEY=my-secret-key ``` After starting the app, configure AI providers via the **Settings → API Keys** UI in the browser. ### 5. Start API ```bash # Terminal 2 make api # or: uv run --env-file .env uvicorn api.main:app --host 0.0.0.0 --port 5055 ``` ### 6. Start Frontend ```bash # Terminal 3 cd frontend && npm install && npm run dev ``` ### 7. Access - **Frontend**: http://localhost:3000 - **API Docs**: http://localhost:5055/docs - **Database**: http://localhost:8000 ### 8. Configure AI Provider 1. Open http://localhost:3000 2. Go to **Settings** → **API Keys** 3. Click **Add Credential** → Select your provider → Paste API key 4. Click **Save**, then **Test Connection** 5. Click **Discover Models** → **Register Models** --- ## Development Workflow ### Code Quality ```bash # Format and lint Python make ruff # or: ruff check . --fix # Type checking make lint # or: uv run python -m mypy . ``` ### Run Tests ```bash uv run pytest tests/ ``` ### Common Commands ```bash # Start everything make start-all # View API docs open http://localhost:5055/docs # Check database migrations # (Auto-run on API startup) # Clean up make clean ``` --- ## Troubleshooting ### Python version too old ```bash python --version # Check version uv sync --python 3.11 # Use specific version ``` ### npm: command not found Install Node.js from https://nodejs.org/ ### Database connection errors ```bash docker ps # Check SurrealDB running docker logs surrealdb # View logs ``` ### Port 5055 already in use ```bash # Use different port uv run uvicorn api.main:app --port 5056 ``` --- ## Next Steps 1. Read [Development Guide](../7-DEVELOPMENT/quick-start.md) 2. See [Architecture Overview](../7-DEVELOPMENT/architecture.md) 3. Check [Contributing Guide](../7-DEVELOPMENT/contributing.md) --- ## Getting Help - **Discord**: [Community](https://discord.gg/37XJPXfz2w) - **Issues**: [GitHub Issues](https://github.com/lfnovo/open-notebook/issues) ================================================ FILE: docs/1-INSTALLATION/index.md ================================================ # Installation Guide Choose your installation route based on your setup and use case. ## Quick Decision: Which Route? ### 🚀 I want the easiest setup (Recommended for most) **→ [Docker Compose](docker-compose.md)** - Multi-container setup, production-ready - ✅ All features working - ✅ Clear separation of services - ✅ Easy to scale - ✅ Works on Mac, Windows, Linux - ⏱️ 5 minutes to running --- ### 🏠 I want everything in one container (Simplified) **→ [Single Container](single-container.md)** - All-in-one for simple deployments - ✅ Minimal configuration - ✅ Lower resource usage - ✅ Good for shared hosting - ✅ Works on PikaPods, Railway, etc. - ⏱️ 3 minutes to running --- ### 👨‍💻 I want to develop/contribute (Developers only) **→ [From Source](from-source.md)** - Clone repo, set up locally - ✅ Full control over code - ✅ Easy to debug - ✅ Can modify and test - ⚠️ Requires Python 3.11+, Node.js - ⏱️ 10 minutes to running --- ## System Requirements ### Minimum - **RAM**: 4GB - **Storage**: 2GB for app + space for documents - **CPU**: Any modern processor - **Network**: Internet (optional for offline setup) ### Recommended - **RAM**: 8GB+ - **Storage**: 10GB+ for documents and models - **CPU**: Multi-core processor - **GPU**: Optional (speeds up local AI models) --- ## AI Provider Options ### Cloud-Based (Pay-as-you-go) - **OpenAI** - GPT-4, GPT-4o, fast and capable - **Anthropic (Claude)** - Claude 3.5 Sonnet, excellent reasoning - **Google Gemini** - Multimodal, cost-effective - **Groq** - Ultra-fast inference - **Others**: Mistral, DeepSeek, xAI, OpenRouter **Cost**: Usually $0.01-$0.10 per 1K tokens **Speed**: Fast (sub-second) **Privacy**: Your data sent to cloud ### Local (Free, Private) - **Ollama** - Run open-source models locally - **LM Studio** - Desktop app for local models - **Hugging Face models** - Download and run **Cost**: $0 (just electricity) **Speed**: Depends on your hardware (slow to medium) **Privacy**: 100% offline --- ## Choose a Route **Already know which way to go?** Pick your installation path: - [Docker Compose](docker-compose.md) - **Most users** - [Single Container](single-container.md) - **Shared hosting** - [From Source](from-source.md) - **Developers** > **Privacy-first?** Any installation method works with Ollama for 100% local AI. See [Local Quick Start](../0-START-HERE/quick-start-local.md). --- ## Pre-Installation Checklist Before installing, you'll need: - [ ] **Docker** (for Docker routes) or **Node.js 18+** (for source) - [ ] **AI Provider API key** (OpenAI, Anthropic, etc.) OR willingness to use free local models - [ ] **At least 4GB RAM** available - [ ] **Stable internet** (or offline setup with Ollama) --- ## Detailed Installation Instructions ### For Docker Users 1. Install [Docker Desktop](https://docker.com/products/docker-desktop) 2. Choose: [Docker Compose](docker-compose.md) or [Single Container](single-container.md) 3. Follow the step-by-step guide 4. Access at `http://localhost:8502` ### For Source Installation (Developers) 1. Have Python 3.11+, Node.js 18+, Git installed 2. Follow [From Source](from-source.md) 3. Run `make start-all` 4. Access at `http://localhost:8502` (frontend) or `http://localhost:5055` (API) --- ## After Installation Once you're up and running: 1. **Configure Models** - Choose your AI provider in Settings 2. **Create First Notebook** - Start organizing research 3. **Add Sources** - PDFs, web links, documents 4. **Explore Features** - Chat, search, transformations 5. **Read Full Guide** - [User Guide](../3-USER-GUIDE/index.md) --- ## Troubleshooting During Installation **Having issues?** Check the troubleshooting section in your chosen installation guide, or see [Quick Fixes](../6-TROUBLESHOOTING/quick-fixes.md). --- ## Need Help? - **Discord**: [Join community](https://discord.gg/37XJPXfz2w) - **GitHub Issues**: [Report problems](https://github.com/lfnovo/open-notebook/issues) - **Docs**: See [Full Documentation](../index.md) --- ## Production Deployment Installing for production use? See additional resources: - [Security Hardening](../5-CONFIGURATION/security.md) - [Reverse Proxy Setup](../5-CONFIGURATION/reverse-proxy.md) - [Performance Tuning](../5-CONFIGURATION/advanced.md) --- **Ready to install?** Pick a route above! ⬆️ ================================================ FILE: docs/1-INSTALLATION/single-container.md ================================================ # Single Container Installation All-in-one container setup. **Simpler than Docker Compose, but less flexible.** **Best for:** PikaPods, Railway, shared hosting, minimal setups > **Alternative Registry:** Images available on both Docker Hub (`lfnovo/open_notebook:v1-latest-single`) and GitHub Container Registry (`ghcr.io/lfnovo/open-notebook:v1-latest-single`). > **Note**: While this is a simple way to get started, we recommend [Docker Compose](docker-compose.md) for most users. Docker Compose is more flexible and will make it easier if we add more services to the setup in the future. This single-container option is best for platforms that specifically require it (PikaPods, Railway, etc.). ## Prerequisites - Docker installed (for local testing) - API key from OpenAI, Anthropic, or another provider - 5 minutes ## Quick Setup ### For Local Testing (Docker) ```yaml # docker-compose.yml services: open_notebook: image: lfnovo/open_notebook:v1-latest-single pull_policy: always ports: - "8502:8502" # Web UI (React frontend) - "5055:5055" # API environment: - OPEN_NOTEBOOK_ENCRYPTION_KEY=change-me-to-a-secret-string - SURREAL_URL=ws://localhost:8000/rpc - SURREAL_USER=root - SURREAL_PASSWORD=root - SURREAL_NAMESPACE=open_notebook - SURREAL_DATABASE=open_notebook volumes: - ./data:/app/data restart: always ``` Run: ```bash docker compose up -d ``` Access: `http://localhost:8502` Then configure your AI provider: 1. Go to **Settings** → **API Keys** 2. Click **Add Credential** → Select your provider → Paste API key 3. Click **Save**, then **Test Connection** 4. Click **Discover Models** → **Register Models** ### For Cloud Platforms **PikaPods:** 1. Click "New App" 2. Search "Open Notebook" 3. Set environment variables (at minimum: `OPEN_NOTEBOOK_ENCRYPTION_KEY`) 4. Click "Deploy" 5. Open the app → Go to **Settings → API Keys** to configure your AI provider **Railway:** 1. Create new project 2. Add `lfnovo/open_notebook:v1-latest-single` 3. Set environment variables (at minimum: `OPEN_NOTEBOOK_ENCRYPTION_KEY`) 4. Deploy 5. Open the app → Go to **Settings → API Keys** to configure your AI provider **Render:** 1. Create new Web Service 2. Use Docker image: `lfnovo/open_notebook:v1-latest-single` 3. Set environment variables in dashboard (at minimum: `OPEN_NOTEBOOK_ENCRYPTION_KEY`) 4. Configure persistent disk for `/app/data` and `/mydata` **DigitalOcean App Platform:** 1. Create new app from Docker Hub 2. Use image: `lfnovo/open_notebook:v1-latest-single` 3. Set port to 8502 4. Add environment variables (at minimum: `OPEN_NOTEBOOK_ENCRYPTION_KEY`) 5. Configure persistent storage **Heroku:** ```bash # Using heroku.yml heroku container:push web heroku container:release web heroku config:set OPEN_NOTEBOOK_ENCRYPTION_KEY=your-secret-key ``` **Coolify:** 1. Add new service → Docker Image 2. Image: `lfnovo/open_notebook:v1-latest-single` 3. Port: 8502 4. Add environment variables (at minimum: `OPEN_NOTEBOOK_ENCRYPTION_KEY`) 5. Enable persistent volumes 6. Coolify handles HTTPS automatically --- ## Environment Variables | Variable | Purpose | Example | |----------|---------|---------| | `OPEN_NOTEBOOK_ENCRYPTION_KEY` | Encryption key for credentials (required) | `my-secret-key` | | `SURREAL_URL` | Database | `ws://localhost:8000/rpc` | | `SURREAL_USER` | DB user | `root` | | `SURREAL_PASSWORD` | DB password | `root` | | `SURREAL_NAMESPACE` | DB namespace | `open_notebook` | | `SURREAL_DATABASE` | DB name | `open_notebook` | | `API_URL` | External URL (for remote access) | `https://myapp.example.com` | AI provider API keys are configured via the **Settings → API Keys** UI after deployment. --- ## Limitations vs Docker Compose | Feature | Single Container | Docker Compose | |---------|------------------|-----------------| | Setup time | 2 minutes | 5 minutes | | Complexity | Minimal | Moderate | | Services | All bundled | Separated | | Scalability | Limited | Excellent | | Memory usage | ~800MB | ~1.2GB | --- ## Next Steps Same as Docker Compose setup - just access via `http://localhost:8502` (local) or your platform's URL (cloud). 1. Go to **Settings → API Keys** to add your AI provider credential 2. **Test Connection** and **Discover Models** See [Docker Compose](docker-compose.md) for full post-install guide. ================================================ FILE: docs/2-CORE-CONCEPTS/ai-context-rag.md ================================================ # AI Context & RAG - How Open Notebook Uses Your Research Open Notebook uses different approaches to make AI models aware of your research depending on the feature. This section explains **RAG** (used in Ask) and **full-content context** (used in Chat). --- ## The Problem: Making AI Aware of Your Data ### Traditional Approaches (and their problems) **Option 1: Fine-Tuning** - Train the model on your data - Pro: Model becomes specialized - Con: Expensive, slow, permanent (can't unlearn) **Option 2: Send Everything to Cloud** - Upload all your data to ChatGPT/Claude API - Pro: Works well, fast - Con: Privacy nightmare, data leaves your control, expensive **Option 3: Ignore Your Data** - Just use the base model without your research - Pro: Private, free - Con: AI doesn't know anything about your specific topic ### Open Notebook's Dual Approach **For Chat**: Sends the entire selected content to the LLM - Simple and transparent: You select sources, they're sent in full - Maximum context: AI sees everything you choose - You control which sources are included **For Ask (RAG)**: Retrieval-Augmented Generation - RAG = Retrieval-Augmented Generation - The insight: *Search your content, find relevant pieces, send only those* - Automatic: AI decides what's relevant based on your question --- ## How RAG Works: Three Stages ### Stage 1: Content Preparation When you upload a source, Open Notebook prepares it for retrieval: ``` 1. EXTRACT TEXT PDF → text URL → webpage text Audio → transcribed text Video → subtitles + transcription 2. CHUNK INTO PIECES Long documents → break into ~500-word chunks Why? AI context has limits; smaller pieces are more precise 3. CREATE EMBEDDINGS Each chunk → semantic vector (numbers representing meaning) Why? Allows finding chunks by similarity, not just keywords 4. STORE IN DATABASE Chunks + embeddings + metadata → searchable storage ``` **Example:** ``` Source: "AI Safety Research 2026" (50-page PDF) ↓ Extracted: 50 pages of text ↓ Chunked: 150 chunks (~500 words each) ↓ Embedded: Each chunk gets a vector (1536 numbers for OpenAI) ↓ Stored: Ready for search ``` --- ### Stage 2: Query Time (What You Search For) When you ask a question, the system finds relevant content: ``` 1. YOU ASK A QUESTION "What does the paper say about alignment?" 2. SYSTEM CONVERTS QUESTION TO EMBEDDING Your question → vector (same way chunks are vectorized) 3. SIMILARITY SEARCH Find chunks most similar to your question (using vector math, not keyword matching) 4. RETURN TOP RESULTS Usually top 5-10 most similar chunks 5. YOU GET BACK ✓ The relevant chunks ✓ Where they came from (sources + page numbers) ✓ Relevance scores ``` **Example:** ``` Q: "What does the paper say about alignment?" ↓ Q vector: [0.23, -0.51, 0.88, ..., 0.12] ↓ Search: Compare to all chunk vectors ↓ Results: - Chunk 47 (alignment section): similarity 0.94 - Chunk 63 (safety approaches): similarity 0.88 - Chunk 12 (related work): similarity 0.71 ``` --- ### Stage 3: Augmentation (How AI Uses It) Now you have the relevant pieces. The AI uses them: ``` SYSTEM BUILDS A PROMPT: "You are an AI research assistant. The user has the following research materials: [CHUNK 47 CONTENT] [CHUNK 63 CONTENT] User question: 'What does the paper say about alignment?' Answer based on the above materials." AI RESPONDS: "Based on the research materials, the paper approaches alignment through [pulls from chunks] and emphasizes [pulls from chunks]..." SYSTEM ADDS CITATIONS: "- See research materials page 15 for approach details - See research materials page 23 for emphasis on X" ``` --- ## Two Search Modes: Exact vs. Semantic Open Notebook provides two different search strategies for different goals. ### 1. Text Search (Keyword Matching) **How it works:** - Uses BM25 ranking (the same algorithm Google uses) - Finds chunks containing your keywords - Ranks by relevance (how often keywords appear, position, etc.) **When to use:** - "I remember the exact phrase 'X' and want to find it" - "I'm looking for a specific name or number" - "I need the exact quote" **Example:** ``` Search: "transformer architecture" Results: 1. Chunk with "transformer architecture" 3 times 2. Chunk with "transformer" and "architecture" separately 3. Chunk with "transformer-based models" ``` ### 2. Vector Search (Semantic Similarity) **How it works:** - Converts your question to a vector (number embedding) - Finds chunks with similar vectors - No keywords needed—finds conceptually similar content **When to use:** - "Find content about X (without saying exact words)" - "I'm exploring a concept" - "Find similar ideas even if worded differently" **Example:** ``` Search: "what's the mechanism for model understanding?" Results (no "understanding" in any chunk): 1. Chunk about interpretability and mechanistic analysis 2. Chunk about feature analysis 3. Chunk about attention mechanisms Why? The vectors are semantically similar to your concept. ``` --- ## Context Management: Your Control Panel Here's where Open Notebook is different: **You decide what the AI sees.** ### The Three Levels | Level | What's Shared | Example Cost | Privacy | Use Case | |-------|---------------|--------------|---------|----------| | **Full Content** | Complete source text | 10,000 tokens | Low | Detailed analysis, close reading | | **Summary Only** | AI-generated summary | 2,000 tokens | High | Background material, references | | **Not in Context** | Nothing | 0 tokens | Max | Confidential, irrelevant, or archived | ### How It Works **Full Content:** ``` You: "What's the methodology in paper A?" System: - Searches paper A - Retrieves full paper content (or large chunks) - Sends to AI: "Here's paper A. Answer about methodology." - AI analyzes complete content - Result: Detailed, precise answer ``` **Summary Only:** ``` You: "I want to chat using paper A and B" System: - For Paper A: Sends AI-generated summary (not full text) - For Paper B: Sends full content (detailed analysis) - AI sees 2 sources but in different detail levels - Result: Uses summaries for context, details for focused content ``` **Not in Context:** ``` You: "I have 10 sources but only want 5 in context" System: - Paper A-E: In context (sent to AI) - Paper F-J: Not in context (AI can't see them, doesn't search them) - AI never knows these 5 sources exist - Result: Tight, focused context ``` ### Why This Matters **Privacy**: You control what leaves your system ``` Scenario: Confidential company docs + public research Control: Public research in context → Confidential docs excluded Result: AI never sees confidential content ``` **Cost**: You control token usage ``` Scenario: 100 sources for background + 5 for detailed analysis Control: Full content for 5 detailed, summaries for 95 background Result: 80% lower token cost than sending everything ``` **Quality**: You control what the AI focuses on ``` Scenario: 20 sources, question requires deep analysis Control: Full content for relevant source, exclude others Result: AI doesn't get distracted; gives better answer ``` --- ## The Difference: Chat vs. Ask **IMPORTANT**: These use completely different approaches! ### Chat: Full-Content Context (NO RAG) **How it works:** ``` YOU: 1. Select which sources to include in context 2. Set context level (full/summary/excluded) 3. Ask question SYSTEM: - Takes ALL selected sources (respecting context levels) - Sends the ENTIRE content to the LLM at once - NO search, NO retrieval, NO chunking - AI sees everything you selected AI: - Responds based on the full content you provided - Can reference any part of selected sources - Conversational: context stays for follow-ups ``` **Use this when**: - You know which sources are relevant - You want conversational back-and-forth - You want AI to see the complete context - You're doing close reading or analysis **Advantages:** - Simple and transparent - AI sees everything (no missed content) - Conversational flow **Limitations:** - Limited by LLM context window - You must manually select relevant sources - Sends more tokens (higher cost with many sources) --- ### Ask: RAG - Automatic Retrieval **How it works:** ``` YOU: Ask one complex question SYSTEM: 1. Analyzes your question 2. Searches across ALL your sources automatically 3. Finds relevant chunks using vector similarity 4. Retrieves only the most relevant pieces 5. Sends ONLY those chunks to the LLM 6. Synthesizes into comprehensive answer AI: - Sees ONLY the retrieved chunks (not full sources) - Answers based on what was found to be relevant - One-shot answer (not conversational) ``` **Use this when**: - You have many sources and don't know which are relevant - You want the AI to search automatically - You need a comprehensive answer to a complex question - You want to minimize tokens sent to LLM **Advantages:** - Automatic search (you don't pick sources) - Works across many sources at once - Cost-effective (sends only relevant chunks) **Limitations:** - Not conversational (single question/answer) - AI only sees retrieved chunks (might miss context) - Search quality depends on how well question matches content --- ## What This Means: Privacy by Design Open Notebook's RAG approach gives you something you don't get with ChatGPT or Claude directly: **You control the boundary between:** - What stays private (on your system) - What goes to AI (explicitly chosen) - What the AI can see (context levels) ### The Audit Trail Because everything is retrieved explicitly, you can ask: - "Which sources did the AI use for this answer?" → See citations - "What exactly did the AI see?" → See chunks in context level - "Is the AI's claim actually in my sources?" → Verify citation This prevents hallucinations or misrepresentation better than most systems. --- ## How Embeddings Work (Simplified) The magic of semantic search comes from embeddings. Here's the intuition: ### The Idea Instead of storing text, store it as a list of numbers (vectors) that represent "meaning." ``` Chunk: "The transformer uses attention mechanisms" Vector: [0.23, -0.51, 0.88, 0.12, ..., 0.34] (1536 numbers for OpenAI) Another chunk: "Attention allows models to focus on relevant parts" Vector: [0.24, -0.48, 0.87, 0.15, ..., 0.35] (similar numbers = similar meaning!) ``` ### Why This Works Words that are semantically similar produce similar vectors. So: - "alignment" and "interpretability" have similar vectors - "transformer" and "attention" have related vectors - "cat" and "dog" are more similar than "cat" and "radiator" ### How Search Works ``` Your question: "How do models understand their decisions?" Question vector: [0.25, -0.50, 0.86, 0.14, ..., 0.33] Compare to all stored vectors. Find the most similar: - Chunk about interpretability: similarity 0.94 - Chunk about explainability: similarity 0.91 - Chunk about feature attribution: similarity 0.88 Return the top matches. ``` This is why semantic search finds conceptually similar content even when words are different. --- ## Key Design Decisions ### 1. Search, Don't Train **Why?** Fine-tuning is slow and permanent. Search is flexible and reversible. ### 2. Explicit Retrieval, Not Implicit Knowledge **Why?** You can verify what the AI saw. You have audit trails. You control what leaves your system. ### 3. Multiple Search Types **Why?** Different questions need different search (keyword vs. semantic). Giving you both is more powerful. ### 4. Context as a Permission System **Why?** Not everything you save needs to reach AI. You control granularly. --- ## Summary Open Notebook gives you **two ways** to work with AI: ### Chat (Full-Content) - Sends entire selected sources to LLM - Manual control: you pick sources - Conversational: back-and-forth dialog - Transparent: you know exactly what AI sees - Best for: focused analysis, close reading ### Ask (RAG) - Searches and retrieves relevant chunks automatically - Automatic: AI finds what's relevant - One-shot: single comprehensive answer - Efficient: sends only relevant pieces - Best for: broad questions across many sources **Both approaches:** 1. Keep your data private (doesn't leave your system by default) 2. Give you control (you choose which features to use) 3. Create audit trails (citations show what was used) 4. Support multiple AI providers **Coming Soon**: The community is working on adding RAG capabilities to Chat as well, giving you the best of both worlds. ================================================ FILE: docs/2-CORE-CONCEPTS/chat-vs-transformations.md ================================================ # Chat vs. Ask vs. Transformations - Which Tool for Which Job? Open Notebook offers different ways to work with your research. Understanding when to use each is key to using the system effectively. --- ## The Three Interaction Modes ### 1. CHAT - Conversational Exploration with Manual Context **What it is:** Have a conversation with AI about selected sources. **The flow:** ``` 1. You select which sources to include ("in context") 2. You ask a question 3. AI responds using ONLY those sources 4. You ask follow-up questions (context stays same) 5. You change sources or context level, then continue ``` **Context management:** You explicitly choose which sources the AI can see. **Conversational:** Multiple questions with shared history. **Example:** ``` You: [Select sources: "paper1.pdf", "research_notes.txt"] [Set context: Full content for paper1, Summary for notes] You: "What's the main argument in these sources?" AI: "Paper 1 argues X [citation]. Your notes emphasize Y [citation]." You: "How do they differ?" AI: "Paper 1 focuses on X [citation], while your notes highlight Y [citation]..." You: [Now select different sources] You: "Compare to this other perspective" AI: "This new source takes a different approach..." ``` **Best for:** - Exploring a focused topic with specific sources - Having a dialogue (multiple back-and-forth questions) - When you know which sources matter - When you want tight control over what goes to AI --- ### 2. ASK - Automated Comprehensive Search **What it is:** Ask one complex question, system automatically finds relevant content. **The flow:** ``` 1. You ask a comprehensive question 2. System analyzes the question 3. System automatically searches your sources 4. System retrieves relevant chunks 5. System synthesizes answer from all results 6. You get one detailed answer (not conversational) ``` **Context management:** Automatic. System figures out what's relevant. **Non-conversational:** One question → one answer. No follow-ups. **Example:** ``` You: "How do these papers compare their approaches to alignment? What does each one recommend?" System: - Breaks down the question into search strategies - Searches all sources for alignment approaches - Searches all sources for recommendations - Retrieves top 10 relevant chunks - Synthesizes: "Paper A recommends X [citation]. Paper B recommends Y [citation]. They differ in Z." You: [Get back one comprehensive answer] [If you want to follow up, use Chat instead] ``` **Best for:** - Comprehensive, one-time questions - Comparing multiple sources at once - When you want the system to decide what's relevant - Complex questions that need multiple search angles - When you don't need a back-and-forth conversation --- ### 3. TRANSFORMATIONS - Template-Based Processing **What it is:** Apply a reusable template to a source and get structured output. **The flow:** ``` 1. You define a transformation (or choose a preset) "Extract: main argument, methodology, limitations" 2. You apply it to ONE source at a time (You can repeat for other sources) 3. For the source: - Source content + transformation prompt → AI - Result stored as new insight/note 4. You get back - Structured output (main argument, methodology, limitations) - Saved as a note in your notebook ``` **Context management:** Works on one source at a time. **Reusable:** Apply the same template to different sources (one by one). **Note**: Currently processes one source at a time. Batch processing (multiple sources at once) is planned for a future release. **Example:** ``` You: Define transformation "For each academic paper, extract: - Main research question - Methodology used - Key findings - Limitations and gaps - Recommended next research" You: Apply to paper 1 System: - Runs the transformation on paper 1 - Result stored as new note You: Apply same transformation to paper 2, 3, etc. After 10 papers: - You have 10 structured notes with consistent format - Perfect for writing a literature review or comparison ``` **Best for:** - Extracting the same information from each source (run repeatedly) - Creating structured summaries with consistent format - Building a knowledge base of categorized insights - When you want reusable templates you can apply to each source --- ## Decision Tree: Which Tool to Use? ``` What are you trying to do? │ ├─→ "I want to have a conversation about this topic" │ └─→ Is the conversation exploratory or fixed? │ ├─→ Exploratory (I'll ask follow-ups) │ │ └─→ USE: CHAT │ │ │ └─→ Fixed (One question → done) │ └─→ Go to next question │ ├─→ "I need to compare these sources or get a comprehensive answer" │ └─→ USE: ASK │ ├─→ "I want to extract the same info from each source (one at a time)" │ └─→ USE: TRANSFORMATIONS (apply to each source) │ └─→ "I just want to read and search" └─→ USE: Search (text or vector) OR read your notes ``` --- ## Side-by-Side Comparison | Aspect | CHAT | ASK | TRANSFORMATIONS | |--------|------|-----|-----------------| | **What's it for?** | Conversational exploration | Comprehensive Q&A | Template-based extraction | | **# of questions** | Multiple (conversational) | One | One template per source | | **Context control** | Manual (you choose) | Automatic (system searches) | One source at a time | | **Conversational?** | Yes (follow-ups work) | No (one question only) | No (single operation) | | **Output** | Natural conversation | Natural answer | Structured note | | **Time** | Quick (back-and-forth) | Longer (comprehensive) | Per source | | **Best when** | Exploring & uncertain | Need full picture | Want consistent format | | **Model speed** | Any | Fast preferred | Any | --- ## Workflow Examples ### Example 1: Academic Research ``` Goal: Write literature review from 15 papers Step 1: TRANSFORMATIONS - Define: "Extract abstract, methodology, findings, relevance" - Apply to paper 1 → get structured note - Apply to paper 2 → get structured note - ... repeat for all 15 papers - Result: 15 structured notes with consistent format Step 2: Read the notes - Now you have consistent summaries Step 3: CHAT or ASK - Chat: "Help me organize these by theme" - Ask: "What are the common methodologies across these papers?" Step 4: Write your review - Use the transformations as foundation - Use chat/ask insights for structure ``` ### Example 2: Product Research ``` Goal: Understand customer feedback from interviews Step 1: Add sources (interview transcripts) Step 2: ASK - "What are the top 10 pain points mentioned?" - Get comprehensive answer with citations Step 3: CHAT - "Can you help me group these by severity?" - Continue conversation to prioritize Step 4: TRANSFORMATIONS (optional) - Define: "Extract: pain point, frequency, who mentioned it" - Apply to each interview (one by one) - Get structured data for analysis ``` ### Example 3: Policy Analysis ``` Goal: Compare policy documents Step 1: Add all policy documents as sources Step 2: ASK - "How do these policies differ on climate measures?" - System searches all docs, gives comprehensive comparison Step 3: CHAT (if needed) - "Which policy is most aligned with X goals?" - Have discussion about trade-offs Step 4: Export notes - Save AI responses as notes for reports ``` --- ## Context Management: The Control Panel All three modes let you control what the AI sees. ### In CHAT and TRANSFORMATIONS ``` You choose: - Which sources to include - Context level for each: ✓ Full Content (send complete text) ✓ Summary Only (send AI summary, not full text) ✓ Not in Context (exclude entirely) Example: Paper A: Full Content (analyzing closely) Paper B: Summary Only (background) Paper C: Not in Context (confidential) ``` ### In ASK ``` Context is automatic: - System searches ALL your sources - Retrieves most relevant chunks - Sends those to AI But you can: - Search in specific notebook - Filter by source type - Use the results to decide context for follow-up Chat ``` --- ## Model Selection Each mode works with different models: ### CHAT - **Any model** works fine - Fast models (GPT-4o mini, Claude Haiku): Quick responses, good for conversation - Powerful models (GPT-4o, Claude Sonnet): Better reasoning, better for complex topics ### ASK - **Fast models preferred** (because it processes multiple searches) - Can use powerful models if you want deep synthesis - Example: GPT-4 for strategy planning, GPT-4o-mini for quick facts ### TRANSFORMATIONS - **Any model** works - Fast models (cost-effective for batch processing) - Powerful models (better quality extractions) --- ## Advanced: Chaining Modes Together You can combine these modes: ``` TRANSFORMATIONS → CHAT 1. Use transformations to extract structured data 2. Use chat to discuss the results ASK → TRANSFORMATIONS 1. Use Ask to understand what matters 2. Use Transformations to extract it from remaining sources CHAT → Save as Note → TRANSFORMATIONS 1. Have conversation (Chat) 2. Save good responses as notes 3. Use those notes as context for transformations ``` --- ## Summary: When to Use Each | Situation | Use | Why | |-----------|-----|-----| | "I want to explore a topic with follow-up questions" | **CHAT** | Conversational, you control context | | "I need a comprehensive answer to one complex question" | **ASK** | Automatic search, synthesized answer | | "I want consistent summaries from each source" | **TRANSFORMATIONS** | Template reuse, apply to each source | | "I'm comparing two specific sources" | **CHAT** | Select just those 2, have discussion | | "I need to categorize each source by X criteria" | **TRANSFORMATIONS** | Extract category from each source | | "I want to understand the big picture across all sources" | **ASK** | Automatic comprehensive search | | "I want to build a knowledge base" | **TRANSFORMATIONS** | Create structured note from each source | | "I want to iterate on understanding" | **CHAT** | Multiple questions, refine thinking | The key insight: **Different questions need different tools.** Open Notebook gives you all three because research rarely fits one mode. ================================================ FILE: docs/2-CORE-CONCEPTS/index.md ================================================ # Core Concepts - Understand the Mental Model Before diving into how to use Open Notebook, it's important to understand **how it thinks**. These core concepts explain the "why" behind the design. ## The Five Mental Models ### 1. [Notebooks, Sources, and Notes](notebooks-sources-notes.md) How Open Notebook organizes your research. Understand the three-tier container structure and how information flows from raw materials to finished insights. **Key idea**: A notebook is a scoped research container. Sources are inputs (PDFs, URLs, etc.). Notes are outputs (your insights, AI-generated summaries, captured responses). --- ### 2. [AI Context & RAG](ai-context-rag.md) How Open Notebook makes AI aware of your research - two different approaches. **Key idea**: **Chat** sends entire selected sources to the LLM (full context, conversational). **Ask** uses RAG (retrieval-augmented generation) to automatically search and retrieve only relevant chunks. Different tools for different needs. --- ### 3. [Chat vs. Transformations](chat-vs-transformations.md) Why Open Notebook has different interaction modes and when to use each one. **Key idea**: Chat is conversational exploration (you control context). Transformations are insight extractions. They reduced content to smaller bits of concentrated/dense information, which is much more suitable for an AI to use. --- ### 4. [Context Management](chat-vs-transformations.md#context-management-the-control-panel) Your control panel for privacy and cost. Decide what data actually reaches AI. **Key idea**: You choose three levels—not in context (private), summary only (condensed), or full content (complete access). This gives you fine-grained control. --- ### 5. [Podcasts Explained](podcasts-explained.md) Why Open Notebook can turn research into audio and why this matters. **Key idea**: Podcasts transform your research into a different consumption format. Instead of reading, someone can listen and absorb your insights passively. --- ## Read This Section If: - **You're new to Open Notebook** — Start here to understand how the system works conceptually before learning the features - **You're confused about Chat vs Ask** — Section 2 explains the difference (full-content vs RAG) - **You're wondering when to use Chat vs Transformations** — Section 3 clarifies the differences - **You want to understand privacy controls** — Section 4 shows you what you can control - **You're curious about podcasts** — Section 5 explains the architecture and why it's different from competitors --- ## The Big Picture Open Notebook is built on a simple insight: **Your research deserves to stay yours**. That means: - **Privacy by default** — Your data doesn't leave your infrastructure unless you explicitly choose - **AI as a tool, not a gatekeeper** — You decide which sources the AI sees, not the AI deciding for you - **Flexible consumption** — Read, listen, search, chat, or transform your research however makes sense These core concepts explain how that works. --- ## Next Steps 1. **Just want to use it?** → Go to [User Guide](../3-USER-GUIDE/index.md) 2. **Want to understand it first?** → Read the 5 sections above (15 min) 3. **Setting up for the first time?** → Go to [Installation](../1-INSTALLATION/index.md) ================================================ FILE: docs/2-CORE-CONCEPTS/notebooks-sources-notes.md ================================================ # Notebooks, Sources, and Notes - The Container Model Open Notebook organizes research in three connected layers. Understanding this hierarchy is key to using the system effectively. ## The Three-Layer Structure ``` ┌─────────────────────────────────────┐ │ NOTEBOOK (The Container) │ │ "My AI Safety Research 2026" │ ├─────────────────────────────────────┤ │ │ │ SOURCES (The Raw Materials) │ │ ├─ safety_paper.pdf │ │ ├─ alignment_video.mp4 │ │ └─ prompt_injection_article.html │ │ │ │ NOTES (The Processed Insights) │ │ ├─ AI Summary (auto-generated) │ │ ├─ Key Concepts (transformation) │ │ ├─ My Research Notes (manual) │ │ └─ Chat Insights (from conversation) │ │ └─────────────────────────────────────┘ ``` --- ## 1. NOTEBOOKS - The Research Container ### What Is a Notebook? A **notebook** is a *scoped container* for a research project or topic. It's your research workspace. Think of it like a physical notebook: everything inside is about the same topic, shares the same context, and builds toward the same goals. ### What Goes In? - **A description** — "This notebook collects research on X topic" - **Sources** — The raw materials you add - **Notes** — Your insights and outputs - **Conversation history** — Your chats and questions ### Why This Matters **Isolation**: Each notebook is completely separate. Sources in Notebook A never appear in Notebook B. This lets you: - Keep different research topics completely isolated - Reuse source names across notebooks without conflicts - Control which AI context applies to which research **Shared Context**: All sources and notes in a notebook inherit the notebook's context. If your notebook is titled "AI Safety 2026" with description "Focusing on alignment and interpretability," that context applies to all AI interactions within that notebook. **Parallel Projects**: You can have 10 notebooks running simultaneously. Each one is its own isolated research environment. ### Example ``` Notebook: "Customer Research - Product Launch" Description: "User interviews and feedback for Q1 2026 launch" → All sources added to this notebook are about customer feedback → All notes generated are in that context → When you chat, the AI knows you're analyzing product launch feedback → Different from your "Market Analysis - Competitors" notebook ``` --- ## 2. SOURCES - The Raw Materials ### What Is a Source? A **source** is a *single piece of input material* — the raw content you bring in. Sources never change; they're just processed and indexed. ### What Can Be a Source? - **PDFs** — Research papers, reports, documents - **Web links** — Articles, blog posts, web pages - **Audio files** — Podcasts, interviews, lectures - **Video files** — Tutorials, presentations, recordings - **Plain text** — Notes, transcripts, passages - **Uploaded text** — Paste content directly ### What Happens When You Add a Source? ``` 1. EXTRACTION File/URL → Extract text and metadata (OCR for PDFs, web scraping for URLs, speech-to-text for audio) 2. CHUNKING Long text → Break into searchable chunks (Prevents "too much context" in single query) 3. EMBEDDING Each chunk → Generate semantic vector (Allows AI to find conceptually similar content) 4. STORAGE Chunks + vectors → Store in database (Ready for search and retrieval) ``` ### Key Properties **Immutable**: Once added, the source doesn't change. If you need a new version, add it as a new source. **Indexed**: Sources are automatically indexed for search (both text and semantic). **Scoped**: A source belongs to exactly one notebook. **Referenceable**: Other sources and notes can reference this source by citation. ### Example ``` Source: "openai_charter.pdf" Type: PDF document What happens: → PDF is uploaded → Text is extracted (including images) → Text is split into 50 chunks (paragraphs, sections) → Each chunk gets an embedding vector → Now searchable by: "OpenAI's approach to safety" ``` --- ## 3. NOTES - The Processed Insights ### What Is a Note? A **note** is a *processed output* — something you created or AI created based on your sources. Notes are the "results" of your research work. ### Types of Notes #### Manual Notes You write them yourself. They're your original thinking, capturing: - What you learned from sources - Your analysis and interpretations - Your next steps and questions #### AI-Generated Notes Created by applying AI processing to sources: - **Transformations** — Structured extraction (main points, key concepts, methodology) - **Chat Responses** — Answers you saved from conversations - **Ask Results** — Comprehensive answers saved to your notebook #### Captured Insights Notes you explicitly saved from interactions: - "Save this response as a note" - "Save this transformation result" - Convert any AI output into a permanent note ### What Can Notes Contain? - **Text** — Your writing or AI-generated content - **Citations** — References to specific sources - **Metadata** — When created, how created (manual/AI), which sources influenced it - **Tags** — Your categorization (optional but useful) ### Why Notes Matter **Knowledge Accumulation**: Notes become your actual knowledge base. They're what you take away from the research. **Searchable**: Notes are searchable along with sources. "Find everything about X" includes your notes, not just sources. **Citable**: Notes can cite sources, creating an audit trail of where insights came from. **Shareable**: Notes are your outputs. You can share them, publish them, or build on them in other projects. --- ## How They Connect: The Data Flow ``` YOU │ ├─→ Create Notebook ("AI Research") │ ├─→ Add Sources (papers, articles, videos) │ └─→ System: Extract, embed, index │ ├─→ Search Sources (text or semantic) │ └─→ System: Find relevant chunks │ ├─→ Apply Transformations (extract insights) │ └─→ Creates Notes │ ├─→ Chat with Sources (explore with context control) │ ├─→ Can save responses as Notes │ └─→ Notes include citations │ ├─→ Ask Questions (automated comprehensive search) │ ├─→ Can save results as Notes │ └─→ Notes include citations │ └─→ Generate Podcast (transform notebook into audio) └─→ Uses all sources + notes for content ``` --- ## Key Design Decisions ### 1. One Notebook Per Source Each source belongs to exactly one notebook. This creates clear boundaries: - No ambiguity about which research project a source is in - Easy to isolate or export a complete project - Clean permissions model (if someone gets access to notebook, they get access to all its sources) ### 2. Immutable Sources, Mutable Notes Sources never change (once added, always the same). But notes can be edited or deleted. Why? - Sources are evidence → evidence shouldn't be altered - Notes are your thinking → thinking evolves as you learn ### 3. Explicit Context Control Sources don't automatically go to AI. You decide which sources are "in context" for each interaction: - Chat: You manually select which sources to include - Ask: System automatically figures out which sources to search - Transformations: You choose which sources to transform This is different from systems that always send everything to AI. --- ## Mental Models Explained ### Notebook as Boundaries Think of a notebook like a Git repository: - Everything in it is about the same topic - You can clone/fork it (copy to new project) - It has clear entry/exit points - You know exactly what's included ### Sources as Evidence Think of sources like exhibits in a legal case: - Once filed, they don't change - They can be cited and referenced - They're the ground truth for what you're basing claims on - Multiple sources can be cross-referenced ### Notes as Synthesis Think of notes like your case brief: - You write them based on evidence - They're your interpretation - You can cite which evidence supports each claim - They're what you actually share or act on --- ## Common Questions ### Can I move a source to a different notebook? Not directly. Each source is tied to one notebook. If you want it in multiple notebooks, add it again (uploads are fast if it's already processed). ### Can a note reference sources from a different notebook? No. Notes stay within their notebook and reference sources within that notebook. This keeps boundaries clean. ### What if I want to group sources within a notebook? Use tags. You can tag sources ("primary research," "background," "methodology") and filter by tags. ### Can I merge two notebooks? Not built-in, but you can manually copy sources from one notebook to another by re-uploading them. --- ## Summary | Concept | Purpose | Lifecycle | Scope | |---------|---------|-----------|-------| | **Notebook** | Container + context | Create once, configure | All its sources + notes | | **Source** | Raw material | Add → Process → Store | One notebook | | **Note** | Processed output | Create/capture → Edit → Share | One notebook | This three-layer model gives you: - **Clear organization** (everything scoped to projects) - **Privacy control** (isolated notebooks) - **Audit trails** (notes cite sources) - **Flexibility** (notes can be manual or AI-generated) ================================================ FILE: docs/2-CORE-CONCEPTS/podcasts-explained.md ================================================ # Podcasts Explained - Research as Audio Dialogue Podcasts are Open Notebook's highest-level transformation: converting your research into audio dialogue for a different consumption pattern. --- ## Why Podcasts Matter ### The Problem Research naturally accumulates as text: PDFs, articles, web pages, notes. This creates a friction point: **To consume research, you must:** - Sit down at a desk - Focus intently - Read actively - Take notes - Set aside dedicated time **But much of life is passive time:** - Commuting - Exercising - Doing dishes - Driving - Walking - Idle moments ### The Solution Convert your research into audio dialogue so you can consume it passively. ``` Before (Text-based): Research pile → Must schedule reading time → Requires focus After (Podcast): Research pile → Podcast → Can listen while commuting → Absorb while exercising → Understand while walking → Engage without screen time ``` --- ## What Makes It Special: Open Notebook vs. Competitors ### Google Notebook LM Podcasts - **Fixed format**: 2 hosts, always conversational - **Limited customization**: You can't choose who the "hosts" are - **One TTS voice per speaker**: Can't customize voices - **Only uses cloud services**: No local options ### Open Notebook Podcasts - **Customizable format**: 1-4 speakers, you design them - **Rich speaker profiles**: Create personas with backstories and expertise - **Multiple TTS options**: - OpenAI (natural, fast) - Google TTS (high quality) - ElevenLabs (beautiful voices, accents) - Local TTS (privacy-first, no API calls) - **Async generation**: Doesn't block your work - **Full control**: Choose outline structure, tone, depth --- ## How Podcast Generation Works ### Stage 1: Content Selection You choose what goes into the podcast: ``` Notebook content → Which sources? → Which notes? → Which topics to focus on? → Depth of coverage? ``` ### Stage 2: Episode Profile You define how you want the podcast structured: ``` Episode Profile ├─ Topic: "AI Safety Approaches" ├─ Length: 20 minutes ├─ Tone: Academic but accessible ├─ Format: Debate (2 speakers with opposing views) ├─ Audience: Researchers new to the field └─ Focus areas: Main approaches, pros/cons, open questions ``` ### Stage 3: Speaker Configuration You create speaker personas (1-4 speakers): ``` Speaker 1: "Expert Alex" ├─ Expertise: "Deep knowledge of alignment research" ├─ Personality: "Rigorous, academic, patient with explanation" ├─ Accent: (Optional) "British English" └─ Voice Model: Selected from model registry (e.g., OpenAI TTS) └─ Optional per-speaker override of the episode's default voice model Speaker 2: "Researcher Sam" ├─ Expertise: "Field observer, pragmatic perspective" ├─ Personality: "Curious, asks clarifying questions" ├─ Accent: "American English" └─ Voice Model: Selected from model registry (e.g., ElevenLabs TTS) ``` ### Stage 4: Outline Generation System generates episode outline: ``` EPISODE: "AI Safety Approaches" 1. Introduction (2 min) Alex: Introduces topic and speakers Sam: What will we cover today? 2. Main Approaches (8 min) Alex: Explains top 3 approaches Sam: Asks about tradeoffs 3. Debate: Best approach? (6 min) Alex: Advocates for approach A Sam: Argues for approach B 4. Open Questions (3 min) Both: What's unsolved? 5. Conclusion (1 min) Recap and where to learn more ``` ### Stage 5: Dialogue Generation System generates dialogue based on outline: ``` Alex: "Today we're exploring three major approaches to AI alignment..." Sam: "That's a great start. Can you break down what we mean by alignment?" Alex: "Good question. Alignment means ensuring AI systems pursue the goals we actually want them to pursue, not just what we literally asked for. There's a classic example of a paperclip maximizer..." Sam: "Interesting. So it's about solving the intention problem?" Alex: "Exactly. And that's where the three approaches come in..." ``` ### Stage 6: Text-to-Speech System converts dialogue to audio using the voice models configured in the model registry. Credentials are automatically resolved from each model's configuration. ``` Alex's text → Voice model (from registry) → Alex's voice (audio file) Sam's text → Voice model (from registry) → Sam's voice (audio file) Audio files → Mix together → Final podcast MP3 ``` --- ## When Things Go Wrong: Failures & Retry Podcast generation involves multiple steps (outline, transcript, TTS) and depends on external AI providers. Sometimes things fail. ### What Happens on Failure When podcast generation fails (e.g., wrong model configured, API key expired, provider outage): - The episode is marked as **Failed** with a red badge - The **error message** from the AI provider is displayed so you can understand what went wrong - No duplicate episodes are created — automatic retries are disabled to prevent confusion ### How to Retry a Failed Episode 1. Go to the podcast's **Episodes** tab 2. Find the failed episode — it shows a red "FAILED" badge and an error details box 3. Click the **Retry** button 4. The failed episode is deleted and a new generation job is submitted 5. The new episode appears with "pending" status ### Common Failure Causes | Error | What to Do | |-------|-----------| | Invalid API key | Check Settings -> Credentials for the TTS and language model providers | | Model not found | Verify the model exists in the model registry and has valid credentials configured | | Rate limit exceeded | Wait a few minutes and retry | | Provider unavailable | Check provider status page; retry later | --- ## Key Architecture Decisions ### 1. Asynchronous Processing Podcasts are generated in the background. You upload → system processes → you download when ready. **Why?** Podcast generation takes time (10+ minutes for a 30-minute episode). Blocking would lock up your interface. ### 2. Multi-Speaker Support Unlike Google Notebook LM (always 2 hosts), you choose 1-4 speakers. **Why?** Different discussions work better with different formats: - Expert monologue (1 speaker) - Interview (2 speakers: host + expert) - Debate (2 speakers: opposing views) - Panel discussion (3-4 speakers: different expertise) ### 3. Speaker Customization You create rich speaker profiles, not just "Host A" and "Host B". **Why?** Makes podcasts more engaging and authentic. Different speakers bring different perspectives. ### 4. Multiple TTS Providers You're not locked into one voice provider. **Why?** - Cost optimization (some providers cheaper) - Quality preferences (some voices more natural) - Privacy options (local TTS for sensitive content) - Accessibility (different accents, genders, styles) ### 5. Local TTS Option Can generate podcasts entirely offline with local text-to-speech. **Why?** For sensitive research, never send audio to external APIs. --- ## Use Cases Show Why This Matters ### Academic Publishing ``` Traditional: Academic paper → PDF Problem: Hard to consume, linear reading required Open Notebook: Research materials → Podcast (expert explaining methodology) → Podcast (debate format: different interpretations) → Different consumption for different audiences ``` ### Content Creation ``` Blog creator: Has research pile on a topic Problem: Doesn't have time to write the article Solution: Add research → Create podcast → Transcribe → Becomes article OR: Podcast BECOMES the content (upload to podcast platforms) ``` ### Educational Content ``` Educator: Has reading materials for a course Problem: Students don't read the papers Solution: Create podcast with expert explaining papers Students listen → Better engagement → Discussions can reference podcast ``` ### Market Research ``` Product manager: Has interviews with customers Problem: Too many hours of audio to review Solution: Create podcast with debate format (customer perspective vs. team perspective) Much more engaging than raw transcripts ``` ### Knowledge Transfer ``` Domain expert: Leaving the organization Problem: How to preserve expertise? Solution: Create expert-mode podcast explaining frameworks, decision-making, context New team member listens, gets context faster than reading 100 documents ``` --- ## The Difference: Active vs. Passive Learning ### Text-Based Research (Active) - **Effort**: High (must focus, read, synthesize) - **When**: Dedicated study time - **Cost**: Time is expensive (can't multitask) - **Best for**: Deep dives, precise information - **Format**: Whatever you write (notes, articles, books) ### Audio Podcast (Passive) - **Effort**: Low (just listen) - **When**: Anywhere, anytime - **Cost**: Low (can multitask) - **Best for**: Overview, context, exploration - **Format**: Dialogue (more engaging than narration) **They complement each other:** 1. **First encounter**: Listen to podcast (passive, get context) 2. **Deep dive**: Read source materials (active, precise) 3. **Mastery**: Both together (understand big picture + details) --- ## How Podcasts Fit Into Your Workflow ``` 1. Build notebook (add sources) ↓ 2. Apply transformations (extract insights) ↓ 3. Chat/Ask (explore content) ↓ 4. Decide on podcast ├─→ Create speaker profiles ├─→ Define episode profile ├─→ Configure voice models (from model registry) └─→ Generate podcast ↓ 5. Listen while commuting/exercising ↓ 6. Reference sources for deep dive ↓ 7. Repeat for different formats/speakers/focus ``` --- ## Advanced: Multiple Podcasts from Same Research You can create different podcasts from the same sources: ### Example: AI Safety Research ``` Podcast 1: "Expert Monologue" Speaker: Researcher explaining field Format: Educational, comprehensive Audience: Students new to field Podcast 2: "Debate Format" Speakers: Optimist vs. skeptic Format: Discussion of tradeoffs Audience: Advanced researchers Podcast 3: "Interview Format" Speakers: Journalist + expert Format: Q&A about practical applications Audience: Industry practitioners ``` Each tells the same story from different angles. --- ## Privacy & Data Considerations ### Where Your Data Goes **Option 1: Cloud TTS (Faster, Higher Quality)** ``` Your outline → API call to TTS provider → Audio returned → Stored in your notebook Provider sees: Your outlined script (not raw sources) Privacy level: Medium (outline is shared, sources aren't) ``` **Option 2: Local TTS (Slower, Maximum Privacy)** ``` Your outline → Local TTS engine (runs on your machine) → Audio generated locally → Stored in your notebook Provider sees: Nothing Privacy level: Maximum (everything local) ``` ### Recommendation - **Sensitive research**: Use local TTS, no API calls - **Less sensitive**: Use ElevenLabs or Google (both handle audio data professionally) - **Mixed**: Use local TTS for speakers reading sensitive content --- ## Cost Considerations ### Cloud TTS Costs | Provider | Cost | Quality | Speed | |----------|------|---------|-------| | OpenAI | ~$0.015 per minute | Good | Fast | | Google | ~$0.004 per minute | Excellent | Fast | | ElevenLabs | ~$0.10 per minute | Exceptional | Medium | | Local TTS | Free | Basic | Slow | A 30-minute podcast costs: - OpenAI: ~$0.45 - Google: ~$0.12 - ElevenLabs: ~$3.00 - Local: Free (but slow) --- ## Summary: Why Podcasts Are Special **Podcasts transform your research consumption:** | Aspect | Text | Podcast | |--------|------|---------| | **How consumed?** | Active reading | Passive listening | | **Where consumed?** | Desk | Anywhere | | **Multitasking** | Hard | Easy | | **Time commitment** | Scheduled | Flexible | | **Format** | Whatever | Natural dialogue | | **Engagement** | Academic | Conversational | | **Accessibility** | Text-based | Audio-based | **In Open Notebook specifically:** - **Full customization** — you create speakers and format - **Privacy options** — local TTS for sensitive content - **Cost control** — choose TTS provider based on budget - **Non-blocking** — generates in background - **Multiple versions** — create different podcasts from same research This is why podcasts matter: they change *when* and *how* you can consume your research. ================================================ FILE: docs/3-USER-GUIDE/adding-sources.md ================================================ # Adding Sources - Getting Content Into Your Notebook Sources are the raw materials of your research. This guide covers how to add different types of content. --- ## Quick-Start: Add Your First Source ### Option 1: Upload a File (PDF, Word, etc.) ``` 1. In your notebook, click "Add Source" 2. Select "Upload File" 3. Choose a file from your computer 4. Click "Upload" 5. Wait 30-60 seconds for processing 6. Done! Source appears in your notebook ``` ### Option 2: Add a Web Link ``` 1. Click "Add Source" 2. Select "Web Link" 3. Paste URL: https://example.com/article 4. Click "Add" 5. Wait for processing (usually faster than files) 6. Done! ``` ### Option 3: Paste Text ``` 1. Click "Add Source" 2. Select "Text" 3. Paste or type your content 4. Click "Save" 5. Done! Immediately available ``` --- ## Supported File Types ### Documents - **PDF** (.pdf) — Best support, including scanned PDFs with OCR - **Word** (.docx, .doc) — Full support - **PowerPoint** (.pptx) — Slides converted to text - **Excel** (.xlsx, .xls) — Spreadsheet data - **EPUB** (.epub) — eBook files - **Markdown** (.md, .txt) — Plain text formats - **HTML** (.html, .htm) — Web page files **File size limits:** Up to ~100MB (varies by system) **Processing time:** 10 seconds - 2 minutes (depending on length and file type) ### Audio & Video - **Audio**: MP3, WAV, M4A, OGG, FLAC (~30 seconds - 3 minutes per hour) - **Video**: MP4, AVI, MOV, MKV, WebM (~3-10 minutes per hour) - **YouTube**: Direct URL support - **Podcasts**: RSS feed URL **Automatic transcription**: Audio/video is transcribed to text automatically. This requires enabling speech-to-text in settings. ### Web Content - **Articles**: Blog posts, news articles, Medium - **YouTube**: Full videos or playlists - **PDFs online**: Direct PDF links - **News**: News site articles **Just paste the URL** in "Web Link" section. ### What Doesn't Work - Paywalled content (WSJ, FT, etc.) — Can't extract - Password-protected PDFs — Can't open - Pure image files (.jpg, .png) — Except scanned PDFs which have OCR - Very large files (>100MB) — Timeout --- ## What Happens When You Add a Source The system automatically does four things: ``` 1. EXTRACT TEXT File/URL → Readable text (PDFs get OCR if scanned) (Videos get transcribed if enabled) 2. BREAK INTO CHUNKS Long text → ~500-word pieces (So search finds specific parts, not whole document) 3. CREATE EMBEDDINGS Each chunk → Vector representation (Enables semantic/concept search) 4. INDEX & STORE Everything → Database (Ready to search and retrieve) ``` **Time to use:** After the progress bar completes, the source is ready immediately. Embeddings are created in the background. --- ## Step-by-Step for Different Types ### PDFs **Best practices:** ``` Clean PDFs: 1. Upload → Done 2. Processing time: ~30-60 seconds Scanned/Image PDFs: 1. Upload same way 2. System auto-detects and uses OCR 3. Processing time: ~2-3 minutes 4. (Higher, due to OCR overhead) Large PDFs (50+ pages): 1. Consider splitting into smaller files 2. Or upload as-is (system handles it) 3. Processing time scales with size ``` **Common issues:** - "Can't extract text" → PDF is corrupted or has copy protection - Solution: Try opening in Adobe. If it won't, the PDF is likely protected. ### Web Links / Articles **Best practices:** ``` 1. Copy full URL from browser: https://example.com/article-title 2. Paste in "Web Link" 3. Click Add 4. Wait for extraction Processing time: Usually 5-15 seconds ``` **What works:** - Standard web articles - Blog posts - News articles - Wikipedia pages - Medium posts - Substack articles **What doesn't work:** - Twitter threads (unreliable) - Paywalled articles (can't access) - JavaScript-heavy sites (content not extracted) **Pro tip:** If it doesn't work, copy the article text and paste as "Text" instead. ### Audio Files **Best practices:** ``` 1. Ensure speech-to-text is enabled in Settings 2. Upload MP3, WAV, or M4A file 3. System automatically transcribes to text 4. Processing time: ~1 minute per 5 minutes of audio Example: - 1-hour podcast → 12 minutes processing - 10-minute recording → 2 minutes processing ``` **Quality matters:** - Clear audio: Fast transcription - Muffled/noisy audio: Slower, less accurate transcription - Background noise: Try to minimize before uploading **Tip:** If audio quality is poor, the AI might misinterpret content. You can manually correct transcription if needed. ### YouTube Videos **Best practices:** ``` Two ways to add: Method 1: Direct URL 1. Copy YouTube URL: https://www.youtube.com/watch?v=... 2. Paste in "Web Link" 3. Click Add 4. System extracts captions (if available) + transcript Method 2: Playlist 1. Paste playlist URL 2. System adds all videos as separate sources 3. Each video processed separately 4. Takes longer (multiple videos) ``` **What's extracted:** - Captions/subtitles (if available) - Transcription (if captions aren't available) - Basic metadata (title, channel, length) **Processing:** - 10-minute video: ~2-3 minutes - 1-hour video: ~10-15 minutes ### Text / Paste Content **Best practices:** ``` 1. Select "Text" when adding source 2. Paste or type content 3. System processes immediately 4. No wait time needed Good for: - Notes you want to reference - Quotes from books - Transcripts you have handy - Quick research snippets ``` --- ## Managing Your Sources ### Viewing Source Details ``` Click on source → See: - Original file name/title - When it was added - Size and format - Processing status - Number of chunks ``` ### Organizing with Metadata You can add to each source: - **Title**: Better name than original filename - **Tags**: Category labels ("primary research", "background", "competitor analysis") - **Description**: A few notes about what it contains **Why this matters:** - Makes sources easier to find - Helps when contextualizing for Chat - Useful for organizing large notebooks ### Searching Within Sources ``` After sources are added, you can: Text search: "Find exact phrase" Vector search: "Find conceptually similar" Both search across all sources in notebook. Results show: - Which source - Which section - Relevance score ``` --- ## Context Management: How Sources Get Used You control how AI accesses sources: ### Three Levels (for Chat) **Full Content:** ``` AI sees: Complete source text Cost: 100% of tokens Use when: Analyzing in detail, need precise citations Example: "Analyze this methodology paper closely" ``` **Summary Only:** ``` AI sees: AI-generated summary (not full text) Cost: ~10-20% of tokens Use when: Background material, reference context Example: "Use this as context but focus on the main source" ``` **Not in Context:** ``` AI sees: Nothing (excluded) Cost: 0 tokens Use when: Confidential, not relevant, or archived Example: "Keep this in notebook but don't use in this conversation" ``` ### How to Set Context (in Chat) ``` 1. Go to Chat 2. Click "Select Context Sources" 3. For each source: - Toggle ON/OFF (include/exclude) - Choose level (Full/Summary/Excluded) 4. Click "Save" 5. Now chat uses these settings ``` --- ## Common Mistakes | Mistake | What Happens | How to Fix | |---------|--------------|-----------| | Upload 200 sources at once | System gets slow, processing stalls | Add 10-20 at a time, wait for processing | | Use full content for all sources | Token usage skyrockets, expensive | Use "Summary" or "Excluded" for background material | | Add huge PDFs without splitting | Processing is slow, search results less precise | Consider splitting large PDFs into chapters | | Forget source titles | Can't distinguish between similar sources | Rename sources with descriptive titles right after uploading | | Don't tag sources | Hard to find and organize later | Add tags immediately: "primary", "background", etc. | | Mix languages in one source | Transcription/embedding quality drops | Keep each language in separate sources | | Use same source multiple times | Takes up space, creates confusion | Add once; reuse in multiple chats/notebooks | --- ## Processing Status & Troubleshooting ### What the Status Indicators Mean ``` 🟡 Processing → Source is being extracted and embedded → Wait 30 seconds - 3 minutes depending on size → Don't use in Chat yet 🟢 Ready → Source is processed and searchable → Can use immediately in Chat → Can apply transformations 🔴 Error → Something went wrong → Common reasons: - Unsupported file format - File too large or corrupted - Network timeout ⚪ Not in Context → Source added but excluded from Chat → Still searchable, not sent to AI ``` ### Common Errors & Solutions **"Unsupported file type"** - You tried to upload a format not in the list (e.g., `.webp` image) - Solution: Convert to supported format (PDF for documents, MP3 for audio) **"Processing timeout"** - Very large file (>100MB) or very long audio - Solution: Split into smaller pieces or try uploading again **"Transcription failed"** - Audio quality too poor or language not detected - Solution: Re-record with better quality, or paste text transcript manually **"Web link won't extract"** - Website blocks automated access or uses JavaScript for content - Solution: Copy the article text and paste as "Text" instead --- ## Tips for Best Results ### For PDFs - Clean, digital PDFs work best - Remove copy protection if present (legally) - Scanned PDFs work but take longer ### For Web Articles - Use full URL including domain - Avoid cookie/popup-laden sites - If extraction fails, copy-paste text instead ### For Audio - Clear, well-recorded audio transcribes better - Remove background noise if possible - YouTube videos usually have good transcriptions built-in ### For Large Documents - Consider splitting into smaller sources - Gives more precise search results - Processing is faster for smaller pieces ### For Organization - Name sources clearly (not "document_2.pdf") - Add tags immediately after uploading - Use descriptions for complex documents --- ## What Comes After: Using Your Sources Once you've added sources, you can: - **Chat** → Ask questions (see [Chat Effectively](chat-effectively.md)) - **Search** → Find specific content (see [Search Effectively](search.md)) - **Transformations** → Extract structured insights (see [Working with Notes](working-with-notes.md)) - **Ask** → Get comprehensive answers (see [Search Effectively](search.md)) - **Podcasts** → Turn into audio (see [Creating Podcasts](creating-podcasts.md)) --- ## Summary Checklist Before adding sources, confirm: - [ ] File is in supported format - [ ] File is under 100MB (or splitting large ones) - [ ] Web links are full URLs (not shortened) - [ ] Audio files have clear speech (if transcription-dependent) - [ ] You've named source clearly - [ ] You've added tags for organization - [ ] You understand context levels (Full/Summary/Excluded) Done! Sources are now ready for Chat, Search, Transformations, and more. ================================================ FILE: docs/3-USER-GUIDE/api-configuration.md ================================================ # API Configuration Configure AI provider credentials through the Settings UI. No file editing required. > **Credential System**: Open Notebook uses encrypted credentials stored in the database. Each credential connects to a provider and allows you to discover, register, and test models. --- ## Overview Open Notebook manages AI provider access through a **credential-based system**: 1. You create a **credential** for each provider (API key + settings) 2. Credentials are **encrypted** and stored in the database 3. You **test connections** to verify credentials work 4. You **discover and register models** from each credential 5. Models are linked to credentials for direct configuration --- ## Encryption Setup Before storing credentials, you must configure an encryption key. ### Setting the Encryption Key Add `OPEN_NOTEBOOK_ENCRYPTION_KEY` to your docker-compose.yml: ```yaml environment: - OPEN_NOTEBOOK_ENCRYPTION_KEY=my-secret-passphrase ``` Any string works as a key — it will be securely derived via SHA-256 internally. > **Warning**: If you change or lose the encryption key, **all stored credentials become unreadable**. Back up your encryption key securely and separately from your database backups. ### Docker Secrets Support Both password and encryption key support Docker secrets: ```yaml # docker-compose.yml services: open_notebook: environment: - OPEN_NOTEBOOK_PASSWORD_FILE=/run/secrets/app_password - OPEN_NOTEBOOK_ENCRYPTION_KEY_FILE=/run/secrets/encryption_key secrets: - app_password - encryption_key secrets: app_password: file: ./secrets/password.txt encryption_key: file: ./secrets/encryption_key.txt ``` ### Encryption Details API keys stored in the database are encrypted using Fernet (AES-128-CBC + HMAC-SHA256). | Configuration | Behavior | |---------------|----------| | Encryption key set | Keys encrypted with your key | | No encryption key set | Storing credentials is disabled | --- ## Accessing Credential Configuration 1. Click **Settings** in the navigation bar 2. Select **API Keys** tab 3. You'll see existing credentials and an **Add Credential** button ``` Navigation: Settings → API Keys ``` --- ## Supported Providers ### Cloud Providers | Provider | Required Fields | Optional Fields | |----------|-----------------|-----------------| | OpenAI | API Key | — | | Anthropic | API Key | — | | Google Gemini | API Key | — | | Groq | API Key | — | | Mistral | API Key | — | | DeepSeek | API Key | — | | xAI | API Key | — | | OpenRouter | API Key | — | | Voyage AI | API Key | — | | ElevenLabs | API Key | — | ### Local/Self-Hosted | Provider | Required Fields | Notes | |----------|-----------------|-------| | Ollama | Base URL | Typically `http://localhost:11434` or `http://ollama:11434` | ### Enterprise | Provider | Required Fields | Optional Fields | |----------|-----------------|-----------------| | Azure OpenAI | API Key, Endpoint, API Version | Service-specific endpoints (LLM, Embedding, STT, TTS) | | OpenAI-Compatible | Base URL | API Key, Service-specific configs | | Vertex AI | Project ID, Location, Credentials Path | — | --- ## Creating a Credential ### Step 1: Add Credential 1. Go to **Settings** → **API Keys** 2. Click **Add Credential** 3. Select your provider 4. Give it a descriptive name (e.g., "My OpenAI Key", "Work Anthropic") 5. Fill in the required fields (API key, base URL, etc.) 6. Click **Save** ### Step 2: Test Connection 1. On your new credential card, click **Test Connection** 2. Wait for the result: | Result | Meaning | |--------|---------| | Success | Key is valid, provider accessible | | Invalid API key | Check key format and value | | Connection failed | Check URL, network, firewall | ### Step 3: Discover Models 1. Click **Discover Models** on the credential card 2. The system queries the provider for available models 3. Review the discovered models ### Step 4: Register Models 1. Select the models you want to use 2. Click **Register Models** 3. The models are now available throughout Open Notebook --- ## Multi-Credential Support Each provider can have **multiple credentials**. This is useful when: - You have different API keys for different projects - You want to test with different endpoints - Multiple team members need separate credentials ### Creating Multiple Credentials 1. Click **Add Credential** again 2. Select the same provider 3. Fill in different credentials 4. Each credential can discover and register its own models ### How Models Link to Credentials When you register models from a credential, those models are linked to that specific credential. This means: - Each model knows which API key to use - You can have models from different credentials for the same provider - Deleting a credential removes its linked models --- ## Testing Connections Click **Test Connection** to verify your credential: | Result | Meaning | |--------|---------| | Success | Key is valid, provider accessible | | Invalid API key | Check key format and value | | Connection failed | Check URL, network, firewall | | Model not available | Key valid but model access restricted | Test uses inexpensive models (e.g., `gpt-3.5-turbo`, `claude-3-haiku`) to minimize cost. --- ## Configuring Specific Providers ### Simple Providers (API Key Only) For OpenAI, Anthropic, Google, Groq, Mistral, DeepSeek, xAI, OpenRouter: 1. Add credential with your API key 2. Test connection 3. Discover and register models ### Ollama (URL-Based) 1. Add credential with provider **Ollama** 2. Enter the base URL (e.g., `http://ollama:11434`) 3. Test connection 4. Discover and register models Ollama allows localhost and private IPs since it runs locally. ### Azure OpenAI Azure requires multiple fields: | Field | Example | Required | |-------|---------|----------| | API Key | `abc123...` | Yes | | Endpoint | `https://myresource.openai.azure.com` | Yes | | API Version | `2024-02-15-preview` | Yes | | LLM Endpoint | `https://myresource-llm.openai.azure.com` | No | | Embedding Endpoint | `https://myresource-embed.openai.azure.com` | No | Service-specific endpoints override the main endpoint for that service type. ### OpenAI-Compatible For custom OpenAI-compatible servers (LM Studio, vLLM, etc.): 1. Add credential with provider **OpenAI-Compatible** 2. Enter the base URL 3. Enter API key (if required) 4. Optionally configure per-service URLs Supports separate configurations for: - LLM (language models) - Embedding - STT (speech-to-text) - TTS (text-to-speech) ### Vertex AI Google Cloud's enterprise AI platform: | Field | Example | |-------|---------| | Project ID | `my-gcp-project` | | Location | `us-central1` | | Credentials Path | `/path/to/service-account.json` | --- ## Migrating from Environment Variables If you have existing API keys in environment variables (from a previous version): 1. Open **Settings → API Keys** 2. A banner appears: "Environment variables detected" 3. Click **Migrate to Database** 4. Keys are copied to the database (encrypted) 5. Original environment variables remain unchanged ### Migration Behavior | Scenario | Action | |----------|--------| | Key in env only | Migrated to database | | Key in database only | No change | | Key in both | Database version kept (skipped) | ### After Migration - Database credentials are used for all operations - You can remove the API key environment variables from your docker-compose.yml - Keep `OPEN_NOTEBOOK_ENCRYPTION_KEY` — it's still required ### Migration Banner Visibility The migration banner only appears when: - You have environment variables configured - Those providers are **not** already in the database - If all env providers are already migrated, the banner won't show --- ## Migrating from ProviderConfig (v1.1 → v1.2) If you're upgrading from an older version that used the ProviderConfig system: - The migration happens automatically on first startup - Your existing configurations are converted to credentials - Check **Settings → API Keys** to verify the migration succeeded - If you see issues, check the API logs for migration messages --- ## Key Storage Security ### Encryption API keys stored in the database are encrypted using Fernet (AES-128-CBC + HMAC-SHA256). | Configuration | Behavior | |---------------|----------| | Encryption key set | Keys encrypted with your key | | No encryption key set | Storing API keys in database is disabled | ### Default Credentials | Setting | Default Value | Production Recommendation | |---------|---------------|---------------------------| | Password | `open-notebook-change-me` | Set `OPEN_NOTEBOOK_PASSWORD` | | Encryption Key | None (must be set) | Set `OPEN_NOTEBOOK_ENCRYPTION_KEY` to any secret string | **For production deployments, always set custom credentials.** --- ## Deleting Credentials 1. Click the **Delete** button on the credential card 2. Confirm deletion 3. Credential and all its linked models are removed from the database --- ## Troubleshooting ### Credential Not Saving | Symptom | Cause | Solution | |---------|-------|----------| | Save button disabled | Empty or invalid input | Enter a valid key | | Error on save | Encryption key not set | Set `OPEN_NOTEBOOK_ENCRYPTION_KEY` in docker-compose.yml | | Error on save | Database connection issue | Check database status | ### Test Connection Fails | Error | Cause | Solution | |-------|-------|----------| | Invalid API key | Wrong key or format | Verify key from provider dashboard | | Connection refused | Wrong URL | Check base URL format | | Timeout | Network issue | Check firewall, proxy settings | | 403 Forbidden | IP restriction | Whitelist your server IP | ### Migration Issues | Problem | Solution | |---------|----------| | No migration banner | No env vars detected, or already migrated | | Partial migration | Check error list, fix and retry | | Keys not working after migration | Clear browser cache, restart services | ### Provider Shows "Not Configured" 1. Check if a credential exists for this provider (Settings → API Keys) 2. Test the credential connection 3. Verify key format matches provider requirements 4. Re-discover and register models if needed --- ## Provider-Specific Notes ### OpenAI - Keys start with `sk-proj-` (project keys) or `sk-` (legacy) - Requires billing enabled on account ### Anthropic - Keys start with `sk-ant-` - Check account has API access enabled ### Google Gemini - Keys start with `AIzaSy` - Free tier has rate limits ### Ollama - No API key required - Default URL: `http://localhost:11434` (local) or `http://ollama:11434` (Docker) - Ensure Ollama server is running ### Azure OpenAI - Endpoint format: `https://{resource-name}.openai.azure.com` - API version format: `YYYY-MM-DD` or `YYYY-MM-DD-preview` - Deployment names configured separately when registering models via the credential's Discover Models dialog --- ## Related - **[AI Providers](../5-CONFIGURATION/ai-providers.md)** — Provider setup instructions and recommendations - **[Security](../5-CONFIGURATION/security.md)** — Password and encryption configuration - **[Environment Reference](../5-CONFIGURATION/environment-reference.md)** — All configuration options ================================================ FILE: docs/3-USER-GUIDE/chat-effectively.md ================================================ # Chat Effectively - Conversations with Your Research Chat is your main tool for exploratory questions and back-and-forth dialogue. This guide covers how to use it effectively. --- ## Quick-Start: Your First Chat ``` 1. Go to your notebook 2. Click "Chat" 3. Select which sources to include (context) 4. Type your question 5. Click "Send" 6. Read the response 7. Ask a follow-up (context stays same) 8. Repeat until satisfied ``` That's it! But doing it *well* requires understanding how context works. --- ## Context Management: The Key to Good Chat Context controls **what the AI is allowed to see**. This is your most important control. ### The Three Levels Explained **FULL CONTENT** - AI sees: Complete source text - Cost: 100 tokens per 1K tokens of source - Best for: Detailed analysis, precise citations - Example: "Analyze this research paper closely" ``` You set: Paper A → Full Content AI sees: Every word of Paper A AI can: Cite specific sentences, notice nuances Result: Precise, detailed answers (higher cost) ``` **SUMMARY ONLY** - AI sees: AI-generated 200-word summary (not full text) - Cost: ~10-20% of full content cost - Best for: Background material, reference context - Example: "Use this for background, focus on the main paper" ``` You set: Paper B → Summary Only AI sees: Condensed summary, key points AI can: Reference main ideas but not details Result: Faster, cheaper answers (loses precision) ``` **NOT IN CONTEXT** - AI sees: Nothing - Cost: 0 tokens - Best for: Confidential, irrelevant, archived content - Example: "Keep this in notebook but don't use now" ``` You set: Paper C → Not in Context AI sees: Nothing (completely excluded) AI can: Never reference it Result: No cost, no privacy risk for that source ``` ### Setting Context (Step by Step) ``` 1. Click "Select Sources" (Shows list of all sources in notebook) 2. For each source: □ Checkbox: Include or exclude Level dropdown: ├─ Full Content ├─ Summary Only └─ Excluded 3. Check your selections Example: ✓ Paper A (Full Content) - "Main focus" ✓ Paper B (Summary Only) - "Background" ✓ Paper C (Excluded) - "Keep private" □ Paper D (Not included) - "Not relevant" 4. Click "Save Context" 5. Now chat uses these settings ``` ### Context Strategies **Strategy 1: Minimalist** - Main source: Full Content - Everything else: Excluded - Result: Focused, cheap, precise ``` Use when: - Analyzing one source deeply - Budget-conscious - Want focused answers ``` **Strategy 2: Comprehensive** - All sources: Full Content - Result: All context considered, expensive ``` Use when: - Comprehensive analysis - Unlimited budget - Want AI to see everything ``` **Strategy 3: Tiered** - Primary sources: Full Content - Secondary sources: Summary Only - Background/reference: Excluded - Result: Balanced cost/quality ``` Use when: - Mix of important and reference material - Want thorough but not expensive - Most common strategy ``` **Strategy 4: Privacy-First** - Sensitive docs: Excluded - Public research: Full Content - Result: Never send confidential data ``` Use when: - Company confidential materials - Personal sensitive data - Complying with data protection ``` --- ## Asking Effective Questions ### Good Questions vs. Poor Questions **Poor Question** ``` "What do you think?" Problems: - Too vague (about what?) - No context (what am I analyzing?) - Can't verify answer (citing what?) Result: Generic, shallow answer ``` **Good Question** ``` "Based on the paper's methodology section, what are the three main limitations the authors acknowledge? Please cite which pages mention each one." Strengths: - Specific about what you want - Clear scope (methodology section) - Asks for citations - Requires deep reading Result: Precise, verifiable, useful answer ``` ### Question Patterns That Work **Factual Questions** ``` "What does the paper say about X?" "Who are the authors?" "What year was this published?" Result: Simple, factual answers with citations ``` **Analysis Questions** ``` "How does this approach differ from the traditional method?" "What are the main assumptions underlying this argument?" "Why do you think the author chose this methodology?" Result: Deeper thinking, comparison, critique ``` **Synthesis Questions** ``` "How do these two sources approach the problem differently?" "What's the common theme across all three papers?" "If we combine these approaches, what would we get?" Result: Cross-source insights, connections ``` **Actionable Questions** ``` "What are the practical implications of this research?" "How could we apply these findings to our situation?" "What's the next logical research direction?" Result: Practical, forward-looking answers ``` ### The SPECIFIC Formula Good questions have: 1. **SCOPE** - What are you analyzing? "In this research paper..." "Looking at these three articles..." "Based on your experience..." 2. **SPECIFICITY** - Exactly what do you want? "...the methodology..." "...main findings..." "...recommended next steps..." 3. **CONSTRAINT** - Any limits? "...in 3 bullet points..." "...with citations to page numbers..." "...comparing these two approaches..." 4. **VERIFICATION** - How can you check it? "...with specific quotes..." "...cite your sources..." "...link to the relevant section..." **Example:** ``` Poor: "What about transformers?" Good: "In this research paper on machine learning, explain the transformer architecture in 2-3 sentences, then cite which page describes the attention mechanism." ``` --- ## Follow-Up Questions (The Real Power of Chat) Chat's strength is dialogue. You ask, get an answer, ask more. ### Building on Responses ``` First question: "What's the main finding?" AI: "The study shows X [citation]" Follow-up question: "How does that compare to Y research?" AI: "The key difference is Z [citation]" Next question: "Why do you think that difference matters?" AI: "Because it affects A, B, C [explained]" ``` ### Iterating Toward Understanding ``` Round 1: Get overview "What's this source about?" Round 2: Get details "What's the most important part?" Round 3: Compare "How does it relate to my notes on X?" Round 4: Apply "What should I do with this information?" ``` ### Changing Direction ``` Context stays same, but you ask new questions: Question 1: "What's the methodology?" Question 2: "What are the limitations?" Question 3: "What about the ethical implications?" Question 4: "Who else has done similar work?" All in one conversation, reusing context. ``` ### Adjusting Context Between Rounds ``` After question 3, you realize: "I need more context from another source" 1. Click "Adjust Context" 2. Add new source or change context level 3. Your conversation history stays 4. Continue asking with new context ``` --- ## Citations and Verification Citations are how you verify that the AI's answer is accurate. ### Understanding Citations ``` AI Response with Citation: "The paper reports a 95% accuracy rate [see page 12]" What this means: ✓ The claim "95% accuracy rate" is from page 12 ✓ You can verify by reading page 12 ✓ If page 12 doesn't say that, the AI hallucinated ``` ### Requesting Better Citations ``` If you get a response without citations: Ask: "Please cite the page number for that claim" or: "Show me where you found that information" AI will: - Find the citation - Provide page numbers - Show you the source ``` ### Verification Workflow ``` 1. Get answer from Chat 2. Check citation (which source? which page?) 3. Click citation link (if available) 4. See the actual text in source 5. Does it really say what AI claimed? If YES: Great, you can use this answer If NO: The AI hallucinated, ask for correction ``` --- ## Common Chat Patterns ### Pattern 1: Deep Dive into One Source ``` 1. Set context: One source (Full Content) 2. Question 1: Overview 3. Question 2: Main argument 4. Question 3: Evidence for argument 5. Question 4: Limitations 6. Question 5: Next steps Result: Complete understanding of one source ``` ### Pattern 2: Comparative Analysis ``` 1. Set context: 2-3 sources (all Full Content) 2. Question 1: What does each source say about X? 3. Question 2: How do they agree? 4. Question 3: How do they disagree? 5. Question 4: Which approach is stronger? Result: Understanding differences and trade-offs ``` ### Pattern 3: Research Exploration ``` 1. Set context: Many sources (mix of Full/Summary) 2. Question 1: What are the main perspectives? 3. Question 2: What's missing from these views? 4. Question 3: What questions does this raise? 5. Question 4: What should I research next? Result: Understanding landscape and gaps ``` ### Pattern 4: Problem Solving ``` 1. Set context: Relevant sources (Full Content) 2. Question 1: What's the problem? 3. Question 2: What approaches exist? 4. Question 3: Pros and cons of each? 5. Question 4: Which would work best for [my situation]? Result: Decision-making informed by research ``` --- ## Optimizing for Cost Chat uses tokens for every response. Here's how to use efficiently: ### Reduce Token Usage **Minimize context** ``` Option A: All sources, Full Content Cost per response: 5,000 tokens Option B: Only relevant sources, Summary Only Cost per response: 1,000 tokens Savings: 80% cheaper, same conversation ``` **Shorter questions** ``` Verbose: "Could you please analyze the methodology section of this paper and explain in detail what the authors did?" Concise: "Summarize the methodology in 2-3 points." Savings: 20-30% per response ``` **Use cheaper models** ``` GPT-4o: $0.15 per 1M input tokens GPT-4o-mini: $0.03 per 1M input tokens Claude Sonnet: $0.90 per 1M input tokens For chat: Mini/Haiku models are usually fine For deep analysis: Sonnet/Opus worth the cost ``` ### Budget Strategies **Exploration budget** - Use cheap model - Broad context (understand landscape) - Short questions - Result: Low cost, good overview **Analysis budget** - Use powerful model - Focused context (main source only) - Detailed questions - Result: Higher cost, deep insights **Synthesis budget** - Use powerful model for final synthesis - Multiple sources (Full Content) - Complex comparative questions - Result: Expensive but valuable output --- ## Troubleshooting Chat Issues ### Poor Responses | Problem | Cause | Solution | |---------|-------|----------| | Generic answers | Vague question | Be specific (see question patterns) | | Missing context | Not enough in context | Add sources or change to Full Content | | Incorrect info | Source not in context | Add the relevant source | | Hallucinating | Model confused | Ask for citations, verify claims | | Shallow analysis | Wrong model | Switch to more powerful model | ### High Costs | Problem | Cause | Solution | |---------|-------|----------| | Expensive per response | Too much context | Use Summary Only or exclude sources | | Many follow-ups | Exploratory chat | Use Ask instead for single comprehensive answer | | Long conversations | Keeping history | Archive old chats, start fresh | | Large sources | Full text in context | Use Summary Only for large documents | --- ## Best Practices ### Before You Chat - [ ] Add sources you'll need - [ ] Decide context strategy (Tiered is usually best) - [ ] Choose model (cheaper for exploration, powerful for analysis) - [ ] Have a question in mind ### During Chat - [ ] Ask specific questions (use SPECIFIC formula) - [ ] Check citations for factual claims - [ ] Follow up on unclear points - [ ] Adjust context if you need different sources ### After Chat - [ ] Save good responses as notes - [ ] Archive conversation if you're done - [ ] Organize notes for future reference - [ ] Use insights in other features (Ask, Transformations, Podcasts) --- ## When to Use Chat vs. Ask **Use CHAT when:** - You want a dialogue - You're exploring a topic - You'll ask multiple related questions - You want to adjust context during conversation - You're not sure exactly what you need **Use ASK when:** - You have one specific question - You want a comprehensive answer - You want the system to auto-search - You want one response, not dialogue - You want maximum tokens spent on search --- ## Summary: Chat as Conversation Chat is fundamentally different from asking ChatGPT directly: | Aspect | ChatGPT | Open Notebook Chat | |--------|---------|-------------------| | **Source control** | None (uses training) | You control which sources are visible | | **Cost control** | Per token | Per token, but context is your choice | | **Iteration** | Works | Works, with your sources changing dynamically | | **Citations** | Made up often | Tied to your sources (verifiable) | | **Privacy** | Your data to OpenAI | Your data stays local (unless you choose) | The key insight: **Chat is retrieval-augmented generation.** AI sees only what you put in context. You control the conversation and the information flow. That's why Chat is powerful for research. You're not just talking to an AI; you're having a conversation with your research itself. ================================================ FILE: docs/3-USER-GUIDE/citations.md ================================================ # Citations - Verify and Trust AI Responses Citations connect AI responses to your source materials. This guide covers how to use and verify them. --- ## Why Citations Matter Every AI-generated response in Open Notebook includes citations to your sources. This lets you: - **Verify claims** - Check that AI actually read what it claims - **Find original context** - See the full passage around a quote - **Catch hallucinations** - Spot when AI makes things up - **Build credibility** - Your notes have traceable sources --- ## Quick Start: Using Citations ### Reading Citations ``` AI Response: "The study found a 95% accuracy rate [1] using the proposed method." [1] = Click to see source What happens when you click: → Opens the source document → Highlights the relevant section → You can verify the claim ``` ### Requesting Better Citations If a response lacks citations, ask: ``` "Please cite the specific page or section for that claim." "Where in the document does it say that?" "Can you quote the exact text?" ``` --- ## How Citations Work ### Automatic Generation When AI references your sources, citations are generated automatically: ``` 1. AI analyzes your question 2. Retrieves relevant source chunks 3. Generates response with inline citations 4. Links citations to original source locations ``` ### Citation Format ``` Inline format: "The researchers concluded X [1] and Y [2]." Reference list: [1] Paper Title - Section 3.2 [2] Report Name - Page 15 Clickable: Each [number] links to the source ``` --- ## Verifying Citations ### The Verification Workflow ``` Step 1: Read AI response "The model achieved 95% accuracy [1]" Step 2: Click citation [1] → Opens source document → Shows relevant passage Step 3: Verify the claim Does source actually say 95%? Is context correct? Any nuance missed? Step 4: Trust or correct ✓ Accurate → Use the insight ✗ Wrong → Ask AI to correct ``` ### What to Check | Check | Why | |-------|-----| | **Exact numbers** | AI sometimes rounds or misremembers | | **Context** | Quote might mean something different in context | | **Attribution** | Is this the source's claim or someone they cited? | | **Completeness** | Did AI miss important caveats? | --- ## Citations in Different Features ### Chat Citations ``` Context: Sources you selected Citations: Reference chunks used in response Verification: Click to see original text Save: Citations preserved when saving as note ``` ### Ask Feature Citations ``` Context: Auto-searched across all sources Citations: Multiple sources synthesized Verification: Each source linked separately Quality: Often more comprehensive than Chat ``` ### Transformation Citations ``` Context: Single source being transformed Citations: Points back to original document Verification: Compare output to source Use: When you need structured extraction ``` --- ## Saving Citations ### In Notes When you save an AI response as a note, citations are preserved: ``` Original response: "According to the paper [1], the method works by..." Saved note includes: - The text - The citation link - Reference to source document ``` ### Exporting Citations work in exports: | Format | Citation Behavior | |--------|-------------------| | **Markdown** | Links preserved as `[text](link)` | | **Copy/Paste** | Plain text with reference numbers | | **PDF** | Clickable references (if supported) | --- ## Citation Quality Tips ### Get Better Citations **Be specific in questions:** ``` Poor: "What does it say about X?" Good: "What does page 15 say about X? Please quote directly." ``` **Request citation format:** ``` "Include page numbers for each claim." "Cite specific sections, not just document names." ``` **Use Full Content context:** ``` Summary Only → Less precise citations Full Content → Exact quotes possible ``` ### When Citations Are Missing | Situation | Cause | Solution | |-----------|-------|----------| | No citations | AI used general knowledge | Ask: "Base your answer only on my sources" | | Vague citations | Source not in Full Content | Change context level | | Wrong citations | AI confused sources | Ask to verify with quotes | --- ## Common Issues ### "Citation doesn't match claim" ``` Problem: AI says X, but source says Y What happened: - AI paraphrased incorrectly - AI combined multiple sources confusingly - Source was taken out of context Solution: 1. Click citation to see original 2. Note the discrepancy 3. Ask AI: "The source says Y, not X. Please correct." ``` ### "Can't find cited section" ``` Problem: Citation link doesn't show relevant text What happened: - Source was chunked differently than expected - Information spread across multiple sections - Processing missed some content Solution: 1. Search within source for key terms 2. Ask AI for more specific location 3. Re-process source if needed ``` ### "No citations at all" ``` Problem: AI response has no source references What happened: - Sources not in context - Question asked for opinion/general knowledge - Model didn't find relevant content Solution: 1. Check context settings 2. Rephrase: "Based on my sources, what..." 3. Add more relevant sources ``` --- ## Best Practices ### For Research Integrity 1. **Always verify important claims** - Don't trust AI blindly 2. **Check context** - Quotes can be misleading out of context 3. **Note limitations** - AI might miss nuance 4. **Keep source access** - Don't delete sources you cite ### For Academic Work 1. **Use Full Content** for documents you'll cite 2. **Request specific page numbers** 3. **Cross-check with original sources** 4. **Document your verification process** ### For Professional Use 1. **Verify before sharing** - Check claims clients will see 2. **Keep citation trail** - Save notes with sources linked 3. **Be transparent** - Note when insights are AI-assisted --- ## Summary ``` Citations = Your verification system How to use: 1. Read AI response 2. Note citation markers [1], [2], etc. 3. Click to see original source 4. Verify claim matches source 5. Trust verified insights When citations fail: - Ask for specific quotes - Change to Full Content - Request page numbers - Verify manually Why it matters: - AI can hallucinate - Context can change meaning - Trust requires verification - Good research needs sources ``` Citations aren't just references — they're your quality control. Use them to build research you can trust. ================================================ FILE: docs/3-USER-GUIDE/creating-podcasts.md ================================================ # Creating Podcasts - Turn Research into Audio Podcasts let you consume your research passively. This guide covers the complete workflow from setup to download. --- ## Quick-Start: Your First Podcast (5 Minutes) ``` 1. Go to your notebook 2. Click "Generate Podcast" 3. Select sources to include 4. Choose a speaker profile (or use default) 5. Click "Generate" 6. Wait 3-10 minutes (non-blocking) 7. Download MP3 when ready 8. Done! ``` That's the minimum. Let's make it better. --- ## Step-by-Step: The Complete Workflow ### Step 1: Prepare Your Notebook ``` Before generating, make sure: ✓ You have sources added (At least 1-2 sources) ✓ Sources have been processed (Green "Ready" status) ✓ Notes are organized (If you want notes included) ✓ You know your message (What's the main story?) Typical preparation: 5-10 minutes ``` ### Step 2: Choose Content ``` Click "Generate Podcast" You'll see: - List of all sources in notebook - List of all notes Select which to include: ☑ Paper A (primary source) ☑ Paper B (supporting source) ☐ Old note (not relevant) ✓ Analysis note (important) What to include: - Primary sources: Always include - Supporting sources: Usually include - Notes: Include your analysis/insights - Everything: Can overload podcast Recommended: 3-5 sources per podcast ``` ### Step 3: Choose Episode Profile An episode profile defines the structure and tone. **Option A: Use Preset Profile** ``` Open Notebook provides preset profiles: Academic Presentation (Monologue) ├─ 1 speaker ├─ Tone: Educational └─ Format: Expert explaining topic Expert Interview (2-speaker) ├─ 2 speakers: Host + Expert ├─ Tone: Q&A, conversational └─ Format: Interview with expert Debate Format (2-speaker) ├─ 2 speakers: Pro vs. Con ├─ Tone: Discussion, disagreement └─ Format: Debate about the topic Panel Discussion (3-4 speaker) ├─ 3-4 speakers: Different perspectives ├─ Tone: Thoughtful discussion └─ Format: Each brings different expertise Solo Explanation (Monologue) ├─ 1 speaker ├─ Tone: Conversational, friendly └─ Format: Personal explanation ``` **Pick based on your content:** - One main idea → Academic Presentation - You want to explain → Solo Explanation - Two competing views → Debate Format - Multiple perspectives → Panel Discussion - Want to explore → Expert Interview ### Step 4: Customize Episode Profile (Optional) If presets don't fit, customize: ``` Episode Profile ├─ Title: "AI Safety in 2026" ├─ Description: "Exploring current approaches" ├─ Length target: 20 minutes ├─ Tone: "Academic but accessible" ├─ Focus areas: │ ├─ Main approaches to alignment │ ├─ Pros and cons comparison │ └─ Open questions ├─ Audience: "Researchers new to field" └─ Format: "Debate between two perspectives" How to set: 1. Click "Customize" 2. Edit each field 3. Click "Save Profile" 4. System uses your profile for outline generation ``` ### Step 5: Create or Select Speakers Speakers are the "voice" of your podcast. **Option A: Use Preset Speakers** ``` Open Notebook provides preset profiles: "Expert Alex" - Expertise: Deep knowledge - Personality: Rigorous, patient - Voice Model: Selected from model registry "Curious Sam" - Expertise: Curious newcomer - Personality: Asks questions - Voice Model: Selected from model registry "Skeptic Jordan" - Expertise: Critical perspective - Personality: Challenges assumptions - Voice Model: Selected from model registry For your first podcast: Use presets For custom podcast: Create your own ``` **Option B: Create Custom Speakers** ``` Click "Add Speaker" Fill in: Name: "Dr. Research Expert" Expertise: "20 years in AI safety research, deep knowledge of alignment approaches" Personality: "Rigorous, academic style, explains clearly, asks good questions" Voice Configuration: - Voice Model: Select from model registry (e.g., OpenAI TTS, Google TTS, ElevenLabs) - Voice: Choose from available voices for the selected model - Per-speaker override: Each speaker can optionally use a different voice model Credentials are automatically resolved from the model configuration. Example: Name: Dr. Research Expert Expertise: AI safety alignment research Personality: Rigorous, academic but accessible Voice Model: ElevenLabs TTS (from registry), Voice: professional male ``` ### Step 6: Generate Podcast ``` 1. Review your setup: Sources: ✓ Selected Profile: ✓ Episode profile chosen Speakers: ✓ Speakers configured 2. Click "Generate Podcast" 3. System begins: - Analyzing your content - Creating outline - Writing dialogue - Generating audio - Mixing speakers 4. Status shows progress: 20% Outline generation 40% Dialogue writing 60% Audio synthesis 80% Mixing 100% Complete Processing time: - 5 minutes of content: 3-5 minutes - 15 minutes of content: 5-10 minutes - 30 minutes of content: 10-20 minutes ``` ### Step 7: Review and Download ``` When complete: Preview: - Play audio sample - Review transcript - Check duration Options: ✓ Download as MP3 - Save to computer ✓ Stream directly - Listen in browser ✓ Share link - Get shareable URL (if public) ✓ Regenerate - Try different speakers/profile Download: 1. Click "Download as MP3" 2. Choose quality: 128kbps / 192kbps / 320kbps 3. Save file: podcast_[notebook]_[date].mp3 4. Listen! ``` --- ## Understanding What Happens Behind the Scenes ### The Generation Pipeline ``` Stage 1: CONTENT ANALYSIS (1 minute) Your sources → What's the main story? → Key themes? → Debate points? Stage 2: OUTLINE CREATION (2-3 minutes) Themes → Episode structure → Section breakdown → Talking points Stage 3: DIALOGUE WRITING (2-3 minutes) Outline → Convert to natural dialogue → Add speaker personalities → Create flow and transitions Stage 4: AUDIO SYNTHESIS (3-5 minutes per speaker) Script + Speaker → Text-to-speech → Individual audio files → High quality audio Stage 5: MIXING & MASTERING (1-2 minutes) Multiple audio → Combine speakers → Level audio → Add polish → Final MP3 Total: 10-20 minutes for typical podcast ``` --- ## Text-to-Speech Providers Different providers, different qualities. ### OpenAI (Recommended) ``` Voices: 5 options (Alloy, Echo, Fable, Onyx, Shimmer) Quality: Good, natural sounding Speed: Fast Cost: ~$0.015 per minute Best for: General purpose, natural speech Example: "I have to say, the research shows..." ``` ### Google TTS ``` Voices: Many options, various accents Quality: Excellent, very natural Speed: Fast Cost: ~$0.004 per minute Best for: High quality output, accents Example: "The research demonstrates that..." ``` ### ElevenLabs ``` Voices: 100+ voices, highly customizable Quality: Exceptional, very expressive Speed: Slower (5-10 seconds per phrase) Cost: ~$0.10 per minute Best for: Premium quality, emotional range Example: [Can convey emotion and tone] ``` ### Local TTS (Free) ``` Voices: Limited, basic options Quality: Basic, robotic Speed: Depends on hardware (slow) Cost: Free (local processing) Best for: Privacy, testing, offline use Example: "The research shows..." Privacy: Everything stays on your computer ``` ### Which Provider to Choose? ``` For your first podcast: Google (quality/cost balance) For privacy-sensitive: Local TTS (free, private) For premium quality: ElevenLabs (best voices) For budget: Google (cheapest quality option) For speed: OpenAI (fast generation) ``` --- ## Tips for Better Podcasts ### Choose Right Profile ``` Single source analysis → Academic Presentation "Explaining one paper to someone new" Comparing two approaches → Debate Format "Pros and cons of different methods" Multiple sources + insights → Panel Discussion "Different experts discussing topic" Narrative exploration → Expert Interview "Host interviewing research expert" Personal take → Solo Explanation "You explaining your analysis" ``` ### Create Good Speakers ``` Good Speaker: ✓ Clear expertise (know what they're talking about) ✓ Distinct personality (not generic) ✓ Good voice choice (matches personality) ✓ Realistic backstory (feels like real person) Bad Speaker: ✗ Generic expertise ("good at research") ✗ No personality ("just reads") ✗ Mismatched voice (deep voice for young person) ✗ Contradicts personality (serious person uses casual voice) ``` ### Focus Content ``` Better: Podcast on ONE specific topic "How transformers work" (15 minutes, focused) Worse: Podcast on everything "All of AI 2025" (2 hours, unfocused) Guideline: - 5-10 minutes: One narrow topic - 15-20 minutes: One broad topic - 30+ minutes: Multiple related subtopics Shorter is usually better for podcasts. ``` ### Optimize Source Selection ``` Too much content: "Here are all 20 papers" → Podcast becomes 2+ hours → Unfocused → Low quality Right amount: "Here are 3 key papers" → Podcast is 15-20 minutes → Focused → High quality Rule: 3-5 sources per podcast Remove long background papers Keep focused on main topic ``` --- ## Quality Troubleshooting ### Audio Sounds Robotic **Problem**: TTS voice sounds unnatural **Solutions**: ``` 1. Switch provider: Try Google or ElevenLabs instead 2. Choose different voice: Some voices more natural 3. Shorter sentences: Very long sentences sound robotic 4. Adjust pacing: Ask for "natural, conversational pacing" ``` ### Audio Sounds Unclear **Problem**: Hard to understand what's being said **Solutions**: ``` 1. Re-generate with different speaker 2. Try different TTS provider 3. Use speakers with clear accents 4. Lower background noise (if any) 5. Increase speech rate (if too slow) ``` ### Missing Content **Problem**: Important information isn't in podcast **Solutions**: ``` 1. Include that source in content selection 2. Review generated outline (check before generating) 3. Regenerate with clearer profile instructions 4. Try different model (more thorough model) ``` ### Speakers Don't Match **Problem**: Speakers sound like same person **Solutions**: ``` 1. Choose different voice models from the registry for each speaker 2. Choose very different voice options 3. Increase personality differences in profile 4. Try different speaker count (2 vs 3 vs 4) ``` ### Generation Failed **Problem**: "Podcast generation failed" **Solutions**: ``` 1. Check internet connection (especially TTS) 2. Try again (might be temporary issue) 3. Use local TTS (doesn't need internet) 4. Reduce source count (less to process) 5. Contact support if persistent ``` --- ## Advanced: Multiple Podcasts from Same Research You can generate different podcasts from one notebook: ``` Podcast 1: Overview Profile: Academic Presentation Sources: Papers A, B, C Speakers: One expert Length: 15 minutes → Use for "What's this about?" understanding Podcast 2: Deep Dive Profile: Expert Interview Sources: Paper A (Full) + B, C (Summary) Speakers: Expert + Interviewer Length: 30 minutes → Use for detailed exploration Podcast 3: Debate Profile: Debate Format Sources: Papers A vs B (different approaches) Speakers: Pro-A speaker + Pro-B speaker Length: 20 minutes → Use for comparing approaches ``` Each tells the same story from different angles. --- ## Exporting and Sharing ### Download MP3 ``` 1. Generation complete 2. Click "Download" 3. Choose quality: - 128 kbps: Smallest file, lower quality - 192 kbps: Balanced (recommended) - 320 kbps: Highest quality, largest file 4. Save to computer 5. Use in podcast app, upload to platform, etc. ``` ### Export Transcript ``` 1. Click "Export Transcript" 2. Get full dialogue as text 3. Useful for: - Blog post content - Show notes - Searchable text version - Accessibility ``` ### Share Link ``` If podcast is public: 1. Click "Share" 2. Get shareable link 3. Others can listen/download 4. Useful for: - Sharing with team - Public distribution - Embedding on website ``` ### Publish to Podcast Platforms ``` If you want to distribute (future feature): 1. Download MP3 2. Upload to platform (Spotify, Apple Podcasts, etc.) 3. Add metadata (title, description, episode notes) 4. Your research becomes a published podcast! ``` --- ## Best Practices ### Before Generation - [ ] Sources are processed and ready - [ ] You've chosen content to include - [ ] You have a clear episode profile - [ ] Speakers are well-defined - [ ] Content is focused (3-5 sources max) ### During Generation - Don't close the browser (use background processing) - Check back in 5-15 minutes - Review transcript when complete - Listen to sample before downloading ### After Generation - [ ] Download MP3 to computer - [ ] Save in organized folder - [ ] Add metadata (title, description, date) - [ ] Test listening in podcast app - [ ] Share with colleagues for feedback --- ## Use Cases ### Academic Researcher ``` Podcast: Explaining your dissertation Speakers: You + colleague Content: Your papers + supporting research Use: Share with advisors, test explanations ``` ### Content Creator ``` Podcast: Research-to-podcast article Speakers: Narrator + expert Content: Articles you've researched Use: Transform article into podcast version ``` ### Team Research ``` Podcast: Weekly research updates Speakers: Multiple team members Content: This week's papers Use: Team updates, knowledge sharing ``` ### Learning/Teaching ``` Podcast: Teaching material Speakers: Teacher + inquisitive student Content: Textbook + examples Use: Students learn while commuting ``` --- ## Cost Breakdown Example ### Generate 15-minute podcast with ElevenLabs ``` Generation (outline + dialogue): No charge (included in service) Text-to-speech: 2 speakers × 15 minutes = 30 minutes TTS ElevenLabs: $0.10 per minute Cost: 30 × $0.10 = $3.00 Processing: Included (no additional cost) Total: $3.00 per podcast Cheaper options: With Google TTS: ~$0.12 With OpenAI: ~$0.45 With Local TTS: ~$0.00 ``` --- ## Summary: Podcasts as Research Tool Podcasts transform how you consume research: ``` Before: Reading papers takes time, focus After: Listen while commuting, exercising, doing chores Before: Can't share complex research easily After: Share audio of your analysis Before: Different consumption styles isolated After: Same research, multiple formats (read/listen) ``` Podcasts aren't just for entertainment—they're a tool for making research more accessible, shareable, and consumable. That's why they're important for Open Notebook. ================================================ FILE: docs/3-USER-GUIDE/index.md ================================================ # User Guide - How to Use Open Notebook This guide covers practical, step-by-step usage of Open Notebook features. You already understand the concepts; now learn how to actually use them. > **Prerequisite**: Review [2-CORE-CONCEPTS](../2-CORE-CONCEPTS/index.md) first to understand the mental models (notebooks, sources, notes, chat, transformations, podcasts). --- ## Start Here ### [Interface Overview](interface-overview.md) Learn the layout before diving in. Understand the three-panel design and where everything is. --- ## Eight Core Features ### 1. [Adding Sources](adding-sources.md) How to bring content into your notebook. Supports PDFs, web links, audio, video, text, and more. **Quick links:** - Upload a PDF or document - Add a web link or article - Transcribe audio or video - Paste text directly - Common mistakes + fixes --- ### 2. [Working with Notes](working-with-notes.md) Creating, organizing, and using notes (both manual and AI-generated). **Quick links:** - Create a manual note - Save AI responses as notes - Apply transformations to generate insights - Organize with tags and naming - Use notes across your notebook --- ### 3. [Chat Effectively](chat-effectively.md) Have conversations with AI about your sources. Manage context to control what AI sees. **Quick links:** - Start your first chat - Select which sources go in context - Ask effective questions - Use follow-ups productively - Understand citations and verify claims --- ### 4. [Creating Podcasts](creating-podcasts.md) Convert your research into audio dialogue for passive consumption. **Quick links:** - Create your first podcast - Choose or customize speakers - Select TTS provider - Generate and download - Common audio quality fixes --- ### 5. [Search Effectively](search.md) Two search modes: text-based (keyword) and vector-based (semantic). Know when to use each. **Quick links:** - Text search vs vector search (when to use) - Running effective searches - Using the Ask feature for comprehensive answers - Saving search results as notes - Troubleshooting poor results --- ### 6. [Transformations](transformations.md) Batch-process sources with predefined templates. Extract the same insights from multiple documents. **Quick links:** - Built-in transformation templates - Creating custom transformations - Applying to single or multiple sources - Managing transformation output --- ### 7. [Citations](citations.md) Verify AI claims by tracing them back to source material. Understand the citation system. **Quick links:** - Reading and clicking citations - Verifying claims against sources - Requesting better citations - Saving cited content as notes --- ### 8. [API Configuration](api-configuration.md) Configure AI provider API keys directly through the Settings UI. **Quick links:** - Add API keys without editing files - Test provider connections - Migrate from environment variables - Manage Azure and OpenAI-compatible providers - Understand key storage and encryption --- ## Which Feature for Which Task? ``` Task: "I want to explore a topic with follow-ups" → Use: Chat (add sources, select context, have conversation) Task: "I want one comprehensive answer" → Use: Search / Ask (system finds relevant content) Task: "I want to extract the same info from many sources" → Use: Transformations (define template, apply to all) Task: "I want summaries of all my sources" → Use: Transformations (with built-in summary template) Task: "I want to share my research in audio form" → Use: Podcasts (create speakers, generate episode) Task: "I want to find that quote I remember" → Use: Search / Text Search (keyword matching) Task: "I'm exploring a concept without knowing exact words" → Use: Search / Vector Search (semantic similarity) Task: "I need to add or change my AI provider API keys" → Use: Settings / API Keys (configure providers without editing files) ``` --- ## Quick-Start Checklist: First 15 Minutes **Step 1: Create a Notebook (1 min)** - Name: Something descriptive ("Q1 Market Research", "AI Safety Papers", etc.) - Description: 1-2 sentences about what you're researching - This is your research container **Step 2: Add Your First Source (3 min)** - Pick one: PDF, web link, or text - Follow [Adding Sources](adding-sources.md) - Wait for processing (usually 30-60 seconds) **Step 3: Chat About It (3 min)** - Go to Chat - Select your source (set context to "Full Content") - Ask a simple question: "What are the main points?" - See AI respond with citations **Step 4: Save Insight as Note (2 min)** - Good response? Click "Save as Note" - Name it something useful ("Main points from source X") - Now you have a captured insight **Step 5: Explore More (6 min)** - Add another source - Chat about both together - Ask a question that compares them - Follow up with clarifying questions **Done!** You've used the core workflow: notebook → sources → chat → notes --- ## Common Mistakes to Avoid | Mistake | Problem | Fix | |---------|---------|-----| | Adding everything to one notebook | No isolation between projects | Create separate notebooks for different topics | | Expecting AI to know your context | Questions get generic answers | Describe your research focus in chat context | | Forgetting to cite sources | You can't verify claims | Click citations to check source chunks | | Using Chat for one-time questions | Slower than Ask | Use Ask for comprehensive Q&A, Chat for exploration | | Adding huge PDFs without chunking | Slow processing, poor search | Break into multiple smaller sources if possible | | Using same context for all chats | Expensive, unfocused | Adjust context level for each chat | | Ignoring vector search | Only finding exact keywords | Use vector search to explore conceptually | --- ## Next Steps 1. **Follow each guide** in order (sources → notes → chat → podcasts → search) 2. **Create your first notebook** with real content 3. **Practice each feature** with your own research 4. **Return to CORE-CONCEPTS** if you need to understand the "why" --- ## Getting Help - **Feature not working?** → Check the feature's guide (look for "Troubleshooting" section) - **Error message?** → Check [6-TROUBLESHOOTING](../6-TROUBLESHOOTING/index.md) - **Understanding how something works?** → Check [2-CORE-CONCEPTS](../2-CORE-CONCEPTS/index.md) - **Setting up for the first time?** → Go back to [1-INSTALLATION](../1-INSTALLATION/index.md) - **For developers** → See [7-DEVELOPMENT](../7-DEVELOPMENT/index.md) --- **Ready to start?** Pick the guide for what you want to do first! ================================================ FILE: docs/3-USER-GUIDE/interface-overview.md ================================================ # Interface Overview - Finding Your Way Around Open Notebook uses a clean three-panel layout. This guide shows you where everything is. --- ## The Main Layout ``` ┌─────────────────────────────────────────────────────────────┐ │ [Logo] Notebooks Search Podcasts Models Settings │ ├──────────────┬──────────────┬───────────────────────────────┤ │ │ │ │ │ SOURCES │ NOTES │ CHAT │ │ │ │ │ │ Your docs │ Your │ Talk to AI about │ │ PDFs, URLs │ insights │ your sources │ │ Videos │ summaries │ │ │ │ │ │ │ [+Add] │ [+Write] │ [Type here...] │ │ │ │ │ └──────────────┴──────────────┴───────────────────────────────┘ ``` --- ## Navigation Bar The top navigation takes you to main sections: | Icon | Page | What It Does | |------|------|--------------| | **Notebooks** | Main workspace | Your research projects | | **Search** | Ask & Search | Query across all notebooks | | **Podcasts** | Audio generation | Manage podcast profiles | | **Models** | AI configuration | Set up providers and models | | **Settings** | Preferences | App configuration | --- ## Left Panel: Sources Your research materials live here. ### What You'll See ``` ┌─────────────────────────┐ │ Sources (5) │ │ [+ Add Source] │ ├─────────────────────────┤ │ ┌─────────────────┐ │ │ │ 📄 Paper.pdf │ │ │ │ 🟢 Full Content │ │ │ │ [⋮ Menu] │ │ │ └─────────────────┘ │ │ │ │ ┌─────────────────┐ │ │ │ 🔗 Article URL │ │ │ │ 🟡 Summary Only │ │ │ │ [⋮ Menu] │ │ │ └─────────────────┘ │ └─────────────────────────┘ ``` ### Source Card Elements - **Icon** - File type (PDF, URL, video, etc.) - **Title** - Document name - **Context indicator** - What AI can see: - 🟢 Full Content - 🟡 Summary Only - ⛔ Not in Context - **Menu (⋮)** - Edit, transform, delete ### Add Source Button Click to add: - File upload (PDF, DOCX, etc.) - Web URL - YouTube video - Plain text --- ## Middle Panel: Notes Your insights and AI-generated content. ### What You'll See ``` ┌─────────────────────────┐ │ Notes (3) │ │ [+ Write Note] │ ├─────────────────────────┤ │ ┌─────────────────┐ │ │ │ 📝 My Analysis │ │ │ │ Manual note │ │ │ │ Jan 3, 2026 │ │ │ └─────────────────┘ │ │ │ │ ┌─────────────────┐ │ │ │ 🤖 Summary │ │ │ │ From transform │ │ │ │ Jan 2, 2026 │ │ │ └─────────────────┘ │ └─────────────────────────┘ ``` ### Note Card Elements - **Icon** - Note type (manual 📝 or AI 🤖) - **Title** - Note name - **Origin** - How it was created - **Date** - When created ### Write Note Button Click to: - Create manual note - Add your own insights - Markdown supported --- ## Right Panel: Chat Your AI conversation space. ### What You'll See ``` ┌───────────────────────────────┐ │ Chat │ │ Session: Research Discussion │ │ [+ New Session] [Sessions ▼] │ ├───────────────────────────────┤ │ │ │ You: What's the main │ │ finding? │ │ │ │ AI: Based on the paper [1], │ │ the main finding is... │ │ [Save as Note] │ │ │ │ You: Tell me more about │ │ the methodology. │ │ │ ├───────────────────────────────┤ │ Context: 3 sources (12K tok) │ ├───────────────────────────────┤ │ [Type your message...] [↑] │ └───────────────────────────────┘ ``` ### Chat Elements - **Session selector** - Switch between conversations - **Message history** - Your conversation - **Save as Note** - Keep good responses - **Context indicator** - What AI can see - **Input field** - Type your questions --- ## Context Indicators These show what AI can access: ### Token Counter ``` Context: 3 sources (12,450 tokens) ↑ ↑ Sources Approximate cost indicator included ``` ### Per-Source Indicators | Indicator | Meaning | AI Access | |-----------|---------|-----------| | 🟢 Full Content | Complete text | Everything | | 🟡 Summary Only | AI summary | Key points only | | ⛔ Not in Context | Excluded | Nothing | Click any source to change its context level. --- ## Podcasts Tab Inside a notebook, switch to Podcasts: ``` ┌───────────────────────────────┐ │ [Chat] [Podcasts] │ ├───────────────────────────────┤ │ Episode Profile: [Select ▼] │ │ │ │ Speakers: │ │ ├─ Host: Alex (voice model) │ │ └─ Guest: Sam (voice model) │ │ │ │ Include: │ │ ☑ Paper.pdf │ │ ☑ My Analysis (note) │ │ ☐ Background article │ │ │ │ [Generate Podcast] │ └───────────────────────────────┘ ``` --- ## Settings Page Access via navigation bar → Settings: ### Key Sections | Section | What It Controls | |---------|------------------| | **Processing** | Document and URL extraction engines | | **Embedding** | Auto-embed settings | | **Files** | Auto-delete uploads after processing | | **YouTube** | Preferred transcript languages | --- ## Models Page Configure AI providers: ``` ┌───────────────────────────────────────┐ │ Models │ ├───────────────────────────────────────┤ │ Language Models │ │ ┌─────────────────────────────────┐ │ │ │ GPT-4o (OpenAI) [Edit] │ │ │ │ Claude Sonnet (Anthropic) │ │ │ │ Llama 3.3 (Ollama) [⭐] │ │ │ └─────────────────────────────────┘ │ │ [+ Add Model] │ │ │ │ Embedding Models │ │ ┌─────────────────────────────────┐ │ │ │ text-embedding-3-small [⭐] │ │ │ └─────────────────────────────────┘ │ │ │ │ Text-to-Speech │ │ ┌─────────────────────────────────┐ │ │ │ OpenAI TTS [⭐] │ │ │ │ Google TTS │ │ │ └─────────────────────────────────┘ │ └───────────────────────────────────────┘ ``` - **⭐** = Default model for that category - **[Edit]** = Modify configuration - **[+ Add]** = Add new model --- ## Search Page Query across all notebooks: ``` ┌───────────────────────────────────────┐ │ Search │ ├───────────────────────────────────────┤ │ [What are you looking for? ] [🔍] │ │ │ │ Search type: [Text ▼] [Vector ▼] │ │ Search in: [Sources] [Notes] │ ├───────────────────────────────────────┤ │ Results (15) │ │ │ │ 📄 Paper.pdf - Notebook: Research │ │ "...the transformer model..." │ │ │ │ 📝 My Analysis - Notebook: Research │ │ "...key findings include..." │ └───────────────────────────────────────┘ ``` --- ## Common Actions ### Create a Notebook ``` Notebooks page → [+ New Notebook] → Enter name → Create ``` ### Add a Source ``` Inside notebook → [+ Add Source] → Choose type → Upload/paste → Wait for processing ``` ### Ask a Question ``` Inside notebook → Chat panel → Type question → Enter → Read response ``` ### Save AI Response ``` Get good response → Click [Save as Note] → Edit title → Save ``` ### Change Context Level ``` Click source → Context dropdown → Select level → Changes apply immediately ``` ### Generate Podcast ``` Podcasts tab → Select profile → Choose sources → [Generate] → Wait → Download ``` --- ## Keyboard Shortcuts | Key | Action | |-----|--------| | `Enter` | Send chat message | | `Shift + Enter` | New line in chat | | `Escape` | Close dialogs | | `Ctrl/Cmd + F` | Browser find | --- ## Mobile View On smaller screens, the three-panel layout stacks vertically: ``` ┌─────────────────┐ │ SOURCES │ │ (tap to expand) ├─────────────────┤ │ NOTES │ │ (tap to expand) ├─────────────────┤ │ CHAT │ │ (always visible) └─────────────────┘ ``` - Panels collapse to save space - Tap headers to expand/collapse - Chat remains accessible - Full functionality preserved --- ## Tips for Efficient Navigation 1. **Use keyboard** - Enter sends messages, Escape closes dialogs 2. **Context first** - Set source context before chatting 3. **Sessions** - Create new sessions for different topics 4. **Search globally** - Use Search page to find across all notebooks 5. **Models page** - Bookmark your preferred models --- Now you know where everything is. Start with [Adding Sources](adding-sources.md) to begin your research! ================================================ FILE: docs/3-USER-GUIDE/search.md ================================================ # Search Effectively - Finding What You Need Search is your gateway into your research. This guide covers two search modes and when to use each. --- ## Quick-Start: Find Something ### Simple Search ``` 1. Go to your notebook 2. Type in search box 3. See results (both sources and notes) 4. Click result to view source/note 5. Done! That works for basic searches. But you can do much better... ``` --- ## Two Search Modes Explained Open Notebook has two fundamentally different search approaches. ### Search Type 1: TEXT SEARCH (Keyword Matching) **How it works:** - You search for words: "transformer" - System finds chunks containing "transformer" - Ranked by relevance: frequency, position, context **Speed:** Very fast (instant) **When to use:** - You remember exact words or phrases - You're looking for specific terms - You want precise keyword matches - You need exact quotes **Example:** ``` Search: "attention mechanism" Results: 1. "The attention mechanism allows..." (perfect match) 2. "Attention and other mechanisms..." (partial match) 3. "How mechanisms work in attention..." (includes words separately) All contain "attention" AND "mechanism" Ranked by how close together they are ``` **What it finds:** - Exact phrases: "transformer model" - Individual words: transformer OR model (too broad) - Names: "Vaswani et al." - Numbers: "1994", "GPT-4" - Technical terms: "LSTM", "convolution" **What it doesn't find:** - Similar words: searching "attention" won't find "focus" - Synonyms: searching "large" won't find "big" - Concepts: searching "similarity" won't find "likeness" --- ### Search Type 2: VECTOR SEARCH (Semantic/Concept Matching) **How it works:** - Your search converted to embedding (vector) - All chunks converted to embeddings - System finds most similar embeddings - Ranked by semantic similarity **Speed:** A bit slower (1-2 seconds) **When to use:** - You're exploring a concept - You don't know exact words - You want semantically similar content - You're discovering, not searching **Example:** ``` Search: "What's the mechanism for understanding in models?" (Notice: No chunk likely says exactly that) Results: 1. "Mechanistic interpretability allows understanding..." (semantic match) 2. "Feature attribution reveals how models work..." (conceptually similar) 3. "Attention visualization shows model decisions..." (same topic) None contain your exact words But all are semantically related ``` **What it finds:** - Similar concepts: "understanding" + "interpretation" + "explainability" (all related) - Paraphrases: "big" and "large" (same meaning) - Related ideas: "safety" relates to "alignment" (connected concepts) - Analogies: content about biological learning when searching "learning" **What it doesn't find:** - Exact keywords: if you search a rare word, vector search might miss it - Specific numbers: "1994" vs "1993" are semantically different - Technical jargon: "LSTM" and "RNN" are different even if related --- ## Decision: Text Search vs. Vector Search? ``` Question: "Do I remember the exact words?" → YES: Use TEXT SEARCH Example: "I remember the paper said 'attention is all you need'" → NO: Use VECTOR SEARCH Example: "I'm looking for content about how models process information" → UNSURE: Try TEXT SEARCH first (faster) If no results, try VECTOR SEARCH Text search: "I know what I'm looking for" Vector search: "I'm exploring an idea" ``` --- ## Step-by-Step: Using Each Search ### Text Search ``` 1. Go to search box 2. Type your keywords: "transformer", "attention", "2017" 3. Press Enter 4. Results appear (usually instant) 5. Click result to see context Results show: - Which source contains it - How many times it appears - Relevance score - Preview of surrounding text ``` ### Vector Search ``` 1. Go to search box 2. Type your concept: "How do models understand language?" 3. Choose "Vector Search" from dropdown 4. Press Enter 5. Results appear (1-2 seconds) 6. Click result to see context Results show: - Semantically related chunks - Similarity score (higher = more related) - Preview of surrounding text - Different sources mixed together ``` --- ## The Ask Feature (Automated Search) Ask is different from simple search. It automatically searches, synthesizes, and answers. ### How Ask Works ``` Stage 1: QUESTION UNDERSTANDING "Compare the approaches in my papers" → System: "This asks for comparison" Stage 2: SEARCH STRATEGY → System: "I should search for each approach separately" Stage 3: PARALLEL SEARCHES → Search 1: "Approach in paper A" → Search 2: "Approach in paper B" (Multiple searches happen at once) Stage 4: ANALYSIS & SYNTHESIS → Per-result analysis: "Based on paper A, the approach is..." → Per-result analysis: "Based on paper B, the approach is..." → Final synthesis: "Comparing A and B: A differs from B in..." Result: Comprehensive answer, not just search results ``` ### When to Use Ask vs. Simple Search | Task | Use | Why | |------|-----|-----| | "Find the quote about X" | **TEXT SEARCH** | Need exact words | | "What does source A say about X?" | **TEXT SEARCH** | Direct, fast answer | | "Find content about X" | **VECTOR SEARCH** | Semantic discovery | | "Compare A and B" | **ASK** | Comprehensive synthesis | | "What's the big picture?" | **ASK** | Full analysis needed | | "How do these sources relate?" | **ASK** | Cross-source synthesis | | "I remember something about X" | **TEXT SEARCH** | Recall memory | | "I'm exploring the topic of X" | **VECTOR SEARCH** | Discovery mode | --- ## Advanced Search Strategies ### Strategy 1: Simple Search with Follow-Up ``` 1. Text search: "attention mechanism" Results: 50 matches 2. Too many. Follow up with vector search: "Why is attention useful?" (concept search) Results: Most relevant papers/notes 3. Better results with less noise ``` ### Strategy 2: Ask for Comprehensive, Then Search for Details ``` 1. Ask: "What are the main approaches to X?" Result: Comprehensive answer about A, B, C 2. Use that to identify specific sources 3. Text search in those specific sources: "Why did they choose method X?" Result: Detailed information ``` ### Strategy 3: Vector Search for Discovery, Text for Verification ``` 1. Vector search: "How do transformers generalize?" Results: Related conceptual papers 2. Skim to understand landscape 3. Text search in promising sources: "generalization", "extrapolation", "transfer" Results: Specific passages to read carefully ``` ### Strategy 4: Combine Search with Chat ``` 1. Vector search: "What's new in AI 2026?" Results: Latest papers 2. Go to Chat 3. Add those papers to context 4. Ask detailed follow-up questions 5. Get deep analysis of results ``` --- ## Search Quality Issues & Fixes ### Getting No Results | Problem | Cause | Solution | |---------|-------|----------| | Text search: no results | Word doesn't appear | Try vector search instead | | Vector search: no results | Concept not in content | Try broader search term | | Both empty | Content not in notebook | Add sources to notebook | | | Sources not processed | Wait for processing to complete | ### Getting Too Many Results | Problem | Cause | Solution | |---------|-------|----------| | 1000+ results | Search too broad | Be more specific | | | All sources | Filter by source | | | Keyword matches rare words | Use vector search instead | ### Getting Wrong Results | Problem | Cause | Solution | |---------|-------|----------| | Results irrelevant | Search term has multiple meanings | Provide more context | | | Using text search for concepts | Try vector search | | Different meaning | Homonym (word means multiple things) | Add context (e.g., "attention mechanism") | ### Getting Low Quality Results | Problem | Cause | Solution | |---------|-------|----------| | Results don't match intent | Vague search term | Be specific ("Who invented X?" vs "X") | | | Concept not well-represented | Add more sources on that topic | | | Vector embedding not trained on domain | Use text search as fallback | --- ## Tips for Better Searches ### For Text Search 1. **Be specific** — "attention mechanism" not just "attention" 2. **Use exact phrases** — Put quotes around: "attention is all you need" 3. **Include context** — "LSTM vs attention" not just "attention" 4. **Use technical terms** — These are usually more precise 5. **Try synonyms** — If first search fails, try related terms ### For Vector Search 1. **Ask a question** — "What's the best way to X?" is better than "best way" 2. **Use natural language** — Explain what you're looking for 3. **Be specific about intent** — "Compare X and Y" not "X and Y" 4. **Include context** — "In machine learning, how..." vs just "how..." 5. **Think conceptually** — What idea are you exploring? ### General Tips 1. **Start broad, then narrow** — "AI papers" → "transformers" → "attention mechanism" 2. **Try both search types** — Each finds different things 3. **Use Ask for complex questions** — Don't just search 4. **Save good results as notes** — Create knowledge base 5. **Filter by source if needed** — "Search in Paper A only" --- ## Search Examples ### Example 1: Finding a Specific Fact **Goal:** "Find the date the transformer was introduced" ``` Step 1: Text search "transformer 2017" (or year you remember) If that works: Done! If no results: Try "attention is all you need" (famous paper title) Check result for exact date ``` ### Example 2: Exploring a Concept **Goal:** "Find content about alignment interpretability" ``` Step 1: Vector search "How do we make AI interpretable?" Results: Papers on interpretability, transparency, alignment Step 2: Review results See which papers are most relevant Step 3: Deep dive Go to Chat, add top 2-3 papers Ask detailed questions about alignment ``` ### Example 3: Comprehensive Answer **Goal:** "How do different approaches to AI safety compare?" ``` Step 1: Ask "Compare the main approaches to AI safety in my sources" Result: Comprehensive analysis comparing approaches Step 2: Identify sources From answer, see which papers were most relevant Step 3: Deep dive Text search in those papers: "limitations", "critiques", "open problems" Step 4: Save as notes Create comparison note from Ask result ``` ### Example 4: Finding Pattern **Goal:** "Find all papers mentioning transformers" ``` Step 1: Text search "transformer" Results: All papers mentioning "transformer" Step 2: Vector search "neural network architecture for sequence processing" Results: Papers that don't say "transformer" but discuss similar concept Step 3: Combine Union of text + vector results shows full landscape Step 4: Analyze Go to Chat with all results Ask: "What's common across all these?" ``` --- ## Search in the Workflow How search fits with other features: ``` SOURCES ↓ SEARCH (find what matters) ├─ Text search (precise) ├─ Vector search (exploration) └─ Ask (comprehensive) ↓ CHAT (explore with follow-ups) ↓ TRANSFORMATIONS (batch extract) ↓ NOTES (save insights) ``` ### Workflow Example ``` 1. Add 10 papers to notebook 2. Search: "What's the state of the art?" (Vector search explores landscape) 3. Ask: "Compare these 3 approaches" (Comprehensive synthesis) 4. Chat: Deep questions about winner (Follow-up exploration) 5. Save best insights as notes (Knowledge capture) 6. Transform remaining papers (Batch extraction for later) 7. Create podcast from notes + sources (Share findings) ``` --- ## Summary: Know Your Search **TEXT SEARCH** — "I know what I'm looking for" - Fast, precise, keyword-based - Use when you remember exact words/phrases - Best for: Finding specific facts, quotes, technical terms - Speed: Instant **VECTOR SEARCH** — "I'm exploring an idea" - Slow-ish, concept-based, semantic - Use when you're discovering connections - Best for: Concept exploration, related ideas, synonyms - Speed: 1-2 seconds **ASK** — "I want a comprehensive answer" - Auto-searches, auto-analyzes, synthesizes - Use for complex questions needing multiple sources - Best for: Comparisons, big-picture questions, synthesis - Speed: 10-30 seconds Pick the right tool for your search goal, and you'll find what you need faster. ================================================ FILE: docs/3-USER-GUIDE/transformations.md ================================================ # Transformations - Batch Processing Your Sources Transformations apply the same analysis to multiple sources at once. Instead of asking the same question repeatedly, define a template and run it across your content. --- ## When to Use Transformations | Use Transformations When | Use Chat Instead When | |-------------------------|----------------------| | Same analysis on many sources | One-off questions | | Consistent output format needed | Exploratory conversation | | Batch processing | Follow-up questions needed | | Creating structured notes | Context changes between questions | **Example**: You have 10 papers and want a summary of each. Transformation does it in one operation. --- ## Quick Start: Your First Transformation ``` 1. Go to your notebook 2. Click "Transformations" in navigation 3. Select a built-in template (e.g., "Summary") 4. Select sources to transform 5. Click "Apply" 6. Wait for processing 7. New notes appear automatically ``` --- ## Built-in Transformations Open Notebook includes ready-to-use templates: ### Summary ``` What it does: Creates a 200-300 word overview Output: Key points, main arguments, conclusions Best for: Quick reference, getting the gist ``` ### Key Concepts ``` What it does: Extracts main ideas and terminology Output: List of concepts with explanations Best for: Learning new topics, building vocabulary ``` ### Methodology ``` What it does: Extracts research approach Output: How the study was conducted Best for: Academic papers, research review ``` ### Takeaways ``` What it does: Extracts actionable insights Output: What you should do with this information Best for: Business documents, practical guides ``` ### Questions ``` What it does: Generates questions the source raises Output: Open questions, gaps, follow-up research Best for: Literature review, research planning ``` --- ## Creating Custom Transformations ### Step-by-Step ``` 1. Go to "Transformations" page 2. Click "Create New" 3. Enter a name: "Academic Paper Analysis" 4. Write your prompt template: "Analyze this academic paper and extract: 1. **Research Question**: What problem does this address? 2. **Hypothesis**: What did they predict? 3. **Methodology**: How did they test it? 4. **Key Findings**: What did they discover? (numbered list) 5. **Limitations**: What caveats do the authors mention? 6. **Future Work**: What do they suggest next? Be specific and cite page numbers where possible." 5. Click "Save" 6. Your transformation appears in the list ``` ### Prompt Template Tips **Be specific about format:** ``` Good: "List 5 key points as bullet points" Bad: "What are the key points?" ``` **Request structure:** ``` Good: "Create sections for: Summary, Methods, Results" Bad: "Tell me about this paper" ``` **Ask for citations:** ``` Good: "Cite page numbers for each claim" Bad: (no citation request) ``` **Set length expectations:** ``` Good: "In 200-300 words, summarize..." Bad: "Summarize this" ``` --- ## Applying Transformations ### To a Single Source ``` 1. In Sources panel, click source menu (⋮) 2. Select "Transform" 3. Choose transformation template 4. Click "Apply" 5. Note appears when done ``` ### To Multiple Sources (Batch) ``` 1. Go to Transformations page 2. Select your template 3. Check multiple sources 4. Click "Apply to Selected" 5. Processing runs in parallel 6. One note per source created ``` ### Processing Time | Sources | Typical Time | |---------|--------------| | 1 source | 30 seconds - 1 minute | | 5 sources | 2-3 minutes | | 10 sources | 4-5 minutes | | 20+ sources | 8-10 minutes | Processing runs in background. You can continue working. --- ## Transformation Examples ### Literature Review Template ``` Name: Literature Review Entry Prompt: "For this research paper, create a literature review entry: **Citation**: [Author(s), Year, Title, Journal] **Research Question**: What problem is addressed? **Methodology**: What approach was used? **Sample**: What population/data was studied? **Key Findings**: 1. [Finding with page citation] 2. [Finding with page citation] 3. [Finding with page citation] **Strengths**: What did this study do well? **Limitations**: What are the gaps? **Relevance**: How does this connect to my research? Keep each section to 2-3 sentences." ``` ### Meeting Notes Template ``` Name: Meeting Summary Prompt: "From this meeting transcript, extract: **Attendees**: Who was present **Date/Time**: When it occurred **Key Decisions**: What was decided (numbered) **Action Items**: - [ ] Task (Owner, Due Date) **Open Questions**: Unresolved issues **Next Steps**: What happens next Format as clear, scannable notes." ``` ### Competitor Analysis Template ``` Name: Competitor Analysis Prompt: "Analyze this company/product document: **Company**: Name and overview **Products/Services**: What they offer **Target Market**: Who they serve **Pricing**: If available **Strengths**: Competitive advantages **Weaknesses**: Gaps or limitations **Opportunities**: How we compare **Threats**: What they do better Be objective and cite specific details." ``` ### Technical Documentation Template ``` Name: API Documentation Summary Prompt: "Extract from this technical document: **Overview**: What does this do? (1-2 sentences) **Authentication**: How to authenticate **Key Endpoints**: - Endpoint 1: [method] [path] - [purpose] - Endpoint 2: ... **Common Parameters**: Frequently used params **Rate Limits**: If mentioned **Error Codes**: Key error responses **Example Usage**: Simple code example if possible Keep technical but concise." ``` --- ## Managing Transformations ### Edit a Transformation ``` 1. Go to Transformations page 2. Find your template 3. Click "Edit" 4. Modify the prompt 5. Click "Save" ``` ### Delete a Transformation ``` 1. Go to Transformations page 2. Find the template 3. Click "Delete" 4. Confirm ``` ### Reorder/Organize Built-in transformations appear first, then custom ones alphabetically. --- ## Transformation Output ### Where Results Go - Each source produces one note - Notes appear in your notebook's Notes panel - Notes are tagged with transformation name - Original source is linked ### Note Naming ``` Default: "[Transformation Name] - [Source Title]" Example: "Summary - Research Paper 2025.pdf" ``` ### Editing Output ``` 1. Click the generated note 2. Click "Edit" 3. Refine the content 4. Save ``` --- ## Best Practices ### Template Design 1. **Start specific** - Vague prompts give vague results 2. **Use formatting** - Headings, bullets, numbered lists 3. **Request citations** - Make results verifiable 4. **Set length** - Prevent overly long or short output 5. **Test first** - Run on one source before batch ### Source Selection 1. **Similar content** - Same transformation on similar sources 2. **Reasonable size** - Very long sources may need splitting 3. **Processed status** - Ensure sources are fully processed ### Quality Control 1. **Review samples** - Check first few outputs before trusting batch 2. **Edit as needed** - Transformations are starting points 3. **Iterate prompts** - Refine based on results --- ## Common Issues ### Generic Output **Problem**: Results are too vague **Solution**: Make prompt more specific, add format requirements ### Missing Information **Problem**: Key details not extracted **Solution**: Explicitly ask for what you need in prompt ### Inconsistent Format **Problem**: Each note looks different **Solution**: Add clear formatting instructions to prompt ### Too Long/Short **Problem**: Output doesn't match expectations **Solution**: Specify word count or section lengths ### Processing Fails **Problem**: Transformation doesn't complete **Solution**: - Check source is processed - Try shorter/simpler prompt - Process sources individually --- ## Transformations vs. Chat vs. Ask | Feature | Transformations | Chat | Ask | |---------|----------------|------|-----| | **Input** | Predefined template | Your questions | Your question | | **Scope** | One source at a time | Selected sources | Auto-searched | | **Output** | Structured note | Conversation | Comprehensive answer | | **Best for** | Batch processing | Exploration | One-shot answers | | **Follow-up** | Run again | Ask more | New query | --- ## Summary ``` Transformations = Batch AI Processing How to use: 1. Define template (or use built-in) 2. Select sources 3. Apply transformation 4. Get structured notes When to use: - Same analysis on many sources - Consistent output needed - Building structured knowledge base - Saving time on repetitive tasks Tips: - Be specific in prompts - Request formatting - Test before batch - Edit output as needed ``` Transformations turn repetitive analysis into one-click operations. Define once, apply many times. ================================================ FILE: docs/3-USER-GUIDE/working-with-notes.md ================================================ # Working with Notes - Capturing and Organizing Insights Notes are your processed knowledge. This guide covers how to create, organize, and use them effectively. --- ## What Are Notes? Notes are your **research output** — the insights you capture from analyzing sources. They can be: - **Manual** — You write them yourself - **AI-Generated** — From Chat responses, Ask results, or Transformations - **Hybrid** — AI insight + your edits and additions Unlike sources (which never change), notes are mutable — you edit, refine, and organize them. --- ## Quick-Start: Create Your First Note ### Method 1: Manual Note (Write Yourself) ``` 1. In your notebook, go to "Notes" section 2. Click "Create New Note" 3. Give it a title: "Key insights from source X" 4. Write your content (markdown supported) 5. Click "Save" 6. Done! Note appears in your notebook ``` ### Method 2: Save from Chat ``` 1. Have a Chat conversation 2. Get a good response from AI 3. Click "Save as Note" button under response 4. Give the note a title 5. Add any additional context 6. Click "Save" 7. Done! Note appears in your notebook ``` ### Method 3: Apply Transformation ``` 1. Go to "Transformations" 2. Select a template (or create custom) 3. Click "Apply to sources" 4. Select which sources to transform 5. Wait for processing 6. New notes automatically appear 7. Done! Each source produces one note ``` --- ## Creating Manual Notes ### Basic Structure ``` Title: "What you're capturing" (Make it descriptive) Content: - Main points - Your analysis - Questions raised - Next steps Metadata: - Tags: How to categorize - Related sources: Which documents influenced this - Date: Auto-added when created ``` ### Markdown Support You can format notes with markdown: ```markdown # Heading ## Subheading ### Sub-subheading **Bold text** for emphasis *Italic text* for secondary emphasis - Bullet lists - Like this 1. Numbered lists 2. Like this > Quotes and important callouts [Links work](https://example.com) ``` ### Example Note Structure ```markdown # Key Findings from "AI Safety Paper 2025" ## Main Argument The paper argues that X approach is better than Y because... ## Methodology The authors use [methodology] to test this hypothesis. ## Key Results - Result 1: [specific finding with citation] - Result 2: [specific finding with citation] - Result 3: [specific finding with citation] ## Gaps & Limitations 1. The paper assumes X, which might not hold in Y scenario 2. Limited to Z population/domain 3. Future work needed on A, B, C ## My Thoughts - This connects to previous research on... - Potential application in... ## Next Steps - [ ] Read the referenced paper on X - [ ] Find similar studies on Y - [ ] Discuss implications with team ``` --- ## AI-Generated Notes: Three Sources ### 1. Save from Chat ``` Workflow: Chat → Good response → "Save as Note" → Edit if needed → Save When to use: - AI response answers your question well - You want to keep the answer for reference - You're building a knowledge base from conversations Quality: - Quality = quality of your Chat question - Better context = better responses = better notes - Ask specific questions for useful notes ``` ### 2. Save from Ask ``` Workflow: Ask → Comprehensive answer → "Save as Note" → Edit if needed → Save When to use: - You need a one-time comprehensive answer - You want to save the synthesized result - Building a knowledge base of comprehensive answers Quality: - System automatically found relevant sources - Results already have citations - Often higher quality than Chat (more thorough) ``` ### 3. Transformations (Batch Processing) ``` Workflow: Define transformation → Apply to sources → Notes auto-created → Review & edit → Organize Example Transformation: Template: "Extract: main argument, methodology, key findings" Apply to: 5 sources Result: 5 new notes with consistent structure When to use: - Same extraction from many sources - Building structured knowledge base - Creating consistent summaries ``` --- ## Using Transformations for Batch Insights ### Built-in Transformations Open Notebook comes with presets: **Summary** ``` Extracts: Main points, key arguments, conclusions Output: 200-300 word summary of source Best for: Quick reference summaries ``` **Key Concepts** ``` Extracts: Main ideas, concepts, terminology Output: List of concepts with explanations Best for: Learning and terminology ``` **Methodology** ``` Extracts: Research approach, methods, data Output: How the research was conducted Best for: Academic sources, methodology review ``` **Takeaways** ``` Extracts: Actionable insights, recommendations Output: What you should do with this information Best for: Practical/business sources ``` ### How to Apply Transformation ``` 1. Go to "Transformations" 2. Select a template 3. Click "Apply" 4. Select which sources (one or many) 5. Wait for processing (usually 30 seconds - 2 minutes) 6. New notes appear in your notebook 7. Edit if needed ``` ### Create Custom Transformation ``` 1. Click "Create Custom Transformation" 2. Write your extraction template: Example: "For this academic paper, extract: - Central research question - Hypothesis tested - Methodology used - Key findings (numbered) - Limitations acknowledged - Recommendations for future work" 3. Click "Save Template" 4. Apply to one or many sources 5. System generates notes with consistent structure ``` --- ## Organizing Notes ### Naming Conventions **Option 1: Date-based** ``` 2026-01-03 - Key points from X source 2026-01-04 - Comparison between A and B Benefit: Easy to see what you did when ``` **Option 2: Topic-based** ``` AI Safety - Alignment approaches AI Safety - Interpretability research Benefit: Groups by subject matter ``` **Option 3: Type-based** ``` SUMMARY: Paper on X QUESTION: What about Y? INSIGHT: Connection between Z and W Benefit: Easy to filter by type ``` **Option 4: Source-based** ``` From: Paper A - Main insights From: Video B - Interesting implications Benefit: Easy to trace back to sources ``` **Best practice:** Combine approaches ``` [Date] [Source] - [Topic] - [Type] 2026-01-03 - Paper A - AI Safety - Takeaways ``` ### Using Tags Tags are labels for categorization. Add them when creating notes: ``` Example tags: - "primary-research" (direct source analysis) - "background" (supporting material) - "methodology" (about research methods) - "insights" (your original thinking) - "questions" (open questions raised) - "follow-up" (needs more work) - "published" (ready to share/use) ``` **Benefits of tags:** - Filter notes by tag - Find all notes of a type - Organize workflow (e.g., find all "follow-up" notes) ### Note Linking & References You can reference sources within notes: ```markdown # Analysis of Paper A As shown in Paper A (see "main argument" section), the authors argue that... ## Related Sources - Paper B discusses similar approach - Video C shows practical application - My note on "Comparative analysis" has more ``` --- ## Editing and Refining Notes ### Improving AI-Generated Notes ``` AI Note: "The paper discusses machine learning" What you might change: "The paper proposes a supervised learning approach to classification problems, using neural networks with attention mechanisms (see pp. 15-18)." How to edit: 1. Click note 2. Click "Edit" 3. Refine the content 4. Click "Save" ``` ### Adding Citations ``` When saving from Chat/Ask: - Citations auto-added - Shows which sources informed answer - You can verify by clicking When manual notes: - Add manually: "From Paper A, page 15: ..." - Or reference: "As discussed in [source]" ``` --- ## Searching Your Notes Notes are fully searchable: ### Text Search ``` Find exact phrase: "attention mechanism" Results: All notes containing that phrase Use when: Looking for specific terms or quotes ``` ### Vector/Semantic Search ``` Find concept: "How do models understand?" Results: Notes about interpretability, mechanistic understanding, etc. Use when: Exploring conceptually (words not exact) ``` ### Combined Search ``` Text search notes → Find keyword matches Vector search notes → Find conceptual matches Both work across sources + notes together ``` --- ## Exporting and Sharing Notes ### Options **Copy to clipboard** ``` Click "Share" → "Copy" → Paste anywhere Good for: Sharing one note via email/chat ``` **Export as Markdown** ``` Click "Share" → "Export as MD" → Saves as .md file Good for: Sharing with others, version control ``` **Create note collection** ``` Select multiple notes → "Export collection" → Creates organized markdown document Good for: Sharing a topic overview ``` **Publish to web** ``` Click "Publish" → Get shareable link Good for: Publishing publicly (if desired) ``` --- ## Organizing Your Notebook's Notes ### By Research Phase **Phase 1: Discovery** - Initial summaries - Questions raised - Interesting findings **Phase 2: Deep Dive** - Detailed analysis - Comparative insights - Methodology reviews **Phase 3: Synthesis** - Connections across sources - Original thinking - Conclusions ### By Content Type **Summaries** - High-level overviews - Generated by transformations - Quick reference **Questions** - Open questions - Things to research more - Gaps to fill **Insights** - Your original analysis - Connections made - Conclusions reached **Tasks** - Follow-up research - Sources to add - People to contact --- ## Using Notes in Other Features ### In Chat ``` You can reference notes: "Based on my note 'Key findings from A', how does this compare to B?" Notes become part of context. Treated like sources but smaller/more focused. ``` ### In Transformations ``` Notes can be transformed: 1. Select notes as input 2. Apply transformation 3. Get new derived notes Example: Transform 5 analysis notes → Create synthesis ``` ### In Podcasts ``` Notes are used to create podcast content: 1. Generate podcast for notebook 2. System includes notes in content selection 3. Notes become part of episode outline ``` --- ## Best Practices ### For Manual Notes 1. **Write clearly** — Future you will appreciate it 2. **Add context** — Why this matters, not just what it says 3. **Link to sources** — You can verify later 4. **Date them** — Track your thinking over time 5. **Tag immediately** — Don't defer organization ### For AI-Generated Notes 1. **Review before saving** — Verify quality 2. **Edit for clarity** — AI might miss nuance 3. **Add your thoughts** — Make it your own 4. **Include citations** — Understand sources 5. **Organize right away** — While context is fresh ### For Organization 1. **Consistent naming** — Your future self will thank you 2. **Tag everything** — Makes filtering later much easier 3. **Link related notes** — Create knowledge network 4. **Review periodically** — Refactor as understanding evolves 5. **Archive old notes** — Keep working space clean --- ## Common Mistakes | Mistake | Problem | Solution | |---------|---------|----------| | Save every Chat response | Notebook becomes cluttered with low-quality notes | Only save good responses that answer your questions | | Don't add tags | Can't find notes later | Tag immediately when creating | | Poor note titles | Can't remember what's in them | Use descriptive titles, include key concept | | Never link notes together | Miss connections between ideas | Add references to related notes | | Forget the source | Can't verify claims later | Always link back to source | | Never edit AI notes | Keep generic AI responses | Refine for clarity and context | | Create one giant note | Too long to be useful | Split into focused notes by subtopic | --- ## Summary: Note Lifecycle ``` 1. CREATE ├─ Manual: Write from scratch ├─ From Chat: Save good response ├─ From Ask: Save synthesis └─ From Transform: Batch process 2. EDIT & REFINE ├─ Improve clarity ├─ Add context ├─ Fix AI mistakes └─ Add citations 3. ORGANIZE ├─ Name clearly ├─ Add tags ├─ Link related └─ Categorize 4. USE ├─ Reference in Chat ├─ Transform for synthesis ├─ Export for sharing └─ Build on with new questions 5. MAINTAIN ├─ Periodically review ├─ Update as understanding grows ├─ Archive when done └─ Learn from organized knowledge ``` Your notes become your actual knowledge base. The more you invest in organizing them, the more valuable they become. ================================================ FILE: docs/4-AI-PROVIDERS/index.md ================================================ # AI Providers - Comparison & Selection Guide Open Notebook supports 15+ AI providers. This guide helps you **choose the right provider** for your needs. > 💡 **Just want to set up a provider?** Skip to the [Configuration Guide](../5-CONFIGURATION/ai-providers.md) for detailed setup instructions. --- ## Quick Decision: Which Provider? ### Cloud Providers (Easiest) **OpenAI (Recommended)** - Cost: ~$0.03-0.15 per 1K tokens - Speed: Very fast - Quality: Excellent - Best for: Most users (best quality/price balance) → [Setup Guide](../5-CONFIGURATION/ai-providers.md#openai) **Anthropic (Claude)** - Cost: ~$0.80-3.00 per 1M tokens - Speed: Fast - Quality: Excellent - Best for: Long context (200K tokens), reasoning, latest AI - Advantage: Superior long-context handling → [Setup Guide](../5-CONFIGURATION/ai-providers.md#anthropic-claude) **Google Gemini** - Cost: ~$0.075-0.30 per 1K tokens - Speed: Very fast - Quality: Good to excellent - Best for: Multimodal (images, audio, video) - Advantage: Longest context (up to 2M tokens) → [Setup Guide](../5-CONFIGURATION/ai-providers.md#google-gemini) **Groq (Ultra-Fast)** - Cost: ~$0.05 per 1M tokens (cheapest) - Speed: Ultra-fast (fastest available) - Quality: Good - Best for: Budget-conscious, transformations, speed-critical tasks - Disadvantage: Limited model selection → [Setup Guide](../5-CONFIGURATION/ai-providers.md#groq) **OpenRouter (100+ Models)** - Cost: Pay-per-model (varies widely) - Speed: Varies by model - Quality: Varies by model - Best for: Model comparison, testing, unified billing - Advantage: One API key for 100+ models from different providers → [Setup Guide](../5-CONFIGURATION/ai-providers.md#openrouter) ### Local / Self-Hosted (Free) **Ollama (Recommended for Local)** - Cost: Free (electricity only) - Speed: Depends on hardware (slow on CPU, fast on GPU) - Quality: Good (open-source models) - Setup: 10 minutes - Best for: Privacy-first, offline use - Privacy: 100% local, nothing leaves your machine → [Setup Guide](../5-CONFIGURATION/ai-providers.md#ollama-recommended-for-local) **LM Studio (Alternative)** - Cost: Free (electricity only) - Speed: Depends on hardware - Quality: Good (same models as Ollama) - Setup: 15 minutes (GUI interface) - Best for: Non-technical users who prefer GUI over CLI - Privacy: 100% local → [Setup Guide](../5-CONFIGURATION/ai-providers.md#lm-studio-local-alternative) ### Enterprise **Azure OpenAI** - Cost: Same as OpenAI (usage-based) - Speed: Very fast - Quality: Excellent (same models as OpenAI) - Setup: 10 minutes (more complex) - Best for: Enterprise, compliance (HIPAA, SOC2), VPC integration → [Setup Guide](../5-CONFIGURATION/ai-providers.md#azure-openai) --- ## Comparison Table | Provider | Speed | Cost | Quality | Privacy | Setup | Context | |----------|-------|------|---------|---------|-------|---------| | **OpenAI** | Very Fast | $$ | Excellent | Low | 5 min | 128K | | **Anthropic** | Fast | $$ | Excellent | Low | 5 min | 200K | | **Google** | Very Fast | $$ | Good-Excellent | Low | 5 min | 2M | | **Groq** | Ultra Fast | $ | Good | Low | 5 min | 32K | | **OpenRouter** | Varies | Varies | Varies | Low | 5 min | Varies | | **Ollama** | Slow-Medium | Free | Good | Max | 10 min | Varies | | **LM Studio** | Slow-Medium | Free | Good | Max | 15 min | Varies | | **Azure** | Very Fast | $$ | Excellent | High | 10 min | 128K | --- ## Choosing Your Provider ### I want the easiest setup → **OpenAI** — Most popular, best community support ### I have unlimited budget → **OpenAI** — Best quality ### I want to save money → **Groq** — Cheapest cloud ($0.05 per 1M tokens) ### I want privacy/offline → **Ollama** — Free, local, private ### I want a GUI (not CLI) → **LM Studio** — Desktop app ### I'm in an enterprise → **Azure OpenAI** — Compliance, support ### I need long context (200K+ tokens) → **Anthropic** — Best long-context model ### I need multimodal (images, audio, video) → **Google Gemini** — Best multimodal support ### I want access to many models with one API key → **OpenRouter** — 100+ models, unified billing --- ## Ready to Set Up Your Provider? Now that you've chosen a provider, follow the detailed setup instructions: → **[AI Providers Configuration Guide](../5-CONFIGURATION/ai-providers.md)** This guide includes: - Step-by-step setup instructions for each provider via the Settings UI - How to add credentials, test connections, and discover models - Model selection and recommendations - Provider-specific troubleshooting - Hardware requirements (for local providers) - Cost optimization tips --- ## Cost Estimator ### OpenAI ``` Light use (10 chats/day): $1-5/month Medium use (50 chats/day): $10-30/month Heavy use (all-day use): $50-100+/month ``` ### Anthropic ``` Light use: $1-3/month Medium use: $5-20/month Heavy use: $20-50+/month ``` ### Groq ``` Light use: $0-1/month Medium use: $2-5/month Heavy use: $5-20/month ``` ### Ollama ``` Any use: Free (electricity only) 8GB GPU running 24/7: ~$10/month electricity ``` --- ## Next Steps 1. **You've chosen a provider** (from this comparison guide) 2. **Follow the setup guide**: [AI Providers Configuration](../5-CONFIGURATION/ai-providers.md) 3. **Add your credential** in Settings → API Keys 4. **Test your connection** and discover models 5. **Start using Open Notebook!** --- ## Need Help? - **Setup issues?** See [AI Providers Configuration](../5-CONFIGURATION/ai-providers.md) for detailed troubleshooting per provider - **General problems?** Check [Troubleshooting Guide](../6-TROUBLESHOOTING/index.md) - **Questions?** Join [Discord community](https://discord.gg/37XJPXfz2w) ================================================ FILE: docs/5-CONFIGURATION/advanced.md ================================================ # Advanced Configuration Performance tuning, debugging, and advanced features. --- ## Performance Tuning ### Concurrency Control ```env # Max concurrent database operations (default: 5) # Increase: Faster processing, more conflicts # Decrease: Slower, fewer conflicts SURREAL_COMMANDS_MAX_TASKS=5 ``` **Guidelines:** - CPU: 2 cores → 2-3 tasks - CPU: 4 cores → 5 tasks (default) - CPU: 8+ cores → 10-20 tasks Higher concurrency = more throughput but more database conflicts (retries handle this). ### Retry Strategy ```env # How to wait between retries SURREAL_COMMANDS_RETRY_WAIT_STRATEGY=exponential_jitter # Options: # - exponential_jitter (recommended) # - exponential # - fixed # - random ``` For high-concurrency deployments, use `exponential_jitter` to prevent thundering herd. ### Timeout Tuning ```env # Client timeout (default: 300 seconds) API_CLIENT_TIMEOUT=300 # LLM timeout (default: 60 seconds) ESPERANTO_LLM_TIMEOUT=60 ``` **Guideline:** Set `API_CLIENT_TIMEOUT` > `ESPERANTO_LLM_TIMEOUT` + buffer ``` Example: ESPERANTO_LLM_TIMEOUT=120 API_CLIENT_TIMEOUT=180 # 120 + 60 second buffer ``` --- ## Batching ### TTS Batch Size For podcast generation, control concurrent TTS requests: ```env # Default: 5 TTS_BATCH_SIZE=2 ``` **Providers and recommendations:** - OpenAI: 5 (can handle many concurrent) - Google: 4 (good concurrency) - ElevenLabs: 2 (limited concurrent requests) - Local TTS: 1 (single-threaded) Lower = slower but more stable. Higher = faster but more load on provider. --- ## Logging & Debugging ### Enable Detailed Logging ```bash # Start with debug logging RUST_LOG=debug # For Rust components LOGLEVEL=DEBUG # For Python components ``` ### Debug Specific Components ```bash # Only surreal operations RUST_LOG=surrealdb=debug # Only langchain LOGLEVEL=langchain:debug # Only specific module RUST_LOG=open_notebook::database=debug ``` ### LangSmith Tracing For debugging LLM workflows: ```env LANGCHAIN_TRACING_V2=true LANGCHAIN_ENDPOINT="https://api.smith.langchain.com" LANGCHAIN_API_KEY=your-key LANGCHAIN_PROJECT="Open Notebook" ``` Then visit https://smith.langchain.com to see traces. --- ## Port Configuration ### Default Ports ``` Frontend: 8502 (Docker deployment) Frontend: 3000 (Development from source) API: 5055 SurrealDB: 8000 ``` ### Changing Frontend Port Edit `docker-compose.yml`: ```yaml services: open-notebook: ports: - "8001:8502" # Change from 8502 to 8001 ``` Access at: `http://localhost:8001` API auto-detects to: `http://localhost:5055` ✓ ### Changing API Port ```yaml services: open-notebook: ports: - "127.0.0.1:8502:8502" # Frontend - "5056:5055" # Change API from 5055 to 5056 environment: - API_URL=http://localhost:5056 # Update API_URL ``` Access API directly: `http://localhost:5056/docs` **Note:** When changing API port, you must set `API_URL` explicitly since auto-detection assumes port 5055. ### Changing SurrealDB Port ```yaml services: surrealdb: ports: - "8001:8000" # Change from 8000 to 8001 environment: - SURREAL_URL=ws://surrealdb:8001/rpc # Update connection URL ``` **Important:** Internal Docker network uses container name (`surrealdb`), not `localhost`. --- ## SSL/TLS Configuration ### Custom CA Certificate For self-signed certs on local providers: ```env ESPERANTO_SSL_CA_BUNDLE=/path/to/ca-bundle.pem ``` ### Disable Verification (Development Only) ```env # WARNING: Only for testing/development # Vulnerable to MITM attacks ESPERANTO_SSL_VERIFY=false ``` --- ## Multi-Provider Setup ### Use Different Providers for Different Tasks Configure multiple AI providers via **Settings → API Keys**. Each provider gets its own credential: 1. Add a credential for your main language model provider (e.g., OpenAI, Anthropic) 2. Add a credential for embeddings (e.g., Voyage AI, or use the same provider) 3. Add a credential for TTS (e.g., ElevenLabs, or OpenAI-Compatible for local Speaches) 4. Each credential's models are registered and available independently ### Multiple Endpoints for OpenAI-Compatible When using OpenAI-Compatible providers, you can configure per-service URLs in a single credential: 1. Go to **Settings** → **API Keys** 2. Click **Add Credential** → Select **OpenAI-Compatible** 3. Configure separate URLs for LLM, Embedding, TTS, and STT 4. Click **Save**, then **Test Connection** --- ## Security Hardening ### Change Default Credentials ```env # Don't use defaults in production SURREAL_USER=your_secure_username SURREAL_PASSWORD=$(openssl rand -base64 32) # Generate secure password ``` ### Add Password Protection ```env # Protect your Open Notebook instance OPEN_NOTEBOOK_PASSWORD=your_secure_password ``` ### Use HTTPS ```env # Always use HTTPS in production API_URL=https://mynotebook.example.com ``` ### Firewall Rules Restrict access to your Open Notebook: - Port 8502 (frontend): Only from your IP - Port 5055 (API): Only from frontend - Port 8000 (SurrealDB): Never expose to internet --- ## Web Scraping & Content Extraction Open Notebook uses multiple services for content extraction: ### Firecrawl For advanced web scraping: ```env FIRECRAWL_API_KEY=your-key ``` Get key from: https://firecrawl.dev/ ### Jina AI Alternative web extraction: ```env JINA_API_KEY=your-key ``` Get key from: https://jina.ai/ --- ## Environment Variable Groups ### Credential Storage (Required) ```env OPEN_NOTEBOOK_ENCRYPTION_KEY # Required for storing credentials ``` AI provider API keys are configured via **Settings → API Keys** (not environment variables). ### Database ```env SURREAL_URL SURREAL_USER SURREAL_PASSWORD SURREAL_NAMESPACE SURREAL_DATABASE ``` ### Performance ```env SURREAL_COMMANDS_MAX_TASKS SURREAL_COMMANDS_RETRY_ENABLED SURREAL_COMMANDS_RETRY_MAX_ATTEMPTS SURREAL_COMMANDS_RETRY_WAIT_STRATEGY SURREAL_COMMANDS_RETRY_WAIT_MIN SURREAL_COMMANDS_RETRY_WAIT_MAX ``` ### API Settings ```env API_URL INTERNAL_API_URL API_CLIENT_TIMEOUT ESPERANTO_LLM_TIMEOUT ``` ### Audio/TTS ```env TTS_BATCH_SIZE ``` > **Note:** `ELEVENLABS_API_KEY` is deprecated. Configure ElevenLabs via **Settings → API Keys**. ### Debugging ```env LANGCHAIN_TRACING_V2 LANGCHAIN_ENDPOINT LANGCHAIN_API_KEY LANGCHAIN_PROJECT ``` --- ## Testing Configuration ### Quick Test ```bash # Test API health curl http://localhost:5055/health # Test with sample (requires configured credential and registered models) curl -X POST http://localhost:5055/api/chat \ -H "Content-Type: application/json" \ -d '{"message":"Hello"}' ``` ### Validate Config ```bash # Check environment variables are set env | grep OPEN_NOTEBOOK_ENCRYPTION_KEY # Verify database connection python -c "import os; print(os.getenv('SURREAL_URL'))" ``` --- ## Troubleshooting Performance ### High Memory Usage ```env # Reduce concurrency SURREAL_COMMANDS_MAX_TASKS=2 # Reduce TTS batch size TTS_BATCH_SIZE=1 ``` ### High CPU Usage ```env # Check worker count SURREAL_COMMANDS_MAX_TASKS # Reduce if maxed out: SURREAL_COMMANDS_MAX_TASKS=5 ``` ### Slow Responses ```env # Check timeout settings API_CLIENT_TIMEOUT=300 # Check retry config SURREAL_COMMANDS_RETRY_MAX_ATTEMPTS=3 ``` ### Database Conflicts ```env # Reduce concurrency SURREAL_COMMANDS_MAX_TASKS=3 # Use jitter strategy SURREAL_COMMANDS_RETRY_WAIT_STRATEGY=exponential_jitter ``` --- ## Backup & Restore ### Data Locations | Path | Contents | |------|----------| | `./data` or `/app/data` | Uploads, podcasts, checkpoints | | `./surreal_data` or `/mydata` | SurrealDB database files | ### Quick Backup ```bash # Stop services (recommended for consistency) docker compose down # Create timestamped backup tar -czf backup-$(date +%Y%m%d-%H%M%S).tar.gz \ notebook_data/ surreal_data/ # Restart services docker compose up -d ``` ### Automated Backup Script ```bash #!/bin/bash # backup.sh - Run daily via cron BACKUP_DIR="/path/to/backups" DATE=$(date +%Y%m%d-%H%M%S) # Create backup tar -czf "$BACKUP_DIR/open-notebook-$DATE.tar.gz" \ /path/to/notebook_data \ /path/to/surreal_data # Keep only last 7 days find "$BACKUP_DIR" -name "open-notebook-*.tar.gz" -mtime +7 -delete echo "Backup complete: open-notebook-$DATE.tar.gz" ``` Add to cron: ```bash # Daily backup at 2 AM 0 2 * * * /path/to/backup.sh >> /var/log/open-notebook-backup.log 2>&1 ``` ### Restore ```bash # Stop services docker compose down # Remove old data (careful!) rm -rf notebook_data/ surreal_data/ # Extract backup tar -xzf backup-20240115-120000.tar.gz # Restart services docker compose up -d ``` ### Migration Between Servers ```bash # On source server docker compose down tar -czf open-notebook-migration.tar.gz notebook_data/ surreal_data/ # Transfer to new server scp open-notebook-migration.tar.gz user@newserver:/path/ # On new server tar -xzf open-notebook-migration.tar.gz docker compose up -d ``` --- ## Container Management ### Common Commands ```bash # Start services docker compose up -d # Stop services docker compose down # View logs (all services) docker compose logs -f # View logs (specific service) docker compose logs -f api # Restart specific service docker compose restart api # Update to latest version docker compose down docker compose pull docker compose up -d # Check resource usage docker stats # Check service health docker compose ps ``` ### Clean Up ```bash # Remove stopped containers docker compose rm # Remove unused images docker image prune # Full cleanup (careful!) docker system prune -a ``` --- ## Summary **Most deployments need:** - One AI provider API key - Default database settings - Default timeouts **Tune performance only if:** - You have specific bottlenecks - High-concurrency workload - Custom hardware (very fast or very slow) **Advanced features:** - Firecrawl for better web scraping - LangSmith for debugging workflows - Custom CA bundles for self-signed certs ================================================ FILE: docs/5-CONFIGURATION/ai-providers.md ================================================ # AI Providers - Configuration Guide Complete setup instructions for each AI provider via the **Settings UI**. > **New in v1.2**: All AI provider credentials are now managed through the Settings UI. Environment variables for API keys are deprecated. --- ## How Provider Setup Works Open Notebook uses a **credential-based system** for managing AI providers: 1. **Get your API key** from the provider's website 2. **Open Settings** → **API Keys** → **Add Credential** 3. **Test the connection** to verify it works 4. **Discover & Register Models** to make them available 5. **Start using** the provider in your notebooks > **Prerequisite**: You must set `OPEN_NOTEBOOK_ENCRYPTION_KEY` in your docker-compose.yml before storing credentials. See [API Configuration](../3-USER-GUIDE/api-configuration.md#encryption-setup) for details. --- ## Cloud Providers (Recommended for Most) ### OpenAI **Cost:** ~$0.03-0.15 per 1K tokens (varies by model) **Get Your API Key:** 1. Go to https://platform.openai.com/api-keys 2. Create account (if needed) 3. Create new API key (starts with "sk-proj-") 4. Add $5+ credits to account **Configure in Open Notebook:** 1. Go to **Settings** → **API Keys** 2. Click **Add Credential** 3. Select provider: **OpenAI** 4. Give it a name (e.g., "My OpenAI Key") 5. Paste your API key 6. Click **Save**, then **Test Connection** 7. Click **Discover Models** to find available models 8. Click **Register Models** to make them available **Available Models (in Open Notebook):** - `gpt-4o` — Best quality, fast (latest version) - `gpt-4o-mini` — Fast, cheap, good for testing - `o1` — Advanced reasoning model (slower, more expensive) - `o1-mini` — Faster reasoning model **Recommended:** - For general use: `gpt-4o` (best balance) - For testing/cheap: `gpt-4o-mini` (90% cheaper) - For complex reasoning: `o1` (best for hard problems) **Cost Estimate:** ``` Light use: $1-5/month Medium use: $10-30/month Heavy use: $50-100+/month ``` **Troubleshooting:** - "Invalid API key" → Check key starts with "sk-proj-" and test the connection in Settings - "Rate limit exceeded" → Wait or upgrade account - "Model not available" → Try gpt-4o-mini instead, or re-discover models --- ### Anthropic (Claude) **Cost:** ~$0.80-3.00 per 1M tokens (cheaper than OpenAI for long context) **Get Your API Key:** 1. Go to https://console.anthropic.com/ 2. Create account or login 3. Go to API keys section 4. Create new API key (starts with "sk-ant-") **Configure in Open Notebook:** 1. Go to **Settings** → **API Keys** 2. Click **Add Credential** 3. Select provider: **Anthropic** 4. Give it a name, paste your API key 5. Click **Save**, then **Test Connection** 6. Click **Discover Models** → **Register Models** **Available Models:** - `claude-sonnet-4-5-20250929` — Latest, best quality (recommended) - `claude-3-5-sonnet-20241022` — Previous generation, still excellent - `claude-3-5-haiku-20241022` — Fast, cheap - `claude-opus-4-5-20251101` — Most powerful, expensive **Recommended:** - For general use: `claude-sonnet-4-5` (best overall, latest) - For cheap: `claude-3-5-haiku` (80% cheaper) - For complex: `claude-opus-4-5` (most capable) **Cost Estimate:** ``` Sonnet: $3-20/month (typical use) Haiku: $0.50-3/month Opus: $10-50+/month ``` **Advantages:** - Great long-context support (200K tokens) - Excellent reasoning - Fast processing **Troubleshooting:** - "Invalid API key" → Check it starts with "sk-ant-" and test in Settings - "Overloaded" → Anthropic is busy, retry later - "Model unavailable" → Re-discover models from the credential --- ### Google Gemini **Cost:** ~$0.075-0.30 per 1K tokens (competitive with OpenAI) **Get Your API Key:** 1. Go to https://aistudio.google.com/app/apikey 2. Create account or login 3. Create new API key **Configure in Open Notebook:** 1. Go to **Settings** → **API Keys** 2. Click **Add Credential** 3. Select provider: **Google Gemini** 4. Give it a name, paste your API key 5. Click **Save**, then **Test Connection** 6. Click **Discover Models** → **Register Models** **Available Models:** - `gemini-2.0-flash-exp` — Latest experimental, fastest (recommended) - `gemini-2.0-flash` — Stable version, fast, cheap **Recommended:** - For general use: `gemini-2.0-flash-exp` (best value, latest) - For cheap: `gemini-1.5-flash` (very cheap) - For complex/long context: `gemini-1.5-pro-latest` (2M token context) **Advantages:** - Very long context (1M tokens) - Multimodal (images, audio, video) - Good for podcasts **Troubleshooting:** - "API key invalid" → Get fresh key from aistudio.google.com - "Quota exceeded" → Free tier limited, upgrade account - "Model not found" → Re-discover models from the credential --- ### Groq **Cost:** ~$0.05 per 1M tokens (cheapest, but limited models) **Get Your API Key:** 1. Go to https://console.groq.com/keys 2. Create account or login 3. Create new API key **Configure in Open Notebook:** 1. Go to **Settings** → **API Keys** 2. Click **Add Credential** 3. Select provider: **Groq** 4. Give it a name, paste your API key 5. Click **Save**, then **Test Connection** 6. Click **Discover Models** → **Register Models** **Available Models:** - `llama-3.3-70b-versatile` — Best on Groq (recommended) - `llama-3.1-70b-versatile` — Fast, capable - `mixtral-8x7b-32768` — Good alternative - `gemma2-9b-it` — Small, very fast **Recommended:** - For quality: `llama-3.3-70b-versatile` (best overall) - For speed: `gemma2-9b-it` (ultra-fast) - For balance: `llama-3.1-70b-versatile` **Advantages:** - Ultra-fast inference - Very cheap - Great for transformations/batch work **Disadvantages:** - Limited model selection - Smaller models than OpenAI/Anthropic **Troubleshooting:** - "Rate limited" → Free tier has limits, upgrade - "Model not available" → Re-discover models from the credential --- ### OpenRouter **Cost:** Varies by model ($0.05-15 per 1M tokens) **Get Your API Key:** 1. Go to https://openrouter.ai/keys 2. Create account or login 3. Add credits to your account 4. Create new API key **Configure in Open Notebook:** 1. Go to **Settings** → **API Keys** 2. Click **Add Credential** 3. Select provider: **OpenRouter** 4. Give it a name, paste your API key 5. Click **Save**, then **Test Connection** 6. Click **Discover Models** → **Register Models** **Available Models (100+ options):** - OpenAI: `openai/gpt-4o`, `openai/o1` - Anthropic: `anthropic/claude-sonnet-4.5`, `anthropic/claude-3.5-haiku` - Google: `google/gemini-2.0-flash-exp`, `google/gemini-1.5-pro` - Meta: `meta-llama/llama-3.3-70b-instruct`, `meta-llama/llama-3.1-405b-instruct` - Mistral: `mistralai/mistral-large-2411` - DeepSeek: `deepseek/deepseek-chat` - And many more... **Recommended:** - For quality: `anthropic/claude-sonnet-4.5` (best overall) - For speed/cost: `google/gemini-2.0-flash-exp` (very fast, cheap) - For open-source: `meta-llama/llama-3.3-70b-instruct` - For reasoning: `openai/o1` **Advantages:** - One API key for 100+ models - Unified billing - Easy model comparison - Access to models that may have waitlists elsewhere **Cost Estimate:** ``` Light use: $1-5/month Medium use: $10-30/month Heavy use: Depends on models chosen ``` **Troubleshooting:** - "Invalid API key" → Check it starts with "sk-or-" - "Insufficient credits" → Add credits at openrouter.ai - "Model not available" → Check model ID spelling (use full path) --- ## Self-Hosted / Local ### Ollama (Recommended for Local) **Cost:** Free (electricity only) **Setup Ollama:** 1. Install Ollama: https://ollama.ai 2. Run Ollama in background: `ollama serve` 3. Download a model: `ollama pull mistral` **Configure in Open Notebook:** 1. Go to **Settings** → **API Keys** 2. Click **Add Credential** 3. Select provider: **Ollama** 4. Give it a name (e.g., "Local Ollama") 5. Enter the base URL: - Same machine (non-Docker): `http://localhost:11434` - Docker with Ollama on host: `http://host.docker.internal:11434` - Docker with Ollama container: `http://ollama:11434` 6. Click **Save**, then **Test Connection** 7. Click **Discover Models** → **Register Models** See [Ollama Setup Guide](ollama.md) for detailed network configuration. **Available Models:** - `llama3.3:70b` — Best quality (requires 40GB+ RAM) - `llama3.1:8b` — Recommended, balanced (8GB RAM) - `qwen2.5:7b` — Excellent for code and reasoning - `mistral:7b` — Good general purpose - `phi3:3.8b` — Small, fast (4GB RAM) - `gemma2:9b` — Google's model, balanced - Many more: `ollama list` to see available **Recommended:** - For quality (with GPU): `llama3.3:70b` (best) - For general use: `llama3.1:8b` (best balance) - For speed/low memory: `phi3:3.8b` (very fast) - For coding: `qwen2.5:7b` (excellent at code) **Hardware Requirements:** ``` GPU (NVIDIA/AMD): 8GB VRAM: Runs most models fine 6GB VRAM: Works, slower 4GB VRAM: Small models only CPU-only: 16GB+ RAM: Slow but works 8GB RAM: Very slow 4GB RAM: Not recommended ``` **Advantages:** - Completely private (runs locally) - Free (electricity only) - No API key needed - Works offline **Disadvantages:** - Slower than cloud (unless on GPU) - Smaller models than cloud - Requires local hardware **Troubleshooting:** - "Connection refused" → Ollama not running or wrong URL in credential - "Model not found" → Download it: `ollama pull modelname` - "Out of memory" → Use smaller model or add more RAM --- ### LM Studio (Local Alternative) **Cost:** Free **Setup LM Studio:** 1. Download LM Studio: https://lmstudio.ai 2. Open app 3. Download a model from library 4. Go to "Local Server" tab 5. Start server (default port: 1234) **Configure in Open Notebook:** 1. Go to **Settings** → **API Keys** 2. Click **Add Credential** 3. Select provider: **OpenAI-Compatible** 4. Give it a name (e.g., "LM Studio") 5. Enter the base URL: `http://host.docker.internal:1234/v1` (Docker) or `http://localhost:1234/v1` (local) 6. API key: `lm-studio` (placeholder, LM Studio doesn't require one) 7. Click **Save**, then **Test Connection** **Advantages:** - GUI interface (easier than Ollama CLI) - Good model selection - Privacy-focused - Works offline **Disadvantages:** - Desktop only (Mac/Windows/Linux) - Slower than cloud - Requires local GPU --- ### Custom OpenAI-Compatible For Text Generation UI, vLLM, or other OpenAI-compatible endpoints: 1. Go to **Settings** → **API Keys** 2. Click **Add Credential** 3. Select provider: **OpenAI-Compatible** 4. Enter the base URL for your endpoint (e.g., `http://localhost:8000/v1`) 5. Enter API key if required 6. Optionally configure per-service URLs (LLM, Embedding, TTS, STT) 7. Click **Save**, then **Test Connection** See [OpenAI-Compatible Setup](openai-compatible.md) for detailed instructions. --- ## Enterprise ### Azure OpenAI **Cost:** Same as OpenAI (usage-based) **Configure in Open Notebook:** 1. Create Azure OpenAI service in Azure portal 2. Deploy GPT-4/3.5-turbo model 3. Get your endpoint and key 4. Go to **Settings** → **API Keys** 5. Click **Add Credential** 6. Select provider: **Azure OpenAI** 7. Fill in: API Key, Endpoint, API Version 8. Optionally configure service-specific endpoints (LLM, Embedding) 9. Click **Save**, then **Test Connection** **Advantages:** - Enterprise support - VPC integration - Compliance (HIPAA, SOC2, etc.) **Disadvantages:** - More complex setup - Higher overhead - Requires Azure account --- ## Embeddings (For Search/Semantic Features) By default, Open Notebook uses the LLM provider's embeddings. Embedding models are discovered and registered through the same credential system — when you discover models from a credential, embedding models are included alongside language models. --- ## Choosing Your Provider **1. Don't want to run locally and don't want to mess around with different providers:** Use OpenAI - Cloud-based - Good quality - Reasonable cost - Simplest setup, supports all modes (text, embedding, tts, stt, etc) **For budget-conscious:** Groq, OpenRouter or Ollama - Groq: Super cheap cloud - Ollama: Free, but local - OpenRouter: many open source models very accessible **For privacy-first:** Ollama or LM Studio and Speaches ([TTS](local-tts.md), [STT](local-stt.md)) - Everything stays local - Works offline - No API keys sent anywhere **For enterprise:** Azure OpenAI - Compliance - VPC integration - Support --- ## Next Steps 1. **Choose your provider** from above 2. **Get API key** (if cloud) or install locally (if Ollama) 3. **Set `OPEN_NOTEBOOK_ENCRYPTION_KEY`** in your docker-compose.yml (required for storing credentials) 4. **Open Settings** → **API Keys** → **Add Credential** 5. **Test Connection** to verify it works 6. **Discover & Register Models** to make them available 7. **Verify it works** with a test chat > **Multiple providers**: You can add credentials for as many providers as you want. Create separate credentials for different projects or team members. Done! --- ## Legacy: Environment Variables (Deprecated) > **Deprecated**: Configuring AI provider API keys via environment variables is deprecated. Use the Settings UI instead. Environment variables may still work as a fallback but are no longer the recommended approach. If you are migrating from an older version that used environment variables, go to **Settings** → **API Keys** and click the **Migrate to Database** button to import your existing keys into the credential system. --- ## Related - **[API Configuration](../3-USER-GUIDE/api-configuration.md)** — Detailed credential management guide - **[Environment Reference](environment-reference.md)** - Complete list of all environment variables - **[Advanced Configuration](advanced.md)** - Timeouts, SSL, performance tuning - **[Ollama Setup](ollama.md)** - Detailed Ollama configuration guide - **[OpenAI-Compatible](openai-compatible.md)** - LM Studio and other compatible providers - **[Local TTS Setup](local-tts.md)** - Text-to-speech with Speaches - **[Local STT Setup](local-stt.md)** - Speech-to-text with Speaches - **[Troubleshooting](../6-TROUBLESHOOTING/quick-fixes.md)** - Common issues and fixes ================================================ FILE: docs/5-CONFIGURATION/database.md ================================================ # Database - SurrealDB Configuration Open Notebook uses SurrealDB for its database needs. --- ## Default Configuration Open Notebook should work out of the box with SurrealDB as long as the environment variables are correctly setup. ### DB running in the same docker compose as Open Notebook (recommended) The example above is for when you are running SurrealDB as a separate docker container, which is the method described [here](../1-INSTALLATION/docker-compose.md) (and our recommended method). ```env SURREAL_URL="ws://surrealdb:8000/rpc" SURREAL_USER="root" SURREAL_PASSWORD="root" SURREAL_NAMESPACE="open_notebook" SURREAL_DATABASE="open_notebook" ``` ### DB running in the host machine and Open Notebook running in Docker If ON is running in docker and SurrealDB is on your host machine, you need to point to it. ```env SURREAL_URL="ws://your-machine-ip:8000/rpc" #or host.docker.internal SURREAL_USER="root" SURREAL_PASSWORD="root" SURREAL_NAMESPACE="open_notebook" SURREAL_DATABASE="open_notebook" ``` ### Open Notebook and Surreal are running on the same machine If you are running both services locally or if you are using the deprecated [single container setup](../1-INSTALLATION/single-container.md) ```env SURREAL_URL="ws://localhost:8000/rpc" SURREAL_USER="root" SURREAL_PASSWORD="root" SURREAL_NAMESPACE="open_notebook" SURREAL_DATABASE="open_notebook" ``` ## Multiple databases You can have multiple namespaces in one SurrealDB instance and you can also have multiple databases in one instance. So, if you want to setup multiple open noteobok deployments for different users, you don't need to deploy multiple databases. ================================================ FILE: docs/5-CONFIGURATION/environment-reference.md ================================================ # Complete Environment Reference Comprehensive list of all environment variables available in Open Notebook. --- ## API Configuration | Variable | Required? | Default | Description | |----------|-----------|---------|-------------| | `API_URL` | No | Auto-detected | URL where frontend reaches API (e.g., http://localhost:5055) | | `INTERNAL_API_URL` | No | http://localhost:5055 | Internal API URL for Next.js server-side proxying | | `API_CLIENT_TIMEOUT` | No | 300 | Client timeout in seconds (how long to wait for API response) | | `OPEN_NOTEBOOK_PASSWORD` | No | None | Password to protect Open Notebook instance | | `OPEN_NOTEBOOK_ENCRYPTION_KEY` | **Yes** | None | Secret string to encrypt credentials stored in database (any string works). **Required** for the credential system. Supports Docker secrets via `_FILE` suffix. | | `HOSTNAME` | No | `0.0.0.0` (in Docker) | Network interface for Next.js to bind to. Default `0.0.0.0` ensures accessibility from reverse proxies | > **Important**: `OPEN_NOTEBOOK_ENCRYPTION_KEY` is required for storing AI provider credentials via the Settings UI. Without it, you cannot save credentials. If you change or lose this key, all stored credentials become unreadable. --- ## Database: SurrealDB | Variable | Required? | Default | Description | |----------|-----------|---------|-------------| | `SURREAL_URL` | Yes | ws://surrealdb:8000/rpc | SurrealDB WebSocket connection URL | | `SURREAL_USER` | Yes | root | SurrealDB username | | `SURREAL_PASSWORD` | Yes | root | SurrealDB password | | `SURREAL_NAMESPACE` | Yes | open_notebook | SurrealDB namespace | | `SURREAL_DATABASE` | Yes | open_notebook | SurrealDB database name | --- ## Database: Retry Configuration | Variable | Required? | Default | Description | |----------|-----------|---------|-------------| | `SURREAL_COMMANDS_RETRY_ENABLED` | No | true | Enable retries on failure | | `SURREAL_COMMANDS_RETRY_MAX_ATTEMPTS` | No | 3 | Maximum retry attempts | | `SURREAL_COMMANDS_RETRY_WAIT_STRATEGY` | No | exponential_jitter | Retry wait strategy (exponential_jitter/exponential/fixed/random) | | `SURREAL_COMMANDS_RETRY_WAIT_MIN` | No | 1 | Minimum wait time between retries (seconds) | | `SURREAL_COMMANDS_RETRY_WAIT_MAX` | No | 30 | Maximum wait time between retries (seconds) | --- ## Database: Concurrency | Variable | Required? | Default | Description | |----------|-----------|---------|-------------| | `SURREAL_COMMANDS_MAX_TASKS` | No | 5 | Maximum concurrent database tasks | --- ## LLM Timeouts | Variable | Required? | Default | Description | |----------|-----------|---------|-------------| | `ESPERANTO_LLM_TIMEOUT` | No | 60 | LLM inference timeout in seconds | | `ESPERANTO_SSL_VERIFY` | No | true | Verify SSL certificates (false = development only) | | `ESPERANTO_SSL_CA_BUNDLE` | No | None | Path to custom CA certificate bundle | --- ## Text-to-Speech (TTS) | Variable | Required? | Default | Description | |----------|-----------|---------|-------------| | `TTS_BATCH_SIZE` | No | 5 | Concurrent TTS requests (1-5, depends on provider) | --- ## Content Extraction | Variable | Required? | Default | Description | |----------|-----------|---------|-------------| | `FIRECRAWL_API_KEY` | No | None | Firecrawl API key for advanced web scraping | | `JINA_API_KEY` | No | None | Jina AI API key for web extraction | **Setup:** - Firecrawl: https://firecrawl.dev/ - Jina: https://jina.ai/ --- ## Network / Proxy | Variable | Required? | Default | Description | |----------|-----------|---------|-------------| | `HTTP_PROXY` | No | None | HTTP proxy URL for outbound HTTP requests | | `HTTPS_PROXY` | No | None | HTTPS proxy URL for outbound HTTPS requests | | `NO_PROXY` | No | None | Comma-separated list of hosts to bypass proxy | Route all outbound HTTP requests through a proxy server. Useful for corporate/firewalled environments. The underlying libraries (esperanto, content-core, podcast-creator) automatically detect proxy settings from these standard environment variables. **Affects:** - AI provider API calls (OpenAI, Anthropic, Google, Groq, etc.) - Content extraction from URLs (web scraping, YouTube transcripts) - Podcast generation (LLM and TTS provider calls) **Format:** `http://[user:pass@]host:port` or `https://[user:pass@]host:port` **Examples:** ```bash # Basic proxy HTTP_PROXY=http://proxy.corp.com:8080 HTTPS_PROXY=http://proxy.corp.com:8080 # Authenticated proxy HTTP_PROXY=http://user:password@proxy.corp.com:8080 HTTPS_PROXY=http://user:password@proxy.corp.com:8080 # Bypass proxy for local hosts NO_PROXY=localhost,127.0.0.1,.local ``` --- ## Debugging & Monitoring | Variable | Required? | Default | Description | |----------|-----------|---------|-------------| | `LANGCHAIN_TRACING_V2` | No | false | Enable LangSmith tracing | | `LANGCHAIN_ENDPOINT` | No | https://api.smith.langchain.com | LangSmith endpoint | | `LANGCHAIN_API_KEY` | No | None | LangSmith API key | | `LANGCHAIN_PROJECT` | No | Open Notebook | LangSmith project name | **Setup:** https://smith.langchain.com/ --- ## Environment Variables by Use Case ### Minimal Setup (New Installation) ``` OPEN_NOTEBOOK_ENCRYPTION_KEY=my-secret-key SURREAL_URL=ws://surrealdb:8000/rpc SURREAL_USER=root SURREAL_PASSWORD=password SURREAL_NAMESPACE=open_notebook SURREAL_DATABASE=open_notebook ``` Then configure AI providers via **Settings → API Keys** in the browser. ### Production Deployment ``` OPEN_NOTEBOOK_ENCRYPTION_KEY=your-strong-secret-key OPEN_NOTEBOOK_PASSWORD=your-secure-password API_URL=https://mynotebook.example.com SURREAL_USER=production_user SURREAL_PASSWORD=secure_password ``` ### Self-Hosted Behind Reverse Proxy ``` OPEN_NOTEBOOK_ENCRYPTION_KEY=your-secret-key API_URL=https://mynotebook.example.com ``` ### Corporate Environment (Behind Proxy) ``` OPEN_NOTEBOOK_ENCRYPTION_KEY=your-secret-key HTTP_PROXY=http://proxy.corp.com:8080 HTTPS_PROXY=http://proxy.corp.com:8080 NO_PROXY=localhost,127.0.0.1 ``` ### High-Performance Deployment ``` OPEN_NOTEBOOK_ENCRYPTION_KEY=your-secret-key SURREAL_COMMANDS_MAX_TASKS=10 TTS_BATCH_SIZE=5 API_CLIENT_TIMEOUT=600 ``` ### Debugging ``` OPEN_NOTEBOOK_ENCRYPTION_KEY=your-secret-key LANGCHAIN_TRACING_V2=true LANGCHAIN_API_KEY=your-key ``` --- ## Validation Check if a variable is set: ```bash # Check single variable echo $OPEN_NOTEBOOK_ENCRYPTION_KEY # Check multiple env | grep -E "OPEN_NOTEBOOK|API_URL" # Print all config env | grep -E "^[A-Z_]+=" | sort ``` --- ## Notes - **Case-sensitive:** `OPEN_NOTEBOOK_ENCRYPTION_KEY` ≠ `open_notebook_encryption_key` - **No spaces:** `OPEN_NOTEBOOK_ENCRYPTION_KEY=my-key` not `OPEN_NOTEBOOK_ENCRYPTION_KEY = my-key` - **Quote values:** Use quotes for values with spaces: `API_URL="http://my server:5055"` - **Restart required:** Changes take effect after restarting services - **Secrets:** Don't commit encryption keys or passwords to git - **AI Providers:** Configure via **Settings → API Keys** in the browser (not via env vars) - **Migration:** Use Settings UI to migrate existing env vars to the credential system. See [API Configuration](../3-USER-GUIDE/api-configuration.md#migrating-from-environment-variables) --- ## Quick Setup Checklist - [ ] Set `OPEN_NOTEBOOK_ENCRYPTION_KEY` in docker-compose.yml - [ ] Set database credentials (`SURREAL_*`) - [ ] Start services - [ ] Open browser → Go to **Settings → API Keys** - [ ] **Add Credential** for your AI provider - [ ] **Test Connection** to verify - [ ] **Discover & Register Models** - [ ] Set `API_URL` if behind reverse proxy - [ ] Change `SURREAL_PASSWORD` in production - [ ] Try a test chat Done! --- ## Legacy: AI Provider Environment Variables (Deprecated) > **Deprecated**: The following AI provider API key environment variables are deprecated. Configure providers via the Settings UI instead. These variables may still work as a fallback but are no longer recommended. If you have these variables configured from a previous installation, click the **Migrate to Database** button in **Settings → API Keys** to import them into the credential system, then remove them from your configuration. | Variable | Provider | Replacement | |----------|----------|-------------| | `OPENAI_API_KEY` | OpenAI | Settings → API Keys → Add OpenAI Credential | | `ANTHROPIC_API_KEY` | Anthropic | Settings → API Keys → Add Anthropic Credential | | `GOOGLE_API_KEY` | Google Gemini | Settings → API Keys → Add Google Credential | | `GEMINI_API_BASE_URL` | Google Gemini | Configure in Google Gemini credential | | `VERTEX_PROJECT` | Vertex AI | Settings → API Keys → Add Vertex AI Credential | | `VERTEX_LOCATION` | Vertex AI | Configure in Vertex AI credential | | `GOOGLE_APPLICATION_CREDENTIALS` | Vertex AI | Configure in Vertex AI credential | | `GROQ_API_KEY` | Groq | Settings → API Keys → Add Groq Credential | | `MISTRAL_API_KEY` | Mistral | Settings → API Keys → Add Mistral Credential | | `DEEPSEEK_API_KEY` | DeepSeek | Settings → API Keys → Add DeepSeek Credential | | `XAI_API_KEY` | xAI | Settings → API Keys → Add xAI Credential | | `OLLAMA_API_BASE` | Ollama | Settings → API Keys → Add Ollama Credential | | `OPENROUTER_API_KEY` | OpenRouter | Settings → API Keys → Add OpenRouter Credential | | `OPENROUTER_BASE_URL` | OpenRouter | Configure in OpenRouter credential | | `VOYAGE_API_KEY` | Voyage AI | Settings → API Keys → Add Voyage AI Credential | | `ELEVENLABS_API_KEY` | ElevenLabs | Settings → API Keys → Add ElevenLabs Credential | | `OPENAI_COMPATIBLE_BASE_URL` | OpenAI-Compatible | Settings → API Keys → Add OpenAI-Compatible Credential | | `OPENAI_COMPATIBLE_API_KEY` | OpenAI-Compatible | Configure in OpenAI-Compatible credential | | `OPENAI_COMPATIBLE_BASE_URL_LLM` | OpenAI-Compatible | Configure per-service URL in credential | | `OPENAI_COMPATIBLE_API_KEY_LLM` | OpenAI-Compatible | Configure per-service key in credential | | `OPENAI_COMPATIBLE_BASE_URL_EMBEDDING` | OpenAI-Compatible | Configure per-service URL in credential | | `OPENAI_COMPATIBLE_API_KEY_EMBEDDING` | OpenAI-Compatible | Configure per-service key in credential | | `OPENAI_COMPATIBLE_BASE_URL_STT` | OpenAI-Compatible | Configure per-service URL in credential | | `OPENAI_COMPATIBLE_API_KEY_STT` | OpenAI-Compatible | Configure per-service key in credential | | `OPENAI_COMPATIBLE_BASE_URL_TTS` | OpenAI-Compatible | Configure per-service URL in credential | | `OPENAI_COMPATIBLE_API_KEY_TTS` | OpenAI-Compatible | Configure per-service key in credential | | `AZURE_OPENAI_API_KEY` | Azure OpenAI | Settings → API Keys → Add Azure OpenAI Credential | | `AZURE_OPENAI_ENDPOINT` | Azure OpenAI | Configure in Azure OpenAI credential | | `AZURE_OPENAI_API_VERSION` | Azure OpenAI | Configure in Azure OpenAI credential | | `AZURE_OPENAI_API_KEY_LLM` | Azure OpenAI | Configure per-service in credential | | `AZURE_OPENAI_ENDPOINT_LLM` | Azure OpenAI | Configure per-service in credential | | `AZURE_OPENAI_API_VERSION_LLM` | Azure OpenAI | Configure per-service in credential | | `AZURE_OPENAI_API_KEY_EMBEDDING` | Azure OpenAI | Configure per-service in credential | | `AZURE_OPENAI_ENDPOINT_EMBEDDING` | Azure OpenAI | Configure per-service in credential | | `AZURE_OPENAI_API_VERSION_EMBEDDING` | Azure OpenAI | Configure per-service in credential | ================================================ FILE: docs/5-CONFIGURATION/index.md ================================================ # Configuration - Essential Settings Configuration is how you customize Open Notebook for your specific setup. This section covers what you need to know. --- ## What Needs Configuration? Three things: 1. **AI Provider** — Which LLM/embedding service you're using (OpenAI, Anthropic, Ollama, etc.) 2. **Database** — How to connect to SurrealDB (usually pre-configured) 3. **Server** — API URL, ports, timeouts (usually auto-detected) --- ## Quick Decision: Which Provider? ### Option 1: Cloud Provider (Fastest) - **OpenRouter (recommended)** (access to all models with one key) - **OpenAI** (GPT) - **Anthropic** (Claude) - **Google Gemini** (multi-modal, long context) - **Groq** (ultra-fast inference) Setup: Get API key → Add credential in Settings UI → Done → Go to **[AI Providers Guide](ai-providers.md)** ### Option 2: Local (Free & Private) - **Ollama** (open-source models, on your machine) → Go to **[Ollama Setup](ollama.md)** ### Option 3: OpenAI-Compatible - **LM Studio** (local) - **Custom endpoints** → Go to **[OpenAI-Compatible Guide](openai-compatible.md)** --- ## Configuration File Use the right file depending on your setup. ### `.env` (Local Development) You will only use .env if you are running Open Notebook locally. ``` Located in: project root Use for: Development on your machine Format: KEY=value, one per line ``` ### `docker.env` (Docker Deployment) You will use this file to hold your environment variables if you are using docker-compose and prefer not to put the variables directly in the compose file. ``` Located in: project root (or ./docker) Use for: Docker deployments Format: Same as .env Loaded by: docker-compose.yml ``` --- ## Most Important Settings All of the settings provided below are to be placed inside your environment file (.env or docker.env depending on your setup). ### Surreal Database This is the database used by the app. ``` SURREAL_URL=ws://surrealdb:8000/rpc SURREAL_USER=root SURREAL_PASSWORD=root # Change in production! SURREAL_NAMESPACE=open_notebook SURREAL_DATABASE=open_notebook ``` > The only thing that is critical to not miss is the hostname in the `SURREAL_URL`. Check what URL to use based on your deployment, [here](database.md). ### AI Provider (Credentials) We need access to LLMs in order for the app to work. AI provider credentials are configured via the **Settings UI**: 1. Set `OPEN_NOTEBOOK_ENCRYPTION_KEY` in your environment (required for storing credentials) 2. Start services 3. Go to **Settings → API Keys → Add Credential** 4. Select your provider, paste your API key 5. **Test Connection** → **Discover Models** → **Register Models** ``` # Required in your .env or docker-compose.yml: OPEN_NOTEBOOK_ENCRYPTION_KEY=my-secret-key ``` > **Ollama users**: Add an Ollama credential in Settings → API Keys with the correct base URL. See [Ollama Setup](ollama.md) for network configuration help. > **LM Studio / OpenAI-Compatible**: Add an OpenAI-Compatible credential in Settings → API Keys. See [OpenAI-Compatible Guide](openai-compatible.md). ### API URL (If Behind Reverse Proxy) You only need to worry about this if you are deploying on a proxy or if you are changing port information. Otherwise, skip this. ``` API_URL=https://your-domain.com # Usually auto-detected. Only set if needed. ``` Auto-detection works for most setups. --- ## Configuration by Scenario ### Scenario 1: Docker on Localhost (Default) ```env # In docker.env: OPEN_NOTEBOOK_ENCRYPTION_KEY=my-secret-key # Everything else uses defaults # Then configure AI provider in Settings → API Keys ``` ### Scenario 2: Docker on Remote Server ```env # In docker.env: OPEN_NOTEBOOK_ENCRYPTION_KEY=my-secret-key API_URL=http://your-server-ip:5055 ``` ### Scenario 3: Behind Reverse Proxy (Nginx/Cloudflare) ```env # In docker.env: OPEN_NOTEBOOK_ENCRYPTION_KEY=my-secret-key API_URL=https://your-domain.com # The reverse proxy handles HTTPS ``` ### Scenario 4: Using Ollama Locally ```env # In .env: OPEN_NOTEBOOK_ENCRYPTION_KEY=my-secret-key # Then add Ollama credential in Settings → API Keys ``` ### Scenario 5: Using Azure OpenAI ```env # In docker.env: OPEN_NOTEBOOK_ENCRYPTION_KEY=my-secret-key # Then add Azure OpenAI credential in Settings → API Keys ``` --- ## Configuration Sections ### [AI Providers](ai-providers.md) - OpenAI configuration - Anthropic configuration - Google Gemini configuration - Groq configuration - Ollama configuration - Azure OpenAI configuration - OpenAI-compatible configuration ### [Database](database.md) - SurrealDB setup - Connection strings - Database vs. namespace - Running your own SurrealDB ### [Advanced](advanced.md) - Ports and networking - Timeouts and concurrency - SSL/security - Retry configuration - Worker concurrency - Language models & embeddings - Speech-to-text & text-to-speech - Debugging and logging ### [Reverse Proxy](reverse-proxy.md) - Nginx, Caddy, Traefik configs - Custom domain setup - SSL/HTTPS configuration - Coolify and other platforms ### [Security](security.md) - Password protection - API authentication - Production hardening - Firewall configuration ### [Local TTS](local-tts.md) - Speaches setup for local text-to-speech - GPU acceleration - Voice options - Docker networking ### [Local STT](local-stt.md) - Speaches setup for local speech-to-text - Whisper model options - GPU acceleration - Docker networking ### [Ollama](ollama.md) - Setting up and pointing to an Ollama server - Downloading models - Using embedding ### [OpenAI-Compatible Providers](openai-compatible.md) - LM Studio, vLLM, Text Generation WebUI - Connection configuration - Docker networking - Troubleshooting ### [Complete Reference](environment-reference.md) - All environment variables - Grouped by category - What each one does - Default values --- ## How to Add Configuration ### Method 1: Settings UI (For AI Provider Credentials) The recommended way to configure AI providers: ``` 1. Open Open Notebook in your browser 2. Go to Settings → API Keys 3. Click "Add Credential" 4. Select provider, enter API key 5. Click Save, then Test Connection 6. Click Discover Models → Register Models ``` No file editing, no restarts. Credentials stored securely (encrypted) in database. → **[Full Guide: API Configuration](../3-USER-GUIDE/api-configuration.md)** ### Method 2: Edit `.env` File (Infrastructure Settings) For database, network, and encryption key settings: ```bash 1. Open .env in your editor 2. Set OPEN_NOTEBOOK_ENCRYPTION_KEY and database vars 3. Save 4. Restart services ``` ### Method 3: Set Docker Environment (Deployment) ```bash # In docker-compose.yml: services: api: environment: - OPEN_NOTEBOOK_ENCRYPTION_KEY=my-secret-key - API_URL=https://your-domain.com ``` --- ## Verification After configuration, verify it works: ``` 1. Open your notebook 2. Go to Settings → Models 3. You should see your configured provider 4. Try a simple Chat question 5. If it responds, configuration is correct! ``` --- ## Common Mistakes | Mistake | Problem | Fix | |---------|---------|-----| | No credential configured | Models not available | Add credential in Settings → API Keys | | Missing encryption key | Can't save credentials | Set OPEN_NOTEBOOK_ENCRYPTION_KEY | | Wrong database URL | Can't start API | Check SURREAL_URL format | | Expose port 5055 | "Can't connect to server" | Expose 5055 in docker-compose | | Typo in env var | Settings ignored | Check spelling (case-sensitive!) | | Don't restart | Old config still used | Restart services after env changes | --- ## What Comes After Configuration Once configured: 1. **[Quick Start](../0-START-HERE/index.md)** — Run your first notebook 2. **[Installation](../1-INSTALLATION/index.md)** — Multi-route deployment guides 3. **[User Guide](../3-USER-GUIDE/index.md)** — How to use each feature --- ## Getting Help - **Configuration error?** → Check [Troubleshooting](../6-TROUBLESHOOTING/quick-fixes.md) - **Provider-specific issue?** → Check [AI Providers](ai-providers.md) - **Need complete reference?** → See [Environment Reference](environment-reference.md) --- ## Summary **Minimal configuration to run:** 1. Set `OPEN_NOTEBOOK_ENCRYPTION_KEY` in your environment 2. Start services 3. Add AI provider credential in Settings → API Keys 4. Done! Everything else is optional optimization. ================================================ FILE: docs/5-CONFIGURATION/local-stt.md ================================================ # Local Speech-to-Text Setup Run speech-to-text locally for free, private audio/video transcription using OpenAI-compatible STT servers. --- ## Why Local STT? | Benefit | Description | |---------|-------------| | **Free** | No per-minute costs after setup | | **Private** | Audio never leaves your machine | | **Unlimited** | No rate limits or quotas | | **Offline** | Works without internet | --- ## Quick Start with Speaches [Speaches](https://github.com/speaches-ai/speaches) is an open-source, OpenAI-compatible server that supports both TTS and STT. It uses [faster-whisper](https://github.com/SYSTRAN/faster-whisper) for transcription. > **💡 Ready-made Docker Compose files available:** > - **[docker-compose-speaches.yml](../../examples/docker-compose-speaches.yml)** - Speaches + Open Notebook > - **[docker-compose-full-local.yml](../../examples/docker-compose-full-local.yml)** - Speaches + Ollama (100% local setup) > > These include complete setup instructions and configuration examples. Just copy and run! ### Step 1: Create Docker Compose File Create a folder and add `docker-compose.yml`: ```yaml services: speaches: image: ghcr.io/speaches-ai/speaches:latest-cpu container_name: speaches ports: - "8969:8000" volumes: - hf-hub-cache:/home/ubuntu/.cache/huggingface/hub restart: unless-stopped volumes: hf-hub-cache: ``` ### Step 2: Start and Download Model ```bash # Start Speaches docker compose up -d # Wait for startup sleep 10 # Download Whisper model (~500MB for small) docker compose exec speaches uv tool run speaches-cli model download Systran/faster-whisper-small ``` Models can also be downloaded automatically on first use, but pre-downloading avoids delays. ### Step 3: Test ```bash # Create a test audio file (or use your own) # Then transcribe it: curl "http://localhost:8969/v1/audio/transcriptions" \ -F "file=@test.mp3" \ -F "model=Systran/faster-whisper-small" ``` You should see the transcribed text in the response. ### Step 4: Configure Open Notebook **Via Settings UI (Recommended):** 1. Go to **Settings** → **API Keys** 2. Click **Add Credential** → Select **OpenAI-Compatible** 3. Enter base URL for STT: `http://host.docker.internal:8969/v1` (Docker) or `http://localhost:8969/v1` (local) 4. Click **Save**, then **Test Connection** **Legacy (Deprecated) — Environment variables:** ```yaml # In your Open Notebook docker-compose.yml environment: - OPENAI_COMPATIBLE_BASE_URL_STT=http://host.docker.internal:8969/v1 ``` ```bash # Local development export OPENAI_COMPATIBLE_BASE_URL_STT=http://localhost:8969/v1 ``` ### Step 5: Add Model in Open Notebook 1. Go to **Settings** → **Models** 2. Click **Add Model** in Speech-to-Text section 3. Configure: - **Provider**: `openai_compatible` - **Model Name**: `Systran/faster-whisper-small` - **Display Name**: `Local Whisper` 4. Click **Save** 5. Set as default if desired --- ## Available Models Speaches supports various Whisper model sizes. Larger models are more accurate but slower: | Model | Size | Speed | Accuracy | VRAM (GPU) | |-------|------|-------|----------|------------| | `Systran/faster-whisper-tiny` | ~75 MB | Fastest | Basic | ~1 GB | | `Systran/faster-whisper-base` | ~150 MB | Fast | Good | ~1 GB | | `Systran/faster-whisper-small` | ~500 MB | Medium | Better | ~2 GB | | `Systran/faster-whisper-medium` | ~1.5 GB | Slow | Great | ~5 GB | | `Systran/faster-whisper-large-v3` | ~3 GB | Slowest | Best | ~10 GB | | `Systran/faster-distil-whisper-small.en` | ~400 MB | Fast | Good (English only) | ~2 GB | ### List Available Models ```bash docker compose exec speaches uv tool run speaches-cli registry ls --task automatic-speech-recognition ``` ### Recommended Models - **For speed**: `Systran/faster-whisper-tiny` or `Systran/faster-whisper-base` - **For balance**: `Systran/faster-whisper-small` (recommended) - **For accuracy**: `Systran/faster-whisper-large-v3` --- ## GPU Acceleration For faster transcription with NVIDIA GPUs: ```yaml services: speaches: image: ghcr.io/speaches-ai/speaches:latest-cuda container_name: speaches ports: - "8969:8000" volumes: - hf-hub-cache:/home/ubuntu/.cache/huggingface/hub environment: - WHISPER__TTL=-1 # Keep model in VRAM (recommended if you have enough memory) restart: unless-stopped deploy: resources: reservations: devices: - driver: nvidia count: 1 capabilities: [gpu] volumes: hf-hub-cache: ``` ### Keep Model in Memory By default, Speaches unloads models after some time. To keep the Whisper model loaded for instant transcription: ```yaml environment: - WHISPER__TTL=-1 # Never unload ``` This is recommended if you have enough RAM/VRAM, as loading the model can take a few seconds. --- ## Docker Networking When configuring your OpenAI-Compatible credential in **Settings → API Keys**, use the appropriate STT base URL for your setup: ### Open Notebook in Docker (macOS/Windows) **STT Base URL:** `http://host.docker.internal:8969/v1` ### Open Notebook in Docker (Linux) **STT Base URL (Option 1 — Docker bridge IP):** `http://172.17.0.1:8969/v1` **Option 2:** Use host networking mode (`docker run --network host ...`), then use: `http://localhost:8969/v1` ### Remote Server Run Speaches on a different machine: **STT Base URL:** `http://server-ip:8969/v1` (replace with your server's IP) --- ## Language Support Whisper supports 99+ languages. Specify the language for better accuracy: ```bash curl "http://localhost:8969/v1/audio/transcriptions" \ -F "file=@audio.mp3" \ -F "model=Systran/faster-whisper-small" \ -F "language=ru" ``` Common language codes: - `en` - English - `ru` - Russian - `es` - Spanish - `fr` - French - `de` - German - `zh` - Chinese - `ja` - Japanese --- ## Troubleshooting ### Service Won't Start ```bash # Check logs docker compose logs speaches # Verify port available lsof -i :8969 # Restart docker compose down && docker compose up -d ``` ### Connection Refused ```bash # Test Speaches is running curl http://localhost:8969/v1/models # From inside Open Notebook container docker exec -it open-notebook curl http://host.docker.internal:8969/v1/models ``` ### Model Download Fails Models are downloaded automatically on first use. If download fails: ```bash # Check available disk space df -h # Check Docker logs for errors docker compose logs speaches # Restart and try again docker compose restart speaches ``` ### Poor Transcription Quality - Use a larger model (`faster-whisper-medium` or `large-v3`) - Specify the correct language - Ensure audio quality is good (clear speech, minimal background noise) - Try different audio formats (WAV often works better than MP3) ### Slow Transcription | Solution | How | |----------|-----| | Use GPU | Switch to `latest-cuda` image | | Smaller model | Use `faster-whisper-tiny` or `base` | | More CPU | Allocate more cores in Docker | | SSD storage | Move Docker volumes to SSD | --- ## Performance Tips ### Recommended Specs | Component | Minimum | Recommended | |-----------|---------|-------------| | CPU | 2 cores | 4+ cores | | RAM | 2 GB | 8+ GB | | Storage | 5 GB | 10 GB (for multiple models) | | GPU | None | NVIDIA (optional, much faster) | ### Resource Limits ```yaml services: speaches: # ... other config mem_limit: 4g cpus: 2 ``` ### Monitor Usage ```bash docker stats speaches ``` --- ## Comparison: Local vs Cloud | Aspect | Local (Speaches) | Cloud (OpenAI Whisper) | |--------|------------------|------------------------| | **Cost** | Free | $0.006/min | | **Privacy** | Complete | Data sent to provider | | **Speed** | Depends on hardware | Usually faster | | **Quality** | Excellent (same Whisper) | Excellent | | **Setup** | Moderate | Simple API key | | **Offline** | Yes | No | | **Languages** | 99+ | 99+ | ### When to Use Local - Privacy-sensitive content - High-volume transcription - Development/testing - Offline environments - Cost control ### When to Use Cloud - Limited hardware - Time-sensitive projects - No GPU available - Simple setup preferred --- ## Using Both TTS and STT Speaches supports both TTS and STT in one server. In **Settings → API Keys**, add a single **OpenAI-Compatible** credential and configure both the TTS and STT base URLs to point to the same Speaches server (e.g., `http://localhost:8969/v1`). See **[Local TTS Setup](local-tts.md)** for TTS configuration. --- ## Other Local STT Options Any OpenAI-compatible STT server works: | Server | Description | |--------|-------------| | [Speaches](https://github.com/speaches-ai/speaches) | TTS + STT in one (recommended) | | [faster-whisper-server](https://github.com/fedirz/faster-whisper-server) | Lightweight STT only | | [whisper.cpp](https://github.com/ggerganov/whisper.cpp) | C++ implementation with server mode | | [LocalAI](https://github.com/mudler/LocalAI) | Multi-model local AI server | The key requirements: 1. Server implements `/v1/audio/transcriptions` endpoint 2. Add an OpenAI-Compatible credential in **Settings → API Keys** with the STT base URL 3. Add model with provider `openai_compatible` --- ## Related - **[Local TTS Setup](local-tts.md)** - Text-to-speech with Speaches - **[OpenAI-Compatible Providers](openai-compatible.md)** - General compatible provider setup - **[AI Providers](ai-providers.md)** - All provider configuration ================================================ FILE: docs/5-CONFIGURATION/local-tts.md ================================================ # Local Text-to-Speech Setup Run text-to-speech locally for free, private podcast generation using OpenAI-compatible TTS servers. --- ## Why Local TTS? | Benefit | Description | |---------|-------------| | **Free** | No per-character costs after setup | | **Private** | Audio never leaves your machine | | **Unlimited** | No rate limits or quotas | | **Offline** | Works without internet | --- ## Quick Start with Speaches [Speaches](https://github.com/speaches-ai/speaches) is an open-source, OpenAI-compatible TTS server. > **💡 Ready-made Docker Compose files available:** > - **[docker-compose-speaches.yml](../../examples/docker-compose-speaches.yml)** - Speaches + Open Notebook > - **[docker-compose-full-local.yml](../../examples/docker-compose-full-local.yml)** - Speaches + Ollama (100% local setup) > > These include complete setup instructions and configuration examples. Just copy and run! ### Step 1: Create Docker Compose File Create a folder and add `docker-compose.yml`: ```yaml services: speaches: image: ghcr.io/speaches-ai/speaches:latest-cpu container_name: speaches ports: - "8969:8000" volumes: - hf-hub-cache:/home/ubuntu/.cache/huggingface/hub restart: unless-stopped volumes: hf-hub-cache: ``` ### Step 2: Start and Download Model ```bash # Start Speaches docker compose up -d # Wait for startup sleep 10 # Download voice model (~500MB) docker compose exec speaches uv tool run speaches-cli model download speaches-ai/Kokoro-82M-v1.0-ONNX ``` ### Step 3: Test ```bash curl "http://localhost:8969/v1/audio/speech" -s \ -H "Content-Type: application/json" \ --output test.mp3 \ --data '{ "input": "Hello! Local TTS is working.", "model": "speaches-ai/Kokoro-82M-v1.0-ONNX", "voice": "af_bella" }' ``` Play `test.mp3` to verify. ### Step 4: Configure Open Notebook **Via Settings UI (Recommended):** 1. Go to **Settings** → **API Keys** 2. Click **Add Credential** → Select **OpenAI-Compatible** 3. Enter base URL for TTS: `http://host.docker.internal:8969/v1` (Docker) or `http://localhost:8969/v1` (local) 4. Click **Save**, then **Test Connection** **Legacy (Deprecated) — Environment variables:** ```yaml # In your Open Notebook docker-compose.yml environment: - OPENAI_COMPATIBLE_BASE_URL_TTS=http://host.docker.internal:8969/v1 ``` ```bash # Local development export OPENAI_COMPATIBLE_BASE_URL_TTS=http://localhost:8969/v1 ``` ### Step 5: Add Model in Open Notebook 1. Go to **Settings** → **Models** 2. Click **Add Model** in Text-to-Speech section 3. Configure: - **Provider**: `openai_compatible` - **Model Name**: `speaches-ai/Kokoro-82M-v1.0-ONNX` - **Display Name**: `Local TTS` 4. Click **Save** 5. Set as default if desired --- ## Available Voices The Kokoro model includes multiple voices: ### Female Voices | Voice ID | Description | |----------|-------------| | `af_bella` | Clear, professional | | `af_sarah` | Warm, friendly | | `af_nicole` | Energetic, expressive | ### Male Voices | Voice ID | Description | |----------|-------------| | `am_adam` | Deep, authoritative | | `am_michael` | Friendly, conversational | ### British Accents | Voice ID | Description | |----------|-------------| | `bf_emma` | British female, professional | | `bm_george` | British male, formal | ### Test Different Voices ```bash for voice in af_bella af_sarah am_adam am_michael; do curl "http://localhost:8969/v1/audio/speech" -s \ -H "Content-Type: application/json" \ --output "test_${voice}.mp3" \ --data "{ \"input\": \"Hello, this is the ${voice} voice.\", \"model\": \"speaches-ai/Kokoro-82M-v1.0-ONNX\", \"voice\": \"${voice}\" }" done ``` --- ## GPU Acceleration For faster generation with NVIDIA GPUs: ```yaml services: speaches: image: ghcr.io/speaches-ai/speaches:latest-cuda container_name: speaches ports: - "8969:8000" volumes: - hf-hub-cache:/home/ubuntu/.cache/huggingface/hub restart: unless-stopped deploy: resources: reservations: devices: - driver: nvidia count: 1 capabilities: [gpu] volumes: hf-hub-cache: ``` --- ## Docker Networking When configuring your OpenAI-Compatible credential in **Settings → API Keys**, use the appropriate TTS base URL for your setup: ### Open Notebook in Docker (macOS/Windows) **TTS Base URL:** `http://host.docker.internal:8969/v1` ### Open Notebook in Docker (Linux) **TTS Base URL (Option 1 — Docker bridge IP):** `http://172.17.0.1:8969/v1` **Option 2:** Use host networking mode (`docker run --network host ...`), then use: `http://localhost:8969/v1` ### Remote Server Run Speaches on a different machine: **TTS Base URL:** `http://server-ip:8969/v1` (replace with your server's IP) --- ## Multi-Speaker Podcasts Configure different voices for each speaker: ``` Speaker 1 (Host): Model: speaches-ai/Kokoro-82M-v1.0-ONNX Voice: af_bella Speaker 2 (Guest): Model: speaches-ai/Kokoro-82M-v1.0-ONNX Voice: am_adam Speaker 3 (Narrator): Model: speaches-ai/Kokoro-82M-v1.0-ONNX Voice: bf_emma ``` --- ## Troubleshooting ### Service Won't Start ```bash # Check logs docker compose logs speaches # Verify port available lsof -i :8969 # Restart docker compose down && docker compose up -d ``` ### Connection Refused ```bash # Test Speaches is running curl http://localhost:8969/v1/models # From inside Open Notebook container docker exec -it open-notebook curl http://host.docker.internal:8969/v1/models ``` ### Model Not Found ```bash # List downloaded models docker compose exec speaches uv tool run speaches-cli model list # Download if missing docker compose exec speaches uv tool run speaches-cli model download speaches-ai/Kokoro-82M-v1.0-ONNX ``` ### Poor Audio Quality - Try different voices - Adjust speed: `"speed": 0.9` to `1.2` - Check model downloaded completely - Allocate more memory ### Slow Generation | Solution | How | |----------|-----| | Use GPU | Switch to `latest-cuda` image | | More CPU | Allocate more cores in Docker | | Faster model | Use smaller/quantized models | | SSD storage | Move Docker volumes to SSD | --- ## Performance Tips ### Recommended Specs | Component | Minimum | Recommended | |-----------|---------|-------------| | CPU | 2 cores | 4+ cores | | RAM | 2 GB | 4+ GB | | Storage | 5 GB | 10 GB (for multiple models) | | GPU | None | NVIDIA (optional) | ### Resource Limits ```yaml services: speaches: # ... other config mem_limit: 4g cpus: 2 ``` ### Monitor Usage ```bash docker stats speaches ``` --- ## Comparison: Local vs Cloud | Aspect | Local (Speaches) | Cloud (OpenAI/ElevenLabs) | |--------|------------------|---------------------------| | **Cost** | Free | $0.015-0.10/min | | **Privacy** | Complete | Data sent to provider | | **Speed** | Depends on hardware | Usually faster | | **Quality** | Good | Excellent | | **Setup** | Moderate | Simple API key | | **Offline** | Yes | No | | **Voices** | Limited | Many options | ### When to Use Local - Privacy-sensitive content - High-volume generation - Development/testing - Offline environments - Cost control ### When to Use Cloud - Premium quality needs - Multiple languages - Time-sensitive projects - Limited hardware --- ## Other Local TTS Options Any OpenAI-compatible TTS server works. The key is: 1. Server implements `/v1/audio/speech` endpoint 2. Add an OpenAI-Compatible credential in **Settings → API Keys** with the TTS base URL 3. Add model with provider `openai_compatible` --- ## Related - **[Local STT Setup](local-stt.md)** - Speech-to-text with Speaches - **[OpenAI-Compatible Providers](openai-compatible.md)** - General compatible provider setup - **[AI Providers](ai-providers.md)** - All provider configuration - **[Creating Podcasts](../3-USER-GUIDE/creating-podcasts.md)** - Using TTS for podcasts ================================================ FILE: docs/5-CONFIGURATION/mcp-integration.md ================================================ # Model Context Protocol (MCP) Integration Open Notebook can be seamlessly integrated into your AI workflows using the **Model Context Protocol (MCP)**, enabling direct access to your notebooks, sources, and chat functionality from AI assistants like Claude Desktop and VS Code extensions. ## What is MCP? The [Model Context Protocol](https://modelcontextprotocol.io) is an open standard that allows AI applications to securely connect to external data sources and tools. With the Open Notebook MCP server, you can: - 📚 **Access your notebooks** directly from Claude Desktop or VS Code - 🔍 **Search your research content** without leaving your AI assistant - 💬 **Create and manage chat sessions** with your research as context - 📝 **Generate notes** and insights on-the-fly - 🤖 **Automate workflows** using the full Open Notebook API ## Quick Setup ### For Claude Desktop 1. **Install the MCP server** (automatically from PyPI): ```bash # No manual installation needed! Claude Desktop will use uvx to run it automatically ``` 2. **Configure Claude Desktop**: **macOS/Linux**: Edit `~/Library/Application Support/Claude/claude_desktop_config.json` ```json { "mcpServers": { "open-notebook": { "command": "uvx", "args": ["open-notebook-mcp"], "env": { "OPEN_NOTEBOOK_URL": "http://localhost:5055", "OPEN_NOTEBOOK_PASSWORD": "your_password_here" } } } } ``` **Windows**: Edit `%APPDATA%\Claude\claude_desktop_config.json` ```json { "mcpServers": { "open-notebook": { "command": "uvx", "args": ["open-notebook-mcp"], "env": { "OPEN_NOTEBOOK_URL": "http://localhost:5055", "OPEN_NOTEBOOK_PASSWORD": "your_password_here" } } } } ``` 3. **Restart Claude Desktop** and start using your notebooks in conversations! ### For VS Code (Cline and other MCP-compatible extensions) Add to your VS Code settings or `.vscode/mcp.json`: ```json { "servers": { "open-notebook": { "command": "uvx", "args": ["open-notebook-mcp"], "env": { "OPEN_NOTEBOOK_URL": "http://localhost:5055", "OPEN_NOTEBOOK_PASSWORD": "your_password_here" } } } } ``` ## Configuration - **OPEN_NOTEBOOK_URL**: URL to your Open Notebook API (default: `http://localhost:5055`) - **OPEN_NOTEBOOK_PASSWORD**: Optional - only needed if you've enabled password protection ### For Remote Servers If your Open Notebook instance is running on a remote server, update the URL accordingly: ```json "OPEN_NOTEBOOK_URL": "http://192.168.1.100:5055" ``` Or with a domain: ```json "OPEN_NOTEBOOK_URL": "https://notebook.yourdomain.com/api" ``` ## What You Can Do Once connected, you can ask Claude or your AI assistant to: - _"Search my research notebooks for information about [topic]"_ - _"Create a new note summarizing the key points from our conversation"_ - _"List all my notebooks"_ - _"Start a chat session about [specific source or topic]"_ - _"What sources do I have in my [notebook name] notebook?"_ - _"Add this PDF to my research notebook"_ - _"Show me all notes in [notebook name]"_ The MCP server provides full access to Open Notebook's capabilities, allowing you to manage your research seamlessly from within your AI assistant. ## Available Tools The Open Notebook MCP server exposes these capabilities: ### Notebooks - List notebooks - Get notebook details - Create new notebooks - Update notebook information - Delete notebooks ### Sources - List sources in a notebook - Get source details - Add new sources (links, files, text) - Update source metadata - Delete sources ### Notes - List notes in a notebook - Get note details - Create new notes - Update notes - Delete notes ### Chat - Create chat sessions - Send messages to chat sessions - Get chat history - List chat sessions ### Search - Vector search across content - Text search across content - Filter by notebook ### Models - List configured AI models - Get model details - Create model configurations - Update model settings ### Settings - Get application settings - Update settings ## MCP Server Repository The Open Notebook MCP server is developed and maintained by the Epochal team: **🔗 GitHub**: [Epochal-dev/open-notebook-mcp](https://github.com/Epochal-dev/open-notebook-mcp) Contributions, issues, and feature requests are welcome! ## Finding the Server The Open Notebook MCP server is published to the official MCP Registry: - **Registry**: Search for "open-notebook" at [registry.modelcontextprotocol.io](https://registry.modelcontextprotocol.io) - **PyPI**: [pypi.org/project/open-notebook-mcp](https://pypi.org/project/open-notebook-mcp) - **GitHub**: [Epochal-dev/open-notebook-mcp](https://github.com/Epochal-dev/open-notebook-mcp) ## Troubleshooting ### Connection Errors 1. Verify the `OPEN_NOTEBOOK_URL` is correct and accessible 2. If using password protection, ensure `OPEN_NOTEBOOK_PASSWORD` is set correctly 3. For remote servers, make sure port 5055 is accessible from your machine 4. Check firewall settings if connecting to a remote server ## Using with Other MCP Clients The Open Notebook MCP server follows the standard MCP protocol and can be used with any MCP-compatible client. Check your client's documentation for configuration details. ## Learn More - [Model Context Protocol Documentation](https://modelcontextprotocol.io) ================================================ FILE: docs/5-CONFIGURATION/ollama.md ================================================ # Ollama Setup Guide Ollama provides free, local AI models that run on your own hardware. This guide covers everything you need to know about setting up Ollama with Open Notebook, including different deployment scenarios and network configurations. ## Why Choose Ollama? - **🆓 Completely Free**: No API costs after initial setup - **🔒 Full Privacy**: Your data never leaves your local network - **📱 Offline Capable**: Works without internet connection - **🚀 Fast**: Local inference with no network latency - **🧠 Reasoning Models**: Support for advanced reasoning models like DeepSeek-R1 - **💾 Model Variety**: Access to hundreds of open-source models ## Quick Start ### 1. Install Ollama **Linux/macOS:** ```bash curl -fsSL https://ollama.ai/install.sh | sh ``` **Windows:** Download and install from [ollama.ai](https://ollama.ai/download) ### 2. Pull Required Models ```bash # Language models (choose one or more) ollama pull qwen3 # Excellent general purpose, 7B parameters ollama pull gemma3 # Google's model, good performance ollama pull deepseek-r1 # Advanced reasoning model ollama pull phi4 # Microsoft's efficient model # Embedding model (required for search) ollama pull mxbai-embed-large # Best embedding model for Ollama ``` ### 3. Configure Open Notebook **Via Settings UI (Recommended):** 1. Go to **Settings** → **API Keys** 2. Click **Add Credential** → Select **Ollama** 3. Enter the base URL (see [Network Configuration](#network-configuration-guide) below for correct URL) 4. Click **Save**, then **Test Connection** 5. Click **Discover Models** → **Register Models** **Legacy (Deprecated) — Environment variables:** ```bash # For local installation: export OLLAMA_API_BASE=http://localhost:11434 # For Docker installation: export OLLAMA_API_BASE=http://host.docker.internal:11434 ``` > **Note**: The `OLLAMA_API_BASE` environment variable is deprecated. Configure Ollama via Settings → API Keys instead. ## Network Configuration Guide When adding an Ollama credential in **Settings → API Keys**, you need to enter the correct base URL. The correct URL depends on your deployment scenario: ### Scenario 1: Local Installation (Same Machine) When both Open Notebook and Ollama run directly on your machine: **Base URL to enter in Settings → API Keys:** `http://localhost:11434` Alternative: `http://127.0.0.1:11434` (use if you have DNS resolution issues with localhost) ### Scenario 2: Open Notebook in Docker, Ollama on Host When Open Notebook runs in Docker but Ollama runs on your host machine: **Base URL to enter in Settings → API Keys:** `http://host.docker.internal:11434` **⚠️ CRITICAL: Ollama must accept external connections:** ```bash # Start Ollama with external access enabled export OLLAMA_HOST=0.0.0.0:11434 ollama serve ``` **⚠️ LINUX USERS: Extra configuration required!** On Linux, `host.docker.internal` doesn't resolve automatically like it does on macOS/Windows. You must add `extra_hosts` to your docker-compose.yml: ```yaml services: open_notebook: image: lfnovo/open_notebook:v1-latest-single # ... other settings ... extra_hosts: - "host.docker.internal:host-gateway" ``` Without this, you'll get connection errors like: ``` httpcore.ConnectError: [Errno -2] Name or service not known ``` **Why `host.docker.internal`?** - Docker containers can't reach `localhost` on the host - `host.docker.internal` is Docker's special hostname for the host machine - Available on Docker Desktop for Mac/Windows; **requires `extra_hosts` on Linux** **Why `OLLAMA_HOST=0.0.0.0:11434`?** - By default, Ollama only binds to localhost and rejects external connections - Docker containers are considered "external" even when running on the same machine - Setting `OLLAMA_HOST=0.0.0.0:11434` allows connections from Docker containers ### Scenario 3: Both in Docker (Same Compose) When both Open Notebook and Ollama run in the same Docker Compose stack: **Base URL to enter in Settings → API Keys:** `http://ollama:11434` **Docker Compose Example:** ```yaml version: '3.8' services: open-notebook: image: lfnovo/open_notebook:v1-latest-single pull_policy: always ports: - "8502:8502" - "5055:5055" environment: - OPEN_NOTEBOOK_ENCRYPTION_KEY=change-me-to-a-secret-string volumes: - ./notebook_data:/app/data - ./surreal_data:/mydata depends_on: - ollama ollama: image: ollama/ollama:latest ports: - "11434:11434" volumes: - ollama_data:/root/.ollama # Optional: GPU support deploy: resources: reservations: devices: - driver: nvidia count: 1 capabilities: [gpu] volumes: ollama_data: ``` ### Scenario 4: Remote Ollama Server When Ollama runs on a different machine in your network: **Base URL to enter in Settings → API Keys:** `http://192.168.1.100:11434` (replace with your Ollama server's IP) **Security Note:** Only use this in trusted networks. Ollama doesn't have built-in authentication. ### Scenario 5: Ollama with Custom Port If you've configured Ollama to use a different port: ```bash # Start Ollama on custom port OLLAMA_HOST=0.0.0.0:8080 ollama serve ``` **Base URL to enter in Settings → API Keys:** `http://localhost:8080` ## Model Recommendations ### Language Models | Model | Size | Best For | Quality | Speed | |-------|------|----------|---------|-------| | **qwen3** | 7B | General purpose, coding | Excellent | Fast | | **deepseek-r1** | 7B | Reasoning, problem-solving | Exceptional | Medium | | **gemma3** | 7B | Balanced performance | Very Good | Fast | | **phi4** | 14B | Efficiency on small hardware | Good | Very Fast | | **llama3** | 8B | General purpose | Very Good | Medium | ### Embedding Models | Model | Best For | Performance | |-------|----------|-------------| | **mxbai-embed-large** | General search | Excellent | | **nomic-embed-text** | Document similarity | Good | | **all-minilm** | Lightweight option | Fair | ### Installation Commands ```bash # Essential models ollama pull qwen3 # Primary language model ollama pull mxbai-embed-large # Search embeddings # Optional reasoning model ollama pull deepseek-r1 # Advanced reasoning # Alternative language models ollama pull gemma3 # Google's model ollama pull phi4 # Microsoft's efficient model ``` ## Hardware Requirements ### Minimum Requirements - **RAM**: 8GB (for 7B models) - **Storage**: 10GB free space per model - **CPU**: Modern multi-core processor ### Recommended Setup - **RAM**: 16GB+ (for multiple models) - **Storage**: SSD with 50GB+ free space - **GPU**: NVIDIA GPU with 8GB+ VRAM (optional but faster) ### GPU Acceleration **NVIDIA GPU (CUDA):** ```bash # Install NVIDIA Container Toolkit for Docker # Then use the Docker Compose example above with GPU support # For local installation, Ollama auto-detects CUDA ollama pull qwen3 ``` **Apple Silicon (M1/M2/M3):** ```bash # Ollama automatically uses Metal acceleration # No additional setup required ollama pull qwen3 ``` **AMD GPUs:** ```bash # ROCm support varies by model and system # Check Ollama documentation for latest compatibility ``` ## Troubleshooting ### Model Name Configuration (Critical) **⚠️ IMPORTANT: Model names must exactly match the output of `ollama list`** This is the most common cause of "Failed to send message" errors. Open Notebook requires the **exact model name** as it appears in Ollama. **Step 1: Get the exact model name** ```bash ollama list ``` Example output: ``` NAME ID SIZE MODIFIED mxbai-embed-large:latest 468836162de7 669 MB 7 minutes ago gemma3:12b f4031aab637d 8.1 GB 2 months ago qwen3:32b 030ee887880f 20 GB 9 days ago ``` **Step 2: Use the exact name when adding the model in Open Notebook** | ✅ Correct | ❌ Wrong | |-----------|----------| | `gemma3:12b` | `gemma3` (missing tag) | | `qwen3:32b` | `qwen3-32b` (wrong format) | | `mxbai-embed-large:latest` | `mxbai-embed-large` (missing tag) | **Note:** Some models use `:latest` as the default tag. If `ollama list` shows `model:latest`, you must use `model:latest` in Open Notebook, not just `model`. **Step 3: Configure in Open Notebook** 1. Go to **Settings → Models** 2. Click **Add Model** 3. Enter the **exact name** from `ollama list` 4. Select provider: `ollama` 5. Select type: `language` (for chat) or `embedding` (for search) 6. Save the model 7. Set it as the default for the appropriate task (chat, transformation, etc.) ### Common Issues **1. "Ollama unavailable" in Open Notebook** **Check Ollama is running:** ```bash curl http://localhost:11434/api/tags ``` **Verify credential is configured:** Check **Settings → API Keys** for an Ollama credential with the correct base URL. **⚠️ IMPORTANT: Enable external connections (most common fix):** ```bash # If Open Notebook runs in Docker or on a different machine, # Ollama must bind to all interfaces, not just localhost export OLLAMA_HOST=0.0.0.0:11434 ollama serve ``` > **Why this is needed:** By default, Ollama only accepts connections from `localhost` (127.0.0.1). When Open Notebook runs in Docker or on a different machine, it can't reach Ollama unless you configure `OLLAMA_HOST=0.0.0.0:11434` to accept external connections. **Restart Ollama:** ```bash # Linux/macOS sudo systemctl restart ollama # or ollama serve # Windows # Restart from system tray or Services ``` **2. Docker networking issues** **From inside Open Notebook container, test Ollama:** ```bash # Get into container docker exec -it open-notebook bash # Test connection curl http://host.docker.internal:11434/api/tags ``` **If this fails on Linux** with "Name or service not known", you need to add `extra_hosts` to your docker-compose.yml. See the [Docker-Specific Troubleshooting](#docker-specific-troubleshooting) section below. **3. Models not downloading** **Check disk space:** ```bash df -h ``` **Manual model pull:** ```bash ollama pull qwen3 --verbose ``` **Clear failed downloads:** ```bash ollama rm qwen3 ollama pull qwen3 ``` **4. Slow performance** **Check model size vs available RAM:** ```bash ollama ps # Show running models free -h # Check available memory ``` **Use smaller models:** ```bash ollama pull phi4 # Instead of larger models ollama pull gemma3:2b # 2B parameter variant ``` **5. Port conflicts** **Check what's using port 11434:** ```bash lsof -i :11434 netstat -tulpn | grep 11434 ``` **Use custom port:** ```bash OLLAMA_HOST=0.0.0.0:8080 ollama serve ``` Then update the base URL in **Settings → API Keys** to `http://localhost:8080` **6. "Failed to send message" in Chat** **Symptom:** Chat shows "Failed to send message" toast notification. Logs may show: ``` Error executing chat: Model is not a LanguageModel: None ``` **Causes (in order of likelihood):** 1. **Model name mismatch**: The model name in Open Notebook doesn't exactly match `ollama list` 2. **No default model configured**: You haven't set a default chat model in Settings → Models 3. **Model was deleted**: You removed the model from Ollama but didn't update Open Notebook's defaults 4. **Model record deleted**: The model was removed from Open Notebook but is still set as default **Solutions:** **Check 1: Verify model names match exactly** ```bash # Get exact model names from Ollama ollama list # Compare with what's configured in Open Notebook # Go to Settings → Models and verify the names match EXACTLY ``` **Check 2: Verify default models are set** 1. Go to **Settings → Models** 2. Scroll to **Default Models** section 3. Ensure **Default Chat Model** has a value selected 4. If empty, select an available language model **Check 3: Refresh after changes** If you've added/removed models in Ollama: 1. Refresh the Open Notebook page 2. Go to Settings → Models 3. Re-add any missing models with exact names from `ollama list` 4. Re-select default models if needed **Check 4: Test the model directly** ```bash # Verify Ollama can use the model ollama run gemma3:12b "Hello, world" ``` ### Docker-Specific Troubleshooting **1. Linux: `host.docker.internal` not resolving (Most Common)** If you see `Name or service not known` errors on Linux, add `extra_hosts` to your docker-compose.yml: ```yaml services: open_notebook: image: lfnovo/open_notebook:v1-latest-single extra_hosts: - "host.docker.internal:host-gateway" environment: # ... rest of your config ``` Then in **Settings → API Keys**, use base URL: `http://host.docker.internal:11434` This maps `host.docker.internal` to your host machine's IP. macOS/Windows Docker Desktop does this automatically, but Linux requires explicit configuration. **2. Host networking on Linux (alternative):** ```bash # Use host networking if host.docker.internal doesn't work docker run --network host lfnovo/open_notebook:v1-latest-single ``` Then in **Settings → API Keys**, use base URL: `http://localhost:11434` **3. Custom bridge network:** ```yaml version: '3.8' networks: ollama_network: driver: bridge services: open-notebook: networks: - ollama_network environment: ollama: networks: - ollama_network ``` Then in **Settings → API Keys**, use base URL: `http://ollama:11434` **4. Firewall issues:** ```bash # Allow Ollama port through firewall sudo ufw allow 11434 # or sudo firewall-cmd --add-port=11434/tcp --permanent ``` ## Performance Optimization ### Model Management **List installed models:** ```bash ollama list ``` **Remove unused models:** ```bash ollama rm model_name ``` **Show running models:** ```bash ollama ps ``` **Preload models for faster startup:** ```bash # Keep model in memory curl http://localhost:11434/api/generate -d '{ "model": "qwen3", "prompt": "test", "keep_alive": -1 }' ``` ### System Optimization **Linux: Increase file limits:** ```bash echo "* soft nofile 65536" >> /etc/security/limits.conf echo "* hard nofile 65536" >> /etc/security/limits.conf ``` **macOS: Increase memory limits:** ```bash # Add to ~/.zshrc or ~/.bash_profile export OLLAMA_MAX_LOADED_MODELS=2 export OLLAMA_NUM_PARALLEL=4 ``` **Docker: Resource allocation:** ```yaml services: ollama: deploy: resources: limits: memory: 8G cpus: '4' ``` ## Advanced Configuration ### Environment Variables ```bash # Ollama server configuration export OLLAMA_HOST=0.0.0.0:11434 # Bind to all interfaces export OLLAMA_KEEP_ALIVE=5m # Keep models in memory export OLLAMA_MAX_LOADED_MODELS=3 # Max concurrent models export OLLAMA_MAX_QUEUE=512 # Request queue size export OLLAMA_NUM_PARALLEL=4 # Parallel request handling export OLLAMA_FLASH_ATTENTION=1 # Enable flash attention (if supported) # Open Notebook configuration (configure via Settings → API Keys instead) # OLLAMA_API_BASE=http://localhost:11434 # Deprecated — use Settings UI ``` ### SSL Configuration (Self-Signed Certificates) If you're running Ollama behind a reverse proxy with self-signed SSL certificates (e.g., Caddy, nginx with custom certs), you may encounter SSL verification errors: ``` [SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: unable to get local issuer certificate ``` **Solutions:** **Option 1: Use a custom CA bundle (recommended)** ```bash # Point to your CA certificate file export ESPERANTO_SSL_CA_BUNDLE=/path/to/your/ca-bundle.pem ``` **Option 2: Disable SSL verification (development only)** ```bash # WARNING: Only use in trusted development environments export ESPERANTO_SSL_VERIFY=false ``` **Docker Compose example with SSL configuration:** ```yaml services: open-notebook: image: lfnovo/open_notebook:v1-latest-single pull_policy: always environment: - OPEN_NOTEBOOK_ENCRYPTION_KEY=change-me-to-a-secret-string # Option 1: Custom CA bundle (if Ollama uses self-signed SSL) - ESPERANTO_SSL_CA_BUNDLE=/certs/ca-bundle.pem # Option 2: Disable verification (dev only) # - ESPERANTO_SSL_VERIFY=false volumes: - /path/to/your/ca-bundle.pem:/certs/ca-bundle.pem:ro ``` > **Security Note:** Disabling SSL verification exposes you to man-in-the-middle attacks. Always prefer using a custom CA bundle in production environments. ### Custom Model Imports **Import custom models:** ```bash # Create Modelfile cat > Modelfile << EOF FROM qwen3 PARAMETER temperature 0.7 PARAMETER top_p 0.9 SYSTEM "You are a helpful research assistant." EOF # Create custom model ollama create my-research-model -f Modelfile ``` **Use in Open Notebook:** 1. Go to Models 2. Add new model: `my-research-model` 3. Set as default for specific tasks ### Monitoring and Logging **Monitor Ollama logs:** ```bash # Linux (systemd) journalctl -u ollama -f # Docker docker logs -f ollama # Manual run with verbose logging OLLAMA_DEBUG=1 ollama serve ``` **Resource monitoring:** ```bash # CPU and memory usage htop # GPU usage (NVIDIA) nvidia-smi -l 1 # Model-specific metrics ollama ps ``` ## Integration Examples ### Python Script Integration ```python import requests import os # Test Ollama connection ollama_base = os.environ.get('OLLAMA_API_BASE', 'http://localhost:11434') response = requests.get(f'{ollama_base}/api/tags') print(f"Available models: {response.json()}") # Generate text payload = { "model": "qwen3", "prompt": "Explain quantum computing", "stream": False } response = requests.post(f'{ollama_base}/api/generate', json=payload) print(response.json()['response']) ``` ### Health Check Script ```bash #!/bin/bash # ollama-health-check.sh OLLAMA_API_BASE=${OLLAMA_API_BASE:-"http://localhost:11434"} echo "Checking Ollama health..." if curl -s "${OLLAMA_API_BASE}/api/tags" > /dev/null; then echo "✅ Ollama is running" echo "Available models:" curl -s "${OLLAMA_API_BASE}/api/tags" | jq -r '.models[].name' else echo "❌ Ollama is not accessible at ${OLLAMA_API_BASE}" exit 1 fi ``` ## Migration from Other Providers ### Coming from OpenAI **Similar performance models:** - GPT-4 → `qwen3` or `deepseek-r1` - GPT-3.5 → `gemma3` or `phi4` - text-embedding-ada-002 → `mxbai-embed-large` **Cost comparison:** - OpenAI: $0.01-0.06 per 1K tokens - Ollama: $0 after hardware investment ### Coming from Anthropic **Claude replacement suggestions:** - Claude 3.5 Sonnet → `deepseek-r1` (reasoning) - Claude 3 Haiku → `phi4` (speed) ## Best Practices ### Security 1. **Network Security:** - Run Ollama only on trusted networks - Use firewall rules to limit access - Consider VPN for remote access 2. **Model Verification:** - Only pull models from trusted sources - Verify model checksums when possible 3. **Resource Limits:** - Set memory and CPU limits in production - Monitor resource usage regularly ### Performance 1. **Model Selection:** - Use appropriate model size for your hardware - Smaller models for simple tasks - Reasoning models only when needed 2. **Resource Management:** - Preload frequently used models - Remove unused models regularly - Monitor system resources 3. **Network Optimization:** - Use local networks for better latency - Consider SSD storage for faster model loading ## Getting Help **Community Resources:** - [Ollama GitHub](https://github.com/jmorganca/ollama) - Official repository - [Ollama Discord](https://discord.gg/ollama) - Community support - [Open Notebook Discord](https://discord.gg/37XJPXfz2w) - Integration help **Debugging Resources:** - Check Ollama logs for error messages - Test connection with curl commands - Verify environment variables - Monitor system resources This comprehensive guide should help you successfully deploy and optimize Ollama with Open Notebook. Start with the Quick Start section and refer to specific scenarios as needed. ================================================ FILE: docs/5-CONFIGURATION/openai-compatible.md ================================================ # OpenAI-Compatible Providers Use any server that implements the OpenAI API format with Open Notebook. This includes LM Studio, Text Generation WebUI, vLLM, and many others. --- ## What is OpenAI-Compatible? Many AI tools implement the same API format as OpenAI: ``` POST /v1/chat/completions POST /v1/embeddings POST /v1/audio/speech ``` Open Notebook can connect to any server using this format. --- ## Common Compatible Servers | Server | Use Case | URL | |--------|----------|-----| | **LM Studio** | Desktop GUI for local models | https://lmstudio.ai | | **Text Generation WebUI** | Full-featured local inference | https://github.com/oobabooga/text-generation-webui | | **vLLM** | High-performance serving | https://github.com/vllm-project/vllm | | **Ollama** | Simple local models | (Use native Ollama provider instead) | | **LocalAI** | Local AI inference | https://github.com/mudler/LocalAI | | **llama.cpp server** | Lightweight inference | https://github.com/ggerganov/llama.cpp | --- ## Quick Setup: LM Studio ### Step 1: Install and Start LM Studio 1. Download from https://lmstudio.ai 2. Install and launch 3. Download a model (e.g., Llama 3) 4. Start the local server (default: port 1234) ### Step 2: Configure in Settings UI (Recommended) 1. Go to **Settings** → **API Keys** 2. Click **Add Credential** → Select **OpenAI-Compatible** 3. Enter base URL: `http://host.docker.internal:1234/v1` (Docker) or `http://localhost:1234/v1` (local) 4. API key: `lm-studio` (placeholder, LM Studio doesn't require one) 5. Click **Save**, then **Test Connection** **Legacy (Deprecated) — Environment variables:** ```bash export OPENAI_COMPATIBLE_BASE_URL=http://localhost:1234/v1 export OPENAI_COMPATIBLE_API_KEY=not-needed ``` ### Step 3: Add Model in Open Notebook 1. Go to **Settings** → **Models** 2. Click **Add Model** 3. Configure: - **Provider**: `openai_compatible` - **Model Name**: Your model name from LM Studio - **Display Name**: `LM Studio - Llama 3` 4. Click **Save** --- ## Configuration via Settings UI The recommended way to configure OpenAI-compatible providers is through the Settings UI: 1. Go to **Settings** → **API Keys** 2. Click **Add Credential** → Select **OpenAI-Compatible** 3. Enter your base URL and API key (if needed) 4. Optionally configure per-service URLs for LLM, Embedding, TTS, and STT 5. Click **Save**, then **Test Connection** ## Legacy: Environment Variables (Deprecated) > **Deprecated**: These environment variables are deprecated. Use the Settings UI instead. ### Language Models (Chat) ```bash OPENAI_COMPATIBLE_BASE_URL=http://localhost:1234/v1 OPENAI_COMPATIBLE_API_KEY=optional-api-key ``` ### Embeddings ```bash OPENAI_COMPATIBLE_BASE_URL_EMBEDDING=http://localhost:1234/v1 OPENAI_COMPATIBLE_API_KEY_EMBEDDING=optional-api-key ``` ### Text-to-Speech ```bash OPENAI_COMPATIBLE_BASE_URL_TTS=http://localhost:8969/v1 OPENAI_COMPATIBLE_API_KEY_TTS=optional-api-key ``` ### Speech-to-Text ```bash OPENAI_COMPATIBLE_BASE_URL_STT=http://localhost:9000/v1 OPENAI_COMPATIBLE_API_KEY_STT=optional-api-key ``` --- ## Docker Networking When Open Notebook runs in Docker and your compatible server runs on the host, use the appropriate base URL when adding your credential in **Settings → API Keys**: ### macOS / Windows **Base URL:** `http://host.docker.internal:1234/v1` ### Linux **Base URL (Option 1 — Docker bridge IP):** `http://172.17.0.1:1234/v1` **Option 2:** Use host networking mode: `docker run --network host ...` Then use base URL: `http://localhost:1234/v1` ### Same Docker Network ```yaml # docker-compose.yml services: open-notebook: # ... lm-studio: # your LM Studio container ports: - "1234:1234" ``` **Base URL in Settings → API Keys:** `http://lm-studio:1234/v1` --- ## Text Generation WebUI Setup ### Start with API Enabled ```bash python server.py --api --listen ``` ### Configure Open Notebook In **Settings → API Keys**, add an **OpenAI-Compatible** credential with base URL: `http://localhost:5000/v1` ### Docker Compose Example ```yaml services: text-gen: image: atinoda/text-generation-webui:default ports: - "5000:5000" - "7860:7860" volumes: - ./models:/app/models command: --api --listen open-notebook: image: lfnovo/open_notebook:v1-latest-single pull_policy: always depends_on: - text-gen ``` Then in **Settings → API Keys**, add an **OpenAI-Compatible** credential with base URL: `http://text-gen:5000/v1` --- ## vLLM Setup ### Start vLLM Server ```bash python -m vllm.entrypoints.openai.api_server \ --model meta-llama/Llama-3.1-8B-Instruct \ --port 8000 ``` ### Configure Open Notebook In **Settings → API Keys**, add an **OpenAI-Compatible** credential with base URL: `http://localhost:8000/v1` ### Docker Compose with GPU ```yaml services: vllm: image: vllm/vllm-openai:latest command: --model meta-llama/Llama-3.1-8B-Instruct ports: - "8000:8000" volumes: - ~/.cache/huggingface:/root/.cache/huggingface deploy: resources: reservations: devices: - driver: nvidia count: 1 capabilities: [gpu] open-notebook: image: lfnovo/open_notebook:v1-latest-single pull_policy: always depends_on: - vllm ``` Then in **Settings → API Keys**, add an **OpenAI-Compatible** credential with base URL: `http://vllm:8000/v1` --- ## Adding Models in Open Notebook ### Via Settings UI 1. Go to **Settings** → **Models** 2. Click **Add Model** in appropriate section 3. Select **Provider**: `openai_compatible` 4. Enter **Model Name**: exactly as the server expects 5. Enter **Display Name**: your preferred name 6. Click **Save** ### Model Name Format The model name must match what your server expects: | Server | Model Name Format | |--------|-------------------| | LM Studio | As shown in LM Studio UI | | vLLM | HuggingFace model path | | Text Gen WebUI | As loaded in UI | | llama.cpp | Model file name | --- ## Testing Connection ### Test API Endpoint ```bash # Test chat completions curl http://localhost:1234/v1/chat/completions \ -H "Content-Type: application/json" \ -d '{ "model": "your-model-name", "messages": [{"role": "user", "content": "Hello"}] }' ``` ### Test from Inside Docker ```bash docker exec -it open-notebook curl http://host.docker.internal:1234/v1/models ``` --- ## Troubleshooting ### Connection Refused ``` Problem: Cannot connect to server Solutions: 1. Verify server is running 2. Check port is correct 3. Test with curl directly 4. Check Docker networking (use host.docker.internal) 5. Verify firewall allows connection ``` ### Model Not Found ``` Problem: Server returns "model not found" Solutions: 1. Check model is loaded in server 2. Verify exact model name spelling 3. List available models: curl http://localhost:1234/v1/models 4. Update model name in Open Notebook ``` ### Slow Responses ``` Problem: Requests take very long Solutions: 1. Check server resources (RAM, GPU) 2. Use smaller/quantized model 3. Reduce context length 4. Enable GPU acceleration if available ``` ### Authentication Errors ``` Problem: 401 or authentication failed Solutions: 1. Check if server requires API key 2. Set the API key in your credential (Settings → API Keys) 3. Some servers need any non-empty key (use a placeholder like "not-needed") ``` ### Timeout Errors ``` Problem: Request times out Solutions: 1. Model may be loading (first request slow) 2. Increase timeout settings 3. Check server logs for errors 4. Reduce request size ``` --- ## Multiple Compatible Endpoints You can use different compatible servers for different purposes. When adding an **OpenAI-Compatible** credential in **Settings → API Keys**, you can configure per-service URLs: - **LLM URL**: e.g., `http://localhost:1234/v1` (LM Studio) - **Embedding URL**: e.g., `http://localhost:8080/v1` (different server) - **TTS URL**: e.g., `http://localhost:8969/v1` (Speaches) - **STT URL**: e.g., `http://localhost:9000/v1` (Speaches) Alternatively, add each as a separate credential with its own base URL. --- ## Performance Tips ### Model Selection | Model Size | RAM Needed | Speed | |------------|------------|-------| | 7B | 8GB | Fast | | 13B | 16GB | Medium | | 70B | 64GB+ | Slow | ### Quantization Use quantized models (Q4, Q5) for faster inference with less RAM: ``` llama-3-8b-q4_k_m.gguf → ~4GB RAM, fast llama-3-8b-f16.gguf → ~16GB RAM, slower ``` ### GPU Acceleration Enable GPU in your server for much faster inference: - LM Studio: Settings → GPU layers - vLLM: Automatic with CUDA - llama.cpp: `--n-gpu-layers 35` --- ## Comparison: Native vs Compatible | Aspect | Native Provider | OpenAI Compatible | |--------|-----------------|-------------------| | **Setup** | API key only | Server + configuration | | **Models** | Provider's models | Any compatible model | | **Cost** | Pay per token | Free (local) | | **Speed** | Usually fast | Depends on hardware | | **Features** | Full support | Basic features | Use OpenAI-compatible when: - Running local models - Using custom/fine-tuned models - Privacy requirements - Cost control --- ## Related - **[Local TTS Setup](local-tts.md)** - Text-to-speech with Speaches - **[Local STT Setup](local-stt.md)** - Speech-to-text with Speaches - **[AI Providers](ai-providers.md)** - All provider options - **[Ollama Setup](ollama.md)** - Native Ollama integration ================================================ FILE: docs/5-CONFIGURATION/reverse-proxy.md ================================================ # Reverse Proxy Configuration Deploy Open Notebook behind nginx, Caddy, Traefik, or other reverse proxies with custom domains and HTTPS. --- ## Simplified Setup (v1.1+) Starting with v1.1, Open Notebook uses Next.js rewrites to simplify configuration. **You only need to proxy to one port** - Next.js handles internal API routing automatically. ### How It Works ``` Browser → Reverse Proxy → Port 8502 (Next.js) ↓ (internal proxy) Port 5055 (FastAPI) ``` Next.js automatically forwards `/api/*` requests to the FastAPI backend, so your reverse proxy only needs one port! --- ## Quick Configuration Examples ### Nginx (Recommended) ```nginx server { listen 443 ssl http2; server_name notebook.example.com; ssl_certificate /etc/nginx/ssl/fullchain.pem; ssl_certificate_key /etc/nginx/ssl/privkey.pem; # Allow file uploads up to 100MB client_max_body_size 100M; # Single location block - that's it! location / { proxy_pass http://open-notebook:8502; proxy_http_version 1.1; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; proxy_set_header Upgrade $http_upgrade; proxy_set_header Connection 'upgrade'; proxy_cache_bypass $http_upgrade; } } # HTTP to HTTPS redirect server { listen 80; server_name notebook.example.com; return 301 https://$server_name$request_uri; } ``` ### Caddy ```caddy notebook.example.com { reverse_proxy open-notebook:8502 { transport http { read_timeout 600s write_timeout 600s } } } ``` Caddy handles HTTPS automatically. The timeout settings ensure long-running operations (transformations, podcast generation) don't fail. ### Traefik ```yaml services: open-notebook: image: lfnovo/open_notebook:v1-latest-single pull_policy: always environment: - API_URL=https://notebook.example.com labels: - "traefik.enable=true" - "traefik.http.routers.notebook.rule=Host(`notebook.example.com`)" - "traefik.http.routers.notebook.entrypoints=websecure" - "traefik.http.routers.notebook.tls.certresolver=myresolver" - "traefik.http.services.notebook.loadbalancer.server.port=8502" # Timeout for long-running operations (transformations, podcasts) - "traefik.http.services.notebook.loadbalancer.responseforwarding.flushinterval=100ms" networks: - traefik-network ``` **Note**: For Traefik v2+, you may also need to configure `serversTransport` timeouts in your static configuration: ```yaml # traefik.yml (static configuration) serversTransport: forwardingTimeouts: dialTimeout: 30s responseHeaderTimeout: 600s idleConnTimeout: 90s ``` ### Coolify 1. Create new service with `lfnovo/open_notebook:v1-latest-single` 2. Set port to **8502** 3. Add environment: `API_URL=https://your-domain.com` 4. Enable HTTPS in Coolify 5. Done! --- ## Environment Variables ```bash # Required for reverse proxy setups API_URL=https://your-domain.com # Optional: For multi-container deployments # INTERNAL_API_URL=http://api-service:5055 ``` **Important**: Set `API_URL` to your public URL (with https://). **Note on HOSTNAME**: The Docker images set `HOSTNAME=0.0.0.0` by default, which ensures Next.js binds to all interfaces and is accessible from reverse proxies. You typically don't need to set this manually. --- ## Understanding API_URL The frontend uses a three-tier priority system to determine the API URL: 1. **Runtime Configuration** (Highest Priority): `API_URL` environment variable set at container runtime 2. **Build-time Configuration**: `NEXT_PUBLIC_API_URL` baked into the Docker image 3. **Auto-detection** (Fallback): Infers from the incoming HTTP request headers ### Auto-Detection Details When `API_URL` is not set, the Next.js frontend: - Analyzes the incoming HTTP request - Extracts the hostname from the `host` header - Respects the `X-Forwarded-Proto` header (for HTTPS behind reverse proxies) - Constructs the API URL as `{protocol}://{hostname}:5055` - Example: Request to `http://10.20.30.20:8502` → API URL becomes `http://10.20.30.20:5055` **Why set API_URL explicitly?** - **Reliability**: Auto-detection can fail with complex proxy setups - **HTTPS**: Ensures frontend uses `https://` when behind SSL-terminating proxy - **Custom domains**: Works correctly with domain names instead of IP addresses - **Port mapping**: Avoids exposing port 5055 in the URL when using reverse proxy **Important**: Don't include `/api` at the end - the system adds this automatically! --- ## Complete Docker Compose Example ```yaml services: open-notebook: image: lfnovo/open_notebook:v1-latest-single pull_policy: always container_name: open-notebook environment: - API_URL=https://notebook.example.com - OPEN_NOTEBOOK_ENCRYPTION_KEY=${OPEN_NOTEBOOK_ENCRYPTION_KEY} - OPEN_NOTEBOOK_PASSWORD=${OPEN_NOTEBOOK_PASSWORD} volumes: - ./notebook_data:/app/data - ./surreal_data:/mydata # Only expose to localhost (nginx handles public access) ports: - "127.0.0.1:8502:8502" restart: unless-stopped nginx: image: nginx:alpine container_name: nginx-proxy ports: - "80:80" - "443:443" volumes: - ./nginx.conf:/etc/nginx/nginx.conf:ro - ./ssl:/etc/nginx/ssl:ro depends_on: - open-notebook restart: unless-stopped ``` --- ## Full Nginx Configuration ```nginx events { worker_connections 1024; } http { upstream notebook { server open-notebook:8502; } # HTTP redirect server { listen 80; server_name notebook.example.com; return 301 https://$server_name$request_uri; } # HTTPS server server { listen 443 ssl http2; server_name notebook.example.com; ssl_certificate /etc/nginx/ssl/fullchain.pem; ssl_certificate_key /etc/nginx/ssl/privkey.pem; ssl_protocols TLSv1.2 TLSv1.3; ssl_ciphers HIGH:!aNULL:!MD5; # Allow file uploads up to 100MB client_max_body_size 100M; # Security headers add_header X-Frame-Options DENY; add_header X-Content-Type-Options nosniff; add_header X-XSS-Protection "1; mode=block"; add_header Strict-Transport-Security "max-age=31536000; includeSubDomains"; # Proxy settings location / { proxy_pass http://notebook; proxy_http_version 1.1; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; proxy_set_header Upgrade $http_upgrade; proxy_set_header Connection 'upgrade'; proxy_cache_bypass $http_upgrade; # Timeouts for long-running operations (transformations, podcasts, etc.) # 600s matches the frontend timeout for slow LLM operations proxy_read_timeout 600s; proxy_connect_timeout 60s; proxy_send_timeout 600s; } } } ``` --- ## Direct API Access (Optional) If external scripts or integrations need direct API access, route `/api/*` directly: ```nginx # Direct API access (for external integrations) location /api/ { proxy_pass http://open-notebook:5055/api/; proxy_http_version 1.1; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; } # Frontend (handles all other traffic) location / { proxy_pass http://open-notebook:8502; # ... same headers as above } ``` **Note**: This is only needed for external API integrations. Browser traffic works fine with single-port setup. --- ## Advanced Scenarios ### Remote Server Access (LAN/VPS) Accessing Open Notebook from a different machine on your network: **Step 1: Get your server IP** ```bash # On the server running Open Notebook: hostname -I # or ifconfig | grep "inet " # Note the IP (e.g., 192.168.1.100) ``` **Step 2: Configure API_URL** ```bash # In docker-compose.yml or .env: API_URL=http://192.168.1.100:5055 ``` **Step 3: Expose ports** ```yaml services: open-notebook: image: lfnovo/open_notebook:v1-latest-single pull_policy: always environment: - API_URL=http://192.168.1.100:5055 ports: - "8502:8502" - "5055:5055" ``` **Step 4: Access from client machine** ```bash # In browser on other machine: http://192.168.1.100:8502 ``` **Troubleshooting**: - Check firewall: `sudo ufw allow 8502 && sudo ufw allow 5055` - Verify connectivity: `ping 192.168.1.100` from client machine - Test port: `telnet 192.168.1.100 8502` from client machine --- ### API on Separate Subdomain Host the API and frontend on different subdomains: **docker-compose.yml:** ```yaml services: open-notebook: image: lfnovo/open_notebook:v1-latest-single pull_policy: always environment: - API_URL=https://api.notebook.example.com - OPEN_NOTEBOOK_ENCRYPTION_KEY=${OPEN_NOTEBOOK_ENCRYPTION_KEY} # Don't expose ports (nginx handles routing) ``` **nginx.conf:** ```nginx # Frontend server server { listen 443 ssl http2; server_name notebook.example.com; ssl_certificate /etc/nginx/ssl/fullchain.pem; ssl_certificate_key /etc/nginx/ssl/privkey.pem; location / { proxy_pass http://open-notebook:8502; proxy_http_version 1.1; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; proxy_set_header Upgrade $http_upgrade; proxy_set_header Connection 'upgrade'; proxy_cache_bypass $http_upgrade; } } # API server (separate subdomain) server { listen 443 ssl http2; server_name api.notebook.example.com; ssl_certificate /etc/nginx/ssl/fullchain.pem; ssl_certificate_key /etc/nginx/ssl/privkey.pem; location / { proxy_pass http://open-notebook:5055; proxy_http_version 1.1; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; } } ``` **Use case**: Separate DNS records, different rate limiting, or isolated API access control. --- ### Multi-Container Deployment (Advanced) For complex deployments with separate frontend and API containers: **docker-compose.yml:** ```yaml services: frontend: image: lfnovo/open_notebook_frontend:v1-latest pull_policy: always environment: - API_URL=https://notebook.example.com ports: - "8502:8502" api: image: lfnovo/open_notebook_api:v1-latest pull_policy: always environment: - OPEN_NOTEBOOK_ENCRYPTION_KEY=${OPEN_NOTEBOOK_ENCRYPTION_KEY} ports: - "5055:5055" depends_on: - surrealdb surrealdb: image: surrealdb/surrealdb:latest command: start --log trace --user root --pass root file:/mydata/database.db ports: - "8000:8000" volumes: - ./surreal_data:/mydata ``` **nginx.conf:** ```nginx http { upstream frontend { server frontend:8502; } upstream api { server api:5055; } server { listen 443 ssl http2; server_name notebook.example.com; # API routes location /api/ { proxy_pass http://api/api/; proxy_http_version 1.1; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; } # Frontend (catch-all) location / { proxy_pass http://frontend; proxy_http_version 1.1; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; proxy_set_header Upgrade $http_upgrade; proxy_set_header Connection 'upgrade'; proxy_cache_bypass $http_upgrade; } } } ``` **Note**: Most users should use the single-container approach (`v1-latest-single`). Multi-container is only needed for custom scaling or isolation requirements. --- ## SSL Certificates ### Let's Encrypt with Certbot ```bash # Install certbot sudo apt install certbot python3-certbot-nginx # Get certificate sudo certbot --nginx -d notebook.example.com # Auto-renewal (usually configured automatically) sudo certbot renew --dry-run ``` ### Let's Encrypt with Caddy Caddy handles SSL automatically - no configuration needed! ### Self-Signed (Development Only) ```bash openssl req -x509 -nodes -days 365 -newkey rsa:2048 \ -keyout ssl/privkey.pem \ -out ssl/fullchain.pem \ -subj "/CN=localhost" ``` --- ## Troubleshooting ### "Unable to connect to server" 1. **Check API_URL is set**: ```bash docker exec open-notebook env | grep API_URL ``` 2. **Verify reverse proxy reaches container**: ```bash curl -I http://localhost:8502 ``` 3. **Check browser console** (F12): - Look for connection errors - Check what URL it's trying to reach ### Mixed Content Errors Frontend using HTTPS but trying to reach HTTP API: ```bash # Ensure API_URL uses https:// API_URL=https://notebook.example.com # Not http:// ``` ### WebSocket Issues Ensure your proxy supports WebSocket upgrades: ```nginx proxy_http_version 1.1; proxy_set_header Upgrade $http_upgrade; proxy_set_header Connection 'upgrade'; ``` ### 502 Bad Gateway 1. Check container is running: `docker ps` 2. Check container logs: `docker logs open-notebook` 3. Verify nginx can reach container (same network) ### Timeout Errors **Symptoms:** - `socket hang up` or `ECONNRESET` errors - `Timeout after 30000ms` errors - Operations fail after exactly 30 seconds **Cause:** Your reverse proxy has a default timeout (often 30s) that's shorter than Open Notebook's operations. **Solutions by proxy:** **Nginx:** ```nginx proxy_read_timeout 600s; proxy_send_timeout 600s; ``` **Caddy:** ```caddy reverse_proxy open-notebook:8502 { transport http { read_timeout 600s write_timeout 600s } } ``` **Traefik (static config):** ```yaml serversTransport: forwardingTimeouts: responseHeaderTimeout: 600s ``` **Application-level timeouts:** If you still experience timeouts after configuring your proxy, you can also adjust the application timeouts: ```bash # In .env file: API_CLIENT_TIMEOUT=600 # API client timeout (default: 300s) ESPERANTO_LLM_TIMEOUT=180 # LLM inference timeout (default: 60s) ``` See [Advanced Configuration](advanced.md) for more timeout options. --- ### How to Debug Configuration Issues **Step 1: Check browser console** (F12 → Console tab) ``` Look for messages starting with 🔧 [Config] These show the configuration detection process You'll see which API URL is being used ``` **Example good output:** ``` ✅ [Config] Runtime API URL from server: https://your-domain.com ``` **Example bad output:** ``` ❌ [Config] Failed to fetch runtime config ⚠️ [Config] Using auto-detected URL: http://localhost:5055 ``` **Step 2: Test API directly** ```bash # Should return JSON config curl https://your-domain.com/api/config # Expected output: {"status":"ok","credentials_configured":true,...} ``` **Step 3: Check Docker logs** ```bash docker logs open-notebook # Look for: # - Frontend startup: "▲ Next.js ready on http://0.0.0.0:8502" # - API startup: "INFO: Uvicorn running on http://0.0.0.0:5055" # - Connection errors or CORS issues ``` **Step 4: Verify environment variable** ```bash docker exec open-notebook env | grep API_URL # Should show: # API_URL=https://your-domain.com ``` --- ### Frontend Adds `:5055` to URL (Versions ≤ 1.0.10) **Symptoms** (only in older versions): - You set `API_URL=https://your-domain.com` - Browser console shows: "Attempted URL: https://your-domain.com:5055/api/config" - CORS errors with "Status code: (null)" **Root Cause:** In versions ≤ 1.0.10, the frontend's config endpoint was at `/api/runtime-config`, which got intercepted by reverse proxies routing all `/api/*` requests to the backend. This prevented the frontend from reading the `API_URL` environment variable. **Solution:** Upgrade to version 1.0.11 or later. The config endpoint has been moved to `/config` which avoids the `/api/*` routing conflict. **Verification:** Check browser console (F12) - should see: `✅ [Config] Runtime API URL from server: https://your-domain.com` **If you can't upgrade**, explicitly configure the `/config` route: ```nginx # Only needed for versions ≤ 1.0.10 location = /config { proxy_pass http://open-notebook:8502; proxy_http_version 1.1; proxy_set_header Host $host; proxy_set_header X-Forwarded-Proto $scheme; } ``` --- ### File Upload Errors (413 Payload Too Large) **Symptoms:** ``` CORS header 'Access-Control-Allow-Origin' missing. Status code: 413. Error creating source. Please try again. ``` **Root Cause:** When uploading files, your reverse proxy may reject the request due to body size limits *before* it reaches the application. Since the error happens at the proxy level, CORS headers are not included in the response. **Version Requirement:** - **Open Notebook v1.3.2+** is required for file uploads >10MB - Uses Next.js 16+ which supports the `proxyClientMaxBodySize` configuration option - Check your version: Settings → About (bottom of settings page) **Solutions:** 1. **Nginx - Increase body size limit**: ```nginx server { # Allow larger file uploads (default is 1MB) client_max_body_size 100M; # Add CORS headers to error responses error_page 413 = @cors_error_413; location @cors_error_413 { add_header 'Access-Control-Allow-Origin' '*' always; add_header 'Access-Control-Allow-Methods' 'GET, POST, PUT, DELETE, OPTIONS' always; add_header 'Access-Control-Allow-Headers' '*' always; return 413 '{"detail": "File too large. Maximum size is 100MB."}'; } location / { # ... your existing proxy configuration } } ``` 2. **Traefik - Increase buffer size**: ```yaml # In your traefik configuration http: middlewares: large-body: buffering: maxRequestBodyBytes: 104857600 # 100MB ``` Apply middleware to your router: ```yaml labels: - "traefik.http.routers.notebook.middlewares=large-body" ``` 3. **Kubernetes Ingress (nginx-ingress)**: ```yaml apiVersion: networking.k8s.io/v1 kind: Ingress metadata: name: open-notebook annotations: nginx.ingress.kubernetes.io/proxy-body-size: "100m" # Add CORS headers for error responses nginx.ingress.kubernetes.io/configuration-snippet: | more_set_headers "Access-Control-Allow-Origin: *"; ``` 4. **Caddy**: ```caddy notebook.example.com { request_body { max_size 100MB } reverse_proxy open-notebook:8502 { transport http { read_timeout 600s write_timeout 600s } } } ``` **Note:** Open Notebook's API includes CORS headers in error responses, but this only works for errors that reach the application. Proxy-level errors (like 413 from nginx) need to be configured at the proxy level. --- ### CORS Errors **Symptoms:** ``` Access-Control-Allow-Origin header is missing Cross-Origin Request Blocked Response to preflight request doesn't pass access control check ``` **Possible Causes:** 1. **Missing proxy headers**: ```nginx # Make sure these are set: proxy_set_header X-Forwarded-Proto $scheme; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header Host $host; ``` 2. **API_URL protocol mismatch**: ```bash # Frontend is HTTPS, but API_URL is HTTP: API_URL=http://notebook.example.com # ❌ Wrong API_URL=https://notebook.example.com # ✅ Correct ``` 3. **Reverse proxy not forwarding `/api/*` correctly**: ```nginx # Make sure this works: location /api/ { proxy_pass http://open-notebook:5055/api/; # Note the trailing slash! } ``` --- ### Missing Authorization Header **Symptoms:** ```json {"detail": "Missing authorization header"} ``` This happens when: - You have set `OPEN_NOTEBOOK_PASSWORD` for authentication - You're trying to access `/api/config` directly without logging in first **Solution:** This is **expected behavior**! The frontend handles authentication automatically. Just: 1. Access the frontend URL (not `/api/` directly) 2. Log in through the UI 3. The frontend will handle authorization headers for all API calls **For API integrations:** Include the password in the Authorization header: ```bash curl -H "Authorization: Bearer your-password-here" \ https://your-domain.com/api/config ``` --- ### SSL/TLS Certificate Errors **Symptoms:** - Browser shows "Your connection is not private" - Certificate warnings - Mixed content errors **Solutions:** 1. **Use Let's Encrypt** (recommended): ```bash sudo certbot --nginx -d notebook.example.com ``` 2. **Check certificate paths** in nginx: ```nginx ssl_certificate /etc/nginx/ssl/fullchain.pem; # Full chain ssl_certificate_key /etc/nginx/ssl/privkey.pem; # Private key ``` 3. **Verify certificate is valid**: ```bash openssl x509 -in /etc/nginx/ssl/fullchain.pem -text -noout ``` 4. **For development**, use self-signed (not for production): ```bash openssl req -x509 -nodes -days 365 -newkey rsa:2048 \ -keyout ssl/privkey.pem -out ssl/fullchain.pem \ -subj "/CN=localhost" ``` --- ## Best Practices 1. **Always use HTTPS** in production 2. **Set API_URL explicitly** when using reverse proxies to avoid auto-detection issues 3. **Bind to localhost** (`127.0.0.1:8502`) and let proxy handle public access for security 4. **Enable security headers** (HSTS, X-Frame-Options, X-Content-Type-Options, X-XSS-Protection) 5. **Set up certificate renewal** for Let's Encrypt (usually automatic with certbot) 6. **Keep ports 5055 and 8502 accessible** from your reverse proxy container (use Docker networks) 7. **Use environment files** (`.env` or `docker.env`) to manage configuration securely 8. **Test your configuration** before going live: - Check browser console for config messages - Test API: `curl https://your-domain.com/api/config` - Verify authentication works - Check long-running operations (podcast generation) 9. **Monitor logs** regularly: `docker logs open-notebook` 10. **Don't include `/api` in API_URL** - the system adds this automatically --- ## Legacy Configurations (Pre-v1.1) If you're running Open Notebook **version 1.0.x or earlier**, you may need to use the legacy two-port configuration where you explicitly route `/api/*` to port 5055. **Check your version:** ```bash docker exec open-notebook cat /app/package.json | grep version ``` **If version < 1.1.0**, you may need: - Explicit `/api/*` routing to port 5055 in reverse proxy - Explicit `/config` endpoint routing for versions ≤ 1.0.10 - See the "Frontend Adds `:5055` to URL" troubleshooting section above **Recommendation:** Upgrade to v1.1+ for simplified configuration and better performance. --- ## Related - **[Security Configuration](security.md)** - Password protection and hardening - **[Advanced Configuration](advanced.md)** - Ports, timeouts, and SSL settings - **[Troubleshooting](../6-TROUBLESHOOTING/connection-issues.md)** - Connection problems - **[Docker Deployment](../1-INSTALLATION/docker-compose.md)** - Complete deployment guide ================================================ FILE: docs/5-CONFIGURATION/security.md ================================================ # Security Configuration Protect your Open Notebook deployment with password authentication and production hardening. --- ## API Key Encryption Open Notebook encrypts API keys stored in the database using Fernet symmetric encryption (AES-128-CBC with HMAC-SHA256). ### Configuration Methods | Method | Documentation | |--------|---------------| | **Settings UI** | [API Configuration Guide](../3-USER-GUIDE/api-configuration.md) | | **Environment Variables** | This page (below) | ### Setup Set the encryption key to any secret string: ```bash # .env or docker.env OPEN_NOTEBOOK_ENCRYPTION_KEY=my-secret-passphrase ``` Any string works — it will be securely derived via SHA-256 internally. Use a strong passphrase for production deployments. ### Default Credentials | Setting | Default | Security Level | |---------|---------|----------------| | Password | `open-notebook-change-me` | Development only | | Encryption Key | **None** (must be configured) | Required for API key storage | **The encryption key has no default.** You must set `OPEN_NOTEBOOK_ENCRYPTION_KEY` before using the API key configuration feature. Without it, encrypting/decrypting API keys will fail. ### Docker Secrets Support Both settings support Docker secrets via `_FILE` suffix: ```yaml environment: - OPEN_NOTEBOOK_PASSWORD_FILE=/run/secrets/app_password - OPEN_NOTEBOOK_ENCRYPTION_KEY_FILE=/run/secrets/encryption_key ``` ### Security Notes | Scenario | Behavior | |----------|----------| | Key configured | API keys encrypted with your key | | No key configured | Encryption/decryption will fail (key is required) | | Key changed | Old encrypted keys become unreadable | | Legacy data | Unencrypted keys still work (graceful fallback) | ### Key Management - **Keep secret**: Never commit the encryption key to version control - **Backup securely**: Store the key separately from database backups - **No rotation yet**: Changing the key requires re-saving all API keys - **Per-deployment**: Each instance should have its own encryption key --- ## When to Use Password Protection ### Use it for: - Public cloud deployments (PikaPods, Railway, DigitalOcean) - Shared network environments - Any deployment accessible beyond localhost ### You can skip it for: - Local development on your machine - Private, isolated networks - Single-user local setups --- ## Quick Setup ### Docker Deployment ```yaml # docker-compose.yml services: open_notebook: image: lfnovo/open_notebook:v1-latest-single pull_policy: always environment: - OPEN_NOTEBOOK_ENCRYPTION_KEY=your-secret-encryption-key - OPEN_NOTEBOOK_PASSWORD=your_secure_password # ... rest of config ``` Or using environment file: ```bash # docker.env OPEN_NOTEBOOK_ENCRYPTION_KEY=your-secret-encryption-key OPEN_NOTEBOOK_PASSWORD=your_secure_password ``` > **Important**: The encryption key is **required** for credential storage. Without it, you cannot save AI provider credentials via the Settings UI. If you change or lose the encryption key, all stored credentials become unreadable. ### Development Setup ```bash # .env OPEN_NOTEBOOK_PASSWORD=your_secure_password ``` --- ## Password Requirements ### Good Passwords ```bash # Strong: 20+ characters, mixed case, numbers, symbols OPEN_NOTEBOOK_PASSWORD=MySecure2024!Research#Tool OPEN_NOTEBOOK_PASSWORD=Notebook$Dev$2024$Strong! # Generated (recommended) OPEN_NOTEBOOK_PASSWORD=$(openssl rand -base64 24) ``` ### Bad Passwords ```bash # DON'T use these OPEN_NOTEBOOK_PASSWORD=password123 OPEN_NOTEBOOK_PASSWORD=opennotebook OPEN_NOTEBOOK_PASSWORD=admin ``` --- ## How It Works ### Frontend Protection 1. Login form appears on first visit 2. Password stored in browser session 3. Session persists until browser closes 4. Clear browser data to log out ### API Protection All API endpoints require authentication: ```bash # Authenticated request curl -H "Authorization: Bearer your_password" \ http://localhost:5055/api/notebooks # Unauthenticated (will fail) curl http://localhost:5055/api/notebooks # Returns: {"detail": "Missing authorization header"} ``` ### Unprotected Endpoints These work without authentication: - `/health` - System health check - `/docs` - API documentation - `/openapi.json` - OpenAPI spec --- ## API Authentication Examples ### curl ```bash # List notebooks curl -H "Authorization: Bearer your_password" \ http://localhost:5055/api/notebooks # Create notebook curl -X POST \ -H "Authorization: Bearer your_password" \ -H "Content-Type: application/json" \ -d '{"name": "My Notebook", "description": "Research notes"}' \ http://localhost:5055/api/notebooks # Upload file curl -X POST \ -H "Authorization: Bearer your_password" \ -F "file=@document.pdf" \ http://localhost:5055/api/sources/upload ``` ### Python ```python import requests class OpenNotebookClient: def __init__(self, base_url: str, password: str): self.base_url = base_url self.headers = {"Authorization": f"Bearer {password}"} def get_notebooks(self): response = requests.get( f"{self.base_url}/api/notebooks", headers=self.headers ) return response.json() def create_notebook(self, name: str, description: str = None): response = requests.post( f"{self.base_url}/api/notebooks", headers=self.headers, json={"name": name, "description": description} ) return response.json() # Usage client = OpenNotebookClient("http://localhost:5055", "your_password") notebooks = client.get_notebooks() ``` ### JavaScript/TypeScript ```javascript const API_URL = 'http://localhost:5055'; const PASSWORD = 'your_password'; async function getNotebooks() { const response = await fetch(`${API_URL}/api/notebooks`, { headers: { 'Authorization': `Bearer ${PASSWORD}` } }); return response.json(); } ``` --- ## Production Hardening ### Docker Security ```yaml services: open_notebook: image: lfnovo/open_notebook:v1-latest-single pull_policy: always ports: - "127.0.0.1:8502:8502" # Bind to localhost only environment: - OPEN_NOTEBOOK_PASSWORD=your_secure_password security_opt: - no-new-privileges:true deploy: resources: limits: memory: 2G cpus: "1.0" restart: always ``` ### Firewall Configuration ```bash # UFW (Ubuntu) sudo ufw allow ssh sudo ufw allow 80/tcp sudo ufw allow 443/tcp sudo ufw deny 8502/tcp # Block direct access sudo ufw deny 5055/tcp # Block direct API access sudo ufw enable # iptables iptables -A INPUT -p tcp --dport 22 -j ACCEPT iptables -A INPUT -p tcp --dport 80 -j ACCEPT iptables -A INPUT -p tcp --dport 443 -j ACCEPT iptables -A INPUT -p tcp --dport 8502 -j DROP iptables -A INPUT -p tcp --dport 5055 -j DROP ``` ### Reverse Proxy with SSL See [Reverse Proxy Configuration](reverse-proxy.md) for complete nginx/Caddy/Traefik setup with HTTPS. --- ## Security Limitations Open Notebook's password protection provides **basic access control**, not enterprise-grade security: | Feature | Status | |---------|--------| | Password transmission | Plain text (use HTTPS!) | | Password storage | In memory | | User management | Single password for all | | Session timeout | None (until browser close) | | Rate limiting | None | | Audit logging | None | ### Risk Mitigation 1. **Always use HTTPS** - Encrypt traffic with TLS 2. **Strong passwords** - 20+ characters, complex 3. **Network security** - Firewall, VPN for sensitive deployments 4. **Regular updates** - Keep containers and dependencies updated 5. **Monitoring** - Check logs for suspicious activity 6. **Backups** - Regular backups of data --- ## Enterprise Considerations For deployments requiring advanced security: | Need | Solution | |------|----------| | SSO/OAuth | Implement OAuth2/SAML proxy | | Role-based access | Custom middleware | | Audit logging | Log aggregation service | | Rate limiting | API gateway or nginx | | Data encryption | Encrypt volumes at rest | | Network segmentation | Docker networks, VPC | --- ## Troubleshooting ### Password Not Working ```bash # Check env var is set docker exec open-notebook env | grep OPEN_NOTEBOOK_PASSWORD # Check logs docker logs open-notebook | grep -i auth # Test API directly curl -H "Authorization: Bearer your_password" \ http://localhost:5055/health ``` ### 401 Unauthorized Errors ```bash # Check header format curl -v -H "Authorization: Bearer your_password" \ http://localhost:5055/api/notebooks # Verify password matches echo "Password length: $(echo -n $OPEN_NOTEBOOK_PASSWORD | wc -c)" ``` ### Cannot Access After Setting Password 1. Clear browser cache and cookies 2. Try incognito/private mode 3. Check browser console for errors 4. Verify password is correct in environment ### Security Testing ```bash # Without password (should fail) curl http://localhost:5055/api/notebooks # Expected: {"detail": "Missing authorization header"} # With correct password (should succeed) curl -H "Authorization: Bearer your_password" \ http://localhost:5055/api/notebooks # Health check (should work without password) curl http://localhost:5055/health ``` --- ## Reporting Security Issues If you discover security vulnerabilities: 1. **Do NOT open public issues** 2. Contact maintainers directly 3. Provide detailed information 4. Allow time for fixes before disclosure --- ## Related - **[Reverse Proxy](reverse-proxy.md)** - HTTPS and SSL setup - **[Advanced Configuration](advanced.md)** - Ports, timeouts, and SSL settings - **[Environment Reference](environment-reference.md)** - All configuration options ================================================ FILE: docs/6-TROUBLESHOOTING/ai-chat-issues.md ================================================ # AI & Chat Issues - Model Configuration & Quality Problems with AI models, chat, and response quality. > **Note:** Open Notebook now shows descriptive error messages for AI provider failures. Instead of a generic "An unexpected error occurred", you'll see specific messages like "Authentication failed. Please check your API key" or "Rate limit exceeded. Please wait a moment and try again." These messages help you diagnose and fix issues faster. --- ## "Failed to send message" Error **Symptom:** Chat shows "Failed to send message" toast. Logs show: ``` Error executing chat: Model is not a LanguageModel: None ``` **Cause:** No valid language model configured for chat **Solutions:** ### Solution 1: Check Default Model Configuration ``` 1. Go to Settings → Models 2. Scroll to "Default Models" section 3. Verify "Default Chat Model" has a model selected 4. If empty, select an available language model 5. Click Save ``` ### Solution 2: Verify Model Names (Ollama Users) ```bash # Get exact model names ollama list # Example output: # NAME SIZE MODIFIED # gemma3:12b 8.1 GB 2 months ago # The model name in Open Notebook must be EXACTLY "gemma3:12b" # NOT "gemma3" or "gemma3-12b" ``` ### Solution 3: Re-add Missing Models ``` 1. Note the exact model names from your provider 2. Go to Settings → Models 3. Delete any misconfigured models 4. Add models with exact names 5. Set new defaults ``` ### Solution 4: Check Model Still Exists ```bash # For Ollama: verify model is installed ollama list # For cloud providers: verify API key is valid # and you have access to the model ``` > **Tip:** This error often occurs when you delete a model from Ollama but forget to update the default models in Open Notebook. Always re-configure defaults after removing models. --- ## "Models not available" or "Models not showing" **Symptom:** Settings → Models shows empty, or "No models configured" **Cause:** No credential configured, or credential has invalid API key **Solutions:** ### Solution 1: Add Credential via Settings UI ``` 1. Go to Settings → API Keys 2. Click "Add Credential" 3. Select your provider (e.g., OpenAI, Anthropic, Google) 4. Enter your API key 5. Click Save, then Test Connection 6. Click Discover Models → Register Models 7. Go to Settings → Models to verify ``` ### Solution 2: Check Key is Valid ``` 1. Go to Settings → API Keys 2. Click "Test Connection" on your credential 3. If it shows "Invalid API key": - Get a fresh key from the provider's website - Delete the credential and create a new one ``` ### Solution 3: Switch Provider ``` 1. Go to Settings → API Keys 2. Add a credential for a different provider 3. Test Connection → Discover Models → Register Models 4. Go to Settings → Models to select the new provider's models ``` --- ## "Invalid API key" or "Unauthorized" **Symptom:** Error when trying to chat: "Invalid API key" **Cause:** Credential has wrong, expired, or revoked API key **Solutions:** ### Step 1: Test Your Credential ``` 1. Go to Settings → API Keys 2. Click "Test Connection" on your credential 3. If it fails, proceed to Step 2 ``` ### Step 2: Get Fresh Key ``` Go to provider's dashboard: - OpenAI: https://platform.openai.com/api-keys (starts with sk-proj-) - Anthropic: https://console.anthropic.com/ (starts with sk-ant-) - Google: https://aistudio.google.com/app/apikey (starts with AIzaSy) Generate new key and copy exactly (no extra spaces) ``` ### Step 3: Update Credential ``` 1. Go to Settings → API Keys 2. Delete the old credential 3. Click "Add Credential" → select provider 4. Paste the new key 5. Click Save, then Test Connection 6. Re-discover and register models if needed ``` ### Step 4: Verify in UI ``` 1. Go to Settings → Models 2. Verify models are available 3. Try a test chat ``` --- ## Chat Returns Generic/Bad Responses **Symptom:** AI responses are shallow, generic, or wrong **Cause:** Bad context, vague question, or wrong model **Solutions:** ### Solution 1: Check Context ``` 1. In Chat, click "Select Sources" 2. Verify sources you want are CHECKED 3. Set them to "Full Content" (not "Summary Only") 4. Click "Save" 5. Try chat again ``` ### Solution 2: Ask Better Question ``` Bad: "What do you think?" Good: "Based on the paper's methodology, what are 3 limitations?" Bad: "Tell me about X" Good: "Summarize X in 3 bullet points with page citations" ``` ### Solution 3: Use Stronger Model ``` OpenAI: Current: gpt-4o-mini → Switch to: gpt-4o Anthropic: Current: claude-3-5-haiku → Switch to: claude-3-5-sonnet To change: 1. Settings → Models 2. Select model 3. Try chat again ``` ### Solution 4: Add More Sources ``` If: "Response seems incomplete" Try: Add more relevant sources to provide context ``` --- ## Chat is Very Slow **Symptom:** Chat responses take minutes **Cause:** Large context, slow model, or overloaded API **Solutions:** ### Solution 1: Use Faster Model ```bash Fastest: Groq (any model) Fast: OpenAI gpt-4o-mini Medium: Anthropic claude-3-5-haiku Slow: Anthropic claude-3-5-sonnet Switch in: Settings → Models ``` ### Solution 2: Reduce Context ``` 1. Chat → Select Sources 2. Uncheck sources you don't need 3. Or switch to "Summary Only" for background sources 4. Save and try again ``` ### Solution 3: Increase Timeout ```bash # In .env: API_CLIENT_TIMEOUT=600 # 10 minutes # Restart: docker compose restart ``` ### Solution 4: Check System Load ```bash # See if API is overloaded: docker stats # If CPU >80% or memory >90%: # Reduce: SURREAL_COMMANDS_MAX_TASKS=2 # Restart: docker compose restart ``` --- ## Chat Doesn't Remember History **Symptom:** Each message treated as separate, no context between questions **Cause:** Chat history not saved or new chat started **Solution:** ``` 1. Make sure you're in same Chat (not new Chat) 2. Check Chat title at top 3. If it's blank, start new Chat with a title 4. Each named Chat keeps its history 5. If you start new Chat, history is separate ``` --- ## "Rate limit exceeded" **Symptom:** Error: "Rate limit exceeded" or "Too many requests" **Cause:** Hit provider's API rate limit **Solutions:** ### For Cloud Providers (OpenAI, Anthropic, etc.) **Immediate:** - Wait 1-2 minutes - Try again **Short term:** - Use cheaper/smaller model - Reduce concurrent operations - Space out requests **Long term:** - Upgrade your account - Switch to different provider - Use Ollama (local, no limits) ### Check Account Status ``` OpenAI: https://platform.openai.com/account/usage/overview Anthropic: https://console.anthropic.com/account/billing/overview Google: Google Cloud Console ``` ### For Ollama (Local) - No rate limits - Use `ollama pull mistral` for best model - Restart if hitting resource limits --- ## "Context length exceeded" or "Token limit" **Symptom:** Error about too many tokens **Cause:** Sources too large for model **Solutions:** ### Solution 1: Use Model with Longer Context ``` Current: GPT-4o (128K tokens) → Switch to: Claude (200K tokens) Current: Claude Haiku (200K) → Switch to: Gemini (1M tokens) To change: Settings → Models ``` ### Solution 2: Reduce Context ``` 1. Select fewer sources 2. Or use "Summary Only" instead of "Full Content" 3. Or split large documents into smaller pieces ``` ### Solution 3: For Ollama (Local) ```bash # Use smaller model: ollama pull phi # Very small # Instead of: ollama pull neural-chat # Large ``` --- ## "API call failed" or Timeout **Symptom:** Generic API error, response times out **Cause:** Provider API down, network issue, or slow service **Solutions:** ### Check Provider Status ``` OpenAI: https://status.openai.com/ Anthropic: Check website Google: Google Cloud Status Groq: Check website ``` ### Retry Operation ``` 1. Wait 30 seconds 2. Try again ``` ### Use Different Model/Provider ``` 1. Settings → Models 2. Try different provider 3. If OpenAI down, use Anthropic ``` ### Check Network ```bash # Verify internet working: ping google.com # Test API endpoint directly: curl https://api.openai.com/v1/models \ -H "Authorization: Bearer YOUR_KEY" ``` --- ## Responses Include Hallucinations **Symptom:** AI makes up facts that aren't in sources **Cause:** Sources not in context, or model guessing **Solutions:** ### Solution 1: Verify Context ``` 1. Click citation in response 2. Check source actually says that 3. If not, sources weren't in context 4. Add source to context and try again ``` ### Solution 2: Request Citations ``` Ask: "Answer this with citations to specific pages" The AI will be more careful if asked for citations ``` ### Solution 3: Use Stronger Model ``` Weaker models hallucinate more Switch to: GPT-4o or Claude Sonnet ``` --- ## High API Costs **Symptom:** API bills are higher than expected **Cause:** Using expensive model, large context, many requests **Solutions:** ### Use Cheaper Model ``` Expensive: gpt-4o Cheaper: gpt-4o-mini (10x cheaper) Expensive: Claude Sonnet Cheaper: Claude Haiku (5x cheaper) Groq: Ultra cheap but fewer models ``` ### Reduce Context ``` In Chat: 1. Select fewer sources 2. Use "Summary Only" for background 3. Ask more specific questions ``` ### Switch to Ollama (Free) ```bash # Install Ollama # Run: ollama serve # Download: ollama pull mistral # Set: OLLAMA_API_BASE=http://localhost:11434 # Cost: Free! ``` --- ## Still Having Chat Issues? - Try [Quick Fixes](quick-fixes.md) - Try [Chat Effectively Guide](../3-USER-GUIDE/chat-effectively.md) - Check logs: `docker compose logs api | grep -i "error"` - Ask for help: [Troubleshooting Index](index.md#getting-help) ================================================ FILE: docs/6-TROUBLESHOOTING/connection-issues.md ================================================ # Connection Issues - Network & API Problems Frontend can't reach API or services won't communicate. --- ## "Cannot connect to server" (Most Common) **What it looks like:** - Browser shows error page - "Unable to reach API" - "Cannot connect to server" - UI loads but can't create notebooks **Diagnosis:** ```bash # Check if API is running docker ps | grep api # Should see "api" service running # Check if API is responding curl http://localhost:5055/health # Should show: {"status":"ok"} # Check if frontend is running docker ps | grep frontend # Should see "frontend" or React service running ``` **Solutions:** ### Solution 1: API Not Running ```bash # Start API docker compose up api -d # Wait 5 seconds sleep 5 # Verify it's running docker compose logs api | tail -20 ``` ### Solution 2: Port Not Exposed ```bash # Check docker-compose.yml has port mapping: # api: # ports: # - "5055:5055" # If missing, add it and restart: docker compose down docker compose up -d ``` ### Solution 3: API_URL Mismatch ```bash # In .env, check API_URL: cat .env | grep API_URL # Should match your frontend URL: # Frontend: http://localhost:8502 # API_URL: http://localhost:5055 # If wrong, fix it: # API_URL=http://localhost:5055 # Then restart: docker compose restart frontend ``` ### Solution 4: Firewall Blocking ```bash # Verify port 5055 is accessible netstat -tlnp | grep 5055 # Should show port listening # If on different machine, try: # Instead of localhost, use your IP: API_URL=http://192.168.1.100:5055 ``` ### Solution 5: Services Not Started ```bash # Restart everything docker compose restart # Wait 10 seconds sleep 10 # Check all services docker compose ps # All should show "Up" ``` --- ## Connection Refused **What it looks like:** ``` Connection refused ECONNREFUSED Error: socket hang up ``` **Diagnosis:** - API port (5055) not open - API crashed - Wrong IP/hostname **Solution:** ```bash # Step 1: Check if API is running docker ps | grep api # Step 2: Check if port is listening lsof -i :5055 # or netstat -tlnp | grep 5055 # Step 3: Check API logs docker compose logs api | tail -30 # Look for errors # Step 4: Restart API docker compose restart api docker compose logs api | grep -i "error" ``` --- ## Timeout / Slow Connection **What it looks like:** - Page loads slowly - Request times out - "Gateway timeout" error **Causes:** - API is overloaded - Network is slow - Reverse proxy issue **Solutions:** ### Check API Performance ```bash # See CPU/memory usage docker stats # Check logs for slow operations docker compose logs api | grep "slow\|timeout" ``` ### Reduce Load ```bash # In .env: SURREAL_COMMANDS_MAX_TASKS=2 API_CLIENT_TIMEOUT=600 # Restart docker compose restart ``` ### Check Network ```bash # Test latency ping localhost # Test API directly time curl http://localhost:5055/health # Should be < 100ms ``` --- ## 502 Bad Gateway (Reverse Proxy) **What it looks like:** ``` 502 Bad Gateway The server is temporarily unable to service the request ``` **Cause:** Reverse proxy can't reach API **Solutions:** ### Check Backend is Running ```bash # From the reverse proxy server curl http://localhost:5055/health # Should work ``` ### Check Reverse Proxy Config ```nginx # Nginx example (correct): location /api { proxy_pass http://localhost:5055/api; proxy_http_version 1.1; } # Common mistake (wrong): location /api { proxy_pass http://localhost:5055; # Missing /api } ``` ### Set API_URL for HTTPS ```bash # In .env: API_URL=https://yourdomain.com # Restart docker compose restart ``` --- ## Intermittent Disconnects **What it looks like:** - Works sometimes, fails other times - Sporadic "cannot connect" errors - Works then stops working **Cause:** Transient network issue or database conflicts **Solutions:** ### Enable Retry Logic ```bash # In .env: SURREAL_COMMANDS_RETRY_ENABLED=true SURREAL_COMMANDS_RETRY_MAX_ATTEMPTS=5 SURREAL_COMMANDS_RETRY_WAIT_STRATEGY=exponential_jitter # Restart docker compose restart ``` ### Reduce Concurrency ```bash # In .env: SURREAL_COMMANDS_MAX_TASKS=2 # Restart docker compose restart ``` ### Check Network Stability ```bash # Monitor connection ping google.com # Long-running test ping -c 100 google.com | grep "packet loss" # Should be 0% loss ``` --- ## Different Machine / Remote Access **You want to access Open Notebook from another computer** **Solution:** ### Step 1: Get Your Machine IP ```bash # On the server running Open Notebook: ifconfig | grep "inet " # or hostname -I # Note the IP (e.g., 192.168.1.100) ``` ### Step 2: Update API_URL ```bash # In .env: API_URL=http://192.168.1.100:5055 # Restart docker compose restart ``` ### Step 3: Access from Other Machine ```bash # In browser on other machine: http://192.168.1.100:8502 # (or your server IP) ``` ### Step 4: Verify Port is Exposed ```bash # On server: docker compose ps # Should show port mapping: # 0.0.0.0:8502->8502/tcp # 0.0.0.0:5055->5055/tcp ``` ### If Still Doesn't Work ```bash # Check firewall on server sudo ufw status # May need to open ports: sudo ufw allow 8502 sudo ufw allow 5055 # Check on different machine: telnet 192.168.1.100 5055 # Should connect ``` --- ## CORS Error (Browser Console) **What it looks like:** ``` Cross-Origin Request Blocked Access-Control-Allow-Origin ``` **In browser console (F12):** ``` CORS policy: Response to preflight request doesn't pass access control check ``` **Cause:** Frontend and API URLs don't match **Solution:** ```bash # Check browser console error for what URLs are being used # The error shows: # - Requesting from: http://localhost:8502 # - Trying to reach: http://localhost:5055 # Make sure API_URL matches: API_URL=http://localhost:5055 # And protocol matches (http/https) # Restart docker compose restart frontend ``` --- ## Testing Connection **Full diagnostic:** ```bash # 1. Services running? docker compose ps # All should show "Up" # 2. Ports listening? netstat -tlnp | grep -E "8502|5055|8000" # 3. API responding? curl http://localhost:5055/health # 4. Frontend accessible? curl http://localhost:8502 | head # 5. Network OK? ping google.com # 6. No firewall? sudo ufw status | grep -E "5055|8502|8000" ``` --- ## Checklist for Remote Access - [ ] Server IP noted (e.g., 192.168.1.100) - [ ] Ports 8502, 5055, 8000 exposed in docker-compose - [ ] API_URL set to server IP - [ ] Firewall allows ports 8502, 5055, 8000 - [ ] Can reach server from client machine (ping IP) - [ ] All services running (docker compose ps) - [ ] Can curl API from client (curl http://IP:5055/health) --- ## SSL Certificate Errors **What it looks like:** ``` [SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed Connection error when using HTTPS endpoints Works with HTTP but fails with HTTPS ``` **Cause:** Self-signed certificates not trusted by Python's SSL verification **Solutions:** ### Solution 1: Use Custom CA Bundle (Recommended) ```bash # In .env: ESPERANTO_SSL_CA_BUNDLE=/path/to/your/ca-bundle.pem # For Docker, mount the certificate: # In docker-compose.yml: volumes: - /path/to/your/ca-bundle.pem:/certs/ca-bundle.pem:ro environment: - ESPERANTO_SSL_CA_BUNDLE=/certs/ca-bundle.pem ``` ### Solution 2: Disable SSL Verification (Development Only) ```bash # WARNING: Only use in trusted development environments # In .env: ESPERANTO_SSL_VERIFY=false ``` ### Solution 3: Use HTTP Instead If services are on a trusted local network, HTTP is acceptable: ``` Change the base URL in your credential (Settings → API Keys) from https:// to http:// Example: http://localhost:1234/v1 ``` > **Security Note:** Disabling SSL verification exposes you to man-in-the-middle attacks. Always prefer custom CA bundle or HTTP on trusted networks. --- ## Still Having Issues? - Check [Quick Fixes](quick-fixes.md) - Check [FAQ](faq.md) - Check logs: `docker compose logs` - Try restart: `docker compose restart` - Check firewall: `sudo ufw status` - Ask for help on [Discord](https://discord.gg/37XJPXfz2w) ================================================ FILE: docs/6-TROUBLESHOOTING/faq.md ================================================ # Frequently Asked Questions Common questions about Open Notebook usage, configuration, and best practices. --- ## General Usage ### What is Open Notebook? Open Notebook is an open-source, privacy-focused alternative to Google's Notebook LM. It allows you to: - Create and manage research notebooks - Chat with your documents using AI - Generate podcasts from your content - Search across all your sources with semantic search - Transform and analyze your content ### How is it different from Google Notebook LM? **Privacy**: Your data stays local by default. Only your chosen AI providers receive queries. **Flexibility**: Support for 15+ AI providers (OpenAI, Anthropic, Google, local models, etc.) **Customization**: Open source, so you can modify and extend functionality **Control**: You control your data, models, and processing ### Can I use Open Notebook offline? **Partially**: The application runs locally, but requires internet for: - AI model API calls (unless using local models like Ollama) - Web content scraping **Fully offline**: Possible with local models (Ollama) for basic functionality. ### What file types are supported? **Documents**: PDF, DOCX, TXT, Markdown **Web Content**: URLs, YouTube videos **Media**: MP3, WAV, M4A (audio), MP4, AVI, MOV (video) **Other**: Direct text input, CSV, code files ### How much does it cost? **Software**: Free (open source) **AI API costs**: Pay-per-use to providers: - OpenAI: ~$0.50-5 per 1M tokens - Anthropic: ~$3-75 per 1M tokens - Google: Often free tier available - Local models: Free after initial setup **Typical monthly costs**: $5-50 for moderate usage. --- ## AI Models and Providers ### Which AI provider should I choose? **For beginners**: OpenAI (reliable, well-documented) **For privacy**: Local models (Ollama) or European providers (Mistral) **For cost optimization**: Groq, Google (free tier), or OpenRouter **For long context**: Anthropic (200K tokens) or Google Gemini (1M tokens) ### Can I use multiple providers? **Yes**: Configure different providers for different tasks: - OpenAI for chat - Google for embeddings - ElevenLabs for text-to-speech - Anthropic for complex reasoning ### What are the best model combinations? **Budget-friendly**: - Language: `gpt-4o-mini` (OpenAI) or `deepseek-chat` - Embedding: `text-embedding-3-small` (OpenAI) **High-quality**: - Language: `claude-3-5-sonnet` (Anthropic) or `gpt-4o` (OpenAI) - Embedding: `text-embedding-3-large` (OpenAI) **Privacy-focused**: - Language: Local Ollama models (mistral, llama3) - Embedding: Local embedding models ### How do I optimize AI costs? **Model selection**: - Use smaller models for simple tasks (gpt-4o-mini, claude-3-5-haiku) - Use larger models only for complex reasoning - Leverage free tiers when available **Usage optimization**: - Use "Summary Only" context for background sources - Ask more specific questions - Use local models (Ollama) for frequent tasks --- ## Data Management ### Where is my data stored? **Local storage**: By default, all data is stored locally: - Database: SurrealDB files in `surreal_data/` - Uploads: Files in `data/uploads/` - Podcasts: Generated audio in `data/podcasts/` - No external data transmission (except to chosen AI providers) ### How do I backup my data? ```bash # Create backup tar -czf backup-$(date +%Y%m%d).tar.gz data/ surreal_data/ # Restore backup tar -xzf backup-20240101.tar.gz ``` ### Can I sync data between devices? **Currently**: No built-in sync functionality. **Workarounds**: - Use shared network storage for data directories - Manual backup/restore between devices ### What happens if I delete a notebook? **Soft deletion**: Notebooks are marked as archived, not permanently deleted. **Recovery**: Archived notebooks can be restored from the database. --- ## Best Practices ### How should I organize my notebooks? - **By topic**: Separate notebooks for different research areas - **By project**: One notebook per project or course - **By time period**: Monthly or quarterly notebooks **Recommended size**: 20-100 sources per notebook for best performance. ### How do I get the best search results? - Use descriptive queries ("data analysis methods" not just "data") - Combine multiple related terms - Use natural language (ask questions as you would to a human) - Try both text search (keywords) and vector search (concepts) ### How can I improve chat responses? - Provide context: Reference specific sources or topics - Be specific: Ask detailed questions rather than general ones - Request citations: "Answer with page citations" - Use follow-up questions: Build on previous responses ### What are the security best practices? - Never share API keys publicly - Use `OPEN_NOTEBOOK_PASSWORD` for public deployments - Use HTTPS for production (via reverse proxy) - Keep Docker images updated - Encrypt backups if they contain sensitive data --- ## Technical Questions ### Can I use Open Notebook programmatically? **Yes**: Open Notebook provides a REST API: - Full API documentation at `http://localhost:5055/docs` - Support for all UI functionality - Authentication via password header ### Can I run Open Notebook in production? **Yes**: Designed for production use with: - Docker deployment - Security features (password protection) - Monitoring and logging - Reverse proxy support (nginx, Caddy, Traefik) ### What are the system requirements? **Minimum**: - 4GB RAM - 2 CPU cores - 10GB disk space **Recommended**: - 8GB+ RAM - 4+ CPU cores - SSD storage - For local models: 16GB+ RAM, GPU recommended --- ## Timeout and Performance ### Why do I get timeout errors? **Common causes**: - Large context (too many sources) - Slow AI provider - Local models on CPU (slow) - First request (model loading) **Solutions**: ```bash # In .env: API_CLIENT_TIMEOUT=600 # 10 minutes for slow setups ESPERANTO_LLM_TIMEOUT=180 # 3 minutes for model inference ``` ### Recommended timeouts by setup: | Setup | API_CLIENT_TIMEOUT | |-------|-------------------| | Cloud APIs (OpenAI, Anthropic) | 300 (default) | | Local Ollama with GPU | 600 | | Local Ollama with CPU | 1200 | | Remote LM Studio | 900 | --- ## Getting Help ### My question isn't answered here 1. Check the troubleshooting guides in this section 2. Search existing GitHub issues 3. Ask in the Discord community 4. Create a GitHub issue with detailed information ### How do I report a bug? Include: - Steps to reproduce - Expected vs actual behavior - Error messages and logs - System information - Configuration details (without API keys) Submit to: [GitHub Issues](https://github.com/lfnovo/open-notebook/issues) ### Where can I get help? - **Discord**: https://discord.gg/37XJPXfz2w (fastest) - **GitHub Issues**: Bug reports and feature requests - **Documentation**: This docs site --- ## Related - [Quick Fixes](quick-fixes.md) - Common issues with 1-minute solutions - [AI & Chat Issues](ai-chat-issues.md) - Model and chat problems - [Connection Issues](connection-issues.md) - Network and API problems ================================================ FILE: docs/6-TROUBLESHOOTING/index.md ================================================ # Troubleshooting - Problem Solving Guide Having issues? Use this guide to diagnose and fix problems. --- ## How to Use This Guide **Step 1: Identify your problem** - What's the symptom? (error message, behavior, something not working?) - When did it happen? (during install, while using, after update?) **Step 2: Find the right guide** - Look below for your symptom - Go to the specific troubleshooting guide **Step 3: Follow the steps** - Guides are organized by symptom, not by root cause - Each has diagnostic steps and solutions --- ## Quick Problem Map ### During Installation - **Docker won't start** → [Quick Fixes](quick-fixes.md#9-services-wont-start-or-docker-error) - **Port already in use** → [Quick Fixes](quick-fixes.md#3-port-x-already-in-use) - **Permission denied** → [Quick Fixes](quick-fixes.md#9-services-wont-start-or-docker-error) - **Can't connect to database** → [Connection Issues](connection-issues.md) ### When Starting - **API won't start** → [Quick Fixes](quick-fixes.md#9-services-wont-start-or-docker-error) - **Frontend won't load** → [Connection Issues](connection-issues.md) - **"Cannot connect to server" error** → [Connection Issues](connection-issues.md) ### Settings / Configuration - **Models not showing** → [AI & Chat Issues](ai-chat-issues.md) - **"Invalid API key"** → [AI & Chat Issues](ai-chat-issues.md) - **Can't find Settings** → [Quick Fixes](quick-fixes.md) ### Using Features - **Chat not working** → [AI & Chat Issues](ai-chat-issues.md) - **Chat responses are slow** → [AI & Chat Issues](ai-chat-issues.md) - **Chat gives bad answers** → [AI & Chat Issues](ai-chat-issues.md) ### Adding Content - **Can't upload PDF** → [Quick Fixes](quick-fixes.md#4-cannot-process-file-or-unsupported-format) - **File won't process** → [Quick Fixes](quick-fixes.md#4-cannot-process-file-or-unsupported-format) - **Web link won't extract** → [Quick Fixes](quick-fixes.md#4-cannot-process-file-or-unsupported-format) ### Search - **Search returns no results** → [Quick Fixes](quick-fixes.md#7-search-returns-nothing) - **Search returns wrong results** → [Quick Fixes](quick-fixes.md#7-search-returns-nothing) ### Podcasts - **Can't generate podcast** → [Quick Fixes](quick-fixes.md#8-podcast-generation-failed) - **Podcast shows "FAILED" badge** → Check the error message displayed on the episode, then use the **Retry** button. See [Podcasts Explained](../2-CORE-CONCEPTS/podcasts-explained.md#when-things-go-wrong-failures--retry) - **Podcast audio is robotic** → [Quick Fixes](quick-fixes.md#8-podcast-generation-failed) - **Podcast generation times out** → [Quick Fixes](quick-fixes.md#8-podcast-generation-failed) --- ## Troubleshooting by Error Message ### "Cannot connect to server" → [Connection Issues](connection-issues.md) — Frontend can't reach API ### "Invalid API key" → [AI & Chat Issues](ai-chat-issues.md) — Wrong or missing API key ### "Models not available" → [AI & Chat Issues](ai-chat-issues.md) — Model not configured ### "Connection refused" → [Connection Issues](connection-issues.md) — Service not running or port wrong ### "Port already in use" → [Quick Fixes](quick-fixes.md#3-port-x-already-in-use) — Port conflict ### "Permission denied" → [Quick Fixes](quick-fixes.md#9-services-wont-start-or-docker-error) — File permissions issue ### "Unsupported file type" → [Quick Fixes](quick-fixes.md#4-cannot-process-file-or-unsupported-format) — File format not supported ### "Processing timeout" → [Quick Fixes](quick-fixes.md#5-chat-is-very-slow) — File too large or slow processing --- ## Troubleshooting by Component ### Frontend (Browser/UI) - Can't access UI → [Connection Issues](connection-issues.md) - UI is slow → [Quick Fixes](quick-fixes.md) - Button/feature missing → [Quick Fixes](quick-fixes.md) ### API (Backend) - API won't start → [Quick Fixes](quick-fixes.md#9-services-wont-start-or-docker-error) - API errors in logs → [Quick Fixes](quick-fixes.md#9-services-wont-start-or-docker-error) - API is slow → [Quick Fixes](quick-fixes.md) ### Database - Can't connect to database → [Connection Issues](connection-issues.md) - Data lost after restart → [FAQ](faq.md#how-do-i-backup-my-data) ### AI / Chat - Chat not working → [AI & Chat Issues](ai-chat-issues.md) - Bad responses → [AI & Chat Issues](ai-chat-issues.md) - Cost too high → [AI & Chat Issues](ai-chat-issues.md#high-api-costs) ### Sources - Can't upload file → [Quick Fixes](quick-fixes.md#4-cannot-process-file-or-unsupported-format) - File won't process → [Quick Fixes](quick-fixes.md#4-cannot-process-file-or-unsupported-format) ### Podcasts - Won't generate → [Quick Fixes](quick-fixes.md#8-podcast-generation-failed) - Bad audio quality → [Quick Fixes](quick-fixes.md#8-podcast-generation-failed) --- ## Diagnostic Checklist **When something isn't working:** - [ ] Check if services are running: `docker ps` - [ ] Check logs: `docker compose logs api` (or frontend, surrealdb) - [ ] Verify ports are exposed: `netstat -tlnp` or `lsof -i :5055` - [ ] Test connectivity: `curl http://localhost:5055/health` - [ ] Check environment variables: `docker inspect ` - [ ] Try restarting: `docker compose restart` - [ ] Check firewall/antivirus isn't blocking --- ## Getting Help If you can't find the answer here: 1. **Check the relevant guide** — Read completely, try all steps 2. **Check the FAQ** — [Frequently Asked Questions](faq.md) 3. **Search our Discord** — Others may have had same issue 4. **Check logs** — Most issues show error messages in logs 5. **Report on GitHub** — Include error message, steps to reproduce ### How to Report an Issue Include: 1. Error message (exact) 2. Steps to reproduce 3. Logs: `docker compose logs` 4. Your setup: Docker/local, provider, OS 5. What you've already tried → [Report on GitHub](https://github.com/lfnovo/open-notebook/issues) --- ## Guides ### [Quick Fixes](quick-fixes.md) Top 10 most common issues with 1-minute solutions. ### [Connection Issues](connection-issues.md) Frontend can't reach API, network problems. ### [AI & Chat Issues](ai-chat-issues.md) Chat not working, bad responses, slow performance. ### [FAQ](faq.md) Frequently asked questions about usage, costs, and best practices. --- ## Common Solutions **Service won't start?** ```bash # Check logs docker compose logs # Restart everything docker compose restart # Nuclear option: rebuild docker compose down docker compose up --build ``` **Port conflict?** ```bash # Find what's using port 5055 lsof -i :5055 # Kill it or use different port ``` **Can't connect?** ```bash # Test API directly curl http://localhost:5055/health # Should return: {"status":"ok"} ``` **Slow performance?** ```bash # Check resource usage docker stats # Reduce concurrency in .env SURREAL_COMMANDS_MAX_TASKS=2 ``` **High costs?** ```bash # Switch to cheaper model # In Settings → Models → Choose gpt-4o-mini (OpenAI) # Or use Ollama (free) ``` --- ## Still Stuck? **Before asking for help:** 1. Read the relevant guide completely 2. Try all the steps 3. Check the logs 4. Restart services 5. Search existing issues on GitHub **Then:** - **Discord**: https://discord.gg/37XJPXfz2w (fastest response) - **GitHub Issues**: https://github.com/lfnovo/open-notebook/issues ================================================ FILE: docs/6-TROUBLESHOOTING/quick-fixes.md ================================================ # Quick Fixes - Top 11 Issues & Solutions Common problems with 1-minute solutions. --- ## #1: "Cannot connect to server" **Symptom:** Browser shows error "Cannot connect to server" or "Unable to reach API" **Cause:** Frontend can't reach API **Solution (1 minute):** ```bash # Step 1: Check if API is running docker ps | grep api # Step 2: Verify port 5055 is accessible curl http://localhost:5055/health # Expected output: {"status":"ok"} # If that doesn't work: # Step 3: Restart services docker compose restart # Step 4: Try again # Open http://localhost:8502 in browser ``` **If still broken:** - Check `API_URL` in .env (should match your frontend URL) - See [Connection Issues](connection-issues.md) --- ## #2: "Invalid API key" or "Models not showing" **Symptom:** Settings → Models shows "No models available" **Cause:** No credential configured, or credential has invalid API key **Solution (1 minute):** ``` 1. Go to Settings → API Keys 2. If no credential exists, click "Add Credential" and add one 3. If a credential exists, click "Test Connection" 4. If test fails, delete and re-create with correct key 5. After test passes, click "Discover Models" → "Register Models" 6. Go to Settings → Models to verify models appear ``` **If still broken:** - Make sure key has no extra spaces - Generate a fresh key from provider dashboard - Check that `OPEN_NOTEBOOK_ENCRYPTION_KEY` is set in docker-compose.yml - See [AI & Chat Issues](ai-chat-issues.md) --- ## #3: "Port X already in use" **Symptom:** Docker error "Port 8502 is already allocated" **Cause:** Another service using that port **Solution (1 minute):** ```bash # Option 1: Stop the other service # Find what's using port 8502 lsof -i :8502 # Kill it or close the app # Option 2: Use different port # Edit docker-compose.yml # Change: - "8502:8502" # To: - "8503:8502" # Then restart docker compose restart # Access at: http://localhost:8503 ``` --- ## #4: "Cannot process file" or "Unsupported format" **Symptom:** Upload fails or says "File format not supported" **Cause:** File type not supported or too large **Solution (1 minute):** ```bash # Check if file format is supported: # ✓ PDF, DOCX, PPTX, XLSX (documents) # ✓ MP3, WAV, M4A (audio) # ✓ MP4, AVI, MOV (video) # ✓ URLs/web links # ✗ Pure images (.jpg without OCR) # ✗ Files > 100MB # Try these: # - Convert to PDF if possible # - Split large files # - Try uploading again ``` --- ## #5: "Chat is very slow" **Symptom:** Chat responses take minutes or timeout **Cause:** Slow AI provider, large context, or overloaded system **Solution (1 minute):** ```bash # Step 1: Check which model you're using # Settings → Models # Note the model name # Step 2: Try a cheaper/faster model # OpenAI: Switch to gpt-4o-mini (10x cheaper, slightly faster) # Anthropic: Switch to claude-3-5-haiku (fastest) # Groq: Use any model (ultra-fast) # Step 3: Reduce context # Chat: Select fewer sources # Use "Summary Only" instead of "Full Content" # Step 4: Check if API is overloaded docker stats # Look at CPU/memory usage ``` For deep dive: See [AI & Chat Issues](ai-chat-issues.md) --- ## #6: "Chat gives bad responses" **Symptom:** AI responses are generic, wrong, or irrelevant **Cause:** Bad context, vague question, or wrong model **Solution (1 minute):** ```bash # Step 1: Make sure sources are in context # Click "Select Sources" in Chat # Verify relevant sources are checked and set to "Full Content" # Step 2: Ask a specific question # Bad: "What do you think?" # Good: "Based on the paper's methodology section, what are the 3 main limitations?" # Step 3: Try a more powerful model # OpenAI: Use gpt-4o (better reasoning) # Anthropic: Use claude-3-5-sonnet (best reasoning) # Step 4: Check citations # Click citations to verify AI actually saw those sources ``` For detailed help: See [Chat Effectively](../3-USER-GUIDE/chat-effectively.md) --- ## #7: "Search returns nothing" **Symptom:** Search shows 0 results even though content exists **Cause:** Wrong search type or poor query **Solution (1 minute):** ```bash # Try a different search type: # If you searched with KEYWORDS: # Try VECTOR SEARCH instead # (Concept-based, not keyword-based) # If you searched for CONCEPTS: # Try TEXT SEARCH instead # (Look for specific words in your query) # Try simpler search: # Instead of: "How do transformers work in neural networks?" # Try: "transformers" or "neural networks" # Check sources are processed: # Go to notebook # All sources should show green "Ready" status ``` For detailed help: See [Search Effectively](../3-USER-GUIDE/search.md) --- ## #8: "Podcast generation failed" **Symptom:** "Podcast generation failed" error **Cause:** Insufficient content, API quota, or network issue **Solution (1 minute):** ```bash # Step 1: Make sure you have content # Select at least 1-2 sources # Avoid single-sentence sources # Step 2: Try again # Sometimes it's a temporary API issue # Wait 30 seconds and retry # Step 3: Check your TTS provider has quota # OpenAI: Check account has credits # ElevenLabs: Check monthly quota # Google: Check API quota # Step 4: Try different TTS provider # In podcast generation, choose "Google" or "Local" # instead of "ElevenLabs" ``` For detailed help: See [FAQ](faq.md) --- ## #9: "Services won't start" or Docker error **Symptom:** Docker error when running `docker compose up` **Cause:** Corrupt configuration, permission issue, or resource issue **Solution (1 minute):** ```bash # Step 1: Check logs docker compose logs # Step 2: Try restart docker compose restart # Step 3: If that fails, rebuild docker compose down docker compose up --build # Step 4: Check disk space df -h # Need at least 5GB free # Step 5: Check Docker has enough memory # Docker settings → Resources → Memory: 4GB+ ``` --- ## #10: "Database says 'too many connections'" **Symptom:** Error about database connections **Cause:** Too many concurrent operations **Solution (1 minute):** ```bash # In .env, reduce concurrency: SURREAL_COMMANDS_MAX_TASKS=2 # Then restart: docker compose restart # This makes it slower but more stable ``` --- ## #11: Slow Startup or Download Timeouts (China/Slow Networks) **Symptom:** Container crashes on startup, worker enters FATAL state, or pip/uv downloads fail **Cause:** Slow network or restricted access to Python package repositories **Solution:** ### Increase Download Timeout ```yaml # In docker-compose.yml environment: environment: - UV_HTTP_TIMEOUT=600 # 10 minutes (default is 30s) ``` ### Use Chinese Mirrors (if in China) ```yaml environment: - UV_HTTP_TIMEOUT=600 - UV_INDEX_URL=https://pypi.tuna.tsinghua.edu.cn/simple - PIP_INDEX_URL=https://pypi.tuna.tsinghua.edu.cn/simple ``` **Alternative Chinese mirrors:** - Tsinghua: `https://pypi.tuna.tsinghua.edu.cn/simple` - Aliyun: `https://mirrors.aliyun.com/pypi/simple/` - Huawei: `https://repo.huaweicloud.com/repository/pypi/simple` **Note:** First startup may take several minutes while dependencies download. Subsequent starts will be faster. --- ## Quick Troubleshooting Checklist When something breaks: - [ ] **Restart services:** `docker compose restart` - [ ] **Check logs:** `docker compose logs` - [ ] **Verify connectivity:** `curl http://localhost:5055/health` - [ ] **Check .env:** API keys set? API_URL correct? - [ ] **Check resources:** `docker stats` (CPU/memory) - [ ] **Clear cache:** `docker system prune` (free space) - [ ] **Rebuild if needed:** `docker compose up --build` --- ## Nuclear Options (Last Resort) **Completely reset (will lose all data in Docker):** ```bash docker compose down -v docker compose up --build ``` **Reset to defaults:** ```bash # Backup your .env first! cp .env .env.backup # Reset to example cp .env.example .env # Edit with your API keys # Restart docker compose up ``` --- ## Prevention Tips 1. **Keep backups** — Export your notebooks regularly 2. **Monitor logs** — Check `docker compose logs` periodically 3. **Update regularly** — Pull latest image: `docker pull lfnovo/open_notebook:latest` 4. **Document changes** — Keep notes on what you configured 5. **Test after updates** — Verify everything works --- ## Still Stuck? - **Look up your exact error** in [Troubleshooting Index](index.md) - **Check the FAQ** in [FAQ](faq.md) - **Check logs:** `docker compose logs | head -50` - **Ask for help:** [Discord](https://discord.gg/37XJPXfz2w) or [GitHub Issues](https://github.com/lfnovo/open-notebook/issues) ================================================ FILE: docs/7-DEVELOPMENT/api-reference.md ================================================ # API Reference Complete REST API for Open Notebook. All endpoints are served from the API backend (default: `http://localhost:5055`). **Base URL**: `http://localhost:5055` (development) or environment-specific production URL **Interactive Docs**: Use FastAPI's built-in Swagger UI at `http://localhost:5055/docs` for live testing and exploration. This is the primary reference for all endpoints, request/response schemas, and real-time testing. --- ## Quick Start ### 1. Authentication Simple password-based (development only): ```bash curl http://localhost:5055/api/notebooks \ -H "Authorization: Bearer your_password" ``` **⚠️ Production**: Replace with OAuth/JWT. See [Security Configuration](../5-CONFIGURATION/security.md) for details. ### 2. Base API Flow Most operations follow this pattern: 1. Create a **Notebook** (container for research) 2. Add **Sources** (PDFs, URLs, text) 3. Query via **Chat** or **Search** 4. View results and **Notes** ### 3. Testing Endpoints Instead of memorizing endpoints, use the interactive API docs: - Navigate to `http://localhost:5055/docs` - Try requests directly in the browser - See request/response schemas in real-time - Test with your own data --- ## API Endpoints Overview ### Main Resource Types **Notebooks** - Research projects containing sources and notes - `GET/POST /notebooks` - List and create - `GET/PUT/DELETE /notebooks/{id}` - Read, update, delete **Sources** - Content items (PDFs, URLs, text) - `GET/POST /sources` - List and add content - `GET /sources/{id}` - Fetch source details - `POST /sources/{id}/retry` - Retry failed processing - `GET /sources/{id}/download` - Download original file **Notes** - User-created or AI-generated research notes - `GET/POST /notes` - List and create - `GET/PUT/DELETE /notes/{id}` - Read, update, delete **Chat** - Conversational AI interface - `GET/POST /chat/sessions` - Manage chat sessions - `POST /chat/execute` - Send message and get response - `POST /chat/context/build` - Prepare context for chat **Search** - Find content by text or semantic similarity - `POST /search` - Full-text or vector search - `POST /ask` - Ask a question (search + synthesize) **Transformations** - Custom prompts for extracting insights - `GET/POST /transformations` - Create custom extraction rules - `POST /sources/{id}/insights` - Apply transformation to source **Models** - Configure AI providers - `GET /models` - Available models - `GET /models/defaults` - Current defaults - `POST /models/config` - Set defaults **Credentials** - Manage AI provider credentials - `GET/POST /credentials` - List and create credentials - `GET/PUT/DELETE /credentials/{id}` - CRUD operations - `POST /credentials/{id}/test` - Test connection - `POST /credentials/{id}/discover` - Discover models from provider - `POST /credentials/{id}/register-models` - Register discovered models - `GET /credentials/status` - Provider status overview - `GET /credentials/env-status` - Environment variable status - `POST /credentials/migrate-from-env` - Migrate env vars to credentials **Health & Status** - `GET /health` - Health check - `GET /commands/{id}` - Track async operations --- ## Authentication ### Current (Development) All requests require password header: ```bash curl -H "Authorization: Bearer your_password" http://localhost:5055/api/notebooks ``` Password configured via `OPEN_NOTEBOOK_PASSWORD` environment variable. > **📖 See [Security Configuration](../5-CONFIGURATION/security.md)** for complete authentication setup, API examples, and production hardening. ### Production **⚠️ Not secure.** Replace with: - OAuth 2.0 (recommended) - JWT tokens - API keys See [Security Configuration](../5-CONFIGURATION/security.md) for production setup. --- ## Common Patterns ### Pagination ```bash # List sources with limit/offset curl 'http://localhost:5055/sources?limit=20&offset=10' ``` ### Filtering & Sorting ```bash # Filter by notebook, sort by date curl 'http://localhost:5055/sources?notebook_id=notebook:abc&sort_by=created&sort_order=asc' ``` ### Async Operations Some operations (source processing, podcast generation) return immediately with a command ID: ```bash # Submit async operation curl -X POST http://localhost:5055/sources -F async_processing=true # Response: {"id": "source:src001", "command_id": "command:cmd123"} # Poll status curl http://localhost:5055/commands/command:cmd123 ``` ### Streaming Responses The `/ask` endpoint streams responses as Server-Sent Events: ```bash curl -N 'http://localhost:5055/ask' \ -H "Content-Type: application/json" \ -d '{"question": "What is AI?"}' # Outputs: data: {"type":"strategy",...} # data: {"type":"answer",...} # data: {"type":"final_answer",...} ``` ### Multipart File Upload ```bash curl -X POST http://localhost:5055/sources \ -F "type=upload" \ -F "notebook_id=notebook:abc" \ -F "file=@document.pdf" ``` --- ## Error Handling All errors return JSON with status code: ```json {"detail": "Notebook not found"} ``` ### Common Status Codes | Code | Meaning | Example | |------|---------|---------| | 200 | Success | Operation completed | | 400 | Bad Request | Invalid input | | 404 | Not Found | Resource doesn't exist | | 409 | Conflict | Resource already exists | | 500 | Server Error | Database/processing error | --- ## Tips for Developers 1. **Start with interactive docs** (`http://localhost:5055/docs`) - this is the definitive reference 2. **Enable logging** for debugging (check API logs: `docker logs`) 3. **Streaming endpoints** require special handling (Server-Sent Events, not standard JSON) 4. **Async operations** return immediately; always poll status before assuming completion 5. **Vector search** requires embedding model configured (check `/models`) 6. **Model overrides** are per-request; set in body, not config 7. **CORS enabled** in development; configure for production --- ## Learning Path 1. **Authentication**: Add `X-Password` header to all requests 2. **Create a notebook**: `POST /notebooks` with name and description 3. **Add a source**: `POST /sources` with file, URL, or text 4. **Query your content**: `POST /chat/execute` to ask questions 5. **Explore advanced features**: Search, transformations, streaming --- ## Production Considerations - Replace password auth with OAuth/JWT (see [Security](../5-CONFIGURATION/security.md)) - Add rate limiting via reverse proxy (Nginx, CloudFlare, Kong) - Enable CORS restrictions (currently allows all origins) - Use HTTPS via reverse proxy (see [Reverse Proxy](../5-CONFIGURATION/reverse-proxy.md)) - Set up API versioning strategy (currently implicit) See [Security Configuration](../5-CONFIGURATION/security.md) and [Reverse Proxy Setup](../5-CONFIGURATION/reverse-proxy.md) for complete production setup. ================================================ FILE: docs/7-DEVELOPMENT/architecture.md ================================================ # Open Notebook Architecture ## High-Level Overview Open Notebook follows a three-tier architecture with clear separation of concerns: ``` ┌─────────────────────────────────────────────────────────┐ │ Your Browser │ │ Access: http://your-server-ip:8502 │ └────────────────┬────────────────────────────────────────┘ │ ▼ ┌───────────────┐ │ Port 8502 │ ← Next.js Frontend (what you see) │ Frontend │ Also proxies API requests internally! └───────┬───────┘ │ proxies /api/* requests ↓ ▼ ┌───────────────┐ │ Port 5055 │ ← FastAPI Backend (handles requests) │ API │ └───────┬───────┘ │ ▼ ┌───────────────┐ │ SurrealDB │ ← Database (internal, auto-configured) │ (Port 8000) │ └───────────────┘ ``` **Key Points:** - **v1.1+**: Next.js automatically proxies `/api/*` requests to the backend, simplifying reverse proxy setup - Your browser loads the frontend from port 8502 - The frontend needs to know where to find the API - when accessing remotely, set: `API_URL=http://your-server-ip:5055` - **Behind reverse proxy?** You only need to proxy to port 8502 now! See [Reverse Proxy Configuration](../5-CONFIGURATION/reverse-proxy.md) --- ## Detailed Architecture Open Notebook is built on a **three-tier, async-first architecture** designed for scalability, modularity, and multi-provider AI flexibility. The system separates concerns across frontend, API, and database layers, with LangGraph powering intelligent workflows and Esperanto enabling seamless integration with 8+ AI providers. **Core Philosophy**: - Privacy-first: Users control their data and AI provider choice - Async/await throughout: Non-blocking operations for responsive UX - Domain-Driven Design: Clear separation between domain models, repositories, and orchestrators - Multi-provider flexibility: Swap AI providers without changing application code - Self-hosted capable: All components deployable in isolated environments --- ## Three-Tier Architecture ### Layer 1: Frontend (React/Next.js @ port 3000) **Purpose**: Responsive, interactive user interface for research, notes, chat, and podcast management. **Technology Stack**: - **Framework**: Next.js 15 with React 19 - **Language**: TypeScript with strict type checking - **State Management**: Zustand (lightweight store) + TanStack Query (server state) - **Styling**: Tailwind CSS + Shadcn/ui component library - **Build Tool**: Webpack (bundled via Next.js) **Key Responsibilities**: - Render notebooks, sources, notes, chat sessions, and podcasts - Handle user interactions (create, read, update, delete operations) - Manage complex UI state (modals, file uploads, real-time search) - Stream responses from API (chat, podcast generation) - Display embeddings, vector search results, and insights **Communication Pattern**: - All data fetched via REST API (async requests to port 5055) - Configured base URL: `http://localhost:5055` (dev) or environment-specific (prod) - TanStack Query handles caching, refetching, and data synchronization - Zustand stores global state (user, notebooks, selected context) - CORS enabled on API side for cross-origin requests **Component Architecture**: - `/src/app/`: Next.js App Router (pages, layouts) - `/src/components/`: Reusable React components (buttons, forms, cards) - `/src/hooks/`: Custom hooks (useNotebook, useChat, useSearch) - `/src/lib/`: Utility functions, API clients, validators - `/src/styles/`: Global CSS, Tailwind config --- ### Layer 2: API (FastAPI @ port 5055) **Purpose**: RESTful backend exposing operations on notebooks, sources, notes, chat sessions, and AI models. **Technology Stack**: - **Framework**: FastAPI 0.104+ (async Python web framework) - **Language**: Python 3.11+ - **Validation**: Pydantic v2 (request/response schemas) - **Logging**: Loguru (structured JSON logging) - **Testing**: Pytest (unit and integration tests) **Architecture**: ``` FastAPI App (main.py) ├── Routers (HTTP endpoints) │ ├── routers/notebooks.py (CRUD operations) │ ├── routers/sources.py (content ingestion, upload) │ ├── routers/notes.py (note management) │ ├── routers/chat.py (conversation sessions) │ ├── routers/search.py (full-text + vector search) │ ├── routers/transformations.py (custom transformations) │ ├── routers/models.py (AI model configuration) │ └── routers/*.py (11 additional routers) │ ├── Services (business logic) │ ├── *_service.py (orchestration, graph invocation) │ ├── command_service.py (async job submission) │ └── middleware (auth, logging) │ ├── Models (Pydantic schemas) │ └── models.py (validation, serialization) │ └── Lifespan (startup/shutdown) └── AsyncMigrationManager (database schema migrations) ``` **Key Responsibilities**: 1. **HTTP Interface**: Accept REST requests, validate, return JSON responses 2. **Business Logic**: Orchestrate domain models, repository operations, and workflows 3. **Async Job Queue**: Submit long-running tasks (podcast generation, source processing) 4. **Database Migrations**: Run schema updates on startup 5. **Error Handling**: Catch exceptions, return appropriate HTTP status codes 6. **Logging**: Track operations for debugging and monitoring **Startup Flow**: 1. Load `.env` environment variables 2. Initialize FastAPI app with CORS + auth middleware 3. Run AsyncMigrationManager (creates/updates database schema) 4. Register all routers (20+ endpoints) 5. Server ready on port 5055 **Request-Response Cycle**: ``` HTTP Request → Router → Service → Domain/Repository → SurrealDB ↓ LangGraph (optional) ↓ Response ← Pydantic serialization ← Service ← Result ``` --- ### Layer 3: Database (SurrealDB @ port 8000) **Purpose**: Graph database with built-in vector embeddings, semantic search, and relationship management. **Technology Stack**: - **Database**: SurrealDB (multi-model, ACID transactions) - **Query Language**: SurrealQL (SQL-like syntax with graph operations) - **Async Driver**: Async Rust client for Python - **Migrations**: Manual `.surql` files in `/migrations/` (auto-run on API startup) **Core Tables**: | Table | Purpose | Key Fields | |-------|---------|-----------| | `notebook` | Research project container | id, name, description, archived, created, updated | | `source` | Content item (PDF, URL, text) | id, title, full_text, topics, asset, created, updated | | `source_embedding` | Vector embeddings for semantic search | id, source, embedding, chunk_text, chunk_index | | `note` | User-created research notes | id, title, content, note_type (human/ai), created, updated | | `chat_session` | Conversation session | id, notebook_id, title, messages (JSON), created, updated | | `transformation` | Custom transformation rules | id, name, description, prompt, created, updated | | `source_insight` | Transformation output | id, source_id, insight_type, content, created, updated | | `reference` | Relationship: source → notebook | out (source), in (notebook) | | `artifact` | Relationship: note → notebook | out (note), in (notebook) | **Relationship Graph**: ``` Notebook ↓ (referenced_by) Source ├→ SourceEmbedding (1:many for chunked text) ├→ SourceInsight (1:many for transformation outputs) └→ Note (via artifact relationship) ├→ Embedding (semantic search) └→ Topics (tags) ChatSession ├→ Notebook └→ Messages (stored as JSON array) ``` **Vector Search Capability**: - Embeddings stored natively in SurrealDB - Full-text search on `source.full_text` and `note.content` - Cosine similarity search on embedding vectors - Semantic search integrates with search endpoint **Connection Management**: - Async connection pooling (configurable size) - Transaction support for multi-record operations - Schema auto-validation via migrations - Query timeout protection (prevent infinite queries) --- ## Tech Stack Rationale ### Why Python + FastAPI? **Python**: - Rich AI/ML ecosystem (LangChain, LangGraph, transformers, scikit-learn) - Rapid prototyping and deployment - Extensive async support (asyncio, async/await) - Strong type hints (Pydantic, mypy) **FastAPI**: - Modern, async-first framework - Automatic OpenAPI documentation (Swagger UI @ /docs) - Built-in request validation (Pydantic) - Excellent performance (benchmarked near C/Rust speeds) - Easy middleware/dependency injection ### Why Next.js + React + TypeScript? **Next.js**: - Full-stack React framework with SSR/SSG - File-based routing (intuitive project structure) - Built-in API routes (optional backend co-location) - Optimized image/code splitting - Easy deployment (Vercel, Docker, self-hosted) **React 19**: - Component-based UI (reusable, testable) - Excellent tooling and community - Client-side state management (Zustand) - Server-side state sync (TanStack Query) **TypeScript**: - Type safety catches errors at compile time - Better IDE autocomplete and refactoring - Documentation via types (self-documenting code) - Easier onboarding for new contributors ### Why SurrealDB? **SurrealDB**: - Native graph database (relationships are first-class) - Built-in vector embeddings (no separate vector DB) - ACID transactions (data consistency) - Multi-model (relational + document + graph) - Full-text search + semantic search in one query - Self-hosted (unlike managed Pinecone/Weaviate) - Flexible SurrealQL (SQL-like syntax) **Alternative Considered**: PostgreSQL + pgvector (more mature but separate extensions) ### Why Esperanto for AI Providers? **Esperanto Library**: - Unified interface to 8+ LLM providers (OpenAI, Anthropic, Google, Groq, Ollama, Mistral, DeepSeek, xAI) - Multi-provider embeddings (OpenAI, Google, Ollama, Mistral, Voyage) - TTS/STT integration (OpenAI, Groq, ElevenLabs, Google) - Smart provider selection (fallback logic, cost optimization) - Per-request model override support - Local Ollama support (completely self-hosted option) **Alternative Considered**: LangChain's provider abstraction (more verbose, less flexible) --- ## LangGraph Workflows LangGraph is a state machine library that orchestrates multi-step AI workflows. Open Notebook uses five core workflows: ### 1. **Source Processing Workflow** (`open_notebook/graphs/source.py`) **Purpose**: Ingest content (PDF, URL, text) and prepare for search/insights. **Flow**: ``` Input (file/URL/text) ↓ Extract Content (content-core library) ↓ Clean & tokenize text ↓ Generate Embeddings (Esperanto) ↓ Create SourceEmbedding records (chunked + indexed) ↓ Extract Topics (LLM summarization) ↓ Save to SurrealDB ↓ Output (Source record with embeddings) ``` **State Dict**: ```python { "content_state": {"file_path" | "url" | "content": str}, "source_id": str, "full_text": str, "embeddings": List[Dict], "topics": List[str], "notebook_ids": List[str], } ``` **Invoked By**: Sources API (`POST /sources`) --- ### 2. **Chat Workflow** (`open_notebook/graphs/chat.py`) **Purpose**: Conduct multi-turn conversations with AI model, referencing notebook context. **Flow**: ``` User Message ↓ Build Context (selected sources/notes) ↓ Add Message to Session ↓ Create Chat Prompt (system + history + context) ↓ Call LLM (via Esperanto) ↓ Stream Response ↓ Save AI Message to ChatSession ↓ Output (complete message) ``` **State Dict**: ```python { "session_id": str, "messages": List[BaseMessage], "context": Dict[str, Any], # sources, notes, snippets "response": str, "model_override": Optional[str], } ``` **Key Features**: - Message history persisted in SurrealDB (SqliteSaver checkpoint) - Context building via `build_context_for_chat()` utility - Token counting to prevent overflow - Per-message model override support **Invoked By**: Chat API (`POST /chat/execute`) --- ### 3. **Ask Workflow** (`open_notebook/graphs/ask.py`) **Purpose**: Answer user questions by searching sources and synthesizing responses. **Flow**: ``` User Question ↓ Plan Search Strategy (LLM generates searches) ↓ Execute Searches (vector + text search) ↓ Score & Rank Results ↓ Provide Answers (LLM synthesizes from results) ↓ Stream Responses ↓ Output (final answer) ``` **State Dict**: ```python { "question": str, "strategy": SearchStrategy, "answers": List[str], "final_answer": str, "sources_used": List[Source], } ``` **Streaming**: Uses `astream()` to emit updates in real-time (strategy → answers → final answer) **Invoked By**: Search API (`POST /ask` with streaming) --- ### 4. **Transformation Workflow** (`open_notebook/graphs/transformation.py`) **Purpose**: Apply custom transformations to sources (extract summaries, key points, etc). **Flow**: ``` Source + Transformation Rule ↓ Generate Prompt (Jinja2 template) ↓ Call LLM ↓ Parse Output ↓ Create SourceInsight record ↓ Output (insight with type + content) ``` **Example Transformations**: - Summary (5-sentence overview) - Key Points (bulleted list) - Quotes (notable excerpts) - Q&A (generated questions and answers) **Invoked By**: Sources API (`POST /sources/{id}/insights`) --- ### 5. **Prompt Workflow** (`open_notebook/graphs/prompt.py`) **Purpose**: Generic LLM task execution (e.g., auto-generate note titles, analyze content). **Flow**: ``` Input Text + Prompt ↓ Call LLM (simple request-response) ↓ Output (completion) ``` **Used For**: Note title generation, content analysis, etc. --- ## AI Provider Integration Pattern ### ModelManager: Centralized Factory Located in `open_notebook/ai/models.py`, ModelManager handles: 1. **Provider Detection**: Check environment variables for available providers 2. **Model Selection**: Choose best model based on context size and task 3. **Fallback Logic**: If primary provider unavailable, try backup 4. **Cost Optimization**: Prefer cheaper models for simple tasks 5. **Token Calculation**: Estimate cost before LLM call **Usage**: ```python from open_notebook.ai.provision import provision_langchain_model # Get best LLM for context size model = await provision_langchain_model( task="chat", # or "search", "extraction" model_override="anthropic/claude-opus-4", # optional context_size=8000, # estimated tokens ) # Invoke model response = await model.ainvoke({"input": prompt}) ``` ### Multi-Provider Support **LLM Providers**: - OpenAI (gpt-4, gpt-4-turbo, gpt-3.5-turbo) - Anthropic (claude-opus, claude-sonnet, claude-haiku) - Google (gemini-pro, gemini-1.5) - Groq (mixtral, llama-2) - Ollama (local models) - Mistral (mistral-large, mistral-medium) - DeepSeek (deepseek-chat) - xAI (grok) **Embedding Providers**: - OpenAI (text-embedding-3-large, text-embedding-3-small) - Google (embedding-001) - Ollama (local embeddings) - Mistral (mistral-embed) - Voyage (voyage-large-2) **TTS Providers**: - OpenAI (tts-1, tts-1-hd) - Groq (no TTS, fallback to OpenAI) - ElevenLabs (multilingual voices) - Google TTS (text-to-speech) ### Per-Request Override Every LangGraph invocation accepts a `config` parameter to override models: ```python result = await graph.ainvoke( input={...}, config={ "configurable": { "model_override": "anthropic/claude-opus-4" # Use Claude instead } } ) ``` --- ## Design Patterns ### 1. **Domain-Driven Design (DDD)** **Domain Objects** (`open_notebook/domain/`): - `Notebook`: Research container with relationships to sources/notes - `Source`: Content item (PDF, URL, text) with embeddings - `Note`: User-created or AI-generated research note - `ChatSession`: Conversation history for a notebook - `Transformation`: Custom rule for extracting insights **Repository Pattern**: - Database access layer (`open_notebook/database/repository.py`) - `repo_query()`: Execute SurrealQL queries - `repo_create()`: Insert records - `repo_upsert()`: Merge records - `repo_delete()`: Remove records **Entity Methods**: ```python # Domain methods (business logic) notebook = await Notebook.get(id) await notebook.save() notes = await notebook.get_notes() sources = await notebook.get_sources() ``` ### 2. **Async-First Architecture** **All I/O is async**: - Database queries: `await repo_query(...)` - LLM calls: `await model.ainvoke(...)` - File I/O: `await upload_file.read()` - Graph invocations: `await graph.ainvoke(...)` **Benefits**: - Non-blocking request handling (FastAPI serves multiple concurrent requests) - Better resource utilization (I/O waiting doesn't block CPU) - Natural fit for Python async/await syntax **Example**: ```python @router.post("/sources") async def create_source(source_data: SourceCreate): # All operations are non-blocking source = Source(title=source_data.title) await source.save() # async database operation await graph.ainvoke({...}) # async LangGraph invocation return SourceResponse(...) ``` ### 3. **Service Pattern** Services orchestrate domain objects, repositories, and workflows: ```python # api/notebook_service.py class NotebookService: async def get_notebook_with_stats(notebook_id: str): notebook = await Notebook.get(notebook_id) sources = await notebook.get_sources() notes = await notebook.get_notes() return { "notebook": notebook, "source_count": len(sources), "note_count": len(notes), } ``` **Responsibilities**: - Validate inputs (Pydantic) - Orchestrate database operations - Invoke workflows (LangGraph graphs) - Handle errors and return appropriate status codes - Log operations ### 4. **Streaming Pattern** For long-running operations (ask workflow, podcast generation), stream results as Server-Sent Events: ```python @router.post("/ask", response_class=StreamingResponse) async def ask(request: AskRequest): async def stream_response(): async for chunk in ask_graph.astream(input={...}): yield f"data: {json.dumps(chunk)}\n\n" return StreamingResponse(stream_response(), media_type="text/event-stream") ``` ### 5. **Job Queue Pattern** For async background tasks (source processing), use Surreal-Commands job queue: ```python # Submit job command_id = await CommandService.submit_command_job( app="open_notebook", command="process_source", input={...} ) # Poll status status = await source.get_status() ``` --- ## Service Communication Patterns ### Frontend → API 1. **REST requests** (HTTP GET/POST/PUT/DELETE) 2. **JSON request/response bodies** 3. **Standard HTTP status codes** (200, 400, 404, 500) 4. **Optional streaming** (Server-Sent Events for long operations) **Example**: ```typescript // Frontend const response = await fetch("http://localhost:5055/sources", { method: "POST", body: formData, // multipart/form-data for file upload }); const source = await response.json(); ``` ### API → SurrealDB 1. **SurrealQL queries** (similar to SQL) 2. **Async driver** with connection pooling 3. **Type-safe record IDs** (record_id syntax) 4. **Transaction support** for multi-step operations **Example**: ```python # API result = await repo_query( "SELECT * FROM source WHERE notebook = $notebook_id", {"notebook_id": ensure_record_id(notebook_id)} ) ``` ### API → AI Providers (via Esperanto) 1. **Esperanto unified interface** 2. **Per-request provider override** 3. **Automatic fallback on failure** 4. **Token counting and cost estimation** **Example**: ```python # API model = await provision_langchain_model(task="chat") response = await model.ainvoke({"input": prompt}) ``` ### API → Job Queue (Surreal-Commands) 1. **Async job submission** 2. **Fire-and-forget pattern** 3. **Status polling via `/commands/{id}` endpoint** 4. **Job completion callbacks (optional)** **Example**: ```python # Submit async source processing command_id = await CommandService.submit_command_job(...) # Client polls status response = await fetch(f"http://localhost:5055/commands/{command_id}") status = await response.json() # returns { status: "running|queued|completed|failed" } ``` --- ## Database Schema Overview ### Core Schema Structure **Tables** (20+): - Notebooks (with soft-delete via `archived` flag) - Sources (content + metadata) - SourceEmbeddings (vector chunks) - Notes (user-created + AI-generated) - ChatSessions (conversation history) - Transformations (custom rules) - SourceInsights (transformation outputs) - Relationships (notebook→source, notebook→note) **Migrations**: - Automatic on API startup - Located in `/migrations/` directory - Numbered sequentially (001_*.surql, 002_*.surql, etc) - Tracked in `_sbl_migrations` table - Rollback via `_down.surql` files (manual) ### Relationship Model **Graph Relationships**: ``` Notebook ← reference ← Source (many:many) ← artifact ← Note (many:many) Source → source_embedding (one:many) → source_insight (one:many) → embedding (via source_embedding) ChatSession → messages (JSON array in database) → notebook_id (reference to Notebook) Transformation → source_insight (one:many) ``` **Query Example** (get all sources in a notebook with counts): ```sql SELECT id, title, count(<-reference.in) as note_count, count(<-embedding.in) as embedded_chunks FROM source WHERE notebook = $notebook_id ORDER BY updated DESC ``` --- ## Key Architectural Decisions ### 1. **Async Throughout** All I/O operations are non-blocking to maximize concurrency and responsiveness. **Trade-off**: Slightly more complex code (async/await syntax) vs. high throughput. ### 2. **Multi-Provider from Day 1** Built-in support for 8+ AI providers prevents vendor lock-in. **Trade-off**: Added complexity in ModelManager vs. flexibility and cost optimization. ### 3. **Graph-First Workflows** LangGraph state machines for complex multi-step operations (ask, chat, transformations). **Trade-off**: Steeper learning curve vs. maintainable, debuggable workflows. ### 4. **Self-Hosted Database** SurrealDB for graph + vector search in one system (no external dependencies). **Trade-off**: Operational responsibility vs. simplified architecture and cost savings. ### 5. **Job Queue for Long-Running Tasks** Async job submission (source processing, podcast generation) prevents request timeouts. **Trade-off**: Eventual consistency vs. responsive user experience. --- ## Important Quirks & Gotchas ### API Startup - **Migrations run automatically** on every startup; check logs for errors - **SurrealDB must be running** before starting API (connection test in lifespan) - **Auth middleware is basic** (password-only); upgrade to OAuth/JWT for production ### Database Operations - **Record IDs use SurrealDB syntax** (table:id format, e.g., "notebook:abc123") - **ensure_record_id()** helper prevents malformed IDs - **Soft deletes** via `archived` field (data not removed, just marked inactive) - **Timestamps in ISO 8601 format** (created, updated fields) ### LangGraph Workflows - **State persistence** via SqliteSaver in `/data/sqlite-db/` - **No built-in timeout**; long workflows may block requests (use streaming for UX) - **Model fallback** automatic if primary provider unavailable - **Checkpoint IDs** must be unique per session (avoid collisions) ### AI Provider Integration - **Esperanto library** handles all provider APIs (no direct API calls) - **Per-request override** via RunnableConfig (temporary, not persistent) - **Cost estimation** via token counting (not 100% accurate, use for guidance) - **Fallback logic** tries cheaper models if primary fails ### File Uploads - **Stored in `/data/uploads/`** directory (not database) - **Unique filename generation** prevents overwrites (counter suffix) - **Content-core library** extracts text from 50+ file types - **Large files** may block API briefly (sync content extraction) --- ## Performance Considerations ### Optimization Strategies 1. **Connection Pooling**: SurrealDB async driver with configurable pool size 2. **Query Caching**: TanStack Query on frontend (client-side caching) 3. **Embedding Reuse**: Vector search uses pre-computed embeddings 4. **Chunking**: Sources split into chunks for better search relevance 5. **Async Operations**: Non-blocking I/O for high concurrency 6. **Lazy Loading**: Frontend requests only needed data (pagination) ### Bottlenecks 1. **LLM Calls**: Latency depends on provider (typically 1-30 seconds) 2. **Embedding Generation**: Time proportional to content size and provider 3. **Vector Search**: Similarity computation over all embeddings 4. **Content Extraction**: Sync operation in source processing ### Monitoring - **API Logs**: Check loguru output for errors and slow operations - **Database Queries**: SurrealDB metrics available via admin UI - **Token Usage**: Estimated via `estimate_tokens()` utility - **Job Status**: Poll `/commands/{id}` for async operations --- ## Extension Points ### Adding a New Workflow 1. Create `open_notebook/graphs/workflow_name.py` 2. Define StateDict and node functions 3. Build graph with `.add_node()` / `.add_edge()` 4. Create service in `api/workflow_service.py` 5. Register router in `api/main.py` 6. Add tests in `tests/test_workflow.py` ### Adding a New Data Model 1. Create model in `open_notebook/domain/model_name.py` 2. Inherit from BaseModel (domain object) 3. Implement `save()`, `get()`, `delete()` methods (CRUD) 4. Add repository functions if complex queries needed 5. Create database migration in `migrations/` 6. Add API routes and models in `api/` ### Adding a New AI Provider 1. Configure Esperanto for new provider (see .env.example) 2. ModelManager automatically detects via environment variables 3. Override via per-request config (no code changes needed) 4. Test fallback logic if provider unavailable --- ## Deployment Considerations ### Development - All services on localhost (3000, 5055, 8000) - Auto-reload on file changes (Next.js, FastAPI) - Hot-reload database migrations - Open API docs at http://localhost:5055/docs ### Production - **Frontend**: Deploy to Vercel, Netlify, or Docker - **API**: Docker container (see Dockerfile) - **Database**: SurrealDB container or managed service - **Environment**: Secure .env file with API keys - **SSL/TLS**: Reverse proxy (Nginx, CloudFlare) - **Rate Limiting**: Add at proxy layer - **Auth**: Replace PasswordAuthMiddleware with OAuth/JWT - **Monitoring**: Log aggregation (CloudWatch, DataDog, etc) --- ## Summary Open Notebook's architecture provides a solid foundation for privacy-focused, AI-powered research. The separation of concerns (frontend/API/database), async-first design, and multi-provider flexibility enable rapid development and easy deployment. LangGraph workflows orchestrate complex AI tasks, while Esperanto abstracts provider details. The result is a scalable, maintainable system that puts users in control of their data and AI provider choice. ================================================ FILE: docs/7-DEVELOPMENT/code-standards.md ================================================ # Code Standards This document outlines coding standards and best practices for Open Notebook contributions. All code should follow these guidelines to ensure consistency, readability, and maintainability. ## Python Standards ### Code Formatting We follow **PEP 8** with some specific guidelines: - Use **Ruff** for linting and formatting - Maximum line length: **88 characters** - Use **double quotes** for strings - Use **trailing commas** in multi-line structures ### Type Hints Always use type hints for function parameters and return values: ```python from typing import List, Optional, Dict, Any from pydantic import BaseModel async def process_content( content: str, options: Optional[Dict[str, Any]] = None ) -> ProcessedContent: """Process content with optional configuration.""" # Implementation ``` ### Async/Await Patterns Use async/await consistently throughout the codebase: ```python # Good async def fetch_data(url: str) -> Dict[str, Any]: async with aiohttp.ClientSession() as session: async with session.get(url) as response: return await response.json() # Bad - mixing sync and async def fetch_data(url: str) -> Dict[str, Any]: loop = asyncio.get_event_loop() return loop.run_until_complete(async_fetch(url)) ``` ### Error Handling Use structured error handling with custom exceptions: ```python from open_notebook.exceptions import DatabaseOperationError, InvalidInputError async def create_notebook(name: str, description: str) -> Notebook: """Create a new notebook with validation.""" if not name.strip(): raise InvalidInputError("Notebook name cannot be empty") try: notebook = Notebook(name=name, description=description) await notebook.save() return notebook except Exception as e: raise DatabaseOperationError(f"Failed to create notebook: {str(e)}") ``` ### Documentation (Google-style Docstrings) Use Google-style docstrings for all functions, classes, and modules: ```python async def vector_search( query: str, limit: int = 10, minimum_score: float = 0.2 ) -> List[SearchResult]: """Perform vector search across embedded content. Args: query: Search query string limit: Maximum number of results to return minimum_score: Minimum similarity score for results Returns: List of search results sorted by relevance score Raises: InvalidInputError: If query is empty or limit is invalid DatabaseOperationError: If search operation fails """ # Implementation ``` #### Module Docstrings ```python """ Notebook domain model and operations. This module contains the core Notebook class and related operations for managing research notebooks within the Open Notebook system. """ ``` #### Class Docstrings ```python class Notebook(BaseModel): """A research notebook containing sources, notes, and chat sessions. Notebooks are the primary organizational unit in Open Notebook, allowing users to group related research materials and maintain separate contexts for different projects. Attributes: name: The notebook's display name description: Optional description of the notebook's purpose archived: Whether the notebook is archived (default: False) created: Timestamp of creation updated: Timestamp of last update """ ``` #### Function Docstrings ```python async def create_notebook( name: str, description: str = "", user_id: Optional[str] = None ) -> Notebook: """Create a new notebook with validation. Args: name: The notebook name (required, non-empty) description: Optional notebook description user_id: Optional user ID for multi-user deployments Returns: The created notebook instance Raises: InvalidInputError: If name is empty or invalid DatabaseOperationError: If creation fails Example: ```python notebook = await create_notebook( name="AI Research", description="Research on AI applications" ) ``` """ ``` ## FastAPI Standards ### Router Organization Organize endpoints by domain: ```python # api/routers/notebooks.py from fastapi import APIRouter, HTTPException, Query from typing import List, Optional router = APIRouter() @router.get("/notebooks", response_model=List[NotebookResponse]) async def get_notebooks( archived: Optional[bool] = Query(None, description="Filter by archived status"), order_by: str = Query("updated desc", description="Order by field and direction"), ): """Get all notebooks with optional filtering and ordering.""" # Implementation ``` ### Request/Response Models Use Pydantic models for validation: ```python from pydantic import BaseModel, Field from typing import Optional class NotebookCreate(BaseModel): name: str = Field(..., description="Name of the notebook", min_length=1) description: str = Field(default="", description="Description of the notebook") class NotebookResponse(BaseModel): id: str name: str description: str archived: bool created: str updated: str ``` ### Error Handling Use consistent error responses: ```python from fastapi import HTTPException from loguru import logger try: result = await some_operation() return result except InvalidInputError as e: raise HTTPException(status_code=400, detail=str(e)) except DatabaseOperationError as e: logger.error(f"Database error: {str(e)}") raise HTTPException(status_code=500, detail="Internal server error") ``` ### API Documentation Use FastAPI's automatic documentation features: ```python @router.post( "/notebooks", response_model=NotebookResponse, summary="Create a new notebook", description="Create a new notebook with the specified name and description.", responses={ 201: {"description": "Notebook created successfully"}, 400: {"description": "Invalid input data"}, 500: {"description": "Internal server error"} } ) async def create_notebook(notebook: NotebookCreate): """Create a new notebook.""" # Implementation ``` ## Database Standards ### SurrealDB Patterns Use the repository pattern consistently: ```python from open_notebook.database.repository import repo_create, repo_query, repo_update # Create records async def create_notebook(data: Dict[str, Any]) -> Dict[str, Any]: """Create a new notebook record.""" return await repo_create("notebook", data) # Query with parameters async def find_notebooks_by_user(user_id: str) -> List[Dict[str, Any]]: """Find notebooks for a specific user.""" return await repo_query( "SELECT * FROM notebook WHERE user_id = $user_id", {"user_id": user_id} ) # Update records async def update_notebook(notebook_id: str, data: Dict[str, Any]) -> Dict[str, Any]: """Update a notebook record.""" return await repo_update("notebook", notebook_id, data) ``` ### Schema Management Use migrations for schema changes: ```surrealql -- migrations/8.surrealql DEFINE TABLE IF NOT EXISTS new_feature SCHEMAFULL; DEFINE FIELD IF NOT EXISTS name ON TABLE new_feature TYPE string; DEFINE FIELD IF NOT EXISTS description ON TABLE new_feature TYPE option; DEFINE FIELD IF NOT EXISTS created ON TABLE new_feature TYPE datetime DEFAULT time::now(); DEFINE FIELD IF NOT EXISTS updated ON TABLE new_feature TYPE datetime DEFAULT time::now(); ``` ## TypeScript Standards ### Basic Guidelines Follow TypeScript best practices: - Use strict mode enabled in `tsconfig.json` - Use proper type annotations for all variables and functions - Avoid using `any` type unless absolutely necessary - Use `interface` for object shapes, `type` for unions and other advanced types ### Component Structure - Use functional components with hooks - Keep components focused and single-responsibility - Extract reusable logic into custom hooks - Use proper TypeScript types for props ### Error Handling - Handle errors explicitly - Provide meaningful error messages - Log errors appropriately - Don't suppress errors silently ## Code Quality Tools We use these tools to maintain code quality: - **Ruff**: Linting and code formatting - Run with: `uv run ruff check . --fix` - Format with: `uv run ruff format .` - **MyPy**: Static type checking - Run with: `uv run python -m mypy .` - **Pytest**: Testing framework - Run with: `uv run pytest` ## Common Patterns ### Async Database Operations ```python async def get_notebook_with_sources(notebook_id: str) -> Notebook: """Retrieve notebook with all related sources.""" notebook_data = await repo_query( "SELECT * FROM notebook WHERE id = $id", {"id": notebook_id} ) if not notebook_data: raise InvalidInputError(f"Notebook {notebook_id} not found") sources_data = await repo_query( "SELECT * FROM source WHERE notebook_id = $notebook_id", {"notebook_id": notebook_id} ) return Notebook( **notebook_data[0], sources=[Source(**s) for s in sources_data] ) ``` ### Model Validation ```python from pydantic import BaseModel, validator class NotebookInput(BaseModel): name: str description: str = "" @validator('name') def name_not_empty(cls, v): if not v.strip(): raise ValueError('Name cannot be empty') return v.strip() ``` ## Code Review Checklist Before submitting code for review, ensure: - [ ] Code follows PEP 8 / TypeScript best practices - [ ] Type hints are present for all functions - [ ] Docstrings are complete and accurate - [ ] Error handling is appropriate - [ ] Tests are included and passing - [ ] No debug code (console.logs, print statements) left behind - [ ] Commit messages are clear and follow conventions - [ ] Documentation is updated if needed --- **See also:** - [Testing Guide](testing.md) - How to write tests - [Contributing Guide](contributing.md) - Overall contribution workflow ================================================ FILE: docs/7-DEVELOPMENT/contributing.md ================================================ # Contributing to Open Notebook Thank you for your interest in contributing to Open Notebook! We welcome contributions from developers of all skill levels. This guide will help you understand our contribution workflow and what makes a good contribution. ## 🚨 Issue-First Workflow **To maintain project coherence and avoid wasted effort, please follow this process:** 1. **Create an issue first** - Before writing any code, create an issue describing the bug or feature 2. **Propose your solution** - Explain how you plan to implement the fix or feature 3. **Wait for assignment** - A maintainer will review and assign the issue to you if approved 4. **Only then start coding** - This ensures your work aligns with the project's vision and architecture **Why this process?** - Prevents duplicate work - Ensures solutions align with our architecture and design principles - Saves your time by getting feedback before coding - Helps maintainers manage the project direction > ⚠️ **Pull requests without an assigned issue may be closed**, even if the code is good. We want to respect your time by making sure work is aligned before it starts. ## Code of Conduct By participating in this project, you are expected to uphold our Code of Conduct. Be respectful, constructive, and collaborative. ## How Can I Contribute? ### Reporting Bugs 1. **Search existing issues** - Check if the bug was already reported 2. **Create a bug report** - Use the [Bug Report template](https://github.com/lfnovo/open-notebook/issues/new?template=bug_report.yml) 3. **Provide details** - Include: - Steps to reproduce - Expected vs actual behavior - Logs, screenshots, or error messages - Your environment (OS, Docker version, Open Notebook version) 4. **Indicate if you want to fix it** - Check the "I would like to work on this" box if you're interested ### Suggesting Features 1. **Search existing issues** - Check if the feature was already suggested 2. **Create a feature request** - Use the [Feature Request template](https://github.com/lfnovo/open-notebook/issues/new?template=feature_request.yml) 3. **Explain the value** - Describe why this feature would be helpful 4. **Propose implementation** - If you have ideas on how to implement it, share them 5. **Indicate if you want to build it** - Check the "I would like to work on this" box if you're interested ### Contributing Code (Pull Requests) **IMPORTANT: Follow the issue-first workflow above before starting any PR** Once your issue is assigned: 1. **Fork the repo** and create your branch from `main` 2. **Understand our vision and principles** - Read [design-principles.md](design-principles.md) to understand what guides our decisions 3. **Follow our architecture** - Refer to the architecture documentation to understand project structure 4. **Write quality code** - Follow the standards outlined in [code-standards.md](code-standards.md) 5. **Test your changes** - See [testing.md](testing.md) for test guidelines 6. **Update documentation** - If you changed functionality, update the relevant docs 7. **Create your PR**: - Reference the issue number (e.g., "Fixes #123") - Describe what changed and why - Include screenshots for UI changes - Keep PRs focused - one issue per PR ### What Makes a Good Contribution? ✅ **We love PRs that:** - Solve a real problem described in an issue - Follow our architecture and coding standards - Include tests and documentation - Are well-scoped (focused on one thing) - Have clear commit messages ❌ **We may close PRs that:** - Don't have an associated approved issue - Introduce breaking changes without discussion - Conflict with our architectural vision - Lack tests or documentation - Try to solve multiple unrelated problems ## Git Commit Messages - Use the present tense ("Add feature" not "Added feature") - Use the imperative mood ("Move cursor to..." not "Moves cursor to...") - Limit the first line to 72 characters or less - Reference issues and pull requests liberally after the first line ## Development Workflow ### Branch Strategy We use a **feature branch workflow**: 1. **Main Branch**: `main` - production-ready code 2. **Feature Branches**: `feature/description` - new features 3. **Bug Fixes**: `fix/description` - bug fixes 4. **Documentation**: `docs/description` - documentation updates ### Making Changes 1. **Create a feature branch**: ```bash git checkout -b feature/amazing-new-feature ``` 2. **Make your changes** following our coding standards 3. **Test your changes**: ```bash # Run tests uv run pytest # Run linting uv run ruff check . # Run formatting uv run ruff format . ``` 4. **Commit your changes**: ```bash git add . git commit -m "feat: add amazing new feature" ``` 5. **Push and create PR**: ```bash git push origin feature/amazing-new-feature # Then create a Pull Request on GitHub ``` ### Keeping Your Fork Updated ```bash # Fetch upstream changes git fetch upstream # Switch to main and merge git checkout main git merge upstream/main # Push to your fork git push origin main ``` ## Pull Request Process When you create a pull request: 1. **Link your issue** - Reference the issue number in PR description 2. **Describe your changes** - Explain what changed and why 3. **Provide test evidence** - Screenshots, test results, or logs 4. **Check PR template** - Ensure you've completed all required sections 5. **Wait for review** - A maintainer will review your PR within a week ### PR Review Expectations - Code review feedback is about the code, not the person - Be open to suggestions and alternative approaches - Address review comments with clarity and respect - Ask questions if feedback is unclear ## Current Priority Areas We're actively looking for contributions in these areas: 1. **Frontend Enhancement** - Help improve the Next.js/React UI with real-time updates and better UX 2. **Testing** - Expand test coverage across all components 3. **Performance** - Async processing improvements and caching 4. **Documentation** - API examples and user guides 5. **Integrations** - New content sources and AI providers ## Getting Help ### Community Support - **Discord**: [Join our Discord server](https://discord.gg/37XJPXfz2w) for real-time help - **GitHub Discussions**: For longer-form questions and ideas - **GitHub Issues**: For bug reports and feature requests ### Documentation References - [Design Principles](design-principles.md) - Understanding our project vision - [Code Standards](code-standards.md) - Coding guidelines by language - [Testing Guide](testing.md) - How to write tests - [Development Setup](development-setup.md) - Getting started locally ## Recognition We recognize contributions through: - **GitHub credits** on releases - **Community recognition** in Discord - **Contribution statistics** in project analytics - **Maintainer consideration** for active contributors --- Thank you for contributing to Open Notebook! Your contributions help make research more accessible and private for everyone. For questions about this guide or contributing in general, please reach out on [Discord](https://discord.gg/37XJPXfz2w) or open a GitHub Discussion. ================================================ FILE: docs/7-DEVELOPMENT/design-principles.md ================================================ # Design Principles & Project Vision This document outlines the core principles, vision, and design philosophy that guide Open Notebook's development. All contributors should read and understand these principles before proposing changes or new features. ## 🎯 Project Vision Open Notebook aims to be a **privacy-focused, self-hosted alternative to Google's Notebook LM** that empowers users to: 1. **Own their research data** - Full control over where data lives and who can access it 2. **Choose their AI providers** - Freedom to use any AI provider or run models locally 3. **Customize their workflows** - Flexibility to adapt the tool to different research needs 4. **Access their work anywhere** - Through web UI, API, or integrations ### What Open Notebook IS - A **research assistant** for managing and understanding content - A **platform** that connects various AI providers - A **privacy-first** tool that keeps your data under your control - An **extensible system** with APIs and customization options ### What Open Notebook IS NOT - A document editor (use Google Docs, Notion, etc. for that) - A file storage system (use Dropbox, S3, etc. for that) - A general-purpose chatbot (use ChatGPT, Claude, etc. for that) - A replacement for your entire workflow (it's one tool in your toolkit) ## 🏗️ Core Design Principles ### 1. Privacy First **Principle**: User data and research should stay under user control by default. **In Practice**: - Self-hosted deployment is the primary use case - No telemetry or analytics without explicit opt-in - No hard dependency on specific cloud services - Clear documentation on what data goes where **Example Decisions**: - ✅ Support for local Ollama models - ✅ Configurable AI provider selection - ❌ Hard-coded cloud service integrations - ❌ Required external service dependencies ### 2. Simplicity Over Features **Principle**: The tool should be easy to understand and use, even if it means fewer features. **In Practice**: - Clear, focused UI with well-defined sections - Sensible defaults that work for most users - Advanced features hidden behind optional configuration - Documentation written for non-technical users **Example Decisions**: - ✅ Three-column layout (Sources, Notes, Chat) - ✅ Default models that work out of the box - ❌ Overwhelming users with too many options upfront - ❌ Complex multi-step workflows for basic tasks ### 3. API-First Architecture **Principle**: All functionality should be accessible via API, not just the UI. **In Practice**: - UI calls the same API that external clients use - Comprehensive REST API with OpenAPI documentation - No "UI-only" features that can't be automated - Clear separation between frontend and backend **Example Decisions**: - ✅ FastAPI backend with full API documentation - ✅ Consistent API patterns across all endpoints - ❌ Business logic in UI components - ❌ Features that require direct database access ### 4. Multi-Provider Flexibility **Principle**: Users should never be locked into a single AI provider. **In Practice**: - Support for multiple AI providers through Esperanto library - Easy switching between providers and models - Clear documentation on provider limitations - Graceful degradation when providers are unavailable **Example Decisions**: - ✅ Support for 16+ AI providers - ✅ Per-feature model selection (chat, embeddings, TTS) - ❌ Features that only work with OpenAI - ❌ Hard-coded API endpoints for specific providers ### 5. Extensibility Through Standards **Principle**: The system should be extensible through well-defined interfaces, not by forking. **In Practice**: - Plugin systems for transformations and commands - Standard data formats (JSON, Markdown) - Clear extension points in the architecture - Documentation for common customization scenarios **Example Decisions**: - ✅ Custom transformation templates - ✅ Background command system - ✅ Jinja2 prompt templates - ❌ Hard-coded business logic without extension points ### 6. Async-First for Performance **Principle**: Long-running operations should not block the user interface or API. **In Practice**: - Async/await patterns throughout the backend - Background job processing for heavy workloads - Status updates and progress tracking - Graceful handling of slow AI provider responses **Example Decisions**: - ✅ AsyncIO for database operations - ✅ Background commands for podcast generation - ✅ Streaming responses for chat - ❌ Synchronous blocking operations in API endpoints ## 🎨 UI/UX Principles ### Focus on Content, Not Chrome - Minimize UI clutter and distractions - Content should occupy most of the screen space - Controls appear when needed, not always visible - Consistent layout across different views ### Progressive Disclosure - Show simple options first, advanced options on demand - Don't overwhelm new users with every possible setting - Provide sensible defaults that work for 80% of use cases - Make power features discoverable but not intrusive ### Responsive and Fast - UI should feel instant for common operations - Show loading states for operations that take time - Cache and optimize where possible - Degrade gracefully on slow connections ## 🔧 Technical Principles ### Clean Separation of Concerns **Layers should not leak**: - Frontend should not know about database structure - API should not contain business logic (delegate to domain layer) - Domain models should not know about HTTP requests - Database layer should not know about AI providers ### Type Safety and Validation **Catch errors early**: - Use Pydantic models for all API boundaries - Type hints throughout Python codebase - TypeScript for frontend code - Validate data at system boundaries ### Test What Matters **Focus on valuable tests**: - Test business logic and domain models - Test API contracts and error handling - Don't test framework code (FastAPI, React, etc.) - Integration tests for critical workflows ### Database as Source of Truth **SurrealDB is our single source of truth**: - All state persisted in database - No business logic in database layer - Use SurrealDB features (record links, queries) appropriately - Schema migrations for all schema changes ## 🚫 Anti-Patterns to Avoid ### Feature Creep **What it looks like**: - Adding features because they're "cool" or "easy" - Building features for edge cases before common cases work well - Trying to be everything to everyone **Why we avoid it**: - Increases complexity and maintenance burden - Makes the tool harder to learn and use - Dilutes the core value proposition **Instead**: - Focus on core use cases - Say no to features that don't align with vision - Build extensibility points for edge cases ### Premature Optimization **What it looks like**: - Optimizing code before knowing if it's slow - Complex caching strategies without measuring impact - Trading code clarity for marginal performance gains **Why we avoid it**: - Makes code harder to understand and maintain - Optimizes the wrong things - Wastes development time **Instead**: - Measure first, optimize second - Focus on algorithmic improvements - Profile before making performance changes ### Over-Engineering **What it looks like**: - Building abstraction layers "in case we need them later" - Implementing design patterns for 3-line functions - Creating frameworks instead of solving problems **Why we avoid it**: - Increases cognitive load for contributors - Makes simple changes require touching many files - Hides the actual business logic **Instead**: - Start simple, refactor when patterns emerge - Optimize for readability and clarity - Use abstractions when they simplify, not complicate ### Breaking Changes Without Migration Path **What it looks like**: - Changing database schema without migration scripts - Modifying API contracts without versioning - Removing features without deprecation warnings **Why we avoid it**: - Breaks existing installations - Frustrates users and contributors - Creates maintenance nightmares **Instead**: - Always provide migration scripts for schema changes - Deprecate before removing - Document breaking changes clearly ## 🤝 Decision-Making Framework When evaluating new features or changes, ask: ### 1. Does it align with our vision? - Does it help users own their research data? - Does it support privacy and self-hosting? - Does it fit our core use cases? ### 2. Does it follow our principles? - Is it simple to use and understand? - Does it work via API? - Does it support multiple providers? - Can it be extended by users? ### 3. Is the implementation sound? - Does it maintain separation of concerns? - Is it properly typed and validated? - Does it include tests? - Is it documented? ### 4. What is the cost? - How much complexity does it add? - How much maintenance burden? - Does it introduce new dependencies? - Will it be used enough to justify the cost? ### 5. Are there alternatives? - Can existing features solve this problem? - Can this be built as a plugin or extension? - Should this be a separate tool instead? ## 📚 Examples of Principle-Driven Decisions ### Why we migrated from Streamlit to Next.js **Principle**: API-First Architecture **Reasoning**: - Streamlit coupled UI and backend logic - Difficult to build external integrations - Limited control over API behavior - Next.js + FastAPI provides clear separation ### Why we use Esperanto for AI providers **Principle**: Multi-Provider Flexibility **Reasoning**: - Abstracts provider-specific details - Easy to add new providers - Consistent interface across providers - No vendor lock-in ### Why we have a Background Command System **Principle**: Async-First for Performance **Reasoning**: - Podcast generation takes minutes - Users shouldn't wait for long operations - Need status tracking and error handling - Supports future batch operations ### Why we support Local Ollama **Principle**: Privacy First **Reasoning**: - Enables fully offline operation - No data sent to external services - Free for users after hardware cost - Aligns with self-hosted philosophy ## 🔄 Evolution of Principles These principles are not set in stone. As the project grows and we learn from users, some principles may evolve. However, changes to core principles should be: 1. **Well-justified** - Clear reasoning for why the change is needed 2. **Discussed openly** - Community input on major changes 3. **Documented** - Updated in this document with explanation 4. **Gradual** - Not implemented as breaking changes when possible --- ## For Contributors When proposing a feature or change: 1. **Reference these principles** - Explain how your proposal aligns 2. **Identify trade-offs** - Be honest about what you're trading for what 3. **Suggest alternatives** - Show you've considered other approaches 4. **Be open to feedback** - Maintainers may see concerns you don't **Remember**: A "no" to a feature isn't a judgment on you or your idea. It means we're staying focused on our core vision. We appreciate all contributions and ideas! --- **Questions about these principles?** Open a discussion on GitHub or join our [Discord](https://discord.gg/37XJPXfz2w). ================================================ FILE: docs/7-DEVELOPMENT/development-setup.md ================================================ # Local Development Setup This guide walks you through setting up Open Notebook for local development. Follow these steps to get the full stack running on your machine. ## Prerequisites Before you start, ensure you have the following installed: - **Python 3.11+** - Check with: `python --version` - **uv** (recommended) or **pip** - Install from: https://github.com/astral-sh/uv - **SurrealDB** - Via Docker or binary (see below) - **Docker** (optional) - For containerized database - **Node.js 18+** (optional) - For frontend development - **Git** - For version control ## Step 1: Clone and Initial Setup ```bash # Clone the repository git clone https://github.com/lfnovo/open-notebook.git cd open-notebook # Add upstream remote for keeping your fork updated git remote add upstream https://github.com/lfnovo/open-notebook.git ``` ## Step 2: Install Python Dependencies ```bash # Using uv (recommended) uv sync # Or using pip pip install -e . ``` ## Step 3: Environment Variables Create a `.env` file in the project root with your configuration: ```bash # Copy from example cp .env.example .env ``` Edit `.env` with your settings: ```bash # Database SURREAL_URL=ws://localhost:8000/rpc SURREAL_USER=root SURREAL_PASSWORD=password SURREAL_NAMESPACE=open_notebook SURREAL_DATABASE=development # Credential encryption (required for storing API keys) OPEN_NOTEBOOK_ENCRYPTION_KEY=my-dev-secret-key # Application APP_PASSWORD= # Optional password protection DEBUG=true LOG_LEVEL=DEBUG ``` ### AI Provider Configuration After starting the API and frontend, configure your AI provider via the Settings UI: 1. Open **http://localhost:3000** → **Settings** → **API Keys** 2. Click **Add Credential** → Select your provider 3. Enter your API key (get from provider dashboard) 4. Click **Save**, then **Test Connection** 5. Click **Discover Models** → **Register Models** Popular providers: - **OpenAI** - https://platform.openai.com/api-keys - **Anthropic (Claude)** - https://console.anthropic.com/ - **Google** - https://ai.google.dev/ - **Groq** - https://console.groq.com/ For local development, you can also use: - **Ollama** - Run locally without API keys (see "Local Ollama" below) > **Note:** API key environment variables (e.g., `OPENAI_API_KEY`) are deprecated. Use the Settings UI to manage credentials instead. ## Step 4: Start SurrealDB ### Option A: Using Docker (Recommended) ```bash # Start SurrealDB in memory docker run -d --name surrealdb -p 8000:8000 \ surrealdb/surrealdb:v2 start \ --user root --pass password \ --bind 0.0.0.0:8000 memory # Or with persistent storage docker run -d --name surrealdb -p 8000:8000 \ -v surrealdb_data:/data \ surrealdb/surrealdb:v2 start \ --user root --pass password \ --bind 0.0.0.0:8000 file:/data/surreal.db ``` ### Option B: Using Make ```bash make database ``` ### Option C: Using Docker Compose ```bash docker compose up -d surrealdb ``` ### Verify SurrealDB is Running ```bash # Should show server information curl http://localhost:8000/ ``` ## Step 5: Run Database Migrations Database migrations run automatically when you start the API. The first startup will apply any pending migrations. To verify migrations manually: ```bash # API will run migrations on startup uv run python -m api.main ``` Check the logs - you should see messages like: ``` Running migration 001_initial_schema Running migration 002_add_vectors ... Migrations completed successfully ``` ## Step 6: Start the API Server In a new terminal window: ```bash # Terminal 2: Start API (port 5055) uv run --env-file .env uvicorn api.main:app --host 0.0.0.0 --port 5055 # Or using the shortcut make api ``` You should see: ``` INFO: Application startup complete INFO: Uvicorn running on http://0.0.0.0:5055 ``` ### Verify API is Running ```bash # Check health endpoint curl http://localhost:5055/health # View API documentation open http://localhost:5055/docs ``` ## Step 7: Start the Frontend (Optional) If you want to work on the frontend, start Next.js in another terminal: ```bash # Terminal 3: Start Next.js frontend (port 3000) cd frontend npm install # First time only npm run dev ``` You should see: ``` > next dev ▲ Next.js 16.x - Local: http://localhost:3000 ``` ### Access the Frontend Open your browser to: http://localhost:3000 ## Verification Checklist After setup, verify everything is working: - [ ] **SurrealDB**: `curl http://localhost:8000/` returns content - [ ] **API**: `curl http://localhost:5055/health` returns `{"status": "ok"}` - [ ] **API Docs**: `open http://localhost:5055/docs` works - [ ] **Database**: API logs show migrations completing - [ ] **Frontend** (optional): `http://localhost:3000` loads ## Starting Services Together ### Quick Start All Services ```bash make start-all ``` This starts SurrealDB, API, and frontend in one command. ### Individual Terminals (Recommended for Development) **Terminal 1 - Database:** ```bash make database ``` **Terminal 2 - API:** ```bash make api ``` **Terminal 3 - Frontend:** ```bash cd frontend && npm run dev ``` ## Development Tools Setup ### Pre-commit Hooks (Optional but Recommended) Install git hooks to automatically check code quality: ```bash uv run pre-commit install ``` Now your commits will be checked before they're made. ### Code Quality Commands ```bash # Lint Python code (auto-fix) make ruff # or: ruff check . --fix # Type check Python code make lint # or: uv run python -m mypy . # Run tests uv run pytest # Run tests with coverage uv run pytest --cov=open_notebook ``` ## Common Development Tasks ### Running Tests ```bash # Run all tests uv run pytest # Run specific test file uv run pytest tests/test_notebooks.py # Run with coverage report uv run pytest --cov=open_notebook --cov-report=html ``` ### Creating a Feature Branch ```bash # Create and switch to new branch git checkout -b feature/my-feature # Make changes, then commit git add . git commit -m "feat: add my feature" # Push to your fork git push origin feature/my-feature ``` ### Updating from Upstream ```bash # Fetch latest changes git fetch upstream # Rebase your branch git rebase upstream/main # Push updated branch git push origin feature/my-feature -f ``` ## Troubleshooting ### "Connection refused" on SurrealDB **Problem**: API can't connect to SurrealDB **Solutions**: 1. Check if SurrealDB is running: `docker ps | grep surrealdb` 2. Verify URL in `.env`: Should be `ws://localhost:8000/rpc` 3. Restart SurrealDB: `docker stop surrealdb && docker rm surrealdb` 4. Then restart with: `docker run -d --name surrealdb -p 8000:8000 surrealdb/surrealdb:v2 start --user root --pass password --bind 0.0.0.0:8000 memory` ### "Address already in use" **Problem**: Port 5055 or 3000 is already in use **Solutions**: ```bash # Find process using port lsof -i :5055 # Check port 5055 # Kill process (macOS/Linux) kill -9 # Or use different port uvicorn api.main:app --port 5056 ``` ### Module not found errors **Problem**: Import errors when running API **Solutions**: ```bash # Reinstall dependencies uv sync # Or with pip pip install -e . ``` ### Database migration failures **Problem**: API fails to start with migration errors **Solutions**: 1. Check SurrealDB is running: `curl http://localhost:8000/` 2. Check credentials in `.env` match your SurrealDB setup 3. Check logs for specific migration error: `make api 2>&1 | grep -i migration` 4. Verify database exists: Check SurrealDB console at http://localhost:8000/ ### Migrations not applying **Problem**: Database schema seems outdated **Solutions**: 1. Restart API - migrations run on startup: `make api` 2. Check logs show "Migrations completed successfully" 3. Verify `/migrations/` folder exists and has files 4. Check SurrealDB is writable and not in read-only mode ## Optional: Local Ollama Setup For testing with local AI models: ```bash # Install Ollama from https://ollama.ai # Pull a model (e.g., Mistral 7B) ollama pull mistral ``` Then configure via the Settings UI: 1. Go to **Settings** → **API Keys** → **Add Credential** → **Ollama** 2. Enter base URL: `http://localhost:11434` 3. Click **Save**, then **Test Connection** 4. Click **Discover Models** → **Register Models** ## Optional: Docker Development Environment Run entire stack in Docker: ```bash # Start all services docker compose --profile multi up # Logs docker compose logs -f # Stop services docker compose down ``` ## Next Steps After setup is complete: 1. **Read the Contributing Guide** - [contributing.md](contributing.md) 2. **Explore the Architecture** - Check the documentation 3. **Find an Issue** - Look for "good first issue" on GitHub 4. **Set Up Pre-commit** - Install git hooks for code quality 5. **Join Discord** - https://discord.gg/37XJPXfz2w ## Getting Help If you get stuck: - **Discord**: [Join our server](https://discord.gg/37XJPXfz2w) for real-time help - **GitHub Issues**: Check existing issues for similar problems - **GitHub Discussions**: Ask questions in discussions - **Documentation**: See [code-standards.md](code-standards.md) and [testing.md](testing.md) --- **Ready to contribute?** Go to [contributing.md](contributing.md) for the contribution workflow. ================================================ FILE: docs/7-DEVELOPMENT/index.md ================================================ # Development Welcome to the Open Notebook development documentation! Whether you're contributing code, understanding our architecture, or maintaining the project, you'll find guidance here. ## 🎯 Pick Your Path ### 👨‍💻 I Want to Contribute Code Start with **[Contributing Guide](contributing.md)** for the workflow, then check: - **[Quick Start](quick-start.md)** - Clone, install, verify in 5 minutes - **[Development Setup](development-setup.md)** - Complete local environment guide - **[Code Standards](code-standards.md)** - How to write code that fits our style - **[Testing](testing.md)** - How to write and run tests **First time?** Check out our [Contributing Guide](contributing.md) for the issue-first workflow. --- ### 🏗️ I Want to Understand the Architecture **[Architecture Overview](architecture.md)** covers: - 3-tier system design - Tech stack and rationale - Key components and workflows - Design patterns we use For deeper dives, check `/open_notebook/` CLAUDE.md for component-specific guidance. --- ### 👨‍🔧 I'm a Maintainer **[Maintainer Guide](maintainer-guide.md)** covers: - Issue triage and management - Pull request review process - Communication templates - Best practices --- ## 📚 Quick Links | Document | For | Purpose | |---|---|---| | [Quick Start](quick-start.md) | New developers | Clone, install, and verify setup (5 min) | | [Development Setup](development-setup.md) | Local development | Complete environment setup guide | | [Contributing](contributing.md) | Code contributors | Workflow: issue → code → PR | | [Code Standards](code-standards.md) | Writing code | Style guides for Python, FastAPI, DB | | [Testing](testing.md) | Testing code | How to write and run tests | | [Architecture](architecture.md) | Understanding system | System design, tech stack, workflows | | [Design Principles](design-principles.md) | All developers | What guides our decisions | | [API Reference](api-reference.md) | Building integrations | Complete REST API documentation | | [Maintainer Guide](maintainer-guide.md) | Maintainers | Managing issues, PRs, releases | --- ## 🚀 Current Development Priorities We're actively looking for help with: 1. **Frontend Enhancement** - Improve Next.js/React UI with real-time updates 2. **Performance** - Async processing and caching optimizations 3. **Testing** - Expand test coverage across components 4. **Documentation** - API examples and developer guides 5. **Integrations** - New content sources and AI providers See GitHub Issues labeled `good first issue` or `help wanted`. --- ## 💬 Getting Help - **Discord**: [Join our server](https://discord.gg/37XJPXfz2w) for real-time discussions - **GitHub Discussions**: For architecture questions - **GitHub Issues**: For bugs and features Don't be shy! We're here to help new contributors succeed. --- ## 📖 Additional Resources ### External Documentation - [FastAPI Docs](https://fastapi.tiangolo.com/) - [SurrealDB Docs](https://surrealdb.com/docs) - [LangChain Docs](https://python.langchain.com/) - [Next.js Docs](https://nextjs.org/docs) ### Our Libraries - [Esperanto](https://github.com/lfnovo/esperanto) - Multi-provider AI abstraction - [Content Core](https://github.com/lfnovo/content-core) - Content processing - [Podcast Creator](https://github.com/lfnovo/podcast-creator) - Podcast generation --- Ready to get started? Head over to **[Quick Start](quick-start.md)**! 🎉 ================================================ FILE: docs/7-DEVELOPMENT/maintainer-guide.md ================================================ # Maintainer Guide This guide is for project maintainers to help manage contributions effectively while maintaining project quality and vision. ## Table of Contents - [Issue Management](#issue-management) - [Pull Request Review](#pull-request-review) - [Common Scenarios](#common-scenarios) - [Communication Templates](#communication-templates) ## Issue Management ### When a New Issue is Created **1. Initial Triage** (within 24-48 hours) - Add appropriate labels: - `bug`, `enhancement`, `documentation`, etc. - `good first issue` for beginner-friendly tasks - `needs-triage` until reviewed - `help wanted` if you'd welcome community contributions - Quick assessment: - Is it clear and well-described? - Is it aligned with project vision? (See [design-principles.md](design-principles.md)) - Does it duplicate an existing issue? **2. Initial Response** ```markdown Thanks for opening this issue! We'll review it and get back to you soon. [If it's a bug] In the meantime, have you checked our troubleshooting guide? [If it's a feature] You might find our [design principles](design-principles.md) helpful for understanding what we're building toward. ``` **3. Decision Making** Ask yourself: - Does this align with our [design principles](design-principles.md)? - Is this something we want in the core project, or better as a plugin/extension? - Do we have the capacity to support this feature long-term? - Will this benefit most users, or just a specific use case? **4. Issue Assignment** If the contributor checked "I am a developer and would like to work on this": **For Accepted Issues:** ```markdown Great idea! This aligns well with our goals, particularly [specific design principle]. I see you'd like to work on this. Before you start: 1. Please share your proposed approach/solution 2. Review our [Contributing Guide](contributing.md) and [Design Principles](design-principles.md) 3. Once we agree on the approach, I'll assign this to you Looking forward to your thoughts! ``` **For Issues Needing Clarification:** ```markdown Thanks for offering to work on this! Before we proceed, we need to clarify a few things: 1. [Question 1] 2. [Question 2] Once we have these details, we can discuss the best approach. ``` **For Issues Not Aligned with Vision:** ```markdown Thank you for the suggestion and for offering to work on this! After reviewing against our [design principles](design-principles.md), we've decided not to pursue this in the core project because [specific reason]. However, you might be able to achieve this through [alternative approach, if applicable]. We appreciate your interest in contributing! Feel free to check out our [open issues](link) for other ways to contribute. ``` ### Labels to Use **Priority:** - `priority: critical` - Security issues, data loss bugs - `priority: high` - Major functionality broken - `priority: medium` - Annoying bugs, useful features - `priority: low` - Nice to have, edge cases **Status:** - `needs-triage` - Not yet reviewed by maintainer - `needs-info` - Waiting for more information from reporter - `needs-discussion` - Requires community/team discussion - `ready` - Approved and ready to be worked on - `in-progress` - Someone is actively working on this - `blocked` - Cannot proceed due to external dependency **Type:** - `bug` - Something is broken - `enhancement` - New feature or improvement - `documentation` - Documentation improvements - `question` - General questions - `refactor` - Code cleanup/restructuring **Difficulty:** - `good first issue` - Good for newcomers - `help wanted` - Community contributions welcome - `advanced` - Requires deep codebase knowledge ## Pull Request Review ### Initial PR Review Checklist **Before diving into code:** - [ ] Is there an associated approved issue? - [ ] Does the PR reference the issue number? - [ ] Is the PR description clear about what changed and why? - [ ] Did the contributor check the relevant boxes in the PR template? - [ ] Are there tests? Screenshots (for UI changes)? **Red Flags** (may require closing PR): - No associated issue - Issue was not assigned to contributor - PR tries to solve multiple unrelated problems - Breaking changes without discussion - Conflicts with project vision ### Code Review Process **1. High-Level Review** - Does the approach align with our architecture? - Is the solution appropriately scoped? - Are there simpler alternatives? - Does it follow our design principles? **2. Code Quality Review** Python: - [ ] Follows PEP 8 - [ ] Has type hints - [ ] Has docstrings - [ ] Proper error handling - [ ] No security vulnerabilities TypeScript/Frontend: - [ ] Follows TypeScript best practices - [ ] Proper component structure - [ ] No console.logs left in production code - [ ] Accessible UI components **3. Testing Review** - [ ] Has appropriate test coverage - [ ] Tests are meaningful (not just for coverage percentage) - [ ] Tests pass locally and in CI - [ ] Edge cases are tested **4. Documentation Review** - [ ] Code is well-commented - [ ] Complex logic is explained - [ ] User-facing documentation updated (if applicable) - [ ] API documentation updated (if API changed) - [ ] Migration guide provided (if breaking change) ### Providing Feedback **Positive Feedback** (important!): ```markdown Thanks for this PR! I really like [specific thing they did well]. [Feedback on what needs to change] ``` **Requesting Changes:** ```markdown This is a great start! A few things to address: 1. **[High-level concern]**: [Explanation and suggested approach] 2. **[Code quality issue]**: [Specific example and fix] 3. **[Testing gap]**: [What scenarios need coverage] Let me know if you have questions about any of this! ``` **Suggesting Alternative Approach:** ```markdown I appreciate the effort you put into this! However, I'm concerned about [specific issue]. Have you considered [alternative approach]? It might be better because [reasons]. What do you think? ``` ## Common Scenarios ### Scenario 1: Good Code, Wrong Approach **Situation**: Contributor wrote quality code, but solved the problem in a way that doesn't fit our architecture. **Response:** ```markdown Thank you for this PR! The code quality is great, and I can see you put thought into this. However, I'm concerned that this approach [specific architectural concern]. In our architecture, we [explain the pattern we follow]. Would you be open to refactoring this to [suggested approach]? I'm happy to provide guidance on the specifics. Alternatively, if you don't have time for a refactor, I can take over and finish this up (with credit to you, of course). Let me know what you prefer! ``` ### Scenario 2: PR Without Assigned Issue **Situation**: Contributor submitted PR without going through issue approval process. **Response:** ```markdown Thanks for the PR! I appreciate you taking the time to contribute. However, to maintain project coherence, we require all PRs to be linked to an approved issue that was assigned to the contributor. This is explained in our [Contributing Guide](contributing.md). This helps us: - Ensure work aligns with project vision - Prevent duplicate efforts - Discuss approach before implementation Could you please: 1. Create an issue describing this change 2. Wait for it to be reviewed and assigned to you 3. We can then reopen this PR or you can create a new one Sorry for the inconvenience - this process helps us manage the project effectively. ``` ### Scenario 3: Feature Request Not Aligned with Vision **Situation**: Well-intentioned feature that doesn't fit project goals. **Response:** ```markdown Thank you for this suggestion! I can see how this would be useful for [specific use case]. After reviewing against our [design principles](design-principles.md), we've decided not to include this in the core project because [specific reason - e.g., "it conflicts with our 'Simplicity Over Features' principle" or "it would require dependencies that conflict with our privacy-first approach"]. Some alternatives: - [If applicable] This could be built as a plugin/extension - [If applicable] This functionality might be achievable through [existing feature] - [If applicable] You might be interested in [other tool] which is designed for this use case We appreciate your contribution and hope you understand. Feel free to check our roadmap or open issues for other ways to contribute! ``` ### Scenario 4: Contributor Ghosts After Feedback **Situation**: You requested changes, but contributor hasn't responded in 2+ weeks. **After 2 weeks:** ```markdown Hey there! Just checking in on this PR. Do you have time to address the feedback, or would you like someone else to take over? No pressure either way - just want to make sure this doesn't fall through the cracks. ``` **After 1 month with no response:** ```markdown Thanks again for starting this work! Since we haven't heard back, I'm going to close this PR for now. If you want to pick this up again in the future, feel free to reopen it or create a new PR. Alternatively, I'll mark the issue as available for someone else to work on. We appreciate your contribution! ``` Then: - Close the PR - Unassign the issue - Add `help wanted` label to the issue ### Scenario 5: Breaking Changes Without Discussion **Situation**: PR introduces breaking changes that weren't discussed. **Response:** ```markdown Thanks for this PR! However, I notice this introduces breaking changes that weren't discussed in the original issue. Breaking changes require: 1. Prior discussion and approval 2. Migration guide for users 3. Deprecation period (when possible) 4. Clear documentation of the change Could we discuss the breaking changes first? Specifically: - [What breaks and why] - [Who will be affected] - [Migration path] We may need to adjust the approach to minimize impact on existing users. ``` ## Communication Templates ### Closing a PR (Misaligned with Vision) ```markdown Thank you for taking the time to contribute! We really appreciate it. After careful review, we've decided not to merge this PR because [specific reason related to design principles]. This isn't a reflection on your code quality - it's about maintaining focus on our core goals as outlined in [design-principles.md](design-principles.md). We'd love to have you contribute in other ways! Check out: - Good first issues - Help wanted issues - Our roadmap Thanks again for your interest in Open Notebook! ``` ### Closing a Stale Issue ```markdown We're closing this issue due to inactivity. If this is still relevant, feel free to reopen it with updated information. Thanks! ``` ### Asking for More Information ```markdown Thanks for reporting this! To help us investigate, could you provide: 1. [Specific information needed] 2. [Logs, screenshots, etc.] 3. [Steps to reproduce] This will help us understand the issue better and find a solution. ``` ### Thanking a Contributor ```markdown Merged! Thank you so much for this contribution, @username! [Specific thing they did well]. This will be included in the next release. ``` ## Best Practices ### Be Kind and Respectful - Thank contributors for their time and effort - Assume good intentions - Be patient with newcomers - Explain *why*, not just *what* ### Be Clear and Direct - Don't leave ambiguity about next steps - Be specific about what needs to change - Explain architectural decisions - Set clear expectations ### Be Consistent - Apply the same standards to all contributors - Follow the process you've defined - Document decisions for future reference ### Be Protective of Project Vision - It's okay to say "no" - Prioritize long-term maintainability - Don't accept features you can't support - Keep the project focused ### Be Responsive - Respond to issues within 48 hours (even just to acknowledge) - Review PRs within a week when possible - Keep contributors updated on status - Close stale issues/PRs to keep things tidy ## When in Doubt Ask yourself: 1. Does this align with our [design principles](design-principles.md)? 2. Will we be able to maintain this feature long-term? 3. Does this benefit most users, or just an edge case? 4. Is there a simpler alternative? 5. Would I want to support this in 2 years? If you're unsure, it's perfectly fine to: - Ask for input from other maintainers - Start a discussion issue - Sleep on it before making a decision --- **Remember**: Good maintainership is about balancing openness to contributions with protection of project vision. You're not being mean by saying "no" to things that don't fit - you're being a responsible steward of the project. ================================================ FILE: docs/7-DEVELOPMENT/quick-start.md ================================================ # Quick Start - Development Get Open Notebook running locally in 5 minutes. ## Prerequisites - **Python 3.11+** - **Git** - **uv** (package manager) - install with `curl -LsSf https://astral.sh/uv/install.sh | sh` - **Docker** (optional, for SurrealDB) ## 1. Clone the Repository (2 min) ```bash # Fork the repository on GitHub first, then clone your fork git clone https://github.com/YOUR_USERNAME/open-notebook.git cd open-notebook # Add upstream remote for updates git remote add upstream https://github.com/lfnovo/open-notebook.git ``` ## 2. Install Dependencies (2 min) ```bash # Install Python dependencies uv sync # Verify uv is working uv --version ``` ## 3. Start Services (1 min) In separate terminal windows: ```bash # Terminal 1: Start SurrealDB (database) make database # or: docker run -d --name surrealdb -p 8000:8000 surrealdb/surrealdb:v2 start --user root --pass password --bind 0.0.0.0:8000 memory # Terminal 2: Start API (backend on port 5055) make api # or: uv run --env-file .env uvicorn api.main:app --host 0.0.0.0 --port 5055 # Terminal 3: Start Frontend (UI on port 3000) cd frontend && npm run dev ``` ## 4. Verify Everything Works (instant) - **API Health**: http://localhost:5055/health → should return `{"status": "ok"}` - **API Docs**: http://localhost:5055/docs → interactive API documentation - **Frontend**: http://localhost:3000 → Open Notebook UI **All three show up?** ✅ You're ready to develop! --- ## Next Steps - **First Issue?** Pick a [good first issue](https://github.com/lfnovo/open-notebook/issues?q=label%3A%22good+first+issue%22) - **Understand the code?** Read [Architecture Overview](architecture.md) - **Make changes?** Follow [Contributing Guide](contributing.md) - **Setup details?** See [Development Setup](development-setup.md) --- ## Troubleshooting ### "Port 5055 already in use" ```bash # Find what's using the port lsof -i :5055 # Use a different port uv run uvicorn api.main:app --port 5056 ``` ### "Can't connect to SurrealDB" ```bash # Check if SurrealDB is running docker ps | grep surrealdb # Restart it make database ``` ### "Python version is too old" ```bash # Check your Python version python --version # Should be 3.11+ # Use Python 3.11 specifically uv sync --python 3.11 ``` ### "npm: command not found" ```bash # Install Node.js from https://nodejs.org/ # Then install frontend dependencies cd frontend && npm install ``` --- ## Common Development Commands ```bash # Run tests uv run pytest # Format code make ruff # Type checking make lint # Run the full stack make start-all # View API documentation open http://localhost:5055/docs ``` --- Need more help? See [Development Setup](development-setup.md) for details or join our [Discord](https://discord.gg/37XJPXfz2w). ================================================ FILE: docs/7-DEVELOPMENT/testing.md ================================================ # Testing Guide This document provides guidelines for writing tests in Open Notebook. Testing is critical to maintaining code quality and preventing regressions. ## Testing Philosophy ### What to Test Focus on testing the things that matter most: - **Business Logic** - Core domain models and their operations - **API Contracts** - HTTP endpoint behavior and error handling - **Critical Workflows** - End-to-end flows that users depend on - **Data Persistence** - Database operations and data integrity - **Error Conditions** - How the system handles failures gracefully ### What NOT to Test Don't waste time testing framework code: - Framework functionality (FastAPI, React, etc.) - Third-party library implementation - Simple getters/setters without logic - View/presentation layer rendering (unless it contains logic) ## Test Structure We use **pytest** with async support for all Python tests: ```python import pytest from httpx import AsyncClient from open_notebook.domain.notebook import Notebook @pytest.mark.asyncio async def test_create_notebook(): """Test notebook creation.""" notebook = Notebook(name="Test Notebook", description="Test description") await notebook.save() assert notebook.id is not None assert notebook.name == "Test Notebook" assert notebook.created is not None @pytest.mark.asyncio async def test_api_create_notebook(): """Test notebook creation via API.""" async with AsyncClient(app=app, base_url="http://test") as client: response = await client.post( "/api/notebooks", json={"name": "Test Notebook", "description": "Test description"} ) assert response.status_code == 200 data = response.json() assert data["name"] == "Test Notebook" ``` ## Test Categories ### 1. Unit Tests Test individual functions and methods in isolation: ```python @pytest.mark.asyncio async def test_notebook_validation(): """Test that notebook name validation works.""" with pytest.raises(InvalidInputError): Notebook(name="", description="test") @pytest.mark.asyncio async def test_notebook_archive(): """Test notebook archiving.""" notebook = Notebook(name="Test", description="") notebook.archive() assert notebook.archived is True ``` **Location**: `tests/unit/` ### 2. Integration Tests Test component interactions and database operations: ```python @pytest.mark.asyncio async def test_create_notebook_with_sources(): """Test creating a notebook and adding sources.""" notebook = await create_notebook(name="Research", description="") source = await add_source(notebook_id=notebook.id, url="https://example.com") retrieved = await get_notebook_with_sources(notebook.id) assert len(retrieved.sources) == 1 assert retrieved.sources[0].id == source.id ``` **Location**: `tests/integration/` ### 3. API Tests Test HTTP endpoints and error responses: ```python @pytest.mark.asyncio async def test_get_notebooks_endpoint(): """Test GET /notebooks endpoint.""" async with AsyncClient(app=app, base_url="http://test") as client: response = await client.get("/api/notebooks") assert response.status_code == 200 data = response.json() assert isinstance(data, list) @pytest.mark.asyncio async def test_create_notebook_validation(): """Test that invalid input is rejected.""" async with AsyncClient(app=app, base_url="http://test") as client: response = await client.post( "/api/notebooks", json={"name": "", "description": ""} ) assert response.status_code == 400 ``` **Location**: `tests/api/` ### 4. Database Tests Test data persistence and query correctness: ```python @pytest.mark.asyncio async def test_save_and_retrieve_notebook(): """Test saving and retrieving a notebook from database.""" notebook = Notebook(name="Test", description="desc") await notebook.save() retrieved = await Notebook.get(notebook.id) assert retrieved.name == "Test" assert retrieved.description == "desc" @pytest.mark.asyncio async def test_query_by_criteria(): """Test querying notebooks by criteria.""" await create_notebook("Active", "") await create_notebook("Archived", "") active = await repo_query( "SELECT * FROM notebook WHERE archived = false" ) assert len(active) >= 1 ``` **Location**: `tests/database/` ## Running Tests ### Run All Tests ```bash uv run pytest ``` ### Run Specific Test File ```bash uv run pytest tests/test_notebooks.py ``` ### Run Specific Test Function ```bash uv run pytest tests/test_notebooks.py::test_create_notebook ``` ### Run with Coverage Report ```bash uv run pytest --cov=open_notebook ``` ### Run Only Unit Tests ```bash uv run pytest tests/unit/ ``` ### Run Only Integration Tests ```bash uv run pytest tests/integration/ ``` ### Run Tests in Verbose Mode ```bash uv run pytest -v ``` ### Run Tests with Output ```bash uv run pytest -s ``` ## Test Fixtures Use pytest fixtures for common setup and teardown: ```python import pytest @pytest.fixture async def test_notebook(): """Create a test notebook.""" notebook = Notebook(name="Test Notebook", description="Test description") await notebook.save() yield notebook await notebook.delete() @pytest.fixture async def api_client(): """Create an API test client.""" async with AsyncClient(app=app, base_url="http://test") as client: yield client @pytest.fixture async def test_notebook_with_sources(test_notebook): """Create a test notebook with sample sources.""" source1 = Source(notebook_id=test_notebook.id, url="https://example.com") source2 = Source(notebook_id=test_notebook.id, url="https://example.org") await source1.save() await source2.save() test_notebook.sources = [source1, source2] yield test_notebook # Cleanup await source1.delete() await source2.delete() ``` ## Best Practices ### 1. Write Descriptive Test Names ```python # Good - clearly describes what is being tested async def test_create_notebook_with_valid_name_succeeds(): ... # Bad - vague about what's being tested async def test_notebook(): ... ``` ### 2. Use Docstrings ```python @pytest.mark.asyncio async def test_vector_search_returns_sorted_results(): """Test that vector search results are sorted by relevance score.""" # Implementation ``` ### 3. Test Edge Cases ```python @pytest.mark.asyncio async def test_search_with_empty_query(): """Test that empty query raises error.""" with pytest.raises(InvalidInputError): await vector_search("") @pytest.mark.asyncio async def test_search_with_very_long_query(): """Test that very long query is handled.""" long_query = "x" * 10000 results = await vector_search(long_query) assert isinstance(results, list) @pytest.mark.asyncio async def test_search_with_special_characters(): """Test that special characters are handled.""" results = await vector_search("@#$%^&*()") assert isinstance(results, list) ``` ### 4. Use Assertions Effectively ```python # Good - specific assertions assert notebook.name == "Test" assert len(notebook.sources) == 3 assert notebook.created is not None # Less good - too broad assert notebook is not None assert notebook # ambiguous what's being tested ``` ### 5. Test Both Success and Failure Cases ```python @pytest.mark.asyncio async def test_create_notebook_success(): """Test successful notebook creation.""" notebook = await create_notebook(name="Research", description="AI") assert notebook.id is not None assert notebook.name == "Research" @pytest.mark.asyncio async def test_create_notebook_empty_name_fails(): """Test that empty name raises error.""" with pytest.raises(InvalidInputError): await create_notebook(name="", description="") @pytest.mark.asyncio async def test_create_notebook_duplicate_fails(): """Test that duplicate names are handled.""" await create_notebook(name="Research", description="") with pytest.raises(DuplicateError): await create_notebook(name="Research", description="") ``` ### 6. Keep Tests Independent ```python # Good - test is self-contained @pytest.mark.asyncio async def test_archive_notebook(): notebook = Notebook(name="Test", description="") await notebook.save() await notebook.archive() assert notebook.archived is True # Bad - depends on another test's state @pytest.mark.asyncio async def test_archive_existing_notebook(): # Assumes test_create_notebook ran first await notebook.archive() # notebook undefined ``` ### 7. Use Fixtures for Reusable Setup ```python # Instead of repeating setup: @pytest.fixture async def client_with_auth(api_client, mock_auth): """Client with authentication set up.""" api_client.headers.update({"Authorization": f"Bearer {mock_auth.token}"}) yield api_client @pytest.mark.asyncio async def test_protected_endpoint(client_with_auth): """Test protected endpoint.""" response = await client_with_auth.get("/api/protected") assert response.status_code == 200 ``` ## Coverage Goals - Aim for 70%+ overall coverage - 90%+ coverage for critical business logic - Don't obsess over 100% - focus on meaningful tests - Use `--cov` flag to check coverage: `uv run pytest --cov=open_notebook` ## Async Test Patterns ### Testing Async Functions ```python @pytest.mark.asyncio async def test_async_operation(): """Test async function.""" result = await some_async_function() assert result is not None ``` ### Testing Concurrent Operations ```python @pytest.mark.asyncio async def test_concurrent_notebook_creation(): """Test creating multiple notebooks concurrently.""" tasks = [ create_notebook(f"Notebook {i}", "") for i in range(10) ] notebooks = await asyncio.gather(*tasks) assert len(notebooks) == 10 assert all(n.id for n in notebooks) ``` ## Common Testing Errors ### Error: "event loop is closed" Solution: Use the async fixture properly: ```python @pytest.fixture async def notebook(): # Use async fixture notebook = Notebook(name="Test", description="") await notebook.save() yield notebook await notebook.delete() ``` ### Error: "object is not awaitable" Solution: Make sure you're using await: ```python # Wrong result = create_notebook("Test", "") # Right result = await create_notebook("Test", "") ``` --- **See also:** - [Code Standards](code-standards.md) - Code formatting and style - [Contributing Guide](contributing.md) - Overall contribution workflow ================================================ FILE: docs/SECURITY_REVIEW.md ================================================ # Security Review - API Configuration UI ## Date: 2026-01-27 (Updated: 2026-01-28) ## Reviewer: Security Audit --- ## Summary Security review of the API key management implementation for Open Notebook. The implementation uses a database-first approach with environment variable fallback. --- ## Encryption | Item | Status | Notes | |------|--------|-------| | Fernet encryption implemented | PASS | `open_notebook/utils/encryption.py` uses AES-128-CBC + HMAC-SHA256 | | Keys encrypted before DB storage | PASS | `encrypt_value()` applied on save | | Keys decrypted only when needed | PASS | `decrypt_value()` called when reading | | Encryption key required | PASS | No default key; ValueError if not configured | | Docker secrets support | PASS | `_FILE` suffix pattern supported | | Documented in .env.example | PASS | Encryption key documented | --- ## API Security | Item | Status | Notes | |------|--------|-------| | Test endpoint implemented | PASS | `connection_tester.py` validates keys | | Test doesn't expose keys | PASS | Only returns success/failure | | Error messages don't leak info | PASS | Generic error messages | | URL validation for SSRF | PASS | Blocks private IPs (except Ollama) | | Rate limiting | NOT IMPL | Future enhancement | --- ## Frontend Security | Item | Status | Notes | |------|--------|-------| | No keys in localStorage | PASS | Keys only in React state during entry | | Keys masked in UI | PASS | Shows `************` placeholder | | No keys in console.log | PASS | No logging of sensitive data | | autocomplete attributes | PARTIAL | Some forms missing autocomplete="off" | --- ## Authentication | Item | Status | Notes | |------|--------|-------| | Password protection | PASS | Bearer token authentication | | Default password | PASS | "open-notebook-change-me" when not set | | Docker secrets support | PASS | `_FILE` suffix for password | | Security warnings | PASS | Logged when using defaults | --- ## Files Reviewed | Component | Path | Status | |-----------|------|--------| | Encryption | `open_notebook/utils/encryption.py` | PASS | | Credential model | `open_notebook/domain/credential.py` | PASS | | Credentials router | `api/routers/credentials.py` | PASS | | Key provider | `open_notebook/ai/key_provider.py` | PASS | | Connection tester | `open_notebook/ai/connection_tester.py` | PASS | | Auth middleware | `api/auth.py` | PASS | | Frontend forms | `frontend/src/components/settings/*.tsx` | PASS | | Environment example | `.env.example` | PASS | --- ## Remaining Recommendations ### Future Improvements 1. **Rate limiting** - Add rate limiting on `/credentials/*` endpoints 2. **Autocomplete attributes** - Add `autocomplete="new-password"` to all password inputs 3. **Show last 4 characters** - Display `********xxxx` format for key identification 4. **Audit logging** - Log API key changes with timestamps --- ## Conclusion The API Configuration UI implementation meets security requirements: - API keys encrypted at rest using Fernet (key must be explicitly configured) - Keys never returned to frontend - URL validation prevents SSRF attacks - Docker secrets supported for production deployments **Review Status: PASS** ================================================ FILE: docs/index.md ================================================ # Open Notebook Documentation Welcome to Open Notebook - a privacy-focused AI research assistant. This documentation is organized for different needs. --- ## 🎯 Choose Your Path ### I'm brand new → Start here: **[0-START-HERE](0-START-HERE/index.md)** - Learn what Open Notebook is - Pick your setup path (OpenAI, cloud, local/Ollama) - 5-minute quick start ### I need to install/deploy → Go here: **[1-INSTALLATION](1-INSTALLATION/index.md)** - Multiple installation routes - Docker Compose (recommended) - From source (developers) - Single container (shared hosting) ### I want to understand how it works → Read this: **[2-CORE-CONCEPTS](2-CORE-CONCEPTS/index.md)** - Mental models and architecture - How RAG (retrieval-augmented generation) works - Notebooks, sources, and notes explained - Chat vs. transformations vs. podcasts ### I want to use it (tutorials) → Follow this: **[3-USER-GUIDE](3-USER-GUIDE/index.md)** - How to add sources (PDFs, URLs, audio, video) - Creating and organizing notes - Chat effectively with your research - Creating podcasts from research - Search techniques ### I need to configure it → Check this: **[5-CONFIGURATION](5-CONFIGURATION/index.md)** - Choose and setup AI provider - API configuration - Database setup - Advanced tuning ### I need provider-specific help → Go here: **[4-AI-PROVIDERS](4-AI-PROVIDERS/index.md)** - OpenAI, Anthropic, Google, Groq, Ollama, Azure - Model comparisons - Cost estimates - Setup paths ### Something's not working → Troubleshoot: **[6-TROUBLESHOOTING](6-TROUBLESHOOTING/index.md)** - Quick fixes (top 10 issues) - Installation problems - Connection issues - AI/chat problems - Content processing issues - Podcast problems ### I want to contribute/develop → Read this: **[7-DEVELOPMENT](7-DEVELOPMENT/index.md)** - Architecture and tech stack - Contributing guidelines - API reference - Testing --- ## 📊 Documentation Overview ### By Section **[0-START-HERE](0-START-HERE/index.md)** — Entry point - What is Open Notebook? - Quick start guides (3 routes) - First 5 minutes **[1-INSTALLATION](1-INSTALLATION/index.md)** — Getting it running - Multiple installation routes - Docker, single-container, from-source - Requirements and setup **[2-CORE-CONCEPTS](2-CORE-CONCEPTS/index.md)** — Understanding the system - Notebooks, sources, notes hierarchy - RAG (retrieval-augmented generation) - Chat, transformations, podcasts - Context management **[3-USER-GUIDE](3-USER-GUIDE/index.md)** — Using features - Adding sources (all types) - Working with notes - Chat effectively - Creating podcasts - Searching (text and semantic) **[4-AI-PROVIDERS](4-AI-PROVIDERS/index.md)** — AI configuration - Provider comparison - Setup for each provider - Model recommendations - Cost estimates **[5-CONFIGURATION](5-CONFIGURATION/index.md)** — Complete reference - AI provider setup (detailed) - Database configuration - Server/API settings - Advanced tuning - Environment variables (complete reference) **[6-TROUBLESHOOTING](6-TROUBLESHOOTING/index.md)** — Problem solving - Quick fixes (top 10) - Installation issues - Connection problems - AI/chat issues - Content processing - Podcast generation - Getting help **[7-DEVELOPMENT](7-DEVELOPMENT/index.md)** — For contributors - Architecture - Contributing guidelines - API reference - Testing & development --- ## 🔍 Find What You Need ### By Problem Type **Installation & Setup** - Fresh install? → [0-START-HERE](0-START-HERE/index.md) - Detailed installation routes? → [1-INSTALLATION](1-INSTALLATION/index.md) - Configuration reference? → [5-CONFIGURATION](5-CONFIGURATION/index.md) - Provider setup? → [4-AI-PROVIDERS](4-AI-PROVIDERS/index.md) **Using Open Notebook** - How to use features? → [3-USER-GUIDE](3-USER-GUIDE/index.md) - Understanding concepts? → [2-CORE-CONCEPTS](2-CORE-CONCEPTS/index.md) - Chat not working? → [6-TROUBLESHOOTING - AI Issues](6-TROUBLESHOOTING/ai-chat-issues.md) - Files won't upload? → [6-TROUBLESHOOTING - Quick Fixes](6-TROUBLESHOOTING/quick-fixes.md#4-cannot-process-file-or-unsupported-format) **Troubleshooting** - Quick fix? → [6-TROUBLESHOOTING - Quick Fixes](6-TROUBLESHOOTING/quick-fixes.md) - Can't connect? → [6-TROUBLESHOOTING - Connection](6-TROUBLESHOOTING/connection-issues.md) - Chat issues? → [6-TROUBLESHOOTING - AI Issues](6-TROUBLESHOOTING/ai-chat-issues.md) - Podcast problems? → [6-TROUBLESHOOTING - Quick Fixes](6-TROUBLESHOOTING/quick-fixes.md#8-podcast-generation-failed) **Development** - Architecture? → [7-DEVELOPMENT - Architecture](7-DEVELOPMENT/architecture.md) - Contributing? → [7-DEVELOPMENT - Contributing](7-DEVELOPMENT/contributing.md) - API reference? → [7-DEVELOPMENT - API Reference](7-DEVELOPMENT/api-reference.md) --- ## 📚 Reading Paths ### Path 1: Complete Beginner (1-2 hours) 1. [0-START-HERE/index.md](0-START-HERE/index.md) — Understand what it is 2. [0-START-HERE Quick Start](0-START-HERE/index.md) — Set it up 3. [2-CORE-CONCEPTS/index.md](2-CORE-CONCEPTS/index.md) — Understand concepts 4. [3-USER-GUIDE/index.md](3-USER-GUIDE/index.md) — Learn features **Result:** Fully understand how to use Open Notebook ### Path 2: Get Running Fast (15 minutes) 1. [0-START-HERE](0-START-HERE/index.md) — Pick your path 2. Follow quick-start guide for your setup 3. Start using! **Result:** Running in 15 minutes, learn details later ### Path 3: DevOps/Deployment (1-2 hours) 1. [1-INSTALLATION](1-INSTALLATION/index.md) — Understand routes 2. [5-CONFIGURATION](5-CONFIGURATION/index.md) — Reference setup 3. [7-DEVELOPMENT - Architecture](../7-DEVELOPMENT/architecture.md) — Understand system **Result:** Ready to deploy to production ### Path 4: Troubleshooting (5-30 minutes) 1. [6-TROUBLESHOOTING/index.md](6-TROUBLESHOOTING/index.md) — Identify problem 2. Find specific guide 3. Follow solutions **Result:** Problem solved! --- ## ❓ Common Questions **Q: Where do I start?** A: → [0-START-HERE](0-START-HERE/index.md) — Choose your setup path **Q: How do I install it?** A: → [1-INSTALLATION](1-INSTALLATION/index.md) — Multiple routes available **Q: How do I use [feature]?** A: → [3-USER-GUIDE](3-USER-GUIDE/index.md) — Step-by-step tutorials **Q: Why does [feature] work like that?** A: → [2-CORE-CONCEPTS](2-CORE-CONCEPTS/index.md) — Understand the mental model **Q: How do I configure [provider]?** A: → [4-AI-PROVIDERS](4-AI-PROVIDERS/index.md) or [5-CONFIGURATION](5-CONFIGURATION/index.md) **Q: Something's broken, what do I do?** A: → [6-TROUBLESHOOTING](6-TROUBLESHOOTING/index.md) — Problem solver **Q: How does the system work?** A: → [2-CORE-CONCEPTS](2-CORE-CONCEPTS/index.md) — Architecture and concepts **Q: Can I contribute?** A: → [7-DEVELOPMENT](../7-DEVELOPMENT/index.md) — Contributing guide --- ## 📖 How This Documentation is Organized ### Principles - **Progressive Disclosure**: Start simple, go deeper if needed - **Multiple Entry Routes**: Different paths for different users - **High Signal-to-Noise**: Focused content, no fluff - **Step-by-Step**: Clear instructions you can follow - **Decision Trees**: Help you pick the right path - **Symptom-Based**: Troubleshooting by what's broken ### Structure - **0-START-HERE** — Entry point (everyone starts here) - **1-INSTALLATION** — Multiple setup routes - **2-CORE-CONCEPTS** — Mental models (understand why) - **3-USER-GUIDE** — How to use (step-by-step) - **4-AI-PROVIDERS** — Provider guides - **5-CONFIGURATION** — Reference material - **6-TROUBLESHOOTING** — Problem solving - **7-DEVELOPMENT** — For contributors --- ## 🚀 Quick Navigation ### First Time? → **[START HERE](0-START-HERE/index.md)** ### Just Want to Use It? → **[QUICK START](0-START-HERE/index.md)** (5 minutes) ### Something Broken? → **[TROUBLESHOOTING](6-TROUBLESHOOTING/index.md)** ### Full Reference? → **[CONFIGURATION](5-CONFIGURATION/index.md)** ### Developer? → **[DEVELOPMENT](7-DEVELOPMENT/index.md)** --- ## 📞 Getting Help - **Discord Community** — https://discord.gg/37XJPXfz2w - **GitHub Issues** — https://github.com/lfnovo/open-notebook/issues - **Documentation** — You're reading it! --- ## 📈 Documentation Stats - **8 major sections** - **35+ focused guides** - **~80,000 words** - **Covers all features** - **Multiple entry paths** - **Progressive difficulty** --- ## 🎯 Start Here **First time using Open Notebook?** → Go to **[0-START-HERE](0-START-HERE/index.md)** **Experienced, looking for specific help?** → Use the navigation above to find your section **Something not working?** → Go to **[TROUBLESHOOTING](6-TROUBLESHOOTING/index.md)** --- Last updated: January 2026 | Open Notebook v1.2.4+ ================================================ FILE: examples/README.md ================================================ # Docker Compose Examples This folder contains different `docker-compose.yml` configurations for various use cases. ## 📋 Available Examples ### `docker-compose-full-local.yml` - 100% Local AI (No Cloud APIs) 🌟 **Use this if:** You want complete privacy with zero external API dependencies **Features:** - **Ollama**: Local LLM and embeddings (mistral, llama, etc.) - **Speaches**: Local TTS (text-to-speech) and STT (speech-to-text) - Everything runs on your machine - nothing sent to cloud - Perfect for privacy, offline work, or air-gapped environments **Setup:** 1. Copy to your project folder as `docker-compose.yml` 2. Run: `docker compose up -d` 3. Download models (see file comments for commands) 4. Configure all providers in UI (detailed instructions in file) **Requirements:** - Minimum: 8GB RAM, 20GB disk, 4 CPU cores - Recommended: 16GB+ RAM, NVIDIA GPU (8GB+ VRAM), 50GB disk **Documentation:** - [Local TTS Guide](../docs/5-CONFIGURATION/local-tts.md) - [Local STT Guide](../docs/5-CONFIGURATION/local-stt.md) --- ### `docker-compose-speaches.yml` - Local Speech Processing **Use this if:** You want free TTS/STT but use cloud LLMs **Features:** - **Speaches**: Local text-to-speech and speech-to-text - Use with cloud LLM providers (OpenAI, Anthropic, etc.) - Great for podcast generation without TTS API costs - Private audio processing **Setup:** 1. Copy to your project folder as `docker-compose.yml` 2. Run: `docker compose up -d` 3. Download speech models (see file for commands) 4. Configure cloud LLM + local Speaches in UI **Documentation:** - [Local TTS Guide](../docs/5-CONFIGURATION/local-tts.md) - [Local STT Guide](../docs/5-CONFIGURATION/local-stt.md) --- ### `docker-compose-ollama.yml` - Free Local AI with Ollama **Use this if:** You want to run AI models locally without API costs **Features:** - Includes Ollama service for local AI models - No external API keys needed (for LLM and embeddings) - Full privacy - everything runs on your machine - Great for testing or privacy-focused deployments **Setup:** 1. Copy to your project folder as `docker-compose.yml` 2. Run: `docker compose up -d` 3. Pull a model: `docker exec open_notebook-ollama-1 ollama pull mistral` 4. Configure in UI: Settings → API Keys → Add Ollama (URL: `http://ollama:11434`) **Recommended models:** - **LLM**: `mistral`, `llama3.1`, `qwen2.5` - **Embeddings**: `nomic-embed-text`, `mxbai-embed-large` --- ### `docker-compose-single.yml` - Single Container (Deprecated) **Use this if:** You need all services in one container (not recommended) **⚠️ Deprecated:** We recommend using the standard multi-container setup (`docker-compose.yml` in root) for better reliability and easier troubleshooting. **Features:** - Single container includes SurrealDB, API, and Frontend - Simpler for very constrained environments - Less flexible for debugging and scaling --- ### `docker-compose-dev.yml` - Development Setup **Use this if:** You're contributing to Open Notebook or developing custom features **Features:** - Hot-reload for code changes - Separate backend and frontend services - Build from source instead of using pre-built images - Includes development tools and debugging **Prerequisites:** - Python 3.11+ - Node.js 18+ - uv (Python package manager) **Setup:** See [Development Guide](../docs/7-DEVELOPMENT/index.md) --- ## 🔄 How to Use These Examples 1. **Choose** the example that fits your use case 2. **Copy** the file to your project folder: ```bash cp examples/docker-compose-ollama.yml docker-compose.yml ``` 3. **Edit** the `OPEN_NOTEBOOK_ENCRYPTION_KEY` value 4. **Run** the services: ```bash docker compose up -d ``` --- ## 💡 Need a Custom Setup? You can combine features from multiple examples. Common customizations: ### Add Ollama to Standard Setup Add this to the main `docker-compose.yml`: ```yaml ollama: image: ollama/ollama:latest ports: - "11434:11434" volumes: - ollama_models:/root/.ollama restart: always volumes: ollama_models: ``` ### Add Reverse Proxy See [Reverse Proxy Guide](../docs/5-CONFIGURATION/reverse-proxy.md) ### Add Basic Auth Add to `open_notebook` service environment: ```yaml - BASIC_AUTH_USERNAME=admin - BASIC_AUTH_PASSWORD=your-secure-password ``` --- ## 📚 Documentation - [Installation Guide](../docs/1-INSTALLATION/index.md) - [Configuration Reference](../docs/5-CONFIGURATION/environment-reference.md) - [Troubleshooting](../docs/6-TROUBLESHOOTING/index.md) --- ## 🆘 Need Help? - **Discord**: [Join our community](https://discord.gg/37XJPXfz2w) - **Issues**: [GitHub Issues](https://github.com/lfnovo/open-notebook/issues) ================================================ FILE: examples/docker-compose-dev.yml ================================================ services: surrealdb: image: surrealdb/surrealdb:v2 volumes: - ./surreal_data:/mydata environment: - SURREAL_EXPERIMENTAL_GRAPHQL=true ports: - "8000:8000" command: start --log info --user root --pass root rocksdb:/mydata/mydatabase.db pull_policy: always user: root restart: always open_notebook: build: context: . dockerfile: Dockerfile ports: - "8502:8502" - "5055:5055" env_file: - ./docker.env depends_on: - surrealdb volumes: - ./notebook_data:/app/data restart: always ================================================ FILE: examples/docker-compose-full-local.yml ================================================ # Docker Compose - 100% Local AI Setup # # This is the complete privacy-focused setup with NO external APIs needed: # - Ollama: Local LLM and embeddings (mistral, llama, nomic-embed, etc.) # - Speaches: Local TTS (text-to-speech) and STT (speech-to-text) # - Open Notebook: Your research assistant # - SurrealDB: Local database # # Perfect for: # - Complete privacy (nothing leaves your machine) # - Offline work # - No API costs # - Air-gapped environments # - Testing and development # # Usage: # 1. Copy this file to your project folder as docker-compose.yml # 2. Change OPEN_NOTEBOOK_ENCRYPTION_KEY below # 3. Run: docker compose up -d # 4. Pull models (see instructions below) # 5. Configure providers in UI # # Full documentation: # - Ollama setup: https://github.com/lfnovo/open-notebook/blob/main/examples/README.md # - TTS setup: https://github.com/lfnovo/open-notebook/blob/main/docs/5-CONFIGURATION/local-tts.md # - STT setup: https://github.com/lfnovo/open-notebook/blob/main/docs/5-CONFIGURATION/local-stt.md services: surrealdb: image: surrealdb/surrealdb:v2 command: start --log info --user root --pass root rocksdb:/mydata/mydatabase.db user: root ports: - "8000:8000" volumes: - ./surreal_data:/mydata environment: - SURREAL_EXPERIMENTAL_GRAPHQL=true restart: always pull_policy: always ollama: image: ollama/ollama:latest ports: - "11434:11434" volumes: - ollama_models:/root/.ollama restart: always pull_policy: always # For GPU acceleration (NVIDIA), add: # deploy: # resources: # reservations: # devices: # - driver: nvidia # count: 1 # capabilities: [gpu] speaches: image: ghcr.io/speaches-ai/speaches:latest-cpu container_name: speaches ports: - "8969:8000" volumes: - hf-hub-cache:/home/ubuntu/.cache/huggingface/hub restart: unless-stopped # For GPU acceleration, use: ghcr.io/speaches-ai/speaches:latest-cuda # and add GPU device mapping (see docs) open_notebook: image: lfnovo/open_notebook:v1-latest ports: - "8502:8502" - "5055:5055" environment: # REQUIRED: Change this to your own secret string - OPEN_NOTEBOOK_ENCRYPTION_KEY=change-me-to-a-secret-string # Database connection - SURREAL_URL=ws://surrealdb:8000/rpc - SURREAL_USER=root - SURREAL_PASSWORD=root - SURREAL_NAMESPACE=open_notebook - SURREAL_DATABASE=open_notebook # Ollama connection (optional, can also configure via UI) - OLLAMA_BASE_URL=http://ollama:11434 volumes: - ./notebook_data:/app/data depends_on: - surrealdb - ollama - speaches restart: always pull_policy: always volumes: ollama_models: hf-hub-cache: # ========================================== # AFTER STARTING: Download Models # ========================================== # # Ollama Models (LLM): # docker exec open_notebook-ollama-1 ollama pull mistral # docker exec open_notebook-ollama-1 ollama pull llama3.1 # docker exec open_notebook-ollama-1 ollama pull qwen2.5 # # Ollama Models (Embeddings): # docker exec open_notebook-ollama-1 ollama pull nomic-embed-text # docker exec open_notebook-ollama-1 ollama pull mxbai-embed-large # # Speaches (TTS): # docker compose exec speaches uv tool run speaches-cli model download speaches-ai/Kokoro-82M-v1.0-ONNX # # Speaches (STT): # docker compose exec speaches uv tool run speaches-cli model download Systran/faster-whisper-small # # ========================================== # CONFIGURATION IN OPEN NOTEBOOK # ========================================== # # 1. Configure Ollama: # - Go to Settings → API Keys # - Add Credential → Select "Ollama" # - Base URL: http://ollama:11434 # - Save → Test Connection → Discover Models → Register Models # # 2. Configure Speaches (TTS/STT): # - Go to Settings → API Keys # - Add Credential → Select "OpenAI-Compatible" # - Name: "Local Speaches" # - Base URL for TTS: http://host.docker.internal:8969/v1 (macOS/Windows) # or: http://172.17.0.1:8969/v1 (Linux) # - Base URL for STT: (same as TTS) # - Save → Test Connection # # 3. Discover Speech Models: # - In the Speaches credential you just created, click Discover Models # - Select and register the models you need (e.g. TTS and STT) # - If models aren't discovered automatically, add them manually: # * TTS: speaches-ai/Kokoro-82M-v1.0-ONNX # * STT: Systran/faster-whisper-small # # ========================================== # RECOMMENDED MODELS # ========================================== # # For LLM (choose based on your hardware): # - Fast: mistral (7B), qwen2.5 (7B) # - Balanced: llama3.1 (8B) # - Best quality: qwen2.5 (14B+), llama3.1 (70B) - requires powerful GPU # # For Embeddings: # - nomic-embed-text (recommended, 137M params) # - mxbai-embed-large (334M params, better quality) # # For TTS: # - speaches-ai/Kokoro-82M-v1.0-ONNX (good quality, fast) # # For STT (Whisper): # - faster-whisper-small (balanced, ~500MB) # - faster-whisper-base (faster, less accurate) # - faster-whisper-large-v3 (best quality, slower, ~3GB) # # ========================================== # HARDWARE REQUIREMENTS # ========================================== # # Minimum (CPU only): # - 8 GB RAM # - 20 GB disk space # - 4 CPU cores # # Recommended (with GPU): # - 16+ GB RAM # - 8+ GB VRAM (NVIDIA GPU) # - 50 GB disk space # - 8+ CPU cores # # ========================================== # COST COMPARISON # ========================================== # # Local (this setup): # - Cost: $0 (after hardware) # - Privacy: 100% private # - Speed: Depends on hardware # # Cloud (OpenAI + ElevenLabs): # - LLM: ~$0.01-0.10 per 1K tokens # - Embeddings: ~$0.0001 per 1K tokens # - TTS: ~$0.015 per minute # - STT: ~$0.006 per minute # - Privacy: Data sent to providers # - Speed: Usually faster ================================================ FILE: examples/docker-compose-ollama.yml ================================================ # Docker Compose with Ollama (Free Local AI) # # This setup includes Ollama for running local AI models without API costs. # Great for privacy-focused deployments or testing without cloud dependencies. # # Usage: # 1. Copy this file to your project folder as docker-compose.yml # 2. Change OPEN_NOTEBOOK_ENCRYPTION_KEY below # 3. Run: docker compose up -d # 4. Pull a model: docker exec open_notebook-ollama-1 ollama pull mistral # 5. Configure Ollama in UI: Settings → API Keys → Add Ollama (URL: http://ollama:11434) services: surrealdb: image: surrealdb/surrealdb:v2 command: start --log info --user root --pass root rocksdb:/mydata/mydatabase.db user: root ports: - "8000:8000" volumes: - ./surreal_data:/mydata environment: - SURREAL_EXPERIMENTAL_GRAPHQL=true restart: always pull_policy: always ollama: image: ollama/ollama:latest ports: - "11434:11434" volumes: - ollama_models:/root/.ollama restart: always pull_policy: always open_notebook: image: lfnovo/open_notebook:v1-latest ports: - "8502:8502" - "5055:5055" environment: # REQUIRED: Change this to your own secret string - OPEN_NOTEBOOK_ENCRYPTION_KEY=change-me-to-a-secret-string # Database connection - SURREAL_URL=ws://surrealdb:8000/rpc - SURREAL_USER=root - SURREAL_PASSWORD=root - SURREAL_NAMESPACE=open_notebook - SURREAL_DATABASE=open_notebook # Ollama connection - OLLAMA_BASE_URL=http://ollama:11434 volumes: - ./notebook_data:/app/data depends_on: - surrealdb - ollama restart: always pull_policy: always volumes: ollama_models: ================================================ FILE: examples/docker-compose-single.yml ================================================ services: open_notebook_single: # image: lfnovo/open_notebook:v1-latest-single build: context: . dockerfile: Dockerfile.single ports: - "8502:8502" # Next.js Frontend - "5055:5055" # REST API env_file: - ./docker.env environment: # Override for single-container mode: SurrealDB runs on localhost inside the same container - SURREAL_URL=ws://localhost:8000/rpc volumes: - ./notebook_data:/app/data # Application data - ./surreal_single_data:/mydata # SurrealDB data restart: always # Single container includes all services: SurrealDB, API, Worker, and Next.js Frontend # Access: # - Next.js UI: http://localhost:8502 # - REST API: http://localhost:5055 # - API Documentation: http://localhost:5055/docs ================================================ FILE: examples/docker-compose-speaches.yml ================================================ # Docker Compose with Speaches (Local TTS/STT) # # This setup includes Speaches for free, private speech processing: # - Text-to-Speech (TTS): Generate podcast audio locally # - Speech-to-Text (STT): Transcribe audio/video content locally # # Why Speaches? # - Free: No per-minute/per-character costs # - Private: Audio never leaves your machine # - Offline: Works without internet # - OpenAI-compatible: Drop-in replacement for OpenAI TTS/STT # # Usage: # 1. Copy this file to your project folder as docker-compose.yml # 2. Change OPEN_NOTEBOOK_ENCRYPTION_KEY below # 3. Run: docker compose up -d # 4. Download models (see instructions below) # 5. Configure in UI: Settings → API Keys → Add OpenAI-Compatible # # Full documentation: # - TTS setup: https://github.com/lfnovo/open-notebook/blob/main/docs/5-CONFIGURATION/local-tts.md # - STT setup: https://github.com/lfnovo/open-notebook/blob/main/docs/5-CONFIGURATION/local-stt.md services: surrealdb: image: surrealdb/surrealdb:v2 command: start --log info --user root --pass root rocksdb:/mydata/mydatabase.db user: root ports: - "8000:8000" volumes: - ./surreal_data:/mydata environment: - SURREAL_EXPERIMENTAL_GRAPHQL=true restart: always pull_policy: always speaches: image: ghcr.io/speaches-ai/speaches:latest-cpu container_name: speaches ports: - "8969:8000" volumes: - hf-hub-cache:/home/ubuntu/.cache/huggingface/hub restart: unless-stopped # For GPU acceleration, use: ghcr.io/speaches-ai/speaches:latest-cuda # and add GPU device mapping (see docs/5-CONFIGURATION/local-tts.md) open_notebook: image: lfnovo/open_notebook:v1-latest ports: - "8502:8502" - "5055:5055" environment: # REQUIRED: Change this to your own secret string - OPEN_NOTEBOOK_ENCRYPTION_KEY=change-me-to-a-secret-string # Database connection - SURREAL_URL=ws://surrealdb:8000/rpc - SURREAL_USER=root - SURREAL_PASSWORD=root - SURREAL_NAMESPACE=open_notebook - SURREAL_DATABASE=open_notebook volumes: - ./notebook_data:/app/data depends_on: - surrealdb - speaches restart: always pull_policy: always volumes: hf-hub-cache: # ========================================== # AFTER STARTING: Download Speech Models # ========================================== # # For TTS (Text-to-Speech): # docker compose exec speaches uv tool run speaches-cli model download speaches-ai/Kokoro-82M-v1.0-ONNX # # For STT (Speech-to-Text): # docker compose exec speaches uv tool run speaches-cli model download Systran/faster-whisper-small # # ========================================== # CONFIGURATION IN OPEN NOTEBOOK # ========================================== # # 1. Go to Settings → API Keys # 2. Click "Add Credential" → Select "OpenAI-Compatible" # 3. Configure: # - Name: "Local Speaches" # - Base URL for TTS: http://host.docker.internal:8969/v1 (macOS/Windows) # or: http://172.17.0.1:8969/v1 (Linux) # - Base URL for STT: (same as TTS) # 4. Click Save → Test Connection # # 5. Go to Settings → Models # 6. Add TTS Model: # - Provider: openai_compatible # - Model Name: speaches-ai/Kokoro-82M-v1.0-ONNX # - Display Name: Local TTS # # 7. Add STT Model: # - Provider: openai_compatible # - Model Name: Systran/faster-whisper-small # - Display Name: Local Whisper # # ========================================== # TESTING # ========================================== # # Test TTS: # curl "http://localhost:8969/v1/audio/speech" -s \ # -H "Content-Type: application/json" \ # --output test.mp3 \ # --data '{"input": "Hello, local TTS works!", "model": "speaches-ai/Kokoro-82M-v1.0-ONNX", "voice": "af_bella"}' # # Test STT: # curl "http://localhost:8969/v1/audio/transcriptions" \ # -F "file=@test.mp3" \ # -F "model=Systran/faster-whisper-small" # # Available voices: af_bella, af_sarah, am_adam, am_michael, bf_emma, bm_george # Available models: See docs/5-CONFIGURATION/local-stt.md for model sizes ================================================ FILE: frontend/.gitignore ================================================ # See https://help.github.com/articles/ignoring-files/ for more about ignoring files. # dependencies /node_modules /.pnp .pnp.* .yarn/* !.yarn/patches !.yarn/plugins !.yarn/releases !.yarn/versions # testing /coverage # next.js /.next/ /out/ # production /build # misc .DS_Store *.pem # debug npm-debug.log* yarn-debug.log* yarn-error.log* .pnpm-debug.log* # env files (can opt-in for committing if needed) .env* # vercel .vercel # typescript *.tsbuildinfo next-env.d.ts doc_exports/ ================================================ FILE: frontend/components.json ================================================ { "$schema": "https://ui.shadcn.com/schema.json", "style": "new-york", "rsc": true, "tsx": true, "tailwind": { "config": "", "css": "src/app/globals.css", "baseColor": "neutral", "cssVariables": true, "prefix": "" }, "aliases": { "components": "@/components", "utils": "@/lib/utils", "ui": "@/components/ui", "lib": "@/lib", "hooks": "@/hooks" }, "iconLibrary": "lucide" } ================================================ FILE: frontend/eslint.config.mjs ================================================ import { dirname } from "path"; import { fileURLToPath } from "url"; import { FlatCompat } from "@eslint/eslintrc"; const __filename = fileURLToPath(import.meta.url); const __dirname = dirname(__filename); const compat = new FlatCompat({ baseDirectory: __dirname, }); const eslintConfig = [ ...compat.extends("next/core-web-vitals", "next/typescript"), ]; export default eslintConfig; ================================================ FILE: frontend/next.config.ts ================================================ import type { NextConfig } from "next"; const nextConfig: NextConfig = { // Enable standalone output for optimized Docker deployment output: "standalone", // Experimental features // Type assertion needed: proxyClientMaxBodySize is valid in Next.js 15 but types lag behind experimental: { // Increase proxy body size limit for file uploads (default is 10MB) // This allows larger files to be uploaded through the /api/* rewrite proxy to FastAPI proxyClientMaxBodySize: '100mb', } as NextConfig['experimental'], // API Rewrites: Proxy /api/* requests to FastAPI backend // This simplifies reverse proxy configuration - users only need to proxy to port 8502 // Next.js handles internal routing to the API backend on port 5055 async rewrites() { // INTERNAL_API_URL: Where Next.js server-side should proxy API requests // Default: http://localhost:5055 (single-container deployment) // Override for multi-container: INTERNAL_API_URL=http://api-service:5055 const internalApiUrl = process.env.INTERNAL_API_URL || 'http://localhost:5055' console.log(`[Next.js Rewrites] Proxying /api/* to ${internalApiUrl}/api/*`) return [ { source: '/api/:path*', destination: `${internalApiUrl}/api/:path*`, }, ] }, }; export default nextConfig; ================================================ FILE: frontend/package.json ================================================ { "name": "frontend", "version": "0.1.0", "private": true, "scripts": { "dev": "next dev", "build": "next build", "start": "node start-server.js", "lint": "eslint src/", "test": "vitest run", "test:watch": "vitest", "test:ui": "vitest --ui" }, "dependencies": { "@hookform/resolvers": "^5.1.1", "@monaco-editor/react": "^4.7.0", "@radix-ui/react-accordion": "^1.2.12", "@radix-ui/react-alert-dialog": "^1.1.14", "@radix-ui/react-checkbox": "^1.3.2", "@radix-ui/react-collapsible": "^1.1.11", "@radix-ui/react-dialog": "^1.1.15", "@radix-ui/react-dropdown-menu": "^2.1.15", "@radix-ui/react-label": "^2.1.7", "@radix-ui/react-popover": "^1.1.15", "@radix-ui/react-progress": "^1.1.7", "@radix-ui/react-radio-group": "^1.3.8", "@radix-ui/react-scroll-area": "^1.2.9", "@radix-ui/react-select": "^2.2.5", "@radix-ui/react-separator": "^1.1.7", "@radix-ui/react-slot": "^1.2.3", "@radix-ui/react-tabs": "^1.1.12", "@radix-ui/react-tooltip": "^1.2.7", "@tailwindcss/typography": "^0.5.16", "@tanstack/react-query": "^5.83.0", "@uiw/react-md-editor": "^4.0.8", "axios": "^1.13.5", "class-variance-authority": "^0.7.1", "clsx": "^2.1.1", "cmdk": "^1.1.1", "date-fns": "^4.1.0", "i18next": "^25.7.3", "i18next-browser-languagedetector": "^8.2.0", "lucide-react": "^0.525.0", "next": "^16.1.5", "next-themes": "^0.4.6", "react": "^19.2.3", "react-dom": "^19.2.3", "react-hook-form": "^7.60.0", "react-i18next": "^16.5.0", "react-markdown": "^10.1.0", "remark-gfm": "^4.0.1", "sonner": "^2.0.6", "tailwind-merge": "^3.3.1", "use-debounce": "^10.0.6", "zod": "^4.0.5", "zustand": "^5.0.6" }, "devDependencies": { "@eslint/eslintrc": "^3", "@tailwindcss/postcss": "^4", "@types/node": "^20", "@types/react": "^19", "@types/react-dom": "^19", "eslint": "^9", "eslint-config-next": "15.4.2", "jsdom": "^26.0.0", "tailwindcss": "^4", "tw-animate-css": "^1.3.5", "typescript": "^5", "vitest": "^3.0.0", "@vitest/ui": "^3.0.0", "@vitejs/plugin-react": "^4.3.4", "@testing-library/react": "^16.2.0", "@testing-library/jest-dom": "^6.6.3" } } ================================================ FILE: frontend/postcss.config.mjs ================================================ const config = { plugins: ["@tailwindcss/postcss"], }; export default config; ================================================ FILE: frontend/src/CLAUDE.md ================================================ # Frontend Architecture Next.js React application providing UI for Open Notebook research assistant. Three-layer architecture: **pages** (Next.js App Router), **components** (feature-specific UI), and **lib** (data fetching, state management, utilities). ## High-Level Data Flow ``` Pages (Next.js) → Components (feature-specific) → Hooks (queries/mutations) ↓ Stores (auth/modal state) → API module → Backend ``` User interactions trigger mutations/queries via hooks, which communicate with the backend through the API module. Store state (auth, modals) flows back to components via hooks. Child CLAUDE.md files document specific modules in detail: - **`lib/api/CLAUDE.md`**: Axios client, FormData handling, interceptors - **`lib/hooks/CLAUDE.md`**: TanStack Query wrappers, SSE streaming, context building - **`lib/stores/CLAUDE.md`**: Zustand auth/modal state, localStorage persistence - **`lib/locales/CLAUDE.md`**: Internationalization (i18n) system, translation files - **`components/ui/CLAUDE.md`**: Radix UI primitives, CVA styling, accessibility ## Architectural Layers ### Pages (`src/app/`) — Next.js App Router - `(auth)/login`: Authentication entry point - `(dashboard)/`: Protected routes (notebooks, sources, search, models, etc.) - Directory-based routing; each `page.tsx` is a route endpoint - **Key pattern**: Pages call hooks to fetch data, render components with state - **Router groups** `(auth)`, `(dashboard)` organize routes by feature without affecting URL ### Components (`src/components/`) — Feature-Specific UI - **layout**: `AppShell.tsx`, `AppSidebar.tsx` — main layout wrapper used by all pages - **providers**: `ThemeProvider`, `QueryProvider`, `ModalProvider` — app-wide context setup - **auth**: `LoginForm.tsx` — authentication UI - **common**: `CommandPalette`, `ErrorBoundary`, `ContextToggle`, `ModelSelector` — shared across pages - **ui**: Reusable Radix UI building blocks (see child CLAUDE.md) - **source**, **notebooks**, **search**, **podcasts**: Feature-specific components consuming hooks **Component composition pattern**: Pages → Feature components → UI components. Feature components handle page-level state (loading, error), UI components remain stateless and styled. ### Lib (`src/lib/`) — Data & State Layer #### `lib/api/` — Backend Communication - **`client.ts`**: Central Axios instance with auth interceptor, FormData handling, 10-min timeout - **`query-client.ts`**: TanStack Query configuration - **Resource modules** (`sources.ts`, `chat.ts`, `notebooks.ts`, etc.): Endpoint-specific functions returning typed responses - **Pattern**: All requests go through `apiClient`; auth token auto-added from localStorage #### `lib/hooks/` — React Query + Custom Logic - **Query hooks**: `useNotebookSources`, `useSources`, `useSource` — TanStack Query wrappers with cache keys - **Mutation hooks**: `useCreateSource`, `useUpdateSource`, `useDeleteSource` — mutations with toast feedback + cache invalidation - **Complex hooks**: `useNotebookChat`, `useSourceChat` — session management, message streaming, context building - **SSE streaming**: `useAsk` — parses newline-delimited JSON from backend for multi-stage workflows - **Pattern**: Hooks return `{ data, isLoading, error, refetch }` + action functions; cache invalidation on mutations #### `lib/stores/` — Application State - **`auth-store.ts`**: Authentication state (token, isAuthenticated) with 30-second check caching - **Zustand + persist middleware**: Auto-syncs sensitive state to localStorage - **Pattern**: Store actions (`login()`, `logout()`, `checkAuth()`) update state; consumed via hooks in components #### `lib/types/` — TypeScript Definitions - API request/response shapes, domain models (Notebook, Source, Note, etc.) - Ensures type safety across API calls and store mutations #### `lib/locales/` — Internationalization (i18n) - **Locale files** (`en-US/`, `pt-BR/`, `zh-CN/`, `zh-TW/`, `ja-JP/`): Translation strings organized by feature - **`i18n.ts`**: i18next configuration with language detection - **`use-translation.ts`**: Custom hook with Proxy-based `t.section.key` access pattern - **Pattern**: Components call `useTranslation()` hook; access strings via `t.common.save`, `t.notebooks.title` ## Data & Control Flow Walkthrough ### Example: Notebook Chat 1. **Page** (`notebooks/[id]/page.tsx`) fetches initial data, passes `notebookId` to `ChatColumn` component 2. **Hook call** (`useNotebookChat()`): - Queries sessions for notebook via TanStack Query - Sets up message state + context building logic - Returns `{ messages, sendMessage(), setModelOverride() }` 3. **Component renders**: `ChatColumn` displays messages, text input 4. **User sends message**: Component calls `sendMessage()` hook 5. **Hook execution**: - Builds context from selected sources/notes via `buildContext()` helper - Calls `chatApi.sendMessage()` (from API module) - Client-side optimistic update: adds message to local state before response 6. **Backend response** arrives, TanStack Query updates cache 7. **Cache invalidation** on other source/note mutations ensures stale UI refreshes ### Example: File Upload with Source Creation 1. **Component** (`SourceDialog`) renders form with file picker 2. **Hook** (`useFileUpload`): - Converts file to FormData (JSON fields stringified) - Calls `sourcesApi.create()` with FormData - API client interceptor deletes Content-Type header (lets browser set multipart boundary) 3. **Toast notifications** show progress 4. **Cache invalidation** on success: `queryClient.invalidateQueries(['sources'])` 5. **Related queries** auto-refetch: notebooks, sources list, etc. ## Key Patterns & Cross-Layer Coordination ### Caching & Invalidation - **Query keys**: `QUERY_KEYS.notebook(id)`, `QUERY_KEYS.sources(notebookId)` — hierarchical structure - **Broad invalidation**: `['sources']` invalidates all source queries; trade-off between accuracy + performance - **Auto-refetch**: `refetchOnWindowFocus: true` on frequently-changing data (sources, notebooks) ### Auth & Protected Routes - **Proxy** (`src/proxy.ts`): Redirects root `/` to `/notebooks` - **Auth store**: Validates token via `/notebooks` API call (actual validation, not JWT decode) - **Interceptor**: Adds `Bearer {token}` to all requests; 401 response clears auth and redirects to login ### Modal State Management - **Modal hooks**: Components query modal state from stores - **Context**: Modals pass data (e.g., notebook ID) to child components - **Pattern**: One store per modal type; triggered by button clicks + data passing via hook arguments ### Error Handling - **API errors**: All request failures propagate to consuming code; components show toast notifications - **Error message resolution** (`lib/utils/error-handler.ts`): `getApiErrorMessage()` tries i18n mapping first via `ERROR_MAP`, then falls back to displaying the backend's descriptive error message directly. This ensures user-friendly error messages from the error classification system are shown as-is. - **Toast feedback**: Mutations show success/error toasts (from `sonner` library) - **Error boundary**: App-level error boundary catches React render errors; shows fallback UI ### FormData Handling - **JSON fields**: Nested objects (arrays, objects) must be JSON stringified before FormData - **Content-Type header**: Removed by interceptor for FormData requests (lets browser set boundary) - **Example**: `sources` array converted to string via `JSON.stringify()` before appending to FormData ## Component Organization Within Features - **Feature folders** (`source/`, `notebooks/`, `podcasts/`): Group related components - **Composition**: Larger components nest smaller ones; no deep prop drilling (state lifted to hooks) - **Dialog patterns**: Features define dialog components for inline actions (edit, create, delete) - **Props**: Components accept data + action callbacks from parent or hooks ## Providers & Context Setup **Root layout** (`app/layout.tsx`) wraps app with (outermost → innermost): 1. `ErrorBoundary` — React error boundary (catches all render errors) 2. `ThemeProvider` — next-themes for light/dark mode 3. `QueryProvider` — TanStack Query client 4. `I18nProvider` — i18next initialization and language loading overlay 5. `ConnectionGuard` — checks backend connectivity on startup 6. `Toaster` — sonner toast notification system (inside ConnectionGuard) ## Important Gotchas & Design Decisions - **Token storage**: Stored in localStorage under `auth-storage` key (Zustand persist); consumed by API interceptor - **Base URL discovery**: API client fetches base URL from runtime config on first request (async; can be slow on startup) - **Optimistic updates**: Chat messages added to state before server confirmation; removed on error - **Modal lifecycle**: Dialogs not auto-reset; parent must clear form state after submit - **Focus management**: Dialog auto-focuses first input; can cause layout shifts if inputs are conditional - **Cache invalidation breadth**: Trade-off between precision + simplicity; broad invalidation simpler but may over-fetch ## How to Add a New Feature 1. **Create page**: `app/(dashboard)/feature/page.tsx` — calls hooks, renders components 2. **Create feature components**: `components/feature/` — compose UI + business logic 3. **Add hooks** (if data needed): `lib/hooks/useFeature.ts` — TanStack Query wrapper 4. **Add API module** (if backend call needed): `lib/api/feature.ts` — resource-specific functions 5. **Add types**: `lib/types/api.ts` — request/response shapes 6. **Use UI components**: Import from `components/ui/` for consistent styling 7. **Handle auth**: Middleware redirects unauthenticated users; no special handling needed in component ## Testing - **Hooks**: Mock API functions, wrap in `QueryClientProvider`, assert query/mutation behavior - **Components**: Mock hooks via `vi.fn()`, test rendering + user interactions - **API calls**: Mock `axios` interceptors; test request/response shapes - **Stores**: Mock store state, test mutations via `act()`, assert state changes See child CLAUDE.md files for module-specific testing patterns. ================================================ FILE: frontend/src/app/(auth)/login/page.tsx ================================================ import { LoginForm } from '@/components/auth/LoginForm' import { ErrorBoundary } from '@/components/common/ErrorBoundary' export default function LoginPage() { return ( ) } ================================================ FILE: frontend/src/app/(dashboard)/advanced/components/RebuildEmbeddings.tsx ================================================ 'use client' import { useState, useEffect, useCallback } from 'react' import { useMutation } from '@tanstack/react-query' import { Card, CardContent, CardDescription, CardHeader, CardTitle } from '@/components/ui/card' import { Button } from '@/components/ui/button' import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from '@/components/ui/select' import { Checkbox } from '@/components/ui/checkbox' import { Label } from '@/components/ui/label' import { Alert, AlertDescription } from '@/components/ui/alert' import { Progress } from '@/components/ui/progress' import { Loader2, AlertCircle, CheckCircle2, XCircle, Clock } from 'lucide-react' import { Accordion, AccordionContent, AccordionItem, AccordionTrigger, } from '@/components/ui/accordion' import { embeddingApi } from '@/lib/api/embedding' import type { RebuildEmbeddingsRequest, RebuildStatusResponse } from '@/lib/api/embedding' import { useTranslation } from '@/lib/hooks/use-translation' export function RebuildEmbeddings() { const { t } = useTranslation() const [mode, setMode] = useState<'existing' | 'all'>('existing') const [includeSources, setIncludeSources] = useState(true) const [includeNotes, setIncludeNotes] = useState(true) const [includeInsights, setIncludeInsights] = useState(true) const [commandId, setCommandId] = useState(null) const [status, setStatus] = useState(null) const [pollingInterval, setPollingInterval] = useState(null) // Rebuild mutation const rebuildMutation = useMutation({ mutationFn: async (request: RebuildEmbeddingsRequest) => { return embeddingApi.rebuildEmbeddings(request) }, onSuccess: (data) => { setCommandId(data.command_id) // Start polling for status startPolling(data.command_id) } }) // Start polling for rebuild status const startPolling = (cmdId: string) => { if (pollingInterval) { clearInterval(pollingInterval) } const interval = setInterval(async () => { try { const statusData = await embeddingApi.getRebuildStatus(cmdId) setStatus(statusData) // Stop polling if completed or failed if (statusData.status === 'completed' || statusData.status === 'failed') { stopPolling() } } catch (error) { console.error('Failed to fetch rebuild status:', error) } }, 5000) // Poll every 5 seconds setPollingInterval(interval) } // Stop polling const stopPolling = useCallback(() => { if (pollingInterval) { clearInterval(pollingInterval) setPollingInterval(null) } }, [pollingInterval]) // Cleanup on unmount useEffect(() => { return () => { stopPolling() } }, [stopPolling]) const handleStartRebuild = () => { const request: RebuildEmbeddingsRequest = { mode, include_sources: includeSources, include_notes: includeNotes, include_insights: includeInsights } rebuildMutation.mutate(request) } const handleReset = () => { stopPolling() setCommandId(null) setStatus(null) rebuildMutation.reset() } const isAnyTypeSelected = includeSources || includeNotes || includeInsights const isRebuildActive = commandId && status && (status.status === 'queued' || status.status === 'running') const progressData = status?.progress const stats = status?.stats const totalItems = progressData?.total_items ?? progressData?.total ?? 0 const processedItems = progressData?.processed_items ?? progressData?.processed ?? 0 const derivedProgressPercent = progressData?.percentage ?? (totalItems > 0 ? (processedItems / totalItems) * 100 : 0) const progressPercent = Number.isFinite(derivedProgressPercent) ? derivedProgressPercent : 0 const sourcesProcessed = stats?.sources_processed ?? stats?.sources ?? 0 const notesProcessed = stats?.notes_processed ?? stats?.notes ?? 0 const insightsProcessed = stats?.insights_processed ?? stats?.insights ?? 0 const failedItems = stats?.failed_items ?? stats?.failed ?? 0 const computedDuration = status?.started_at && status?.completed_at ? (new Date(status.completed_at).getTime() - new Date(status.started_at).getTime()) / 1000 : undefined const processingTimeSeconds = stats?.processing_time ?? computedDuration return ( {t.advanced.rebuildEmbeddings} {t.advanced.rebuildEmbeddingsDesc} {/* Configuration Form */} {!isRebuildActive && (

{mode === 'existing' ? t.advanced.rebuild.existingDesc : t.advanced.rebuild.allDesc}

{t.advanced.rebuild.include}
setIncludeSources(checked === true)} />
setIncludeNotes(checked === true)} />
setIncludeInsights(checked === true)} />
{!isAnyTypeSelected && ( {t.advanced.rebuild.selectOneError} )}
{rebuildMutation.isError && ( {t.advanced.rebuild.failed}: {(rebuildMutation.error as Error)?.message || t.common.error} )}
)} {/* Status Display */} {status && (
{status.status === 'queued' && } {status.status === 'running' && } {status.status === 'completed' && } {status.status === 'failed' && }
{status.status === 'queued' && t.advanced.rebuild.queued} {status.status === 'running' && t.advanced.rebuild.running} {status.status === 'completed' && t.advanced.rebuild.completed} {status.status === 'failed' && t.advanced.rebuild.failed} {status.status === 'running' && ( {t.advanced.rebuild.leavePageHint} )}
{(status.status === 'completed' || status.status === 'failed') && ( )}
{progressData && (
{t.common.progress} {t.advanced.rebuild.itemsProcessed .replace('{processed}', processedItems.toString()) .replace('{total}', totalItems.toString()) .replace('{percent}', progressPercent.toFixed(1))}
{failedItems > 0 && (

⚠️ {t.advanced.rebuild.failedItems.replace('{count}', failedItems.toString())}

)}
)} {stats && (

{t.navigation.sources}

{sourcesProcessed}

{t.common.notes}

{notesProcessed}

{t.common.insights}

{insightsProcessed}

{t.advanced.rebuild.time}

{processingTimeSeconds !== undefined ? `${processingTimeSeconds.toFixed(1)}s` : '—'}

)} {status.error_message && ( {status.error_message} )} {status.started_at && (

{t.common.created.replace('{time}', new Date(status.started_at).toLocaleString())}

{status.completed_at && (

{t.notebooks.updated}: {new Date(status.completed_at).toLocaleString()}

)}
)}
)} {/* Help Section */} {t.advanced.rebuild.whenToRebuild}

{t.advanced.rebuild.whenToRebuildAns}

 {t.advanced.rebuild.howLong}

{t.advanced.rebuild.howLongAns}

 {t.advanced.rebuild.isSafe}

{t.advanced.rebuild.isSafeAns}

 ) } ================================================ FILE: frontend/src/app/(dashboard)/advanced/components/SystemInfo.tsx ================================================ 'use client' import { useEffect, useState } from 'react' import { Card } from '@/components/ui/card' import { getConfig } from '@/lib/config' import { Badge } from '@/components/ui/badge' import { useTranslation } from '@/lib/hooks/use-translation' export function SystemInfo() { const { t } = useTranslation() const [config, setConfig] = useState<{ version: string latestVersion?: string | null hasUpdate?: boolean } | null>(null) const [isLoading, setIsLoading] = useState(true) useEffect(() => { const loadConfig = async () => { try { const cfg = await getConfig() setConfig(cfg) } catch (error) { console.error('Failed to load config:', error) } finally { setIsLoading(false) } } loadConfig() }, []) if (isLoading) { return (

{t.advanced.systemInfo}

{t.common.loading}
 ) } return (

{t.advanced.systemInfo}

{/* Current Version */}
{t.advanced.currentVersion} {config?.version || t.advanced.unknown}
{/* Latest Version */} {config?.latestVersion && (
{t.advanced.latestVersion} {config.latestVersion}
)} {/* Update Status */}
{t.advanced.status} {config?.hasUpdate ? ( {t.advanced.updateAvailable.replace('{version}', config.latestVersion || '')} ) : config?.latestVersion ? ( {t.advanced.upToDate} ) : ( {t.advanced.unknown} )}
{/* GitHub Repository Link */} {config?.hasUpdate && ( )} {/* Version Check Failed Message */} {!config?.latestVersion && config?.version && (
{t.advanced.updateCheckFailed}
)}
) } ================================================ FILE: frontend/src/app/(dashboard)/advanced/page.tsx ================================================ 'use client' import { AppShell } from '@/components/layout/AppShell' import { RebuildEmbeddings } from './components/RebuildEmbeddings' import { SystemInfo } from './components/SystemInfo' import { useTranslation } from '@/lib/hooks/use-translation' export default function AdvancedPage() { const { t } = useTranslation() return (

{t.advanced.title}

{t.advanced.desc}

) } ================================================ FILE: frontend/src/app/(dashboard)/layout.tsx ================================================ 'use client' import { useAuth } from '@/lib/hooks/use-auth' import { useVersionCheck } from '@/lib/hooks/use-version-check' import { useRouter } from 'next/navigation' import { useEffect, useState } from 'react' import { LoadingSpinner } from '@/components/common/LoadingSpinner' import { ErrorBoundary } from '@/components/common/ErrorBoundary' import { ModalProvider } from '@/components/providers/ModalProvider' import { CreateDialogsProvider } from '@/lib/hooks/use-create-dialogs' import { CommandPalette } from '@/components/common/CommandPalette' export default function DashboardLayout({ children, }: { children: React.ReactNode }) { const { isAuthenticated, isLoading } = useAuth() const router = useRouter() const [hasCheckedAuth, setHasCheckedAuth] = useState(false) // Check for version updates once per session useVersionCheck() useEffect(() => { // Mark that we've completed the initial auth check if (!isLoading) { setHasCheckedAuth(true) // Redirect to login if not authenticated if (!isAuthenticated) { // Store the current path to redirect back after login const currentPath = window.location.pathname + window.location.search sessionStorage.setItem('redirectAfterLogin', currentPath) router.push('/login') } } }, [isAuthenticated, isLoading, router]) // Show loading spinner during initial auth check or while loading if (isLoading || !hasCheckedAuth) { return (
) } // Don't render anything if not authenticated (during redirect) if (!isAuthenticated) { return null } return ( {children} ) } ================================================ FILE: frontend/src/app/(dashboard)/notebooks/[id]/page.tsx ================================================ 'use client' import { useState, useEffect } from 'react' import { useParams } from 'next/navigation' import { AppShell } from '@/components/layout/AppShell' import { NotebookHeader } from '../components/NotebookHeader' import { SourcesColumn } from '../components/SourcesColumn' import { NotesColumn } from '../components/NotesColumn' import { ChatColumn } from '../components/ChatColumn' import { useNotebook } from '@/lib/hooks/use-notebooks' import { useNotebookSources } from '@/lib/hooks/use-sources' import { useNotes } from '@/lib/hooks/use-notes' import { LoadingSpinner } from '@/components/common/LoadingSpinner' import { useNotebookColumnsStore } from '@/lib/stores/notebook-columns-store' import { useIsDesktop } from '@/lib/hooks/use-media-query' import { useTranslation } from '@/lib/hooks/use-translation' import { cn } from '@/lib/utils' import { Tabs, TabsList, TabsTrigger } from '@/components/ui/tabs' import { FileText, StickyNote, MessageSquare } from 'lucide-react' export type ContextMode = 'off' | 'insights' | 'full' export interface ContextSelections { sources: Record notes: Record } export default function NotebookPage() { const { t } = useTranslation() const params = useParams() // Ensure the notebook ID is properly decoded from URL const notebookId = params?.id ? decodeURIComponent(params.id as string) : '' const { data: notebook, isLoading: notebookLoading } = useNotebook(notebookId) const { sources, isLoading: sourcesLoading, refetch: refetchSources, hasNextPage, isFetchingNextPage, fetchNextPage, } = useNotebookSources(notebookId) const { data: notes, isLoading: notesLoading } = useNotes(notebookId) // Get collapse states for dynamic layout const { sourcesCollapsed, notesCollapsed } = useNotebookColumnsStore() // Detect desktop to avoid double-mounting ChatColumn const isDesktop = useIsDesktop() // Mobile tab state (Sources, Notes, or Chat) const [mobileActiveTab, setMobileActiveTab] = useState<'sources' | 'notes' | 'chat'>('chat') // Context selection state const [contextSelections, setContextSelections] = useState({ sources: {}, notes: {} }) // Initialize and update selections when sources load or change useEffect(() => { if (sources && sources.length > 0) { setContextSelections(prev => { const newSourceSelections = { ...prev.sources } sources.forEach(source => { const currentMode = newSourceSelections[source.id] const hasInsights = source.insights_count > 0 if (currentMode === undefined) { // Initial setup - default based on insights availability newSourceSelections[source.id] = hasInsights ? 'insights' : 'full' } else if (currentMode === 'full' && hasInsights) { // Source gained insights while in 'full' mode - auto-switch to 'insights' newSourceSelections[source.id] = 'insights' } }) return { ...prev, sources: newSourceSelections } }) } }, [sources]) useEffect(() => { if (notes && notes.length > 0) { setContextSelections(prev => { const newNoteSelections = { ...prev.notes } notes.forEach(note => { // Only set default if not already set if (!(note.id in newNoteSelections)) { // Notes default to 'full' newNoteSelections[note.id] = 'full' } }) return { ...prev, notes: newNoteSelections } }) } }, [notes]) // Handler to update context selection const handleContextModeChange = (itemId: string, mode: ContextMode, type: 'source' | 'note') => { setContextSelections(prev => ({ ...prev, [type === 'source' ? 'sources' : 'notes']: { ...(type === 'source' ? prev.sources : prev.notes), [itemId]: mode } })) } if (notebookLoading) { return (
) } if (!notebook) { return (

{t.notebooks.notFound}

{t.notebooks.notFoundDesc}

 ) } return (
{/* Mobile: Tabbed interface - only render on mobile to avoid double-mounting */} {!isDesktop && ( <>
setMobileActiveTab(value as 'sources' | 'notes' | 'chat')}> {t.navigation.sources} {t.common.notes} {t.common.chat}
{/* Mobile: Show only active tab */}
{mobileActiveTab === 'sources' && ( handleContextModeChange(sourceId, mode, 'source')} hasNextPage={hasNextPage} isFetchingNextPage={isFetchingNextPage} fetchNextPage={fetchNextPage} /> )} {mobileActiveTab === 'notes' && ( handleContextModeChange(noteId, mode, 'note')} /> )} {mobileActiveTab === 'chat' && ( )}
)} {/* Desktop: Collapsible columns layout */}
{/* Sources Column */}
handleContextModeChange(sourceId, mode, 'source')} hasNextPage={hasNextPage} isFetchingNextPage={isFetchingNextPage} fetchNextPage={fetchNextPage} />
{/* Notes Column */}
handleContextModeChange(noteId, mode, 'note')} />
{/* Chat Column - always expanded, takes remaining space */}
) } ================================================ FILE: frontend/src/app/(dashboard)/notebooks/components/ChatColumn.test.tsx ================================================ import { render, screen } from '@testing-library/react' import { describe, it, expect, vi } from 'vitest' import { ChatColumn } from './ChatColumn' import { useNotes } from '@/lib/hooks/use-notes' import { useNotebookChat } from '@/lib/hooks/useNotebookChat' // Mock the hooks vi.mock('@/lib/hooks/use-notes') vi.mock('@/lib/hooks/useNotebookChat') vi.mock('@/components/source/ChatPanel', () => ({ ChatPanel: () =>
})) // Type-safe mock factory for useNotes hook function createNotesMock(overrides: { isLoading?: boolean } = {}) { return { data: [], isLoading: overrides.isLoading ?? false, } as unknown as ReturnType } // Type-safe mock factory for useNotebookChat hook function createChatMock() { return { messages: [], isSending: false, tokenCount: 0, charCount: 0, sessions: [], currentSessionId: null, } as unknown as ReturnType } describe('ChatColumn', () => { const baseProps = { notebookId: 'test-notebook', contextSelections: { sources: {}, notes: {} }, sources: [], } it('shows loading spinner when fetching data', () => { vi.mocked(useNotes).mockReturnValue(createNotesMock({ isLoading: true })) vi.mocked(useNotebookChat).mockReturnValue(createChatMock()) render() // Should show loading spinner expect(screen.getByTestId('loading-spinner')).toBeInTheDocument() }) it('renders chat panel when data is loaded', () => { vi.mocked(useNotes).mockReturnValue(createNotesMock({ isLoading: false })) vi.mocked(useNotebookChat).mockReturnValue(createChatMock()) render() // Should show chat panel expect(screen.getByTestId('chat-panel')).toBeInTheDocument() }) }) ================================================ FILE: frontend/src/app/(dashboard)/notebooks/components/ChatColumn.tsx ================================================ 'use client' import { useMemo } from 'react' import { useNotebookChat } from '@/lib/hooks/useNotebookChat' import { useNotes } from '@/lib/hooks/use-notes' import { ChatPanel } from '@/components/source/ChatPanel' import { LoadingSpinner } from '@/components/common/LoadingSpinner' import { Card, CardContent } from '@/components/ui/card' import { AlertCircle } from 'lucide-react' import { ContextSelections } from '../[id]/page' import { useTranslation } from '@/lib/hooks/use-translation' import { SourceListResponse } from '@/lib/types/api' interface ChatColumnProps { notebookId: string contextSelections: ContextSelections sources: SourceListResponse[] sourcesLoading: boolean } export function ChatColumn({ notebookId, contextSelections, sources, sourcesLoading }: ChatColumnProps) { const { t } = useTranslation() // Fetch notes for this notebook const { data: notes = [], isLoading: notesLoading } = useNotes(notebookId) // Initialize notebook chat hook const chat = useNotebookChat({ notebookId, sources, notes, contextSelections }) // Calculate context stats for indicator const contextStats = useMemo(() => { let sourcesInsights = 0 let sourcesFull = 0 let notesCount = 0 // Count sources by mode sources.forEach(source => { const mode = contextSelections.sources[source.id] if (mode === 'insights') { sourcesInsights++ } else if (mode === 'full') { sourcesFull++ } }) // Count notes that are included (not 'off') notes.forEach(note => { const mode = contextSelections.notes[note.id] if (mode === 'full') { notesCount++ } }) return { sourcesInsights, sourcesFull, notesCount, tokenCount: chat.tokenCount, charCount: chat.charCount } }, [sources, notes, contextSelections, chat.tokenCount, chat.charCount]) // Show loading state while sources/notes are being fetched if (sourcesLoading || notesLoading) { return ( ) } // Show error state if data fetch failed (unlikely but good to handle) if (!sources && !notes) { return (

{t.chat.unableToLoadChat}

{t.common.refreshPage || 'Please try refreshing the page'}

 ) } return ( chat.sendMessage(message, modelOverride)} modelOverride={chat.currentSession?.model_override ?? chat.pendingModelOverride ?? undefined} onModelChange={(model) => chat.setModelOverride(model ?? null)} sessions={chat.sessions} currentSessionId={chat.currentSessionId} onCreateSession={(title) => chat.createSession(title)} onSelectSession={chat.switchSession} onUpdateSession={(sessionId, title) => chat.updateSession(sessionId, { title })} onDeleteSession={chat.deleteSession} loadingSessions={chat.loadingSessions} notebookContextStats={contextStats} notebookId={notebookId} /> ) } ================================================ FILE: frontend/src/app/(dashboard)/notebooks/components/NoteEditorDialog.tsx ================================================ 'use client' import { Controller, useForm, useWatch } from 'react-hook-form' import { useEffect, useState } from 'react' import { useQueryClient } from '@tanstack/react-query' import { zodResolver } from '@hookform/resolvers/zod' import { z } from 'zod' import { Dialog, DialogContent, DialogTitle } from '@/components/ui/dialog' import { Button } from '@/components/ui/button' import { useCreateNote, useUpdateNote, useNote } from '@/lib/hooks/use-notes' import { QUERY_KEYS } from '@/lib/api/query-client' import { MarkdownEditor } from '@/components/ui/markdown-editor' import { InlineEdit } from '@/components/common/InlineEdit' import { cn } from "@/lib/utils"; import { useTranslation } from '@/lib/hooks/use-translation' const createNoteSchema = z.object({ title: z.string().optional(), content: z.string().min(1, 'Content is required'), }) type CreateNoteFormData = z.infer interface NoteEditorDialogProps { open: boolean onOpenChange: (open: boolean) => void notebookId: string note?: { id: string; title: string | null; content: string | null } } export function NoteEditorDialog({ open, onOpenChange, notebookId, note }: NoteEditorDialogProps) { const { t } = useTranslation() const createNote = useCreateNote() const updateNote = useUpdateNote() const queryClient = useQueryClient() const isEditing = Boolean(note) // Ensure note ID has 'note:' prefix for API calls const noteIdWithPrefix = note?.id ? (note.id.includes(':') ? note.id : `note:${note.id}`) : '' const { data: fetchedNote, isLoading: noteLoading } = useNote(noteIdWithPrefix, { enabled: open && !!note?.id }) const isSaving = isEditing ? updateNote.isPending : createNote.isPending const { handleSubmit, control, formState: { errors }, reset, setValue, } = useForm({ resolver: zodResolver(createNoteSchema), defaultValues: { title: '', content: '', }, }) const watchTitle = useWatch({ control, name: 'title' }) const [isEditorFullscreen, setIsEditorFullscreen] = useState(false) useEffect(() => { if (!open) { reset({ title: '', content: '' }) return } const source = fetchedNote ?? note const title = source?.title ?? '' const content = source?.content ?? '' reset({ title, content }) }, [open, note, fetchedNote, reset]) useEffect(() => { if (!open) return const observer = new MutationObserver(() => { setIsEditorFullscreen(!!document.querySelector('.w-md-editor-fullscreen')) }) observer.observe(document.body, { subtree: true, attributes: true, attributeFilter: ['class'] }) return () => observer.disconnect() }, [open]) const onSubmit = async (data: CreateNoteFormData) => { if (note) { await updateNote.mutateAsync({ id: noteIdWithPrefix, data: { title: data.title || undefined, content: data.content, }, }) // Only invalidate notebook-specific queries if we have a notebookId if (notebookId) { queryClient.invalidateQueries({ queryKey: QUERY_KEYS.notes(notebookId) }) } } else { // Creating a note requires a notebookId if (!notebookId) { console.error('Cannot create note without notebook_id') return } await createNote.mutateAsync({ title: data.title || undefined, content: data.content, note_type: 'human', notebook_id: notebookId, }) } reset() onOpenChange(false) } const handleClose = () => { reset() setIsEditorFullscreen(false) onOpenChange(false) } return ( {isEditing ? t.sources.editNote : t.sources.createNote}
{isEditing && noteLoading ? (
{t.common.loading}
) : ( <>
setValue('title', value || '')} placeholder={t.sources.addTitle} emptyText={t.sources.untitledNote} className="text-xl font-semibold" inputClassName="text-xl font-semibold" />
( )} /> {errors.content && (

{errors.content.message}

)}
)}
) } ================================================ FILE: frontend/src/app/(dashboard)/notebooks/components/NotebookCard.tsx ================================================ 'use client' import { useRouter } from 'next/navigation' import { NotebookResponse } from '@/lib/types/api' import { Card, CardContent, CardDescription, CardHeader, CardTitle } from '@/components/ui/card' import { Button } from '@/components/ui/button' import { Badge } from '@/components/ui/badge' import { MoreHorizontal, Archive, ArchiveRestore, Trash2, FileText, StickyNote } from 'lucide-react' import { formatDistanceToNow } from 'date-fns' import { DropdownMenu, DropdownMenuContent, DropdownMenuItem, DropdownMenuTrigger, } from '@/components/ui/dropdown-menu' import { useUpdateNotebook } from '@/lib/hooks/use-notebooks' import { NotebookDeleteDialog } from './NotebookDeleteDialog' import { useState } from 'react' import { useTranslation } from '@/lib/hooks/use-translation' import { getDateLocale } from '@/lib/utils/date-locale' interface NotebookCardProps { notebook: NotebookResponse } export function NotebookCard({ notebook }: NotebookCardProps) { const { t, language } = useTranslation() const [showDeleteDialog, setShowDeleteDialog] = useState(false) const router = useRouter() const updateNotebook = useUpdateNotebook() const handleArchiveToggle = (e: React.MouseEvent) => { e.stopPropagation() updateNotebook.mutate({ id: notebook.id, data: { archived: !notebook.archived } }) } const handleCardClick = () => { router.push(`/notebooks/${encodeURIComponent(notebook.id)}`) } return ( <>
{notebook.name} {notebook.archived && ( {t.notebooks.archived} )}
e.stopPropagation()}> {notebook.archived ? ( <> {t.notebooks.unarchive} ) : ( <> {t.notebooks.archive} )} { e.stopPropagation() setShowDeleteDialog(true) }} className="text-red-600" > {t.common.delete}
{notebook.description || t.chat.noDescription}
{t.common.updated.replace('{time}', formatDistanceToNow(new Date(notebook.updated), { addSuffix: true, locale: getDateLocale(language) }))}
{/* Item counts footer */}
{notebook.source_count} {notebook.note_count}
) } ================================================ FILE: frontend/src/app/(dashboard)/notebooks/components/NotebookDeleteDialog.tsx ================================================ 'use client' import { useState, useEffect } from 'react' import { AlertDialog, AlertDialogAction, AlertDialogCancel, AlertDialogContent, AlertDialogDescription, AlertDialogFooter, AlertDialogHeader, AlertDialogTitle, } from '@/components/ui/alert-dialog' import { RadioGroup, RadioGroupItem } from '@/components/ui/radio-group' import { Label } from '@/components/ui/label' import { useTranslation } from '@/lib/hooks/use-translation' import { LoadingSpinner } from '@/components/common/LoadingSpinner' import { useNotebookDeletePreview, useDeleteNotebook } from '@/lib/hooks/use-notebooks' import { useRouter } from 'next/navigation' interface NotebookDeleteDialogProps { open: boolean onOpenChange: (open: boolean) => void notebookId: string notebookName: string redirectAfterDelete?: boolean } export function NotebookDeleteDialog({ open, onOpenChange, notebookId, notebookName, redirectAfterDelete = false, }: NotebookDeleteDialogProps) { const { t } = useTranslation() const router = useRouter() const [sourceAction, setSourceAction] = useState<'keep' | 'delete'>('keep') // Reset state when dialog opens useEffect(() => { if (open) { setSourceAction('keep') } }, [open, notebookId]) // Fetch delete preview when dialog is open const { data: preview, isLoading: isLoadingPreview, error: previewError } = useNotebookDeletePreview( notebookId, open ) const deleteNotebook = useDeleteNotebook() const handleConfirm = async () => { await deleteNotebook.mutateAsync({ id: notebookId, deleteExclusiveSources: sourceAction === 'delete', }) onOpenChange(false) if (redirectAfterDelete) { router.push('/notebooks') } } const isDeleting = deleteNotebook.isPending return ( {t.notebooks.deleteNotebook} {t.notebooks.deleteNotebookDesc.replace('{name}', notebookName)}
{isLoadingPreview ? (
{t.notebooks.deleteNotebookLoading}
) : previewError ? (
{t.common.error}: {previewError.message || 'Failed to load preview'}
) : preview ? ( <> {/* Notes section */}
{preview.note_count > 0 ? (

{t.notebooks.deleteNotebookNotes.replace( '{count}', String(preview.note_count) )}

) : (

{t.notebooks.deleteNotebookNoNotes}

)}
{/* Shared sources - always above the line */} {preview.shared_source_count > 0 && (

{t.notebooks.deleteNotebookSharedSources.replace( '{count}', String(preview.shared_source_count) )}

)} {/* No sources message */} {preview.exclusive_source_count === 0 && preview.shared_source_count === 0 && (

{t.notebooks.deleteNotebookNoSources}

)} {/* Exclusive sources section - below the line with radio buttons */} {preview.exclusive_source_count > 0 && (

{t.notebooks.deleteNotebookExclusiveSources.replace( '{count}', String(preview.exclusive_source_count) )}

setSourceAction(value as 'keep' | 'delete')} disabled={isDeleting} >
)} ) : null}
{t.common.cancel} {isDeleting ? ( <> {t.common.deleting} ) : ( t.common.delete )} ) } ================================================ FILE: frontend/src/app/(dashboard)/notebooks/components/NotebookHeader.tsx ================================================ 'use client' import { useState } from 'react' import { NotebookResponse } from '@/lib/types/api' import { Button } from '@/components/ui/button' import { Badge } from '@/components/ui/badge' import { Archive, ArchiveRestore, Trash2 } from 'lucide-react' import { useUpdateNotebook } from '@/lib/hooks/use-notebooks' import { NotebookDeleteDialog } from './NotebookDeleteDialog' import { formatDistanceToNow } from 'date-fns' import { getDateLocale } from '@/lib/utils/date-locale' import { InlineEdit } from '@/components/common/InlineEdit' import { useTranslation } from '@/lib/hooks/use-translation' interface NotebookHeaderProps { notebook: NotebookResponse } export function NotebookHeader({ notebook }: NotebookHeaderProps) { const { t, language } = useTranslation() const dfLocale = getDateLocale(language) const [showDeleteDialog, setShowDeleteDialog] = useState(false) const updateNotebook = useUpdateNotebook() const handleUpdateName = async (name: string) => { if (!name || name === notebook.name) return await updateNotebook.mutateAsync({ id: notebook.id, data: { name } }) } const handleUpdateDescription = async (description: string) => { if (description === notebook.description) return await updateNotebook.mutateAsync({ id: notebook.id, data: { description: description || undefined } }) } const handleArchiveToggle = () => { updateNotebook.mutate({ id: notebook.id, data: { archived: !notebook.archived } }) } return ( <>
{notebook.archived && ( {t.notebooks.archived} )}
{t.common.created.replace('{time}', formatDistanceToNow(new Date(notebook.created), { addSuffix: true, locale: dfLocale }))} • {t.common.updated.replace('{time}', formatDistanceToNow(new Date(notebook.updated), { addSuffix: true, locale: dfLocale }))}
) } ================================================ FILE: frontend/src/app/(dashboard)/notebooks/components/NotebookList.tsx ================================================ 'use client' import { NotebookResponse } from '@/lib/types/api' import { NotebookCard } from './NotebookCard' import { LoadingSpinner } from '@/components/common/LoadingSpinner' import { EmptyState } from '@/components/common/EmptyState' import { Book, ChevronDown, ChevronRight, Plus } from 'lucide-react' import { Button } from '@/components/ui/button' import { useState } from 'react' import { useTranslation } from '@/lib/hooks/use-translation' interface NotebookListProps { notebooks?: NotebookResponse[] isLoading: boolean title: string collapsible?: boolean emptyTitle?: string emptyDescription?: string onAction?: () => void actionLabel?: string } export function NotebookList({ notebooks, isLoading, title, collapsible = false, emptyTitle, emptyDescription, onAction, actionLabel, }: NotebookListProps) { const { t } = useTranslation() const [isExpanded, setIsExpanded] = useState(!collapsible) if (isLoading) { return (
) } if (!notebooks || notebooks.length === 0) { return ( {actionLabel} ) : undefined} /> ) } return (
{collapsible && ( )}

{title}

({notebooks.length})
{isExpanded && (
{notebooks.map((notebook) => ( ))}
)}
) } ================================================ FILE: frontend/src/app/(dashboard)/notebooks/components/NotesColumn.tsx ================================================ 'use client' import { useState, useMemo } from 'react' import { NoteResponse } from '@/lib/types/api' import { Button } from '@/components/ui/button' import { Card, CardContent, CardHeader, CardTitle } from '@/components/ui/card' import { DropdownMenu, DropdownMenuContent, DropdownMenuItem, DropdownMenuTrigger, } from '@/components/ui/dropdown-menu' import { Plus, StickyNote, Bot, User, MoreVertical, Trash2 } from 'lucide-react' import { LoadingSpinner } from '@/components/common/LoadingSpinner' import { EmptyState } from '@/components/common/EmptyState' import { Badge } from '@/components/ui/badge' import { NoteEditorDialog } from './NoteEditorDialog' import { getDateLocale } from '@/lib/utils/date-locale' import { formatDistanceToNow } from 'date-fns' import { ContextToggle } from '@/components/common/ContextToggle' import { ContextMode } from '../[id]/page' import { useDeleteNote } from '@/lib/hooks/use-notes' import { ConfirmDialog } from '@/components/common/ConfirmDialog' import { CollapsibleColumn, createCollapseButton } from '@/components/notebooks/CollapsibleColumn' import { useNotebookColumnsStore } from '@/lib/stores/notebook-columns-store' import { useTranslation } from '@/lib/hooks/use-translation' interface NotesColumnProps { notes?: NoteResponse[] isLoading: boolean notebookId: string contextSelections?: Record onContextModeChange?: (noteId: string, mode: ContextMode) => void } export function NotesColumn({ notes, isLoading, notebookId, contextSelections, onContextModeChange }: NotesColumnProps) { const { t, language } = useTranslation() const [showAddDialog, setShowAddDialog] = useState(false) const [editingNote, setEditingNote] = useState(null) const [deleteDialogOpen, setDeleteDialogOpen] = useState(false) const [noteToDelete, setNoteToDelete] = useState(null) const deleteNote = useDeleteNote() // Collapsible column state const { notesCollapsed, toggleNotes } = useNotebookColumnsStore() const collapseButton = useMemo( () => createCollapseButton(toggleNotes, t.common.notes), [toggleNotes, t.common.notes] ) const handleDeleteClick = (noteId: string) => { setNoteToDelete(noteId) setDeleteDialogOpen(true) } const handleDeleteConfirm = async () => { if (!noteToDelete) return try { await deleteNote.mutateAsync(noteToDelete) setDeleteDialogOpen(false) setNoteToDelete(null) } catch (error) { console.error('Failed to delete note:', error) } } return ( <>
{t.common.notes}
{collapseButton}
{isLoading ? (
) : !notes || notes.length === 0 ? ( ) : (
{notes.map((note) => (
setEditingNote(note)} >
{note.note_type === 'ai' ? ( ) : ( )} {note.note_type === 'ai' ? t.common.aiGenerated : t.common.human}
{formatDistanceToNow(new Date(note.updated), { addSuffix: true, locale: getDateLocale(language) })} {/* Context toggle - only show if handler provided */} {onContextModeChange && contextSelections?.[note.id] && (
event.stopPropagation()}> onContextModeChange(note.id, mode)} />
)} {/* Ellipsis menu for delete action */} { e.stopPropagation() handleDeleteClick(note.id) }} className="text-red-600 focus:text-red-600" > {t.notebooks.deleteNote}
{note.title && (

{note.title}

)} {note.content && (

{note.content}

)}
))}
)} { if (!open) { setShowAddDialog(false) setEditingNote(null) } else { setShowAddDialog(true) } }} notebookId={notebookId} note={editingNote ?? undefined} /> ) } ================================================ FILE: frontend/src/app/(dashboard)/notebooks/components/SourcesColumn.tsx ================================================ 'use client' import { useState, useMemo, useRef, useCallback, useEffect } from 'react' import { SourceListResponse } from '@/lib/types/api' import { Button } from '@/components/ui/button' import { Card, CardContent, CardHeader, CardTitle } from '@/components/ui/card' import { DropdownMenu, DropdownMenuContent, DropdownMenuItem, DropdownMenuTrigger, } from '@/components/ui/dropdown-menu' import { Plus, FileText, Link2, ChevronDown, Loader2 } from 'lucide-react' import { LoadingSpinner } from '@/components/common/LoadingSpinner' import { EmptyState } from '@/components/common/EmptyState' import { AddSourceDialog } from '@/components/sources/AddSourceDialog' import { AddExistingSourceDialog } from '@/components/sources/AddExistingSourceDialog' import { SourceCard } from '@/components/sources/SourceCard' import { useDeleteSource, useRetrySource, useRemoveSourceFromNotebook } from '@/lib/hooks/use-sources' import { ConfirmDialog } from '@/components/common/ConfirmDialog' import { useModalManager } from '@/lib/hooks/use-modal-manager' import { ContextMode } from '../[id]/page' import { CollapsibleColumn, createCollapseButton } from '@/components/notebooks/CollapsibleColumn' import { useNotebookColumnsStore } from '@/lib/stores/notebook-columns-store' import { useTranslation } from '@/lib/hooks/use-translation' interface SourcesColumnProps { sources?: SourceListResponse[] isLoading: boolean notebookId: string notebookName?: string onRefresh?: () => void contextSelections?: Record onContextModeChange?: (sourceId: string, mode: ContextMode) => void // Pagination props hasNextPage?: boolean isFetchingNextPage?: boolean fetchNextPage?: () => void } export function SourcesColumn({ sources, isLoading, notebookId, onRefresh, contextSelections, onContextModeChange, hasNextPage, isFetchingNextPage, fetchNextPage, }: SourcesColumnProps) { const { t } = useTranslation() const [dropdownOpen, setDropdownOpen] = useState(false) const [addDialogOpen, setAddDialogOpen] = useState(false) const [addExistingDialogOpen, setAddExistingDialogOpen] = useState(false) const [deleteDialogOpen, setDeleteDialogOpen] = useState(false) const [sourceToDelete, setSourceToDelete] = useState(null) const [removeDialogOpen, setRemoveDialogOpen] = useState(false) const [sourceToRemove, setSourceToRemove] = useState(null) const { openModal } = useModalManager() const deleteSource = useDeleteSource() const retrySource = useRetrySource() const removeFromNotebook = useRemoveSourceFromNotebook() // Collapsible column state const { sourcesCollapsed, toggleSources } = useNotebookColumnsStore() const collapseButton = useMemo( () => createCollapseButton(toggleSources, t.navigation.sources), [toggleSources, t.navigation.sources] ) // Scroll container ref for infinite scroll const scrollContainerRef = useRef(null) // Handle scroll for infinite loading const handleScroll = useCallback(() => { const container = scrollContainerRef.current if (!container || !hasNextPage || isFetchingNextPage || !fetchNextPage) return const { scrollTop, scrollHeight, clientHeight } = container // Load more when user scrolls within 200px of the bottom if (scrollHeight - scrollTop - clientHeight < 200) { fetchNextPage() } }, [hasNextPage, isFetchingNextPage, fetchNextPage]) // Attach scroll listener useEffect(() => { const container = scrollContainerRef.current if (!container) return container.addEventListener('scroll', handleScroll) return () => container.removeEventListener('scroll', handleScroll) }, [handleScroll]) const handleDeleteClick = (sourceId: string) => { setSourceToDelete(sourceId) setDeleteDialogOpen(true) } const handleDeleteConfirm = async () => { if (!sourceToDelete) return try { await deleteSource.mutateAsync(sourceToDelete) setDeleteDialogOpen(false) setSourceToDelete(null) onRefresh?.() } catch (error) { console.error('Failed to delete source:', error) } } const handleRemoveFromNotebook = (sourceId: string) => { setSourceToRemove(sourceId) setRemoveDialogOpen(true) } const handleRemoveConfirm = async () => { if (!sourceToRemove) return try { await removeFromNotebook.mutateAsync({ notebookId, sourceId: sourceToRemove }) setRemoveDialogOpen(false) setSourceToRemove(null) } catch (error) { console.error('Failed to remove source from notebook:', error) // Error toast is handled by the hook } } const handleRetry = async (sourceId: string) => { try { await retrySource.mutateAsync(sourceId) } catch (error) { console.error('Failed to retry source:', error) } } const handleSourceClick = (sourceId: string) => { openModal('source', sourceId) } return ( <>
{t.navigation.sources}
{ setDropdownOpen(false); setAddDialogOpen(true); }}> {t.sources.addSource} { setDropdownOpen(false); setAddExistingDialogOpen(true); }}> {t.sources.addExistingTitle} {collapseButton}
{isLoading ? (
) : !sources || sources.length === 0 ? ( ) : (
{sources.map((source) => ( onContextModeChange(source.id, mode) : undefined } /> ))} {/* Loading indicator for infinite scroll */} {isFetchingNextPage && (
)}
)} ) } ================================================ FILE: frontend/src/app/(dashboard)/notebooks/page.tsx ================================================ 'use client' import { useMemo, useState } from 'react' import { AppShell } from '@/components/layout/AppShell' import { NotebookList } from './components/NotebookList' import { Button } from '@/components/ui/button' import { Plus, RefreshCw } from 'lucide-react' import { useNotebooks } from '@/lib/hooks/use-notebooks' import { CreateNotebookDialog } from '@/components/notebooks/CreateNotebookDialog' import { Input } from '@/components/ui/input' import { useTranslation } from '@/lib/hooks/use-translation' export default function NotebooksPage() { const { t } = useTranslation() const [createDialogOpen, setCreateDialogOpen] = useState(false) const [searchTerm, setSearchTerm] = useState('') const { data: notebooks, isLoading, refetch } = useNotebooks(false) const { data: archivedNotebooks } = useNotebooks(true) const normalizedQuery = searchTerm.trim().toLowerCase() const filteredActive = useMemo(() => { if (!notebooks) { return undefined } if (!normalizedQuery) { return notebooks } return notebooks.filter((notebook) => notebook.name.toLowerCase().includes(normalizedQuery) ) }, [notebooks, normalizedQuery]) const filteredArchived = useMemo(() => { if (!archivedNotebooks) { return undefined } if (!normalizedQuery) { return archivedNotebooks } return archivedNotebooks.filter((notebook) => notebook.name.toLowerCase().includes(normalizedQuery) ) }, [archivedNotebooks, normalizedQuery]) const hasArchived = (archivedNotebooks?.length ?? 0) > 0 const isSearching = normalizedQuery.length > 0 return (

{t.notebooks.title}

setSearchTerm(event.target.value)} placeholder={t.notebooks.searchPlaceholder} autoComplete="off" aria-label={t.common.accessibility?.searchNotebooks || "Search notebooks"} className="w-full sm:w-64" />
setCreateDialogOpen(true) : undefined} actionLabel={!isSearching ? t.notebooks.newNotebook : undefined} /> {hasArchived && ( )}
) } ================================================ FILE: frontend/src/app/(dashboard)/page.tsx ================================================ import { redirect } from 'next/navigation' export default function DashboardPage() { redirect('/notebooks') } ================================================ FILE: frontend/src/app/(dashboard)/podcasts/page.tsx ================================================ 'use client' import { useMemo, useState } from 'react' import { AlertTriangle } from 'lucide-react' import { AppShell } from '@/components/layout/AppShell' import { Tabs, TabsContent, TabsList, TabsTrigger } from '@/components/ui/tabs' import { Alert, AlertDescription, AlertTitle } from '@/components/ui/alert' import { EpisodesTab } from '@/components/podcasts/EpisodesTab' import { TemplatesTab } from '@/components/podcasts/TemplatesTab' import { Mic, LayoutTemplate } from 'lucide-react' import { useTranslation } from '@/lib/hooks/use-translation' import { useEpisodeProfiles, useSpeakerProfiles } from '@/lib/hooks/use-podcasts' import { needsModelSetup } from '@/lib/types/podcasts' export default function PodcastsPage() { const { t } = useTranslation() const [activeTab, setActiveTab] = useState<'episodes' | 'templates'>('episodes') const { episodeProfiles } = useEpisodeProfiles() const { speakerProfiles } = useSpeakerProfiles(episodeProfiles) const hasUnconfiguredProfiles = useMemo(() => { return episodeProfiles.some(needsModelSetup) || speakerProfiles.some(needsModelSetup) }, [episodeProfiles, speakerProfiles]) return (

{t.podcasts.listTitle}

{t.podcasts.listDesc}

{hasUnconfiguredProfiles ? ( {t.podcasts.setupRequired} {t.podcasts.setupRequiredDesc} ) : null} setActiveTab(value as 'episodes' | 'templates')} className="space-y-6" >

{t.podcasts.chooseAView}

{t.podcasts.episodesTab} {t.podcasts.templatesTab}
) } ================================================ FILE: frontend/src/app/(dashboard)/search/page.tsx ================================================ 'use client' import { useCallback, useEffect, useMemo, useRef, useState } from 'react' import { useSearchParams } from 'next/navigation' import { useTranslation } from '@/lib/hooks/use-translation' import { AppShell } from '@/components/layout/AppShell' import { Tabs, TabsContent, TabsList, TabsTrigger } from '@/components/ui/tabs' import { Input } from '@/components/ui/input' import { Textarea } from '@/components/ui/textarea' import { Button } from '@/components/ui/button' import { RadioGroup, RadioGroupItem } from '@/components/ui/radio-group' import { Label } from '@/components/ui/label' import { Checkbox } from '@/components/ui/checkbox' import { Card, CardContent, CardHeader, CardTitle } from '@/components/ui/card' import { Badge } from '@/components/ui/badge' import { Collapsible, CollapsibleContent, CollapsibleTrigger } from '@/components/ui/collapsible' import { Search, ChevronDown, AlertCircle, Settings, Save, MessageCircleQuestion } from 'lucide-react' import { useSearch } from '@/lib/hooks/use-search' import { useAsk } from '@/lib/hooks/use-ask' import { useModelDefaults, useModels } from '@/lib/hooks/use-models' import { useModalManager } from '@/lib/hooks/use-modal-manager' import { LoadingSpinner } from '@/components/common/LoadingSpinner' import { StreamingResponse } from '@/components/search/StreamingResponse' import { AdvancedModelsDialog } from '@/components/search/AdvancedModelsDialog' import { SaveToNotebooksDialog } from '@/components/search/SaveToNotebooksDialog' export default function SearchPage() { const { t } = useTranslation() // URL params const searchParams = useSearchParams() const urlQuery = searchParams?.get('q') || '' const rawMode = searchParams?.get('mode') const urlMode = rawMode === 'search' ? 'search' : 'ask' // Tab state (controlled) const [activeTab, setActiveTab] = useState<'ask' | 'search'>( urlMode === 'search' ? 'search' : 'ask' ) // Search state const [searchQuery, setSearchQuery] = useState(urlMode === 'search' ? urlQuery : '') const [searchType, setSearchType] = useState<'text' | 'vector'>('text') const [searchSources, setSearchSources] = useState(true) const [searchNotes, setSearchNotes] = useState(true) // Ask state const [askQuestion, setAskQuestion] = useState(urlMode === 'ask' ? urlQuery : '') // Advanced models dialog const [showAdvancedModels, setShowAdvancedModels] = useState(false) const [customModels, setCustomModels] = useState<{ strategy: string answer: string finalAnswer: string } | null>(null) // Save to notebooks dialog const [showSaveDialog, setShowSaveDialog] = useState(false) // Hooks const searchMutation = useSearch() const ask = useAsk() const { data: modelDefaults, isLoading: modelsLoading } = useModelDefaults() const { data: availableModels } = useModels() const { openModal } = useModalManager() const modelNameById = useMemo(() => { if (!availableModels) { return new Map() } return new Map(availableModels.map((model) => [model.id, model.name])) }, [availableModels]) const resolveModelName = (id?: string | null) => { if (!id) return t.searchPage.notSet return modelNameById.get(id) ?? id } const hasEmbeddingModel = !!modelDefaults?.default_embedding_model // Track if we've already auto-triggered from URL params const hasAutoTriggeredRef = useRef(false) const lastUrlParamsRef = useRef({ q: '', mode: '' }) const handleSearch = useCallback(() => { if (!searchQuery.trim()) return searchMutation.mutate({ query: searchQuery, type: searchType, limit: 100, search_sources: searchSources, search_notes: searchNotes, minimum_score: 0.2 }) }, [searchQuery, searchType, searchSources, searchNotes, searchMutation]) const handleKeyPress = (e: React.KeyboardEvent) => { if (e.key === 'Enter') { handleSearch() } } const handleAsk = useCallback(() => { if (!askQuestion.trim() || !modelDefaults?.default_chat_model) return const models = customModels || { strategy: modelDefaults.default_chat_model, answer: modelDefaults.default_chat_model, finalAnswer: modelDefaults.default_chat_model } ask.sendAsk(askQuestion, models) }, [askQuestion, modelDefaults, customModels, ask]) // Auto-trigger search/ask when arriving with URL params useEffect(() => { // Skip if already triggered or no query if (hasAutoTriggeredRef.current || !urlQuery) return // Wait for models to load before triggering ask if (urlMode === 'ask' && modelsLoading) return if (urlMode === 'search') { handleSearch() hasAutoTriggeredRef.current = true } else if (urlMode === 'ask' && modelDefaults?.default_chat_model) { handleAsk() hasAutoTriggeredRef.current = true } }, [urlQuery, urlMode, modelsLoading, modelDefaults, handleSearch, handleAsk]) // Handle URL param changes while on page (e.g., from command palette again) useEffect(() => { const currentQ = searchParams?.get('q') || '' const rawCurrentMode = searchParams?.get('mode') const currentMode = rawCurrentMode === 'search' ? 'search' : 'ask' // Check if URL params have changed if (currentQ !== lastUrlParamsRef.current.q || currentMode !== lastUrlParamsRef.current.mode) { lastUrlParamsRef.current = { q: currentQ, mode: currentMode } if (currentQ) { // Update state based on mode if (currentMode === 'search') { setSearchQuery(currentQ) setActiveTab('search') // Reset trigger flag so we auto-trigger with new params hasAutoTriggeredRef.current = false } else { setAskQuestion(currentQ) setActiveTab('ask') hasAutoTriggeredRef.current = false } } } }, [searchParams]) return (

{t.searchPage.askAndSearch}

setActiveTab(v as 'ask' | 'search')} className="w-full space-y-6">

{t.searchPage.chooseAMode}

{t.searchPage.askBeta} {t.searchPage.search}
{t.searchPage.askYourKb}

{t.searchPage.askYourKbDesc}

{/* Question Input */}