[ { "path": ".github/dependabot.yml", "content": "# To get started with Dependabot version updates, you'll need to specify which\n# package ecosystems to update and where the package manifests are located.\n# Please see the documentation for all configuration options:\n# https://docs.github.com/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file\n\nversion: 2\nupdates:\n - package-ecosystem: \"pip\" # See documentation for possible values\n directory: \"/\" # Location of package manifests\n schedule:\n interval: \"weekly\"\n" }, { "path": ".github/workflows/claude-code-review.yml", "content": "name: Claude Code Review\n\non:\n pull_request:\n types: [opened, synchronize]\n # Optional: Only run on specific file changes\n # paths:\n # - \"src/**/*.ts\"\n # - \"src/**/*.tsx\"\n # - \"src/**/*.js\"\n # - \"src/**/*.jsx\"\n\njobs:\n claude-review:\n if: github.actor != 'dependabot[bot]'\n\n runs-on: ubuntu-latest\n permissions:\n contents: read\n pull-requests: read\n issues: read\n id-token: write\n \n steps:\n - name: Checkout repository\n uses: actions/checkout@v4\n with:\n fetch-depth: 1\n\n - name: Run Claude Code Review\n id: claude-review\n uses: anthropics/claude-code-action@beta\n with:\n anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }}\n\n # Optional: Specify model (defaults to Claude Sonnet 4, uncomment for Claude Opus 4)\n # model: \"claude-opus-4-20250514\"\n \n # Direct prompt for automated review (no @claude mention needed)\n direct_prompt: |\n Please review this pull request and provide feedback on:\n - Code quality and best practices\n - Potential bugs or issues\n - Performance considerations\n - Security concerns\n - Test coverage\n \n Be constructive and helpful in your feedback.\n\n # Optional: Use sticky comments to make Claude reuse the same comment on subsequent pushes to the same PR\n # use_sticky_comment: true\n \n # Optional: Customize review based on file types\n # direct_prompt: |\n # Review this PR focusing on:\n # - For TypeScript files: Type safety and proper interface usage\n # - For API endpoints: Security, input validation, and error handling\n # - For React components: Performance, accessibility, and best practices\n # - For tests: Coverage, edge cases, and test quality\n \n # Optional: Different prompts for different authors\n # direct_prompt: |\n # ${{ github.event.pull_request.author_association == 'FIRST_TIME_CONTRIBUTOR' && \n # 'Welcome! Please review this PR from a first-time contributor. Be encouraging and provide detailed explanations for any suggestions.' ||\n # 'Please provide a thorough code review focusing on our coding standards and best practices.' }}\n \n # Optional: Add specific tools for running tests or linting\n # allowed_tools: \"Bash(npm run test),Bash(npm run lint),Bash(npm run typecheck)\"\n \n # Optional: Skip review for certain conditions\n # if: |\n # !contains(github.event.pull_request.title, '[skip-review]') &&\n # !contains(github.event.pull_request.title, '[WIP]')\n\n" }, { "path": ".github/workflows/claude.yml", "content": "name: Claude Code\n\non:\n issue_comment:\n types: [created]\n pull_request_review_comment:\n types: [created]\n issues:\n types: [opened, assigned]\n pull_request_review:\n types: [submitted]\n\njobs:\n claude:\n if: |\n (github.event_name == 'issue_comment' && contains(github.event.comment.body, '@claude')) ||\n (github.event_name == 'pull_request_review_comment' && contains(github.event.comment.body, '@claude')) ||\n (github.event_name == 'pull_request_review' && contains(github.event.review.body, '@claude')) ||\n (github.event_name == 'issues' && (contains(github.event.issue.body, '@claude') || contains(github.event.issue.title, '@claude')))\n runs-on: ubuntu-latest\n permissions:\n contents: read\n pull-requests: read\n issues: read\n id-token: write\n actions: read # Required for Claude to read CI results on PRs\n steps:\n - name: Checkout repository\n uses: actions/checkout@v4\n with:\n fetch-depth: 1\n\n - name: Run Claude Code\n id: claude\n uses: anthropics/claude-code-action@beta\n with:\n anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }}\n\n # This is an optional setting that allows Claude to read CI results on PRs\n additional_permissions: |\n actions: read\n \n # Optional: Specify model (defaults to Claude Sonnet 4, uncomment for Claude Opus 4)\n # model: \"claude-opus-4-20250514\"\n \n # Optional: Customize the trigger phrase (default: @claude)\n # trigger_phrase: \"/claude\"\n \n # Optional: Trigger when specific user is assigned to an issue\n # assignee_trigger: \"claude-bot\"\n \n # Optional: Allow Claude to run specific commands\n # allowed_tools: \"Bash(npm install),Bash(npm run build),Bash(npm run test:*),Bash(npm run lint:*)\"\n \n # Optional: Add custom instructions for Claude to customize its behavior for your project\n # custom_instructions: |\n # Follow our coding standards\n # Ensure all new code has tests\n # Use TypeScript for new files\n \n # Optional: Custom environment variables for Claude\n # claude_env: |\n # NODE_ENV: test\n\n" }, { "path": ".github/workflows/docker-image.yml", "content": "name: Docker Image CI\n\non:\n push:\n branches: [ \"main\" ]\n pull_request:\n branches: [ \"main\" ]\n\npermissions:\n contents: read\n\nenv:\n # Use docker.io for Docker Hub if empty\n REGISTRY: ghcr.io\n # github.repository as /\n IMAGE_NAME: ${{ github.repository }}\n\njobs:\n build:\n runs-on: ubuntu-latest\n\n steps:\n - name: Checkout repository\n uses: actions/checkout@v4\n\n # Setup QEMU for multi-platform build support\n # https://docs.docker.com/build/ci/github-actions/multi-platform/\n - name: Set up QEMU\n uses: docker/setup-qemu-action@v3 \n\n # Set up BuildKit Docker container builder to be able to build\n # multi-platform images and export cache\n # https://github.com/docker/setup-buildx-action\n - name: Setup Docker buildx\n uses: docker/setup-buildx-action@v3\n\n # Login against a Docker registry except on PR\n # https://github.com/docker/login-action\n - name: Log into registry ${{ env.REGISTRY }}\n if: github.event_name != 'pull_request'\n uses: docker/login-action@v3\n with:\n registry: ${{ env.REGISTRY }}\n username: ${{ github.actor }}\n password: ${{ secrets.GH_TOKEN }}\n\n # Extract metadata (tags, labels) for Docker\n # https://github.com/docker/metadata-action\n - name: Extract Docker metadata\n id: meta\n uses: docker/metadata-action@v5\n with:\n images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}\n\n # Build and push Docker image with Buildx (don't push on PR)\n # https://github.com/docker/build-push-action\n - name: Build and push Docker image\n id: build-and-push\n uses: docker/build-push-action@v5\n with:\n context: .\n platforms: linux/amd64,linux/arm64\n push: ${{ github.event_name != 'pull_request' }}\n tags: |\n ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest\n labels: ${{ steps.meta.outputs.labels }}\n\n" }, { "path": "Dockerfile", "content": "FROM --platform=$BUILDPLATFORM python:3.11-slim\n\nWORKDIR /app\n\nRUN apt-get update && apt-get install -y --no-install-recommends \\\n curl \\\n ca-certificates \\\n build-essential \\\n python3-dev \\\n libssl-dev \\\n libffi-dev \\\n rustc \\\n cargo \\\n && rm -rf /var/lib/apt/lists/*\n\n# Install uv package manager (use pip for safer cross-arch install)\nRUN pip install uv\nENV PATH=\"/root/.local/bin:${PATH}\"\n\n# 2) Copy the repository content\nCOPY . /app\n\n# 3) Provide default environment variables to point to Ollama (running elsewhere)\n# Adjust the OLLAMA_URL to match your actual Ollama container or service.\nENV OLLAMA_BASE_URL=\"http://localhost:11434/\"\n\n# 4) Expose the port that LangGraph dev server uses (default: 2024)\nEXPOSE 2024\n\n# 5) Launch the assistant with the LangGraph dev server:\n# Equivalent to the quickstart: uvx --refresh --from \"langgraph-cli[inmem]\" --with-editable . --python 3.11 langgraph dev\nCMD [\"uvx\", \\\n \"--refresh\", \\\n \"--from\", \"langgraph-cli[inmem]\", \\\n \"--with-editable\", \".\", \\\n \"--python\", \"3.11\", \\\n \"langgraph\", \\\n \"dev\", \\\n \"--host\", \"0.0.0.0\"]" }, { "path": "LICENSE", "content": "MIT License\n\nCopyright (c) 2025 Lance Martin\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the \"Software\"), to deal\nin the Software without restriction, including without limitation the rights\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\ncopies of the Software, and to permit persons to whom the Software is\nfurnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\nSOFTWARE.MIT License\n\n" }, { "path": "README.md", "content": "# Local Deep Researcher\n\nLocal Deep Researcher is a fully local web research assistant that uses any LLM hosted by [Ollama](https://ollama.com/search) or [LMStudio](https://lmstudio.ai/). Give it a topic and it will generate a web search query, gather web search results, summarize the results of web search, reflect on the summary to examine knowledge gaps, generate a new search query to address the gaps, and repeat for a user-defined number of cycles. It will provide the user a final markdown summary with all sources used to generate the summary.\n\n![ollama-deep-research](https://github.com/user-attachments/assets/1c6b28f8-6b64-42ba-a491-1ab2875d50ea)\n\nShort summary video:\n\n\n## 🔥 Updates \n\n* 8/6/25: Added support for tool calling and [gpt-oss](https://openai.com/index/introducing-gpt-oss/). \n\n> ⚠️ **WARNING (8/6/25)**: The `gpt-oss` models do not support JSON mode in Ollama. Select `use_tool_calling` in the configuration to use tool calling instead of JSON mode.\n\n## 📺 Video Tutorials\n\nSee it in action or build it yourself? Check out these helpful video tutorials:\n- [Overview of Local Deep Researcher with R1](https://www.youtube.com/watch?v=sGUjmyfof4Q) - Load and test [DeepSeek R1](https://api-docs.deepseek.com/news/news250120) [distilled models](https://ollama.com/library/deepseek-r1).\n- [Building Local Deep Researcher from Scratch](https://www.youtube.com/watch?v=XGuTzHoqlj8) - Overview of how this is built.\n\n## 🚀 Quickstart\n\nClone the repository:\n```shell\ngit clone https://github.com/langchain-ai/local-deep-researcher.git\ncd local-deep-researcher\n```\n\nThen edit the `.env` file to customize the environment variables according to your needs. These environment variables control the model selection, search tools, and other configuration settings. When you run the application, these values will be automatically loaded via `python-dotenv` (because `langgraph.json` point to the \"env\" file).\n```shell\ncp .env.example .env\n```\n\n### Selecting local model with Ollama\n\n1. Download the Ollama app for Mac [here](https://ollama.com/download).\n\n2. Pull a local LLM from [Ollama](https://ollama.com/search). As an [example](https://ollama.com/library/deepseek-r1:8b):\n```shell\nollama pull deepseek-r1:8b\n```\n\n3. Optionally, update the `.env` file with the following Ollama configuration settings. \n\n* If set, these values will take precedence over the defaults set in the `Configuration` class in `configuration.py`. \n```shell\nLLM_PROVIDER=ollama\nOLLAMA_BASE_URL=\"http://localhost:11434\" # Ollama service endpoint, defaults to `http://localhost:11434` \nLOCAL_LLM=model # the model to use, defaults to `llama3.2` if not set\n```\n\n### Selecting local model with LMStudio\n\n1. Download and install LMStudio from [here](https://lmstudio.ai/).\n\n2. In LMStudio:\n - Download and load your preferred model (e.g., qwen_qwq-32b)\n - Go to the \"Local Server\" tab\n - Start the server with the OpenAI-compatible API\n - Note the server URL (default: http://localhost:1234/v1)\n\n3. Optionally, update the `.env` file with the following LMStudio configuration settings. \n\n* If set, these values will take precedence over the defaults set in the `Configuration` class in `configuration.py`. \n```shell\nLLM_PROVIDER=lmstudio\nLOCAL_LLM=qwen_qwq-32b # Use the exact model name as shown in LMStudio\nLMSTUDIO_BASE_URL=http://localhost:1234/v1\n```\n\n### Selecting search tool\n\nBy default, it will use [DuckDuckGo](https://duckduckgo.com/) for web search, which does not require an API key. But you can also use [SearXNG](https://docs.searxng.org/), [Tavily](https://tavily.com/) or [Perplexity](https://www.perplexity.ai/hub/blog/introducing-the-sonar-pro-api) by adding their API keys to the environment file. Optionally, update the `.env` file with the following search tool configuration and API keys. If set, these values will take precedence over the defaults set in the `Configuration` class in `configuration.py`. \n```shell\nSEARCH_API=xxx # the search API to use, such as `duckduckgo` (default)\nTAVILY_API_KEY=xxx # the tavily API key to use\nPERPLEXITY_API_KEY=xxx # the perplexity API key to use\nMAX_WEB_RESEARCH_LOOPS=xxx # the maximum number of research loop steps, defaults to `3`\nFETCH_FULL_PAGE=xxx # fetch the full page content (with `duckduckgo`), defaults to `false`\n```\n\n### Running with LangGraph Studio\n\n#### Mac\n\n1. (Recommended) Create a virtual environment:\n```bash\npython -m venv .venv\nsource .venv/bin/activate\n```\n\n2. Launch LangGraph server:\n\n```bash\n# Install uv package manager\ncurl -LsSf https://astral.sh/uv/install.sh | sh\nuvx --refresh --from \"langgraph-cli[inmem]\" --with-editable . --python 3.11 langgraph dev\n```\n\n#### Windows\n\n1. (Recommended) Create a virtual environment: \n\n* Install `Python 3.11` (and add to PATH during installation). \n* Restart your terminal to ensure Python is available, then create and activate a virtual environment:\n\n```powershell\npython -m venv .venv\n.venv\\Scripts\\Activate.ps1\n```\n\n2. Launch LangGraph server:\n\n```powershell\n# Install dependencies\npip install -e .\npip install -U \"langgraph-cli[inmem]\" \n\n# Start the LangGraph server\nlanggraph dev\n```\n\n### Using the LangGraph Studio UI\n\nWhen you launch LangGraph server, you should see the following output and Studio will open in your browser:\n> Ready!\n\n> API: http://127.0.0.1:2024\n\n> Docs: http://127.0.0.1:2024/docs\n\n> LangGraph Studio Web UI: https://smith.langchain.com/studio/?baseUrl=http://127.0.0.1:2024\n\nOpen `LangGraph Studio Web UI` via the URL above. In the `configuration` tab, you can directly set various assistant configurations. Keep in mind that the priority order for configuration values is:\n\n```\n1. Environment variables (highest priority)\n2. LangGraph UI configuration\n3. Default values in the Configuration class (lowest priority)\n```\n\n\"Screenshot\n\nGive the assistant a topic for research, and you can visualize its process!\n\n\"Screenshot\n\n### Model Compatibility Note\n\nWhen selecting a local LLM, set steps use structured JSON output. Some models may have difficulty with this requirement, and the assistant has fallback mechanisms to handle this. As an example, the [DeepSeek R1 (7B)](https://ollama.com/library/deepseek-llm:7b) and [DeepSeek R1 (1.5B)](https://ollama.com/library/deepseek-r1:1.5b) models have difficulty producing required JSON output, and the assistant will use a fallback mechanism to handle this.\n \n### Browser Compatibility Note\n\nWhen accessing the LangGraph Studio UI:\n- Firefox is recommended for the best experience\n- Safari users may encounter security warnings due to mixed content (HTTPS/HTTP)\n- If you encounter issues, try:\n 1. Using Firefox or another browser\n 2. Disabling ad-blocking extensions\n 3. Checking browser console for specific error messages\n\n## How it works\n\nLocal Deep Researcher is inspired by [IterDRAG](https://arxiv.org/html/2410.04343v1#:~:text=To%20tackle%20this%20issue%2C%20we,used%20to%20generate%20intermediate%20answers.). This approach will decompose a query into sub-queries, retrieve documents for each one, answer the sub-query, and then build on the answer by retrieving docs for the second sub-query. Here, we do similar:\n- Given a user-provided topic, use a local LLM (via [Ollama](https://ollama.com/search) or [LMStudio](https://lmstudio.ai/)) to generate a web search query\n- Uses a search engine / tool to find relevant sources\n- Uses LLM to summarize the findings from web search related to the user-provided research topic\n- Then, it uses the LLM to reflect on the summary, identifying knowledge gaps\n- It generates a new search query to address the knowledge gaps\n- The process repeats, with the summary being iteratively updated with new information from web search\n- Runs for a configurable number of iterations (see `configuration` tab)\n\n## Outputs\n\nThe output of the graph is a markdown file containing the research summary, with citations to the sources used. All sources gathered during research are saved to the graph state. You can visualize them in the graph state, which is visible in LangGraph Studio:\n\n![Screenshot 2024-12-05 at 4 08 59 PM](https://github.com/user-attachments/assets/e8ac1c0b-9acb-4a75-8c15-4e677e92f6cb)\n\nThe final summary is saved to the graph state as well:\n\n![Screenshot 2024-12-05 at 4 10 11 PM](https://github.com/user-attachments/assets/f6d997d5-9de5-495f-8556-7d3891f6bc96)\n\n## Deployment Options\n\nThere are [various ways](https://langchain-ai.github.io/langgraph/concepts/#deployment-options) to deploy this graph. See [Module 6](https://github.com/langchain-ai/langchain-academy/tree/main/module-6) of LangChain Academy for a detailed walkthrough of deployment options with LangGraph.\n\n## TypeScript Implementation\n\nA TypeScript port of this project (without Perplexity search) is available at:\nhttps://github.com/PacoVK/ollama-deep-researcher-ts\n\n## Running as a Docker container\n\nThe included `Dockerfile` only runs LangChain Studio with local-deep-researcher as a service, but does not include Ollama as a dependant service. You must run Ollama separately and configure the `OLLAMA_BASE_URL` environment variable. Optionally you can also specify the Ollama model to use by providing the `LOCAL_LLM` environment variable.\n\nClone the repo and build an image:\n```\n$ docker build -t local-deep-researcher .\n```\n\nRun the container:\n```\n$ docker run --rm -it -p 2024:2024 \\\n -e SEARCH_API=\"tavily\" \\ \n -e TAVILY_API_KEY=\"tvly-***YOUR_KEY_HERE***\" \\\n -e LLM_PROVIDER=ollama \\\n -e OLLAMA_BASE_URL=\"http://host.docker.internal:11434/\" \\\n -e LOCAL_LLM=\"llama3.2\" \\ \n local-deep-researcher\n```\n\nNOTE: You will see log message:\n```\n2025-02-10T13:45:04.784915Z [info ] 🎨 Opening Studio in your browser... [browser_opener] api_variant=local_dev message=🎨 Opening Studio in your browser...\nURL: https://smith.langchain.com/studio/?baseUrl=http://0.0.0.0:2024\n```\n...but the browser will not launch from the container.\n\nInstead, visit this link with the correct baseUrl IP address: [`https://smith.langchain.com/studio/thread?baseUrl=http://127.0.0.1:2024`](https://smith.langchain.com/studio/thread?baseUrl=http://127.0.0.1:2024)\n" }, { "path": "langgraph.json", "content": "{\n \"dockerfile_lines\": [],\n \"graphs\": {\n \"ollama_deep_researcher\": \"./src/ollama_deep_researcher/graph.py:graph\"\n },\n \"python_version\": \"3.11\",\n \"env\": \"./.env\",\n \"dependencies\": [\n \".\"\n ]\n }" }, { "path": "pyproject.toml", "content": "[project]\nname = \"ollama-deep-researcher\"\nversion = \"0.0.1\"\ndescription = \"Fully local web research and summarization assistant with Ollama and LangGraph.\"\nauthors = [\n { name = \"Lance Martin\" }\n]\nreadme = \"README.md\"\nlicense = { text = \"MIT\" }\nrequires-python = \">=3.9\"\ndependencies = [\n \"langgraph>=0.2.55\",\n \"langchain-community>=0.3.9\",\n \"tavily-python>=0.5.0\",\n \"langchain-ollama>=0.3.6\",\n \"duckduckgo-search>=7.3.0\",\n \"langchain-openai>=0.1.1\",\n \"openai>=1.12.0\",\n \"langchain_openai>=0.3.9\",\n \"httpx>=0.28.1\",\n \"markdownify>=0.11.0\",\n \"python-dotenv==1.2.1\",\n]\n\n[project.optional-dependencies]\ndev = [\"mypy>=1.11.1\", \"ruff>=0.6.1\"]\n\n[build-system]\nrequires = [\"setuptools>=73.0.0\", \"wheel\"]\nbuild-backend = \"setuptools.build_meta\"\n\n[tool.setuptools]\npackages = [\"ollama_deep_researcher\"]\n\n[tool.setuptools.package-dir]\n\"ollama_deep_researcher\" = \"src/ollama_deep_researcher\"\n\n[tool.setuptools.package-data]\n\"*\" = [\"py.typed\"]\n\n[tool.ruff]\nlint.select = [\n \"E\", # pycodestyle\n \"F\", # pyflakes\n \"I\", # isort\n \"D\", # pydocstyle\n \"D401\", # First line should be in imperative mood\n \"T201\",\n \"UP\",\n]\nlint.ignore = [\n \"UP006\",\n \"UP007\",\n \"UP035\",\n \"D417\",\n \"E501\",\n]\n\n[tool.ruff.lint.per-file-ignores]\n\"tests/*\" = [\"D\", \"UP\"]\n\n[tool.ruff.lint.pydocstyle]\nconvention = \"google\"\n\n[dependency-groups]\ndev = [\n \"ruff>=0.12.7\",\n]\n" }, { "path": "src/ollama_deep_researcher/__init__.py", "content": "version = \"0.0.1\"\n" }, { "path": "src/ollama_deep_researcher/configuration.py", "content": "import os\nfrom enum import Enum\nfrom pydantic import BaseModel, Field\nfrom typing import Any, Optional, Literal\n\nfrom langchain_core.runnables import RunnableConfig\n\n\nclass SearchAPI(Enum):\n PERPLEXITY = \"perplexity\"\n TAVILY = \"tavily\"\n DUCKDUCKGO = \"duckduckgo\"\n SEARXNG = \"searxng\"\n\n\nclass Configuration(BaseModel):\n \"\"\"The configurable fields for the research assistant.\"\"\"\n\n max_web_research_loops: int = Field(\n default=3,\n title=\"Research Depth\",\n description=\"Number of research iterations to perform\",\n )\n local_llm: str = Field(\n default=\"llama3.2\",\n title=\"LLM Model Name\",\n description=\"Name of the LLM model to use\",\n )\n llm_provider: Literal[\"ollama\", \"lmstudio\"] = Field(\n default=\"ollama\",\n title=\"LLM Provider\",\n description=\"Provider for the LLM (Ollama or LMStudio)\",\n )\n search_api: Literal[\"perplexity\", \"tavily\", \"duckduckgo\", \"searxng\"] = Field(\n default=\"duckduckgo\", title=\"Search API\", description=\"Web search API to use\"\n )\n fetch_full_page: bool = Field(\n default=True,\n title=\"Fetch Full Page\",\n description=\"Include the full page content in the search results\",\n )\n ollama_base_url: str = Field(\n default=\"http://localhost:11434/\",\n title=\"Ollama Base URL\",\n description=\"Base URL for Ollama API\",\n )\n lmstudio_base_url: str = Field(\n default=\"http://localhost:1234/v1\",\n title=\"LMStudio Base URL\",\n description=\"Base URL for LMStudio OpenAI-compatible API\",\n )\n strip_thinking_tokens: bool = Field(\n default=True,\n title=\"Strip Thinking Tokens\",\n description=\"Whether to strip tokens from model responses\",\n )\n use_tool_calling: bool = Field(\n default=False,\n title=\"Use Tool Calling\",\n description=\"Use tool calling instead of JSON mode for structured output\",\n )\n\n @classmethod\n def from_runnable_config(\n cls, config: Optional[RunnableConfig] = None\n ) -> \"Configuration\":\n \"\"\"Create a Configuration instance from a RunnableConfig.\"\"\"\n configurable = (\n config[\"configurable\"] if config and \"configurable\" in config else {}\n )\n\n # Get raw values from environment or config\n raw_values: dict[str, Any] = {\n name: os.environ.get(name.upper(), configurable.get(name))\n for name in cls.model_fields.keys()\n }\n\n # Filter out None values\n values = {k: v for k, v in raw_values.items() if v is not None}\n\n return cls(**values)\n" }, { "path": "src/ollama_deep_researcher/graph.py", "content": "import json\n\nfrom pydantic import BaseModel, Field\nfrom typing_extensions import Literal\n\nfrom langchain_core.messages import HumanMessage, SystemMessage\nfrom langchain_core.runnables import RunnableConfig\nfrom langchain_core.tools import tool\nfrom langchain_ollama import ChatOllama\nfrom langgraph.graph import START, END, StateGraph\n\nfrom ollama_deep_researcher.configuration import Configuration, SearchAPI\nfrom ollama_deep_researcher.utils import (\n deduplicate_and_format_sources,\n tavily_search,\n format_sources,\n perplexity_search,\n duckduckgo_search,\n searxng_search,\n strip_thinking_tokens,\n get_config_value,\n)\nfrom ollama_deep_researcher.state import (\n SummaryState,\n SummaryStateInput,\n SummaryStateOutput,\n)\nfrom ollama_deep_researcher.prompts import (\n query_writer_instructions,\n summarizer_instructions,\n reflection_instructions,\n get_current_date,\n json_mode_query_instructions,\n tool_calling_query_instructions,\n json_mode_reflection_instructions,\n tool_calling_reflection_instructions,\n)\nfrom ollama_deep_researcher.lmstudio import ChatLMStudio\n\n# Constants\nMAX_TOKENS_PER_SOURCE = 1000\nCHARS_PER_TOKEN = 4\n\ndef generate_search_query_with_structured_output(\n configurable: Configuration,\n messages: list,\n tool_class,\n fallback_query: str,\n tool_query_field: str,\n json_query_field: str,\n):\n \"\"\"Helper function to generate search queries using either tool calling or JSON mode.\n \n Args:\n configurable: Configuration object\n messages: List of messages to send to LLM\n tool_class: Tool class for tool calling mode\n fallback_query: Fallback search query if extraction fails\n tool_query_field: Field name in tool args containing the query\n json_query_field: Field name in JSON response containing the query\n \n Returns:\n Dictionary with \"search_query\" key\n \"\"\"\n if configurable.use_tool_calling:\n llm = get_llm(configurable).bind_tools([tool_class])\n result = llm.invoke(messages)\n\n if not result.tool_calls:\n return {\"search_query\": fallback_query}\n \n try:\n tool_data = result.tool_calls[0][\"args\"]\n search_query = tool_data.get(tool_query_field)\n return {\"search_query\": search_query}\n except (IndexError, KeyError):\n return {\"search_query\": fallback_query}\n \n else:\n # Use JSON mode\n llm = get_llm(configurable)\n result = llm.invoke(messages)\n print(f\"result: {result}\")\n content = result.content\n\n try:\n parsed_json = json.loads(content)\n search_query = parsed_json.get(json_query_field)\n if not search_query:\n return {\"search_query\": fallback_query}\n return {\"search_query\": search_query}\n except (json.JSONDecodeError, KeyError):\n if configurable.strip_thinking_tokens:\n content = strip_thinking_tokens(content)\n return {\"search_query\": fallback_query}\n\ndef get_llm(configurable: Configuration):\n \"\"\"Helper function to initialize LLM based on configuration.\n\n Uses JSON mode if use_tool_calling is False, otherwise regular mode for tool calling.\n\n Args:\n configurable: Configuration object containing LLM settings\n\n Returns:\n Configured LLM instance\n \"\"\"\n if configurable.llm_provider == \"lmstudio\":\n if configurable.use_tool_calling:\n return ChatLMStudio(\n base_url=configurable.lmstudio_base_url,\n model=configurable.local_llm,\n temperature=0,\n )\n else:\n return ChatLMStudio(\n base_url=configurable.lmstudio_base_url,\n model=configurable.local_llm,\n temperature=0,\n format=\"json\",\n )\n else: # Default to Ollama\n if configurable.use_tool_calling:\n return ChatOllama(\n base_url=configurable.ollama_base_url,\n model=configurable.local_llm,\n temperature=0,\n )\n else:\n return ChatOllama(\n base_url=configurable.ollama_base_url,\n model=configurable.local_llm,\n temperature=0,\n format=\"json\",\n )\n\n# Nodes\ndef generate_query(state: SummaryState, config: RunnableConfig):\n \"\"\"LangGraph node that generates a search query based on the research topic.\n\n Uses an LLM to create an optimized search query for web research based on\n the user's research topic. Supports both LMStudio and Ollama as LLM providers.\n\n Args:\n state: Current graph state containing the research topic\n config: Configuration for the runnable, including LLM provider settings\n\n Returns:\n Dictionary with state update, including search_query key containing the generated query\n \"\"\"\n\n # Format the prompt\n current_date = get_current_date()\n formatted_prompt = query_writer_instructions.format(\n current_date=current_date, research_topic=state.research_topic\n )\n\n # Generate a query\n configurable = Configuration.from_runnable_config(config)\n\n @tool\n class Query(BaseModel):\n \"\"\"\n This tool is used to generate a query for web search.\n \"\"\"\n\n query: str = Field(description=\"The actual search query string\")\n rationale: str = Field(\n description=\"Brief explanation of why this query is relevant\"\n )\n\n messages = [\n SystemMessage(\n content=formatted_prompt + (\n tool_calling_query_instructions if configurable.use_tool_calling \n else json_mode_query_instructions\n )\n ),\n HumanMessage(content=\"Generate a query for web search:\"),\n ]\n\n return generate_search_query_with_structured_output(\n configurable=configurable,\n messages=messages,\n tool_class=Query,\n fallback_query=f\"Tell me more about {state.research_topic}\",\n tool_query_field=\"query\",\n json_query_field=\"query\",\n )\n\n\ndef web_research(state: SummaryState, config: RunnableConfig):\n \"\"\"LangGraph node that performs web research using the generated search query.\n\n Executes a web search using the configured search API (tavily, perplexity,\n duckduckgo, or searxng) and formats the results for further processing.\n\n Args:\n state: Current graph state containing the search query and research loop count\n config: Configuration for the runnable, including search API settings\n\n Returns:\n Dictionary with state update, including sources_gathered, research_loop_count, and web_research_results\n \"\"\"\n\n # Configure\n configurable = Configuration.from_runnable_config(config)\n\n # Get the search API\n search_api = get_config_value(configurable.search_api)\n\n # Search the web\n if search_api == \"tavily\":\n search_results = tavily_search(\n state.search_query,\n fetch_full_page=configurable.fetch_full_page,\n max_results=1,\n )\n search_str = deduplicate_and_format_sources(\n search_results,\n max_tokens_per_source=MAX_TOKENS_PER_SOURCE,\n fetch_full_page=configurable.fetch_full_page,\n )\n elif search_api == \"perplexity\":\n search_results = perplexity_search(\n state.search_query, state.research_loop_count\n )\n search_str = deduplicate_and_format_sources(\n search_results,\n max_tokens_per_source=MAX_TOKENS_PER_SOURCE,\n fetch_full_page=configurable.fetch_full_page,\n )\n elif search_api == \"duckduckgo\":\n search_results = duckduckgo_search(\n state.search_query,\n max_results=3,\n fetch_full_page=configurable.fetch_full_page,\n )\n search_str = deduplicate_and_format_sources(\n search_results,\n max_tokens_per_source=MAX_TOKENS_PER_SOURCE,\n fetch_full_page=configurable.fetch_full_page,\n )\n elif search_api == \"searxng\":\n search_results = searxng_search(\n state.search_query,\n max_results=3,\n fetch_full_page=configurable.fetch_full_page,\n )\n search_str = deduplicate_and_format_sources(\n search_results,\n max_tokens_per_source=MAX_TOKENS_PER_SOURCE,\n fetch_full_page=configurable.fetch_full_page,\n )\n else:\n raise ValueError(f\"Unsupported search API: {configurable.search_api}\")\n\n return {\n \"sources_gathered\": [format_sources(search_results)],\n \"research_loop_count\": state.research_loop_count + 1,\n \"web_research_results\": [search_str],\n }\n\n\ndef summarize_sources(state: SummaryState, config: RunnableConfig):\n \"\"\"LangGraph node that summarizes web research results.\n\n Uses an LLM to create or update a running summary based on the newest web research\n results, integrating them with any existing summary.\n\n Args:\n state: Current graph state containing research topic, running summary,\n and web research results\n config: Configuration for the runnable, including LLM provider settings\n\n Returns:\n Dictionary with state update, including running_summary key containing the updated summary\n \"\"\"\n\n # Existing summary\n existing_summary = state.running_summary\n\n # Most recent web research\n most_recent_web_research = state.web_research_results[-1]\n\n # Build the human message\n if existing_summary:\n human_message_content = (\n f\" \\n {existing_summary} \\n \\n\\n\"\n f\" \\n {most_recent_web_research} \\n \"\n f\"Update the Existing Summary with the New Context on this topic: \\n \\n {state.research_topic} \\n \\n\\n\"\n )\n else:\n human_message_content = (\n f\" \\n {most_recent_web_research} \\n \"\n f\"Create a Summary using the Context on this topic: \\n \\n {state.research_topic} \\n \\n\\n\"\n )\n\n # Run the LLM\n configurable = Configuration.from_runnable_config(config)\n\n # For summarization, we don't need structured output, so always use regular mode\n if configurable.llm_provider == \"lmstudio\":\n llm = ChatLMStudio(\n base_url=configurable.lmstudio_base_url,\n model=configurable.local_llm,\n temperature=0,\n )\n else: # Default to Ollama\n llm = ChatOllama(\n base_url=configurable.ollama_base_url,\n model=configurable.local_llm,\n temperature=0,\n )\n\n result = llm.invoke(\n [\n SystemMessage(content=summarizer_instructions),\n HumanMessage(content=human_message_content),\n ]\n )\n\n # Strip thinking tokens if configured\n running_summary = result.content\n if configurable.strip_thinking_tokens:\n running_summary = strip_thinking_tokens(running_summary)\n\n return {\"running_summary\": running_summary}\n\n\ndef reflect_on_summary(state: SummaryState, config: RunnableConfig):\n \"\"\"LangGraph node that identifies knowledge gaps and generates follow-up queries.\n\n Analyzes the current summary to identify areas for further research and generates\n a new search query to address those gaps. Uses structured output to extract\n the follow-up query in JSON format.\n\n Args:\n state: Current graph state containing the running summary and research topic\n config: Configuration for the runnable, including LLM provider settings\n\n Returns:\n Dictionary with state update, including search_query key containing the generated follow-up query\n \"\"\"\n\n # Generate a query\n configurable = Configuration.from_runnable_config(config)\n formatted_prompt = reflection_instructions.format(\n research_topic=state.research_topic\n )\n\n @tool\n class FollowUpQuery(BaseModel):\n \"\"\"\n This tool is used to generate a follow-up query to address a knowledge gap.\n \"\"\"\n\n follow_up_query: str = Field(\n description=\"Write a specific question to address this gap\"\n )\n knowledge_gap: str = Field(\n description=\"Describe what information is missing or needs clarification\"\n )\n\n messages = [\n SystemMessage(\n content=formatted_prompt + (\n tool_calling_reflection_instructions if configurable.use_tool_calling \n else json_mode_reflection_instructions\n )\n ),\n HumanMessage(\n content=f\"Reflect on our existing knowledge: \\n === \\n {state.running_summary}, \\n === \\n And now identify a knowledge gap and generate a follow-up web search query:\"\n ),\n ]\n\n return generate_search_query_with_structured_output(\n configurable=configurable,\n messages=messages,\n tool_class=FollowUpQuery,\n fallback_query=f\"Tell me more about {state.research_topic}\",\n tool_query_field=\"follow_up_query\",\n json_query_field=\"follow_up_query\",\n )\n\n\ndef finalize_summary(state: SummaryState):\n \"\"\"LangGraph node that finalizes the research summary.\n\n Prepares the final output by deduplicating and formatting sources, then\n combining them with the running summary to create a well-structured\n research report with proper citations.\n\n Args:\n state: Current graph state containing the running summary and sources gathered\n\n Returns:\n Dictionary with state update, including running_summary key containing the formatted final summary with sources\n \"\"\"\n\n # Deduplicate sources before joining\n seen_sources = set()\n unique_sources = []\n\n for source in state.sources_gathered:\n # Split the source into lines and process each individually\n for line in source.split(\"\\n\"):\n # Only process non-empty lines\n if line.strip() and line not in seen_sources:\n seen_sources.add(line)\n unique_sources.append(line)\n\n # Join the deduplicated sources\n all_sources = \"\\n\".join(unique_sources)\n state.running_summary = (\n f\"## Summary\\n{state.running_summary}\\n\\n ### Sources:\\n{all_sources}\"\n )\n return {\"running_summary\": state.running_summary}\n\n\ndef route_research(\n state: SummaryState, config: RunnableConfig\n) -> Literal[\"finalize_summary\", \"web_research\"]:\n \"\"\"LangGraph routing function that determines the next step in the research flow.\n\n Controls the research loop by deciding whether to continue gathering information\n or to finalize the summary based on the configured maximum number of research loops.\n\n Args:\n state: Current graph state containing the research loop count\n config: Configuration for the runnable, including max_web_research_loops setting\n\n Returns:\n String literal indicating the next node to visit (\"web_research\" or \"finalize_summary\")\n \"\"\"\n\n configurable = Configuration.from_runnable_config(config)\n if state.research_loop_count <= configurable.max_web_research_loops:\n return \"web_research\"\n else:\n return \"finalize_summary\"\n\n\n# Add nodes and edges\nbuilder = StateGraph(\n SummaryState,\n input=SummaryStateInput,\n output=SummaryStateOutput,\n config_schema=Configuration,\n)\nbuilder.add_node(\"generate_query\", generate_query)\nbuilder.add_node(\"web_research\", web_research)\nbuilder.add_node(\"summarize_sources\", summarize_sources)\nbuilder.add_node(\"reflect_on_summary\", reflect_on_summary)\nbuilder.add_node(\"finalize_summary\", finalize_summary)\n\n# Add edges\nbuilder.add_edge(START, \"generate_query\")\nbuilder.add_edge(\"generate_query\", \"web_research\")\nbuilder.add_edge(\"web_research\", \"summarize_sources\")\nbuilder.add_edge(\"summarize_sources\", \"reflect_on_summary\")\nbuilder.add_conditional_edges(\"reflect_on_summary\", route_research)\nbuilder.add_edge(\"finalize_summary\", END)\n\ngraph = builder.compile()\n" }, { "path": "src/ollama_deep_researcher/lmstudio.py", "content": "\"\"\"LMStudio integration for the research assistant.\"\"\"\n\nimport json\nimport logging\nfrom typing import Any, List, Optional\n\nfrom langchain_core.callbacks.manager import CallbackManagerForLLMRun\nfrom langchain_core.messages import (\n BaseMessage,\n)\nfrom langchain_core.outputs import ChatResult\nfrom langchain_openai import ChatOpenAI\nfrom pydantic import Field\n\n# Set up logging\nlogger = logging.getLogger(__name__)\n\n\nclass ChatLMStudio(ChatOpenAI):\n \"\"\"Chat model that uses LMStudio's OpenAI-compatible API.\"\"\"\n\n format: Optional[str] = Field(\n default=None, description=\"Format for the response (e.g., 'json')\"\n )\n\n def __init__(\n self,\n base_url: str = \"http://localhost:1234/v1\",\n model: str = \"qwen_qwq-32b\",\n temperature: float = 0.7,\n format: Optional[str] = None,\n api_key: str = \"not-needed-for-local-models\",\n **kwargs: Any,\n ):\n \"\"\"Initialize the ChatLMStudio.\n\n Args:\n base_url: Base URL for LMStudio's OpenAI-compatible API\n model: Model name to use\n temperature: Temperature for sampling\n format: Format for the response (e.g., \"json\")\n api_key: API key (not actually used, but required by OpenAI client)\n **kwargs: Additional arguments to pass to the OpenAI client\n \"\"\"\n # Initialize the base class\n super().__init__(\n base_url=base_url,\n model=model,\n temperature=temperature,\n api_key=api_key,\n **kwargs,\n )\n self.format = format\n\n def _generate(\n self,\n messages: List[BaseMessage],\n stop: Optional[List[str]] = None,\n run_manager: Optional[CallbackManagerForLLMRun] = None,\n **kwargs: Any,\n ) -> ChatResult:\n \"\"\"Generate a chat response using LMStudio's OpenAI-compatible API.\"\"\"\n\n if self.format == \"json\":\n # Set response_format for JSON mode\n kwargs[\"response_format\"] = {\"type\": \"json_object\"}\n logger.info(f\"Using response_format={kwargs['response_format']}\")\n\n # Call the parent class's _generate method\n result = super()._generate(messages, stop, run_manager, **kwargs)\n\n # If JSON format is requested, try to clean up the response\n if self.format == \"json\" and result.generations:\n try:\n # Get the raw text\n raw_text = result.generations[0][0].text\n logger.info(f\"Raw model response: {raw_text}\")\n\n # Try to find JSON in the response\n json_start = raw_text.find(\"{\")\n json_end = raw_text.rfind(\"}\") + 1\n\n if json_start >= 0 and json_end > json_start:\n # Extract just the JSON part\n json_text = raw_text[json_start:json_end]\n # Validate it's proper JSON\n json.loads(json_text)\n logger.info(f\"Cleaned JSON: {json_text}\")\n # Update the generation with the cleaned JSON\n result.generations[0][0].text = json_text\n else:\n logger.warning(\"Could not find JSON in response\")\n except Exception as e:\n logger.error(f\"Error processing JSON response: {str(e)}\")\n # If any error occurs during cleanup, just use the original response\n pass\n\n return result\n" }, { "path": "src/ollama_deep_researcher/prompts.py", "content": "from datetime import datetime\n\n\n# Get current date in a readable format\ndef get_current_date():\n return datetime.now().strftime(\"%B %d, %Y\")\n\n\nquery_writer_instructions = \"\"\"Your goal is to generate a targeted web search query.\n\n\nCurrent date: {current_date}\nPlease ensure your queries account for the most current information available as of this date.\n\n\n\n{research_topic}\n\n\n\nExample output:\n{{\n \"query\": \"machine learning transformer architecture explained\",\n \"rationale\": \"Understanding the fundamental structure of transformer models\"\n}}\n\"\"\"\n\njson_mode_query_instructions = \"\"\"\nFormat your response as a JSON object with ALL three of these exact keys:\n- \"query\": The actual search query string\n- \"rationale\": Brief explanation of why this query is relevant\n\n\nProvide your response in JSON format:\"\"\"\n\ntool_calling_query_instructions = \"\"\"\nCall the Query tool to format your response with the following keys:\n - \"query\": The actual search query string\n - \"rationale\": Brief explanation of why this query is relevant\n\n\nCall the Query Tool to generate a query for this request:\"\"\"\n\nsummarizer_instructions = \"\"\"\n\nGenerate a high-quality summary of the provided context.\n\n\n\nWhen creating a NEW summary:\n1. Highlight the most relevant information related to the user topic from the search results\n2. Ensure a coherent flow of information\n\nWhen EXTENDING an existing summary: \n1. Read the existing summary and new search results carefully. \n2. Compare the new information with the existing summary. \n3. For each piece of new information: \n a. If it's related to existing points, integrate it into the relevant paragraph. \n b. If it's entirely new but relevant, add a new paragraph with a smooth transition. \n c. If it's not relevant to the user topic, skip it. \n4. Ensure all additions are relevant to the user's topic. \n5. Verify that your final output differs from the input summary. \n< /REQUIREMENTS >\n\n< FORMATTING >\n- Start directly with the updated summary, without preamble or titles. Do not use XML tags in the output. \n< /FORMATTING >\n\n\nThink carefully about the provided Context first. Then generate a summary of the context to address the User Input.\n\n\"\"\"\n\nreflection_instructions = \"\"\"You are an expert research assistant analyzing a summary about {research_topic}.\n\n\n1. Identify knowledge gaps or areas that need deeper exploration\n2. Generate a follow-up question that would help expand your understanding\n3. Focus on technical details, implementation specifics, or emerging trends that weren't fully covered\n\n\n\nEnsure the follow-up question is self-contained and includes necessary context for web search.\n\"\"\"\n\njson_mode_reflection_instructions = \"\"\"\nFormat your response as a JSON object with these exact keys:\n- knowledge_gap: Describe what information is missing or needs clarification\n- follow_up_query: Write a specific question to address this gap\n\n\n\nReflect carefully on the Summary to identify knowledge gaps and produce a follow-up query. Then, produce your output following this JSON format:\n{{\n \"knowledge_gap\": \"The summary lacks information about performance metrics and benchmarks\",\n \"follow_up_query\": \"What are typical performance benchmarks and metrics used to evaluate [specific technology]?\"\n}}\n\n\nProvide your analysis in JSON format:\"\"\"\n\ntool_calling_reflection_instructions = \"\"\"\nCall the FollowUpQuery tool to format your response with the following keys:\n- follow_up_query: Write a specific question to address this gap\n- knowledge_gap: Describe what information is missing or needs clarification\n\n\n\nReflect carefully on the Summary to identify knowledge gaps and produce a follow-up query.\n\n\nCall the FollowUpQuery Tool to generate a reflection for this request:\"\"\"" }, { "path": "src/ollama_deep_researcher/state.py", "content": "import operator\nfrom dataclasses import dataclass, field\nfrom typing_extensions import Annotated\n\n\n@dataclass(kw_only=True)\nclass SummaryState:\n research_topic: str = field(default=None) # Report topic\n search_query: str = field(default=None) # Search query\n web_research_results: Annotated[list, operator.add] = field(default_factory=list)\n sources_gathered: Annotated[list, operator.add] = field(default_factory=list)\n research_loop_count: int = field(default=0) # Research loop count\n running_summary: str = field(default=None) # Final report\n\n\n@dataclass(kw_only=True)\nclass SummaryStateInput:\n research_topic: str = field(default=None) # Report topic\n\n\n@dataclass(kw_only=True)\nclass SummaryStateOutput:\n running_summary: str = field(default=None) # Final report\n" }, { "path": "src/ollama_deep_researcher/utils.py", "content": "import os\nimport httpx\nimport requests\nfrom typing import Dict, Any, List, Union, Optional\n\nfrom markdownify import markdownify\nfrom langsmith import traceable\nfrom tavily import TavilyClient\nfrom duckduckgo_search import DDGS\n\nfrom langchain_community.utilities import SearxSearchWrapper\n\n# Constants\nCHARS_PER_TOKEN = 4\n\n\ndef get_config_value(value: Any) -> str:\n \"\"\"\n Convert configuration values to string format, handling both string and enum types.\n\n Args:\n value (Any): The configuration value to process. Can be a string or an Enum.\n\n Returns:\n str: The string representation of the value.\n\n Examples:\n >>> get_config_value(\"tavily\")\n 'tavily'\n >>> get_config_value(SearchAPI.TAVILY)\n 'tavily'\n \"\"\"\n return value if isinstance(value, str) else value.value\n\n\ndef strip_thinking_tokens(text: str) -> str:\n \"\"\"\n Remove and tags and their content from the text.\n\n Iteratively removes all occurrences of content enclosed in thinking tokens.\n\n Args:\n text (str): The text to process\n\n Returns:\n str: The text with thinking tokens and their content removed\n \"\"\"\n while \"\" in text and \"\" in text:\n start = text.find(\"\")\n end = text.find(\"\") + len(\"\")\n text = text[:start] + text[end:]\n return text\n\n\ndef deduplicate_and_format_sources(\n search_response: Union[Dict[str, Any], List[Dict[str, Any]]],\n max_tokens_per_source: int,\n fetch_full_page: bool = False,\n) -> str:\n \"\"\"\n Format and deduplicate search responses from various search APIs.\n\n Takes either a single search response or list of responses from search APIs,\n deduplicates them by URL, and formats them into a structured string.\n\n Args:\n search_response (Union[Dict[str, Any], List[Dict[str, Any]]]): Either:\n - A dict with a 'results' key containing a list of search results\n - A list of dicts, each containing search results\n max_tokens_per_source (int): Maximum number of tokens to include for each source's content\n fetch_full_page (bool, optional): Whether to include the full page content. Defaults to False.\n\n Returns:\n str: Formatted string with deduplicated sources\n\n Raises:\n ValueError: If input is neither a dict with 'results' key nor a list of search results\n \"\"\"\n # Convert input to list of results\n if isinstance(search_response, dict):\n sources_list = search_response[\"results\"]\n elif isinstance(search_response, list):\n sources_list = []\n for response in search_response:\n if isinstance(response, dict) and \"results\" in response:\n sources_list.extend(response[\"results\"])\n else:\n sources_list.extend(response)\n else:\n raise ValueError(\n \"Input must be either a dict with 'results' or a list of search results\"\n )\n\n # Deduplicate by URL\n unique_sources = {}\n for source in sources_list:\n if source[\"url\"] not in unique_sources:\n unique_sources[source[\"url\"]] = source\n\n # Format output\n formatted_text = \"Sources:\\n\\n\"\n for i, source in enumerate(unique_sources.values(), 1):\n formatted_text += f\"Source: {source['title']}\\n===\\n\"\n formatted_text += f\"URL: {source['url']}\\n===\\n\"\n formatted_text += (\n f\"Most relevant content from source: {source['content']}\\n===\\n\"\n )\n if fetch_full_page:\n # Using rough estimate of characters per token\n char_limit = max_tokens_per_source * CHARS_PER_TOKEN\n # Handle None raw_content\n raw_content = source.get(\"raw_content\", \"\")\n if raw_content is None:\n raw_content = \"\"\n print(f\"Warning: No raw_content found for source {source['url']}\")\n if len(raw_content) > char_limit:\n raw_content = raw_content[:char_limit] + \"... [truncated]\"\n formatted_text += f\"Full source content limited to {max_tokens_per_source} tokens: {raw_content}\\n\\n\"\n\n return formatted_text.strip()\n\n\ndef format_sources(search_results: Dict[str, Any]) -> str:\n \"\"\"\n Format search results into a bullet-point list of sources with URLs.\n\n Creates a simple bulleted list of search results with title and URL for each source.\n\n Args:\n search_results (Dict[str, Any]): Search response containing a 'results' key with\n a list of search result objects\n\n Returns:\n str: Formatted string with sources as bullet points in the format \"* title : url\"\n \"\"\"\n return \"\\n\".join(\n f\"* {source['title']} : {source['url']}\" for source in search_results[\"results\"]\n )\n\n\ndef fetch_raw_content(url: str) -> Optional[str]:\n \"\"\"\n Fetch HTML content from a URL and convert it to markdown format.\n\n Uses a 10-second timeout to avoid hanging on slow sites or large pages.\n\n Args:\n url (str): The URL to fetch content from\n\n Returns:\n Optional[str]: The fetched content converted to markdown if successful,\n None if any error occurs during fetching or conversion\n \"\"\"\n try:\n # Create a client with reasonable timeout\n with httpx.Client(timeout=10.0) as client:\n response = client.get(url)\n response.raise_for_status()\n return markdownify(response.text)\n except Exception as e:\n print(f\"Warning: Failed to fetch full page content for {url}: {str(e)}\")\n return None\n\n\n@traceable\ndef duckduckgo_search(\n query: str, max_results: int = 3, fetch_full_page: bool = False\n) -> Dict[str, List[Dict[str, Any]]]:\n \"\"\"\n Search the web using DuckDuckGo and return formatted results.\n\n Uses the DDGS library to perform web searches through DuckDuckGo.\n\n Args:\n query (str): The search query to execute\n max_results (int, optional): Maximum number of results to return. Defaults to 3.\n fetch_full_page (bool, optional): Whether to fetch full page content from result URLs.\n Defaults to False.\n Returns:\n Dict[str, List[Dict[str, Any]]]: Search response containing:\n - results (list): List of search result dictionaries, each containing:\n - title (str): Title of the search result\n - url (str): URL of the search result\n - content (str): Snippet/summary of the content\n - raw_content (str or None): Full page content if fetch_full_page is True,\n otherwise same as content\n \"\"\"\n try:\n with DDGS() as ddgs:\n results = []\n search_results = list(ddgs.text(query, max_results=max_results))\n\n for r in search_results:\n url = r.get(\"href\")\n title = r.get(\"title\")\n content = r.get(\"body\")\n\n if not all([url, title, content]):\n print(f\"Warning: Incomplete result from DuckDuckGo: {r}\")\n continue\n\n raw_content = content\n if fetch_full_page:\n raw_content = fetch_raw_content(url)\n\n # Add result to list\n result = {\n \"title\": title,\n \"url\": url,\n \"content\": content,\n \"raw_content\": raw_content,\n }\n results.append(result)\n\n return {\"results\": results}\n except Exception as e:\n print(f\"Error in DuckDuckGo search: {str(e)}\")\n print(f\"Full error details: {type(e).__name__}\")\n return {\"results\": []}\n\n\n@traceable\ndef searxng_search(\n query: str, max_results: int = 3, fetch_full_page: bool = False\n) -> Dict[str, List[Dict[str, Any]]]:\n \"\"\"\n Search the web using SearXNG and return formatted results.\n\n Uses the SearxSearchWrapper to perform searches through a SearXNG instance.\n The SearXNG host URL is read from the SEARXNG_URL environment variable\n or defaults to http://localhost:8888.\n\n Args:\n query (str): The search query to execute\n max_results (int, optional): Maximum number of results to return. Defaults to 3.\n fetch_full_page (bool, optional): Whether to fetch full page content from result URLs.\n Defaults to False.\n\n Returns:\n Dict[str, List[Dict[str, Any]]]: Search response containing:\n - results (list): List of search result dictionaries, each containing:\n - title (str): Title of the search result\n - url (str): URL of the search result\n - content (str): Snippet/summary of the content\n - raw_content (str or None): Full page content if fetch_full_page is True,\n otherwise same as content\n \"\"\"\n host = os.environ.get(\"SEARXNG_URL\", \"http://localhost:8888\")\n s = SearxSearchWrapper(searx_host=host)\n\n results = []\n search_results = s.results(query, num_results=max_results)\n for r in search_results:\n url = r.get(\"link\")\n title = r.get(\"title\")\n content = r.get(\"snippet\")\n\n if not all([url, title, content]):\n print(f\"Warning: Incomplete result from SearXNG: {r}\")\n continue\n\n raw_content = content\n if fetch_full_page:\n raw_content = fetch_raw_content(url)\n\n # Add result to list\n result = {\n \"title\": title,\n \"url\": url,\n \"content\": content,\n \"raw_content\": raw_content,\n }\n results.append(result)\n return {\"results\": results}\n\n\n@traceable\ndef tavily_search(\n query: str, fetch_full_page: bool = True, max_results: int = 3\n) -> Dict[str, List[Dict[str, Any]]]:\n \"\"\"\n Search the web using the Tavily API and return formatted results.\n\n Uses the TavilyClient to perform searches. Tavily API key must be configured\n in the environment.\n\n Args:\n query (str): The search query to execute\n fetch_full_page (bool, optional): Whether to include raw content from sources.\n Defaults to True.\n max_results (int, optional): Maximum number of results to return. Defaults to 3.\n\n Returns:\n Dict[str, List[Dict[str, Any]]]: Search response containing:\n - results (list): List of search result dictionaries, each containing:\n - title (str): Title of the search result\n - url (str): URL of the search result\n - content (str): Snippet/summary of the content\n - raw_content (str or None): Full content of the page if available and\n fetch_full_page is True\n \"\"\"\n\n tavily_client = TavilyClient()\n return tavily_client.search(\n query, max_results=max_results, include_raw_content=fetch_full_page\n )\n\n\n@traceable\ndef perplexity_search(\n query: str, perplexity_search_loop_count: int = 0\n) -> Dict[str, Any]:\n \"\"\"\n Search the web using the Perplexity API and return formatted results.\n\n Uses the Perplexity API to perform searches with the 'sonar-pro' model.\n Requires a PERPLEXITY_API_KEY environment variable to be set.\n\n Args:\n query (str): The search query to execute\n perplexity_search_loop_count (int, optional): The loop step for perplexity search\n (used for source labeling). Defaults to 0.\n\n Returns:\n Dict[str, Any]: Search response containing:\n - results (list): List of search result dictionaries, each containing:\n - title (str): Title of the search result (includes search counter)\n - url (str): URL of the citation source\n - content (str): Content of the response or reference to main content\n - raw_content (str or None): Full content for the first source, None for additional\n citation sources\n\n Raises:\n requests.exceptions.HTTPError: If the API request fails\n \"\"\"\n\n headers = {\n \"accept\": \"application/json\",\n \"content-type\": \"application/json\",\n \"Authorization\": f\"Bearer {os.getenv('PERPLEXITY_API_KEY')}\",\n }\n\n payload = {\n \"model\": \"sonar-pro\",\n \"messages\": [\n {\n \"role\": \"system\",\n \"content\": \"Search the web and provide factual information with sources.\",\n },\n {\"role\": \"user\", \"content\": query},\n ],\n }\n\n response = requests.post(\n \"https://api.perplexity.ai/chat/completions\", headers=headers, json=payload\n )\n response.raise_for_status() # Raise exception for bad status codes\n\n # Parse the response\n data = response.json()\n content = data[\"choices\"][0][\"message\"][\"content\"]\n\n # Perplexity returns a list of citations for a single search result\n citations = data.get(\"citations\", [\"https://perplexity.ai\"])\n\n # Return first citation with full content, others just as references\n results = [\n {\n \"title\": f\"Perplexity Search {perplexity_search_loop_count + 1}, Source 1\",\n \"url\": citations[0],\n \"content\": content,\n \"raw_content\": content,\n }\n ]\n\n # Add additional citations without duplicating content\n for i, citation in enumerate(citations[1:], start=2):\n results.append(\n {\n \"title\": f\"Perplexity Search {perplexity_search_loop_count + 1}, Source {i}\",\n \"url\": citation,\n \"content\": \"See above for full content\",\n \"raw_content\": None,\n }\n )\n\n return {\"results\": results}\n" } ]