[ { "path": ".azdo/pipelines/azure-dev.yml", "content": "# Run when commits are pushed to mainline branch (main or master)\n# Set this to the mainline branch you are using\ntrigger:\n - main\n - master\n\n# Azure Pipelines workflow to deploy to Azure using azd\n# To configure required secrets and service connection for connecting to Azure, simply run `azd pipeline config --provider azdo`\n# Task \"Install azd\" needs to install setup-azd extension for azdo - https://marketplace.visualstudio.com/items?itemName=ms-azuretools.azd\n# See below for alternative task to install azd if you can't install above task in your organization\n\npool:\n vmImage: ubuntu-latest\n\nsteps:\n - task: setup-azd@0\n displayName: Install azd\n\n # If you can't install above task in your organization, you can comment it and uncomment below task to install azd\n # - task: Bash@3\n # displayName: Install azd\n # inputs:\n # targetType: 'inline'\n # script: |\n # curl -fsSL https://aka.ms/install-azd.sh | bash\n\n # azd delegate auth to az to use service connection with AzureCLI@2\n - pwsh: |\n azd config set auth.useAzCliAuth \"true\"\n displayName: Configure AZD to Use AZ CLI Authentication.\n\n - task: AzureCLI@2\n displayName: Provision Infrastructure\n inputs:\n # azconnection is the service connection created by azd. You can change it to any service connection you have in your organization.\n azureSubscription: azconnection\n scriptType: bash\n scriptLocation: inlineScript\n inlineScript: |\n azd provision --no-prompt\n env:\n AZURE_SUBSCRIPTION_ID: $(AZURE_SUBSCRIPTION_ID)\n AZURE_ENV_NAME: $(AZURE_ENV_NAME)\n AZURE_LOCATION: $(AZURE_LOCATION)\n AZD_INITIAL_ENVIRONMENT_CONFIG: $(AZD_INITIAL_ENVIRONMENT_CONFIG)\n AZURE_OPENAI_SERVICE: $(AZURE_OPENAI_SERVICE)\n AZURE_OPENAI_LOCATION: $(AZURE_OPENAI_LOCATION)\n AZURE_OPENAI_RESOURCE_GROUP: $(AZURE_OPENAI_RESOURCE_GROUP)\n AZURE_DOCUMENTINTELLIGENCE_SERVICE: $(AZURE_DOCUMENTINTELLIGENCE_SERVICE)\n AZURE_DOCUMENTINTELLIGENCE_RESOURCE_GROUP: $(AZURE_DOCUMENTINTELLIGENCE_RESOURCE_GROUP)\n AZURE_DOCUMENTINTELLIGENCE_SKU: $(AZURE_DOCUMENTINTELLIGENCE_SKU)\n AZURE_DOCUMENTINTELLIGENCE_LOCATION: $(AZURE_DOCUMENTINTELLIGENCE_LOCATION)\n AZURE_SEARCH_INDEX: $(AZURE_SEARCH_INDEX)\n AZURE_SEARCH_SERVICE: $(AZURE_SEARCH_SERVICE)\n AZURE_SEARCH_SERVICE_RESOURCE_GROUP: $(AZURE_SEARCH_SERVICE_RESOURCE_GROUP)\n AZURE_SEARCH_SERVICE_LOCATION: $(AZURE_SEARCH_SERVICE_LOCATION)\n AZURE_SEARCH_SERVICE_SKU: $(AZURE_SEARCH_SERVICE_SKU)\n AZURE_SEARCH_QUERY_LANGUAGE: $(AZURE_SEARCH_QUERY_LANGUAGE)\n AZURE_SEARCH_QUERY_SPELLER: $(AZURE_SEARCH_QUERY_SPELLER)\n AZURE_SEARCH_SEMANTIC_RANKER: $(AZURE_SEARCH_SEMANTIC_RANKER)\n AZURE_SEARCH_QUERY_REWRITING: $(AZURE_SEARCH_QUERY_REWRITING)\n AZURE_SEARCH_FIELD_NAME_EMBEDDING: $(AZURE_SEARCH_FIELD_NAME_EMBEDDING)\n AZURE_STORAGE_ACCOUNT: $(AZURE_STORAGE_ACCOUNT)\n AZURE_STORAGE_RESOURCE_GROUP: $(AZURE_STORAGE_RESOURCE_GROUP)\n AZURE_STORAGE_SKU: $(AZURE_STORAGE_SKU)\n AZURE_APP_SERVICE_SKU: $(AZURE_APP_SERVICE_SKU)\n AZURE_OPENAI_CHATGPT_MODEL: $(AZURE_OPENAI_CHATGPT_MODEL)\n AZURE_OPENAI_CHATGPT_DEPLOYMENT: $(AZURE_OPENAI_CHATGPT_DEPLOYMENT)\n AZURE_OPENAI_CHATGPT_DEPLOYMENT_CAPACITY: $(AZURE_OPENAI_CHATGPT_DEPLOYMENT_CAPACITY)\n AZURE_OPENAI_CHATGPT_DEPLOYMENT_VERSION: $(AZURE_OPENAI_CHATGPT_DEPLOYMENT_VERSION)\n AZURE_OPENAI_CHATGPT_DEPLOYMENT_SKU: $(AZURE_OPENAI_CHATGPT_DEPLOYMENT_SKU)\n AZURE_OPENAI_REASONING_EFFORT: $(AZURE_OPENAI_REASONING_EFFORT)\n AGENTIC_KNOWLEDGEBASE_REASONING_EFFORT: $(AGENTIC_KNOWLEDGEBASE_REASONING_EFFORT)\n AZURE_OPENAI_EMB_MODEL_NAME: $(AZURE_OPENAI_EMB_MODEL_NAME)\n AZURE_OPENAI_EMB_DEPLOYMENT: $(AZURE_OPENAI_EMB_DEPLOYMENT)\n AZURE_OPENAI_EMB_DEPLOYMENT_CAPACITY: $(AZURE_OPENAI_EMB_DEPLOYMENT_CAPACITY)\n AZURE_OPENAI_EMB_DEPLOYMENT_VERSION: $(AZURE_OPENAI_EMB_DEPLOYMENT_VERSION)\n AZURE_OPENAI_EMB_DEPLOYMENT_SKU: $(AZURE_OPENAI_EMB_DEPLOYMENT_SKU)\n AZURE_OPENAI_EMB_DIMENSIONS: $(AZURE_OPENAI_EMB_DIMENSIONS)\n AZURE_OPENAI_DISABLE_KEYS: $(AZURE_OPENAI_DISABLE_KEYS)\n OPENAI_HOST: $(OPENAI_HOST)\n OPENAI_API_KEY: $(OPENAI_API_KEY)\n OPENAI_ORGANIZATION: $(OPENAI_ORGANIZATION)\n AZURE_USE_APPLICATION_INSIGHTS: $(AZURE_USE_APPLICATION_INSIGHTS)\n AZURE_APPLICATION_INSIGHTS: $(AZURE_APPLICATION_INSIGHTS)\n AZURE_APPLICATION_INSIGHTS_DASHBOARD: $(AZURE_APPLICATION_INSIGHTS_DASHBOARD)\n AZURE_LOG_ANALYTICS: $(AZURE_LOG_ANALYTICS)\n USE_VECTORS: $(USE_VECTORS)\n USE_MULTIMODAL: $(USE_MULTIMODAL)\n USE_CLOUD_INGESTION: $(USE_CLOUD_INGESTION)\n USE_CLOUD_INGESTION_ACLS: $(USE_CLOUD_INGESTION_ACLS)\n USE_EXISTING_ADLS_STORAGE: $(USE_EXISTING_ADLS_STORAGE)\n AZURE_ADLS_GEN2_STORAGE_ACCOUNT: $(AZURE_ADLS_GEN2_STORAGE_ACCOUNT)\n AZURE_ADLS_GEN2_STORAGE_RESOURCE_GROUP: $(AZURE_ADLS_GEN2_STORAGE_RESOURCE_GROUP)\n AZURE_VISION_ENDPOINT: $(AZURE_VISION_ENDPOINT)\n VISION_SECRET_NAME: $(VISION_SECRET_NAME)\n AZURE_VISION_SERVICE: $(AZURE_VISION_SERVICE)\n AZURE_VISION_RESOURCE_GROUP: $(AZURE_VISION_RESOURCE_GROUP)\n AZURE_VISION_LOCATION: $(AZURE_VISION_LOCATION)\n AZURE_VISION_SKU: $(AZURE_VISION_SKU)\n ENABLE_LANGUAGE_PICKER: $(ENABLE_LANGUAGE_PICKER)\n USE_SPEECH_INPUT_BROWSER: $(USE_SPEECH_INPUT_BROWSER)\n USE_SPEECH_OUTPUT_BROWSER: $(USE_SPEECH_OUTPUT_BROWSER)\n USE_SPEECH_OUTPUT_AZURE: $(USE_SPEECH_OUTPUT_AZURE)\n AZURE_SPEECH_SERVICE: $(AZURE_SPEECH_SERVICE)\n AZURE_SPEECH_SERVICE_RESOURCE_GROUP: $(AZURE_SPEECH_SERVICE_RESOURCE_GROUP)\n AZURE_SPEECH_SERVICE_LOCATION: $(AZURE_SPEECH_SERVICE_LOCATION)\n AZURE_SPEECH_SERVICE_SKU: $(AZURE_SPEECH_SERVICE_SKU)\n AZURE_SPEECH_SERVICE_VOICE: $(AZURE_SPEECH_SERVICE_VOICE)\n AZURE_KEY_VAULT_NAME: $(AZURE_KEY_VAULT_NAME)\n AZURE_USE_AUTHENTICATION: $(AZURE_USE_AUTHENTICATION)\n AZURE_ENFORCE_ACCESS_CONTROL: $(AZURE_ENFORCE_ACCESS_CONTROL)\n AZURE_ENABLE_GLOBAL_DOCUMENT_ACCESS: $(AZURE_ENABLE_GLOBAL_DOCUMENT_ACCESS)\n AZURE_ENABLE_UNAUTHENTICATED_ACCESS: $(AZURE_ENABLE_UNAUTHENTICATED_ACCESS)\n AZURE_TENANT_ID: $(AZURE_TENANT_ID)\n AZURE_AUTH_TENANT_ID: $(AZURE_AUTH_TENANT_ID)\n AZURE_SERVER_APP_ID: $(AZURE_SERVER_APP_ID)\n AZURE_CLIENT_APP_ID: $(AZURE_CLIENT_APP_ID)\n ALLOWED_ORIGIN: $(ALLOWED_ORIGIN)\n AZURE_SERVER_APP_SECRET: $(AZURE_SERVER_APP_SECRET)\n AZURE_CLIENT_APP_SECRET: $(AZURE_CLIENT_APP_SECRET)\n AZURE_ADLS_GEN2_FILESYSTEM: $(AZURE_ADLS_GEN2_FILESYSTEM)\n DEPLOYMENT_TARGET: $(DEPLOYMENT_TARGET)\n AZURE_CONTAINER_APPS_WORKLOAD_PROFILE: $(AZURE_CONTAINER_APPS_WORKLOAD_PROFILE)\n USE_CHAT_HISTORY_BROWSER: $(USE_CHAT_HISTORY_BROWSER)\n USE_MEDIA_DESCRIBER_AZURE_CU: $(USE_MEDIA_DESCRIBER_AZURE_CU)\n RAG_SEARCH_TEXT_EMBEDDINGS: $(RAG_SEARCH_TEXT_EMBEDDINGS)\n RAG_SEARCH_IMAGE_EMBEDDINGS: $(RAG_SEARCH_IMAGE_EMBEDDINGS)\n RAG_SEND_TEXT_SOURCES: $(RAG_SEND_TEXT_SOURCES)\n RAG_SEND_IMAGE_SOURCES: $(RAG_SEND_IMAGE_SOURCES)\n USE_AGENTIC_KNOWLEDGEBASE: $(USE_AGENTIC_KNOWLEDGEBASE)\n USE_WEB_SOURCE: $(USE_WEB_SOURCE)\n USE_SHAREPOINT_SOURCE: $(USE_SHAREPOINT_SOURCE)\n - task: AzureCLI@2\n displayName: Deploy Application\n inputs:\n azureSubscription: azconnection\n scriptType: bash\n scriptLocation: inlineScript\n inlineScript: |\n azd deploy --no-prompt\n" }, { "path": ".devcontainer/devcontainer.json", "content": "{\n \"name\": \"Azure Search OpenAI Demo\",\n \"image\": \"mcr.microsoft.com/devcontainers/python:3.13-bookworm\",\n \"features\": {\n \"ghcr.io/devcontainers/features/node:1\": {\n // This should match the version of Node.js in Github Actions workflows\n \"version\": \"22\",\n \"nodeGypDependencies\": false\n },\n \"ghcr.io/devcontainers/features/azure-cli:1.2.5\": {},\n \"ghcr.io/devcontainers/features/docker-in-docker:2\": {},\n \"ghcr.io/azure/azure-dev/azd:latest\": {}\n },\n \"customizations\": {\n \"vscode\": {\n \"extensions\": [\n \"ms-azuretools.azure-dev\",\n \"ms-azuretools.vscode-bicep\",\n \"ms-python.python\",\n \"astral-sh.ty\",\n \"esbenp.prettier-vscode\",\n \"DavidAnson.vscode-markdownlint\"\n ]\n }\n },\n \"forwardPorts\": [\n 50505\n ],\n \"postCreateCommand\": \"\",\n \"remoteUser\": \"vscode\",\n \"hostRequirements\": {\n \"memory\": \"8gb\"\n }\n}\n" }, { "path": ".gitattributes", "content": "*.sh text eol=lf\n*.jsonlines text eol=lf\n" }, { "path": ".github/CODE_OF_CONDUCT.md", "content": "# Microsoft Open Source Code of Conduct\n\nThis project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/).\n\nResources:\n\n- [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/)\n- [Microsoft Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/)\n- Contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with questions or concerns\n" }, { "path": ".github/ISSUE_TEMPLATE.md", "content": "\n> Please provide us with the following information:\n> ---------------------------------------------------------------\n\n### This issue is for a: (mark with an `x`)\n```\n- [ ] bug report -> please search issues before submitting\n- [ ] feature request\n- [ ] documentation issue or request\n- [ ] regression (a behavior that used to work and stopped in a new release)\n```\n\n### Minimal steps to reproduce\n>\n\n### Any log messages given by the failure\n>\n\n### Expected/desired behavior\n>\n\n### OS and Version?\n> Windows 7, 8 or 10. Linux (which distribution). macOS (Yosemite? El Capitan? Sierra?)\n\n### azd version?\n> run `azd version` and copy paste here.\n\n### Versions\n>\n\n### Mention any other details that might be useful\n\n> ---------------------------------------------------------------\n> Thanks! We'll be in touch soon.\n" }, { "path": ".github/PULL_REQUEST_TEMPLATE.md", "content": "## Purpose\n\n\n\n\n## Does this introduce a breaking change?\n\nWhen developers merge from main and run the server, azd up, or azd deploy, will this produce an error?\nIf you're not sure, try it out on an old environment.\n\n```\n[ ] Yes\n[ ] No\n```\n\n## Does this require changes to learn.microsoft.com docs?\n\nThis repository is referenced by [this tutorial](https://learn.microsoft.com/azure/developer/python/get-started-app-chat-template)\nwhich includes deployment, settings and usage instructions. If text or screenshot need to change in the tutorial,\ncheck the box below and notify the tutorial author. A Microsoft employee can do this for you if you're an external contributor.\n\n```\n[ ] Yes\n[ ] No\n```\n\n## Type of change\n\n```\n[ ] Bugfix\n[ ] Feature\n[ ] Code style update (formatting, local variables)\n[ ] Refactoring (no functional changes, no api changes)\n[ ] Documentation content changes\n[ ] Other... Please describe:\n```\n\n## Code quality checklist\n\nSee [CONTRIBUTING.md](https://github.com/Azure-Samples/azure-search-openai-demo/blob/main/CONTRIBUTING.md#submit-pr) for more details.\n\n- [ ] The current tests all pass (`python -m pytest`).\n- [ ] I added tests that prove my fix is effective or that my feature works\n- [ ] I ran `python -m pytest --cov` to verify 100% coverage of added lines\n- [ ] I ran `ty check` to check for type errors\n- [ ] I either used the pre-commit hooks or ran `ruff` and `black` manually on my code.\n" }, { "path": ".github/agents/fixer.agent.md", "content": "---\ndescription: 'Fix and verify issues in app'\ntools: ['vscode', 'execute', 'read', 'edit', 'search', 'web', 'agent', 'azure-mcp/search', 'github/create_pull_request', 'github/issue_read', 'github/list_issues', 'github/search_issues', 'playwright/*', 'pylance-mcp-server/*', 'microsoftdocs/mcp/*']\n---\n\n# Fixer Mode Instructions\n\nYou are in fixer mode. When given an issue to fix, follow these steps:\n\n1. **Gather context**: Read error messages/stack traces/related code. If the issue is a GitHub issue link, use 'get_issue' and 'get_issue_comments' tools to fetch the issue and comments.\n2. **Make targeted fix**: Make minimal changes to fix the issue. Do not fix any issues that weren't identified. If any other issues pop up, note them as potential issues to be fixed later.\n3. **Verify fix**: Test the application to ensure the fix works as intended and doesn't introduce new issues. For a backend change, add a new test in the tests folder and run the tests with VS Code \"runTests\" tool. RUN all the tests using that tool, not just the tests you added. Try to add tests to existing test files when possible, like test_app.py. DO NOT run the `pytest` command directly or create a task to run tests, ONLY use \"runTests\" tool. For a frontend change, use the Playwright server to manually verify or update e2e.py tests.\n\n## Local server setup\n\nYou MUST check task output readiness before debugging, testing, or declaring work complete.\n\n- Start the app: Run the \"Development\" compound task (which runs both frontend and backend tasks) and check readiness from task output. Both must be in ready state:\n\t- Frontend task: \"Frontend: npm run dev\"\n\t- Backend task: \"Backend: quart run\"\n- Investigate and fix errors shown in the corresponding task terminal before proceeding. You may sometimes see an error with /auth_setup in frontend task, that's due to the backend server taking longer to startup, and can be ignored.\n- Both of the tasks provide hot reloading behavior:\n\t- Frontend: Vite provides HMR; changes in the frontend are picked up automatically without restarting the task.\n\t- Backend: Quart was started with --reload; Python changes trigger an automatic restart.\n\t- If watchers seem stuck or output stops updating, stop the tasks and run the \"Development\" task again.\n- To interact with a running application, use the Playwright MCP server. If testing login, you will need to navigate to 'localhost' instead of '127.0.0.1' since that's the URL allowed by the Entra application.\n\n## Running Python scripts\n\nIf you are running Python scripts that depend on installed requirements, you must run them using the virtual environment in `.venv`.\n\n## Committing the change\n\nWhen change is complete, offer to make a new branch, git commit, and pull request.\nDO NOT check out a new branch unless explicitly confirmed - sometimes user is already in a branch\n\n## Making the PR\n\n* Use the `github/create_pull_request` tool to create the PR.\n* Follow the `.github/PULL_REQUEST_TEMPLATE.md` format, with all sections filled out and appropriate checkboxes checked. If any section does not apply, write \"N/A\" in that section.\n* Includes \"Fixes #\" sentence in the PR description to auto-close the issue when the PR is merged.\n" }, { "path": ".github/agents/triager.agent.md", "content": "---\ndescription: 'Triage old stale issues for obsolescence and recommend closures'\ntools: ['edit', 'search/usages', 'web', 'azure-mcp/search', 'github/add_issue_comment', 'github/get_commit', 'github/get_file_contents', 'github/get_latest_release', 'github/get_me', 'github/get_release_by_tag', 'github/get_tag', 'github/issue_read', 'github/issue_write', 'github/list_branches', 'github/list_commits', 'github/list_issue_types', 'github/list_issues', 'github/list_pull_requests', 'github/list_releases', 'github/list_tags', 'github/pull_request_read', 'github/search_code', 'github/search_issues', 'github/search_pull_requests', 'github/search_repositories', 'github/search_users', 'github/assign_copilot_to_issue', 'todo']\n---\n\n# Issue Triager\n\nYou are a GitHub issue triage specialist tasked with finding old stale issues that can be safely closed as obsolete. DO NOT actually close them yourself unless specifically told to do so. Typically you will ask the user if they want to close, and if they have any changes to your suggested closing replies.\n\n## Task Requirements\n\n### Primary Objective\nFind the specified number of stale issues in the Azure-Samples/azure-search-openai-demo repository that can be closed due to being obsolete or resolved by subsequent improvements.\n\n### Analysis Process\n1. **Search for stale issues**: Use GitHub tools to list issues with \"Stale\" label, sorted by creation date (oldest first)\n2. **Examine each issue**: Get detailed information including:\n - Creation date and last update\n - Issue description and problem reported\n - Comments and any attempted solutions\n - Current relevance to the codebase\n3. **Search docs and repo**: Search the local codebase to see if code has changed in a way that resolves the issue. Also look at README.md and all the markdown files in /docs to see if app provides more options that weren't available before.\n4. **Categorize obsolescence**: Identify issues that are obsolete due to:\n - Infrastructure/deployment changes since the issue was reported\n - Migration to newer libraries/frameworks (e.g., OpenAI SDK updates)\n - Cross-platform compatibility improvements\n - Configuration system redesigns\n - API changes that resolve the underlying problem\n\n### Output Format\nFor each recommended issue closure, provide:\n\n1. **Issue Number and Title**\n2. **GitHub Link**: Direct URL to the issue\n3. **Brief Summary** (2 sentences):\n - What the original problem was\n - Why it's now obsolete\n4. **Suggested Closing Reply**: A professional comment explaining:\n - Why the issue is being closed as obsolete\n - What changes have made it irrelevant (Only high confidence changes)\n - Invitation to open a new issue if the problem persists with current version\n\n### Success Criteria\n- Issues should be at least 1 year old\n- Issues should have \"Stale\" label\n- Must provide clear rationale for why each issue is obsolete\n- Closing replies should be professional and helpful\n- Focus on issues that won't recur with current codebase\n\n### Constraints\n- Do not recommend closing issues that represent ongoing valid feature requests\n- Avoid closing issues that highlight fundamental design limitations\n- Skip issues that could still affect current users even if less common\n- Ensure the obsolescence is due to actual code/infrastructure changes, not just age\n\n### Example Categories to Target\n- Deployment failures from early 2023 that were fixed by infrastructure improvements\n- Cross-platform compatibility issues resolved by script migrations\n- API errors from old library versions that have been updated\n- Configuration issues resolved by azd template redesigns\n- Authentication/permissions errors fixed by improved role assignment logic\n" }, { "path": ".github/dependabot.yaml", "content": "version: 2\nupdates:\n\n # Maintain dependencies for GitHub Actions\n - package-ecosystem: \"github-actions\"\n directory: \"/\"\n schedule:\n interval: \"weekly\"\n groups:\n github-actions:\n patterns:\n - \"*\"\n\n # Maintain dependencies for npm\n - package-ecosystem: \"npm\"\n directory: \"/app/frontend\"\n schedule:\n interval: \"weekly\"\n # Ignore Vite 7.x for now: Vite >=7 requires Node >=20.19.0 while project engines.node is currently >=20.0.0\n ignore:\n - dependency-name: \"vite\"\n versions:\n - \">=7.0.0\"\n\n # Maintain dependencies for pip\n - package-ecosystem: \"pip\"\n directory: \"/\"\n schedule:\n interval: \"weekly\"\n groups:\n telemetry:\n patterns:\n - \"opentelemetry-*\"\n - \"azure-monitor-opentelemetry*\"\n pydantic:\n patterns:\n - \"pydantic\"\n - \"pydantic-*\"\n - \"typing-extensions\"\n pallets:\n patterns:\n - \"flask\"\n - \"werkzeug\"\n - \"blinker\"\n - \"quart\"\n - \"jinja2\"\n - \"click\"\n - \"itsdangerous\"\n - \"markupsafe\"\n" }, { "path": ".github/instructions/bicep.instructions.md", "content": "---\ndescription: 'Infrastructure as Code with Bicep'\napplyTo: '**/*.bicep'\n---\n\n# Bicep best-practices\nThis list of best-practices builds on top of information available at https://learn.microsoft.com/azure/azure-resource-manager/bicep. It provides a more opinionated and up-to-date set of rules for generating high-quality Bicep code. You should aim to follow these rules whenever generating or modifying Bicep code.\n\n## Rules\n### General\n1. Avoid setting the `name` field for `module` statements - it is no longer required.\n1. If you need to input or output a set of logically-grouped values, generate a single `param` or `output` statement with a User-defined type instead of emitting a `param` or `output` statement for each value.\n1. If generating parameters, default to generating Bicep parameters files (`*.bicepparam`), instead of ARM parameters files (`*.json`).\n\n### Resources\n1. Do not add references from child resources to parent resources by using `/` characters in the child resource `name` property. Instead, use the `parent` property with a symbolic reference to the parent resource.\n1. If you are generating a child resource type, sometimes this may require you to add an `existing` resource for the parent if the parent is not already present in the file.\n1. If you see diagnostic codes `BCP036`, `BCP037` or `BCP081`, this may indicate you have hallucinated resource types or resource type properties. You may need to double-check against available resource type schema to tune your output.\n1. Avoid using multiple `resourceId()` functions and `reference()` function where possible. Instead use symbolic names to refer to ids or properties, creating `existing` resources if needed. For example, write `foo.id` or `foo.properties.id`, instead of `resourceId('...')` or `reference('...').id`.\n\n### Types\n1. Avoid using open types such as `array` or `object` when referencing types where possible (e.g. in `output` or `param` statements). Instead, use User-defined types to define a more precise type.\n1. Use typed variables instead of untyped variables when exporting values with the `@export()` decorator. For example, use `var foo string = 'blah'` instead of `var foo = bar`.\n1. When using User-defined types, aim to avoid repetition, and comment properties with `@description()` where the context is unclear.\n1. If you are passing data directly to or from a resource body via a `param` or `output` statement, try to use existing Resource-derived types (`resourceInput<'type@version'>` and `resourceOutput<'type@version'>`) instead of writing User-defined types.\n\n### Security\n1. When generating `param` or `output` statements, ALWAYS use the `@secure()` decorator if sensitive data is present.\n\n### Syntax\n1. If you hit warnings or errors with null properties, prefer solving them with the safe-dereference (`.?`) operator, in conjunction with the coalesce (`??`) operator. For example, `a.?b ?? c` is better than `a!.b` which may cause runtime errors, or `a != null ? a.b : c` which is unnecessarily verbose.\n\n## Glossary\n* Child resource: an Azure resource type with type name consisting of more than 1 `/` characters. For example, `Microsoft.Network/virtualNetworks/subnets` is a child resource. `Microsoft.Network/virtualNetworks` is not.\n" }, { "path": ".github/prompts/review_pr_comments.prompt.md", "content": "---\nagent: agent\n---\nWe have received comments on the current active pull request. Together, we will go through each comment one by one and discuss whether to accept the change, iterate on it, or reject the change.\n\n## Steps to follow:\n\n1. Fetch the active pull request: If available, use the `activePullRequest` tool from the `GitHub Pull Requests` toolset to get the details of the active pull request including the comments. If not, use the GitHub MCP server or GitHub CLI to get the details of the active pull request. Fetch both top level comments and inline comments.\n2. Present a list of the comments with a one-sentence summary of each.\n3. One at a time, present each comment in full detail and ask me whether to accept, iterate, or reject the change. Provide your recommendation for each comment based on best practices, code quality, and project guidelines. Await user's decision before proceeding to the next comment. DO NOT make any changes to the code or files until I have responded with my decision for each comment.\n4. If the decision is to accept or iterate, make the necessary code changes to address the comment. If the decision is to reject, provide a brief explanation of why the change was not made.\n5. Wait for user to affirm completion of any code changes made before moving to the next comment.\n6. Reply to each comment on the pull request with the outcome of our discussion (accepted, iterated, or rejected) along with any relevant explanations.\n" }, { "path": ".github/skills/github-pr-inline-reply/SKILL.md", "content": "---\nname: github-pr-inline-reply\ndescription: Reply to inline PR review comments on GitHub pull requests using the GitHub API. Use this skill when you need to respond to individual review comments on a PR, acknowledge feedback, or mark comments as resolved by posting direct replies to comment threads.\n---\n\n# GitHub PR Inline Reply Skill\n\nThis skill enables replying directly to inline review comments on GitHub pull requests.\n\n## When to use\n\n- Replying to individual PR review comments\n- Acknowledging reviewer feedback on specific lines of code\n- Marking review comments as addressed with a reply\n\n## API Endpoint\n\nTo reply to an inline PR comment, use:\n\n```http\nPOST /repos/{owner}/{repo}/pulls/{pull_number}/comments/{comment_id}/replies\n```\n\nWith body:\n\n```json\n{\n \"body\": \"Your reply message\"\n}\n```\n\n## Using gh CLI\n\n```bash\ngh api repos/{owner}/{repo}/pulls/{pull_number}/comments/{comment_id}/replies \\\n -X POST \\\n -f body=\"Your reply message\"\n```\n\n## Workflow\n\n1. **Get PR comments**: First fetch the PR review comments to get their IDs:\n\n ```bash\n gh api repos/{owner}/{repo}/pulls/{pull_number}/comments\n ```\n\n2. **Identify comment IDs**: Each comment has an `id` field. For threaded comments, use the root comment's `id`.\n\n3. **Post replies**: For each comment you want to reply to:\n\n ```bash\n gh api repos/{owner}/{repo}/pulls/{pull_number}/comments/{comment_id}/replies \\\n -X POST \\\n -f body=\"Fixed in commit abc123\"\n ```\n\n## Example Replies\n\nFor accepted changes:\n\n- \"Fixed in {commit_sha}\"\n- \"Accepted - fixed in {commit_sha}\"\n\nFor rejected changes:\n\n- \"Rejected - {reason}\"\n- \"Won't fix - {explanation}\"\n\nFor questions:\n\n- \"Good catch, addressed in {commit_sha}\"\n\n## Notes\n\n- The `comment_id` is the numeric ID from the comment object, NOT the `node_id`\n- Replies appear as threaded responses under the original comment\n- You can reply to any comment, including bot comments (like Copilot reviews)\n\n## Resolving Conversations\n\nTo resolve (mark as resolved) PR review threads, use the GraphQL API:\n\n1. **Get thread IDs**: Query for unresolved threads:\n\n ```bash\n gh api graphql -f query='\n query {\n repository(owner: \"{owner}\", name: \"{repo}\") {\n pullRequest(number: {pull_number}) {\n reviewThreads(first: 50) {\n nodes {\n id\n isResolved\n comments(first: 1) {\n nodes { body path }\n }\n }\n }\n }\n }\n }'\n ```\n\n2. **Resolve threads**: Use the `resolveReviewThread` mutation:\n\n ```bash\n gh api graphql -f query='\n mutation {\n resolveReviewThread(input: {threadId: \"PRRT_xxx\"}) {\n thread { isResolved }\n }\n }'\n ```\n\n3. **Resolve multiple threads at once**:\n\n ```bash\n gh api graphql -f query='\n mutation {\n t1: resolveReviewThread(input: {threadId: \"PRRT_xxx\"}) { thread { isResolved } }\n t2: resolveReviewThread(input: {threadId: \"PRRT_yyy\"}) { thread { isResolved } }\n }'\n ```\n\nThe thread ID starts with `PRRT_` and can be found in the GraphQL query response.\n\nNote: This skill can be removed once the GitHub MCP server has added built-in support for replying to PR review comments and resolving threads.\nSee:\nhttps://github.com/github/github-mcp-server/issues/1323\nhttps://github.com/github/github-mcp-server/issues/1768\n" }, { "path": ".github/workflows/azure-dev-validation.yaml", "content": "name: Validate AZD template\non:\n push:\n branches: [ main ]\n paths:\n - \"infra/**\"\n pull_request:\n branches: [ main ]\n paths:\n - \"infra/**\"\n workflow_dispatch:\n\njobs:\n bicep:\n runs-on: ubuntu-latest\n permissions:\n security-events: write\n steps:\n - name: Checkout\n uses: actions/checkout@v6\n\n - name: Build Bicep for linting\n uses: azure/CLI@v2\n with:\n inlineScript: |\n export DOTNET_SYSTEM_GLOBALIZATION_INVARIANT=1\n az config set bicep.use_binary_from_path=false && az bicep build -f infra/main.bicep --stdout\n\n psrule:\n runs-on: ubuntu-latest\n permissions:\n security-events: write\n steps:\n - name: Checkout\n uses: actions/checkout@v6\n\n - name: Run PSRule analysis\n uses: microsoft/ps-rule@v2.9.0\n with:\n modules: PSRule.Rules.Azure\n baseline: Azure.Pillar.Security\n inputPath: infra/*.test.bicep\n outputFormat: Sarif\n outputPath: reports/ps-rule-results.sarif\n summary: true\n continue-on-error: true\n\n env:\n PSRULE_CONFIGURATION_AZURE_BICEP_FILE_EXPANSION: 'true'\n PSRULE_CONFIGURATION_AZURE_BICEP_FILE_EXPANSION_TIMEOUT: '30'\n\n - name: Upload results to security tab\n uses: github/codeql-action/upload-sarif@v4\n if: github.repository == 'Azure-Samples/azure-search-openai-demo'\n with:\n sarif_file: reports/ps-rule-results.sarif\n" }, { "path": ".github/workflows/azure-dev.yml", "content": "name: Deploy\n\non:\n workflow_dispatch:\n push:\n # Run when commits are pushed to mainline branch (main or master)\n # Set this to the mainline branch you are using\n branches:\n - main\n - master\n\n# GitHub Actions workflow to deploy to Azure using azd\n# To configure required secrets for connecting to Azure, simply run `azd pipeline config`\n\n# Set up permissions for deploying with secretless Azure federated credentials\n# https://learn.microsoft.com/azure/developer/github/connect-from-azure?tabs=azure-portal%2Clinux#set-up-azure-login-with-openid-connect-authentication\npermissions:\n id-token: write\n contents: read\n\njobs:\n build:\n runs-on: ubuntu-latest\n env:\n # azd required\n AZURE_CLIENT_ID: ${{ vars.AZURE_CLIENT_ID }}\n AZURE_TENANT_ID: ${{ vars.AZURE_TENANT_ID }}\n AZURE_SUBSCRIPTION_ID: ${{ vars.AZURE_SUBSCRIPTION_ID }}\n AZURE_ENV_NAME: ${{ vars.AZURE_ENV_NAME }}\n AZURE_LOCATION: ${{ vars.AZURE_LOCATION }}\n # project specific\n AZURE_OPENAI_SERVICE: ${{ vars.AZURE_OPENAI_SERVICE }}\n AZURE_OPENAI_LOCATION: ${{ vars.AZURE_OPENAI_LOCATION }}\n AZURE_OPENAI_RESOURCE_GROUP: ${{ vars.AZURE_OPENAI_RESOURCE_GROUP }}\n AZURE_DOCUMENTINTELLIGENCE_SERVICE: ${{ vars.AZURE_DOCUMENTINTELLIGENCE_SERVICE }}\n AZURE_DOCUMENTINTELLIGENCE_RESOURCE_GROUP: ${{ vars.AZURE_DOCUMENTINTELLIGENCE_RESOURCE_GROUP }}\n AZURE_DOCUMENTINTELLIGENCE_SKU: ${{ vars.AZURE_DOCUMENTINTELLIGENCE_SKU }}\n AZURE_DOCUMENTINTELLIGENCE_LOCATION: ${{ vars.AZURE_DOCUMENTINTELLIGENCE_LOCATION }}\n AZURE_VISION_SERVICE: ${{ vars.AZURE_VISION_SERVICE }}\n AZURE_VISION_RESOURCE_GROUP: ${{ vars.AZURE_VISION_RESOURCE_GROUP }}\n AZURE_VISION_LOCATION: ${{ vars.AZURE_VISION_LOCATION }}\n AZURE_VISION_SKU: ${{ vars.AZURE_VISION_SKU }}\n AZURE_SEARCH_INDEX: ${{ vars.AZURE_SEARCH_INDEX }}\n AZURE_SEARCH_SERVICE: ${{ vars.AZURE_SEARCH_SERVICE }}\n AZURE_SEARCH_SERVICE_RESOURCE_GROUP: ${{ vars.AZURE_SEARCH_SERVICE_RESOURCE_GROUP }}\n AZURE_SEARCH_SERVICE_LOCATION: ${{ vars.AZURE_SEARCH_SERVICE_LOCATION }}\n AZURE_SEARCH_SERVICE_SKU: ${{ vars.AZURE_SEARCH_SERVICE_SKU }}\n AZURE_SEARCH_QUERY_LANGUAGE: ${{ vars.AZURE_SEARCH_QUERY_LANGUAGE }}\n AZURE_SEARCH_QUERY_SPELLER: ${{ vars.AZURE_SEARCH_QUERY_SPELLER }}\n AZURE_SEARCH_SEMANTIC_RANKER: ${{ vars.AZURE_SEARCH_SEMANTIC_RANKER }}\n AZURE_SEARCH_QUERY_REWRITING: ${{ vars.AZURE_SEARCH_QUERY_REWRITING }}\n AZURE_SEARCH_FIELD_NAME_EMBEDDING: ${{ vars.AZURE_SEARCH_FIELD_NAME_EMBEDDING }}\n AZURE_STORAGE_ACCOUNT: ${{ vars.AZURE_STORAGE_ACCOUNT }}\n AZURE_STORAGE_RESOURCE_GROUP: ${{ vars.AZURE_STORAGE_RESOURCE_GROUP }}\n AZURE_STORAGE_SKU: ${{ vars.AZURE_STORAGE_SKU }}\n AZURE_APP_SERVICE_PLAN: ${{ vars.AZURE_APP_SERVICE_PLAN }}\n AZURE_APP_SERVICE_SKU: ${{ vars.AZURE_APP_SERVICE_SKU }}\n AZURE_APP_SERVICE: ${{ vars.AZURE_APP_SERVICE }}\n AZURE_OPENAI_CHATGPT_MODEL: ${{ vars.AZURE_OPENAI_CHATGPT_MODEL }}\n AZURE_OPENAI_CHATGPT_DEPLOYMENT: ${{ vars.AZURE_OPENAI_CHATGPT_DEPLOYMENT }}\n AZURE_OPENAI_CHATGPT_DEPLOYMENT_CAPACITY: ${{ vars.AZURE_OPENAI_CHATGPT_DEPLOYMENT_CAPACITY }}\n AZURE_OPENAI_CHATGPT_DEPLOYMENT_VERSION: ${{ vars.AZURE_OPENAI_CHATGPT_DEPLOYMENT_VERSION }}\n AZURE_OPENAI_REASONING_EFFORT: ${{ vars.AZURE_OPENAI_REASONING_EFFORT }}\n AGENTIC_KNOWLEDGEBASE_REASONING_EFFORT: ${{ vars.AGENTIC_KNOWLEDGEBASE_REASONING_EFFORT }}\n AZURE_OPENAI_EMB_MODEL_NAME: ${{ vars.AZURE_OPENAI_EMB_MODEL_NAME }}\n AZURE_OPENAI_EMB_DEPLOYMENT: ${{ vars.AZURE_OPENAI_EMB_DEPLOYMENT }}\n AZURE_OPENAI_EMB_DEPLOYMENT_CAPACITY: ${{ vars.AZURE_OPENAI_EMB_DEPLOYMENT_CAPACITY }}\n AZURE_OPENAI_EMB_DEPLOYMENT_VERSION: ${{ vars.AZURE_OPENAI_EMB_DEPLOYMENT_VERSION }}\n AZURE_OPENAI_EMB_DIMENSIONS: ${{ vars.AZURE_OPENAI_EMB_DIMENSIONS }}\n USE_EVAL: ${{ vars.USE_EVAL }}\n AZURE_OPENAI_EVAL_MODEL: ${{ vars.AZURE_OPENAI_EVAL_MODEL }}\n AZURE_OPENAI_EVAL_MODEL_VERSION: ${{ vars.AZURE_OPENAI_EVAL_MODEL_VERSION }}\n AZURE_OPENAI_EVAL_DEPLOYMENT: ${{ vars.AZURE_OPENAI_EVAL_DEPLOYMENT }}\n AZURE_OPENAI_EVAL_DEPLOYMENT_SKU: ${{ vars.AZURE_OPENAI_EVAL_DEPLOYMENT_SKU }}\n AZURE_OPENAI_EVAL_DEPLOYMENT_CAPACITY: ${{ vars.AZURE_OPENAI_EVAL_DEPLOYMENT_CAPACITY }}\n AZURE_OPENAI_DISABLE_KEYS: ${{ vars.AZURE_OPENAI_DISABLE_KEYS }}\n OPENAI_HOST: ${{ vars.OPENAI_HOST }}\n OPENAI_API_KEY: ${{ vars.OPENAI_API_KEY }}\n OPENAI_ORGANIZATION: ${{ vars.OPENAI_ORGANIZATION }}\n AZURE_USE_APPLICATION_INSIGHTS: ${{ vars.AZURE_USE_APPLICATION_INSIGHTS }}\n AZURE_APPLICATION_INSIGHTS: ${{ vars.AZURE_APPLICATION_INSIGHTS }}\n AZURE_APPLICATION_INSIGHTS_DASHBOARD: ${{ vars.AZURE_APPLICATION_INSIGHTS_DASHBOARD }}\n AZURE_LOG_ANALYTICS: ${{ vars.AZURE_LOG_ANALYTICS }}\n USE_VECTORS: ${{ vars.USE_VECTORS }}\n USE_MULTIMODAL: ${{ vars.USE_MULTIMODAL }}\n USE_CLOUD_INGESTION: ${{ vars.USE_CLOUD_INGESTION }}\n USE_CLOUD_INGESTION_ACLS: ${{ vars.USE_CLOUD_INGESTION_ACLS }}\n USE_EXISTING_ADLS_STORAGE: ${{ vars.USE_EXISTING_ADLS_STORAGE }}\n AZURE_ADLS_GEN2_STORAGE_ACCOUNT: ${{ vars.AZURE_ADLS_GEN2_STORAGE_ACCOUNT }}\n AZURE_ADLS_GEN2_STORAGE_RESOURCE_GROUP: ${{ vars.AZURE_ADLS_GEN2_STORAGE_RESOURCE_GROUP }}\n AZURE_VISION_ENDPOINT: ${{ vars.AZURE_VISION_ENDPOINT }}\n VISION_SECRET_NAME: ${{ vars.VISION_SECRET_NAME }}\n ENABLE_LANGUAGE_PICKER: ${{ vars.ENABLE_LANGUAGE_PICKER }}\n USE_SPEECH_INPUT_BROWSER: ${{ vars.USE_SPEECH_INPUT_BROWSER }}\n USE_SPEECH_OUTPUT_BROWSER: ${{ vars.USE_SPEECH_OUTPUT_BROWSER }}\n USE_SPEECH_OUTPUT_AZURE: ${{ vars.USE_SPEECH_OUTPUT_AZURE }}\n AZURE_SPEECH_SERVICE: ${{ vars.AZURE_SPEECH_SERVICE }}\n AZURE_SPEECH_SERVICE_RESOURCE_GROUP: ${{ vars.AZURE_SPEECH_RESOURCE_GROUP }}\n AZURE_SPEECH_SERVICE_LOCATION: ${{ vars.AZURE_SPEECH_SERVICE_LOCATION }}\n AZURE_SPEECH_SERVICE_SKU: ${{ vars.AZURE_SPEECH_SERVICE_SKU }}\n AZURE_SPEECH_SERVICE_VOICE: ${{ vars.AZURE_SPEECH_SERVICE_VOICE }}\n AZURE_KEY_VAULT_NAME: ${{ vars.AZURE_KEY_VAULT_NAME }}\n AZURE_USE_AUTHENTICATION: ${{ vars.AZURE_USE_AUTHENTICATION }}\n AZURE_ENFORCE_ACCESS_CONTROL: ${{ vars.AZURE_ENFORCE_ACCESS_CONTROL }}\n AZURE_ENABLE_GLOBAL_DOCUMENT_ACCESS: ${{ vars.AZURE_ENABLE_GLOBAL_DOCUMENT_ACCESS }}\n AZURE_ENABLE_UNAUTHENTICATED_ACCESS: ${{ vars.AZURE_ENABLE_UNAUTHENTICATED_ACCESS }}\n AZURE_AUTH_TENANT_ID: ${{ vars.AZURE_AUTH_TENANT_ID }}\n AZURE_SERVER_APP_ID: ${{ vars.AZURE_SERVER_APP_ID }}\n AZURE_CLIENT_APP_ID: ${{ vars.AZURE_CLIENT_APP_ID }}\n ALLOWED_ORIGIN: ${{ vars.ALLOWED_ORIGIN }}\n AZURE_ADLS_GEN2_FILESYSTEM: ${{ vars.AZURE_ADLS_GEN2_FILESYSTEM }}\n DEPLOYMENT_TARGET: ${{ vars.DEPLOYMENT_TARGET }}\n AZURE_CONTAINER_APPS_WORKLOAD_PROFILE: ${{ vars.AZURE_CONTAINER_APPS_WORKLOAD_PROFILE }}\n USE_CHAT_HISTORY_BROWSER: ${{ vars.USE_CHAT_HISTORY_BROWSER }}\n USE_MEDIA_DESCRIBER_AZURE_CU: ${{ vars.USE_MEDIA_DESCRIBER_AZURE_CU }}\n USE_AI_PROJECT: ${{ vars.USE_AI_PROJECT }}\n RAG_SEARCH_TEXT_EMBEDDINGS: ${{ vars.RAG_SEARCH_TEXT_EMBEDDINGS }}\n RAG_SEARCH_IMAGE_EMBEDDINGS: ${{ vars.RAG_SEARCH_IMAGE_EMBEDDINGS }}\n RAG_SEND_TEXT_SOURCES: ${{ vars.RAG_SEND_TEXT_SOURCES }}\n RAG_SEND_IMAGE_SOURCES: ${{ vars.RAG_SEND_IMAGE_SOURCES }}\n USE_AGENTIC_KNOWLEDGEBASE: ${{ vars.USE_AGENTIC_KNOWLEDGEBASE }}\n USE_WEB_SOURCE: ${{ vars.USE_WEB_SOURCE }}\n USE_SHAREPOINT_SOURCE: ${{ vars.USE_SHAREPOINT_SOURCE }}\n steps:\n - name: Checkout\n uses: actions/checkout@v6\n\n - name: Install azd\n uses: Azure/setup-azd@v2.2.1\n\n - name: Install Nodejs\n uses: actions/setup-node@v6\n with:\n node-version: 20\n\n - name: Log in with Azure (Federated Credentials)\n run: |\n azd auth login `\n --client-id \"$Env:AZURE_CLIENT_ID\" `\n --federated-credential-provider \"github\" `\n --tenant-id \"$Env:AZURE_TENANT_ID\"\n shell: pwsh\n\n - name: Provision Infrastructure\n run: azd provision --no-prompt\n env:\n AZD_INITIAL_ENVIRONMENT_CONFIG: ${{ secrets.AZD_INITIAL_ENVIRONMENT_CONFIG }}\n AZURE_SERVER_APP_SECRET: ${{ secrets.AZURE_SERVER_APP_SECRET }}\n AZURE_CLIENT_APP_SECRET: ${{ secrets.AZURE_CLIENT_APP_SECRET }}\n\n - name: Deploy Application\n run: azd deploy --no-prompt\n" }, { "path": ".github/workflows/evaluate.yaml", "content": "name: Evaluate RAG answer flow\n\non:\n issue_comment:\n types: [created]\n\n# Set up permissions for deploying with secretless Azure federated credentials\n# https://learn.microsoft.com/azure/developer/github/connect-from-azure?tabs=azure-portal%2Clinux#set-up-azure-login-with-openid-connect-authentication\npermissions:\n id-token: write\n contents: read\n issues: write\n pull-requests: write\n\njobs:\n evaluate:\n if: |\n contains('[\"OWNER\", \"CONTRIBUTOR\", \"COLLABORATOR\", \"MEMBER\"]', github.event.comment.author_association) &&\n github.event.issue.pull_request &&\n github.event.comment.body == '/evaluate'\n runs-on: ubuntu-latest\n env:\n # azd required\n AZURE_CLIENT_ID: ${{ vars.AZURE_CLIENT_ID }}\n AZURE_TENANT_ID: ${{ vars.AZURE_TENANT_ID }}\n AZURE_SUBSCRIPTION_ID: ${{ vars.AZURE_SUBSCRIPTION_ID }}\n AZURE_ENV_NAME: ${{ vars.AZURE_ENV_NAME }}\n AZURE_LOCATION: ${{ vars.AZURE_LOCATION }}\n # project specific\n AZURE_OPENAI_SERVICE: ${{ vars.AZURE_OPENAI_SERVICE }}\n AZURE_OPENAI_LOCATION: ${{ vars.AZURE_OPENAI_LOCATION }}\n AZURE_OPENAI_RESOURCE_GROUP: ${{ vars.AZURE_OPENAI_RESOURCE_GROUP }}\n AZURE_DOCUMENTINTELLIGENCE_SERVICE: ${{ vars.AZURE_DOCUMENTINTELLIGENCE_SERVICE }}\n AZURE_DOCUMENTINTELLIGENCE_RESOURCE_GROUP: ${{ vars.AZURE_DOCUMENTINTELLIGENCE_RESOURCE_GROUP }}\n AZURE_DOCUMENTINTELLIGENCE_SKU: ${{ vars.AZURE_DOCUMENTINTELLIGENCE_SKU }}\n AZURE_DOCUMENTINTELLIGENCE_LOCATION: ${{ vars.AZURE_DOCUMENTINTELLIGENCE_LOCATION }}\n AZURE_VISION_SERVICE: ${{ vars.AZURE_VISION_SERVICE }}\n AZURE_VISION_RESOURCE_GROUP: ${{ vars.AZURE_VISION_RESOURCE_GROUP }}\n AZURE_VISION_LOCATION: ${{ vars.AZURE_VISION_LOCATION }}\n AZURE_VISION_SKU: ${{ vars.AZURE_VISION_SKU }}\n AZURE_SEARCH_INDEX: ${{ vars.AZURE_SEARCH_INDEX }}\n AZURE_SEARCH_SERVICE: ${{ vars.AZURE_SEARCH_SERVICE }}\n AZURE_SEARCH_SERVICE_RESOURCE_GROUP: ${{ vars.AZURE_SEARCH_SERVICE_RESOURCE_GROUP }}\n AZURE_SEARCH_SERVICE_LOCATION: ${{ vars.AZURE_SEARCH_SERVICE_LOCATION }}\n AZURE_SEARCH_SERVICE_SKU: ${{ vars.AZURE_SEARCH_SERVICE_SKU }}\n AZURE_SEARCH_QUERY_LANGUAGE: ${{ vars.AZURE_SEARCH_QUERY_LANGUAGE }}\n AZURE_SEARCH_QUERY_SPELLER: ${{ vars.AZURE_SEARCH_QUERY_SPELLER }}\n AZURE_SEARCH_SEMANTIC_RANKER: ${{ vars.AZURE_SEARCH_SEMANTIC_RANKER }}\n AZURE_STORAGE_ACCOUNT: ${{ vars.AZURE_STORAGE_ACCOUNT }}\n AZURE_STORAGE_RESOURCE_GROUP: ${{ vars.AZURE_STORAGE_RESOURCE_GROUP }}\n AZURE_STORAGE_SKU: ${{ vars.AZURE_STORAGE_SKU }}\n AZURE_APP_SERVICE_PLAN: ${{ vars.AZURE_APP_SERVICE_PLAN }}\n AZURE_APP_SERVICE_SKU: ${{ vars.AZURE_APP_SERVICE_SKU }}\n AZURE_APP_SERVICE: ${{ vars.AZURE_APP_SERVICE }}\n AZURE_OPENAI_CHATGPT_MODEL: ${{ vars.AZURE_OPENAI_CHATGPT_MODEL }}\n AZURE_OPENAI_CHATGPT_DEPLOYMENT: ${{ vars.AZURE_OPENAI_CHATGPT_DEPLOYMENT }}\n AZURE_OPENAI_CHATGPT_DEPLOYMENT_CAPACITY: ${{ vars.AZURE_OPENAI_CHATGPT_DEPLOYMENT_CAPACITY }}\n AZURE_OPENAI_CHATGPT_DEPLOYMENT_VERSION: ${{ vars.AZURE_OPENAI_CHATGPT_DEPLOYMENT_VERSION }}\n AZURE_OPENAI_EMB_MODEL_NAME: ${{ vars.AZURE_OPENAI_EMB_MODEL_NAME }}\n AZURE_OPENAI_EMB_DEPLOYMENT: ${{ vars.AZURE_OPENAI_EMB_DEPLOYMENT }}\n AZURE_OPENAI_EMB_DEPLOYMENT_CAPACITY: ${{ vars.AZURE_OPENAI_EMB_DEPLOYMENT_CAPACITY }}\n AZURE_OPENAI_EMB_DEPLOYMENT_VERSION: ${{ vars.AZURE_OPENAI_EMB_DEPLOYMENT_VERSION }}\n AZURE_OPENAI_EMB_DIMENSIONS: ${{ vars.AZURE_OPENAI_EMB_DIMENSIONS }}\n USE_EVAL: ${{ vars.USE_EVAL }}\n AZURE_OPENAI_EVAL_MODEL: ${{ vars.AZURE_OPENAI_EVAL_MODEL }}\n AZURE_OPENAI_EVAL_MODEL_VERSION: ${{ vars.AZURE_OPENAI_EVAL_MODEL_VERSION }}\n AZURE_OPENAI_EVAL_DEPLOYMENT: ${{ vars.AZURE_OPENAI_EVAL_DEPLOYMENT }}\n AZURE_OPENAI_EVAL_DEPLOYMENT_SKU: ${{ vars.AZURE_OPENAI_EVAL_DEPLOYMENT_SKU }}\n AZURE_OPENAI_EVAL_DEPLOYMENT_CAPACITY: ${{ vars.AZURE_OPENAI_EVAL_DEPLOYMENT_CAPACITY }}\n AZURE_OPENAI_DISABLE_KEYS: ${{ vars.AZURE_OPENAI_DISABLE_KEYS }}\n OPENAI_HOST: ${{ vars.OPENAI_HOST }}\n OPENAI_API_KEY: ${{ vars.OPENAI_API_KEY }}\n OPENAI_ORGANIZATION: ${{ vars.OPENAI_ORGANIZATION }}\n AZURE_USE_APPLICATION_INSIGHTS: ${{ vars.AZURE_USE_APPLICATION_INSIGHTS }}\n AZURE_APPLICATION_INSIGHTS: ${{ vars.AZURE_APPLICATION_INSIGHTS }}\n AZURE_APPLICATION_INSIGHTS_DASHBOARD: ${{ vars.AZURE_APPLICATION_INSIGHTS_DASHBOARD }}\n AZURE_LOG_ANALYTICS: ${{ vars.AZURE_LOG_ANALYTICS }}\n USE_VECTORS: ${{ vars.USE_VECTORS }}\n USE_MULTIMODAL: ${{ vars.USE_MULTIMODAL }}\n AZURE_VISION_ENDPOINT: ${{ vars.AZURE_VISION_ENDPOINT }}\n VISION_SECRET_NAME: ${{ vars.VISION_SECRET_NAME }}\n ENABLE_LANGUAGE_PICKER: ${{ vars.ENABLE_LANGUAGE_PICKER }}\n USE_SPEECH_INPUT_BROWSER: ${{ vars.USE_SPEECH_INPUT_BROWSER }}\n USE_SPEECH_OUTPUT_BROWSER: ${{ vars.USE_SPEECH_OUTPUT_BROWSER }}\n USE_SPEECH_OUTPUT_AZURE: ${{ vars.USE_SPEECH_OUTPUT_AZURE }}\n AZURE_SPEECH_SERVICE: ${{ vars.AZURE_SPEECH_SERVICE }}\n AZURE_SPEECH_SERVICE_RESOURCE_GROUP: ${{ vars.AZURE_SPEECH_RESOURCE_GROUP }}\n AZURE_SPEECH_SERVICE_LOCATION: ${{ vars.AZURE_SPEECH_SERVICE_LOCATION }}\n AZURE_SPEECH_SERVICE_SKU: ${{ vars.AZURE_SPEECH_SERVICE_SKU }}\n AZURE_SPEECH_SERVICE_VOICE: ${{ vars.AZURE_SPEECH_SERVICE_VOICE }}\n AZURE_KEY_VAULT_NAME: ${{ vars.AZURE_KEY_VAULT_NAME }}\n AZURE_USE_AUTHENTICATION: ${{ vars.AZURE_USE_AUTHENTICATION }}\n AZURE_ENFORCE_ACCESS_CONTROL: ${{ vars.AZURE_ENFORCE_ACCESS_CONTROL }}\n AZURE_ENABLE_GLOBAL_DOCUMENT_ACCESS: ${{ vars.AZURE_ENABLE_GLOBAL_DOCUMENT_ACCESS }}\n AZURE_ENABLE_UNAUTHENTICATED_ACCESS: ${{ vars.AZURE_ENABLE_UNAUTHENTICATED_ACCESS }}\n AZURE_AUTH_TENANT_ID: ${{ vars.AZURE_AUTH_TENANT_ID }}\n AZURE_SERVER_APP_ID: ${{ vars.AZURE_SERVER_APP_ID }}\n AZURE_CLIENT_APP_ID: ${{ vars.AZURE_CLIENT_APP_ID }}\n ALLOWED_ORIGIN: ${{ vars.ALLOWED_ORIGIN }}\n AZURE_ADLS_GEN2_STORAGE_ACCOUNT: ${{ vars.AZURE_ADLS_GEN2_STORAGE_ACCOUNT }}\n AZURE_ADLS_GEN2_FILESYSTEM: ${{ vars.AZURE_ADLS_GEN2_FILESYSTEM }}\n DEPLOYMENT_TARGET: ${{ vars.DEPLOYMENT_TARGET }}\n AZURE_CONTAINER_APPS_WORKLOAD_PROFILE: ${{ vars.AZURE_CONTAINER_APPS_WORKLOAD_PROFILE }}\n USE_CHAT_HISTORY_BROWSER: ${{ vars.USE_CHAT_HISTORY_BROWSER }}\n USE_MEDIA_DESCRIBER_AZURE_CU: ${{ vars.USE_MEDIA_DESCRIBER_AZURE_CU }}\n USE_AI_PROJECT: ${{ vars.USE_AI_PROJECT }}\n steps:\n\n - name: Comment on pull request\n uses: actions/github-script@v8\n with:\n script: |\n github.rest.issues.createComment({\n issue_number: context.issue.number,\n owner: context.repo.owner,\n repo: context.repo.repo,\n body: \"Starting evaluation! Check the Actions tab for progress, or wait for a comment with the results.\"\n })\n\n - name: Checkout pull request\n uses: actions/checkout@v6\n with:\n ref: refs/pull/${{ github.event.issue.number }}/head\n\n - name: Install uv\n uses: astral-sh/setup-uv@v7\n with:\n enable-cache: true\n version: \"0.9.5\"\n cache-dependency-glob: \"requirements**.txt\"\n python-version: \"3.11\"\n\n - name: Setup node\n uses: actions/setup-node@v6\n with:\n node-version: 20\n\n - name: Install azd\n uses: Azure/setup-azd@v2.2.1\n\n - name: Login to Azure with az CLI\n uses: azure/login@v2\n with:\n client-id: ${{ env.AZURE_CLIENT_ID }}\n tenant-id: ${{ env.AZURE_TENANT_ID }}\n subscription-id: ${{ env.AZURE_SUBSCRIPTION_ID }}\n\n - name: Set az account\n uses: azure/CLI@v2\n with:\n inlineScript: |\n az account set --subscription ${{env.AZURE_SUBSCRIPTION_ID}}\n\n - name: Login to with Azure with azd (Federated Credentials)\n run: |\n azd auth login `\n --client-id \"$Env:AZURE_CLIENT_ID\" `\n --federated-credential-provider \"github\" `\n --tenant-id \"$Env:AZURE_TENANT_ID\"\n shell: pwsh\n\n - name: Refresh azd environment variables\n run: |\n azd env refresh -e $AZURE_ENV_NAME --no-prompt\n env:\n AZD_INITIAL_ENVIRONMENT_CONFIG: ${{ secrets.AZD_INITIAL_ENVIRONMENT_CONFIG }}\n\n - name: Build frontend\n run: |\n cd ./app/frontend\n npm install\n npm run build\n\n - name: Install dependencies\n run: |\n uv pip install -r requirements-dev.txt\n\n - name: Run local server in background\n run: |\n cd app/backend\n RUNNER_TRACKING_ID=\"\" && (nohup python3 -m quart --app main:app run --port 50505 > serverlogs.out 2> serverlogs.err &)\n cd ../..\n\n - name: Install evaluate dependencies\n run: |\n uv pip install -r evals/requirements.txt\n\n - name: Evaluate local RAG flow\n run: |\n python evals/evaluate.py --targeturl=http://127.0.0.1:50505/chat --resultsdir=evals/results/pr${{ github.event.issue.number }}\n\n - name: Upload eval results as build artifact\n if: ${{ success() }}\n uses: actions/upload-artifact@v7\n with:\n name: eval_result\n path: ./evals/results/pr${{ github.event.issue.number }}\n\n - name: Upload server logs as build artifact\n uses: actions/upload-artifact@v7\n with:\n name: server_logs\n path: ./app/backend/serverlogs.out\n\n - name: Upload server error logs as build artifact\n uses: actions/upload-artifact@v7\n with:\n name: server_error_logs\n path: ./app/backend/serverlogs.err\n\n - name: Summarize results\n if: ${{ success() }}\n run: |\n echo \"## Evaluation results\" >> eval-summary.md\n python -m evaltools summary evals/results --output=markdown >> eval-summary.md\n echo \"## Answer differences across runs\" >> run-diff.md\n python -m evaltools diff evals/results/baseline evals/results/pr${{ github.event.issue.number }} --output=markdown >> run-diff.md\n cat eval-summary.md >> $GITHUB_STEP_SUMMARY\n cat run-diff.md >> $GITHUB_STEP_SUMMARY\n\n - name: Comment on pull request\n uses: actions/github-script@v8\n with:\n script: |\n const fs = require('fs');\n const summaryPath = \"eval-summary.md\";\n const summary = fs.readFileSync(summaryPath, 'utf8');\n const runId = process.env.GITHUB_RUN_ID;\n const repo = process.env.GITHUB_REPOSITORY;\n const actionsUrl = `https://github.com/${repo}/actions/runs/${runId}`;\n github.rest.issues.createComment({\n issue_number: context.issue.number,\n owner: context.repo.owner,\n repo: context.repo.repo,\n body: `${summary}\\n\\n[Check the workflow run for more details](${actionsUrl}).`\n })\n" }, { "path": ".github/workflows/frontend.yaml", "content": "name: Frontend linting\n\non:\n push:\n branches: [ main ]\n paths:\n - \"app/frontend/**\"\n pull_request:\n branches: [ main ]\n paths:\n - \"app/frontend/**\"\n\njobs:\n prettier:\n runs-on: ubuntu-latest\n steps:\n - uses: actions/checkout@v6\n - name: Run prettier on frontend\n run: |\n cd ./app/frontend\n npm install\n npx prettier --check .\n" }, { "path": ".github/workflows/lint-markdown.yml", "content": "name: Validate Markdown\n\non:\n pull_request:\n branches:\n - main\n paths:\n - '**.md'\n\njobs:\n lint-markdown:\n name: Check for Markdown linting errors\n runs-on: ubuntu-latest\n steps:\n - name: Checkout repo\n uses: actions/checkout@v6\n - name: Run markdownlint-cli2\n uses: DavidAnson/markdownlint-cli2-action@v22\n with:\n config: .markdownlint-cli2.jsonc\n globs: |\n **/*.md\n !data/**\n !.github/**\n" }, { "path": ".github/workflows/nightly-jobs.yaml", "content": "name: Nightly Jobs\n\non:\n schedule:\n - cron: '0 0 * * *'\n workflow_dispatch:\n\njobs:\n python-test:\n uses: ./.github/workflows/python-test.yaml\n" }, { "path": ".github/workflows/python-test.yaml", "content": "name: Python check\n\non:\n push:\n branches: [ main ]\n paths-ignore:\n - \"**.md\"\n - \".azdo/**\"\n - \".devcontainer/**\"\n - \".github/**\"\n pull_request:\n branches: [ main ]\n paths-ignore:\n - \"**.md\"\n - \".azdo/**\"\n - \".devcontainer/**\"\n - \".github/**\"\n workflow_call:\n\njobs:\n test_package:\n name: Test ${{ matrix.os }} Python ${{ matrix.python_version }} Node ${{ matrix.node_version }}\n runs-on: ${{ matrix.os }}\n strategy:\n fail-fast: false\n matrix:\n os: [\"ubuntu-latest\", \"windows-latest\"]\n python_version: [\"3.10\", \"3.11\", \"3.12\", \"3.13\", \"3.14\"]\n node_version: [\"20.14\", \"22\"]\n steps:\n - uses: actions/checkout@v6\n with:\n # Fetch full history so diff-cover can compute a merge base with origin/main\n fetch-depth: 0\n - name: Install uv\n uses: astral-sh/setup-uv@v7\n with:\n enable-cache: true\n version: \"0.9.5\"\n cache-dependency-glob: \"requirements**.txt\"\n python-version: ${{ matrix.python_version }}\n activate-environment: true\n - name: Setup node\n uses: actions/setup-node@v6\n with:\n node-version: ${{ matrix.node_version }}\n - name: Build frontend\n run: |\n cd ./app/frontend\n npm install\n npm run build\n - name: Check i18n translations\n run: npx -y @lingual/i18n-check@0.8.12 --locales app/frontend/src/locales -s en -f i18next -r summary\n - name: Install dependencies\n run: |\n uv pip install -r requirements-dev.txt\n - name: Lint with ruff\n run: ruff check .\n - name: Check types with ty\n run: ty check\n - name: Check formatting with black\n run: black . --check --verbose\n - name: Run Python tests\n if: runner.os != 'Windows'\n run: pytest -s -vv --cov --cov-report=xml --cov-fail-under=90\n - name: Check diff coverage\n if: runner.os != 'Windows'\n run: |\n BASE_REF=\"${{ github.base_ref }}\"\n if [ -z \"$BASE_REF\" ]; then BASE_REF=\"main\"; fi\n git fetch origin \"$BASE_REF:refs/remotes/origin/$BASE_REF\"\n diff-cover coverage.xml --compare-branch=\"origin/$BASE_REF\" --fail-under=90\n - name: Run E2E tests with Playwright\n id: e2e\n if: runner.os != 'Windows'\n run: |\n playwright install chromium --with-deps\n pytest tests/e2e.py --tracing=retain-on-failure\n - name: Upload test artifacts\n if: ${{ failure() && steps.e2e.conclusion == 'failure' }}\n uses: actions/upload-artifact@v7\n with:\n name: playwright-traces${{ matrix.python_version }}\n path: test-results\n" }, { "path": ".github/workflows/stale-bot.yml", "content": "name: 'Close stale issues and PRs'\non:\n schedule:\n - cron: '30 1 * * *'\n\njobs:\n stale:\n runs-on: ubuntu-latest\n steps:\n - uses: actions/stale@v10\n with:\n stale-issue-message: 'This issue is stale because it has been open 60 days with no activity. Remove stale label or comment or this issue will be closed.'\n stale-pr-message: 'This PR is stale because it has been open 60 days with no activity. Remove stale label or comment or this will be closed.'\n close-issue-message: 'This issue was closed because it has been stalled for 7 days with no activity.'\n close-pr-message: 'This PR was closed because it has been stalled for 10 days with no activity.'\n days-before-issue-stale: 60\n days-before-pr-stale: 60\n days-before-issue-close: -1\n days-before-pr-close: -1\n" }, { "path": ".github/workflows/validate-markdown.yml", "content": "name: Validate Markdown\n\non:\n # Trigger the workflow on pull request\n pull_request_target:\n branches:\n - main\n paths:\n - '**.md'\n - '**.ipynb'\n\npermissions:\n contents: read\n pull-requests: write\n\njobs:\n check-broken-paths:\n name: Check Broken Relative Paths\n runs-on: ubuntu-latest\n steps:\n - name: Checkout Repo\n uses: actions/checkout@v6\n with:\n ref: ${{ github.event.pull_request.head.sha }}\n - name: Check broken Paths\n id: check-broken-paths\n uses: john0isaac/action-check-markdown@v1.1.0\n with:\n command: check_broken_paths\n directory: ./\n guide-url: 'https://github.com/Azure-Samples/azure-search-openai-demo/blob/main/CONTRIBUTING.md'\n github-token: ${{ secrets.GITHUB_TOKEN }}\n check-urls-locale:\n if: ${{ always() }}\n needs: check-broken-paths\n name: Check URLs Don't Have Locale\n runs-on: ubuntu-latest\n steps:\n - name: Checkout Repo\n uses: actions/checkout@v6\n with:\n ref: ${{ github.event.pull_request.head.sha }}\n - name: Run Check URLs Country Locale\n id: check-urls-locale\n uses: john0isaac/action-check-markdown@v1.1.0\n with:\n command: check_urls_locale\n directory: ./\n guide-url: 'https://github.com/Azure-Samples/azure-search-openai-demo/blob/main/CONTRIBUTING.md'\n github-token: ${{ secrets.GITHUB_TOKEN }}\n check-broken-urls:\n if: ${{ always() }}\n name: Check Broken URLs\n runs-on: ubuntu-latest\n steps:\n - name: Checkout Repo\n uses: actions/checkout@v6\n with:\n ref: ${{ github.event.pull_request.head.sha }}\n - name: Run Check Broken URLs\n id: check-broken-urls\n uses: john0isaac/action-check-markdown@v1.1.0\n with:\n command: check_broken_urls\n directory: ./\n guide-url: 'https://github.com/Azure-Samples/azure-search-openai-demo/blob/main/CONTRIBUTING.md'\n github-token: ${{ secrets.GITHUB_TOKEN }}\n" }, { "path": ".gitignore", "content": "# Azure az webapp deployment details\n.azure\n*_env\n\n# Byte-compiled / optimized / DLL files\n__pycache__/\n*.py[cod]\n*$py.class\n\n# C extensions\n*.so\n\n# Distribution / packaging\n.Python\nbuild/\ndevelop-eggs/\ndist/\ndownloads/\neggs/\n.eggs/\nlib/\nlib64/\nparts/\nsdist/\nvar/\nwheels/\nshare/python-wheels/\n*.egg-info/\n.installed.cfg\n*.egg\nMANIFEST\n\n# PyInstaller\n# Usually these files are written by a python script from a template\n# before PyInstaller builds the exe, so as to inject date/other infos into it.\n*.manifest\n*.spec\n\n# Installer logs\npip-log.txt\npip-delete-this-directory.txt\n\n# Unit test / coverage reports\nhtmlcov/\n.tox/\n.nox/\n.coverage\n.coverage.*\n.cache\nnosetests.xml\ncoverage.xml\n*.cover\n*.py,cover\n.hypothesis/\n.pytest_cache/\ncover/\ncoverage_report.html\n\n# Translations\n*.mo\n*.pot\n\n# Django stuff:\n*.log\nlocal_settings.py\ndb.sqlite3\ndb.sqlite3-journal\n\n# Flask stuff:\ninstance/\n.webassets-cache\n\n# Scrapy stuff:\n.scrapy\n\n# Sphinx documentation\ndocs/_build/\n\n# PyBuilder\n.pybuilder/\ntarget/\n\n# Jupyter Notebook\n.ipynb_checkpoints\n\n# IPython\nprofile_default/\nipython_config.py\n\n# pyenv\n# For a library or package, you might want to ignore these files since the code is\n# intended to run in multiple environments; otherwise, check them in:\n# .python-version\n\n# pipenv\n# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.\n# However, in case of collaboration, if having platform-specific dependencies or dependencies\n# having no cross-platform support, pipenv may install dependencies that don't work, or not\n# install all needed dependencies.\n#Pipfile.lock\n\n# PEP 582; used by e.g. github.com/David-OConnor/pyflow\n__pypackages__/\n\n# Celery stuff\ncelerybeat-schedule\ncelerybeat.pid\n\n# SageMath parsed files\n*.sage.py\n\n# Environments\n.env\n.venv\n.evalenv\nenv/\nvenv/\nENV/\nenv.bak/\nvenv.bak/\n\n# Spyder project settings\n.spyderproject\n.spyproject\n\n# Rope project settings\n.ropeproject\n\n# mkdocs documentation\n/site\n\n# mypy\n.mypy_cache/\n.dmypy.json\ndmypy.json\n\n# ty\n.ty_cache/\n\n# Pyre type checker\n.pyre/\n\n# pytype static type analyzer\n.pytype/\n\n# Cython debug symbols\ncython_debug/\n\n# NPM\nnpm-debug.log*\nnode_modules\nstatic/\n\napp/functions/*/prepdocslib/\napp/functions/*/requirements.txt\n\ndata/**/*.md5\n\n.DS_Store\n" }, { "path": ".markdownlint-cli2.jsonc", "content": "{\n \"config\": {\n \"default\": true,\n \"line-length\": false,\n \"table-column-style\": false,\n \"MD033\": { \"allowed_elements\": [\"br\", \"details\", \"summary\"] }\n }\n}\n" }, { "path": ".pre-commit-config.yaml", "content": "exclude: '^tests/snapshots/'\nrepos:\n- repo: https://github.com/pre-commit/pre-commit-hooks\n rev: v5.0.0\n hooks:\n - id: check-yaml\n - id: end-of-file-fixer\n - id: trailing-whitespace\n- repo: https://github.com/astral-sh/ruff-pre-commit\n rev: v0.14.2\n hooks:\n - id: ruff\n- repo: https://github.com/psf/black\n rev: 26.1.0\n hooks:\n - id: black\n- repo: https://github.com/pre-commit/mirrors-prettier\n rev: v3.1.0\n hooks:\n - id: prettier\n types_or: [css, javascript, ts, tsx, html]\n" }, { "path": ".vscode/extensions.json", "content": "{\n \"recommendations\": [\n \"ms-azuretools.azure-dev\",\n \"ms-azuretools.vscode-bicep\",\n \"ms-python.python\",\n \"astral-sh.ty\",\n \"esbenp.prettier-vscode\",\n \"DavidAnson.vscode-markdownlint\"\n ]\n}\n" }, { "path": ".vscode/launch.json", "content": "{\n // Use IntelliSense to learn about possible attributes.\n // Hover to view descriptions of existing attributes.\n // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387\n \"version\": \"0.2.0\",\n \"configurations\": [\n {\n \"name\": \"Backend (Python)\",\n \"type\": \"debugpy\",\n \"request\": \"launch\",\n \"module\": \"quart\",\n \"cwd\": \"${workspaceFolder}/app/backend\",\n // Use the currently selected interpreter in VS Code. MAC/Linux use /bin, Windows uses /Scripts\n \"python\": \"${command:python.interpreterPath}\",\n \"env\": {\n \"QUART_APP\": \"main:app\",\n \"QUART_ENV\": \"development\",\n \"QUART_DEBUG\": \"0\",\n // Set this to \"no-override\" if you want env vars here to override AZD env vars\n \"LOADING_MODE_FOR_AZD_ENV_VARS\": \"override\"\n },\n \"args\": [\n \"run\",\n \"--no-reload\",\n \"-p 50505\"\n ],\n \"console\": \"integratedTerminal\",\n \"justMyCode\": false\n },\n {\n \"name\": \"Frontend\",\n \"type\": \"node-terminal\",\n \"request\": \"launch\",\n \"command\": \"npm run dev\",\n \"cwd\": \"${workspaceFolder}/app/frontend\",\n },\n {\n \"name\": \"Tests (Python)\",\n \"type\": \"debugpy\",\n \"request\": \"launch\",\n \"program\": \"${file}\",\n \"purpose\": [\"debug-test\"],\n \"console\": \"integratedTerminal\",\n \"justMyCode\": false\n }\n ],\n \"compounds\": [\n {\n \"name\": \"Frontend & Backend\",\n \"configurations\": [\"Backend (Python)\", \"Frontend\"],\n \"stopAll\": true\n }\n ]\n}\n" }, { "path": ".vscode/settings.json", "content": "{\n \"python.languageServer\": \"None\", // Disabling due to ty using its own full-featured language server\n \"[javascript]\": {\n \"editor.defaultFormatter\": \"esbenp.prettier-vscode\",\n \"editor.formatOnSave\": true\n },\n \"[typescript]\": {\n \"editor.defaultFormatter\": \"esbenp.prettier-vscode\",\n \"editor.formatOnSave\": true\n },\n \"[typescriptreact]\": {\n \"editor.defaultFormatter\": \"esbenp.prettier-vscode\",\n \"editor.formatOnSave\": true\n },\n \"[css]\": {\n \"editor.defaultFormatter\": \"esbenp.prettier-vscode\",\n \"editor.formatOnSave\": true\n },\n \"files.exclude\": {\n \"**/__pycache__\": true,\n \"**/.coverage\": true,\n \"**/.pytest_cache\": true,\n \"**/.ruff_cache\": true,\n \"**/.mypy_cache\": true,\n \"**/.ty_cache\": true\n },\n \"search.exclude\": {\n \"**/node_modules\": true,\n \"static\": true\n },\n \"python.testing.pytestArgs\": [\n \"tests\"\n ],\n \"python.testing.unittestEnabled\": false,\n \"python.testing.pytestEnabled\": true\n}\n" }, { "path": ".vscode/tasks.json", "content": "{\n \"version\": \"2.0.0\",\n \"tasks\": [\n {\n \"label\": \"Start App\",\n \"type\": \"shell\",\n \"command\": \"${workspaceFolder}/app/start.sh\",\n \"windows\": {\n \"command\": \"pwsh ${workspaceFolder}/app/start.ps1\"\n },\n \"presentation\": {\n \"reveal\": \"silent\"\n },\n \"options\": {\n \"cwd\": \"${workspaceFolder}/app\"\n },\n \"problemMatcher\": []\n },\n {\n \"label\": \"Development\",\n \"dependsOn\": [\n \"Frontend: npm run dev\",\n \"Backend: quart run\"\n ],\n \"group\": {\n \"kind\": \"build\",\n \"isDefault\": true\n }\n },\n {\n \"label\": \"Frontend: npm run dev\",\n \"type\": \"npm\",\n \"script\": \"dev\",\n \"isBackground\": true,\n \"options\": {\n \"cwd\": \"${workspaceFolder}/app/frontend\"\n },\n \"presentation\": {\n \"reveal\": \"always\",\n \"group\": \"buildWatchers\",\n \"panel\": \"dedicated\",\n \"clear\": false\n },\n \"problemMatcher\": {\n \"pattern\": {\n \"regexp\": \"\"\n },\n \"background\": {\n \"activeOnStart\": true,\n \"beginsPattern\": \".*VITE v.*\",\n \"endsPattern\": \".*(?:➜\\\\s*)?Local:\\\\s+https?://.*\"\n }\n }\n },\n {\n \"label\": \"Backend: quart run\",\n \"type\": \"shell\",\n \"command\": \"${workspaceFolder}/.venv/bin/python\",\n \"windows\": {\n \"command\": \"${workspaceFolder}\\\\.venv\\\\Scripts\\\\python.exe\"\n },\n \"args\": [\"-m\", \"quart\", \"run\", \"--reload\", \"-p\", \"50505\"],\n \"options\": {\n \"cwd\": \"${workspaceFolder}/app/backend\",\n \"env\": {\n \"QUART_APP\": \"main:app\",\n \"QUART_ENV\": \"development\",\n \"QUART_DEBUG\": \"0\",\n \"LOADING_MODE_FOR_AZD_ENV_VARS\": \"override\"\n }\n },\n \"isBackground\": true,\n \"presentation\": {\n \"reveal\": \"always\",\n \"group\": \"buildWatchers\",\n \"panel\": \"dedicated\"\n },\n \"problemMatcher\": {\n \"pattern\": { \"regexp\": \"\" },\n \"background\": {\n \"activeOnStart\": true,\n \"beginsPattern\": \".*Serving Quart app.*\",\n \"endsPattern\": \".*hypercorn.*Running on http://.*\"\n }\n }\n }\n ]\n}\n" }, { "path": "AGENTS.md", "content": "# Instructions for Coding Agents\n\nThis file contains instructions for developers working on the Azure Search and OpenAI demo application. It covers the overall code layout, how to add new data, how to add new azd environment variables, how to add new developer settings, and how to add tests for new features.\n\nAlways keep this file up to date with any changes to the codebase or development process.\nIf necessary, edit this file to ensure it accurately reflects the current state of the project.\n\n## Overall code layout\n\n* app: Contains the main application code, including frontend and backend.\n * app/backend: Contains the Python backend code, written with Quart framework.\n * app/backend/approaches: Contains the different approaches\n * app/backend/approaches/approach.py: Base class for all approaches\n * app/backend/approaches/chatreadretrieveread.py: Chat approach, includes query rewriting step first\n * app/backend/approaches/promptmanager.py: Manages loading and rendering of Jinja2 prompt templates\n * app/backend/approaches/prompts/query_rewrite.system.jinja2: Jinja2 template used to rewrite the query based off search history into a better search query\n * app/backend/approaches/prompts/chat_query_rewrite_tools.json: Tools used by the query rewriting prompt\n * app/backend/approaches/prompts/chat_answer.system.jinja2: Jinja2 template for the system message used by the Chat approach to answer questions\n * app/backend/approaches/prompts/chat_answer.user.jinja2: Jinja2 template for the user message used by the Chat approach, including sources\n * app/backend/prepdocslib: Contains the document ingestion library used by both local and cloud ingestion\n * app/backend/prepdocslib/blobmanager.py: Manages uploads to Azure Blob Storage\n * app/backend/prepdocslib/cloudingestionstrategy.py: Builds the Azure AI Search indexer and skillset for the cloud ingestion pipeline\n * app/backend/prepdocslib/csvparser.py: Parses CSV files\n * app/backend/prepdocslib/embeddings.py: Generates embeddings for text and images using Azure OpenAI\n * app/backend/prepdocslib/figureprocessor.py: Generates figure descriptions for both local ingestion and the cloud figure-processor skill\n * app/backend/prepdocslib/fileprocessor.py: Orchestrates parsing and chunking of individual files\n * app/backend/prepdocslib/filestrategy.py: Strategy for uploading and indexing files (local ingestion)\n * app/backend/prepdocslib/htmlparser.py: Parses HTML files\n * app/backend/prepdocslib/integratedvectorizerstrategy.py: Strategy using Azure AI Search integrated vectorization\n * app/backend/prepdocslib/jsonparser.py: Parses JSON files\n * app/backend/prepdocslib/listfilestrategy.py: Lists files from local filesystem or Azure Data Lake\n * app/backend/prepdocslib/mediadescriber.py: Interfaces for describing images (Azure OpenAI GPT-4o, Content Understanding)\n * app/backend/prepdocslib/page.py: Data classes for pages, images, and chunks\n * app/backend/prepdocslib/parser.py: Base parser interface\n * app/backend/prepdocslib/pdfparser.py: Parses PDFs using Azure Document Intelligence or local parser\n * app/backend/prepdocslib/searchmanager.py: Manages Azure AI Search index creation and updates\n * app/backend/prepdocslib/servicesetup.py: Shared service setup helpers for OpenAI, embeddings, blob storage, etc.\n * app/backend/prepdocslib/strategy.py: Base strategy interface for document ingestion\n * app/backend/prepdocslib/textparser.py: Parses plain text and markdown files\n * app/backend/prepdocslib/textprocessor.py: Processes text chunks for cloud ingestion (merges figures, generates embeddings)\n * app/backend/prepdocslib/textsplitter.py: Splits text into chunks using different strategies\n * app/backend/app.py: The main entry point for the backend application.\n * app/functions: Azure Functions used for cloud ingestion custom skills (document extraction, figure processing, text processing). Each function bundles a synchronized copy of `prepdocslib`; run `python scripts/copy_prepdocslib.py` to refresh the local copies if you modify the library.\n * app/frontend: Contains the React frontend code, built with TypeScript, built with vite.\n * app/frontend/src/api: Contains the API client code for communicating with the backend.\n * app/frontend/src/components: Contains the React components for the frontend.\n * app/frontend/src/locales: Contains the translation files for internationalization.\n * app/frontend/src/locales/da/translation.json: Danish translations\n * app/frontend/src/locales/en/translation.json: English translations\n * app/frontend/src/locales/es/translation.json: Spanish translations\n * app/frontend/src/locales/fr/translation.json: French translations\n * app/frontend/src/locales/it/translation.json: Italian translations\n * app/frontend/src/locales/ja/translation.json: Japanese translations\n * app/frontend/src/locales/nl/translation.json: Dutch translations\n * app/frontend/src/locales/ptBR/translation.json: Portuguese translations\n * app/frontend/src/locales/tr/translation.json: Turkish translations\n * app/frontend/src/pages: Contains the main pages of the application\n* infra: Contains the Bicep templates for provisioning Azure resources.\n* tests: Contains the test code, including e2e tests, app integration tests, and unit tests.\n\n## Adding new data\n\nNew files should be added to the `data` folder, and then either run scripts/prepdocs.sh or scripts/prepdocs.ps1 to ingest the data.\n\n## Adding a new azd environment variable\n\nAn azd environment variable is stored by the azd CLI for each environment. It is passed to the \"azd up\" command and can configure both provisioning options and application settings.\nWhen adding new azd environment variables, update:\n\n1. infra/main.parameters.json : Add the new parameter with a Bicep-friendly variable name and map to the new environment variable\n1. infra/main.bicep: Add the new Bicep parameter at the top, and add it to the `appEnvVariables` object\n1. .azdo/pipelines/azure-dev.yml: Add the new environment variable under `env` section\n1. .github/workflows/azure-dev.yml: Add the new environment variable under `env` section\n\nYou may also need to update:\n\n1. app/backend/prepdocs.py: If the variable is used in the ingestion script, retrieve it from environment variables here. Not always needed.\n1. app/backend/app.py: If the variable is used in the backend application, retrieve it from environment variables in setup_clients() function. Not always needed.\n\n## Adding a new setting to \"Developer Settings\" in RAG app\n\nWhen adding a new developer setting, update:\n\n* frontend:\n * app/frontend/src/api/models.ts : Add to ChatAppRequestOverrides\n * app/frontend/src/components/Settings.tsx : Add a UI element for the setting\n * app/frontend/src/locales/*/translations.json: Add a translation for the setting label/tooltip for all languages\n * app/frontend/src/pages/chat/Chat.tsx: Add the setting to the component, pass it to Settings\n\n* backend:\n * app/backend/approaches/chatreadretrieveread.py : Retrieve from overrides parameter\n * app/backend/app.py: Some settings may need to be sent down in the /config route.\n\n## When adding tests for a new feature\n\nAll tests are in the `tests` folder and use the pytest framework.\nThere are three styles of tests:\n\n* e2e tests: These use playwright to run the app in a browser and test the UI end-to-end. They are in e2e.py and they mock the backend using the snapshots from the app tests. (Before running e2e tests, make sure to run `npm run build` in app/frontend first to build the frontend code.)\n* app integration tests: Mostly in test_app.py, these test the app's API endpoints and use mocks for services like Azure OpenAI and Azure Search.\n* unit tests: The rest of the tests are unit tests that test individual functions and methods. They are in test_*.py files.\n\nWhen adding a new feature, add tests for it in the appropriate file.\nIf the feature is a UI element, add an e2e test for it.\nIf it is an API endpoint, add an app integration test for it.\nIf it is a function or method, add a unit test for it.\nUse mocks from tests/conftest.py to mock external services. Prefer mocking at the HTTP/requests level when possible.\n\nWhen you're running tests, make sure you activate the .venv virtual environment first:\n\n```shell\nsource .venv/bin/activate\n```\n\nTo check for coverage, run the following command:\n\n```shell\npytest --cov --cov-report=annotate:cov_annotate\n```\n\nOpen the cov_annotate directory to view the annotated source code. There will be one file per source file. If a file has 100% source coverage, it means all lines are covered by tests, so you do not need to open the file.\n\nFor each file that has less than 100% test coverage, find the matching file in cov_annotate and review the file.\n\nIf a line starts with a ! (exclamation mark), it means that the line is not covered by tests. Add tests to cover the missing lines.\n\n## Sending pull requests\n\nWhen sending pull requests, make sure to follow the PULL_REQUEST_TEMPLATE.md format.\n\n## Upgrading dependencies\n\n### Python backend dependencies\n\nTo upgrade a particular package in the backend, use the following command, replacing `` with the name of the package you want to upgrade:\n\n```shell\ncd app/backend && uv pip compile requirements.in -o requirements.txt --python-version 3.10 --upgrade-package \n```\n\nAfter upgrading, run tests to verify compatibility:\n\n```shell\nsource .venv/bin/activate\npytest tests/\n```\n\n### npm frontend dependencies\n\nTo upgrade a particular package in the frontend:\n\n1. **Navigate to the frontend directory**:\n\n ```shell\n cd app/frontend\n ```\n\n2. **Upgrade the package** (replace `` with the package you want to upgrade):\n\n ```shell\n npm install @latest\n ```\n\n3. **Build the frontend** to verify the upgrade works:\n\n ```shell\n npm run build\n ```\n\n4. **Run all tests** to ensure nothing broke:\n\n ```shell\n # Run e2e tests from the root directory\n cd ../..\n source .venv/bin/activate\n pytest tests/e2e.py\n ```\n\n5. **Commit changes** if the upgrade is successful:\n\n ```shell\n git add package.json package-lock.json\n git commit -m \"chore: upgrade to \"\n ```\n\n**Important notes for frontend upgrades**:\n\n* When upgrading React or related core packages, you may need to upgrade multiple packages together (e.g., `react`, `react-dom`, `@types/react`, `@types/react-dom`)\n* Some upgrades may require code changes for API compatibility - check the package's changelog\n* For major version upgrades of UI libraries like Fluent UI or MSAL, review breaking changes carefully. Manual tests are required for any MSAL changes since the E2E tests do not cover authentication flows.\n* If npm reports peer dependency conflicts, the `.npmrc` file has `legacy-peer-deps=true` which allows the install to proceed. This is currently needed because `react-helmet-async` declares peer dependencies on React 17/18, but works fine with React 19.\n\n## Checking Python type hints\n\nTo check Python type hints, use the following command:\n\n```shell\nty check\n```\n\nNote that we do not currently enforce type hints in the tests folder, as it would require adding a lot of `# type: ignore` comments to the existing tests.\nWe only enforce type hints in the main application code and scripts.\n\n## Python code style\n\nDo not use single underscores in front of \"private\" methods or variables in Python code. We do not follow that convention in this codebase, since this is an application and not a library.\n\n## Deploying the application\n\nTo deploy the application, use the `azd` CLI tool. Make sure you have the latest version of the `azd` CLI installed. Then, run the following command from the root of the repository:\n\n```shell\nazd up\n```\n\nThat command will BOTH provision the Azure resources AND deploy the application code.\n\nIf you only changed the Bicep templates and want to re-provision the Azure resources, run:\n\n```shell\nazd provision\n```\n\nIf you only changed the application code and want to re-deploy the code, run:\n\n```shell\nazd deploy\n```\n\nIf you are using cloud ingestion and only want to deploy individual functions, run the necessary deploy commands, for example:\n\n```shell\nazd deploy document-extractor\nazd deploy figure-processor\nazd deploy text-processor\n```\n" }, { "path": "CONTRIBUTING.md", "content": "# Contributing\n\nThis project welcomes contributions and suggestions. Most contributions require you to agree to a\nContributor License Agreement (CLA) declaring that you have the right to, and actually do, grant us\nthe rights to use your contribution. For details, visit .\n\nWhen you submit a pull request, a CLA bot will automatically determine whether you need to provide\na CLA and decorate the PR appropriately (e.g., status check, comment). Simply follow the instructions\nprovided by the bot. You will only need to do this once across all repos using our CLA.\n\nThis project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/).\nFor more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or\ncontact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments.\n\n- [Submitting a Pull Request (PR)](#submitting-a-pull-request-pr)\n- [Setting up the development environment](#setting-up-the-development-environment)\n- [Running unit tests](#running-unit-tests)\n- [Running E2E tests](#running-e2e-tests)\n- [Code style](#code-style)\n- [Adding new features](#adding-new-features)\n - [Adding new azd environment variables](#adding-new-azd-environment-variables)\n - [Adding new UI strings](#adding-new-ui-strings)\n\n## Submitting a Pull Request (PR)\n\nBefore you submit your Pull Request (PR) consider the following guidelines:\n\n- Search the repository (]/[repository-name]/pulls) for an open or closed PR\n that relates to your submission. You don't want to duplicate effort.\n- Make your changes in a new git fork\n- Follow [Code style conventions](#code-style)\n- [Run the tests](#running-unit-tests) (and write new ones, if needed)\n- Commit your changes using a descriptive commit message\n- Push your fork to GitHub\n- In GitHub, create a pull request to the `main` branch of the repository\n- Ask a maintainer to review your PR and address any comments they might have\n\n## Setting up the development environment\n\nInstall the development dependencies:\n\n```shell\npython -m pip install -r requirements-dev.txt\n```\n\nInstall the pre-commit hooks:\n\n```shell\npre-commit install\n```\n\nCompile the JavaScript:\n\n```shell\n( cd ./app/frontend ; npm install ; npm run build )\n```\n\n## Running unit tests\n\nRun the tests:\n\n```shell\npython -m pytest\n```\n\nIf test snapshots need updating (and the changes are expected), you can update them by running:\n\n```shell\npython -m pytest --snapshot-update\n```\n\nOnce tests are passing, generate a coverage report to make sure your changes are covered:\n\n```shell\npytest --cov --cov-report=xml && \\\ndiff-cover coverage.xml --html-report coverage_report.html && \\\nopen coverage_report.html\n```\n\n## Running E2E tests\n\nInstall Playwright browser dependencies:\n\n```shell\nplaywright install --with-deps\n```\n\nRun the tests:\n\n```shell\npython -m pytest tests/e2e.py --tracing=retain-on-failure\n```\n\nWhen a failure happens, the trace zip will be saved in the test-results folder.\nYou can view that using the Playwright CLI:\n\n```shell\nplaywright show-trace test-results/\n```\n\nYou can also use the online trace viewer at \n\n## Code style\n\nThis codebase includes several languages: TypeScript, Python, Bicep, Powershell, and Bash.\nCode should follow the standard conventions of each language.\n\nFor Python, you can enforce the conventions using `ruff` and `black`.\n\nInstall the development dependencies:\n\n```shell\npython -m pip install -r requirements-dev.txt\n```\n\nRun `ruff` to lint a file:\n\n```shell\npython -m ruff \n```\n\nRun `black` to format a file:\n\n```shell\npython -m black \n```\n\nIf you followed the steps above to install the pre-commit hooks, then you can just wait for those hooks to run `ruff` and `black` for you.\n\n## Adding new features\n\nWe recommend using GitHub Copilot Agent mode when adding new features,\nas this project includes an [AGENTS.md](AGENTS.md) file\nthat instructs Copilot (and other coding agents) about how to generate code for common code changes.\n\nIf you are not using Copilot Agent mode, consult both that file and suggestions below.\n\n### Adding new azd environment variables\n\nWhen adding new azd environment variables, please remember to update:\n\n1. [main.parameters.json](./infra/main.parameters.json)\n1. [appEnvVariables in main.bicep](./infra/main.bicep)\n1. [ADO pipeline](.azdo/pipelines/azure-dev.yml).\n1. [Github workflows](.github/workflows/azure-dev.yml)\n\n### Adding new UI strings\n\nWhen adding new UI strings, please remember to update all translations.\nFor any translations that you generate with an AI tool,\nplease indicate in the PR description which language's strings were AI-generated.\n\nHere are community contributors that can review translations:\n\n| Language | Contributor |\n|----------|---------------------|\n| Danish | @EMjetrot |\n| French | @manekinekko |\n| Japanese | @bnodir |\n| Norwegian| @@jeannotdamoiseaux |\n| Portugese| @glaucia86 |\n| Spanish | @miguelmsft |\n| Turkish | @mertcakdogan |\n| Italian | @ivanvaccarics |\n| Dutch | |\n| Polish | @michuhu |\n" }, { "path": "LICENSE", "content": "MIT License\n\nCopyright (c) 2023 Azure Samples\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the \"Software\"), to deal\nin the Software without restriction, including without limitation the rights\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\ncopies of the Software, and to permit persons to whom the Software is\nfurnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\nSOFTWARE.\n" }, { "path": "README.md", "content": "\n\n# RAG chat app with Azure OpenAI and Azure AI Search (Python)\n\nThis solution creates a ChatGPT-like frontend experience over your own documents using RAG (Retrieval Augmented Generation). It uses Azure OpenAI Service to access GPT models, and Azure AI Search for data indexing and retrieval.\n\nThis solution's backend is written in Python. There are also [**JavaScript**](https://aka.ms/azai/js/code), [**.NET**](https://aka.ms/azai/net/code), and [**Java**](https://aka.ms/azai/java/code) samples based on this one. Learn more about [developing AI apps using Azure AI Services](https://aka.ms/azai).\n\n[![Open in GitHub Codespaces](https://img.shields.io/static/v1?style=for-the-badge&label=GitHub+Codespaces&message=Open&color=brightgreen&logo=github)](https://github.com/codespaces/new?hide_repo_select=true&ref=main&repo=599293758&machine=standardLinux32gb&devcontainer_path=.devcontainer%2Fdevcontainer.json&location=WestUs2)\n[![Open in Dev Containers](https://img.shields.io/static/v1?style=for-the-badge&label=Dev%20Containers&message=Open&color=blue&logo=visualstudiocode)](https://vscode.dev/redirect?url=vscode://ms-vscode-remote.remote-containers/cloneInVolume?url=https://github.com/azure-samples/azure-search-openai-demo)\n[![Open in VS Code for the Web](https://img.shields.io/static/v1?style=for-the-badge&label=VS+Code+for+the+Web&message=Open&color=purple&logo=visualstudiocode)](https://vscode.dev/azure?azdTemplateUrl=https://github.com/azure-samples/azure-search-openai-demo)\n\n## Important Security Notice\n\nThis template, the application code and configuration it contains, has been built to showcase Microsoft Azure specific services and tools. We strongly advise our customers not to make this code part of their production environments without implementing or enabling additional security features. See our [productionizing guide](docs/productionizing.md) for tips, and consult the [Azure OpenAI Landing Zone reference architecture](https://techcommunity.microsoft.com/blog/azurearchitectureblog/azure-openai-landing-zone-reference-architecture/3882102) for more best practices.\n\n## Table of Contents\n\n- [Features](#features)\n- [Azure account requirements](#azure-account-requirements)\n - [Cost estimation](#cost-estimation)\n- [Getting Started](#getting-started)\n - [GitHub Codespaces](#github-codespaces)\n - [VS Code Dev Containers](#vs-code-dev-containers)\n - [Local environment](#local-environment)\n- [Deploying](#deploying)\n - [Deploying again](#deploying-again)\n- [Running the development server](#running-the-development-server)\n- [Using the app](#using-the-app)\n- [Clean up](#clean-up)\n- [Guidance](#guidance)\n - [Resources](#resources)\n\n![Chat screen](docs/images/chatscreen.png)\n\n[📺 Watch a video overview of the app.](https://youtu.be/3acB0OWmLvM)\n\nThis sample demonstrates a few approaches for creating ChatGPT-like experiences over your own data using the Retrieval Augmented Generation pattern. It uses Azure OpenAI Service to access a GPT model (gpt-4.1-mini), and Azure AI Search for data indexing and retrieval.\n\nThe repo includes sample data so it's ready to try end to end. In this sample application we use a fictitious company called Zava, and the experience allows its employees to ask questions about the benefits, internal policies, as well as job descriptions and roles.\n\n## Features\n\n- Chat (multi-turn) interface\n- Renders citations and thought process for each answer\n- Includes settings directly in the UI to tweak the behavior and experiment with options\n- Integrates Azure AI Search for indexing and retrieval of documents, with support for [many document formats](/docs/data_ingestion.md#supported-document-formats) as well as [cloud data ingestion](/docs/data_ingestion.md#cloud-data-ingestion)\n- Optional usage of [multimodal models](/docs/multimodal.md) to reason over image-heavy documents\n- Optional addition of [speech input/output](/docs/deploy_features.md#enabling-speech-inputoutput) for accessibility\n- Optional automation of [user login and data access](/docs/login_and_acl.md) via Microsoft Entra\n- Performance tracing and monitoring with Application Insights\n\n### Architecture Diagram\n\n![RAG Architecture](docs/images/appcomponents.png)\n\n## Azure account requirements\n\n**IMPORTANT:** In order to deploy and run this example, you'll need:\n\n- **Azure account**. If you're new to Azure, [get an Azure account for free](https://azure.microsoft.com/free/cognitive-search/) and you'll get some free Azure credits to get started. See [guide to deploying with the free trial](docs/deploy_freetrial.md).\n- **Azure account permissions**:\n - Your Azure account must have `Microsoft.Authorization/roleAssignments/write` permissions, such as [Role Based Access Control Administrator](https://learn.microsoft.com/azure/role-based-access-control/built-in-roles#role-based-access-control-administrator-preview), [User Access Administrator](https://learn.microsoft.com/azure/role-based-access-control/built-in-roles#user-access-administrator), or [Owner](https://learn.microsoft.com/azure/role-based-access-control/built-in-roles#owner). If you don't have subscription-level permissions, you must be granted [RBAC](https://learn.microsoft.com/azure/role-based-access-control/built-in-roles#role-based-access-control-administrator-preview) for an existing resource group and [deploy to that existing group](docs/deploy_existing.md#resource-group).\n - Your Azure account also needs `Microsoft.Resources/deployments/write` permissions on the subscription level.\n\n### Cost estimation\n\nPricing varies per region and usage, so it isn't possible to predict exact costs for your usage.\nHowever, you can try the [Azure pricing calculator](https://azure.com/e/e3490de2372a4f9b909b0d032560e41b) for the resources below.\n\n- Azure Container Apps: Default host for app deployment as of 10/28/2024. See more details in [the ACA deployment guide](docs/azure_container_apps.md). Consumption plan with 1 CPU core, 2 GB RAM, minimum of 0 replicas. Pricing with Pay-as-You-Go. [Pricing](https://azure.microsoft.com/pricing/details/container-apps/)\n- Azure Container Registry: Basic tier. [Pricing](https://azure.microsoft.com/pricing/details/container-registry/)\n- Azure App Service: Only provisioned if you deploy to Azure App Service following [the App Service deployment guide](docs/azure_app_service.md). Basic Tier with 1 CPU core, 1.75 GB RAM. Pricing per hour. [Pricing](https://azure.microsoft.com/pricing/details/app-service/linux/)\n- Azure OpenAI: Standard tier, GPT and Ada models. Pricing per 1K tokens used, and at least 1K tokens are used per question. [Pricing](https://azure.microsoft.com/pricing/details/cognitive-services/openai-service/)\n- Azure AI Document Intelligence: SO (Standard) tier using pre-built layout. Pricing per document page, sample documents have 261 pages total. [Pricing](https://azure.microsoft.com/pricing/details/form-recognizer/)\n- Azure AI Search: Basic tier, 1 replica, free level of semantic search. Pricing per hour. [Pricing](https://azure.microsoft.com/pricing/details/search/)\n- Azure Blob Storage: Standard tier with ZRS (Zone-redundant storage). Pricing per storage and read operations. [Pricing](https://azure.microsoft.com/pricing/details/storage/blobs/)\n- Azure Cosmos DB: Only provisioned if you enabled [chat history with Cosmos DB](docs/deploy_features.md#enabling-persistent-chat-history-with-azure-cosmos-db). Serverless tier. Pricing per request unit and storage. [Pricing](https://azure.microsoft.com/pricing/details/cosmos-db/)\n- Azure AI Vision: Only provisioned if you enabled [multimodal approach](docs/multimodal.md). Pricing per 1K transactions. [Pricing](https://azure.microsoft.com/pricing/details/cognitive-services/computer-vision/)\n- Azure AI Content Understanding: Only provisioned if you enabled [media description](docs/deploy_features.md#enabling-media-description-with-azure-content-understanding). Pricing per 1K images. [Pricing](https://azure.microsoft.com/pricing/details/content-understanding/)\n- Azure Monitor: Pay-as-you-go tier. Costs based on data ingested. [Pricing](https://azure.microsoft.com/pricing/details/monitor/)\n\nTo reduce costs, you can switch to free SKUs for various services, but those SKUs have limitations.\nSee this guide on [deploying with minimal costs](docs/deploy_lowcost.md) for more details.\n\n⚠️ To avoid unnecessary costs, remember to take down your app if it's no longer in use,\neither by deleting the resource group in the Portal or running `azd down`.\n\n## Getting Started\n\nYou have a few options for setting up this project.\nThe easiest way to get started is GitHub Codespaces, since it will setup all the tools for you,\nbut you can also [set it up locally](#local-environment) if desired.\n\n### GitHub Codespaces\n\nYou can run this repo virtually by using GitHub Codespaces, which will open a web-based VS Code in your browser:\n\n[![Open in GitHub Codespaces](https://img.shields.io/static/v1?style=for-the-badge&label=GitHub+Codespaces&message=Open&color=brightgreen&logo=github)](https://github.com/codespaces/new?hide_repo_select=true&ref=main&repo=599293758&machine=standardLinux32gb&devcontainer_path=.devcontainer%2Fdevcontainer.json&location=WestUs2)\n\nOnce the codespace opens (this may take several minutes), open a terminal window.\n\n### VS Code Dev Containers\n\nA related option is VS Code Dev Containers, which will open the project in your local VS Code using the [Dev Containers extension](https://marketplace.visualstudio.com/items?itemName=ms-vscode-remote.remote-containers):\n\n1. Start Docker Desktop (install it if not already installed)\n2. Open the project:\n [![Open in Dev Containers](https://img.shields.io/static/v1?style=for-the-badge&label=Dev%20Containers&message=Open&color=blue&logo=visualstudiocode)](https://vscode.dev/redirect?url=vscode://ms-vscode-remote.remote-containers/cloneInVolume?url=https://github.com/azure-samples/azure-search-openai-demo)\n\n3. In the VS Code window that opens, once the project files show up (this may take several minutes), open a terminal window.\n\n### Local environment\n\n1. Install the required tools:\n\n - [Azure Developer CLI](https://aka.ms/azure-dev/install)\n - [Python 3.10, 3.11, 3.12, 3.13, or 3.14](https://www.python.org/downloads/)\n - **Important**: Python and the pip package manager must be in the path in Windows for the setup scripts to work.\n - **Important**: Ensure you can run `python --version` from console. On Ubuntu, you might need to run `sudo apt install python-is-python3` to link `python` to `python3`.\n - [Node.js 20+](https://nodejs.org/download/)\n - [Git](https://git-scm.com/downloads)\n - [Powershell 7+ (pwsh)](https://github.com/powershell/powershell) - For Windows users only.\n - **Important**: Ensure you can run `pwsh.exe` from a PowerShell terminal. If this fails, you likely need to upgrade PowerShell.\n\n2. Create a new folder and switch to it in the terminal.\n3. Run this command to download the project code:\n\n ```shell\n azd init -t azure-search-openai-demo\n ```\n\n Note that this command will initialize a git repository, so you do not need to clone this repository.\n\n## Deploying\n\nThe steps below will provision Azure resources and deploy the application code to Azure Container Apps. To deploy to Azure App Service instead, follow [the app service deployment guide](docs/azure_app_service.md).\n\n1. Login to your Azure account:\n\n ```shell\n azd auth login\n ```\n\n For GitHub Codespaces users, if the previous command fails, try:\n\n ```shell\n azd auth login --use-device-code\n ```\n\n1. Create a new azd environment:\n\n ```shell\n azd env new\n ```\n\n Enter a name that will be used for the resource group.\n This will create a new folder in the `.azure` folder, and set it as the active environment for any calls to `azd` going forward.\n1. (Optional) This is the point where you can customize the deployment by setting environment variables, in order to [use existing resources](docs/deploy_existing.md), [enable optional features (such as auth or vision)](docs/deploy_features.md), or [deploy low-cost options](docs/deploy_lowcost.md), or [deploy with the Azure free trial](docs/deploy_freetrial.md).\n1. Run `azd up` - This will provision Azure resources and deploy this sample to those resources, including building the search index based on the files found in the `./data` folder.\n - **Important**: Beware that the resources created by this command will incur immediate costs, primarily from the AI Search resource. These resources may accrue costs even if you interrupt the command before it is fully executed. You can run `azd down` or delete the resources manually to avoid unnecessary spending.\n - You will be prompted to select two locations, one for the majority of resources and one for the OpenAI resource, which is currently a short list. That location list is based on the [OpenAI model availability table](https://learn.microsoft.com/azure/cognitive-services/openai/concepts/models#model-summary-table-and-region-availability) and may become outdated as availability changes.\n1. After the application has been successfully deployed you will see a URL printed to the console. Click that URL to interact with the application in your browser.\nIt will look like the following:\n\n!['Output from running azd up'](docs/images/endpoint.png)\n\n> NOTE: It may take 5-10 minutes after you see 'SUCCESS' for the application to be fully deployed. If you see a \"Python Developer\" welcome screen or an error page, then wait a bit and refresh the page.\n\n### Deploying again\n\nIf you've only changed the backend/frontend code in the `app` folder, then you don't need to re-provision the Azure resources. You can just run:\n\n```shell\nazd deploy\n```\n\nIf you've changed the infrastructure files (`infra` folder or `azure.yaml`), then you'll need to re-provision the Azure resources. You can do that by running:\n\n```shell\nazd up\n```\n\n## Running the development server\n\nYou can only run a development server locally **after** having successfully run the `azd up` command. If you haven't yet, follow the [deploying](#deploying) steps above.\n\n1. Run `azd auth login` if you have not logged in recently.\n2. Start the server:\n\n Windows:\n\n ```shell\n ./app/start.ps1\n ```\n\n Linux/Mac:\n\n ```shell\n ./app/start.sh\n ```\n\n VS Code: Run the \"VS Code Task: Start App\" task.\n\nIt's also possible to enable hotloading or the VS Code debugger.\nSee more tips in [the local development guide](docs/localdev.md).\n\n## Using the app\n\n- In Azure: navigate to the Azure WebApp deployed by azd. The URL is printed out when azd completes (as \"Endpoint\"), or you can find it in the Azure portal.\n- Running locally: navigate to 127.0.0.1:50505\n\nOnce in the web app:\n\n- Try different topics in chat. Try follow up questions, clarifications, ask to simplify or elaborate on answer, etc.\n- Explore citations and sources\n- Click on \"settings\" to try different options, tweak prompts, etc.\n\n## Clean up\n\nTo clean up all the resources created by this sample:\n\n1. Run `azd down`\n2. When asked if you are sure you want to continue, enter `y`\n3. When asked if you want to permanently delete the resources, enter `y`\n\nThe resource group and all the resources will be deleted.\n\n## Guidance\n\nYou can find extensive documentation in the [docs](docs/README.md) folder:\n\n- Deploying:\n - [Troubleshooting deployment](docs/deploy_troubleshooting.md)\n - [Debugging the app on App Service](docs/appservice.md)\n - [Deploying with azd: deep dive and CI/CD](docs/azd.md)\n - [Deploying with existing Azure resources](docs/deploy_existing.md)\n - [Deploying from a free account](docs/deploy_lowcost.md)\n - [Enabling optional features](docs/deploy_features.md)\n - [All features](docs/deploy_features.md)\n - [Login and access control](docs/login_and_acl.md)\n - [Multimodal](docs/multimodal.md)\n - [Reasoning](docs/reasoning.md)\n - [Private endpoints](docs/deploy_private.md)\n - [Agentic retrieval](docs/agentic_retrieval.md)\n - [Sharing deployment environments](docs/sharing_environments.md)\n- [Local development](docs/localdev.md)\n- [Customizing the app](docs/customization.md)\n- [App architecture](docs/architecture.md)\n- [HTTP Protocol](docs/http_protocol.md)\n- [Data ingestion](docs/data_ingestion.md)\n- [Evaluation](docs/evaluation.md)\n- [Safety evaluation](docs/safety_evaluation.md)\n- [Monitoring with Application Insights](docs/monitoring.md)\n- [Productionizing](docs/productionizing.md)\n- [Alternative RAG chat samples](docs/other_samples.md)\n\n### Resources\n\n- [📖 Docs: Get started using the chat with your data sample](https://learn.microsoft.com/azure/developer/python/get-started-app-chat-template?toc=%2Fazure%2Fdeveloper%2Fai%2Ftoc.json&bc=%2Fazure%2Fdeveloper%2Fai%2Fbreadcrumb%2Ftoc.json&tabs=github-codespaces)\n- [📖 Blog: Revolutionize your Enterprise Data with ChatGPT: Next-gen Apps w/ Azure OpenAI and AI Search](https://techcommunity.microsoft.com/blog/azure-ai-services-blog/revolutionize-your-enterprise-data-with-chatgpt-next-gen-apps-w-azure-openai-and/3762087)\n- [📖 Docs: Azure AI Search](https://learn.microsoft.com/azure/search/search-what-is-azure-search)\n- [📖 Docs: Azure OpenAI Service](https://learn.microsoft.com/azure/cognitive-services/openai/overview)\n- [📖 Docs: Comparing Azure OpenAI and OpenAI](https://learn.microsoft.com/azure/cognitive-services/openai/overview#comparing-azure-openai-and-openai/)\n- [📖 Blog: Access Control in Generative AI applications with Azure AI Search](https://techcommunity.microsoft.com/blog/azure-ai-services-blog/access-control-in-generative-ai-applications-with-azure-ai-search/3956408)\n- [📺 Talk: Quickly build and deploy OpenAI apps on Azure, infused with your own data](https://www.youtube.com/watch?v=j8i-OM5kwiY)\n- [📺 Video: RAG Deep Dive Series](https://techcommunity.microsoft.com/blog/azuredevcommunityblog/rag-deep-dive-watch-all-the-recordings/4383171)\n\n### Getting help\n\nThis is a sample built to demonstrate the capabilities of modern Generative AI apps and how they can be built in Azure.\nFor help with deploying this sample, please post in [GitHub Issues](/issues). If you're a Microsoft employee, you can also post in [our Teams channel](https://aka.ms/azai-python-help).\n\nThis repository is supported by the maintainers, _not_ by Microsoft Support,\nso please use the support mechanisms described above, and we will do our best to help you out.\n\nFor general questions about developing AI solutions on Azure,\njoin the Azure AI Foundry Developer Community:\n\n[![Azure AI Foundry Discord](https://img.shields.io/badge/Discord-Azure_AI_Foundry_Community_Discord-blue?style=for-the-badge&logo=discord&color=5865f2&logoColor=fff)](https://aka.ms/foundry/discord)\n[![Azure AI Foundry Developer Forum](https://img.shields.io/badge/GitHub-Azure_AI_Foundry_Developer_Forum-blue?style=for-the-badge&logo=github&color=000000&logoColor=fff)](https://aka.ms/foundry/forum)\n\n### Note\n\n>Note: The PDF documents used in this demo contain information generated using a language model (Azure OpenAI Service). The information contained in these documents is only for demonstration purposes and does not reflect the opinions or beliefs of Microsoft. Microsoft makes no representations or warranties of any kind, express or implied, about the completeness, accuracy, reliability, suitability or availability with respect to the information contained in this document. All rights reserved to Microsoft.\n" }, { "path": "SECURITY.md", "content": "\n# Security\n\nMicrosoft takes the security of our software products and services seriously, which includes all source code repositories managed through our GitHub organizations, which include [Microsoft](https://github.com/Microsoft), [Azure](https://github.com/Azure), [DotNet](https://github.com/dotnet), [AspNet](https://github.com/aspnet), [Xamarin](https://github.com/xamarin), and [our GitHub organizations](https://opensource.microsoft.com/).\n\nIf you believe you have found a security vulnerability in any Microsoft-owned repository that meets [Microsoft's definition of a security vulnerability](), please report it to us as described below.\n\n## Reporting Security Issues\n\n**Please do not report security vulnerabilities through public GitHub issues.**\n\nInstead, please report them to the Microsoft Security Response Center (MSRC) at [https://msrc.microsoft.com/create-report](https://msrc.microsoft.com/create-report).\n\nIf you prefer to submit without logging in, send email to [secure@microsoft.com](mailto:secure@microsoft.com). If possible, encrypt your message with our PGP key; please download it from the [Microsoft Security Response Center PGP Key page](https://www.microsoft.com/msrc/pgp-key-msrc).\n\nYou should receive a response within 24 hours. If for some reason you do not, please follow up via email to ensure we received your original message. Additional information can be found at [microsoft.com/msrc](https://www.microsoft.com/msrc).\n\nPlease include the requested information listed below (as much as you can provide) to help us better understand the nature and scope of the possible issue:\n\n- Type of issue (e.g. buffer overflow, SQL injection, cross-site scripting, etc.)\n- Full paths of source file(s) related to the manifestation of the issue\n- The location of the affected source code (tag/branch/commit or direct URL)\n- Any special configuration required to reproduce the issue\n- Step-by-step instructions to reproduce the issue\n- Proof-of-concept or exploit code (if possible)\n- Impact of the issue, including how an attacker might exploit the issue\n\nThis information will help us triage your report more quickly.\n\nIf you are reporting for a bug bounty, more complete reports can contribute to a higher bounty award. Please visit our [Microsoft Bug Bounty Program](https://microsoft.com/msrc/bounty) page for more details about our active programs.\n\n## Preferred Languages\n\nWe prefer all communications to be in English.\n\n## Policy\n\nMicrosoft follows the principle of [Coordinated Vulnerability Disclosure](https://www.microsoft.com/msrc/cvd).\n\n\n" }, { "path": "app/backend/.dockerignore", "content": ".git\n__pycache__\n*.pyc\n*.pyo\n*.pyd\n.Python\nenv\n" }, { "path": "app/backend/Dockerfile", "content": "FROM python:3.13-bookworm\n\nWORKDIR /app\n\nCOPY ./ /app\n\nRUN python -m pip install -r requirements.txt\n\nRUN python -m pip install gunicorn\n\nCMD [\"python3\", \"-m\", \"gunicorn\", \"-b\", \"0.0.0.0:8000\", \"main:app\"]\n" }, { "path": "app/backend/app.py", "content": "import dataclasses\nimport io\nimport json\nimport logging\nimport mimetypes\nimport os\nimport time\nfrom collections.abc import AsyncGenerator, Awaitable, Callable\nfrom pathlib import Path\nfrom typing import Any, cast\n\nfrom azure.cognitiveservices.speech import (\n ResultReason,\n SpeechConfig,\n SpeechSynthesisOutputFormat,\n SpeechSynthesisResult,\n SpeechSynthesizer,\n)\nfrom azure.identity.aio import (\n AzureDeveloperCliCredential,\n ManagedIdentityCredential,\n get_bearer_token_provider,\n)\nfrom azure.monitor.opentelemetry import configure_azure_monitor\nfrom azure.search.documents.aio import SearchClient\nfrom azure.search.documents.indexes.aio import SearchIndexClient\nfrom azure.search.documents.knowledgebases.aio import KnowledgeBaseRetrievalClient\nfrom opentelemetry.instrumentation.aiohttp_client import AioHttpClientInstrumentor\nfrom opentelemetry.instrumentation.asgi import OpenTelemetryMiddleware\nfrom opentelemetry.instrumentation.httpx import (\n HTTPXClientInstrumentor,\n)\nfrom opentelemetry.instrumentation.openai import OpenAIInstrumentor\nfrom quart import (\n Blueprint,\n Quart,\n abort,\n current_app,\n jsonify,\n make_response,\n request,\n send_file,\n send_from_directory,\n)\nfrom quart_cors import cors\n\nfrom approaches.approach import Approach, DataPoints\nfrom approaches.chatreadretrieveread import ChatReadRetrieveReadApproach\nfrom approaches.promptmanager import PromptManager\nfrom chat_history.cosmosdb import chat_history_cosmosdb_bp\nfrom config import (\n CONFIG_AGENTIC_KNOWLEDGEBASE_ENABLED,\n CONFIG_AUTH_CLIENT,\n CONFIG_CHAT_APPROACH,\n CONFIG_CHAT_HISTORY_BROWSER_ENABLED,\n CONFIG_CHAT_HISTORY_COSMOS_ENABLED,\n CONFIG_CREDENTIAL,\n CONFIG_DEFAULT_REASONING_EFFORT,\n CONFIG_DEFAULT_RETRIEVAL_REASONING_EFFORT,\n CONFIG_GLOBAL_BLOB_MANAGER,\n CONFIG_INGESTER,\n CONFIG_KNOWLEDGEBASE_CLIENT,\n CONFIG_KNOWLEDGEBASE_CLIENT_WITH_SHAREPOINT,\n CONFIG_KNOWLEDGEBASE_CLIENT_WITH_WEB,\n CONFIG_KNOWLEDGEBASE_CLIENT_WITH_WEB_AND_SHAREPOINT,\n CONFIG_LANGUAGE_PICKER_ENABLED,\n CONFIG_MULTIMODAL_ENABLED,\n CONFIG_OPENAI_CLIENT,\n CONFIG_QUERY_REWRITING_ENABLED,\n CONFIG_RAG_SEARCH_IMAGE_EMBEDDINGS,\n CONFIG_RAG_SEARCH_TEXT_EMBEDDINGS,\n CONFIG_RAG_SEND_IMAGE_SOURCES,\n CONFIG_RAG_SEND_TEXT_SOURCES,\n CONFIG_REASONING_EFFORT_ENABLED,\n CONFIG_SEARCH_CLIENT,\n CONFIG_SEMANTIC_RANKER_DEPLOYED,\n CONFIG_SHAREPOINT_SOURCE_ENABLED,\n CONFIG_SPEECH_INPUT_ENABLED,\n CONFIG_SPEECH_OUTPUT_AZURE_ENABLED,\n CONFIG_SPEECH_OUTPUT_BROWSER_ENABLED,\n CONFIG_SPEECH_SERVICE_ID,\n CONFIG_SPEECH_SERVICE_LOCATION,\n CONFIG_SPEECH_SERVICE_TOKEN,\n CONFIG_SPEECH_SERVICE_VOICE,\n CONFIG_STREAMING_ENABLED,\n CONFIG_USER_BLOB_MANAGER,\n CONFIG_USER_UPLOAD_ENABLED,\n CONFIG_VECTOR_SEARCH_ENABLED,\n CONFIG_WEB_SOURCE_ENABLED,\n)\nfrom core.authentication import AuthenticationHelper\nfrom core.sessionhelper import create_session_id\nfrom decorators import authenticated, authenticated_path\nfrom error import error_dict, error_response\nfrom prepdocs import (\n OpenAIHost,\n setup_embeddings_service,\n setup_file_processors,\n setup_image_embeddings_service,\n setup_openai_client,\n setup_search_info,\n)\nfrom prepdocslib.blobmanager import AdlsBlobManager, BlobManager\nfrom prepdocslib.embeddings import ImageEmbeddings\nfrom prepdocslib.filestrategy import UploadUserFileStrategy\nfrom prepdocslib.listfilestrategy import File\n\nbp = Blueprint(\"routes\", __name__, static_folder=\"static\")\n# Fix Windows registry issue with mimetypes\nmimetypes.add_type(\"application/javascript\", \".js\")\nmimetypes.add_type(\"text/css\", \".css\")\n\n\n@bp.route(\"/\")\nasync def index():\n return await bp.send_static_file(\"index.html\")\n\n\n# Empty page is recommended for login redirect to work.\n# See https://github.com/AzureAD/microsoft-authentication-library-for-js/blob/dev/lib/msal-browser/docs/initialization.md#redirecturi-considerations for more information\n@bp.route(\"/redirect\")\nasync def redirect():\n return \"\"\n\n\n@bp.route(\"/favicon.ico\")\nasync def favicon():\n return await bp.send_static_file(\"favicon.ico\")\n\n\n@bp.route(\"/assets/\")\nasync def assets(path):\n return await send_from_directory(Path(__file__).resolve().parent / \"static\" / \"assets\", path)\n\n\n@bp.route(\"/content/\")\n@authenticated_path\nasync def content_file(path: str, auth_claims: dict[str, Any]):\n \"\"\"\n Serve content files from blob storage from within the app to keep the example self-contained.\n *** NOTE *** if you are using app services authentication, this route will return unauthorized to all users that are not logged in\n if AZURE_ENFORCE_ACCESS_CONTROL is not set or false, logged in users can access all files regardless of access control\n if AZURE_ENFORCE_ACCESS_CONTROL is set to true, logged in users can only access files they have access to\n This is also slow and memory hungry.\n \"\"\"\n # Remove page number from path, filename-1.txt -> filename.txt\n # This shouldn't typically be necessary as browsers don't send hash fragments to servers\n if path.find(\"#page=\") > 0:\n path_parts = path.rsplit(\"#page=\", 1)\n path = path_parts[0]\n current_app.logger.info(\"Opening file %s\", path)\n blob_manager: BlobManager = current_app.config[CONFIG_GLOBAL_BLOB_MANAGER]\n\n # Get bytes and properties from the blob manager\n result = await blob_manager.download_blob(path)\n\n if result is None:\n current_app.logger.info(\"Path not found in general Blob container: %s\", path)\n if current_app.config[CONFIG_USER_UPLOAD_ENABLED]:\n user_oid = auth_claims[\"oid\"]\n user_blob_manager: AdlsBlobManager = current_app.config[CONFIG_USER_BLOB_MANAGER]\n result = await user_blob_manager.download_blob(path, user_oid=user_oid)\n if result is None:\n current_app.logger.exception(\"Path not found in DataLake: %s\", path)\n\n if not result:\n abort(404)\n\n content, properties = result\n\n if not properties or \"content_settings\" not in properties:\n abort(404)\n\n mime_type = properties[\"content_settings\"][\"content_type\"]\n if mime_type == \"application/octet-stream\":\n mime_type = mimetypes.guess_type(path)[0] or \"application/octet-stream\"\n\n # Create a BytesIO object from the bytes\n blob_file = io.BytesIO(content)\n return await send_file(blob_file, mimetype=mime_type, as_attachment=False, attachment_filename=path)\n\n\nclass JSONEncoder(json.JSONEncoder):\n def default(self, o):\n if dataclasses.is_dataclass(o) and not isinstance(o, type):\n as_dict = dataclasses.asdict(o)\n if isinstance(o, DataPoints):\n # Drop optional data point collections that are not populated to keep API surface stable\n return {k: v for k, v in as_dict.items() if v is not None}\n data_points_payload = as_dict.get(\"data_points\") if isinstance(as_dict, dict) else None\n if isinstance(data_points_payload, dict) and data_points_payload.get(\"citation_activity_details\") is None:\n data_points_payload.pop(\"citation_activity_details\")\n return as_dict\n return super().default(o)\n\n\nasync def format_as_ndjson(r: AsyncGenerator[dict, None]) -> AsyncGenerator[str, None]:\n try:\n async for event in r:\n yield json.dumps(event, ensure_ascii=False, cls=JSONEncoder) + \"\\n\"\n except Exception as error:\n logging.exception(\"Exception while generating response stream: %s\", error)\n yield json.dumps(error_dict(error))\n\n\n@bp.route(\"/chat\", methods=[\"POST\"])\n@authenticated\nasync def chat(auth_claims: dict[str, Any]):\n if not request.is_json:\n return jsonify({\"error\": \"request must be json\"}), 415\n request_json = await request.get_json()\n context = request_json.get(\"context\", {})\n context[\"auth_claims\"] = auth_claims\n try:\n approach: Approach = cast(Approach, current_app.config[CONFIG_CHAT_APPROACH])\n\n # If session state is provided, persists the session state,\n # else creates a new session_id depending on the chat history options enabled.\n session_state = request_json.get(\"session_state\")\n if session_state is None:\n session_state = create_session_id(\n current_app.config[CONFIG_CHAT_HISTORY_COSMOS_ENABLED],\n current_app.config[CONFIG_CHAT_HISTORY_BROWSER_ENABLED],\n )\n result = await approach.run(\n request_json[\"messages\"],\n context=context,\n session_state=session_state,\n )\n return jsonify(result)\n except Exception as error:\n return error_response(error, \"/chat\")\n\n\n@bp.route(\"/chat/stream\", methods=[\"POST\"])\n@authenticated\nasync def chat_stream(auth_claims: dict[str, Any]):\n if not request.is_json:\n return jsonify({\"error\": \"request must be json\"}), 415\n request_json = await request.get_json()\n context = request_json.get(\"context\", {})\n context[\"auth_claims\"] = auth_claims\n try:\n approach: Approach = cast(Approach, current_app.config[CONFIG_CHAT_APPROACH])\n\n # If session state is provided, persists the session state,\n # else creates a new session_id depending on the chat history options enabled.\n session_state = request_json.get(\"session_state\")\n if session_state is None:\n session_state = create_session_id(\n current_app.config[CONFIG_CHAT_HISTORY_COSMOS_ENABLED],\n current_app.config[CONFIG_CHAT_HISTORY_BROWSER_ENABLED],\n )\n result = await approach.run_stream(\n request_json[\"messages\"],\n context=context,\n session_state=session_state,\n )\n response = await make_response(format_as_ndjson(result))\n response.timeout = None # type: ignore\n response.mimetype = \"application/json-lines\"\n return response\n except Exception as error:\n return error_response(error, \"/chat\")\n\n\n# Send MSAL.js settings to the client UI\n@bp.route(\"/auth_setup\", methods=[\"GET\"])\ndef auth_setup():\n auth_helper = current_app.config[CONFIG_AUTH_CLIENT]\n return jsonify(auth_helper.get_auth_setup_for_client())\n\n\n@bp.route(\"/config\", methods=[\"GET\"])\ndef config():\n return jsonify(\n {\n \"showMultimodalOptions\": current_app.config[CONFIG_MULTIMODAL_ENABLED],\n \"showSemanticRankerOption\": current_app.config[CONFIG_SEMANTIC_RANKER_DEPLOYED],\n \"showQueryRewritingOption\": current_app.config[CONFIG_QUERY_REWRITING_ENABLED],\n \"showReasoningEffortOption\": current_app.config[CONFIG_REASONING_EFFORT_ENABLED],\n \"streamingEnabled\": current_app.config[CONFIG_STREAMING_ENABLED],\n \"defaultReasoningEffort\": current_app.config[CONFIG_DEFAULT_REASONING_EFFORT],\n \"defaultRetrievalReasoningEffort\": current_app.config[CONFIG_DEFAULT_RETRIEVAL_REASONING_EFFORT],\n \"showVectorOption\": current_app.config[CONFIG_VECTOR_SEARCH_ENABLED],\n \"showUserUpload\": current_app.config[CONFIG_USER_UPLOAD_ENABLED],\n \"showLanguagePicker\": current_app.config[CONFIG_LANGUAGE_PICKER_ENABLED],\n \"showSpeechInput\": current_app.config[CONFIG_SPEECH_INPUT_ENABLED],\n \"showSpeechOutputBrowser\": current_app.config[CONFIG_SPEECH_OUTPUT_BROWSER_ENABLED],\n \"showSpeechOutputAzure\": current_app.config[CONFIG_SPEECH_OUTPUT_AZURE_ENABLED],\n \"showChatHistoryBrowser\": current_app.config[CONFIG_CHAT_HISTORY_BROWSER_ENABLED],\n \"showChatHistoryCosmos\": current_app.config[CONFIG_CHAT_HISTORY_COSMOS_ENABLED],\n \"showAgenticRetrievalOption\": current_app.config[CONFIG_AGENTIC_KNOWLEDGEBASE_ENABLED],\n \"ragSearchTextEmbeddings\": current_app.config[CONFIG_RAG_SEARCH_TEXT_EMBEDDINGS],\n \"ragSearchImageEmbeddings\": current_app.config[CONFIG_RAG_SEARCH_IMAGE_EMBEDDINGS],\n \"ragSendTextSources\": current_app.config[CONFIG_RAG_SEND_TEXT_SOURCES],\n \"ragSendImageSources\": current_app.config[CONFIG_RAG_SEND_IMAGE_SOURCES],\n \"webSourceEnabled\": current_app.config[CONFIG_WEB_SOURCE_ENABLED],\n \"sharepointSourceEnabled\": current_app.config[CONFIG_SHAREPOINT_SOURCE_ENABLED],\n }\n )\n\n\n@bp.route(\"/speech\", methods=[\"POST\"])\nasync def speech():\n if not request.is_json:\n return jsonify({\"error\": \"request must be json\"}), 415\n\n speech_token = current_app.config.get(CONFIG_SPEECH_SERVICE_TOKEN)\n if speech_token is None or speech_token.expires_on < time.time() + 60:\n speech_token = await current_app.config[CONFIG_CREDENTIAL].get_token(\n \"https://cognitiveservices.azure.com/.default\"\n )\n current_app.config[CONFIG_SPEECH_SERVICE_TOKEN] = speech_token\n\n request_json = await request.get_json()\n text = request_json[\"text\"]\n try:\n # Construct a token as described in documentation:\n # https://learn.microsoft.com/azure/ai-services/speech-service/how-to-configure-azure-ad-auth?pivots=programming-language-python\n auth_token = (\n \"aad#\"\n + current_app.config[CONFIG_SPEECH_SERVICE_ID]\n + \"#\"\n + current_app.config[CONFIG_SPEECH_SERVICE_TOKEN].token\n )\n speech_config = SpeechConfig(auth_token=auth_token, region=current_app.config[CONFIG_SPEECH_SERVICE_LOCATION])\n speech_config.speech_synthesis_voice_name = current_app.config[CONFIG_SPEECH_SERVICE_VOICE]\n speech_config.set_speech_synthesis_output_format(SpeechSynthesisOutputFormat.Audio16Khz32KBitRateMonoMp3)\n synthesizer = SpeechSynthesizer(speech_config=speech_config, audio_config=None)\n result: SpeechSynthesisResult = synthesizer.speak_text_async(text).get()\n if result.reason == ResultReason.SynthesizingAudioCompleted:\n return result.audio_data, 200, {\"Content-Type\": \"audio/mp3\"}\n elif result.reason == ResultReason.Canceled:\n cancellation_details = result.cancellation_details\n current_app.logger.error(\n \"Speech synthesis canceled: %s %s\", cancellation_details.reason, cancellation_details.error_details\n )\n raise Exception(\"Speech synthesis canceled. Check logs for details.\")\n else:\n current_app.logger.error(\"Unexpected result reason: %s\", result.reason)\n raise Exception(\"Speech synthesis failed. Check logs for details.\")\n except Exception as e:\n current_app.logger.exception(\"Exception in /speech\")\n return jsonify({\"error\": str(e)}), 500\n\n\n@bp.post(\"/upload\")\n@authenticated\nasync def upload(auth_claims: dict[str, Any]):\n request_files = await request.files\n if \"file\" not in request_files:\n return jsonify({\"message\": \"No file part in the request\", \"status\": \"failed\"}), 400\n\n try:\n user_oid = auth_claims[\"oid\"]\n file = request_files.getlist(\"file\")[0]\n adls_manager: AdlsBlobManager = current_app.config[CONFIG_USER_BLOB_MANAGER]\n file_url = await adls_manager.upload_blob(file, file.filename, user_oid)\n ingester: UploadUserFileStrategy = current_app.config[CONFIG_INGESTER]\n await ingester.add_file(File(content=file, url=file_url, acls={\"oids\": [user_oid]}), user_oid=user_oid)\n return jsonify({\"message\": \"File uploaded successfully\"}), 200\n except Exception as error:\n current_app.logger.error(\"Error uploading file: %s\", error)\n return jsonify({\"message\": \"Error uploading file, check server logs for details.\", \"status\": \"failed\"}), 500\n\n\n@bp.post(\"/delete_uploaded\")\n@authenticated\nasync def delete_uploaded(auth_claims: dict[str, Any]):\n request_json = await request.get_json()\n filename = request_json.get(\"filename\")\n user_oid = auth_claims[\"oid\"]\n adls_manager: AdlsBlobManager = current_app.config[CONFIG_USER_BLOB_MANAGER]\n await adls_manager.remove_blob(filename, user_oid)\n ingester: UploadUserFileStrategy = current_app.config[CONFIG_INGESTER]\n await ingester.remove_file(filename, user_oid)\n return jsonify({\"message\": f\"File {filename} deleted successfully\"}), 200\n\n\n@bp.get(\"/list_uploaded\")\n@authenticated\nasync def list_uploaded(auth_claims: dict[str, Any]):\n \"\"\"Lists the uploaded documents for the current user.\n Only returns files directly in the user's directory, not in subdirectories.\n Excludes image files and the images directory.\"\"\"\n user_oid = auth_claims[\"oid\"]\n adls_manager: AdlsBlobManager = current_app.config[CONFIG_USER_BLOB_MANAGER]\n files = await adls_manager.list_blobs(user_oid)\n return jsonify(files), 200\n\n\n@bp.before_app_serving\nasync def setup_clients():\n # Replace these with your own values, either in environment variables or directly here\n AZURE_STORAGE_ACCOUNT = os.environ[\"AZURE_STORAGE_ACCOUNT\"]\n AZURE_STORAGE_CONTAINER = os.environ[\"AZURE_STORAGE_CONTAINER\"]\n AZURE_IMAGESTORAGE_CONTAINER = os.environ.get(\"AZURE_IMAGESTORAGE_CONTAINER\")\n AZURE_USERSTORAGE_ACCOUNT = os.environ.get(\"AZURE_USERSTORAGE_ACCOUNT\")\n AZURE_USERSTORAGE_CONTAINER = os.environ.get(\"AZURE_USERSTORAGE_CONTAINER\")\n AZURE_SEARCH_SERVICE = os.environ[\"AZURE_SEARCH_SERVICE\"]\n AZURE_SEARCH_ENDPOINT = f\"https://{AZURE_SEARCH_SERVICE}.search.windows.net\"\n AZURE_SEARCH_INDEX = os.environ[\"AZURE_SEARCH_INDEX\"]\n AZURE_SEARCH_KNOWLEDGEBASE_NAME = os.getenv(\"AZURE_SEARCH_KNOWLEDGEBASE_NAME\", \"\")\n # Shared by all OpenAI deployments\n OPENAI_HOST = OpenAIHost(os.getenv(\"OPENAI_HOST\", \"azure\"))\n OPENAI_CHATGPT_MODEL = os.environ[\"AZURE_OPENAI_CHATGPT_MODEL\"]\n AZURE_OPENAI_KNOWLEDGEBASE_MODEL = os.getenv(\"AZURE_OPENAI_KNOWLEDGEBASE_MODEL\")\n AZURE_OPENAI_KNOWLEDGEBASE_DEPLOYMENT = os.getenv(\"AZURE_OPENAI_KNOWLEDGEBASE_DEPLOYMENT\")\n OPENAI_EMB_MODEL = os.getenv(\"AZURE_OPENAI_EMB_MODEL_NAME\", \"text-embedding-ada-002\")\n OPENAI_EMB_DIMENSIONS = int(os.getenv(\"AZURE_OPENAI_EMB_DIMENSIONS\") or 1536)\n OPENAI_REASONING_EFFORT = os.getenv(\"AZURE_OPENAI_REASONING_EFFORT\")\n # Used with Azure OpenAI deployments\n AZURE_OPENAI_SERVICE = os.getenv(\"AZURE_OPENAI_SERVICE\")\n AZURE_OPENAI_CHATGPT_DEPLOYMENT = (\n os.getenv(\"AZURE_OPENAI_CHATGPT_DEPLOYMENT\")\n if OPENAI_HOST in [OpenAIHost.AZURE, OpenAIHost.AZURE_CUSTOM]\n else None\n )\n AZURE_OPENAI_EMB_DEPLOYMENT = (\n os.getenv(\"AZURE_OPENAI_EMB_DEPLOYMENT\") if OPENAI_HOST in [OpenAIHost.AZURE, OpenAIHost.AZURE_CUSTOM] else None\n )\n AZURE_OPENAI_CUSTOM_URL = os.getenv(\"AZURE_OPENAI_CUSTOM_URL\")\n AZURE_VISION_ENDPOINT = os.getenv(\"AZURE_VISION_ENDPOINT\", \"\")\n AZURE_OPENAI_API_KEY_OVERRIDE = os.getenv(\"AZURE_OPENAI_API_KEY_OVERRIDE\")\n # Used only with non-Azure OpenAI deployments\n OPENAI_API_KEY = os.getenv(\"OPENAI_API_KEY\")\n OPENAI_ORGANIZATION = os.getenv(\"OPENAI_ORGANIZATION\")\n\n AZURE_TENANT_ID = os.getenv(\"AZURE_TENANT_ID\")\n AZURE_USE_AUTHENTICATION = os.getenv(\"AZURE_USE_AUTHENTICATION\", \"\").lower() == \"true\"\n AZURE_ENFORCE_ACCESS_CONTROL = os.getenv(\"AZURE_ENFORCE_ACCESS_CONTROL\", \"\").lower() == \"true\"\n AZURE_ENABLE_UNAUTHENTICATED_ACCESS = os.getenv(\"AZURE_ENABLE_UNAUTHENTICATED_ACCESS\", \"\").lower() == \"true\"\n AZURE_SERVER_APP_ID = os.getenv(\"AZURE_SERVER_APP_ID\")\n AZURE_SERVER_APP_SECRET = os.getenv(\"AZURE_SERVER_APP_SECRET\")\n AZURE_CLIENT_APP_ID = os.getenv(\"AZURE_CLIENT_APP_ID\")\n AZURE_AUTH_TENANT_ID = os.getenv(\"AZURE_AUTH_TENANT_ID\", AZURE_TENANT_ID)\n\n KB_FIELDS_CONTENT = os.getenv(\"KB_FIELDS_CONTENT\", \"content\")\n KB_FIELDS_SOURCEPAGE = os.getenv(\"KB_FIELDS_SOURCEPAGE\", \"sourcepage\")\n\n AZURE_SEARCH_QUERY_LANGUAGE = os.getenv(\"AZURE_SEARCH_QUERY_LANGUAGE\") or \"en-us\"\n AZURE_SEARCH_QUERY_SPELLER = os.getenv(\"AZURE_SEARCH_QUERY_SPELLER\") or \"lexicon\"\n AZURE_SEARCH_SEMANTIC_RANKER = os.getenv(\"AZURE_SEARCH_SEMANTIC_RANKER\", \"free\").lower()\n AZURE_SEARCH_QUERY_REWRITING = os.getenv(\"AZURE_SEARCH_QUERY_REWRITING\", \"false\").lower()\n # This defaults to the previous field name \"embedding\", for backwards compatibility\n AZURE_SEARCH_FIELD_NAME_EMBEDDING = os.getenv(\"AZURE_SEARCH_FIELD_NAME_EMBEDDING\", \"embedding\")\n\n AZURE_SPEECH_SERVICE_ID = os.getenv(\"AZURE_SPEECH_SERVICE_ID\")\n AZURE_SPEECH_SERVICE_LOCATION = os.getenv(\"AZURE_SPEECH_SERVICE_LOCATION\")\n AZURE_SPEECH_SERVICE_VOICE = os.getenv(\"AZURE_SPEECH_SERVICE_VOICE\") or \"en-US-AndrewMultilingualNeural\"\n\n USE_MULTIMODAL = os.getenv(\"USE_MULTIMODAL\", \"\").lower() == \"true\"\n RAG_SEARCH_TEXT_EMBEDDINGS = os.getenv(\"RAG_SEARCH_TEXT_EMBEDDINGS\", \"true\").lower() == \"true\"\n RAG_SEARCH_IMAGE_EMBEDDINGS = os.getenv(\"RAG_SEARCH_IMAGE_EMBEDDINGS\", \"true\").lower() == \"true\"\n RAG_SEND_TEXT_SOURCES = os.getenv(\"RAG_SEND_TEXT_SOURCES\", \"true\").lower() == \"true\"\n RAG_SEND_IMAGE_SOURCES = os.getenv(\"RAG_SEND_IMAGE_SOURCES\", \"true\").lower() == \"true\"\n USE_USER_UPLOAD = os.getenv(\"USE_USER_UPLOAD\", \"\").lower() == \"true\"\n ENABLE_LANGUAGE_PICKER = os.getenv(\"ENABLE_LANGUAGE_PICKER\", \"\").lower() == \"true\"\n USE_SPEECH_INPUT_BROWSER = os.getenv(\"USE_SPEECH_INPUT_BROWSER\", \"\").lower() == \"true\"\n USE_SPEECH_OUTPUT_BROWSER = os.getenv(\"USE_SPEECH_OUTPUT_BROWSER\", \"\").lower() == \"true\"\n USE_SPEECH_OUTPUT_AZURE = os.getenv(\"USE_SPEECH_OUTPUT_AZURE\", \"\").lower() == \"true\"\n USE_CHAT_HISTORY_BROWSER = os.getenv(\"USE_CHAT_HISTORY_BROWSER\", \"\").lower() == \"true\"\n USE_CHAT_HISTORY_COSMOS = os.getenv(\"USE_CHAT_HISTORY_COSMOS\", \"\").lower() == \"true\"\n USE_AGENTIC_KNOWLEDGEBASE = os.getenv(\"USE_AGENTIC_KNOWLEDGEBASE\", \"\").lower() == \"true\"\n USE_WEB_SOURCE = os.getenv(\"USE_WEB_SOURCE\", \"\").lower() == \"true\"\n USE_SHAREPOINT_SOURCE = os.getenv(\"USE_SHAREPOINT_SOURCE\", \"\").lower() == \"true\"\n AGENTIC_KNOWLEDGEBASE_REASONING_EFFORT = os.getenv(\"AGENTIC_KNOWLEDGEBASE_REASONING_EFFORT\", \"low\")\n USE_VECTORS = os.getenv(\"USE_VECTORS\", \"\").lower() != \"false\"\n\n # WEBSITE_HOSTNAME is always set by App Service, RUNNING_IN_PRODUCTION is set in main.bicep\n RUNNING_ON_AZURE = os.getenv(\"WEBSITE_HOSTNAME\") is not None or os.getenv(\"RUNNING_IN_PRODUCTION\") is not None\n\n # Use the current user identity for keyless authentication to Azure services.\n # This assumes you use 'azd auth login' locally, and managed identity when deployed on Azure.\n # The managed identity is setup in the infra/ folder.\n azure_credential: AzureDeveloperCliCredential | ManagedIdentityCredential\n azure_ai_token_provider: Callable[[], Awaitable[str]]\n if RUNNING_ON_AZURE:\n current_app.logger.info(\"Setting up Azure credential using ManagedIdentityCredential\")\n if AZURE_CLIENT_ID := os.getenv(\"AZURE_CLIENT_ID\"):\n # ManagedIdentityCredential should use AZURE_CLIENT_ID if set in env, but its not working for some reason,\n # so we explicitly pass it in as the client ID here. This is necessary for user-assigned managed identities.\n current_app.logger.info(\n \"Setting up Azure credential using ManagedIdentityCredential with client_id %s\", AZURE_CLIENT_ID\n )\n azure_credential = ManagedIdentityCredential(client_id=AZURE_CLIENT_ID)\n else:\n current_app.logger.info(\"Setting up Azure credential using ManagedIdentityCredential\")\n azure_credential = ManagedIdentityCredential()\n elif AZURE_TENANT_ID:\n current_app.logger.info(\n \"Setting up Azure credential using AzureDeveloperCliCredential with tenant_id %s\", AZURE_TENANT_ID\n )\n azure_credential = AzureDeveloperCliCredential(tenant_id=AZURE_TENANT_ID, process_timeout=60)\n else:\n current_app.logger.info(\"Setting up Azure credential using AzureDeveloperCliCredential for home tenant\")\n azure_credential = AzureDeveloperCliCredential(process_timeout=60)\n azure_ai_token_provider = get_bearer_token_provider(\n azure_credential, \"https://cognitiveservices.azure.com/.default\"\n )\n\n # Set the Azure credential in the app config for use in other parts of the app\n current_app.config[CONFIG_CREDENTIAL] = azure_credential\n\n # Set up clients for AI Search and Storage\n search_client = SearchClient(\n endpoint=AZURE_SEARCH_ENDPOINT,\n index_name=AZURE_SEARCH_INDEX,\n credential=azure_credential,\n )\n\n knowledgebase_client = KnowledgeBaseRetrievalClient(\n endpoint=AZURE_SEARCH_ENDPOINT, knowledge_base_name=AZURE_SEARCH_KNOWLEDGEBASE_NAME, credential=azure_credential\n )\n knowledgebase_client_with_web = None\n knowledgebase_client_with_sharepoint = None\n knowledgebase_client_with_web_and_sharepoint = None\n\n if AZURE_SEARCH_KNOWLEDGEBASE_NAME:\n if USE_WEB_SOURCE:\n knowledgebase_client_with_web = KnowledgeBaseRetrievalClient(\n endpoint=AZURE_SEARCH_ENDPOINT,\n knowledge_base_name=f\"{AZURE_SEARCH_KNOWLEDGEBASE_NAME}-with-web\",\n credential=azure_credential,\n )\n if USE_SHAREPOINT_SOURCE:\n knowledgebase_client_with_sharepoint = KnowledgeBaseRetrievalClient(\n endpoint=AZURE_SEARCH_ENDPOINT,\n knowledge_base_name=f\"{AZURE_SEARCH_KNOWLEDGEBASE_NAME}-with-sp\",\n credential=azure_credential,\n )\n if USE_WEB_SOURCE and USE_SHAREPOINT_SOURCE:\n knowledgebase_client_with_web_and_sharepoint = KnowledgeBaseRetrievalClient(\n endpoint=AZURE_SEARCH_ENDPOINT,\n knowledge_base_name=f\"{AZURE_SEARCH_KNOWLEDGEBASE_NAME}-with-web-and-sp\",\n credential=azure_credential,\n )\n\n # Set up the global blob storage manager (used for global content/images, but not user uploads)\n global_blob_manager = BlobManager(\n endpoint=f\"https://{AZURE_STORAGE_ACCOUNT}.blob.core.windows.net\",\n credential=azure_credential,\n container=AZURE_STORAGE_CONTAINER,\n image_container=AZURE_IMAGESTORAGE_CONTAINER,\n )\n current_app.config[CONFIG_GLOBAL_BLOB_MANAGER] = global_blob_manager\n\n # Set up authentication helper\n search_index = None\n if AZURE_USE_AUTHENTICATION:\n current_app.logger.info(\"AZURE_USE_AUTHENTICATION is true, setting up search index client\")\n search_index_client = SearchIndexClient(\n endpoint=AZURE_SEARCH_ENDPOINT,\n credential=azure_credential,\n )\n search_index = await search_index_client.get_index(AZURE_SEARCH_INDEX)\n await search_index_client.close()\n auth_helper = AuthenticationHelper(\n search_index=search_index,\n use_authentication=AZURE_USE_AUTHENTICATION,\n server_app_id=AZURE_SERVER_APP_ID,\n server_app_secret=AZURE_SERVER_APP_SECRET,\n client_app_id=AZURE_CLIENT_APP_ID,\n tenant_id=AZURE_AUTH_TENANT_ID,\n enforce_access_control=AZURE_ENFORCE_ACCESS_CONTROL,\n enable_unauthenticated_access=AZURE_ENABLE_UNAUTHENTICATED_ACCESS,\n )\n\n if USE_SPEECH_OUTPUT_AZURE:\n current_app.logger.info(\"USE_SPEECH_OUTPUT_AZURE is true, setting up Azure speech service\")\n if not AZURE_SPEECH_SERVICE_ID or AZURE_SPEECH_SERVICE_ID == \"\":\n raise ValueError(\"Azure speech resource not configured correctly, missing AZURE_SPEECH_SERVICE_ID\")\n if not AZURE_SPEECH_SERVICE_LOCATION or AZURE_SPEECH_SERVICE_LOCATION == \"\":\n raise ValueError(\"Azure speech resource not configured correctly, missing AZURE_SPEECH_SERVICE_LOCATION\")\n current_app.config[CONFIG_SPEECH_SERVICE_ID] = AZURE_SPEECH_SERVICE_ID\n current_app.config[CONFIG_SPEECH_SERVICE_LOCATION] = AZURE_SPEECH_SERVICE_LOCATION\n current_app.config[CONFIG_SPEECH_SERVICE_VOICE] = AZURE_SPEECH_SERVICE_VOICE\n # Wait until token is needed to fetch for the first time\n current_app.config[CONFIG_SPEECH_SERVICE_TOKEN] = None\n\n openai_client, azure_openai_endpoint = setup_openai_client(\n openai_host=OPENAI_HOST,\n azure_credential=azure_credential,\n azure_openai_service=AZURE_OPENAI_SERVICE,\n azure_openai_custom_url=AZURE_OPENAI_CUSTOM_URL,\n azure_openai_api_key=AZURE_OPENAI_API_KEY_OVERRIDE,\n openai_api_key=OPENAI_API_KEY,\n openai_organization=OPENAI_ORGANIZATION,\n )\n\n user_blob_manager = None\n if USE_USER_UPLOAD:\n current_app.logger.info(\"USE_USER_UPLOAD is true, setting up user upload feature\")\n if not AZURE_USERSTORAGE_ACCOUNT or not AZURE_USERSTORAGE_CONTAINER:\n raise ValueError(\n \"AZURE_USERSTORAGE_ACCOUNT and AZURE_USERSTORAGE_CONTAINER must be set when USE_USER_UPLOAD is true\"\n )\n if not AZURE_ENFORCE_ACCESS_CONTROL:\n raise ValueError(\"AZURE_ENFORCE_ACCESS_CONTROL must be true when USE_USER_UPLOAD is true\")\n user_blob_manager = AdlsBlobManager(\n endpoint=f\"https://{AZURE_USERSTORAGE_ACCOUNT}.dfs.core.windows.net\",\n container=AZURE_USERSTORAGE_CONTAINER,\n credential=azure_credential,\n )\n current_app.config[CONFIG_USER_BLOB_MANAGER] = user_blob_manager\n\n # Set up ingester\n file_processors, figure_processor = setup_file_processors(\n azure_credential=azure_credential,\n document_intelligence_service=os.getenv(\"AZURE_DOCUMENTINTELLIGENCE_SERVICE\"),\n local_pdf_parser=os.getenv(\"USE_LOCAL_PDF_PARSER\", \"\").lower() == \"true\",\n local_html_parser=os.getenv(\"USE_LOCAL_HTML_PARSER\", \"\").lower() == \"true\",\n use_content_understanding=os.getenv(\"USE_CONTENT_UNDERSTANDING\", \"\").lower() == \"true\",\n content_understanding_endpoint=os.getenv(\"AZURE_CONTENTUNDERSTANDING_ENDPOINT\"),\n use_multimodal=USE_MULTIMODAL,\n openai_client=openai_client,\n openai_model=OPENAI_CHATGPT_MODEL,\n openai_deployment=AZURE_OPENAI_CHATGPT_DEPLOYMENT if OPENAI_HOST == OpenAIHost.AZURE else None,\n )\n search_info = setup_search_info(\n search_service=AZURE_SEARCH_SERVICE,\n index_name=AZURE_SEARCH_INDEX,\n azure_credential=azure_credential,\n use_agentic_knowledgebase=USE_AGENTIC_KNOWLEDGEBASE,\n azure_openai_endpoint=azure_openai_endpoint,\n knowledgebase_name=AZURE_SEARCH_KNOWLEDGEBASE_NAME,\n azure_openai_knowledgebase_deployment=AZURE_OPENAI_KNOWLEDGEBASE_DEPLOYMENT,\n azure_openai_knowledgebase_model=AZURE_OPENAI_KNOWLEDGEBASE_MODEL,\n )\n\n text_embeddings_service = None\n if USE_VECTORS:\n text_embeddings_service = setup_embeddings_service(\n open_ai_client=openai_client,\n openai_host=OPENAI_HOST,\n emb_model_name=OPENAI_EMB_MODEL,\n emb_model_dimensions=OPENAI_EMB_DIMENSIONS,\n azure_openai_deployment=AZURE_OPENAI_EMB_DEPLOYMENT,\n azure_openai_endpoint=azure_openai_endpoint,\n )\n\n image_embeddings_service = setup_image_embeddings_service(\n azure_credential=azure_credential,\n vision_endpoint=AZURE_VISION_ENDPOINT,\n use_multimodal=USE_MULTIMODAL,\n )\n ingester = UploadUserFileStrategy(\n search_info=search_info,\n file_processors=file_processors,\n embeddings=text_embeddings_service,\n image_embeddings=image_embeddings_service,\n search_field_name_embedding=AZURE_SEARCH_FIELD_NAME_EMBEDDING,\n blob_manager=user_blob_manager,\n figure_processor=figure_processor,\n )\n current_app.config[CONFIG_INGESTER] = ingester\n\n image_embeddings_client = None\n if USE_MULTIMODAL:\n image_embeddings_client = ImageEmbeddings(AZURE_VISION_ENDPOINT, azure_ai_token_provider)\n\n current_app.config[CONFIG_OPENAI_CLIENT] = openai_client\n current_app.config[CONFIG_SEARCH_CLIENT] = search_client\n current_app.config[CONFIG_KNOWLEDGEBASE_CLIENT] = knowledgebase_client\n current_app.config[CONFIG_KNOWLEDGEBASE_CLIENT_WITH_WEB] = knowledgebase_client_with_web\n current_app.config[CONFIG_KNOWLEDGEBASE_CLIENT_WITH_SHAREPOINT] = knowledgebase_client_with_sharepoint\n current_app.config[CONFIG_KNOWLEDGEBASE_CLIENT_WITH_WEB_AND_SHAREPOINT] = (\n knowledgebase_client_with_web_and_sharepoint\n )\n current_app.config[CONFIG_AUTH_CLIENT] = auth_helper\n\n current_app.config[CONFIG_SEMANTIC_RANKER_DEPLOYED] = AZURE_SEARCH_SEMANTIC_RANKER != \"disabled\"\n current_app.config[CONFIG_QUERY_REWRITING_ENABLED] = (\n AZURE_SEARCH_QUERY_REWRITING == \"true\" and AZURE_SEARCH_SEMANTIC_RANKER != \"disabled\"\n )\n current_app.config[CONFIG_DEFAULT_REASONING_EFFORT] = OPENAI_REASONING_EFFORT\n current_app.config[CONFIG_DEFAULT_RETRIEVAL_REASONING_EFFORT] = AGENTIC_KNOWLEDGEBASE_REASONING_EFFORT\n current_app.config[CONFIG_REASONING_EFFORT_ENABLED] = OPENAI_CHATGPT_MODEL in Approach.GPT_REASONING_MODELS\n current_app.config[CONFIG_STREAMING_ENABLED] = (\n OPENAI_CHATGPT_MODEL not in Approach.GPT_REASONING_MODELS\n or Approach.GPT_REASONING_MODELS[OPENAI_CHATGPT_MODEL].streaming\n )\n current_app.config[CONFIG_VECTOR_SEARCH_ENABLED] = bool(USE_VECTORS)\n current_app.config[CONFIG_USER_UPLOAD_ENABLED] = bool(USE_USER_UPLOAD)\n current_app.config[CONFIG_LANGUAGE_PICKER_ENABLED] = ENABLE_LANGUAGE_PICKER\n current_app.config[CONFIG_SPEECH_INPUT_ENABLED] = USE_SPEECH_INPUT_BROWSER\n current_app.config[CONFIG_SPEECH_OUTPUT_BROWSER_ENABLED] = USE_SPEECH_OUTPUT_BROWSER\n current_app.config[CONFIG_SPEECH_OUTPUT_AZURE_ENABLED] = USE_SPEECH_OUTPUT_AZURE\n current_app.config[CONFIG_CHAT_HISTORY_BROWSER_ENABLED] = USE_CHAT_HISTORY_BROWSER\n current_app.config[CONFIG_CHAT_HISTORY_COSMOS_ENABLED] = USE_CHAT_HISTORY_COSMOS\n current_app.config[CONFIG_AGENTIC_KNOWLEDGEBASE_ENABLED] = USE_AGENTIC_KNOWLEDGEBASE\n current_app.config[CONFIG_MULTIMODAL_ENABLED] = USE_MULTIMODAL\n current_app.config[CONFIG_RAG_SEARCH_TEXT_EMBEDDINGS] = RAG_SEARCH_TEXT_EMBEDDINGS\n current_app.config[CONFIG_RAG_SEARCH_IMAGE_EMBEDDINGS] = RAG_SEARCH_IMAGE_EMBEDDINGS\n current_app.config[CONFIG_RAG_SEND_TEXT_SOURCES] = RAG_SEND_TEXT_SOURCES\n current_app.config[CONFIG_RAG_SEND_IMAGE_SOURCES] = RAG_SEND_IMAGE_SOURCES\n current_app.config[CONFIG_WEB_SOURCE_ENABLED] = USE_WEB_SOURCE\n if AGENTIC_KNOWLEDGEBASE_REASONING_EFFORT == \"minimal\" and current_app.config[CONFIG_WEB_SOURCE_ENABLED]:\n raise ValueError(\"Web source cannot be used with minimal retrieval reasoning effort\")\n current_app.config[CONFIG_SHAREPOINT_SOURCE_ENABLED] = USE_SHAREPOINT_SOURCE\n\n prompt_manager = PromptManager()\n\n # ChatReadRetrieveReadApproach is used by /chat for multi-turn conversation\n current_app.config[CONFIG_CHAT_APPROACH] = ChatReadRetrieveReadApproach(\n search_client=search_client,\n search_index_name=AZURE_SEARCH_INDEX,\n knowledgebase_model=AZURE_OPENAI_KNOWLEDGEBASE_MODEL,\n knowledgebase_deployment=AZURE_OPENAI_KNOWLEDGEBASE_DEPLOYMENT,\n knowledgebase_client=knowledgebase_client,\n knowledgebase_client_with_web=knowledgebase_client_with_web,\n knowledgebase_client_with_sharepoint=knowledgebase_client_with_sharepoint,\n knowledgebase_client_with_web_and_sharepoint=knowledgebase_client_with_web_and_sharepoint,\n openai_client=openai_client,\n chatgpt_model=OPENAI_CHATGPT_MODEL,\n chatgpt_deployment=AZURE_OPENAI_CHATGPT_DEPLOYMENT,\n embedding_model=OPENAI_EMB_MODEL,\n embedding_deployment=AZURE_OPENAI_EMB_DEPLOYMENT,\n embedding_dimensions=OPENAI_EMB_DIMENSIONS,\n embedding_field=AZURE_SEARCH_FIELD_NAME_EMBEDDING,\n sourcepage_field=KB_FIELDS_SOURCEPAGE,\n content_field=KB_FIELDS_CONTENT,\n query_language=AZURE_SEARCH_QUERY_LANGUAGE,\n query_speller=AZURE_SEARCH_QUERY_SPELLER,\n prompt_manager=prompt_manager,\n reasoning_effort=OPENAI_REASONING_EFFORT,\n multimodal_enabled=USE_MULTIMODAL,\n image_embeddings_client=image_embeddings_client,\n global_blob_manager=global_blob_manager,\n user_blob_manager=user_blob_manager,\n use_web_source=current_app.config[CONFIG_WEB_SOURCE_ENABLED],\n use_sharepoint_source=current_app.config[CONFIG_SHAREPOINT_SOURCE_ENABLED],\n retrieval_reasoning_effort=AGENTIC_KNOWLEDGEBASE_REASONING_EFFORT,\n )\n\n\n@bp.after_app_serving\nasync def close_clients():\n await current_app.config[CONFIG_SEARCH_CLIENT].close()\n await current_app.config[CONFIG_GLOBAL_BLOB_MANAGER].close_clients()\n if user_blob_manager := current_app.config.get(CONFIG_USER_BLOB_MANAGER):\n await user_blob_manager.close_clients()\n await current_app.config[CONFIG_CREDENTIAL].close()\n\n\ndef create_app():\n app = Quart(__name__)\n app.register_blueprint(bp)\n app.register_blueprint(chat_history_cosmosdb_bp)\n\n if os.getenv(\"APPLICATIONINSIGHTS_CONNECTION_STRING\"):\n app.logger.info(\"APPLICATIONINSIGHTS_CONNECTION_STRING is set, enabling Azure Monitor\")\n configure_azure_monitor(\n instrumentation_options={\n \"django\": {\"enabled\": False},\n \"psycopg2\": {\"enabled\": False},\n \"fastapi\": {\"enabled\": False},\n }\n )\n # This tracks HTTP requests made by aiohttp:\n AioHttpClientInstrumentor().instrument()\n # This tracks HTTP requests made by httpx:\n HTTPXClientInstrumentor().instrument()\n # This tracks OpenAI SDK requests:\n OpenAIInstrumentor().instrument()\n # This middleware tracks app route requests:\n app.asgi_app = OpenTelemetryMiddleware(app.asgi_app) # type: ignore[assignment]\n\n # Log levels should be one of https://docs.python.org/3/library/logging.html#logging-levels\n # Set root level to WARNING to avoid seeing overly verbose logs from SDKS\n logging.basicConfig(level=logging.WARNING)\n # Set our own logger levels to INFO by default\n app_level = os.getenv(\"APP_LOG_LEVEL\", \"INFO\")\n app.logger.setLevel(os.getenv(\"APP_LOG_LEVEL\", app_level))\n logging.getLogger(\"scripts\").setLevel(app_level)\n\n if allowed_origin := os.getenv(\"ALLOWED_ORIGIN\"):\n allowed_origins = allowed_origin.split(\";\")\n if len(allowed_origins) > 0:\n app.logger.info(\"CORS enabled for %s\", allowed_origins)\n cors(app, allow_origin=allowed_origins, allow_methods=[\"GET\", \"POST\"])\n\n return app\n" }, { "path": "app/backend/approaches/__init__.py", "content": "" }, { "path": "app/backend/approaches/approach.py", "content": "import base64\nimport json\nimport re\nfrom abc import ABC\nfrom collections.abc import AsyncGenerator, Awaitable\nfrom dataclasses import asdict, dataclass, field\nfrom typing import Any, Optional, TypedDict, cast\n\nfrom azure.search.documents.aio import SearchClient\nfrom azure.search.documents.knowledgebases.aio import KnowledgeBaseRetrievalClient\nfrom azure.search.documents.knowledgebases.models import (\n KnowledgeBaseMessage,\n KnowledgeBaseMessageTextContent,\n KnowledgeBaseRemoteSharePointActivityRecord,\n KnowledgeBaseRemoteSharePointReference,\n KnowledgeBaseRetrievalRequest,\n KnowledgeBaseRetrievalResponse,\n KnowledgeBaseSearchIndexActivityRecord,\n KnowledgeBaseSearchIndexReference,\n KnowledgeBaseWebActivityRecord,\n KnowledgeBaseWebReference,\n KnowledgeRetrievalLowReasoningEffort,\n KnowledgeRetrievalMediumReasoningEffort,\n KnowledgeRetrievalMinimalReasoningEffort,\n KnowledgeRetrievalSemanticIntent,\n KnowledgeSourceParams,\n RemoteSharePointKnowledgeSourceParams,\n SearchIndexKnowledgeSourceParams,\n WebKnowledgeSourceParams,\n)\nfrom azure.search.documents.models import (\n QueryCaptionResult,\n QueryType,\n VectorizedQuery,\n VectorQuery,\n)\nfrom openai import AsyncOpenAI, AsyncStream\nfrom openai.types import CompletionUsage\nfrom openai.types.chat import (\n ChatCompletion,\n ChatCompletionChunk,\n ChatCompletionMessageFunctionToolCall,\n ChatCompletionMessageParam,\n ChatCompletionReasoningEffort,\n ChatCompletionToolParam,\n)\n\nfrom approaches.promptmanager import PromptManager\nfrom prepdocslib.blobmanager import AdlsBlobManager, BlobManager\nfrom prepdocslib.embeddings import ImageEmbeddings\n\n\n@dataclass\nclass ActivityDetail:\n id: int\n number: int\n type: str\n source: str\n query: str\n\n\n@dataclass\nclass Document:\n id: Optional[str] = None\n ref_id: Optional[str] = None # Reference id from agentic retrieval (if applicable)\n content: Optional[str] = None\n category: Optional[str] = None\n sourcepage: Optional[str] = None\n sourcefile: Optional[str] = None\n oids: Optional[list[str]] = None\n groups: Optional[list[str]] = None\n captions: Optional[list[QueryCaptionResult]] = None\n score: Optional[float] = None\n reranker_score: Optional[float] = None\n activity: Optional[ActivityDetail] = None\n images: Optional[list[dict[str, Any]]] = None\n\n def serialize_for_results(self) -> dict[str, Any]:\n result_dict = {\n \"type\": \"searchIndex\",\n \"id\": self.id,\n \"content\": self.content,\n \"category\": self.category,\n \"sourcepage\": self.sourcepage,\n \"sourcefile\": self.sourcefile,\n \"oids\": self.oids,\n \"groups\": self.groups,\n \"captions\": (\n [\n {\n \"additional_properties\": caption.additional_properties,\n \"text\": caption.text,\n \"highlights\": caption.highlights,\n }\n for caption in self.captions\n ]\n if self.captions\n else []\n ),\n \"score\": self.score,\n \"reranker_score\": self.reranker_score,\n \"activity\": asdict(self.activity) if self.activity else None,\n \"images\": self.images,\n }\n return result_dict\n\n\n@dataclass\nclass WebResult:\n id: Optional[str] = None\n title: Optional[str] = None\n url: Optional[str] = None\n activity: Optional[ActivityDetail] = None\n\n def serialize_for_results(self) -> dict[str, Any]:\n return {\n \"type\": \"web\",\n \"id\": self.id,\n \"ref_id\": str(self.id),\n \"title\": self.title,\n \"url\": self.url,\n \"activity\": asdict(self.activity) if self.activity else None,\n }\n\n\n@dataclass\nclass SharePointResult:\n id: Optional[str] = None\n web_url: Optional[str] = None\n content: Optional[str] = None\n title: Optional[str] = None\n reranker_score: Optional[float] = None\n activity: Optional[ActivityDetail] = None\n\n def serialize_for_results(self) -> dict[str, Any]:\n return {\n \"type\": \"remoteSharePoint\",\n \"id\": self.id,\n \"ref_id\": str(self.id),\n \"web_url\": self.web_url,\n \"content\": self.content,\n \"title\": self.title,\n \"reranker_score\": self.reranker_score,\n \"activity\": asdict(self.activity) if self.activity else None,\n }\n\n\n@dataclass\nclass RewriteQueryResult:\n query: str\n messages: list[ChatCompletionMessageParam]\n completion: ChatCompletion\n reasoning_effort: ChatCompletionReasoningEffort\n\n\n@dataclass\nclass ThoughtStep:\n title: str\n description: Optional[Any]\n props: Optional[dict[str, Any]] = None\n\n def update_token_usage(self, usage: CompletionUsage) -> None:\n if self.props:\n self.props[\"token_usage\"] = TokenUsageProps.from_completion_usage(usage)\n\n\n@dataclass\nclass AgenticRetrievalResults:\n \"\"\"Results from agentic retrieval including activities, documents, web results, SharePoint results, and optional answer.\"\"\"\n\n response: KnowledgeBaseRetrievalResponse\n documents: list[Document]\n web_results: list[WebResult]\n sharepoint_results: list[SharePointResult] = field(default_factory=list)\n answer: Optional[str] = None # Synthesized answer when web knowledge source is used\n rewrite_result: Optional[RewriteQueryResult] = None\n activity_details_by_id: Optional[dict[int, ActivityDetail]] = None\n thoughts: list[ThoughtStep] = field(default_factory=list)\n\n\n@dataclass\nclass DataPoints:\n text: Optional[list[str]] = None\n images: Optional[list] = None\n citations: Optional[list[str]] = None\n external_results_metadata: Optional[list[dict[str, Any]]] = None\n citation_activity_details: Optional[dict[str, dict[str, Any]]] = None\n\n\n@dataclass\nclass ExtraInfo:\n data_points: DataPoints\n thoughts: list[ThoughtStep] = field(default_factory=list)\n followup_questions: Optional[list[Any]] = None\n answer: Optional[str] = None # Only when web knowledge source is used\n\n\n@dataclass\nclass TokenUsageProps:\n prompt_tokens: int\n completion_tokens: int\n reasoning_tokens: Optional[int]\n total_tokens: int\n\n @classmethod\n def from_completion_usage(cls, usage: CompletionUsage) -> \"TokenUsageProps\":\n return cls(\n prompt_tokens=usage.prompt_tokens,\n completion_tokens=usage.completion_tokens,\n reasoning_tokens=(\n usage.completion_tokens_details.reasoning_tokens if usage.completion_tokens_details else None\n ),\n total_tokens=usage.total_tokens,\n )\n\n\n# GPT reasoning models don't support the same set of parameters as other models\n# https://learn.microsoft.com/azure/ai-services/openai/how-to/reasoning\n@dataclass\nclass GPTReasoningModelSupport:\n streaming: bool\n minimal_effort: bool\n\n\nclass Approach(ABC):\n # List of GPT reasoning models support\n GPT_REASONING_MODELS = {\n \"o1\": GPTReasoningModelSupport(streaming=False, minimal_effort=False),\n \"o3\": GPTReasoningModelSupport(streaming=True, minimal_effort=False),\n \"o3-mini\": GPTReasoningModelSupport(streaming=True, minimal_effort=False),\n \"o4-mini\": GPTReasoningModelSupport(streaming=True, minimal_effort=False),\n \"gpt-5\": GPTReasoningModelSupport(streaming=True, minimal_effort=True),\n \"gpt-5-nano\": GPTReasoningModelSupport(streaming=True, minimal_effort=True),\n \"gpt-5-mini\": GPTReasoningModelSupport(streaming=True, minimal_effort=True),\n }\n # Set a higher token limit for GPT reasoning models\n RESPONSE_DEFAULT_TOKEN_LIMIT = 1024\n RESPONSE_REASONING_DEFAULT_TOKEN_LIMIT = 8192\n QUERY_REWRITE_NO_RESPONSE = \"0\"\n\n def __init__(\n self,\n search_client: SearchClient,\n openai_client: AsyncOpenAI,\n knowledgebase_model: Optional[str],\n knowledgebase_deployment: Optional[str],\n query_language: Optional[str],\n query_speller: Optional[str],\n embedding_deployment: Optional[str], # Not needed for non-Azure OpenAI or for retrieval_mode=\"text\"\n embedding_model: str,\n embedding_dimensions: int,\n embedding_field: str,\n openai_host: str,\n chatgpt_model: str,\n chatgpt_deployment: Optional[str], # Not needed for non-Azure OpenAI\n prompt_manager: PromptManager,\n reasoning_effort: Optional[str] = None,\n multimodal_enabled: bool = False,\n image_embeddings_client: Optional[ImageEmbeddings] = None,\n global_blob_manager: Optional[BlobManager] = None,\n user_blob_manager: Optional[AdlsBlobManager] = None,\n ):\n self.search_client = search_client\n self.openai_client = openai_client\n self.query_language = query_language\n self.query_speller = query_speller\n self.knowledgebase_model = knowledgebase_model\n self.knowledgebase_deployment = knowledgebase_deployment\n self.embedding_deployment = embedding_deployment\n self.embedding_model = embedding_model\n self.embedding_dimensions = embedding_dimensions\n self.embedding_field = embedding_field\n self.openai_host = openai_host\n self.chatgpt_model = chatgpt_model\n self.chatgpt_deployment = chatgpt_deployment\n self.prompt_manager = prompt_manager\n self.query_rewrite_tools = self.prompt_manager.load_tools(\"chat_query_rewrite_tools.json\")\n self.reasoning_effort = reasoning_effort\n self.include_token_usage = True\n self.multimodal_enabled = multimodal_enabled\n self.image_embeddings_client = image_embeddings_client\n self.global_blob_manager = global_blob_manager\n self.user_blob_manager = user_blob_manager\n\n def build_filter(self, overrides: dict[str, Any]) -> Optional[str]:\n include_category = overrides.get(\"include_category\")\n exclude_category = overrides.get(\"exclude_category\")\n filters = []\n if include_category:\n filters.append(\"category eq '{}'\".format(include_category.replace(\"'\", \"''\")))\n if exclude_category:\n filters.append(\"category ne '{}'\".format(exclude_category.replace(\"'\", \"''\")))\n return None if not filters else \" and \".join(filters)\n\n async def search(\n self,\n top: int,\n query_text: Optional[str],\n filter: Optional[str],\n vectors: list[VectorQuery],\n use_text_search: bool,\n use_vector_search: bool,\n use_semantic_ranker: bool,\n use_semantic_captions: bool,\n minimum_search_score: Optional[float] = None,\n minimum_reranker_score: Optional[float] = None,\n use_query_rewriting: Optional[bool] = None,\n access_token: Optional[str] = None,\n ) -> list[Document]:\n search_text = query_text if use_text_search else \"\"\n search_vectors = vectors if use_vector_search else []\n if use_semantic_ranker:\n results = await self.search_client.search(\n search_text=search_text,\n filter=filter,\n top=top,\n query_caption=\"extractive|highlight-false\" if use_semantic_captions else None,\n query_rewrites=\"generative\" if use_query_rewriting else None,\n vector_queries=search_vectors,\n query_type=QueryType.SEMANTIC,\n query_language=self.query_language,\n query_speller=self.query_speller,\n semantic_configuration_name=\"default\",\n semantic_query=query_text,\n x_ms_query_source_authorization=access_token,\n )\n else:\n results = await self.search_client.search(\n search_text=search_text,\n filter=filter,\n top=top,\n vector_queries=search_vectors,\n x_ms_query_source_authorization=access_token,\n )\n\n documents: list[Document] = []\n async for page in results.by_page():\n async for document in page:\n documents.append(\n Document(\n id=document.get(\"id\"),\n content=document.get(\"content\"),\n category=document.get(\"category\"),\n sourcepage=document.get(\"sourcepage\"),\n sourcefile=document.get(\"sourcefile\"),\n oids=document.get(\"oids\"),\n groups=document.get(\"groups\"),\n captions=cast(list[QueryCaptionResult], document.get(\"@search.captions\")),\n score=document.get(\"@search.score\"),\n reranker_score=document.get(\"@search.reranker_score\"),\n images=document.get(\"images\"),\n )\n )\n\n qualified_documents = [\n doc\n for doc in documents\n if (\n (doc.score or 0) >= (minimum_search_score or 0)\n and (doc.reranker_score or 0) >= (minimum_reranker_score or 0)\n )\n ]\n\n return qualified_documents\n\n def extract_rewritten_query(\n self,\n chat_completion: ChatCompletion,\n user_query: str,\n no_response_token: Optional[str] = None,\n ) -> str:\n response_message = chat_completion.choices[0].message\n\n if response_message.tool_calls:\n for tool_call in response_message.tool_calls:\n if tool_call.type != \"function\":\n continue\n arguments_payload = cast(ChatCompletionMessageFunctionToolCall, tool_call).function.arguments or \"{}\"\n try:\n parsed_arguments = json.loads(arguments_payload)\n except json.JSONDecodeError:\n continue\n search_query = parsed_arguments.get(\"search_query\")\n if search_query and (no_response_token is None or search_query != no_response_token):\n return search_query\n\n if response_message.content:\n candidate = response_message.content.strip()\n if candidate and (no_response_token is None or candidate != no_response_token):\n return candidate\n\n return user_query\n\n async def rewrite_query(\n self,\n *,\n prompt_template: str,\n prompt_variables: dict[str, Any],\n overrides: dict[str, Any],\n chatgpt_model: str,\n chatgpt_deployment: Optional[str],\n user_query: str,\n response_token_limit: int,\n tools: Optional[list[ChatCompletionToolParam]] = None,\n temperature: float = 0.0,\n no_response_token: Optional[str] = None,\n ) -> RewriteQueryResult:\n query_messages = [self.prompt_manager.build_system_prompt(prompt_template, prompt_variables)]\n rewrite_reasoning_effort = self.get_lowest_reasoning_effort(self.chatgpt_model)\n\n chat_completion = cast(\n ChatCompletion,\n await self.create_chat_completion(\n chatgpt_deployment,\n chatgpt_model,\n messages=query_messages,\n overrides=overrides,\n response_token_limit=response_token_limit,\n temperature=temperature,\n tools=tools,\n reasoning_effort=rewrite_reasoning_effort,\n ),\n )\n\n rewritten_query = self.extract_rewritten_query(\n chat_completion,\n user_query,\n no_response_token=no_response_token,\n )\n\n return RewriteQueryResult(\n query=rewritten_query,\n messages=query_messages,\n completion=chat_completion,\n reasoning_effort=rewrite_reasoning_effort,\n )\n\n async def run_agentic_retrieval(\n self,\n messages: list[ChatCompletionMessageParam],\n knowledgebase_client: KnowledgeBaseRetrievalClient,\n search_index_name: str,\n filter_add_on: Optional[str] = None,\n minimum_reranker_score: Optional[float] = None,\n access_token: Optional[str] = None,\n use_web_source: bool = False,\n use_sharepoint_source: bool = False,\n retrieval_reasoning_effort: Optional[str] = None,\n should_rewrite_query: bool = True,\n ) -> AgenticRetrievalResults:\n # STEP 1: Invoke agentic retrieval\n thoughts = []\n\n knowledge_source_params = [\n SearchIndexKnowledgeSourceParams(\n knowledge_source_name=search_index_name,\n filter_add_on=filter_add_on,\n include_references=True,\n include_reference_source_data=True,\n always_query_source=False,\n reranker_threshold=minimum_reranker_score,\n )\n ]\n # Build list as KnowledgeSourceParams for type variance\n knowledge_source_params_list: list[KnowledgeSourceParams] = cast(\n list[KnowledgeSourceParams], knowledge_source_params\n )\n\n if use_web_source:\n knowledge_source_params_list.append(\n WebKnowledgeSourceParams(\n knowledge_source_name=\"web\",\n include_references=True,\n include_reference_source_data=True,\n always_query_source=False,\n )\n )\n\n if use_sharepoint_source:\n knowledge_source_params_list.append(\n RemoteSharePointKnowledgeSourceParams(\n knowledge_source_name=\"sharepoint\",\n include_references=True,\n include_reference_source_data=True,\n always_query_source=False,\n )\n )\n\n agentic_retrieval_input: dict[str, Any] = {}\n rewrite_result = None\n if retrieval_reasoning_effort == \"minimal\" and should_rewrite_query:\n original_user_query = messages[-1][\"content\"]\n if not isinstance(original_user_query, str):\n raise ValueError(\"The most recent message content must be a string.\")\n\n rewrite_result = await self.rewrite_query(\n prompt_template=\"query_rewrite.system.jinja2\",\n prompt_variables={\"user_query\": original_user_query, \"past_messages\": messages[:-1]},\n overrides={},\n chatgpt_model=self.chatgpt_model,\n chatgpt_deployment=self.chatgpt_deployment,\n user_query=original_user_query,\n response_token_limit=self.get_response_token_limit(\n self.chatgpt_model, 100\n ), # Setting too low risks malformed JSON, setting too high may affect performance\n tools=self.query_rewrite_tools,\n temperature=0.0, # Minimize creativity for search query generation\n no_response_token=self.QUERY_REWRITE_NO_RESPONSE,\n )\n thoughts.append(\n self.format_thought_step_for_chatcompletion(\n title=\"Prompt to generate search query\",\n messages=rewrite_result.messages,\n overrides={},\n model=self.chatgpt_model,\n deployment=self.chatgpt_deployment,\n usage=rewrite_result.completion.usage,\n reasoning_effort=rewrite_result.reasoning_effort,\n )\n )\n agentic_retrieval_input[\"intents\"] = [KnowledgeRetrievalSemanticIntent(search=rewrite_result.query)]\n elif retrieval_reasoning_effort == \"minimal\":\n last_content = messages[-1][\"content\"]\n if not isinstance(last_content, str):\n raise ValueError(\"The most recent message content must be a string.\")\n agentic_retrieval_input[\"intents\"] = [KnowledgeRetrievalSemanticIntent(search=last_content)]\n else:\n kb_messages: list[KnowledgeBaseMessage] = [\n KnowledgeBaseMessage(\n role=str(msg[\"role\"]), content=[KnowledgeBaseMessageTextContent(text=str(msg[\"content\"]))]\n )\n for msg in messages\n if msg[\"role\"] != \"system\"\n ]\n agentic_retrieval_input[\"messages\"] = kb_messages\n # When we're not using a web source, set output mode to extractiveData to avoid synthesized answer\n if not use_web_source:\n agentic_retrieval_input[\"output_mode\"] = \"extractiveData\"\n\n retrieval_effort: Optional[\n KnowledgeRetrievalMinimalReasoningEffort\n | KnowledgeRetrievalLowReasoningEffort\n | KnowledgeRetrievalMediumReasoningEffort\n ] = None\n if retrieval_reasoning_effort == \"minimal\":\n retrieval_effort = KnowledgeRetrievalMinimalReasoningEffort()\n elif retrieval_reasoning_effort == \"low\":\n retrieval_effort = KnowledgeRetrievalLowReasoningEffort()\n elif retrieval_reasoning_effort == \"medium\":\n retrieval_effort = KnowledgeRetrievalMediumReasoningEffort()\n\n request_kwargs: dict[str, Any] = {\n \"knowledge_source_params\": knowledge_source_params_list,\n \"include_activity\": True,\n \"retrieval_reasoning_effort\": retrieval_effort,\n }\n request_kwargs.update(agentic_retrieval_input)\n\n response = await knowledgebase_client.retrieve(\n retrieval_request=KnowledgeBaseRetrievalRequest(**request_kwargs),\n x_ms_query_source_authorization=access_token,\n )\n\n # Map activity id -> agent's internal search query and citation\n activities = response.activity or []\n activity_details_by_id: dict[int, ActivityDetail] = {}\n\n for index, activity in enumerate(activities):\n search_query = None\n if isinstance(activity, KnowledgeBaseSearchIndexActivityRecord):\n if activity.search_index_arguments:\n search_query = activity.search_index_arguments.search\n elif isinstance(activity, KnowledgeBaseWebActivityRecord):\n if activity.web_arguments:\n search_query = activity.web_arguments.search\n elif isinstance(activity, KnowledgeBaseRemoteSharePointActivityRecord):\n if activity.remote_share_point_arguments:\n search_query = activity.remote_share_point_arguments.search\n\n activity_details_by_id[activity.id] = ActivityDetail(\n id=activity.id,\n number=index + 1,\n type=activity.type or \"\",\n source=getattr(activity, \"knowledge_source_name\", \"\")\n or \"\", # Not all activity types have knowledge_source_name\n query=search_query or \"\",\n )\n\n # Extract references\n references = response.references or []\n\n document_refs = [\n r for r in references if isinstance(r, KnowledgeBaseSearchIndexReference) or hasattr(r, \"doc_key\")\n ]\n document_results: list[Document] = []\n # Create documents from reference source data\n for ref in document_refs:\n if ref.source_data and ref.doc_key:\n # Note that ref.doc_key is the same as source_data[\"id\"]\n document_results.append(\n Document(\n id=cast(str, ref.doc_key),\n ref_id=ref.id,\n content=ref.source_data.get(\"content\"),\n category=ref.source_data.get(\"category\"),\n sourcepage=ref.source_data.get(\"sourcepage\"),\n sourcefile=ref.source_data.get(\"sourcefile\"),\n oids=ref.source_data.get(\"oids\"),\n groups=ref.source_data.get(\"groups\"),\n reranker_score=getattr(ref, \"reranker_score\", None),\n images=ref.source_data.get(\"images\"),\n activity=activity_details_by_id[ref.activity_source],\n )\n )\n\n # We need to handle KnowledgeBaseWebReference separately if web knowledge source is used\n web_refs = [r for r in references if isinstance(r, KnowledgeBaseWebReference)]\n web_results: list[WebResult] = []\n for ref in web_refs:\n web_result = WebResult(\n id=ref.id, title=ref.title, url=ref.url, activity=activity_details_by_id[ref.activity_source]\n )\n web_results.append(web_result)\n\n # Handle KnowledgeBaseRemoteSharePointReference if SharePoint knowledge source is used\n sharepoint_refs = [r for r in references if isinstance(r, KnowledgeBaseRemoteSharePointReference)]\n sharepoint_results: list[SharePointResult] = []\n for ref in sharepoint_refs:\n # Extract content from all sourceData.extracts[].text and concatenate\n content = None\n if ref.source_data and \"extracts\" in ref.source_data and len(ref.source_data[\"extracts\"]) > 0:\n extracts = [extract.get(\"text\", \"\") for extract in ref.source_data[\"extracts\"]]\n content = \"\\n\\n\".join(extracts) if extracts else None\n\n # Extract title from sourceData.resourceMetadata.title\n title = None\n if ref.source_data and \"resourceMetadata\" in ref.source_data:\n title = ref.source_data[\"resourceMetadata\"].get(\"title\")\n\n sharepoint_result = SharePointResult(\n id=ref.id,\n web_url=ref.web_url,\n content=content,\n title=title,\n reranker_score=getattr(ref, \"reranker_score\", None),\n activity=activity_details_by_id[ref.activity_source],\n )\n sharepoint_results.append(sharepoint_result)\n\n # Extract answer from response if web knowledge source provided one\n answer: Optional[str] = None\n if (\n use_web_source\n and response.response\n and len(response.response) > 0\n and len(response.response[0].content) > 0\n ):\n message_content = response.response[0].content[0]\n if isinstance(message_content, KnowledgeBaseMessageTextContent):\n raw_answer: Optional[str] = message_content.text\n # Replace all ref_id tokens (web -> URL, documents -> sourcepage, SharePoint -> web_url)\n if raw_answer:\n answer = self.replace_all_ref_ids(raw_answer, document_results, web_results, sharepoint_results)\n\n thoughts.append(\n ThoughtStep(\n \"Agentic retrieval response\",\n [result.serialize_for_results() for result in document_results + web_results + sharepoint_results],\n {\n \"query_plan\": (\n [activity.as_dict() for activity in response.activity] if response.activity else None\n ),\n \"model\": self.knowledgebase_model,\n \"deployment\": self.knowledgebase_deployment,\n \"reranker_threshold\": minimum_reranker_score,\n \"filter\": filter_add_on,\n },\n )\n )\n\n return AgenticRetrievalResults(\n response=response,\n documents=document_results,\n web_results=web_results,\n sharepoint_results=sharepoint_results,\n answer=answer,\n rewrite_result=rewrite_result,\n activity_details_by_id=activity_details_by_id,\n thoughts=thoughts,\n )\n\n def replace_all_ref_ids(\n self,\n answer: str,\n documents: list[Document],\n web_results: list[WebResult],\n sharepoint_results: Optional[list[SharePointResult]] = None,\n ) -> str:\n \"\"\"Replace [ref_id:] tokens with document sourcepage, web URL, or SharePoint web_url.\n\n Priority: web result -> SharePoint result -> document.\n Unknown ids left untouched.\n \"\"\"\n doc_map = {d.ref_id: d.sourcepage for d in documents if d.ref_id and d.sourcepage}\n web_map = {str(w.id): w.url for w in web_results if w.id and w.url}\n sharepoint_entries = sharepoint_results or []\n sharepoint_map = {str(sp.id): sp.web_url.split(\"/\")[-1] for sp in sharepoint_entries if sp.id and sp.web_url}\n\n def _sub(match: re.Match) -> str:\n ref_id = match.group(1)\n if ref_id in web_map and web_map[ref_id]:\n return f\"[{web_map[ref_id]}]\"\n if ref_id in sharepoint_map and sharepoint_map[ref_id]:\n return f\"[{sharepoint_map[ref_id]}]\"\n if ref_id in doc_map and doc_map[ref_id]:\n return f\"[{doc_map[ref_id]}]\"\n return match.group(0)\n\n return re.sub(r\"\\[ref_id:([^\\]]+)\\]\", _sub, answer)\n\n async def get_sources_content(\n self,\n results: list[Document],\n use_semantic_captions: bool,\n include_text_sources: bool,\n download_image_sources: bool,\n user_oid: Optional[str] = None,\n web_results: Optional[list[WebResult]] = None,\n sharepoint_results: Optional[list[SharePointResult]] = None,\n ) -> DataPoints:\n \"\"\"Extract text/image sources & citations from documents.\n\n Args:\n results: List of retrieved Document objects.\n use_semantic_captions: Whether to use semantic captions instead of full content text.\n download_image_sources: Whether to attempt downloading & base64 encoding referenced images.\n user_oid: Optional user object id for per-user storage access (ADLS scenarios).\n web_results: Optional list of web retrieval results to expose to clients.\n sharepoint_results: Optional list of SharePoint retrieval results to expose to clients.\n\n Returns:\n DataPoints: with text (list[str]), images (list[str - base64 data URI]), citations (list[str]).\n \"\"\"\n\n def clean_source(s: str) -> str:\n s = s.replace(\"\\n\", \" \").replace(\"\\r\", \" \") # normalize newlines to spaces\n s = s.replace(\":::\", \":::\") # escape DocFX/markdown triple colons\n return s\n\n citations = []\n text_sources = []\n image_sources = []\n seen_urls = set()\n external_results_metadata: list[dict[str, Any]] = []\n citation_activity_details: dict[str, dict[str, Any]] = {}\n\n for doc in results:\n # Get the citation for the source page\n citation = self.get_citation(doc.sourcepage)\n if citation not in citations:\n citations.append(citation)\n # Add activity details if available\n if doc.activity:\n citation_activity_details[citation] = asdict(doc.activity)\n\n # If semantic captions are used, extract captions; otherwise, use content\n if include_text_sources:\n if use_semantic_captions and doc.captions:\n cleaned = clean_source(\" . \".join([cast(str, c.text) for c in doc.captions]))\n else:\n cleaned = clean_source(doc.content or \"\")\n text_sources.append(f\"{citation}: {cleaned}\")\n\n if download_image_sources and hasattr(doc, \"images\") and doc.images:\n for img in doc.images:\n # Skip if we've already processed this URL\n if img[\"url\"] in seen_urls or not img[\"url\"]:\n continue\n seen_urls.add(img[\"url\"])\n url = await self.download_blob_as_base64(img[\"url\"], user_oid=user_oid)\n if url:\n image_sources.append(url)\n image_citation = self.get_image_citation(doc.sourcepage or \"\", img[\"url\"])\n citations.append(image_citation)\n if web_results:\n for web in web_results:\n citation = self.get_citation(web.url)\n if citation and citation not in citations:\n citations.append(citation)\n # Add activity details if available\n if web.activity:\n citation_activity_details[citation] = asdict(web.activity)\n external_results_metadata.append(\n {\n \"id\": web.id,\n \"title\": web.title,\n \"url\": web.url,\n \"activity\": asdict(web.activity) if web.activity else None,\n }\n )\n if sharepoint_results:\n for sp in sharepoint_results:\n # Extract filename from web_url for citation\n filename = sp.web_url.split(\"/\")[-1] if sp.web_url else \"\"\n citation = self.get_citation(filename)\n if citation and citation not in citations:\n citations.append(citation)\n # Add activity details if available\n if sp.activity:\n citation_activity_details[citation] = asdict(sp.activity)\n if include_text_sources and sp.content:\n text_sources.append(f\"{citation}: {clean_source(sp.content)}\")\n external_results_metadata.append(\n {\n \"id\": sp.id,\n \"title\": sp.title or \"\",\n \"url\": sp.web_url or \"\",\n \"snippet\": clean_source(sp.content or \"\"),\n \"activity\": asdict(sp.activity) if sp.activity else None,\n }\n )\n\n return DataPoints(\n text=text_sources,\n images=image_sources,\n citations=citations,\n external_results_metadata=external_results_metadata,\n citation_activity_details=citation_activity_details if citation_activity_details else None,\n )\n\n def get_citation(self, sourcepage: Optional[str]):\n return sourcepage or \"\"\n\n def get_image_citation(self, sourcepage: Optional[str], image_url: str):\n sourcepage_citation = self.get_citation(sourcepage)\n image_filename = image_url.split(\"/\")[-1]\n return f\"{sourcepage_citation}({image_filename})\"\n\n async def download_blob_as_base64(self, blob_url: str, user_oid: Optional[str] = None) -> Optional[str]:\n \"\"\"\n Downloads a blob from either Azure Blob Storage or Azure Data Lake Storage and returns it as a base64 encoded string.\n\n Args:\n blob_url: The URL or path to the blob to download\n user_oid: The user's object ID, required for Data Lake Storage operations and access control\n\n Returns:\n Optional[str]: The base64 encoded image data with data URI scheme prefix, or None if the blob cannot be downloaded\n \"\"\"\n\n # Handle full URLs for both Blob Storage and Data Lake Storage\n container: Optional[str] = None\n if blob_url.startswith(\"http\"):\n url_parts = blob_url.split(\"/\")\n # Extract container name from URL\n # For blob: https://{account}.blob.core.windows.net/{container}/{blob_path}\n # For dfs: https://{account}.dfs.core.windows.net/{filesystem}/{path}\n container = url_parts[3]\n # Extract the blob path portion (everything after the container/filesystem segment)\n blob_path = \"/\".join(url_parts[4:])\n # If %20 in URL, replace it with a space\n blob_path = blob_path.replace(\"%20\", \" \")\n else:\n # Treat as a direct blob path\n blob_path = blob_url\n\n # Download the blob using the appropriate client\n result = None\n if \".dfs.core.windows.net\" in blob_url and self.user_blob_manager:\n result = await self.user_blob_manager.download_blob(blob_path, user_oid=user_oid, container=container)\n elif self.global_blob_manager:\n result = await self.global_blob_manager.download_blob(blob_path, container=container)\n\n if result:\n content, _ = result # Unpack the tuple, ignoring properties\n img = base64.b64encode(content).decode(\"utf-8\")\n return f\"data:image/png;base64,{img}\"\n return None\n\n async def compute_text_embedding(self, q: str):\n SUPPORTED_DIMENSIONS_MODEL = {\n \"text-embedding-ada-002\": False,\n \"text-embedding-3-small\": True,\n \"text-embedding-3-large\": True,\n }\n\n class ExtraArgs(TypedDict, total=False):\n dimensions: int\n\n dimensions_args: ExtraArgs = (\n {\"dimensions\": self.embedding_dimensions} if SUPPORTED_DIMENSIONS_MODEL[self.embedding_model] else {}\n )\n embedding = await self.openai_client.embeddings.create(\n # Azure OpenAI takes the deployment name as the model name\n model=self.embedding_deployment if self.embedding_deployment else self.embedding_model,\n input=q,\n **dimensions_args,\n )\n query_vector = embedding.data[0].embedding\n # This performs an oversampling due to how the search index was setup,\n # so we do not need to explicitly pass in an oversampling parameter here\n return VectorizedQuery(vector=query_vector, k=50, fields=self.embedding_field)\n\n async def compute_multimodal_embedding(self, q: str):\n if not self.image_embeddings_client:\n raise ValueError(\"Approach is missing an image embeddings client for multimodal queries\")\n multimodal_query_vector = await self.image_embeddings_client.create_embedding_for_text(q)\n return VectorizedQuery(vector=multimodal_query_vector, k=50, fields=\"images/embedding\")\n\n def get_system_prompt_variables(self, override_prompt: Optional[str]) -> dict[str, str]:\n # Allows client to replace the entire prompt, or to inject into the existing prompt using >>>\n if override_prompt is None:\n return {}\n elif override_prompt.startswith(\">>>\"):\n return {\"injected_prompt\": override_prompt[3:]}\n else:\n return {\"override_prompt\": override_prompt}\n\n def get_response_token_limit(self, model: str, default_limit: int) -> int:\n if model in self.GPT_REASONING_MODELS:\n return self.RESPONSE_REASONING_DEFAULT_TOKEN_LIMIT\n\n return default_limit\n\n def get_lowest_reasoning_effort(self, model: str) -> ChatCompletionReasoningEffort:\n \"\"\"\n Return the lowest valid reasoning_effort for the given model.\n \"\"\"\n if model not in self.GPT_REASONING_MODELS:\n return None\n if self.GPT_REASONING_MODELS[model].minimal_effort:\n return \"minimal\"\n return \"low\"\n\n def create_chat_completion(\n self,\n chatgpt_deployment: Optional[str],\n chatgpt_model: str,\n messages: list[ChatCompletionMessageParam],\n overrides: dict[str, Any],\n response_token_limit: int,\n should_stream: bool = False,\n tools: Optional[list[ChatCompletionToolParam]] = None,\n temperature: Optional[float] = None,\n n: Optional[int] = None,\n reasoning_effort: Optional[ChatCompletionReasoningEffort] = None,\n ) -> Awaitable[ChatCompletion] | Awaitable[AsyncStream[ChatCompletionChunk]]:\n if chatgpt_model in self.GPT_REASONING_MODELS:\n params: dict[str, Any] = {\n # max_tokens is not supported\n \"max_completion_tokens\": response_token_limit\n }\n\n # Adjust parameters for reasoning models\n supported_features = self.GPT_REASONING_MODELS[chatgpt_model]\n if supported_features.streaming and should_stream:\n params[\"stream\"] = True\n params[\"stream_options\"] = {\"include_usage\": True}\n params[\"reasoning_effort\"] = reasoning_effort or overrides.get(\"reasoning_effort\") or self.reasoning_effort\n\n else:\n # Include parameters that may not be supported for reasoning models\n params = {\n \"max_tokens\": response_token_limit,\n \"temperature\": temperature or overrides.get(\"temperature\", 0.3),\n }\n if should_stream:\n params[\"stream\"] = True\n params[\"stream_options\"] = {\"include_usage\": True}\n\n if tools is not None:\n params[\"tools\"] = tools\n\n # Azure OpenAI takes the deployment name as the model name\n seed_value: Optional[int] = overrides.get(\"seed\", None)\n return self.openai_client.chat.completions.create( # type: ignore[no-matching-overload]\n model=chatgpt_deployment if chatgpt_deployment else chatgpt_model,\n messages=messages,\n seed=seed_value,\n n=n or 1,\n **params,\n )\n\n def format_thought_step_for_chatcompletion(\n self,\n title: str,\n messages: list[ChatCompletionMessageParam],\n overrides: dict[str, Any],\n model: str,\n deployment: Optional[str],\n usage: Optional[CompletionUsage] = None,\n reasoning_effort: Optional[ChatCompletionReasoningEffort] = None,\n ) -> ThoughtStep:\n properties: dict[str, Any] = {\"model\": model}\n if deployment:\n properties[\"deployment\"] = deployment\n # Only add reasoning_effort setting if the model supports it\n if model in self.GPT_REASONING_MODELS:\n properties[\"reasoning_effort\"] = reasoning_effort or overrides.get(\n \"reasoning_effort\", self.reasoning_effort\n )\n if usage:\n properties[\"token_usage\"] = TokenUsageProps.from_completion_usage(usage)\n return ThoughtStep(title, messages, properties)\n\n async def run(\n self,\n messages: list[ChatCompletionMessageParam],\n session_state: Any = None,\n context: dict[str, Any] = {},\n ) -> dict[str, Any]:\n raise NotImplementedError\n\n async def run_stream(\n self,\n messages: list[ChatCompletionMessageParam],\n session_state: Any = None,\n context: dict[str, Any] = {},\n ) -> AsyncGenerator[dict[str, Any], None]:\n raise NotImplementedError\n" }, { "path": "app/backend/approaches/chatreadretrieveread.py", "content": "import re\nfrom collections.abc import AsyncGenerator, Awaitable\nfrom dataclasses import asdict\nfrom typing import Any, Optional, cast\n\nfrom azure.search.documents.aio import SearchClient\nfrom azure.search.documents.knowledgebases.aio import KnowledgeBaseRetrievalClient\nfrom azure.search.documents.models import VectorQuery\nfrom openai import AsyncOpenAI, AsyncStream\nfrom openai.types.chat import (\n ChatCompletion,\n ChatCompletionChunk,\n ChatCompletionMessageParam,\n)\nfrom openai.types.chat.chat_completion import Choice\nfrom openai.types.chat.chat_completion_message import ChatCompletionMessage\n\nfrom approaches.approach import (\n Approach,\n ExtraInfo,\n ThoughtStep,\n)\nfrom approaches.promptmanager import PromptManager\nfrom prepdocslib.blobmanager import AdlsBlobManager, BlobManager\nfrom prepdocslib.embeddings import ImageEmbeddings\n\n\nclass ChatReadRetrieveReadApproach(Approach):\n \"\"\"\n A multi-step approach that first uses OpenAI to turn the user's question into a search query,\n then uses Azure AI Search to retrieve relevant documents, and then sends the conversation history,\n original user question, and search results to OpenAI to generate a response.\n \"\"\"\n\n NO_RESPONSE = Approach.QUERY_REWRITE_NO_RESPONSE\n\n def __init__(\n self,\n *,\n search_client: SearchClient,\n search_index_name: str,\n knowledgebase_model: Optional[str],\n knowledgebase_deployment: Optional[str],\n knowledgebase_client: Optional[KnowledgeBaseRetrievalClient],\n knowledgebase_client_with_web: Optional[KnowledgeBaseRetrievalClient] = None,\n knowledgebase_client_with_sharepoint: Optional[KnowledgeBaseRetrievalClient] = None,\n knowledgebase_client_with_web_and_sharepoint: Optional[KnowledgeBaseRetrievalClient] = None,\n openai_client: AsyncOpenAI,\n chatgpt_model: str,\n chatgpt_deployment: Optional[str], # Not needed for non-Azure OpenAI\n embedding_deployment: Optional[str], # Not needed for non-Azure OpenAI or for retrieval_mode=\"text\"\n embedding_model: str,\n embedding_dimensions: int,\n embedding_field: str,\n sourcepage_field: str,\n content_field: str,\n query_language: str,\n query_speller: str,\n prompt_manager: PromptManager,\n reasoning_effort: Optional[str] = None,\n multimodal_enabled: bool = False,\n image_embeddings_client: Optional[ImageEmbeddings] = None,\n global_blob_manager: Optional[BlobManager] = None,\n user_blob_manager: Optional[AdlsBlobManager] = None,\n use_web_source: bool = False,\n use_sharepoint_source: bool = False,\n retrieval_reasoning_effort: Optional[str] = None,\n ):\n self.search_client = search_client\n self.search_index_name = search_index_name\n self.knowledgebase_model = knowledgebase_model\n self.knowledgebase_deployment = knowledgebase_deployment\n self.knowledgebase_client = knowledgebase_client\n self.knowledgebase_client_with_web = knowledgebase_client_with_web\n self.knowledgebase_client_with_sharepoint = knowledgebase_client_with_sharepoint\n self.knowledgebase_client_with_web_and_sharepoint = knowledgebase_client_with_web_and_sharepoint\n self.openai_client = openai_client\n self.chatgpt_model = chatgpt_model\n self.chatgpt_deployment = chatgpt_deployment\n self.embedding_deployment = embedding_deployment\n self.embedding_model = embedding_model\n self.embedding_dimensions = embedding_dimensions\n self.embedding_field = embedding_field\n self.sourcepage_field = sourcepage_field\n self.content_field = content_field\n self.query_language = query_language\n self.query_speller = query_speller\n self.prompt_manager = prompt_manager\n self.query_rewrite_tools = self.prompt_manager.load_tools(\"chat_query_rewrite_tools.json\")\n self.reasoning_effort = reasoning_effort\n self.include_token_usage = True\n self.multimodal_enabled = multimodal_enabled\n self.image_embeddings_client = image_embeddings_client\n self.global_blob_manager = global_blob_manager\n self.user_blob_manager = user_blob_manager\n # Track whether web source retrieval is enabled for this deployment; overrides may only disable it.\n self.web_source_enabled = use_web_source\n self.use_sharepoint_source = use_sharepoint_source\n self.retrieval_reasoning_effort = retrieval_reasoning_effort\n\n def extract_followup_questions(self, content: Optional[str]):\n if content is None:\n return content, []\n return content.split(\"<<\")[0], re.findall(r\"<<([^>>]+)>>\", content)\n\n def get_search_query(self, chat_completion: ChatCompletion, default_query: str) -> str:\n \"\"\"Read the optimized search query from a chat completion tool call.\"\"\"\n try:\n return self.extract_rewritten_query(chat_completion, default_query, no_response_token=self.NO_RESPONSE)\n except Exception:\n return default_query\n\n async def run_without_streaming(\n self,\n messages: list[ChatCompletionMessageParam],\n overrides: dict[str, Any],\n auth_claims: dict[str, Any],\n session_state: Any = None,\n ) -> dict[str, Any]:\n extra_info, chat_coroutine = await self.run_until_final_call(\n messages, overrides, auth_claims, should_stream=False\n )\n chat_completion_response: ChatCompletion = await cast(Awaitable[ChatCompletion], chat_coroutine)\n content = chat_completion_response.choices[0].message.content\n role = chat_completion_response.choices[0].message.role\n if overrides.get(\"suggest_followup_questions\"):\n content, followup_questions = self.extract_followup_questions(content)\n extra_info.followup_questions = followup_questions\n # Assume last thought is for generating answer\n # TODO: Update for agentic? This isn't still true?\n if self.include_token_usage and extra_info.thoughts and chat_completion_response.usage:\n extra_info.thoughts[-1].update_token_usage(chat_completion_response.usage)\n chat_app_response = {\n \"message\": {\"content\": content, \"role\": role},\n \"context\": {\n \"thoughts\": extra_info.thoughts,\n \"data_points\": {\n key: value for key, value in asdict(extra_info.data_points).items() if value is not None\n },\n \"followup_questions\": extra_info.followup_questions,\n },\n \"session_state\": session_state,\n }\n return chat_app_response\n\n async def run_with_streaming(\n self,\n messages: list[ChatCompletionMessageParam],\n overrides: dict[str, Any],\n auth_claims: dict[str, Any],\n session_state: Any = None,\n ) -> AsyncGenerator[dict, None]:\n extra_info, chat_coroutine = await self.run_until_final_call(\n messages, overrides, auth_claims, should_stream=True\n )\n yield {\"delta\": {\"role\": \"assistant\"}, \"context\": extra_info, \"session_state\": session_state}\n\n followup_questions_started = False\n followup_content = \"\"\n chat_result = await chat_coroutine\n\n if isinstance(chat_result, ChatCompletion):\n message = chat_result.choices[0].message\n content = message.content or \"\"\n role = message.role or \"assistant\"\n\n followup_questions: list[str] = []\n if overrides.get(\"suggest_followup_questions\"):\n content, followup_questions = self.extract_followup_questions(content)\n extra_info.followup_questions = followup_questions\n\n if self.include_token_usage and extra_info.thoughts and chat_result.usage:\n extra_info.thoughts[-1].update_token_usage(chat_result.usage)\n\n delta_payload: dict[str, Any] = {\"role\": role}\n if content:\n delta_payload[\"content\"] = content\n yield {\"delta\": delta_payload}\n\n yield {\"delta\": {\"role\": \"assistant\"}, \"context\": extra_info, \"session_state\": session_state}\n\n if followup_questions:\n yield {\n \"delta\": {\"role\": \"assistant\"},\n \"context\": {\"context\": extra_info, \"followup_questions\": followup_questions},\n }\n return\n\n chat_result = cast(AsyncStream[ChatCompletionChunk], chat_result)\n\n async for event_chunk in chat_result:\n # \"2023-07-01-preview\" API version has a bug where first response has empty choices\n event = event_chunk.model_dump() # Convert pydantic model to dict\n if event[\"choices\"]:\n # No usage during streaming\n completion = {\n \"delta\": {\n \"content\": event[\"choices\"][0][\"delta\"].get(\"content\"),\n \"role\": event[\"choices\"][0][\"delta\"][\"role\"],\n }\n }\n # if event contains << and not >>, it is start of follow-up question, truncate\n delta_content_raw = completion[\"delta\"].get(\"content\")\n delta_content: str = (\n delta_content_raw or \"\"\n ) # content may either not exist in delta, or explicitly be None\n if overrides.get(\"suggest_followup_questions\") and \"<<\" in delta_content:\n followup_questions_started = True\n earlier_content = delta_content[: delta_content.index(\"<<\")]\n if earlier_content:\n completion[\"delta\"][\"content\"] = earlier_content\n yield completion\n followup_content += delta_content[delta_content.index(\"<<\") :]\n elif followup_questions_started:\n followup_content += delta_content\n else:\n yield completion\n else:\n # Final chunk at end of streaming should contain usage\n # https://cookbook.openai.com/examples/how_to_stream_completions#4-how-to-get-token-usage-data-for-streamed-chat-completion-response\n if event_chunk.usage and extra_info.thoughts and self.include_token_usage:\n extra_info.thoughts[-1].update_token_usage(event_chunk.usage)\n yield {\"delta\": {\"role\": \"assistant\"}, \"context\": extra_info, \"session_state\": session_state}\n\n if followup_content:\n _, followup_questions = self.extract_followup_questions(followup_content)\n yield {\n \"delta\": {\"role\": \"assistant\"},\n \"context\": {\"context\": extra_info, \"followup_questions\": followup_questions},\n }\n\n async def run(\n self,\n messages: list[ChatCompletionMessageParam],\n session_state: Any = None,\n context: dict[str, Any] = {},\n ) -> dict[str, Any]:\n overrides = context.get(\"overrides\", {})\n auth_claims = context.get(\"auth_claims\", {})\n return await self.run_without_streaming(messages, overrides, auth_claims, session_state)\n\n async def run_stream(\n self,\n messages: list[ChatCompletionMessageParam],\n session_state: Any = None,\n context: dict[str, Any] = {},\n ) -> AsyncGenerator[dict[str, Any], None]:\n overrides = context.get(\"overrides\", {})\n auth_claims = context.get(\"auth_claims\", {})\n return self.run_with_streaming(messages, overrides, auth_claims, session_state)\n\n async def run_until_final_call(\n self,\n messages: list[ChatCompletionMessageParam],\n overrides: dict[str, Any],\n auth_claims: dict[str, Any],\n should_stream: bool = False,\n ) -> tuple[ExtraInfo, Awaitable[ChatCompletion] | Awaitable[AsyncStream[ChatCompletionChunk]]]:\n use_agentic_knowledgebase = True if overrides.get(\"use_agentic_knowledgebase\") else False\n original_user_query = messages[-1][\"content\"]\n\n reasoning_model_support = self.GPT_REASONING_MODELS.get(self.chatgpt_model)\n if reasoning_model_support and (not reasoning_model_support.streaming and should_stream):\n raise Exception(\n f\"{self.chatgpt_model} does not support streaming. Please use a different model or disable streaming.\"\n )\n if use_agentic_knowledgebase:\n if should_stream and overrides.get(\"use_web_source\"):\n raise Exception(\n \"Streaming is not supported with agentic retrieval when web source is enabled. Please disable streaming or web source.\"\n )\n extra_info = await self.run_agentic_retrieval_approach(messages, overrides, auth_claims)\n else:\n extra_info = await self.run_search_approach(messages, overrides, auth_claims)\n\n if extra_info.answer:\n # If agentic retrieval already provided an answer, skip final call to LLM\n async def return_answer() -> ChatCompletion:\n return ChatCompletion(\n id=\"no-final-call\",\n object=\"chat.completion\",\n created=0,\n model=self.chatgpt_model,\n choices=[\n Choice(\n message=ChatCompletionMessage(\n role=\"assistant\",\n content=extra_info.answer,\n ),\n finish_reason=\"stop\",\n index=0,\n )\n ],\n )\n\n return (extra_info, return_answer())\n\n messages = self.prompt_manager.build_conversation(\n system_template_path=\"chat_answer.system.jinja2\",\n system_template_variables=self.get_system_prompt_variables(overrides.get(\"prompt_template\"))\n | {\n \"include_follow_up_questions\": bool(overrides.get(\"suggest_followup_questions\")),\n \"image_sources\": extra_info.data_points.images,\n \"citations\": extra_info.data_points.citations,\n },\n user_template_path=\"chat_answer.user.jinja2\",\n user_template_variables={\n \"user_query\": original_user_query,\n \"text_sources\": extra_info.data_points.text,\n },\n user_image_sources=extra_info.data_points.images,\n past_messages=messages[:-1],\n )\n\n chat_coroutine = cast(\n Awaitable[ChatCompletion] | Awaitable[AsyncStream[ChatCompletionChunk]],\n self.create_chat_completion(\n self.chatgpt_deployment,\n self.chatgpt_model,\n messages,\n overrides,\n self.get_response_token_limit(self.chatgpt_model, 1024),\n should_stream,\n ),\n )\n extra_info.thoughts.append(\n self.format_thought_step_for_chatcompletion(\n title=\"Prompt to generate answer\",\n messages=messages,\n overrides=overrides,\n model=self.chatgpt_model,\n deployment=self.chatgpt_deployment,\n usage=None,\n )\n )\n return (extra_info, chat_coroutine)\n\n async def run_search_approach(\n self, messages: list[ChatCompletionMessageParam], overrides: dict[str, Any], auth_claims: dict[str, Any]\n ):\n use_text_search = overrides.get(\"retrieval_mode\") in [\"text\", \"hybrid\", None]\n use_vector_search = overrides.get(\"retrieval_mode\") in [\"vectors\", \"hybrid\", None]\n use_semantic_ranker = True if overrides.get(\"semantic_ranker\") else False\n use_semantic_captions = True if overrides.get(\"semantic_captions\") else False\n use_query_rewriting = True if overrides.get(\"query_rewriting\") else False\n top = overrides.get(\"top\", 3)\n minimum_search_score = overrides.get(\"minimum_search_score\", 0.0)\n minimum_reranker_score = overrides.get(\"minimum_reranker_score\", 0.0)\n search_index_filter = self.build_filter(overrides)\n access_token = auth_claims.get(\"access_token\")\n send_text_sources = overrides.get(\"send_text_sources\", True)\n send_image_sources = overrides.get(\"send_image_sources\", self.multimodal_enabled) and self.multimodal_enabled\n search_text_embeddings = overrides.get(\"search_text_embeddings\", True)\n search_image_embeddings = (\n overrides.get(\"search_image_embeddings\", self.multimodal_enabled) and self.multimodal_enabled\n )\n\n original_user_query = messages[-1][\"content\"]\n if not isinstance(original_user_query, str):\n raise ValueError(\"The most recent message content must be a string.\")\n\n # STEP 1: Generate an optimized keyword search query based on the chat history and the last question\n\n rewrite_result = await self.rewrite_query(\n prompt_template=\"query_rewrite.system.jinja2\",\n prompt_variables={\n \"user_query\": original_user_query,\n \"past_messages\": messages[:-1],\n },\n overrides=overrides,\n chatgpt_model=self.chatgpt_model,\n chatgpt_deployment=self.chatgpt_deployment,\n user_query=original_user_query,\n response_token_limit=self.get_response_token_limit(\n self.chatgpt_model, 100\n ), # Setting too low risks malformed JSON, setting too high may affect performance\n tools=self.query_rewrite_tools,\n temperature=0.0, # Minimize creativity for search query generation\n no_response_token=self.NO_RESPONSE,\n )\n\n query_text = rewrite_result.query\n\n # STEP 2: Retrieve relevant documents from the search index with the GPT optimized query\n\n vectors: list[VectorQuery] = []\n if use_vector_search:\n if search_text_embeddings:\n vectors.append(await self.compute_text_embedding(query_text))\n if search_image_embeddings:\n vectors.append(await self.compute_multimodal_embedding(query_text))\n\n results = await self.search(\n top,\n query_text,\n search_index_filter,\n vectors,\n use_text_search,\n use_vector_search,\n use_semantic_ranker,\n use_semantic_captions,\n minimum_search_score,\n minimum_reranker_score,\n use_query_rewriting,\n access_token,\n )\n\n # STEP 3: Generate a contextual and content specific answer using the search results and chat history\n data_points = await self.get_sources_content(\n results,\n use_semantic_captions,\n include_text_sources=send_text_sources,\n download_image_sources=send_image_sources,\n user_oid=auth_claims.get(\"oid\"),\n )\n extra_info = ExtraInfo(\n data_points,\n thoughts=[\n self.format_thought_step_for_chatcompletion(\n title=\"Prompt to generate search query\",\n messages=rewrite_result.messages,\n overrides=overrides,\n model=self.chatgpt_model,\n deployment=self.chatgpt_deployment,\n usage=rewrite_result.completion.usage,\n reasoning_effort=rewrite_result.reasoning_effort,\n ),\n ThoughtStep(\n \"Search using generated search query\",\n query_text,\n {\n \"use_semantic_captions\": use_semantic_captions,\n \"use_semantic_ranker\": use_semantic_ranker,\n \"use_query_rewriting\": use_query_rewriting,\n \"top\": top,\n \"filter\": search_index_filter,\n \"use_vector_search\": use_vector_search,\n \"use_text_search\": use_text_search,\n \"search_text_embeddings\": search_text_embeddings,\n \"search_image_embeddings\": search_image_embeddings,\n },\n ),\n ThoughtStep(\n \"Search results\",\n [result.serialize_for_results() for result in results],\n ),\n ],\n )\n return extra_info\n\n async def run_agentic_retrieval_approach(\n self,\n messages: list[ChatCompletionMessageParam],\n overrides: dict[str, Any],\n auth_claims: dict[str, Any],\n ):\n search_index_filter = self.build_filter(overrides)\n access_token = auth_claims.get(\"access_token\")\n minimum_reranker_score = overrides.get(\"minimum_reranker_score\", 0)\n send_text_sources = overrides.get(\"send_text_sources\", True)\n send_image_sources = overrides.get(\"send_image_sources\", self.multimodal_enabled) and self.multimodal_enabled\n retrieval_reasoning_effort = overrides.get(\"retrieval_reasoning_effort\", self.retrieval_reasoning_effort)\n # Overrides can only disable web source support configured at construction time.\n use_web_source = self.web_source_enabled\n override_use_web_source = overrides.get(\"use_web_source\")\n if isinstance(override_use_web_source, bool):\n use_web_source = use_web_source and override_use_web_source\n # Overrides can only disable sharepoint source support configured at construction time.\n use_sharepoint_source = self.use_sharepoint_source\n override_use_sharepoint_source = overrides.get(\"use_sharepoint_source\")\n if isinstance(override_use_sharepoint_source, bool):\n use_sharepoint_source = use_sharepoint_source and override_use_sharepoint_source\n if use_web_source and retrieval_reasoning_effort == \"minimal\":\n raise Exception(\"Web source cannot be used with minimal retrieval reasoning effort.\")\n\n selected_client, effective_web_source, effective_sharepoint_source = self._select_knowledgebase_client(\n use_web_source,\n use_sharepoint_source,\n )\n\n agentic_results = await self.run_agentic_retrieval(\n messages=messages,\n knowledgebase_client=selected_client,\n search_index_name=self.search_index_name,\n filter_add_on=search_index_filter,\n minimum_reranker_score=minimum_reranker_score,\n access_token=access_token,\n use_web_source=effective_web_source,\n use_sharepoint_source=effective_sharepoint_source,\n retrieval_reasoning_effort=retrieval_reasoning_effort,\n )\n\n data_points = await self.get_sources_content(\n agentic_results.documents,\n use_semantic_captions=False,\n include_text_sources=send_text_sources,\n download_image_sources=send_image_sources,\n user_oid=auth_claims.get(\"oid\"),\n web_results=agentic_results.web_results,\n sharepoint_results=agentic_results.sharepoint_results,\n )\n\n return ExtraInfo(\n data_points,\n thoughts=agentic_results.thoughts,\n answer=agentic_results.answer,\n )\n\n def _select_knowledgebase_client(\n self,\n use_web_source: bool,\n use_sharepoint_source: bool,\n ) -> tuple[KnowledgeBaseRetrievalClient, bool, bool]:\n if use_web_source and use_sharepoint_source:\n if self.knowledgebase_client_with_web_and_sharepoint:\n return self.knowledgebase_client_with_web_and_sharepoint, True, True\n if self.knowledgebase_client_with_web:\n return self.knowledgebase_client_with_web, True, False\n if self.knowledgebase_client_with_sharepoint:\n return self.knowledgebase_client_with_sharepoint, False, True\n\n if use_web_source and self.knowledgebase_client_with_web:\n return self.knowledgebase_client_with_web, True, False\n\n if use_sharepoint_source and self.knowledgebase_client_with_sharepoint:\n return self.knowledgebase_client_with_sharepoint, False, True\n\n if self.knowledgebase_client:\n return self.knowledgebase_client, False, False\n raise ValueError(\"Agentic retrieval requested but no knowledge base is configured\")\n" }, { "path": "app/backend/approaches/promptmanager.py", "content": "import json\nimport pathlib\nfrom typing import Any, cast\n\nfrom jinja2 import Environment, FileSystemLoader\nfrom openai.types.chat import (\n ChatCompletionMessageParam,\n ChatCompletionSystemMessageParam,\n ChatCompletionToolParam,\n ChatCompletionUserMessageParam,\n)\n\n\nclass PromptManager:\n \"\"\"Builds OpenAI chat completion messages from Jinja2 templates.\"\"\"\n\n PROMPTS_DIRECTORY = pathlib.Path(__file__).parent / \"prompts\"\n\n def __init__(self):\n self.env = Environment(\n loader=FileSystemLoader(self.PROMPTS_DIRECTORY),\n autoescape=False,\n trim_blocks=True,\n lstrip_blocks=True,\n )\n\n def build_system_prompt(\n self, template_path: str, template_variables: dict[str, Any]\n ) -> ChatCompletionSystemMessageParam:\n \"\"\"Build a single system message. Use for simple prompts like query rewrite.\n\n Args:\n template_path: Path to the system message template file\n template_variables: Dictionary of variables to pass to the template\n\n Returns:\n A system message\n \"\"\"\n content = self.env.get_template(template_path).render(**template_variables).strip()\n return {\"role\": \"system\", \"content\": content}\n\n def build_user_prompt(\n self,\n template_path: str,\n template_variables: dict[str, Any],\n image_sources: list[str] | None = None,\n ) -> ChatCompletionUserMessageParam:\n \"\"\"Build a single user message with optional images.\n\n Args:\n template_path: Path to the user message template file\n template_variables: Dictionary of variables to pass to the template\n image_sources: Optional list of image URLs to include in the message\n\n Returns:\n A user message\n \"\"\"\n user_text = self.env.get_template(template_path).render(**template_variables).strip()\n if image_sources:\n user_content: list[dict[str, Any]] = [{\"type\": \"text\", \"text\": user_text}]\n for image in image_sources:\n user_content.append({\"type\": \"image_url\", \"image_url\": {\"url\": image, \"detail\": \"auto\"}})\n return cast(ChatCompletionUserMessageParam, {\"role\": \"user\", \"content\": user_content})\n return {\"role\": \"user\", \"content\": user_text}\n\n def build_conversation(\n self,\n system_template_path: str,\n system_template_variables: dict[str, Any],\n user_template_path: str,\n user_template_variables: dict[str, Any],\n user_image_sources: list[str] | None = None,\n past_messages: list[ChatCompletionMessageParam] | None = None,\n ) -> list[ChatCompletionMessageParam]:\n \"\"\"Build a full conversation with system, history, and user message.\n\n Args:\n system_template_path: Path to the system message template file\n system_template_variables: Dictionary of variables to pass to the system template\n user_template_path: Path to the user message template file\n user_template_variables: Dictionary of variables to pass to the user template\n user_image_sources: Optional list of image URLs to include in the user message\n past_messages: Optional list of past messages to include as conversation history\n\n Returns:\n A list of ChatCompletionMessageParam messages\n \"\"\"\n messages: list[ChatCompletionMessageParam] = []\n\n # System message\n messages.append(self.build_system_prompt(system_template_path, system_template_variables))\n\n # Past messages (conversation history)\n for msg in past_messages or []:\n messages.append(cast(ChatCompletionMessageParam, msg))\n\n # User message (with optional images)\n messages.append(self.build_user_prompt(user_template_path, user_template_variables, user_image_sources))\n\n return messages\n\n def load_tools(self, path: str) -> list[ChatCompletionToolParam]:\n \"\"\"Load tools from a JSON file.\"\"\"\n with open(self.PROMPTS_DIRECTORY / path) as f:\n return cast(list[ChatCompletionToolParam], json.load(f))\n" }, { "path": "app/backend/approaches/prompts/chat_answer.system.jinja2", "content": "{% if override_prompt %}\n{{ override_prompt }}\n{% else %}\nAssistant helps the company employees with their questions about internal documents. Be brief in your answers.\nAnswer ONLY with the facts listed in the list of sources below. If there isn't enough information below, say you don't know. Do not generate answers that don't use the sources below.\nIf asking a clarifying question to the user would help, ask the question.\nIf the question is not in English, answer in the language used in the question.\nEach source has a name followed by colon and the actual information, always include the source name for each fact you use in the response. Use square brackets to reference the source, for example [info1.txt]. Don't combine sources, list each source separately, for example [info1.txt][info2.pdf].\n{% if image_sources %}\nEach image source has the document file name in the top left corner of the image with coordinates (10,10) pixels with format ,\nand the image figure name is right-aligned in the top right corner of the image.\nThe filename of the actual image is in the top right corner of the image and is in the format .\nEach text source starts in a new line and has the file name followed by colon and the actual information\nAlways include the source document filename for each fact you use in the response in the format: [document_name.ext#page=N].\nIf you are referencing an image, add the image filename in the format: [document_name.ext#page=N(image_name.png)].\n{% endif %}\n\nPossible citations for current question: {% for citation in citations %} [{{ citation }}] {% endfor %}\n{{ injected_prompt }}\n{% endif %}\n\n{% if include_follow_up_questions %}\nGenerate 3 very brief follow-up questions that the user would likely ask next.\nEnclose the follow-up questions in double angle brackets. Example:\n<>\n<>\n<>\nDo not repeat questions that have already been asked.\nMake sure the last question ends with \">>\".\n{% endif %}\n" }, { "path": "app/backend/approaches/prompts/chat_answer.user.jinja2", "content": "{{ user_query }}\n\n{% if text_sources is defined and text_sources %}\n\nSources:\n\n{% for text_source in text_sources %}\n{{ text_source }}\n\n{% endfor %}\n{% endif %}\n" }, { "path": "app/backend/approaches/prompts/chat_query_rewrite_tools.json", "content": "[{\n \"type\": \"function\",\n \"function\": {\n \"name\": \"search_sources\",\n \"description\": \"Retrieve sources from the Azure AI Search index\",\n \"parameters\": {\n \"type\": \"object\",\n \"properties\": {\n \"search_query\": {\n \"type\": \"string\",\n \"description\": \"Query string to retrieve documents from azure search eg: 'Health care plan'\"\n }\n },\n \"required\": [\"search_query\"]\n }\n }\n}]\n" }, { "path": "app/backend/approaches/prompts/query_rewrite.system.jinja2", "content": "Below is a history of the conversation so far, and a new question asked by the user that needs to be answered by searching in a knowledge base.\nYou have access to Azure AI Search index with 100's of documents.\nGenerate a search query based on the conversation and the new question.\nDo not include cited source filenames and document names e.g. info.txt or doc.pdf in the search query terms.\nDo not include any text inside [] or <<>> in the search query terms.\nDo not include any special characters like '+'.\nIf the question is not in English, translate the question to English before generating the search query.\nIf you cannot generate a search query, return just the number 0.\n\nGenerate search query for: How did crypto do last year?\n\nSearch query: Summarize Cryptocurrency Market Dynamics from last year\n\nGenerate search query for: What are my health plans?\n\nSearch query: Show available health plans\n\n{% if past_messages %}\nConversation history:\n\n{% for message in past_messages %}\n{{ message[\"role\"] }}:\n{{ message[\"content\"] }}\n{% endfor %}\n{% endif %}\n\nGenerate search query for: {{ user_query }}\n" }, { "path": "app/backend/chat_history/__init__.py", "content": "" }, { "path": "app/backend/chat_history/cosmosdb.py", "content": "import os\nimport time\nfrom typing import Any\n\nfrom azure.cosmos.aio import ContainerProxy, CosmosClient\nfrom azure.identity.aio import AzureDeveloperCliCredential, ManagedIdentityCredential\nfrom quart import Blueprint, current_app, jsonify, make_response, request\n\nfrom config import (\n CONFIG_CHAT_HISTORY_COSMOS_ENABLED,\n CONFIG_COSMOS_HISTORY_CLIENT,\n CONFIG_COSMOS_HISTORY_CONTAINER,\n CONFIG_COSMOS_HISTORY_VERSION,\n CONFIG_CREDENTIAL,\n)\nfrom decorators import authenticated\nfrom error import error_response\n\nchat_history_cosmosdb_bp = Blueprint(\"chat_history_cosmos\", __name__, static_folder=\"static\")\n\n\n@chat_history_cosmosdb_bp.post(\"/chat_history\")\n@authenticated\nasync def post_chat_history(auth_claims: dict[str, Any]):\n if not current_app.config[CONFIG_CHAT_HISTORY_COSMOS_ENABLED]:\n return jsonify({\"error\": \"Chat history not enabled\"}), 400\n\n container: ContainerProxy = current_app.config[CONFIG_COSMOS_HISTORY_CONTAINER]\n if not container:\n return jsonify({\"error\": \"Chat history not enabled\"}), 400\n\n entra_oid = auth_claims.get(\"oid\")\n if not entra_oid:\n return jsonify({\"error\": \"User OID not found\"}), 401\n\n try:\n request_json = await request.get_json()\n session_id = request_json.get(\"id\")\n message_pairs = request_json.get(\"answers\")\n first_question = message_pairs[0][0]\n title = first_question + \"...\" if len(first_question) > 50 else first_question\n timestamp = int(time.time() * 1000)\n\n # Insert the session item:\n session_item = {\n \"id\": session_id,\n \"version\": current_app.config[CONFIG_COSMOS_HISTORY_VERSION],\n \"session_id\": session_id,\n \"entra_oid\": entra_oid,\n \"type\": \"session\",\n \"title\": title,\n \"timestamp\": timestamp,\n }\n\n message_pair_items = []\n # Now insert a message item for each question/response pair:\n for ind, message_pair in enumerate(message_pairs):\n message_pair_items.append(\n {\n \"id\": f\"{session_id}-{ind}\",\n \"version\": current_app.config[CONFIG_COSMOS_HISTORY_VERSION],\n \"session_id\": session_id,\n \"entra_oid\": entra_oid,\n \"type\": \"message_pair\",\n \"question\": message_pair[0],\n \"response\": message_pair[1],\n }\n )\n\n batch_operations = [(\"upsert\", (session_item,))] + [\n (\"upsert\", (message_pair_item,)) for message_pair_item in message_pair_items\n ]\n await container.execute_item_batch(batch_operations=batch_operations, partition_key=[entra_oid, session_id])\n return jsonify({}), 201\n except Exception as error:\n return error_response(error, \"/chat_history\")\n\n\n@chat_history_cosmosdb_bp.get(\"/chat_history/sessions\")\n@authenticated\nasync def get_chat_history_sessions(auth_claims: dict[str, Any]):\n if not current_app.config[CONFIG_CHAT_HISTORY_COSMOS_ENABLED]:\n return jsonify({\"error\": \"Chat history not enabled\"}), 400\n\n container: ContainerProxy = current_app.config[CONFIG_COSMOS_HISTORY_CONTAINER]\n if not container:\n return jsonify({\"error\": \"Chat history not enabled\"}), 400\n\n entra_oid = auth_claims.get(\"oid\")\n if not entra_oid:\n return jsonify({\"error\": \"User OID not found\"}), 401\n\n try:\n count = int(request.args.get(\"count\", 10))\n continuation_token = request.args.get(\"continuation_token\")\n\n res = container.query_items(\n query=\"SELECT c.id, c.entra_oid, c.title, c.timestamp FROM c WHERE c.entra_oid = @entra_oid AND c.type = @type ORDER BY c.timestamp DESC\",\n parameters=[dict(name=\"@entra_oid\", value=entra_oid), dict(name=\"@type\", value=\"session\")],\n partition_key=[entra_oid],\n max_item_count=count,\n )\n\n pager = res.by_page(continuation_token)\n\n # Get the first page, and the continuation token\n sessions = []\n try:\n page = await pager.__anext__()\n continuation_token = pager.continuation_token\n\n async for item in page:\n sessions.append(\n {\n \"id\": item.get(\"id\"),\n \"entra_oid\": item.get(\"entra_oid\"),\n \"title\": item.get(\"title\", \"untitled\"),\n \"timestamp\": item.get(\"timestamp\"),\n }\n )\n\n # If there are no more pages, StopAsyncIteration is raised\n except StopAsyncIteration:\n continuation_token = None\n\n return jsonify({\"sessions\": sessions, \"continuation_token\": continuation_token}), 200\n\n except Exception as error:\n return error_response(error, \"/chat_history/sessions\")\n\n\n@chat_history_cosmosdb_bp.get(\"/chat_history/sessions/\")\n@authenticated\nasync def get_chat_history_session(auth_claims: dict[str, Any], session_id: str):\n if not current_app.config[CONFIG_CHAT_HISTORY_COSMOS_ENABLED]:\n return jsonify({\"error\": \"Chat history not enabled\"}), 400\n\n container: ContainerProxy = current_app.config[CONFIG_COSMOS_HISTORY_CONTAINER]\n if not container:\n return jsonify({\"error\": \"Chat history not enabled\"}), 400\n\n entra_oid = auth_claims.get(\"oid\")\n if not entra_oid:\n return jsonify({\"error\": \"User OID not found\"}), 401\n\n try:\n res = container.query_items(\n query=\"SELECT * FROM c WHERE c.session_id = @session_id AND c.type = @type\",\n parameters=[dict(name=\"@session_id\", value=session_id), dict(name=\"@type\", value=\"message_pair\")],\n partition_key=[entra_oid, session_id],\n )\n\n message_pairs = []\n async for page in res.by_page():\n async for item in page:\n message_pairs.append([item[\"question\"], item[\"response\"]])\n\n return (\n jsonify(\n {\n \"id\": session_id,\n \"entra_oid\": entra_oid,\n \"answers\": message_pairs,\n }\n ),\n 200,\n )\n except Exception as error:\n return error_response(error, f\"/chat_history/sessions/{session_id}\")\n\n\n@chat_history_cosmosdb_bp.delete(\"/chat_history/sessions/\")\n@authenticated\nasync def delete_chat_history_session(auth_claims: dict[str, Any], session_id: str):\n if not current_app.config[CONFIG_CHAT_HISTORY_COSMOS_ENABLED]:\n return jsonify({\"error\": \"Chat history not enabled\"}), 400\n\n container: ContainerProxy = current_app.config[CONFIG_COSMOS_HISTORY_CONTAINER]\n if not container:\n return jsonify({\"error\": \"Chat history not enabled\"}), 400\n\n entra_oid = auth_claims.get(\"oid\")\n if not entra_oid:\n return jsonify({\"error\": \"User OID not found\"}), 401\n\n try:\n res = container.query_items(\n query=\"SELECT c.id FROM c WHERE c.session_id = @session_id\",\n parameters=[dict(name=\"@session_id\", value=session_id)],\n partition_key=[entra_oid, session_id],\n )\n\n ids_to_delete = []\n async for page in res.by_page():\n async for item in page:\n ids_to_delete.append(item[\"id\"])\n\n batch_operations = [(\"delete\", (id,)) for id in ids_to_delete]\n await container.execute_item_batch(batch_operations=batch_operations, partition_key=[entra_oid, session_id])\n return await make_response(\"\", 204)\n except Exception as error:\n return error_response(error, f\"/chat_history/sessions/{session_id}\")\n\n\n@chat_history_cosmosdb_bp.before_app_serving\nasync def setup_clients():\n USE_CHAT_HISTORY_COSMOS = os.getenv(\"USE_CHAT_HISTORY_COSMOS\", \"\").lower() == \"true\"\n AZURE_COSMOSDB_ACCOUNT = os.getenv(\"AZURE_COSMOSDB_ACCOUNT\")\n AZURE_CHAT_HISTORY_DATABASE = os.getenv(\"AZURE_CHAT_HISTORY_DATABASE\")\n AZURE_CHAT_HISTORY_CONTAINER = os.getenv(\"AZURE_CHAT_HISTORY_CONTAINER\")\n\n azure_credential: AzureDeveloperCliCredential | ManagedIdentityCredential = current_app.config[CONFIG_CREDENTIAL]\n\n if USE_CHAT_HISTORY_COSMOS:\n current_app.logger.info(\"USE_CHAT_HISTORY_COSMOS is true, setting up CosmosDB client\")\n if not AZURE_COSMOSDB_ACCOUNT:\n raise ValueError(\"AZURE_COSMOSDB_ACCOUNT must be set when USE_CHAT_HISTORY_COSMOS is true\")\n if not AZURE_CHAT_HISTORY_DATABASE:\n raise ValueError(\"AZURE_CHAT_HISTORY_DATABASE must be set when USE_CHAT_HISTORY_COSMOS is true\")\n if not AZURE_CHAT_HISTORY_CONTAINER:\n raise ValueError(\"AZURE_CHAT_HISTORY_CONTAINER must be set when USE_CHAT_HISTORY_COSMOS is true\")\n cosmos_client = CosmosClient(\n url=f\"https://{AZURE_COSMOSDB_ACCOUNT}.documents.azure.com:443/\", credential=azure_credential\n )\n cosmos_db = cosmos_client.get_database_client(AZURE_CHAT_HISTORY_DATABASE)\n cosmos_container = cosmos_db.get_container_client(AZURE_CHAT_HISTORY_CONTAINER)\n\n current_app.config[CONFIG_COSMOS_HISTORY_CLIENT] = cosmos_client\n current_app.config[CONFIG_COSMOS_HISTORY_CONTAINER] = cosmos_container\n current_app.config[CONFIG_COSMOS_HISTORY_VERSION] = os.environ[\"AZURE_CHAT_HISTORY_VERSION\"]\n\n\n@chat_history_cosmosdb_bp.after_app_serving\nasync def close_clients():\n if current_app.config.get(CONFIG_COSMOS_HISTORY_CLIENT):\n cosmos_client: CosmosClient = current_app.config[CONFIG_COSMOS_HISTORY_CLIENT]\n await cosmos_client.close()\n" }, { "path": "app/backend/config.py", "content": "CONFIG_OPENAI_TOKEN = \"openai_token\"\nCONFIG_CREDENTIAL = \"azure_credential\"\nCONFIG_CHAT_APPROACH = \"chat_approach\"\nCONFIG_GLOBAL_BLOB_MANAGER = \"global_blob_manager\"\nCONFIG_USER_BLOB_MANAGER = \"user_blob_manager\"\nCONFIG_USER_UPLOAD_ENABLED = \"user_upload_enabled\"\nCONFIG_AUTH_CLIENT = \"auth_client\"\nCONFIG_SEMANTIC_RANKER_DEPLOYED = \"semantic_ranker_deployed\"\nCONFIG_QUERY_REWRITING_ENABLED = \"query_rewriting_enabled\"\nCONFIG_REASONING_EFFORT_ENABLED = \"reasoning_effort_enabled\"\nCONFIG_DEFAULT_REASONING_EFFORT = \"default_reasoning_effort\"\nCONFIG_DEFAULT_RETRIEVAL_REASONING_EFFORT = \"default_retrieval_reasoning_effort\"\nCONFIG_VECTOR_SEARCH_ENABLED = \"vector_search_enabled\"\nCONFIG_SEARCH_CLIENT = \"search_client\"\nCONFIG_OPENAI_CLIENT = \"openai_client\"\nCONFIG_KNOWLEDGEBASE_CLIENT = \"knowledgebase_client\"\nCONFIG_KNOWLEDGEBASE_CLIENT_WITH_WEB = \"knowledgebase_client_with_web\"\nCONFIG_KNOWLEDGEBASE_CLIENT_WITH_SHAREPOINT = \"knowledgebase_client_with_sharepoint\"\nCONFIG_KNOWLEDGEBASE_CLIENT_WITH_WEB_AND_SHAREPOINT = \"knowledgebase_client_with_web_and_sharepoint\"\nCONFIG_INGESTER = \"ingester\"\nCONFIG_LANGUAGE_PICKER_ENABLED = \"language_picker_enabled\"\nCONFIG_SPEECH_INPUT_ENABLED = \"speech_input_enabled\"\nCONFIG_SPEECH_OUTPUT_BROWSER_ENABLED = \"speech_output_browser_enabled\"\nCONFIG_SPEECH_OUTPUT_AZURE_ENABLED = \"speech_output_azure_enabled\"\nCONFIG_SPEECH_SERVICE_ID = \"speech_service_id\"\nCONFIG_SPEECH_SERVICE_LOCATION = \"speech_service_location\"\nCONFIG_SPEECH_SERVICE_TOKEN = \"speech_service_token\"\nCONFIG_SPEECH_SERVICE_VOICE = \"speech_service_voice\"\nCONFIG_STREAMING_ENABLED = \"streaming_enabled\"\nCONFIG_CHAT_HISTORY_BROWSER_ENABLED = \"chat_history_browser_enabled\"\nCONFIG_CHAT_HISTORY_COSMOS_ENABLED = \"chat_history_cosmos_enabled\"\nCONFIG_AGENTIC_KNOWLEDGEBASE_ENABLED = \"agentic_knowledgebase_enabled\"\nCONFIG_COSMOS_HISTORY_CLIENT = \"cosmos_history_client\"\nCONFIG_COSMOS_HISTORY_CONTAINER = \"cosmos_history_container\"\nCONFIG_COSMOS_HISTORY_VERSION = \"cosmos_history_version\"\nCONFIG_MULTIMODAL_ENABLED = \"multimodal_enabled\"\nCONFIG_RAG_SEARCH_TEXT_EMBEDDINGS = \"rag_search_text_embeddings\"\nCONFIG_RAG_SEARCH_IMAGE_EMBEDDINGS = \"rag_search_image_embeddings\"\nCONFIG_RAG_SEND_TEXT_SOURCES = \"rag_send_text_sources\"\nCONFIG_RAG_SEND_IMAGE_SOURCES = \"rag_send_image_sources\"\nCONFIG_WEB_SOURCE_ENABLED = \"web_source_enabled\"\nCONFIG_SHAREPOINT_SOURCE_ENABLED = \"sharepoint_source_enabled\"\n" }, { "path": "app/backend/core/__init__.py", "content": "" }, { "path": "app/backend/core/authentication.py", "content": "# Refactored from https://github.com/Azure-Samples/ms-identity-python-on-behalf-of\n\nimport base64\nimport json\nimport logging\nfrom typing import Any, Optional\n\nimport aiohttp\nimport jwt\nfrom azure.search.documents.aio import SearchClient\nfrom azure.search.documents.indexes.models import SearchIndex\nfrom cryptography.hazmat.primitives import serialization\nfrom cryptography.hazmat.primitives.asymmetric import rsa\nfrom msal import ConfidentialClientApplication\nfrom msal.token_cache import TokenCache\nfrom tenacity import (\n AsyncRetrying,\n retry_if_exception_type,\n stop_after_attempt,\n wait_random_exponential,\n)\n\n\n# AuthError is raised when the authentication token sent by the client UI cannot be parsed or there is an authentication error accessing the graph API\nclass AuthError(Exception):\n def __init__(self, error, status_code):\n self.error = error\n self.status_code = status_code\n\n def __str__(self) -> str:\n return self.error or \"\"\n\n\nclass AuthenticationHelper:\n scope: str = \"https://search.azure.com/.default\"\n\n def __init__(\n self,\n search_index: Optional[SearchIndex],\n use_authentication: bool,\n server_app_id: Optional[str],\n server_app_secret: Optional[str],\n client_app_id: Optional[str],\n tenant_id: Optional[str],\n enforce_access_control: bool = False,\n enable_unauthenticated_access: bool = False,\n ):\n self.use_authentication = use_authentication\n self.server_app_id = server_app_id\n self.server_app_secret = server_app_secret\n self.client_app_id = client_app_id\n self.tenant_id = tenant_id\n self.authority = f\"https://login.microsoftonline.com/{tenant_id}\"\n # Depending on if requestedAccessTokenVersion is 1 or 2, the issuer and audience of the token may be different\n # See https://learn.microsoft.com/graph/api/resources/apiapplication\n self.valid_issuers = [\n f\"https://sts.windows.net/{tenant_id}/\",\n f\"https://login.microsoftonline.com/{tenant_id}/v2.0\",\n ]\n self.valid_audiences = [f\"api://{server_app_id}\", str(server_app_id)]\n # See https://learn.microsoft.com/entra/identity-platform/access-tokens#validate-the-issuer for more information on token validation\n self.key_url = f\"{self.authority}/discovery/v2.0/keys\"\n\n if self.use_authentication:\n field_names = [field.name for field in search_index.fields] if search_index else []\n self.has_auth_fields = \"oids\" in field_names and \"groups\" in field_names\n self.enforce_access_control = enforce_access_control\n self.enable_unauthenticated_access = enable_unauthenticated_access\n self.confidential_client = ConfidentialClientApplication(\n server_app_id, authority=self.authority, client_credential=server_app_secret, token_cache=TokenCache()\n )\n else:\n self.has_auth_fields = False\n self.enforce_access_control = False\n self.enable_unauthenticated_access = True\n\n def get_auth_setup_for_client(self) -> dict[str, Any]:\n # returns MSAL.js settings used by the client app\n return {\n \"useLogin\": self.use_authentication, # Whether or not login elements are enabled on the UI\n \"requireAccessControl\": self.enforce_access_control, # Whether or not access control is required to access documents with access control lists\n \"enableUnauthenticatedAccess\": self.enable_unauthenticated_access, # Whether or not the user can access the app without login\n \"msalConfig\": {\n \"auth\": {\n \"clientId\": self.client_app_id, # Client app id used for login\n \"authority\": self.authority, # Directory to use for login https://learn.microsoft.com/entra/identity-platform/msal-client-application-configuration#authority\n \"redirectUri\": \"/redirect\", # Points to window.location.origin. You must register this URI on Azure Portal/App Registration.\n \"postLogoutRedirectUri\": \"/\", # Indicates the page to navigate after logout.\n \"navigateToLoginRequestUrl\": False, # If \"true\", will navigate back to the original request location before processing the auth code response.\n },\n \"cache\": {\n # Configures cache location. \"sessionStorage\" is more secure, but \"localStorage\" gives you SSO between tabs.\n \"cacheLocation\": \"localStorage\",\n # Set this to \"true\" if you are having issues on IE11 or Edge\n \"storeAuthStateInCookie\": False,\n },\n },\n \"loginRequest\": {\n # Scopes you add here will be prompted for user consent during sign-in.\n # By default, MSAL.js will add OIDC scopes (openid, profile, email) to any login request.\n # For more information about OIDC scopes, visit:\n # https://learn.microsoft.com/entra/identity-platform/permissions-consent-overview#openid-connect-scopes\n \"scopes\": [\".default\"],\n # Uncomment the following line to cause a consent dialog to appear on every login\n # For more information, please visit https://learn.microsoft.com/entra/identity-platform/v2-oauth2-auth-code-flow#request-an-authorization-code\n # \"prompt\": \"consent\"\n },\n \"tokenRequest\": {\n \"scopes\": [f\"api://{self.server_app_id}/access_as_user\"],\n },\n }\n\n @staticmethod\n def get_token_auth_header(headers: dict) -> str:\n # Obtains the Access Token from the Authorization Header\n auth = headers.get(\"Authorization\")\n if auth:\n parts = auth.split()\n\n if parts[0].lower() != \"bearer\":\n raise AuthError(error=\"Authorization header must start with Bearer\", status_code=401)\n elif len(parts) == 1:\n raise AuthError(error=\"Token not found\", status_code=401)\n elif len(parts) > 2:\n raise AuthError(error=\"Authorization header must be Bearer token\", status_code=401)\n\n token = parts[1]\n return token\n\n # App services built-in authentication passes the access token directly as a header\n # To learn more, please visit https://learn.microsoft.com/azure/app-service/configure-authentication-oauth-tokens\n token = headers.get(\"x-ms-token-aad-access-token\")\n if token:\n return token\n\n raise AuthError(error=\"Authorization header is expected\", status_code=401)\n\n async def get_auth_claims_if_enabled(self, headers: dict) -> dict[str, Any]:\n if not self.use_authentication:\n return {}\n try:\n # Read the authentication token from the authorization header and exchange it using the On Behalf Of Flow\n # The scope is set to Azure Search for authentication\n # https://learn.microsoft.com/entra/identity-platform/v2-oauth2-on-behalf-of-flow\n auth_token = AuthenticationHelper.get_token_auth_header(headers)\n # Validate the token before use\n await self.validate_access_token(auth_token)\n\n # Use the on-behalf-of-flow to acquire another token for use with Azure Search\n # See https://learn.microsoft.com/entra/identity-platform/v2-oauth2-on-behalf-of-flow for more information\n search_resource_access_token = self.confidential_client.acquire_token_on_behalf_of(\n user_assertion=auth_token, scopes=[self.scope]\n )\n if \"error\" in search_resource_access_token:\n raise AuthError(error=str(search_resource_access_token), status_code=401)\n\n id_token_claims = search_resource_access_token[\"id_token_claims\"]\n auth_claims = {\"oid\": id_token_claims[\"oid\"]}\n # Only pass on the access token if access control is required\n # See https://learn.microsoft.com/azure/search/search-query-access-control-rbac-enforcement for more information\n if self.enforce_access_control:\n access_token = search_resource_access_token[\"access_token\"]\n auth_claims[\"access_token\"] = access_token\n return auth_claims\n except AuthError as e:\n logging.exception(\"Exception getting authorization information - \" + json.dumps(e.error))\n if not self.enable_unauthenticated_access:\n raise\n return {}\n except Exception:\n logging.exception(\"Exception getting authorization information\")\n if not self.enable_unauthenticated_access:\n raise\n return {}\n\n async def check_path_auth(self, path: str, auth_claims: dict[str, Any], search_client: SearchClient) -> bool:\n # If there was no access control or no path, then the path is allowed\n if not self.enforce_access_control or len(path) == 0:\n return True\n\n # Remove any fragment string from the path before checking\n fragment_index = path.find(\"#\")\n if fragment_index != -1:\n path = path[:fragment_index]\n\n # Filter down to only chunks that are from the specific source file\n # Sourcepage is used for GPT-4V\n # Replace ' with '' to escape the single quote for the filter\n # https://learn.microsoft.com/azure/search/query-odata-filter-orderby-syntax#escaping-special-characters-in-string-constants\n path_for_filter = path.replace(\"'\", \"''\")\n filter = f\"(sourcefile eq '{path_for_filter}') or (sourcepage eq '{path_for_filter}')\"\n\n # If the filter returns any results, the user is allowed to access the document\n # Otherwise, access is denied\n results = await search_client.search(\n search_text=\"*\", top=1, filter=filter, x_ms_query_source_authorization=auth_claims[\"access_token\"]\n )\n allowed = False\n async for _ in results:\n allowed = True\n break\n\n return allowed\n\n async def create_pem_format(self, jwks, token):\n unverified_header = jwt.get_unverified_header(token)\n for key in jwks[\"keys\"]:\n if key[\"kid\"] == unverified_header[\"kid\"]:\n # Construct the RSA public key\n public_numbers = rsa.RSAPublicNumbers(\n e=int.from_bytes(base64.urlsafe_b64decode(key[\"e\"] + \"==\"), byteorder=\"big\"),\n n=int.from_bytes(base64.urlsafe_b64decode(key[\"n\"] + \"==\"), byteorder=\"big\"),\n )\n public_key = public_numbers.public_key()\n\n # Convert to PEM format\n pem_key = public_key.public_bytes(\n encoding=serialization.Encoding.PEM, format=serialization.PublicFormat.SubjectPublicKeyInfo\n )\n rsa_key = pem_key\n return rsa_key\n\n # See https://github.com/Azure-Samples/ms-identity-python-on-behalf-of/blob/939be02b11f1604814532fdacc2c2eccd198b755/FlaskAPI/helpers/authorization.py#L44\n async def validate_access_token(self, token: str):\n \"\"\"\n Validate an access token is issued by Entra\n \"\"\"\n jwks = None\n async for attempt in AsyncRetrying(\n retry=retry_if_exception_type(AuthError),\n wait=wait_random_exponential(min=15, max=60),\n stop=stop_after_attempt(5),\n ):\n with attempt:\n async with aiohttp.ClientSession() as session:\n async with session.get(url=self.key_url) as resp:\n resp_status = resp.status\n if resp_status in [500, 502, 503, 504]:\n raise AuthError(\n error=f\"Failed to get keys info: {await resp.text()}\", status_code=resp_status\n )\n jwks = await resp.json()\n\n if not jwks or \"keys\" not in jwks:\n raise AuthError(\"Unable to get keys to validate auth token.\", 401)\n\n rsa_key = None\n issuer = None\n audience = None\n try:\n unverified_claims = jwt.decode(token, options={\"verify_signature\": False})\n issuer = unverified_claims.get(\"iss\")\n audience = unverified_claims.get(\"aud\")\n rsa_key = await self.create_pem_format(jwks, token)\n except jwt.PyJWTError as exc:\n raise AuthError(\"Unable to parse authorization token.\", 401) from exc\n if not rsa_key:\n raise AuthError(\"Unable to find appropriate key\", 401)\n\n if issuer not in self.valid_issuers:\n raise AuthError(f\"Issuer {issuer} not in {','.join(self.valid_issuers)}\", 401)\n\n if audience not in self.valid_audiences:\n raise AuthError(\n f\"Audience {audience} not in {','.join(self.valid_audiences)}\",\n 401,\n )\n\n try:\n jwt.decode(token, rsa_key, algorithms=[\"RS256\"], audience=audience, issuer=issuer)\n except jwt.ExpiredSignatureError as jwt_expired_exc:\n raise AuthError(\"Token is expired\", 401) from jwt_expired_exc\n except (jwt.InvalidAudienceError, jwt.InvalidIssuerError) as jwt_claims_exc:\n raise AuthError(\n \"Incorrect claims: please check the audience and issuer\",\n 401,\n ) from jwt_claims_exc\n except Exception as exc:\n raise AuthError(\"Unable to parse authorization token.\", 401) from exc\n" }, { "path": "app/backend/core/sessionhelper.py", "content": "import uuid\nfrom typing import Optional\n\n\ndef create_session_id(\n config_chat_history_cosmos_enabled: bool, config_chat_history_browser_enabled: bool\n) -> Optional[str]:\n if config_chat_history_cosmos_enabled:\n return str(uuid.uuid4())\n if config_chat_history_browser_enabled:\n return str(uuid.uuid4())\n return None\n" }, { "path": "app/backend/custom_uvicorn_worker.py", "content": "from uvicorn.workers import UvicornWorker\n\nlogconfig_dict = {\n \"version\": 1,\n \"disable_existing_loggers\": False,\n \"formatters\": {\n \"default\": {\n \"()\": \"uvicorn.logging.DefaultFormatter\",\n \"format\": \"%(asctime)s - %(levelname)s - %(message)s\",\n },\n \"access\": {\n \"()\": \"uvicorn.logging.AccessFormatter\",\n \"format\": \"%(asctime)s - %(message)s\",\n },\n },\n \"handlers\": {\n \"default\": {\n \"formatter\": \"default\",\n \"class\": \"logging.StreamHandler\",\n \"stream\": \"ext://sys.stderr\",\n },\n \"access\": {\n \"formatter\": \"access\",\n \"class\": \"logging.StreamHandler\",\n \"stream\": \"ext://sys.stdout\",\n },\n },\n \"loggers\": {\n \"root\": {\"handlers\": [\"default\"]},\n \"uvicorn.error\": {\n \"level\": \"INFO\",\n \"handlers\": [\"default\"],\n \"propagate\": False,\n },\n \"uvicorn.access\": {\n \"level\": \"INFO\",\n \"handlers\": [\"access\"],\n \"propagate\": False,\n },\n },\n}\n\n\nclass CustomUvicornWorker(UvicornWorker):\n CONFIG_KWARGS = {\n \"log_config\": logconfig_dict,\n }\n" }, { "path": "app/backend/decorators.py", "content": "import logging\nfrom collections.abc import Callable\nfrom functools import wraps\nfrom typing import Any, TypeVar, cast\n\nfrom quart import abort, current_app, request\n\nfrom config import CONFIG_AUTH_CLIENT, CONFIG_SEARCH_CLIENT\nfrom core.authentication import AuthError\nfrom error import error_response\n\n\ndef authenticated_path(route_fn: Callable[[str, dict[str, Any]], Any]):\n \"\"\"\n Decorator for routes that request a specific file that might require access control enforcement\n \"\"\"\n\n @wraps(route_fn)\n async def auth_handler(path=\"\"):\n # If authentication is enabled, validate the user can access the file\n auth_helper = current_app.config[CONFIG_AUTH_CLIENT]\n search_client = current_app.config[CONFIG_SEARCH_CLIENT]\n authorized = False\n try:\n auth_claims = await auth_helper.get_auth_claims_if_enabled(request.headers)\n authorized = await auth_helper.check_path_auth(path, auth_claims, search_client)\n except AuthError:\n abort(403)\n except Exception as error:\n logging.exception(\"Problem checking path auth %s\", error)\n return error_response(error, route=\"/content\")\n\n if not authorized:\n abort(403)\n\n return await route_fn(path, auth_claims)\n\n return auth_handler\n\n\n_C = TypeVar(\"_C\", bound=Callable[..., Any])\n\n\ndef authenticated(route_fn: _C) -> _C:\n \"\"\"\n Decorator for routes that might require access control. Unpacks Authorization header information into an auth_claims dictionary\n \"\"\"\n\n @wraps(route_fn)\n async def auth_handler(*args, **kwargs):\n auth_helper = current_app.config[CONFIG_AUTH_CLIENT]\n try:\n auth_claims = await auth_helper.get_auth_claims_if_enabled(request.headers)\n except AuthError:\n abort(403)\n\n return await route_fn(auth_claims, *args, **kwargs)\n\n return cast(_C, auth_handler)\n" }, { "path": "app/backend/error.py", "content": "import logging\n\nfrom openai import APIError\nfrom quart import jsonify\n\nERROR_MESSAGE = \"\"\"The app encountered an error processing your request.\nIf you are an administrator of the app, check the application logs for a full traceback.\nError type: {error_type}\n\"\"\"\nERROR_MESSAGE_FILTER = \"\"\"Your message contains content that was flagged by the OpenAI content filter.\"\"\"\n\nERROR_MESSAGE_LENGTH = \"\"\"Your message exceeded the context length limit for this OpenAI model. Please shorten your message or change your settings to retrieve fewer search results.\"\"\"\n\n\ndef error_dict(error: Exception) -> dict:\n if isinstance(error, APIError) and error.code == \"content_filter\":\n return {\"error\": ERROR_MESSAGE_FILTER}\n if isinstance(error, APIError) and error.code == \"context_length_exceeded\":\n return {\"error\": ERROR_MESSAGE_LENGTH}\n return {\"error\": ERROR_MESSAGE.format(error_type=type(error))}\n\n\ndef error_response(error: Exception, route: str, status_code: int = 500):\n logging.exception(\"Exception in %s: %s\", route, error)\n if isinstance(error, APIError) and error.code == \"content_filter\":\n status_code = 400\n return jsonify(error_dict(error)), status_code\n" }, { "path": "app/backend/gunicorn.conf.py", "content": "import multiprocessing\nimport os\n\nmax_requests = 1000\nmax_requests_jitter = 50\nlog_file = \"-\"\nbind = \"0.0.0.0\"\n\ntimeout = 230\n# https://learn.microsoft.com/troubleshoot/azure/app-service/web-apps-performance-faqs#why-does-my-request-time-out-after-230-seconds\n\nnum_cpus = multiprocessing.cpu_count()\nif os.getenv(\"WEBSITE_SKU\") == \"LinuxFree\":\n # Free tier reports 2 CPUs but can't handle multiple workers\n workers = 1\nelse:\n workers = (num_cpus * 2) + 1\nworker_class = \"custom_uvicorn_worker.CustomUvicornWorker\"\n" }, { "path": "app/backend/load_azd_env.py", "content": "import json\nimport logging\nimport os\nimport subprocess\n\nfrom dotenv import load_dotenv\n\nlogger = logging.getLogger(\"scripts\")\n\n\ndef load_azd_env():\n \"\"\"Get path to current azd env file and load file using python-dotenv\"\"\"\n result = subprocess.run(\"azd env list -o json\", shell=True, capture_output=True, text=True)\n if result.returncode != 0:\n raise Exception(\"Error loading azd env\")\n env_json = json.loads(result.stdout)\n env_file_path = None\n for entry in env_json:\n if entry[\"IsDefault\"]:\n env_file_path = entry[\"DotEnvPath\"]\n if not env_file_path:\n raise Exception(\"No default azd env file found\")\n loading_mode = os.getenv(\"LOADING_MODE_FOR_AZD_ENV_VARS\") or \"override\"\n if loading_mode == \"no-override\":\n logger.info(\"Loading azd env from %s, but not overriding existing environment variables\", env_file_path)\n load_dotenv(env_file_path, override=False)\n else:\n logger.info(\"Loading azd env from %s, which may override existing environment variables\", env_file_path)\n load_dotenv(env_file_path, override=True)\n" }, { "path": "app/backend/main.py", "content": "import os\n\nfrom app import create_app\nfrom load_azd_env import load_azd_env\n\n# WEBSITE_HOSTNAME is always set by App Service, RUNNING_IN_PRODUCTION is set in main.bicep\nRUNNING_ON_AZURE = os.getenv(\"WEBSITE_HOSTNAME\") is not None or os.getenv(\"RUNNING_IN_PRODUCTION\") is not None\n\nif not RUNNING_ON_AZURE:\n load_azd_env()\n\napp = create_app()\n" }, { "path": "app/backend/prepdocs.py", "content": "import argparse\nimport asyncio\nimport logging\nimport os\nfrom typing import Optional\n\nimport aiohttp\nfrom azure.core.credentials_async import AsyncTokenCredential\nfrom azure.identity.aio import AzureDeveloperCliCredential\nfrom openai import AsyncOpenAI\nfrom rich.logging import RichHandler\n\nfrom load_azd_env import load_azd_env\nfrom prepdocslib.filestrategy import FileStrategy\nfrom prepdocslib.integratedvectorizerstrategy import (\n IntegratedVectorizerStrategy,\n)\nfrom prepdocslib.listfilestrategy import (\n LocalListFileStrategy,\n)\nfrom prepdocslib.servicesetup import (\n OpenAIHost,\n build_file_processors,\n clean_key_if_exists,\n setup_blob_manager,\n setup_embeddings_service,\n setup_figure_processor,\n setup_image_embeddings_service,\n setup_openai_client,\n setup_search_info,\n)\nfrom prepdocslib.strategy import DocumentAction, Strategy\n\nlogger = logging.getLogger(\"scripts\")\n\n\nasync def check_search_service_connectivity(search_service: str) -> bool:\n \"\"\"Check if the search service is accessible by hitting the /ping endpoint.\"\"\"\n ping_url = f\"https://{search_service}.search.windows.net/ping\"\n\n try:\n async with aiohttp.ClientSession() as session:\n async with session.get(ping_url, timeout=aiohttp.ClientTimeout(total=10)) as response:\n return response.status == 200\n except Exception as e:\n logger.debug(f\"Search service ping failed: {e}\")\n return False\n\n\ndef setup_list_file_strategy(\n azure_credential: AsyncTokenCredential,\n local_files: str,\n enable_global_documents: bool = False,\n):\n logger.info(\"Using local files: %s\", local_files)\n list_file_strategy = LocalListFileStrategy(\n path_pattern=local_files, enable_global_documents=enable_global_documents\n )\n return list_file_strategy\n\n\ndef setup_file_processors(\n azure_credential: AsyncTokenCredential,\n document_intelligence_service: Optional[str],\n document_intelligence_key: Optional[str] = None,\n local_pdf_parser: bool = False,\n local_html_parser: bool = False,\n use_content_understanding: bool = False,\n use_multimodal: bool = False,\n openai_client: Optional[AsyncOpenAI] = None,\n openai_model: Optional[str] = None,\n openai_deployment: Optional[str] = None,\n content_understanding_endpoint: Optional[str] = None,\n):\n \"\"\"Setup file processors and figure processor for document ingestion.\n\n Uses build_file_processors from servicesetup to ensure consistent parser/splitter\n selection logic with the Azure Functions cloud ingestion pipeline.\n \"\"\"\n file_processors = build_file_processors(\n azure_credential=azure_credential,\n document_intelligence_service=document_intelligence_service,\n document_intelligence_key=document_intelligence_key,\n use_local_pdf_parser=local_pdf_parser,\n use_local_html_parser=local_html_parser,\n process_figures=use_multimodal,\n )\n\n figure_processor = setup_figure_processor(\n credential=azure_credential,\n use_multimodal=use_multimodal,\n use_content_understanding=use_content_understanding,\n content_understanding_endpoint=content_understanding_endpoint,\n openai_client=openai_client,\n openai_model=openai_model,\n openai_deployment=openai_deployment,\n )\n\n return file_processors, figure_processor\n\n\nasync def main(strategy: Strategy, setup_index: bool = True):\n if setup_index:\n await strategy.setup()\n\n await strategy.run()\n\n\nif __name__ == \"__main__\": # pragma: no cover\n parser = argparse.ArgumentParser(\n description=\"Prepare documents by extracting content from PDFs, splitting content into sections, uploading to blob storage, and indexing in a search index.\"\n )\n parser.add_argument(\"files\", nargs=\"?\", help=\"Files to be processed\")\n\n parser.add_argument(\n \"--category\", help=\"Value for the category field in the search index for all sections indexed in this run\"\n )\n parser.add_argument(\n \"--disablebatchvectors\", action=\"store_true\", help=\"Don't compute embeddings in batch for the sections\"\n )\n parser.add_argument(\n \"--remove\",\n action=\"store_true\",\n help=\"Remove references to this document from blob storage and the search index\",\n )\n parser.add_argument(\n \"--removeall\",\n action=\"store_true\",\n help=\"Remove all blobs from blob storage and documents from the search index\",\n )\n\n # Optional key specification:\n parser.add_argument(\n \"--searchkey\",\n required=False,\n help=\"Optional. Use this Azure AI Search account key instead of the current user identity to login (use az login to set current user for Azure)\",\n )\n parser.add_argument(\n \"--storagekey\",\n required=False,\n help=\"Optional. Use this Azure Blob Storage account key instead of the current user identity to login (use az login to set current user for Azure)\",\n )\n parser.add_argument(\n \"--documentintelligencekey\",\n required=False,\n help=\"Optional. Use this Azure Document Intelligence account key instead of the current user identity to login (use az login to set current user for Azure)\",\n )\n\n parser.add_argument(\"--verbose\", \"-v\", action=\"store_true\", help=\"Verbose output\")\n args = parser.parse_args()\n\n if args.verbose:\n logging.basicConfig(format=\"%(message)s\", datefmt=\"[%X]\", handlers=[RichHandler(rich_tracebacks=True)])\n # We only set the level to INFO for our logger,\n # to avoid seeing the noisy INFO level logs from the Azure SDKs\n logger.setLevel(logging.DEBUG)\n\n load_azd_env()\n\n if os.getenv(\"USE_CLOUD_INGESTION\", \"\").lower() == \"true\":\n logger.warning(\n \"Cloud ingestion is enabled. Please use setup_cloud_ingestion.py instead of prepdocs.py. Exiting.\"\n )\n exit(0)\n\n if (\n os.getenv(\"AZURE_PUBLIC_NETWORK_ACCESS\") == \"Disabled\"\n and os.getenv(\"AZURE_USE_VPN_GATEWAY\", \"\").lower() != \"true\"\n ):\n logger.error(\"AZURE_PUBLIC_NETWORK_ACCESS is set to Disabled. Exiting.\")\n exit(0)\n\n use_int_vectorization = os.getenv(\"USE_FEATURE_INT_VECTORIZATION\", \"\").lower() == \"true\"\n use_multimodal = os.getenv(\"USE_MULTIMODAL\", \"\").lower() == \"true\"\n use_acls = os.getenv(\"AZURE_USE_AUTHENTICATION\", \"\").lower() == \"true\"\n enforce_access_control = os.getenv(\"AZURE_ENFORCE_ACCESS_CONTROL\", \"\").lower() == \"true\"\n enable_global_documents = os.getenv(\"AZURE_ENABLE_GLOBAL_DOCUMENT_ACCESS\", \"\").lower() == \"true\"\n dont_use_vectors = os.getenv(\"USE_VECTORS\", \"\").lower() == \"false\"\n use_agentic_knowledgebase = os.getenv(\"USE_AGENTIC_KNOWLEDGEBASE\", \"\").lower() == \"true\"\n use_content_understanding = os.getenv(\"USE_MEDIA_DESCRIBER_AZURE_CU\", \"\").lower() == \"true\"\n use_web_source = os.getenv(\"USE_WEB_SOURCE\", \"\").lower() == \"true\"\n use_sharepoint_source = os.getenv(\"USE_SHAREPOINT_SOURCE\", \"\").lower() == \"true\"\n\n # Use the current user identity to connect to Azure services. See infra/main.bicep for role assignments.\n if tenant_id := os.getenv(\"AZURE_TENANT_ID\"):\n logger.info(\"Connecting to Azure services using the azd credential for tenant %s\", tenant_id)\n azd_credential = AzureDeveloperCliCredential(tenant_id=tenant_id, process_timeout=60)\n else:\n logger.info(\"Connecting to Azure services using the azd credential for home tenant\")\n azd_credential = AzureDeveloperCliCredential(process_timeout=60)\n\n if args.removeall:\n document_action = DocumentAction.RemoveAll\n elif args.remove:\n document_action = DocumentAction.Remove\n else:\n document_action = DocumentAction.Add\n\n loop = asyncio.new_event_loop()\n asyncio.set_event_loop(loop)\n\n OPENAI_HOST = OpenAIHost(os.environ[\"OPENAI_HOST\"])\n # Check for incompatibility\n # if openai host is not azure\n if use_agentic_knowledgebase and OPENAI_HOST not in [OpenAIHost.AZURE, OpenAIHost.AZURE_CUSTOM]:\n raise Exception(\"Agentic retrieval requires an Azure OpenAI chat completion service\")\n\n search_info = setup_search_info(\n search_service=os.environ[\"AZURE_SEARCH_SERVICE\"],\n index_name=os.environ[\"AZURE_SEARCH_INDEX\"],\n use_agentic_knowledgebase=use_agentic_knowledgebase,\n knowledgebase_name=os.getenv(\"AZURE_SEARCH_KNOWLEDGEBASE_NAME\"),\n azure_openai_endpoint=os.environ[\"AZURE_OPENAI_ENDPOINT\"],\n azure_openai_knowledgebase_deployment=os.getenv(\"AZURE_OPENAI_KNOWLEDGEBASE_DEPLOYMENT\"),\n azure_openai_knowledgebase_model=os.getenv(\"AZURE_OPENAI_KNOWLEDGEBASE_MODEL\"),\n azure_credential=azd_credential,\n search_key=clean_key_if_exists(args.searchkey),\n azure_vision_endpoint=os.getenv(\"AZURE_VISION_ENDPOINT\"),\n )\n\n # Check search service connectivity\n search_service = os.environ[\"AZURE_SEARCH_SERVICE\"]\n is_connected = loop.run_until_complete(check_search_service_connectivity(search_service))\n\n if not is_connected:\n if os.getenv(\"AZURE_USE_PRIVATE_ENDPOINT\"):\n logger.error(\n \"Unable to connect to Azure AI Search service, which indicates either a network issue or a misconfiguration. You have AZURE_USE_PRIVATE_ENDPOINT enabled. Perhaps you're not yet connected to the VPN? Download the VPN configuration from the Azure portal here: %s\",\n os.getenv(\"AZURE_VPN_CONFIG_DOWNLOAD_LINK\"),\n )\n else:\n logger.error(\n \"Unable to connect to Azure AI Search service, which indicates either a network issue or a misconfiguration.\"\n )\n exit(1)\n\n blob_manager = setup_blob_manager(\n azure_credential=azd_credential,\n storage_account=os.environ[\"AZURE_STORAGE_ACCOUNT\"],\n storage_container=os.environ[\"AZURE_STORAGE_CONTAINER\"],\n storage_resource_group=os.environ[\"AZURE_STORAGE_RESOURCE_GROUP\"],\n subscription_id=os.environ[\"AZURE_SUBSCRIPTION_ID\"],\n storage_key=clean_key_if_exists(args.storagekey),\n image_storage_container=os.environ.get(\"AZURE_IMAGESTORAGE_CONTAINER\"), # Pass the image container\n )\n list_file_strategy = setup_list_file_strategy(\n azure_credential=azd_credential,\n local_files=args.files,\n enable_global_documents=enable_global_documents,\n )\n\n emb_model_dimensions = 1536\n if os.getenv(\"AZURE_OPENAI_EMB_DIMENSIONS\"):\n emb_model_dimensions = int(os.environ[\"AZURE_OPENAI_EMB_DIMENSIONS\"])\n\n openai_client, azure_openai_endpoint = setup_openai_client(\n openai_host=OPENAI_HOST,\n azure_credential=azd_credential,\n azure_openai_service=os.getenv(\"AZURE_OPENAI_SERVICE\"),\n azure_openai_custom_url=os.getenv(\"AZURE_OPENAI_CUSTOM_URL\"),\n azure_openai_api_key=os.getenv(\"AZURE_OPENAI_API_KEY_OVERRIDE\"),\n openai_api_key=clean_key_if_exists(os.getenv(\"OPENAI_API_KEY\")),\n openai_organization=os.getenv(\"OPENAI_ORGANIZATION\"),\n )\n openai_embeddings_service = None\n if not dont_use_vectors:\n openai_embeddings_service = setup_embeddings_service(\n OPENAI_HOST,\n openai_client,\n emb_model_name=os.environ[\"AZURE_OPENAI_EMB_MODEL_NAME\"],\n emb_model_dimensions=emb_model_dimensions,\n azure_openai_deployment=os.getenv(\"AZURE_OPENAI_EMB_DEPLOYMENT\"),\n azure_openai_endpoint=azure_openai_endpoint,\n disable_batch=args.disablebatchvectors,\n )\n\n ingestion_strategy: Strategy\n if use_int_vectorization:\n\n if not openai_embeddings_service or OPENAI_HOST not in [OpenAIHost.AZURE, OpenAIHost.AZURE_CUSTOM]:\n raise Exception(\"Integrated vectorization strategy requires an Azure OpenAI embeddings service\")\n\n ingestion_strategy = IntegratedVectorizerStrategy(\n search_info=search_info,\n list_file_strategy=list_file_strategy,\n blob_manager=blob_manager,\n document_action=document_action,\n embeddings=openai_embeddings_service,\n search_field_name_embedding=os.environ[\"AZURE_SEARCH_FIELD_NAME_EMBEDDING\"],\n subscription_id=os.environ[\"AZURE_SUBSCRIPTION_ID\"],\n search_analyzer_name=os.getenv(\"AZURE_SEARCH_ANALYZER_NAME\"),\n use_acls=use_acls,\n category=args.category,\n enforce_access_control=enforce_access_control,\n )\n else:\n file_processors, figure_processor = setup_file_processors(\n azure_credential=azd_credential,\n document_intelligence_service=os.getenv(\"AZURE_DOCUMENTINTELLIGENCE_SERVICE\"),\n document_intelligence_key=clean_key_if_exists(args.documentintelligencekey),\n local_pdf_parser=os.getenv(\"USE_LOCAL_PDF_PARSER\") == \"true\",\n local_html_parser=os.getenv(\"USE_LOCAL_HTML_PARSER\") == \"true\",\n use_content_understanding=use_content_understanding,\n use_multimodal=use_multimodal,\n content_understanding_endpoint=os.getenv(\"AZURE_CONTENTUNDERSTANDING_ENDPOINT\"),\n openai_client=openai_client,\n openai_model=os.getenv(\"AZURE_OPENAI_CHATGPT_MODEL\"),\n openai_deployment=os.getenv(\"AZURE_OPENAI_CHATGPT_DEPLOYMENT\") if OPENAI_HOST == OpenAIHost.AZURE else None,\n )\n\n image_embeddings_service = setup_image_embeddings_service(\n azure_credential=azd_credential,\n vision_endpoint=os.getenv(\"AZURE_VISION_ENDPOINT\"),\n use_multimodal=use_multimodal,\n )\n\n ingestion_strategy = FileStrategy(\n search_info=search_info,\n list_file_strategy=list_file_strategy,\n blob_manager=blob_manager,\n file_processors=file_processors,\n document_action=document_action,\n embeddings=openai_embeddings_service,\n image_embeddings=image_embeddings_service,\n search_analyzer_name=os.getenv(\"AZURE_SEARCH_ANALYZER_NAME\"),\n # Default to the previous field names for backward compatibility\n search_field_name_embedding=os.getenv(\"AZURE_SEARCH_FIELD_NAME_EMBEDDING\", \"embedding\"),\n use_acls=use_acls,\n category=args.category,\n figure_processor=figure_processor,\n enforce_access_control=enforce_access_control,\n use_web_source=use_web_source,\n use_sharepoint_source=use_sharepoint_source,\n )\n\n try:\n loop.run_until_complete(main(ingestion_strategy, setup_index=not args.remove and not args.removeall))\n finally:\n # Gracefully close any async clients/credentials to avoid noisy destructor warnings\n try:\n loop.run_until_complete(blob_manager.close_clients())\n loop.run_until_complete(openai_client.close())\n loop.run_until_complete(azd_credential.close())\n except Exception as e:\n logger.debug(f\"Failed to close async clients cleanly: {e}\")\n loop.close()\n" }, { "path": "app/backend/prepdocslib/__init__.py", "content": "" }, { "path": "app/backend/prepdocslib/blobmanager.py", "content": "import io\nimport logging\nimport os\nimport re\nfrom pathlib import Path\nfrom typing import IO, Any, Optional, TypedDict, cast\nfrom urllib.parse import unquote\n\nfrom azure.core.credentials_async import AsyncTokenCredential\nfrom azure.core.exceptions import ResourceNotFoundError\nfrom azure.storage.blob.aio import BlobServiceClient\nfrom azure.storage.filedatalake.aio import (\n DataLakeDirectoryClient,\n FileSystemClient,\n)\nfrom PIL import Image, ImageDraw, ImageFont\n\nfrom .listfilestrategy import File\n\nlogger = logging.getLogger(\"scripts\")\n\n\nclass BlobProperties(TypedDict, total=False):\n \"\"\"Properties of a blob, with optional fields for content settings\"\"\"\n\n content_settings: dict[str, Any]\n\n\nclass BaseBlobManager:\n \"\"\"\n Base class for Azure Storage operations, providing common file naming and path utilities\n \"\"\"\n\n @classmethod\n def sourcepage_from_file_page(cls, filename, page=0) -> str:\n if os.path.splitext(filename)[1].lower() == \".pdf\":\n return f\"{os.path.basename(filename)}#page={page+1}\"\n else:\n return os.path.basename(filename)\n\n @classmethod\n def blob_name_from_file_name(cls, filename) -> str:\n return os.path.basename(filename)\n\n @classmethod\n def add_image_citation(\n cls, image_bytes: bytes, document_filename: str, image_filename: str, page_num: int\n ) -> bytes:\n \"\"\"\n Adds citation text to an image from a document.\n Args:\n image_bytes: The original image bytes\n document_filename: The name of the document containing the image\n image_filename: The name of the image file\n page_num: The page number where the image appears\n Returns:\n A tuple containing (BytesIO of the modified image, format of the image)\n \"\"\"\n # Load and modify the image to add text\n image = Image.open(io.BytesIO(image_bytes))\n line_height = 30\n text_height = line_height * 2 # Two lines of text\n new_img = Image.new(\"RGB\", (image.width, image.height + text_height), \"white\")\n new_img.paste(image, (0, text_height))\n\n # Add text\n draw = ImageDraw.Draw(new_img)\n sourcepage = cls.sourcepage_from_file_page(document_filename, page=page_num)\n text = sourcepage\n figure_text = image_filename\n\n # Load the Jupiteroid font which is included in the repo\n font_path = Path(__file__).parent / \"Jupiteroid-Regular.ttf\"\n font = ImageFont.truetype(str(font_path), 20) # Slightly smaller font for better fit\n\n # Calculate text widths for right alignment\n fig_width = draw.textlength(figure_text, font=font)\n\n # Left align document name, right align figure name\n padding = 20 # Padding from edges\n draw.text((padding, 5), text, font=font, fill=\"black\") # Left aligned\n draw.text(\n (new_img.width - fig_width - padding, line_height + 5), figure_text, font=font, fill=\"black\"\n ) # Right aligned\n\n # Convert back to bytes\n output = io.BytesIO()\n format = image.format or \"PNG\"\n new_img.save(output, format=format)\n\n return output.getvalue()\n\n async def upload_document_image(\n self,\n document_filename: str,\n image_bytes: bytes,\n image_filename: str,\n image_page_num: int,\n user_oid: Optional[str] = None,\n ) -> Optional[str]:\n raise NotImplementedError(\"Subclasses must implement this method\")\n\n async def download_blob(\n self, blob_path: str, user_oid: Optional[str] = None, container: Optional[str] = None\n ) -> Optional[tuple[bytes, BlobProperties]]:\n \"\"\"\n Downloads a blob from Azure Storage.\n If user_oid is provided, it checks if the blob belongs to the user.\n\n Args:\n blob_path: The path to the blob in the storage\n user_oid: The user's object ID (optional)\n container: Optional container name override (defaults to the manager's configured container)\n\n Returns:\n Optional[tuple[bytes, BlobProperties]]:\n - A tuple containing the blob content as bytes and the blob properties\n - None if blob not found or access denied\n \"\"\"\n raise NotImplementedError(\"Subclasses must implement this method\")\n\n\nclass AdlsBlobManager(BaseBlobManager):\n \"\"\"\n Manager for Azure Data Lake Storage blob operations, particularly for user-specific file operations.\n Documents are stored directly in the user's directory for backwards compatibility.\n Images are stored in a separate images subdirectory for better organization.\n \"\"\"\n\n def __init__(self, endpoint: str, container: str, credential: AsyncTokenCredential):\n \"\"\"\n Initializes the AdlsBlobManager with the necessary parameters.\n\n Args:\n endpoint: The ADLS endpoint URL\n container: The name of the container (file system)\n credential: The credential for accessing ADLS\n \"\"\"\n self.endpoint = endpoint\n self.container = container\n self.credential = credential\n self.file_system_client = FileSystemClient(\n account_url=self.endpoint,\n file_system_name=self.container,\n credential=self.credential,\n )\n\n async def close_clients(self):\n await self.file_system_client.close()\n\n async def _ensure_directory(self, directory_path: str, user_oid: str) -> DataLakeDirectoryClient:\n \"\"\"\n Ensures that a directory path exists and has proper permissions.\n Creates the entire path in a single operation if it doesn't exist.\n\n Args:\n directory_path: Full path of directory to create (e.g., 'user123/images/mydoc')\n user_oid: The owner to set for all created directories\n \"\"\"\n directory_client = self.file_system_client.get_directory_client(directory_path)\n try:\n await directory_client.get_directory_properties()\n # Check directory properties to ensure it has the correct owner\n props = await directory_client.get_access_control()\n if props.get(\"owner\") != user_oid:\n raise PermissionError(f\"User {user_oid} does not have permission to access {directory_path}\")\n except ResourceNotFoundError:\n logger.info(\"Creating directory path %s\", directory_path)\n await directory_client.create_directory()\n await directory_client.set_access_control(owner=user_oid)\n return directory_client\n\n async def upload_blob(self, file: File | IO, filename: str, user_oid: str) -> str:\n \"\"\"\n Uploads a file directly to the user's directory in ADLS (no subdirectory).\n\n Args:\n file: Either a File object or an IO object to upload\n filename: The name of the file to upload\n user_oid: The user's object ID\n\n Returns:\n str: The URL of the uploaded file, with forward slashes (not URL-encoded)\n \"\"\"\n # Ensure user directory exists but don't create a subdirectory\n user_directory_client = await self._ensure_directory(directory_path=user_oid, user_oid=user_oid)\n\n # Create file directly in user directory\n file_client = user_directory_client.get_file_client(filename)\n\n # Handle both File and IO objects\n if isinstance(file, File):\n file_io = file.content\n else:\n file_io = file\n\n # Ensure the file is at the beginning\n file_io.seek(0)\n\n await file_client.upload_data(file_io, overwrite=True)\n\n # Reset the file position for any subsequent reads\n file_io.seek(0)\n\n # Decode the URL to convert %2F back to / and other escaped characters\n return unquote(file_client.url)\n\n def _get_image_directory_path(self, document_filename: str, user_oid: str, page_num: Optional[int] = None) -> str:\n \"\"\"\n Returns the standardized path for storing document images.\n\n Args:\n document_filename: The name of the document\n user_oid: The user's object ID\n page_num: Optional page number. If provided, includes a page-specific subdirectory\n\n Returns:\n str: Full path to the image directory\n \"\"\"\n if page_num is not None:\n return f\"{user_oid}/images/{document_filename}/page_{page_num}\"\n return f\"{user_oid}/images/{document_filename}\"\n\n async def upload_document_image(\n self,\n document_filename: str,\n image_bytes: bytes,\n image_filename: str,\n image_page_num: int,\n user_oid: Optional[str] = None,\n ) -> Optional[str]:\n \"\"\"\n Uploads an image from a document to ADLS in a directory structure:\n {user_oid}/{document_name}/images/{image_name}\n This structure allows for easy cleanup when documents are deleted.\n\n Args:\n document_filename: The name of the document containing the image\n image_bytes: The image data to upload\n image_filename: The name to give the image file\n image_page_num: The page number where the image appears in the document\n user_oid: The user's object ID\n\n Returns:\n str: The URL of the uploaded file, with forward slashes (not URL-encoded)\n \"\"\"\n if user_oid is None:\n raise ValueError(\"user_oid must be provided for user-specific operations.\")\n await self._ensure_directory(directory_path=user_oid, user_oid=user_oid)\n image_directory_path = self._get_image_directory_path(document_filename, user_oid, image_page_num)\n image_directory_client = await self._ensure_directory(directory_path=image_directory_path, user_oid=user_oid)\n file_client = image_directory_client.get_file_client(image_filename)\n image_bytes = BaseBlobManager.add_image_citation(image_bytes, document_filename, image_filename, image_page_num)\n logger.info(\"Uploading document image '%s' to '%s'\", image_filename, image_directory_path)\n await file_client.upload_data(image_bytes, overwrite=True, metadata={\"UploadedBy\": user_oid})\n return unquote(file_client.url)\n\n async def download_blob(\n self, blob_path: str, user_oid: Optional[str] = None, container: Optional[str] = None\n ) -> Optional[tuple[bytes, BlobProperties]]:\n \"\"\"\n Downloads a blob from Azure Data Lake Storage.\n\n Args:\n blob_path: The path to the blob in the format {user_oid}/{document_name}/images/{image_name}\n user_oid: The user's object ID\n container: Optional filesystem name override (ignored; this manager uses its configured filesystem)\n\n Returns:\n Optional[tuple[bytes, BlobProperties]]:\n - A tuple containing the blob content as bytes and the blob properties as a dictionary\n - None if blob not found or access denied\n \"\"\"\n if user_oid is None:\n logger.warning(\"user_oid must be provided for Data Lake Storage operations.\")\n return None\n\n # Get the directory path and file name from the blob path\n path_parts = blob_path.split(\"/\")\n if len(path_parts) < 2:\n # If no slashes in path, we assume it's a file in the user's root directory\n filename = blob_path\n directory_path = user_oid\n else:\n # First verify that the root directory matches the user_oid\n root_dir = path_parts[0]\n if root_dir != user_oid:\n logger.warning(f\"User {user_oid} does not have permission to access {blob_path}\")\n return None\n\n # Get the directory client for the full path except the filename\n directory_path = \"/\".join(path_parts[:-1])\n filename = path_parts[-1]\n\n try:\n user_directory_client = await self._ensure_directory(directory_path=directory_path, user_oid=user_oid)\n file_client = user_directory_client.get_file_client(filename)\n download_response = await file_client.download_file()\n content = await download_response.readall()\n\n # Convert FileProperties to our BlobProperties format\n properties: BlobProperties = {\n \"content_settings\": {\n \"content_type\": download_response.properties.get(\"content_type\", \"application/octet-stream\")\n }\n }\n\n return content, properties\n except ResourceNotFoundError:\n logger.warning(f\"Directory or file not found: {directory_path}/{filename}\")\n return None\n except Exception as e:\n logging.error(f\"Error accessing directory {directory_path}: {str(e)}\")\n return None\n\n async def remove_blob(self, filename: str, user_oid: str) -> None:\n \"\"\"\n Deletes a file from the user's directory in ADLS and any associated image directories.\n The following will be deleted:\n - {user_oid}/{filename}\n - {user_oid}/images/{filename}/* (recursively)\n\n Args:\n filename: The name of the file to delete\n user_oid: The user's object ID\n\n Raises:\n ResourceNotFoundError: If the file does not exist\n \"\"\"\n # Ensure the user directory exists\n user_directory_client = await self._ensure_directory(directory_path=user_oid, user_oid=user_oid)\n # Delete the main document file\n file_client = user_directory_client.get_file_client(filename)\n await file_client.delete_file()\n\n # Try to delete any associated image directories\n image_directory_path = self._get_image_directory_path(filename, user_oid)\n try:\n image_directory_client = await self._ensure_directory(\n directory_path=image_directory_path, user_oid=user_oid\n )\n await image_directory_client.delete_directory()\n logger.info(f\"Deleted associated image directory: {image_directory_path}\")\n except ResourceNotFoundError:\n # It's okay if there was no image directory\n logger.debug(f\"No image directory found at {image_directory_path}\")\n pass\n\n async def list_blobs(self, user_oid: str) -> list[str]:\n \"\"\"\n Lists the uploaded documents for the given user.\n Only returns files directly in the user's directory, not in subdirectories.\n Excludes image files and the images directory.\n\n Args:\n user_oid: The user's object ID\n\n Returns:\n list[str]: List of filenames that belong to the user\n \"\"\"\n await self._ensure_directory(directory_path=user_oid, user_oid=user_oid)\n files = []\n try:\n all_paths = self.file_system_client.get_paths(path=user_oid, recursive=True)\n async for path in all_paths:\n # Split path into parts (user_oid/filename or user_oid/directory/files)\n path_parts = path.name.split(\"/\", 1)\n if len(path_parts) != 2:\n continue\n\n filename = path_parts[1]\n # Only include files that are:\n # 1. Directly in the user's directory (no additional slashes)\n # 2. Not image files\n # 3. Not in a directory containing 'images'\n if (\n \"/\" not in filename\n and not any(filename.lower().endswith(ext) for ext in [\".png\", \".jpg\", \".jpeg\", \".gif\", \".bmp\"])\n and \"images\" not in filename\n ):\n files.append(filename)\n except ResourceNotFoundError as error:\n if error.status_code != 404:\n logger.exception(\"Error listing uploaded files\", error)\n # Return empty list for 404 (no directory) as this is expected for new users\n return files\n\n\nclass BlobManager(BaseBlobManager):\n \"\"\"\n Class to manage uploading and deleting blobs containing citation information from a blob storage account\n \"\"\"\n\n def __init__(\n self,\n endpoint: str,\n container: str,\n credential: AsyncTokenCredential | str,\n image_container: Optional[str] = None,\n account: Optional[str] = None,\n resource_group: Optional[str] = None,\n subscription_id: Optional[str] = None,\n ):\n self.endpoint = endpoint\n self.credential = credential\n self.account = account\n self.container = container\n self.resource_group = resource_group\n self.subscription_id = subscription_id\n self.image_container = image_container\n self.blob_service_client = BlobServiceClient(\n account_url=self.endpoint, credential=self.credential, max_single_put_size=4 * 1024 * 1024\n )\n\n async def close_clients(self):\n await self.blob_service_client.close()\n\n def get_managedidentity_connectionstring(self):\n if not self.account or not self.resource_group or not self.subscription_id:\n raise ValueError(\"Account, resource group, and subscription ID must be set to generate connection string.\")\n return f\"ResourceId=/subscriptions/{self.subscription_id}/resourceGroups/{self.resource_group}/providers/Microsoft.Storage/storageAccounts/{self.account};\"\n\n async def upload_blob(self, file: File) -> str:\n container_client = self.blob_service_client.get_container_client(self.container)\n if not await container_client.exists():\n await container_client.create_container()\n\n # Re-open and upload the original file\n # URL may be a path to a local file or already set to a blob URL\n if file.url is None or os.path.exists(file.url):\n with open(file.content.name, \"rb\") as reopened_file:\n blob_name = self.blob_name_from_file_name(file.content.name)\n logger.info(\"Uploading blob for document '%s'\", blob_name)\n blob_client = await container_client.upload_blob(blob_name, reopened_file, overwrite=True)\n file.url = blob_client.url\n\n if file.url is None:\n raise ValueError(\"file.url must be set after upload\")\n return unquote(file.url)\n\n async def upload_document_image(\n self,\n document_filename: str,\n image_bytes: bytes,\n image_filename: str,\n image_page_num: int,\n user_oid: Optional[str] = None,\n ) -> Optional[str]:\n if self.image_container is None:\n raise ValueError(\n \"Image container name is not set. Re-run `azd provision` to automatically set up the images container.\"\n )\n if user_oid is not None:\n raise ValueError(\n \"user_oid is not supported for BlobManager. Use AdlsBlobManager for user-specific operations.\"\n )\n container_client = self.blob_service_client.get_container_client(self.image_container)\n if not await container_client.exists():\n await container_client.create_container()\n image_bytes = self.add_image_citation(image_bytes, document_filename, image_filename, image_page_num)\n blob_name = f\"{self.blob_name_from_file_name(document_filename)}/page{image_page_num}/{image_filename}\"\n logger.info(\"Uploading blob for document image '%s'\", blob_name)\n blob_client = await container_client.upload_blob(blob_name, image_bytes, overwrite=True)\n return blob_client.url\n\n async def download_blob(\n self, blob_path: str, user_oid: Optional[str] = None, container: Optional[str] = None\n ) -> Optional[tuple[bytes, BlobProperties]]:\n \"\"\"\n Downloads a blob from Azure Blob Storage.\n\n Args:\n blob_path: The path to the blob in the storage\n user_oid: Not used in BlobManager, but included for API compatibility\n container: Optional container name override (defaults to self.container)\n\n Returns:\n Optional[tuple[bytes, BlobProperties]]:\n - A tuple containing the blob content as bytes and the blob properties\n - None if blob not found\n\n Raises:\n ValueError: If user_oid is provided (not supported for BlobManager)\n \"\"\"\n if user_oid is not None:\n raise ValueError(\n \"user_oid is not supported for BlobManager. Use AdlsBlobManager for user-specific operations.\"\n )\n container_client = self.blob_service_client.get_container_client(container or self.container)\n if not await container_client.exists():\n return None\n if len(blob_path) == 0:\n logger.warning(\"Blob path is empty\")\n return None\n\n blob_client = container_client.get_blob_client(blob_path)\n try:\n download_response = await blob_client.download_blob()\n if not download_response.properties:\n logger.warning(f\"No blob exists for {blob_path}\")\n return None\n\n # Get the content as bytes\n content = await download_response.readall()\n\n # Convert BlobProperties to our internal BlobProperties format\n properties: BlobProperties = {\n \"content_settings\": {\n \"content_type\": (\n download_response.properties.content_settings.content_type\n if (\n hasattr(download_response.properties, \"content_settings\")\n and download_response.properties.content_settings\n and hasattr(download_response.properties.content_settings, \"content_type\")\n )\n else \"application/octet-stream\"\n )\n }\n }\n\n return cast(bytes, content), properties\n except ResourceNotFoundError:\n logger.warning(\"Blob not found: %s\", blob_path)\n return None\n\n async def remove_blob(self, path: Optional[str] = None):\n container_client = self.blob_service_client.get_container_client(self.container)\n if not await container_client.exists():\n return\n if path is None:\n prefix = None\n blobs = container_client.list_blob_names()\n else:\n prefix = os.path.splitext(os.path.basename(path))[0]\n blobs = container_client.list_blob_names(name_starts_with=os.path.splitext(os.path.basename(prefix))[0])\n async for blob_path in blobs:\n # This still supports PDFs split into individual pages, but we could remove in future to simplify code\n if (\n prefix is not None\n and (not re.match(rf\"{prefix}-\\d+\\.pdf\", blob_path) or not re.match(rf\"{prefix}-\\d+\\.png\", blob_path))\n ) or (path is not None and blob_path == os.path.basename(path)):\n continue\n logger.info(\"Removing blob %s\", blob_path)\n await container_client.delete_blob(blob_path)\n" }, { "path": "app/backend/prepdocslib/cloudingestionstrategy.py", "content": "\"\"\"Cloud ingestion strategy using Azure AI Search custom skills.\"\"\"\n\nimport logging\nfrom dataclasses import dataclass\nfrom datetime import timedelta\n\nfrom azure.search.documents.indexes._generated.models import (\n NativeBlobSoftDeleteDeletionDetectionPolicy,\n)\nfrom azure.search.documents.indexes.models import (\n IndexingParameters,\n IndexingParametersConfiguration,\n IndexProjectionMode,\n InputFieldMappingEntry,\n OutputFieldMappingEntry,\n SearchIndexer,\n SearchIndexerDataContainer,\n SearchIndexerDataSourceConnection,\n SearchIndexerDataSourceType,\n SearchIndexerDataUserAssignedIdentity,\n SearchIndexerIndexProjection,\n SearchIndexerIndexProjectionSelector,\n SearchIndexerIndexProjectionsParameters,\n SearchIndexerSkillset,\n ShaperSkill,\n WebApiSkill,\n)\n\nfrom .blobmanager import BlobManager\nfrom .embeddings import OpenAIEmbeddings\nfrom .listfilestrategy import ListFileStrategy\nfrom .searchmanager import SearchManager\nfrom .strategy import DocumentAction, SearchInfo, Strategy\n\nlogger = logging.getLogger(\"scripts\")\n\nDEFAULT_SKILL_TIMEOUT = timedelta(seconds=230)\nDEFAULT_BATCH_SIZE = 1\n\n\n@dataclass(slots=True)\nclass SkillConfig:\n \"\"\"Configuration for a custom Web API skill.\"\"\"\n\n name: str\n description: str\n uri: str\n auth_resource_id: str\n\n\nclass CloudIngestionStrategy(Strategy): # pragma: no cover\n \"\"\"Ingestion strategy that wires Azure Function custom skills into an indexer.\"\"\"\n\n def __init__(\n self,\n *,\n list_file_strategy: ListFileStrategy,\n blob_manager: BlobManager,\n search_info: SearchInfo,\n embeddings: OpenAIEmbeddings,\n search_field_name_embedding: str,\n document_extractor_uri: str,\n document_extractor_auth_resource_id: str,\n figure_processor_uri: str,\n figure_processor_auth_resource_id: str,\n text_processor_uri: str,\n text_processor_auth_resource_id: str,\n subscription_id: str,\n document_action: DocumentAction = DocumentAction.Add,\n search_analyzer_name: str | None = None,\n use_acls: bool = False,\n use_multimodal: bool = False,\n enforce_access_control: bool = False,\n use_web_source: bool = False,\n search_user_assigned_identity_resource_id: str,\n ) -> None:\n self.list_file_strategy = list_file_strategy\n self.blob_manager = blob_manager\n self.document_action = document_action\n self.embeddings = embeddings\n self.search_field_name_embedding = search_field_name_embedding\n self.search_info = search_info\n self.search_analyzer_name = search_analyzer_name\n self.use_acls = use_acls\n self.use_multimodal = use_multimodal\n self.enforce_access_control = enforce_access_control\n self.use_web_source = use_web_source\n self.subscription_id = subscription_id\n\n prefix = f\"{self.search_info.index_name}-cloud\"\n self.skillset_name = f\"{prefix}-skillset\"\n self.indexer_name = f\"{prefix}-indexer\"\n self.data_source_name = f\"{prefix}-blob\"\n\n self.document_extractor = SkillConfig(\n name=f\"{prefix}-document-extractor-skill\",\n description=\"Custom skill that downloads and parses source documents\",\n uri=document_extractor_uri,\n auth_resource_id=document_extractor_auth_resource_id,\n )\n self.figure_processor = SkillConfig(\n name=f\"{prefix}-figure-processor-skill\",\n description=\"Custom skill that enriches individual figures\",\n uri=figure_processor_uri,\n auth_resource_id=figure_processor_auth_resource_id,\n )\n self.text_processor = SkillConfig(\n name=f\"{prefix}-text-processor-skill\",\n description=\"Custom skill that merges figures, chunks text, and generates embeddings\",\n uri=text_processor_uri,\n auth_resource_id=text_processor_auth_resource_id,\n )\n\n self._search_manager: SearchManager | None = None\n self.search_user_assigned_identity_resource_id = search_user_assigned_identity_resource_id\n\n def _build_skillset(self) -> SearchIndexerSkillset:\n prefix = f\"{self.search_info.index_name}-cloud\"\n\n # NOTE: Do NOT map the chunk id directly to the index key field. Azure AI Search\n # index projections forbid mapping an input field onto the target index key when\n # using parent/child projections. The service will generate keys for projected\n # child documents automatically. Removing the explicit 'id' mapping resolves\n # HttpResponseError: \"Input 'id' cannot map to the key field\".\n mappings = [\n InputFieldMappingEntry(name=\"content\", source=\"/document/chunks/*/content\"),\n InputFieldMappingEntry(name=\"sourcepage\", source=\"/document/chunks/*/sourcepage\"),\n InputFieldMappingEntry(name=\"sourcefile\", source=\"/document/chunks/*/sourcefile\"),\n InputFieldMappingEntry(name=self.search_field_name_embedding, source=\"/document/chunks/*/embedding\"),\n InputFieldMappingEntry(name=\"storageUrl\", source=\"/document/metadata_storage_path\"),\n ]\n if self.use_multimodal:\n mappings.append(InputFieldMappingEntry(name=\"images\", source=\"/document/chunks/*/images\"))\n if self.use_acls:\n mappings.append(InputFieldMappingEntry(name=\"oids\", source=\"/document/chunks/*/oids\"))\n mappings.append(InputFieldMappingEntry(name=\"groups\", source=\"/document/chunks/*/groups\"))\n\n index_projection = SearchIndexerIndexProjection(\n selectors=[\n SearchIndexerIndexProjectionSelector(\n target_index_name=self.search_info.index_name,\n parent_key_field_name=\"parent_id\",\n source_context=\"/document/chunks/*\",\n mappings=mappings,\n )\n ],\n parameters=SearchIndexerIndexProjectionsParameters(\n projection_mode=IndexProjectionMode.SKIP_INDEXING_PARENT_DOCUMENTS,\n ),\n )\n\n document_extractor_skill = WebApiSkill(\n name=self.document_extractor.name,\n description=self.document_extractor.description,\n context=\"/document\",\n uri=self.document_extractor.uri,\n http_method=\"POST\",\n timeout=DEFAULT_SKILL_TIMEOUT,\n batch_size=DEFAULT_BATCH_SIZE,\n degree_of_parallelism=1,\n # Managed identity: Search service authenticates against the function app using this resource ID.\n auth_resource_id=self.document_extractor.auth_resource_id,\n auth_identity=SearchIndexerDataUserAssignedIdentity(\n resource_id=self.search_user_assigned_identity_resource_id\n ),\n inputs=[\n # Always provide the blob URL so the function can download large files (> 16MB)\n InputFieldMappingEntry(name=\"metadata_storage_path\", source=\"/document/metadata_storage_path\"),\n # We are not using the SAS token since the functions have RBAC access via managed identity\n ],\n outputs=[\n OutputFieldMappingEntry(name=\"pages\", target_name=\"pages\"),\n OutputFieldMappingEntry(name=\"figures\", target_name=\"figures\"),\n ]\n + (\n [\n # ACL outputs for document-level access control (populated by manual ADLS Gen2 extraction)\n OutputFieldMappingEntry(name=\"oids\", target_name=\"oids\"),\n OutputFieldMappingEntry(name=\"groups\", target_name=\"groups\"),\n ]\n if self.use_acls\n else []\n ),\n )\n\n figure_processor_skill = WebApiSkill(\n name=self.figure_processor.name,\n description=self.figure_processor.description,\n context=\"/document/figures/*\",\n uri=self.figure_processor.uri,\n http_method=\"POST\",\n timeout=DEFAULT_SKILL_TIMEOUT,\n batch_size=DEFAULT_BATCH_SIZE,\n degree_of_parallelism=1,\n # Managed identity: Search service authenticates against the function app using this resource ID.\n auth_resource_id=self.figure_processor.auth_resource_id,\n auth_identity=SearchIndexerDataUserAssignedIdentity(\n resource_id=self.search_user_assigned_identity_resource_id\n ),\n inputs=[\n InputFieldMappingEntry(name=\"figure_id\", source=\"/document/figures/*/figure_id\"),\n InputFieldMappingEntry(name=\"document_file_name\", source=\"/document/figures/*/document_file_name\"),\n InputFieldMappingEntry(name=\"filename\", source=\"/document/figures/*/filename\"),\n InputFieldMappingEntry(name=\"mime_type\", source=\"/document/figures/*/mime_type\"),\n InputFieldMappingEntry(name=\"bytes_base64\", source=\"/document/figures/*/bytes_base64\"),\n InputFieldMappingEntry(name=\"page_num\", source=\"/document/figures/*/page_num\"),\n InputFieldMappingEntry(name=\"bbox\", source=\"/document/figures/*/bbox\"),\n InputFieldMappingEntry(name=\"placeholder\", source=\"/document/figures/*/placeholder\"),\n InputFieldMappingEntry(name=\"title\", source=\"/document/figures/*/title\"),\n ],\n outputs=[\n # Only output the enriched fields to avoid cyclic dependency\n OutputFieldMappingEntry(name=\"description\", target_name=\"description\"),\n OutputFieldMappingEntry(name=\"url\", target_name=\"url\"),\n OutputFieldMappingEntry(name=\"embedding\", target_name=\"embedding\"),\n ],\n )\n\n # Shaper skill to consolidate pages and enriched figures into a single object\n shaper_skill = ShaperSkill(\n name=f\"{prefix}-document-shaper-skill\",\n description=\"Consolidates pages and enriched figures into a single document object\",\n context=\"/document\",\n inputs=[\n InputFieldMappingEntry(name=\"pages\", source=\"/document/pages\"),\n InputFieldMappingEntry(\n name=\"figures\",\n source_context=\"/document/figures/*\",\n inputs=[\n InputFieldMappingEntry(name=\"figure_id\", source=\"/document/figures/*/figure_id\"),\n InputFieldMappingEntry(\n name=\"document_file_name\", source=\"/document/figures/*/document_file_name\"\n ),\n InputFieldMappingEntry(name=\"filename\", source=\"/document/figures/*/filename\"),\n InputFieldMappingEntry(name=\"mime_type\", source=\"/document/figures/*/mime_type\"),\n InputFieldMappingEntry(name=\"page_num\", source=\"/document/figures/*/page_num\"),\n InputFieldMappingEntry(name=\"bbox\", source=\"/document/figures/*/bbox\"),\n InputFieldMappingEntry(name=\"placeholder\", source=\"/document/figures/*/placeholder\"),\n InputFieldMappingEntry(name=\"title\", source=\"/document/figures/*/title\"),\n InputFieldMappingEntry(name=\"description\", source=\"/document/figures/*/description\"),\n InputFieldMappingEntry(name=\"url\", source=\"/document/figures/*/url\"),\n InputFieldMappingEntry(name=\"embedding\", source=\"/document/figures/*/embedding\"),\n ],\n ),\n InputFieldMappingEntry(name=\"file_name\", source=\"/document/metadata_storage_name\"),\n InputFieldMappingEntry(name=\"storageUrl\", source=\"/document/metadata_storage_path\"),\n ]\n + (\n [\n # ACL fields from document_extractor's manual ADLS Gen2 ACL extraction\n InputFieldMappingEntry(name=\"oids\", source=\"/document/oids\"),\n InputFieldMappingEntry(name=\"groups\", source=\"/document/groups\"),\n ]\n if self.use_acls\n else []\n ),\n outputs=[OutputFieldMappingEntry(name=\"output\", target_name=\"consolidated_document\")],\n )\n\n text_processor_skill = WebApiSkill(\n name=self.text_processor.name,\n description=self.text_processor.description,\n context=\"/document\",\n uri=self.text_processor.uri,\n http_method=\"POST\",\n timeout=DEFAULT_SKILL_TIMEOUT,\n batch_size=DEFAULT_BATCH_SIZE,\n degree_of_parallelism=1,\n # Managed identity: Search service authenticates against the function app using this resource ID.\n auth_resource_id=self.text_processor.auth_resource_id,\n auth_identity=SearchIndexerDataUserAssignedIdentity(\n resource_id=self.search_user_assigned_identity_resource_id\n ),\n inputs=[\n InputFieldMappingEntry(name=\"consolidated_document\", source=\"/document/consolidated_document\"),\n ],\n outputs=[OutputFieldMappingEntry(name=\"chunks\", target_name=\"chunks\")],\n )\n\n return SearchIndexerSkillset(\n name=self.skillset_name,\n description=\"Skillset linking document extraction, figure enrichment, and text processing functions\",\n skills=[document_extractor_skill, figure_processor_skill, shaper_skill, text_processor_skill],\n index_projection=index_projection,\n )\n\n async def setup(self) -> None:\n logger.info(\"Setting up search index and skillset for cloud ingestion\")\n\n if not self.embeddings.azure_endpoint or not self.embeddings.azure_deployment_name:\n raise ValueError(\"Cloud ingestion requires Azure OpenAI endpoint and deployment\")\n\n if not isinstance(self.embeddings, OpenAIEmbeddings):\n raise TypeError(\"Cloud ingestion requires Azure OpenAI embeddings to configure the search index.\")\n\n # Warn if access control is enforced but ACL extraction is not enabled\n if self.enforce_access_control and not self.use_acls:\n logger.warning(\n \"AZURE_ENFORCE_ACCESS_CONTROL is enabled but USE_CLOUD_INGESTION_ACLS is not. \"\n \"Documents will not have ACLs extracted automatically from ADLS Gen2. \"\n \"If you intend to use document-level access control, either set USE_CLOUD_INGESTION_ACLS=true \"\n \"or manually set ACLs using scripts/manageacl.py after ingestion.\"\n )\n\n # Verify the storage container exists before attempting to create the data source\n container_client = self.blob_manager.blob_service_client.get_container_client(self.blob_manager.container)\n if not await container_client.exists():\n raise ValueError(\n f\"Storage container '{self.blob_manager.container}' does not exist in storage account '{self.blob_manager.account}'. \"\n f\"Please create the container first, or set AZURE_STORAGE_CONTAINER to an existing container name.\"\n )\n\n self._search_manager = SearchManager(\n search_info=self.search_info,\n search_analyzer_name=self.search_analyzer_name,\n use_acls=self.use_acls,\n use_parent_index_projection=True,\n embeddings=self.embeddings,\n field_name_embedding=self.search_field_name_embedding,\n search_images=self.use_multimodal,\n enforce_access_control=self.enforce_access_control,\n use_web_source=self.use_web_source,\n )\n\n await self._search_manager.create_index()\n\n async with self.search_info.create_search_indexer_client() as indexer_client:\n # Use ADLS_GEN2 when ACLs are enabled (requires hierarchical namespace storage)\n # Note: We do NOT use indexer_permission_options because that's incompatible with\n # Custom WebAPI skills. Instead, ACLs are extracted manually in document_extractor.\n data_source_type = (\n SearchIndexerDataSourceType.ADLS_GEN2 if self.use_acls else SearchIndexerDataSourceType.AZURE_BLOB\n )\n data_source_connection = SearchIndexerDataSourceConnection(\n name=self.data_source_name,\n type=data_source_type,\n connection_string=self.blob_manager.get_managedidentity_connectionstring(),\n container=SearchIndexerDataContainer(name=self.blob_manager.container),\n data_deletion_detection_policy=NativeBlobSoftDeleteDeletionDetectionPolicy(),\n )\n await indexer_client.create_or_update_data_source_connection(data_source_connection)\n\n skillset = self._build_skillset()\n await indexer_client.create_or_update_skillset(skillset)\n\n indexer = SearchIndexer(\n name=self.indexer_name,\n description=\"Indexer orchestrating cloud ingestion pipeline\",\n data_source_name=self.data_source_name,\n target_index_name=self.search_info.index_name,\n skillset_name=self.skillset_name,\n parameters=IndexingParameters(\n configuration=IndexingParametersConfiguration(\n query_timeout=None, # type: ignore\n data_to_extract=\"storageMetadata\",\n allow_skillset_to_read_file_data=False,\n )\n ),\n )\n await indexer_client.create_or_update_indexer(indexer)\n\n async def run(self) -> None:\n files = self.list_file_strategy.list()\n async for file in files:\n try:\n await self.blob_manager.upload_blob(file)\n finally:\n if file:\n file.close()\n\n async with self.search_info.create_search_indexer_client() as indexer_client:\n await indexer_client.run_indexer(self.indexer_name)\n logger.info(\"Triggered indexer '%s' for cloud ingestion\", self.indexer_name)\n" }, { "path": "app/backend/prepdocslib/csvparser.py", "content": "import csv\nfrom collections.abc import AsyncGenerator\nfrom typing import IO\n\nfrom .page import Page\nfrom .parser import Parser\n\n\nclass CsvParser(Parser):\n \"\"\"\n Concrete parser that can parse CSV into Page objects. Each row becomes a Page object.\n \"\"\"\n\n async def parse(self, content: IO) -> AsyncGenerator[Page, None]:\n # Check if content is in bytes (binary file) and decode to string\n content_str: str\n if isinstance(content, (bytes, bytearray)):\n content_str = content.decode(\"utf-8\")\n elif hasattr(content, \"read\"): # Handle BufferedReader\n content_str = content.read().decode(\"utf-8\")\n\n # Create a CSV reader from the text content\n reader = csv.reader(content_str.splitlines())\n offset = 0\n\n # Skip the header row\n next(reader, None)\n\n for i, row in enumerate(reader):\n page_text = \",\".join(row)\n yield Page(i, offset, page_text)\n offset += len(page_text) + 1 # Account for newline character\n" }, { "path": "app/backend/prepdocslib/embeddings.py", "content": "import logging\nfrom abc import ABC\nfrom collections.abc import Awaitable, Callable\nfrom urllib.parse import urljoin\n\nimport aiohttp\nimport tiktoken\nfrom openai import AsyncOpenAI, RateLimitError\nfrom tenacity import (\n AsyncRetrying,\n retry_if_exception_type,\n stop_after_attempt,\n wait_random_exponential,\n)\nfrom typing_extensions import TypedDict\n\nlogger = logging.getLogger(\"scripts\")\n\n\nclass EmbeddingBatch:\n \"\"\"Represents a batch of text that is going to be embedded.\"\"\"\n\n def __init__(self, texts: list[str], token_length: int):\n self.texts = texts\n self.token_length = token_length\n\n\nclass ExtraArgs(TypedDict, total=False):\n dimensions: int\n\n\nclass OpenAIEmbeddings(ABC):\n \"\"\"Client wrapper that handles batching, retries, and token accounting.\"\"\"\n\n SUPPORTED_BATCH_MODEL = {\n \"text-embedding-ada-002\": {\"token_limit\": 8100, \"max_batch_size\": 16},\n \"text-embedding-3-small\": {\"token_limit\": 8100, \"max_batch_size\": 16},\n \"text-embedding-3-large\": {\"token_limit\": 8100, \"max_batch_size\": 16},\n }\n SUPPORTED_DIMENSIONS_MODEL = {\n \"text-embedding-ada-002\": False,\n \"text-embedding-3-small\": True,\n \"text-embedding-3-large\": True,\n }\n\n def __init__(\n self,\n open_ai_client: AsyncOpenAI,\n open_ai_model_name: str,\n open_ai_dimensions: int,\n *,\n disable_batch: bool = False,\n azure_deployment_name: str | None = None,\n azure_endpoint: str | None = None,\n ):\n self.open_ai_client = open_ai_client\n self.open_ai_model_name = open_ai_model_name\n self.open_ai_dimensions = open_ai_dimensions\n self.disable_batch = disable_batch\n self.azure_deployment_name = azure_deployment_name\n self.azure_endpoint = azure_endpoint.rstrip(\"/\") if azure_endpoint else None\n\n @property\n def _api_model(self) -> str:\n return self.azure_deployment_name or self.open_ai_model_name\n\n def before_retry_sleep(self, retry_state):\n logger.info(\"Rate limited on the OpenAI embeddings API, sleeping before retrying...\")\n\n def calculate_token_length(self, text: str):\n encoding = tiktoken.encoding_for_model(self.open_ai_model_name)\n return len(encoding.encode(text))\n\n def split_text_into_batches(self, texts: list[str]) -> list[EmbeddingBatch]:\n batch_info = OpenAIEmbeddings.SUPPORTED_BATCH_MODEL.get(self.open_ai_model_name)\n if not batch_info:\n raise NotImplementedError(\n f\"Model {self.open_ai_model_name} is not supported with batch embedding operations\"\n )\n\n batch_token_limit = batch_info[\"token_limit\"]\n batch_max_size = batch_info[\"max_batch_size\"]\n batches: list[EmbeddingBatch] = []\n batch: list[str] = []\n batch_token_length = 0\n for text in texts:\n text_token_length = self.calculate_token_length(text)\n if batch_token_length + text_token_length >= batch_token_limit and len(batch) > 0:\n batches.append(EmbeddingBatch(batch, batch_token_length))\n batch = []\n batch_token_length = 0\n\n batch.append(text)\n batch_token_length = batch_token_length + text_token_length\n if len(batch) == batch_max_size:\n batches.append(EmbeddingBatch(batch, batch_token_length))\n batch = []\n batch_token_length = 0\n\n if len(batch) > 0:\n batches.append(EmbeddingBatch(batch, batch_token_length))\n\n return batches\n\n async def create_embedding_batch(self, texts: list[str], dimensions_args: ExtraArgs) -> list[list[float]]:\n batches = self.split_text_into_batches(texts)\n embeddings = []\n for batch in batches:\n async for attempt in AsyncRetrying(\n retry=retry_if_exception_type(RateLimitError),\n wait=wait_random_exponential(min=15, max=60),\n stop=stop_after_attempt(15),\n before_sleep=self.before_retry_sleep,\n ):\n with attempt:\n emb_response = await self.open_ai_client.embeddings.create(\n model=self._api_model, input=batch.texts, **dimensions_args\n )\n embeddings.extend([data.embedding for data in emb_response.data])\n logger.info(\n \"Computed embeddings in batch. Batch size: %d, Token count: %d\",\n len(batch.texts),\n batch.token_length,\n )\n\n return embeddings\n\n async def create_embedding_single(self, text: str, dimensions_args: ExtraArgs) -> list[float]:\n async for attempt in AsyncRetrying(\n retry=retry_if_exception_type(RateLimitError),\n wait=wait_random_exponential(min=15, max=60),\n stop=stop_after_attempt(15),\n before_sleep=self.before_retry_sleep,\n ):\n with attempt:\n emb_response = await self.open_ai_client.embeddings.create(\n model=self._api_model, input=text, **dimensions_args\n )\n logger.info(\"Computed embedding for text section. Character count: %d\", len(text))\n\n return emb_response.data[0].embedding\n\n async def create_embeddings(self, texts: list[str]) -> list[list[float]]:\n\n dimensions_args: ExtraArgs = (\n {\"dimensions\": self.open_ai_dimensions}\n if OpenAIEmbeddings.SUPPORTED_DIMENSIONS_MODEL.get(self.open_ai_model_name)\n else {}\n )\n\n if not self.disable_batch and self.open_ai_model_name in OpenAIEmbeddings.SUPPORTED_BATCH_MODEL:\n return await self.create_embedding_batch(texts, dimensions_args)\n\n return [await self.create_embedding_single(text, dimensions_args) for text in texts]\n\n\nclass ImageEmbeddings:\n \"\"\"\n Class for using image embeddings from Azure AI Vision\n To learn more, please visit https://learn.microsoft.com/azure/ai-services/computer-vision/how-to/image-retrieval#call-the-vectorize-image-api\n \"\"\"\n\n def __init__(self, endpoint: str, token_provider: Callable[[], Awaitable[str]]):\n self.token_provider = token_provider\n self.endpoint = endpoint\n\n async def create_embedding_for_image(self, image_bytes: bytes) -> list[float]:\n endpoint = urljoin(self.endpoint, \"computervision/retrieval:vectorizeImage\")\n params = {\"api-version\": \"2024-02-01\", \"model-version\": \"2023-04-15\"}\n headers = {\"Authorization\": \"Bearer \" + await self.token_provider()}\n\n async with aiohttp.ClientSession(headers=headers) as session:\n async for attempt in AsyncRetrying(\n retry=retry_if_exception_type(Exception),\n wait=wait_random_exponential(min=15, max=60),\n stop=stop_after_attempt(15),\n before_sleep=self.before_retry_sleep,\n ):\n with attempt:\n async with session.post(url=endpoint, params=params, data=image_bytes) as resp:\n resp_json = await resp.json()\n return resp_json[\"vector\"]\n raise ValueError(\"Failed to get image embedding after multiple retries.\")\n\n async def create_embedding_for_text(self, q: str):\n endpoint = urljoin(self.endpoint, \"computervision/retrieval:vectorizeText\")\n headers = {\"Content-Type\": \"application/json\"}\n params = {\"api-version\": \"2024-02-01\", \"model-version\": \"2023-04-15\"}\n data = {\"text\": q}\n headers[\"Authorization\"] = \"Bearer \" + await self.token_provider()\n\n async with aiohttp.ClientSession() as session:\n async with session.post(\n url=endpoint, params=params, headers=headers, json=data, raise_for_status=True\n ) as response:\n json = await response.json()\n return json[\"vector\"]\n raise ValueError(\"Failed to get text embedding after multiple retries.\")\n\n def before_retry_sleep(self, retry_state):\n logger.info(\"Rate limited on the Vision embeddings API, sleeping before retrying...\")\n" }, { "path": "app/backend/prepdocslib/figureprocessor.py", "content": "\"\"\"Utilities for describing and enriching figures extracted from documents.\"\"\"\n\nimport logging\nfrom enum import Enum\nfrom typing import Any, Optional\n\nfrom azure.core.credentials import AzureKeyCredential\nfrom azure.core.credentials_async import AsyncTokenCredential\n\nfrom .blobmanager import BaseBlobManager\nfrom .embeddings import ImageEmbeddings\nfrom .mediadescriber import (\n ContentUnderstandingDescriber,\n MediaDescriber,\n MultimodalModelDescriber,\n)\nfrom .page import ImageOnPage\n\nlogger = logging.getLogger(\"scripts\")\n\n\nclass MediaDescriptionStrategy(Enum):\n \"\"\"Supported mechanisms for describing images extracted from documents.\"\"\"\n\n NONE = \"none\"\n OPENAI = \"openai\"\n CONTENTUNDERSTANDING = \"content_understanding\"\n\n\nclass FigureProcessor:\n \"\"\"Helper that lazily creates a media describer and captions figures on demand.\"\"\"\n\n def __init__(\n self,\n *,\n credential: AsyncTokenCredential | AzureKeyCredential | None = None,\n strategy: MediaDescriptionStrategy = MediaDescriptionStrategy.NONE,\n openai_client: Any | None = None,\n openai_model: str | None = None,\n openai_deployment: str | None = None,\n content_understanding_endpoint: str | None = None,\n ) -> None:\n self.credential = credential\n self.strategy = strategy\n self.openai_client = openai_client\n self.openai_model = openai_model\n self.openai_deployment = openai_deployment\n self.content_understanding_endpoint = content_understanding_endpoint\n self.media_describer: MediaDescriber | None = None\n self.content_understanding_ready = False\n\n async def get_media_describer(self) -> MediaDescriber | None:\n \"\"\"Return (and lazily create) the media describer for this processor.\"\"\"\n\n if self.strategy == MediaDescriptionStrategy.NONE:\n return None\n\n if self.media_describer is not None:\n return self.media_describer\n\n if self.strategy == MediaDescriptionStrategy.CONTENTUNDERSTANDING:\n if self.content_understanding_endpoint is None:\n raise ValueError(\"Content Understanding strategy requires an endpoint\")\n if self.credential is None:\n raise ValueError(\"Content Understanding strategy requires a credential\")\n if isinstance(self.credential, AzureKeyCredential):\n raise ValueError(\n \"Content Understanding does not support key credentials; provide a token credential instead\"\n )\n self.media_describer = ContentUnderstandingDescriber(self.content_understanding_endpoint, self.credential)\n return self.media_describer\n\n if self.strategy == MediaDescriptionStrategy.OPENAI:\n if self.openai_client is None or self.openai_model is None:\n raise ValueError(\"OpenAI strategy requires both a client and a model name\")\n self.media_describer = MultimodalModelDescriber(\n self.openai_client, model=self.openai_model, deployment=self.openai_deployment\n )\n return self.media_describer\n\n logger.warning(\"Unknown media description strategy '%s'; skipping description\", self.strategy)\n return None\n\n def mark_content_understanding_ready(self) -> None:\n \"\"\"Record that the Content Understanding analyzer exists to avoid recreating it.\"\"\"\n\n self.content_understanding_ready = True\n\n async def describe(self, image_bytes: bytes) -> str | None:\n \"\"\"Generate a description for the provided image bytes if a describer is available.\"\"\"\n\n describer = await self.get_media_describer()\n if describer is None:\n return None\n if isinstance(describer, ContentUnderstandingDescriber) and not self.content_understanding_ready:\n await describer.create_analyzer()\n self.content_understanding_ready = True\n return await describer.describe_image(image_bytes)\n\n\ndef build_figure_markup(image: \"ImageOnPage\", description: Optional[str] = None) -> str:\n \"\"\"Create consistent HTML markup for a figure description on demand.\"\"\"\n\n caption_parts = [image.figure_id]\n if image.title:\n caption_parts.append(image.title)\n caption = \" \".join(part for part in caption_parts if part)\n if description:\n return f\"
{caption}
{description}
\"\n return f\"
{caption}
\"\n\n\nasync def process_page_image(\n *,\n image: \"ImageOnPage\",\n document_filename: str,\n blob_manager: Optional[BaseBlobManager],\n image_embeddings_client: Optional[ImageEmbeddings],\n figure_processor: Optional[FigureProcessor] = None,\n user_oid: Optional[str] = None,\n) -> \"ImageOnPage\":\n \"\"\"Generate description, upload image, and optionally compute embedding for a figure.\"\"\"\n\n if blob_manager is None:\n raise ValueError(\"BlobManager must be provided to process images.\")\n\n # Generate plain (model) description text only; do not wrap in HTML markup here.\n description_text: str | None = None\n if figure_processor is not None:\n description_text = await figure_processor.describe(image.bytes)\n\n # Store plain descriptive text (can be None). HTML rendering is deferred to build_figure_markup.\n image.description = description_text\n\n if image.url is None:\n image.url = await blob_manager.upload_document_image(\n document_filename, image.bytes, image.filename, image.page_num, user_oid=user_oid\n )\n\n if image_embeddings_client is not None:\n try:\n image.embedding = await image_embeddings_client.create_embedding_for_image(image.bytes)\n except Exception: # pragma: no cover - embedding failures shouldn't abort figure processing\n logger.warning(\"Image embedding generation failed for figure %s\", image.figure_id, exc_info=True)\n\n return image\n" }, { "path": "app/backend/prepdocslib/fileprocessor.py", "content": "from dataclasses import dataclass\n\nfrom .parser import Parser\nfrom .textsplitter import TextSplitter\n\n\n@dataclass(frozen=True)\nclass FileProcessor:\n parser: Parser\n splitter: TextSplitter\n" }, { "path": "app/backend/prepdocslib/filestrategy.py", "content": "import logging\nfrom typing import Optional\n\nfrom .blobmanager import AdlsBlobManager, BaseBlobManager, BlobManager\nfrom .embeddings import ImageEmbeddings, OpenAIEmbeddings\nfrom .figureprocessor import (\n FigureProcessor,\n MediaDescriptionStrategy,\n process_page_image,\n)\nfrom .fileprocessor import FileProcessor\nfrom .listfilestrategy import File, ListFileStrategy\nfrom .mediadescriber import ContentUnderstandingDescriber\nfrom .searchmanager import SearchManager, Section\nfrom .strategy import DocumentAction, SearchInfo, Strategy\nfrom .textprocessor import process_text\n\nlogger = logging.getLogger(\"scripts\")\n\n\nasync def parse_file(\n file: File,\n file_processors: dict[str, FileProcessor],\n category: Optional[str] = None,\n blob_manager: Optional[BaseBlobManager] = None,\n image_embeddings_client: Optional[ImageEmbeddings] = None,\n figure_processor: Optional[FigureProcessor] = None,\n user_oid: Optional[str] = None,\n) -> list[Section]:\n\n key = file.file_extension().lower()\n processor = file_processors.get(key)\n if processor is None:\n logger.info(\"Skipping '%s', no parser found.\", file.filename())\n return []\n logger.info(\"Ingesting '%s'\", file.filename())\n pages = [page async for page in processor.parser.parse(content=file.content)]\n for page in pages:\n for image in page.images:\n logger.info(\"Processing image '%s' on page %d\", image.filename, page.page_num)\n await process_page_image(\n image=image,\n document_filename=file.filename(),\n blob_manager=blob_manager,\n image_embeddings_client=image_embeddings_client,\n figure_processor=figure_processor,\n user_oid=user_oid,\n )\n sections = process_text(pages, file, processor.splitter, category)\n return sections\n\n\nclass FileStrategy(Strategy):\n \"\"\"\n Strategy for ingesting documents into a search service from files stored either locally or in a data lake storage account\n \"\"\"\n\n def __init__(\n self,\n list_file_strategy: ListFileStrategy,\n blob_manager: BlobManager,\n search_info: SearchInfo,\n file_processors: dict[str, FileProcessor],\n document_action: DocumentAction = DocumentAction.Add,\n embeddings: Optional[OpenAIEmbeddings] = None,\n image_embeddings: Optional[ImageEmbeddings] = None,\n search_analyzer_name: Optional[str] = None,\n search_field_name_embedding: Optional[str] = None,\n use_acls: bool = False,\n category: Optional[str] = None,\n figure_processor: Optional[FigureProcessor] = None,\n enforce_access_control: bool = False,\n use_web_source: bool = False,\n use_sharepoint_source: bool = False,\n ):\n self.list_file_strategy = list_file_strategy\n self.blob_manager = blob_manager\n self.file_processors = file_processors\n self.document_action = document_action\n self.embeddings = embeddings\n self.image_embeddings = image_embeddings\n self.search_analyzer_name = search_analyzer_name\n self.search_field_name_embedding = search_field_name_embedding\n self.search_info = search_info\n self.use_acls = use_acls\n self.category = category\n self.figure_processor = figure_processor\n self.enforce_access_control = enforce_access_control\n self.use_web_source = use_web_source\n self.use_sharepoint_source = use_sharepoint_source\n\n def setup_search_manager(self):\n self.search_manager = SearchManager(\n self.search_info,\n self.search_analyzer_name,\n self.use_acls,\n False, # use_parent_index_projection disabled for file-based ingestion\n self.embeddings,\n field_name_embedding=self.search_field_name_embedding,\n search_images=self.image_embeddings is not None,\n enforce_access_control=self.enforce_access_control,\n use_web_source=self.use_web_source,\n use_sharepoint_source=self.use_sharepoint_source,\n )\n\n async def setup(self):\n self.setup_search_manager()\n await self.search_manager.create_index()\n\n if (\n self.figure_processor is not None\n and self.figure_processor.strategy == MediaDescriptionStrategy.CONTENTUNDERSTANDING\n ):\n media_describer = await self.figure_processor.get_media_describer()\n if isinstance(media_describer, ContentUnderstandingDescriber):\n await media_describer.create_analyzer()\n self.figure_processor.mark_content_understanding_ready()\n\n async def run(self):\n self.setup_search_manager()\n if self.document_action == DocumentAction.Add:\n files = self.list_file_strategy.list()\n async for file in files:\n try:\n blob_url = await self.blob_manager.upload_blob(file)\n sections = await parse_file(\n file,\n self.file_processors,\n self.category,\n self.blob_manager,\n self.image_embeddings,\n figure_processor=self.figure_processor,\n )\n if sections:\n await self.search_manager.update_content(sections, url=blob_url)\n finally:\n if file:\n file.close()\n elif self.document_action == DocumentAction.Remove:\n paths = self.list_file_strategy.list_paths()\n async for path in paths:\n await self.blob_manager.remove_blob(path)\n await self.search_manager.remove_content(path)\n elif self.document_action == DocumentAction.RemoveAll:\n await self.blob_manager.remove_blob()\n await self.search_manager.remove_content()\n\n\nclass UploadUserFileStrategy:\n \"\"\"\n Strategy for ingesting a file that has already been uploaded to a ADLS2 storage account\n \"\"\"\n\n def __init__(\n self,\n search_info: SearchInfo,\n file_processors: dict[str, FileProcessor],\n blob_manager: AdlsBlobManager,\n search_field_name_embedding: Optional[str] = None,\n embeddings: Optional[OpenAIEmbeddings] = None,\n image_embeddings: Optional[ImageEmbeddings] = None,\n enforce_access_control: bool = False,\n figure_processor: Optional[FigureProcessor] = None,\n ):\n self.file_processors = file_processors\n self.embeddings = embeddings\n self.image_embeddings = image_embeddings\n self.search_info = search_info\n self.blob_manager = blob_manager\n self.figure_processor = figure_processor\n self.search_manager = SearchManager(\n search_info=self.search_info,\n search_analyzer_name=None,\n use_acls=True,\n use_parent_index_projection=False,\n embeddings=self.embeddings,\n field_name_embedding=search_field_name_embedding,\n search_images=image_embeddings is not None,\n enforce_access_control=enforce_access_control,\n )\n self.search_field_name_embedding = search_field_name_embedding\n\n async def add_file(self, file: File, user_oid: str):\n sections = await parse_file(\n file,\n self.file_processors,\n None,\n self.blob_manager,\n self.image_embeddings,\n figure_processor=self.figure_processor,\n user_oid=user_oid,\n )\n if sections:\n await self.search_manager.update_content(sections, url=file.url)\n\n async def remove_file(self, filename: str, oid: str):\n if filename is None or filename == \"\":\n logging.warning(\"Filename is required to remove a file\")\n return\n await self.search_manager.remove_content(filename, oid)\n" }, { "path": "app/backend/prepdocslib/htmlparser.py", "content": "import logging\nimport re\nfrom collections.abc import AsyncGenerator\nfrom typing import IO\n\nfrom bs4 import BeautifulSoup\n\nfrom .page import Page\nfrom .parser import Parser\n\nlogger = logging.getLogger(\"scripts\")\n\n\ndef cleanup_data(data: str) -> str:\n \"\"\"Cleans up the given content using regexes\n Args:\n data: (str): The data to clean up.\n Returns:\n str: The cleaned up data.\n \"\"\"\n # match two or more newlines and replace them with one new line\n output = re.sub(r\"\\n{2,}\", \"\\n\", data)\n # match two or more spaces that are not newlines and replace them with one space\n output = re.sub(r\"[^\\S\\n]{2,}\", \" \", output)\n # match two or more hyphens and replace them with two hyphens\n output = re.sub(r\"-{2,}\", \"--\", output)\n\n return output.strip()\n\n\nclass LocalHTMLParser(Parser):\n \"\"\"Parses HTML text into Page objects.\"\"\"\n\n async def parse(self, content: IO) -> AsyncGenerator[Page, None]:\n \"\"\"Parses the given content.\n To learn more, please visit https://pypi.org/project/beautifulsoup4/\n Args:\n content (IO): The content to parse.\n Returns:\n Page: The parsed html Page.\n \"\"\"\n logger.info(\"Extracting text from '%s' using local HTML parser (BeautifulSoup)\", content.name)\n\n data = content.read()\n soup = BeautifulSoup(data, \"html.parser\")\n\n # Get text only from html file\n result = soup.get_text()\n\n yield Page(0, 0, text=cleanup_data(result))\n" }, { "path": "app/backend/prepdocslib/integratedvectorizerstrategy.py", "content": "import logging\nfrom typing import Optional\n\nfrom azure.search.documents.indexes._generated.models import (\n NativeBlobSoftDeleteDeletionDetectionPolicy,\n)\nfrom azure.search.documents.indexes.models import (\n AzureOpenAIEmbeddingSkill,\n IndexProjectionMode,\n InputFieldMappingEntry,\n OutputFieldMappingEntry,\n SearchIndexer,\n SearchIndexerDataContainer,\n SearchIndexerDataSourceConnection,\n SearchIndexerDataSourceType,\n SearchIndexerIndexProjection,\n SearchIndexerIndexProjectionSelector,\n SearchIndexerIndexProjectionsParameters,\n SearchIndexerSkillset,\n SplitSkill,\n)\n\nfrom .blobmanager import BlobManager\nfrom .embeddings import OpenAIEmbeddings\nfrom .listfilestrategy import ListFileStrategy\nfrom .searchmanager import SearchManager\nfrom .strategy import DocumentAction, SearchInfo, Strategy\n\nlogger = logging.getLogger(\"scripts\")\n\n\nclass IntegratedVectorizerStrategy(Strategy): # pragma: no cover\n \"\"\"\n Strategy for ingesting and vectorizing documents into a search service from files stored storage account\n \"\"\"\n\n def __init__(\n self,\n list_file_strategy: ListFileStrategy,\n blob_manager: BlobManager,\n search_info: SearchInfo,\n embeddings: OpenAIEmbeddings,\n search_field_name_embedding: str,\n subscription_id: str,\n document_action: DocumentAction = DocumentAction.Add,\n search_analyzer_name: Optional[str] = None,\n use_acls: bool = False,\n category: Optional[str] = None,\n enforce_access_control: bool = False,\n use_web_source: bool = False,\n ):\n\n self.list_file_strategy = list_file_strategy\n self.blob_manager = blob_manager\n self.document_action = document_action\n self.embeddings = embeddings\n self.search_field_name_embedding = search_field_name_embedding\n self.subscription_id = subscription_id\n self.search_analyzer_name = search_analyzer_name\n self.use_acls = use_acls\n self.category = category\n self.search_info = search_info\n prefix = f\"{self.search_info.index_name}-{self.search_field_name_embedding}\"\n self.skillset_name = f\"{prefix}-skillset\"\n self.indexer_name = f\"{prefix}-indexer\"\n self.data_source_name = f\"{prefix}-blob\"\n self.enforce_access_control = enforce_access_control\n self.use_web_source = use_web_source\n\n async def create_embedding_skill(self, index_name: str) -> SearchIndexerSkillset:\n \"\"\"\n Create a skillset for the indexer to chunk documents and generate embeddings\n \"\"\"\n\n split_skill = SplitSkill(\n name=\"split-skill\",\n description=\"Split skill to chunk documents\",\n text_split_mode=\"pages\",\n context=\"/document\",\n maximum_page_length=2048,\n page_overlap_length=20,\n inputs=[\n InputFieldMappingEntry(name=\"text\", source=\"/document/content\"),\n ],\n outputs=[OutputFieldMappingEntry(name=\"textItems\", target_name=\"pages\")],\n )\n\n if not self.embeddings.azure_endpoint or not self.embeddings.azure_deployment_name:\n raise ValueError(\"Integrated vectorization requires Azure OpenAI endpoint and deployment\")\n\n embedding_skill = AzureOpenAIEmbeddingSkill(\n name=\"embedding-skill\",\n description=\"Skill to generate embeddings via Azure OpenAI\",\n context=\"/document/pages/*\",\n resource_url=self.embeddings.azure_endpoint,\n deployment_name=self.embeddings.azure_deployment_name,\n model_name=self.embeddings.open_ai_model_name,\n dimensions=self.embeddings.open_ai_dimensions,\n inputs=[\n InputFieldMappingEntry(name=\"text\", source=\"/document/pages/*\"),\n ],\n outputs=[OutputFieldMappingEntry(name=\"embedding\", target_name=\"vector\")],\n )\n\n index_projection = SearchIndexerIndexProjection(\n selectors=[\n SearchIndexerIndexProjectionSelector(\n target_index_name=index_name,\n parent_key_field_name=\"parent_id\",\n source_context=\"/document/pages/*\",\n mappings=[\n InputFieldMappingEntry(name=\"content\", source=\"/document/pages/*\"),\n InputFieldMappingEntry(name=\"sourcepage\", source=\"/document/metadata_storage_name\"),\n InputFieldMappingEntry(name=\"sourcefile\", source=\"/document/metadata_storage_name\"),\n InputFieldMappingEntry(name=\"storageUrl\", source=\"/document/metadata_storage_path\"),\n InputFieldMappingEntry(\n name=self.search_field_name_embedding, source=\"/document/pages/*/vector\"\n ),\n ],\n ),\n ],\n parameters=SearchIndexerIndexProjectionsParameters(\n projection_mode=IndexProjectionMode.SKIP_INDEXING_PARENT_DOCUMENTS\n ),\n )\n\n skillset = SearchIndexerSkillset(\n name=self.skillset_name,\n description=\"Skillset to chunk documents and generate embeddings\",\n skills=[split_skill, embedding_skill],\n index_projection=index_projection,\n )\n\n return skillset\n\n async def setup(self):\n logger.info(\"Setting up search index using integrated vectorization...\")\n search_manager = SearchManager(\n search_info=self.search_info,\n search_analyzer_name=self.search_analyzer_name,\n use_acls=self.use_acls,\n use_parent_index_projection=True,\n embeddings=self.embeddings,\n field_name_embedding=self.search_field_name_embedding,\n search_images=False,\n enforce_access_control=self.enforce_access_control,\n use_web_source=self.use_web_source,\n )\n\n await search_manager.create_index()\n\n ds_client = self.search_info.create_search_indexer_client()\n ds_container = SearchIndexerDataContainer(name=self.blob_manager.container)\n data_source_connection = SearchIndexerDataSourceConnection(\n name=self.data_source_name,\n type=SearchIndexerDataSourceType.AZURE_BLOB,\n connection_string=self.blob_manager.get_managedidentity_connectionstring(),\n container=ds_container,\n data_deletion_detection_policy=NativeBlobSoftDeleteDeletionDetectionPolicy(),\n )\n\n await ds_client.create_or_update_data_source_connection(data_source_connection)\n\n embedding_skillset = await self.create_embedding_skill(self.search_info.index_name)\n await ds_client.create_or_update_skillset(embedding_skillset)\n await ds_client.close()\n\n async def run(self):\n if self.document_action == DocumentAction.Add:\n files = self.list_file_strategy.list()\n async for file in files:\n try:\n await self.blob_manager.upload_blob(file)\n finally:\n if file:\n file.close()\n elif self.document_action == DocumentAction.Remove:\n paths = self.list_file_strategy.list_paths()\n async for path in paths:\n await self.blob_manager.remove_blob(path)\n elif self.document_action == DocumentAction.RemoveAll:\n await self.blob_manager.remove_blob()\n\n # Create an indexer\n indexer = SearchIndexer(\n name=self.indexer_name,\n description=\"Indexer to index documents and generate embeddings\",\n skillset_name=self.skillset_name,\n target_index_name=self.search_info.index_name,\n data_source_name=self.data_source_name,\n )\n\n indexer_client = self.search_info.create_search_indexer_client()\n indexer_result = await indexer_client.create_or_update_indexer(indexer)\n\n # Run the indexer\n await indexer_client.run_indexer(self.indexer_name)\n await indexer_client.close()\n\n logger.info(\n f\"Successfully created index, indexer: {indexer_result.name}, and skillset. Please navigate to search service in Azure Portal to view the status of the indexer.\"\n )\n" }, { "path": "app/backend/prepdocslib/jsonparser.py", "content": "import json\nfrom collections.abc import AsyncGenerator\nfrom typing import IO\n\nfrom .page import Page\nfrom .parser import Parser\n\n\nclass JsonParser(Parser):\n \"\"\"\n Concrete parser that can parse JSON into Page objects. A top-level object becomes a single Page, while a top-level array becomes multiple Page objects.\n \"\"\"\n\n async def parse(self, content: IO) -> AsyncGenerator[Page, None]:\n offset = 0\n data = json.loads(content.read())\n if isinstance(data, list):\n for i, obj in enumerate(data):\n offset += 1 # For opening bracket or comma before object\n page_text = json.dumps(obj)\n yield Page(i, offset, page_text)\n offset += len(page_text)\n elif isinstance(data, dict):\n yield Page(0, 0, json.dumps(data))\n" }, { "path": "app/backend/prepdocslib/listfilestrategy.py", "content": "import base64\nimport hashlib\nimport logging\nimport os\nimport re\nfrom abc import ABC\nfrom collections.abc import AsyncGenerator\nfrom glob import glob\nfrom typing import IO, Optional\n\nlogger = logging.getLogger(\"scripts\")\n\n\nclass File:\n \"\"\"\n Represents a file stored either locally or in a data lake storage account\n This file might contain access control information about which users or groups can access it\n \"\"\"\n\n def __init__(self, content: IO, acls: Optional[dict[str, list]] = None, url: Optional[str] = None):\n self.content = content\n self.acls = acls or {}\n self.url = url\n\n def filename(self) -> str:\n \"\"\"\n Get the filename from the content object.\n\n Different file-like objects store the filename in different attributes:\n - File objects from open() have a .name attribute\n - HTTP uploaded files (werkzeug.datastructures.FileStorage) have a .filename attribute\n\n Returns:\n str: The basename of the file\n \"\"\"\n content_name = None\n\n # Try to get filename attribute (common for HTTP uploaded files)\n if hasattr(self.content, \"filename\"):\n content_name = getattr(self.content, \"filename\")\n if content_name:\n return os.path.basename(content_name)\n\n # Try to get name attribute (common for file objects from open())\n if hasattr(self.content, \"name\"):\n content_name = getattr(self.content, \"name\")\n if content_name and content_name != \"file\":\n return os.path.basename(content_name)\n\n raise ValueError(\"The content object does not have a filename or name attribute. \")\n\n def file_extension(self):\n return os.path.splitext(self.filename())[1]\n\n def filename_to_id(self):\n filename_ascii = re.sub(\"[^0-9a-zA-Z_-]\", \"_\", self.filename())\n filename_hash = base64.b16encode(self.filename().encode(\"utf-8\")).decode(\"ascii\")\n acls_hash = \"\"\n if self.acls:\n acls_hash = base64.b16encode(str(self.acls).encode(\"utf-8\")).decode(\"ascii\")\n return f\"file-{filename_ascii}-{filename_hash}{acls_hash}\"\n\n def close(self):\n if self.content:\n self.content.close()\n\n\nclass ListFileStrategy(ABC):\n \"\"\"\n Abstract strategy for listing files that are located somewhere. For example, on a local computer or remotely in a storage account\n \"\"\"\n\n async def list(self) -> AsyncGenerator[File, None]:\n if False: # pragma: no cover - this is necessary for mypy to type check\n yield\n\n async def list_paths(self) -> AsyncGenerator[str, None]:\n if False: # pragma: no cover - this is necessary for mypy to type check\n yield\n\n\nclass LocalListFileStrategy(ListFileStrategy):\n \"\"\"\n Concrete strategy for listing files that are located in a local filesystem\n \"\"\"\n\n def __init__(self, path_pattern: str, enable_global_documents: bool = False):\n self.path_pattern = path_pattern\n self.enable_global_documents = enable_global_documents\n\n async def list_paths(self) -> AsyncGenerator[str, None]:\n async for p in self._list_paths(self.path_pattern):\n yield p\n\n async def _list_paths(self, path_pattern: str) -> AsyncGenerator[str, None]:\n for path in glob(path_pattern):\n if os.path.isdir(path):\n async for p in self._list_paths(f\"{path}/*\"):\n yield p\n else:\n # Only list files, not directories\n yield path\n\n async def list(self) -> AsyncGenerator[File, None]:\n acls = {\"oids\": [\"all\"], \"groups\": [\"all\"]} if self.enable_global_documents else {}\n async for path in self.list_paths():\n if not self.check_md5(path):\n yield File(content=open(path, mode=\"rb\"), acls=acls, url=path)\n\n def check_md5(self, path: str) -> bool:\n # if filename ends in .md5 skip\n if path.endswith(\".md5\"):\n return True\n\n # if there is a file called .md5 in this directory, see if its updated\n stored_hash = None\n with open(path, \"rb\") as file:\n existing_hash = hashlib.md5(file.read()).hexdigest()\n hash_path = f\"{path}.md5\"\n if os.path.exists(hash_path):\n with open(hash_path, encoding=\"utf-8\") as md5_f:\n stored_hash = md5_f.read()\n\n if stored_hash and stored_hash.strip() == existing_hash.strip():\n logger.info(\"Skipping '%s', no changes detected.\", path)\n return True\n\n # Write the hash\n with open(hash_path, \"w\", encoding=\"utf-8\") as md5_f:\n md5_f.write(existing_hash)\n\n return False\n" }, { "path": "app/backend/prepdocslib/mediadescriber.py", "content": "import base64\nimport logging\nfrom abc import ABC\nfrom typing import Optional\n\nimport aiohttp\nfrom azure.core.credentials_async import AsyncTokenCredential\nfrom azure.identity.aio import get_bearer_token_provider\nfrom openai import AsyncOpenAI, RateLimitError\nfrom rich.progress import Progress\nfrom tenacity import (\n AsyncRetrying,\n retry,\n retry_if_exception_type,\n stop_after_attempt,\n wait_fixed,\n wait_random_exponential,\n)\n\nlogger = logging.getLogger(\"scripts\")\n\n\nclass MediaDescriber(ABC):\n\n async def describe_image(self, image_bytes) -> str:\n raise NotImplementedError # pragma: no cover\n\n\nclass ContentUnderstandingDescriber(MediaDescriber):\n CU_API_VERSION = \"2024-12-01-preview\"\n\n analyzer_schema = {\n \"analyzerId\": \"image_analyzer\",\n \"name\": \"Image understanding\",\n \"description\": \"Extract detailed structured information from images extracted from documents.\",\n \"baseAnalyzerId\": \"prebuilt-image\",\n \"scenario\": \"image\",\n \"config\": {\"returnDetails\": False},\n \"fieldSchema\": {\n \"name\": \"ImageInformation\",\n \"descriptions\": \"Description of image.\",\n \"fields\": {\n \"Description\": {\n \"type\": \"string\",\n \"description\": \"Description of the image. If the image has a title, start with the title. Include a 2-sentence summary. If the image is a chart, diagram, or table, include the underlying data in an HTML table tag, with accurate numbers. If the image is a chart, describe any axis or legends. The only allowed HTML tags are the table/thead/tr/td/tbody tags.\",\n },\n },\n },\n }\n\n def __init__(self, endpoint: str, credential: AsyncTokenCredential):\n self.endpoint = endpoint\n self.credential = credential\n\n async def poll_api(self, session, poll_url, headers):\n\n @retry(stop=stop_after_attempt(60), wait=wait_fixed(2), retry=retry_if_exception_type(ValueError))\n async def poll():\n async with session.get(poll_url, headers=headers) as response:\n response.raise_for_status()\n response_json = await response.json()\n if response_json[\"status\"] == \"Failed\":\n raise Exception(\"Failed\")\n if response_json[\"status\"] == \"Running\":\n raise ValueError(\"Running\")\n return response_json\n\n return await poll()\n\n async def create_analyzer(self):\n logger.info(\"Creating analyzer '%s'...\", self.analyzer_schema[\"analyzerId\"])\n\n token_provider = get_bearer_token_provider(self.credential, \"https://cognitiveservices.azure.com/.default\")\n token = await token_provider()\n headers = {\"Authorization\": f\"Bearer {token}\", \"Content-Type\": \"application/json\"}\n params = {\"api-version\": self.CU_API_VERSION}\n analyzer_id = self.analyzer_schema[\"analyzerId\"]\n cu_endpoint = f\"{self.endpoint}/contentunderstanding/analyzers/{analyzer_id}\"\n async with aiohttp.ClientSession() as session:\n async with session.put(\n url=cu_endpoint, params=params, headers=headers, json=self.analyzer_schema\n ) as response:\n if response.status == 409:\n logger.info(\"Analyzer '%s' already exists.\", analyzer_id)\n return\n elif response.status != 201:\n data = await response.text()\n raise Exception(\"Error creating analyzer\", data)\n else:\n poll_url = response.headers.get(\"Operation-Location\")\n\n with Progress() as progress:\n progress.add_task(\"Creating analyzer...\", total=None, start=False)\n await self.poll_api(session, poll_url, headers)\n\n async def describe_image(self, image_bytes: bytes) -> str:\n async with aiohttp.ClientSession() as session:\n token = await self.credential.get_token(\"https://cognitiveservices.azure.com/.default\")\n headers = {\"Authorization\": \"Bearer \" + token.token}\n params = {\"api-version\": self.CU_API_VERSION}\n analyzer_name = self.analyzer_schema[\"analyzerId\"]\n async with session.post(\n url=f\"{self.endpoint}/contentunderstanding/analyzers/{analyzer_name}:analyze\",\n params=params,\n headers=headers,\n data=image_bytes,\n ) as response:\n response.raise_for_status()\n poll_url = response.headers[\"Operation-Location\"]\n\n with Progress() as progress:\n progress.add_task(\"Processing...\", total=None, start=False)\n results = await self.poll_api(session, poll_url, headers)\n\n fields = results[\"result\"][\"contents\"][0][\"fields\"]\n return fields[\"Description\"][\"valueString\"]\n\n\nclass MultimodalModelDescriber(MediaDescriber):\n def __init__(self, openai_client: AsyncOpenAI, model: str, deployment: Optional[str] = None):\n self.openai_client = openai_client\n self.model = model\n self.deployment = deployment\n\n async def describe_image(self, image_bytes: bytes) -> str:\n def before_retry_sleep(retry_state):\n logger.info(\"Rate limited on the OpenAI chat completions API, sleeping before retrying...\")\n\n image_base64 = base64.b64encode(image_bytes).decode(\"utf-8\")\n image_datauri = f\"data:image/png;base64,{image_base64}\"\n\n async for attempt in AsyncRetrying(\n retry=retry_if_exception_type(RateLimitError),\n wait=wait_random_exponential(min=15, max=60),\n stop=stop_after_attempt(15),\n before_sleep=before_retry_sleep,\n ):\n with attempt:\n response = await self.openai_client.chat.completions.create(\n model=self.model if self.deployment is None else self.deployment,\n max_tokens=500,\n seed=42, # Keep responses more consistent across runs\n messages=[\n {\n \"role\": \"system\",\n \"content\": \"You are a helpful assistant that describes images from organizational documents.\",\n },\n {\n \"role\": \"user\",\n \"content\": [\n {\n \"text\": \"Describe image with no more than 5 sentences. Do not speculate about anything you don't know.\",\n \"type\": \"text\",\n },\n {\"image_url\": {\"url\": image_datauri, \"detail\": \"auto\"}, \"type\": \"image_url\"},\n ],\n },\n ],\n )\n description = \"\"\n if response.choices and response.choices[0].message.content:\n description = response.choices[0].message.content.strip()\n return description\n" }, { "path": "app/backend/prepdocslib/page.py", "content": "import base64\nfrom dataclasses import asdict, dataclass, field\nfrom typing import Any, Optional\n\n\n@dataclass\nclass ImageOnPage:\n bytes: bytes\n bbox: tuple[float, float, float, float] # Pixels\n filename: str\n figure_id: str\n page_num: int # 0-indexed\n placeholder: str # HTML placeholder in page text, e.g. '
'\n mime_type: str = \"image/png\" # Set by parser; default assumes PNG rendering\n url: Optional[str] = None\n title: str = \"\"\n embedding: Optional[list[float]] = None\n description: Optional[str] = None\n\n def to_skill_payload(\n self,\n file_name: str,\n *,\n include_bytes_base64: bool = True,\n ) -> dict[str, Any]:\n data = asdict(self)\n\n # Remove raw bytes to keep payload lean (and JSON-friendly without extra handling).\n data.pop(\"bytes\", None)\n\n # Optionally include base64-encoded bytes for skills that need it\n if include_bytes_base64:\n b = self.bytes if isinstance(self.bytes, (bytes, bytearray)) else b\"\"\n data[\"bytes_base64\"] = base64.b64encode(b).decode(\"utf-8\")\n\n data[\"document_file_name\"] = file_name\n return data\n\n @classmethod\n def from_skill_payload(cls, data: dict[str, Any]) -> tuple[\"ImageOnPage\", str]:\n # Decode base64 image data (optional - may be omitted if already persisted to blob)\n bytes_base64 = data.get(\"bytes_base64\")\n if bytes_base64:\n try:\n raw_bytes = base64.b64decode(bytes_base64)\n except Exception as exc: # pragma: no cover - defensive\n raise ValueError(\"Invalid bytes_base64 image data\") from exc\n else:\n raw_bytes = b\"\" # Empty bytes if not provided (already uploaded to blob)\n\n # page_num may arrive as str; coerce\n try:\n page_num = int(data.get(\"page_num\") or 0)\n except Exception:\n page_num = 0\n\n # bbox may arrive as list; coerce into tuple\n bbox_val = data.get(\"bbox\")\n if isinstance(bbox_val, list) and len(bbox_val) == 4:\n bbox = tuple(bbox_val)\n else:\n bbox = (0, 0, 0, 0)\n\n filename = data.get(\"filename\")\n figure_id = data.get(\"figure_id\")\n placeholder = data.get(\"placeholder\")\n if filename is None:\n raise ValueError(\"filename is required\")\n if figure_id is None:\n raise ValueError(\"figure_id is required for ImageOnPage deserialization\")\n\n # Generate placeholder if not provided\n if placeholder is None:\n placeholder = f'
'\n\n image = cls(\n bytes=raw_bytes,\n bbox=bbox,\n page_num=page_num,\n filename=filename,\n figure_id=figure_id,\n placeholder=placeholder,\n mime_type=data.get(\"mime_type\") or \"image/png\",\n title=data.get(\"title\") or \"\",\n description=data.get(\"description\"),\n url=data.get(\"url\"),\n )\n return image, data.get(\"document_file_name\", \"\")\n\n\n@dataclass\nclass Page:\n \"\"\"\n A single page from a document\n\n Attributes:\n page_num (int): Page number (0-indexed)\n offset (int): If the text of the entire Document was concatenated into a single string, the index of the first character on the page. For example, if page 1 had the text \"hello\" and page 2 had the text \"world\", the offset of page 2 is 5 (\"hellow\")\n text (str): The text of the page\n \"\"\"\n\n page_num: int\n offset: int\n text: str\n images: list[ImageOnPage] = field(default_factory=list)\n tables: list[str] = field(default_factory=list)\n\n\n@dataclass\nclass Chunk:\n \"\"\"Semantic chunk emitted by the splitter (may originate wholly within one page\n or be the result of a cross-page merge / trailing fragment carry-forward).\n\n Attributes:\n page_num (int): Logical source page number (0-indexed) for the originating\n portion of content. For merged content spanning pages we keep the earliest\n contributing page number for stable attribution.\n text (str): Textual content of the chunk.\n images (list[ImageOnPage]): Images associated with this chunk, if any.\n \"\"\"\n\n page_num: int\n text: str\n images: list[ImageOnPage] = field(default_factory=list)\n" }, { "path": "app/backend/prepdocslib/parser.py", "content": "from abc import ABC\nfrom collections.abc import AsyncGenerator\nfrom typing import IO\n\nfrom .page import Page\n\n\nclass Parser(ABC):\n \"\"\"\n Abstract parser that parses content into Page objects\n \"\"\"\n\n async def parse(self, content: IO) -> AsyncGenerator[Page, None]:\n if False:\n yield # pragma: no cover - this is necessary for mypy to type check\n" }, { "path": "app/backend/prepdocslib/pdfparser.py", "content": "import html\nimport io\nimport logging\nimport uuid\nfrom collections.abc import AsyncGenerator\nfrom enum import Enum\nfrom typing import IO, Optional, cast\n\nimport pymupdf\nfrom azure.ai.documentintelligence.aio import DocumentIntelligenceClient\nfrom azure.ai.documentintelligence.models import (\n AnalyzeDocumentRequest,\n AnalyzeResult,\n DocumentFigure,\n DocumentTable,\n)\nfrom azure.core.credentials import AzureKeyCredential\nfrom azure.core.credentials_async import AsyncTokenCredential\nfrom azure.core.exceptions import HttpResponseError\nfrom PIL import Image\nfrom pypdf import PdfReader\n\nfrom .page import ImageOnPage, Page\nfrom .parser import Parser\n\nlogger = logging.getLogger(\"scripts\")\n\n\nclass LocalPdfParser(Parser):\n \"\"\"\n Concrete parser backed by PyPDF that can parse PDFs into pages\n To learn more, please visit https://pypi.org/project/pypdf/\n \"\"\"\n\n async def parse(self, content: IO) -> AsyncGenerator[Page, None]:\n logger.info(\"Extracting text from '%s' using local PDF parser (pypdf)\", content.name)\n\n reader = PdfReader(content)\n pages = reader.pages\n offset = 0\n for page_num, p in enumerate(pages):\n page_text = p.extract_text()\n yield Page(page_num=page_num, offset=offset, text=page_text)\n offset += len(page_text)\n\n\nclass DocumentAnalysisParser(Parser):\n \"\"\"\n Concrete parser backed by Azure AI Document Intelligence that can parse many document formats into pages\n To learn more, please visit https://learn.microsoft.com/azure/ai-services/document-intelligence/overview\n \"\"\"\n\n def __init__(\n self,\n endpoint: str,\n credential: AsyncTokenCredential | AzureKeyCredential,\n model_id: str = \"prebuilt-layout\",\n process_figures: bool = False,\n ) -> None:\n self.model_id = model_id\n self.endpoint = endpoint\n self.credential = credential\n self.process_figures = process_figures\n\n async def parse(self, content: IO) -> AsyncGenerator[Page, None]:\n logger.info(\"Extracting text from '%s' using Azure Document Intelligence\", content.name)\n\n async with DocumentIntelligenceClient(\n endpoint=self.endpoint, credential=self.credential\n ) as document_intelligence_client:\n # Always convert to bytes up front to avoid passing a FileStorage/stream object\n try:\n content.seek(0)\n except Exception:\n pass\n content_bytes = content.read()\n\n poller = None\n doc_for_pymupdf = None\n\n if self.process_figures:\n try:\n poller = await document_intelligence_client.begin_analyze_document(\n model_id=\"prebuilt-layout\",\n body=AnalyzeDocumentRequest(bytes_source=content_bytes),\n output=[\"figures\"],\n features=[\"ocrHighResolution\"],\n output_content_format=\"markdown\",\n )\n doc_for_pymupdf = pymupdf.open(stream=io.BytesIO(content_bytes))\n except HttpResponseError as e:\n if e.error and e.error.code == \"InvalidArgument\":\n logger.error(\n \"This document type does not support media description. Proceeding with standard analysis.\"\n )\n else:\n logger.error(\n \"Unexpected error analyzing document for media description: %s. Proceeding with standard analysis.\",\n e,\n )\n poller = None\n\n if poller is None:\n poller = await document_intelligence_client.begin_analyze_document(\n model_id=self.model_id,\n body=AnalyzeDocumentRequest(bytes_source=content_bytes),\n )\n analyze_result: AnalyzeResult = await poller.result()\n\n offset = 0\n\n for page in analyze_result.pages:\n tables_on_page = [\n table\n for table in (analyze_result.tables or [])\n if table.bounding_regions and table.bounding_regions[0].page_number == page.page_number\n ]\n figures_on_page = []\n if self.process_figures:\n figures_on_page = [\n figure\n for figure in (analyze_result.figures or [])\n if figure.bounding_regions and figure.bounding_regions[0].page_number == page.page_number\n ]\n page_images: list[ImageOnPage] = []\n page_tables: list[str] = []\n\n class ObjectType(Enum):\n NONE = -1\n TABLE = 0\n FIGURE = 1\n\n MaskEntry = tuple[ObjectType, Optional[int]]\n\n page_offset = page.spans[0].offset\n page_length = page.spans[0].length\n mask_chars: list[MaskEntry] = cast(list[MaskEntry], [(ObjectType.NONE, None)] * page_length)\n # mark all positions of the table spans in the page\n for table_idx, table in enumerate(tables_on_page):\n for span in table.spans:\n # replace all table spans with \"table_id\" in table_chars array\n for i in range(span.length):\n idx = span.offset - page_offset + i\n if idx >= 0 and idx < page_length:\n mask_chars[idx] = (ObjectType.TABLE, table_idx)\n # mark all positions of the figure spans in the page\n for figure_idx, figure in enumerate(figures_on_page):\n for span in figure.spans:\n # replace all figure spans with \"figure_id\" in figure_chars array\n for i in range(span.length):\n idx = span.offset - page_offset + i\n if idx >= 0 and idx < page_length:\n mask_chars[idx] = (ObjectType.FIGURE, figure_idx)\n\n # build page text by replacing characters in table spans with table html\n page_text = \"\"\n added_objects: set[MaskEntry] = set()\n for idx, mask_char in enumerate(mask_chars):\n object_type, object_idx = mask_char\n if object_type == ObjectType.NONE:\n page_text += analyze_result.content[page_offset + idx]\n elif object_type == ObjectType.TABLE:\n if object_idx is None:\n raise ValueError(\"Expected object_idx to be set\")\n if mask_char not in added_objects:\n table_html = DocumentAnalysisParser.table_to_html(tables_on_page[object_idx])\n page_tables.append(table_html)\n page_text += table_html\n added_objects.add(mask_char)\n elif object_type == ObjectType.FIGURE:\n if object_idx is None:\n raise ValueError(\"Expected object_idx to be set\")\n if doc_for_pymupdf is None: # pragma: no cover\n raise ValueError(\"Expected doc_for_pymupdf to be set for figure processing\")\n if mask_char not in added_objects:\n image_on_page = await DocumentAnalysisParser.figure_to_image(\n doc_for_pymupdf, figures_on_page[object_idx]\n )\n page_images.append(image_on_page)\n page_text += image_on_page.placeholder\n added_objects.add(mask_char)\n\n # We remove these comments since they are not needed and skew the page numbers\n page_text = page_text.replace(\"\", \"\")\n # We remove excess newlines at the beginning and end of the page\n page_text = page_text.strip()\n yield Page(\n page_num=page.page_number - 1,\n offset=offset,\n text=page_text,\n images=page_images,\n tables=page_tables,\n )\n offset += len(page_text)\n\n @staticmethod\n async def figure_to_image(doc: pymupdf.Document, figure: DocumentFigure) -> ImageOnPage:\n figure_title = figure.caption.content if figure.caption and figure.caption.content else \"\"\n # Generate a random UUID if figure.id is None\n figure_id = figure.id or f\"fig_{uuid.uuid4().hex[:8]}\"\n figure_filename = f\"figure{figure_id.replace('.', '_')}.png\"\n logger.info(\"Cropping figure %s with title '%s'\", figure_id, figure_title)\n placeholder = f'
'\n if not figure.bounding_regions:\n return ImageOnPage(\n bytes=b\"\",\n page_num=0, # 0-indexed\n figure_id=figure_id,\n bbox=(0, 0, 0, 0),\n filename=figure_filename,\n title=figure_title,\n placeholder=placeholder,\n mime_type=\"image/png\",\n )\n if len(figure.bounding_regions) > 1:\n logger.warning(\"Figure %s has more than one bounding region, using the first one\", figure_id)\n first_region = figure.bounding_regions[0]\n # To learn more about bounding regions, see https://aka.ms/bounding-region\n bounding_box = (\n first_region.polygon[0], # x0 (left)\n first_region.polygon[1], # y0 (top\n first_region.polygon[4], # x1 (right)\n first_region.polygon[5], # y1 (bottom)\n )\n page_number = first_region[\"pageNumber\"] # 1-indexed\n cropped_img, bbox_pixels = DocumentAnalysisParser.crop_image_from_pdf_page(doc, page_number - 1, bounding_box)\n return ImageOnPage(\n bytes=cropped_img,\n page_num=page_number - 1, # Convert to 0-indexed\n figure_id=figure_id,\n bbox=bbox_pixels,\n filename=figure_filename,\n title=figure_title,\n placeholder=placeholder,\n mime_type=\"image/png\",\n )\n\n @staticmethod\n def table_to_html(table: DocumentTable):\n table_html = \"
\"\n rows = [\n sorted([cell for cell in table.cells if cell.row_index == i], key=lambda cell: cell.column_index)\n for i in range(table.row_count)\n ]\n for row_cells in rows:\n table_html += \"\"\n for cell in row_cells:\n tag = \"th\" if (cell.kind == \"columnHeader\" or cell.kind == \"rowHeader\") else \"td\"\n cell_spans = \"\"\n if cell.column_span is not None and cell.column_span > 1:\n cell_spans += f\" colSpan={cell.column_span}\"\n if cell.row_span is not None and cell.row_span > 1:\n cell_spans += f\" rowSpan={cell.row_span}\"\n table_html += f\"<{tag}{cell_spans}>{html.escape(cell.content)}\"\n table_html += \"\"\n table_html += \"
\"\n return table_html\n\n @staticmethod\n def crop_image_from_pdf_page(\n doc: pymupdf.Document, page_number: int, bbox_inches: tuple[float, float, float, float]\n ) -> tuple[bytes, tuple[float, float, float, float]]:\n \"\"\"\n Crops a region from a given page in a PDF and returns it as an image.\n\n :param pdf_path: Path to the PDF file.\n :param page_number: The page number to crop from (0-indexed).\n :param bbox_inches: A tuple of (x0, y0, x1, y1) coordinates for the bounding box, in inches.\n :return: A tuple of (image_bytes, bbox_pixels).\n \"\"\"\n # Scale the bounding box to 72 DPI\n bbox_dpi = 72\n # We multiply using unpacking to ensure the resulting tuple has the correct number of elements\n x0, y0, x1, y1 = (round(x * bbox_dpi, 2) for x in bbox_inches)\n bbox_pixels = (x0, y0, x1, y1)\n rect = pymupdf.Rect(bbox_pixels)\n # Assume that the PDF has 300 DPI,\n # and use the matrix to convert between the 2 DPIs\n page_dpi = 300\n page = doc.load_page(page_number)\n pix = page.get_pixmap(matrix=pymupdf.Matrix(page_dpi / bbox_dpi, page_dpi / bbox_dpi), clip=rect)\n\n img = Image.frombytes(\"RGB\", (pix.width, pix.height), pix.samples)\n bytes_io = io.BytesIO()\n img.save(bytes_io, format=\"PNG\")\n return bytes_io.getvalue(), bbox_pixels\n" }, { "path": "app/backend/prepdocslib/searchmanager.py", "content": "import asyncio\nimport logging\nimport os\nfrom typing import Optional\n\nfrom azure.search.documents.indexes.models import (\n AIServicesVisionParameters,\n AIServicesVisionVectorizer,\n AzureOpenAIVectorizer,\n AzureOpenAIVectorizerParameters,\n BinaryQuantizationCompression,\n HnswAlgorithmConfiguration,\n HnswParameters,\n KnowledgeBase,\n KnowledgeBaseAzureOpenAIModel,\n KnowledgeRetrievalOutputMode,\n KnowledgeSourceReference,\n PermissionFilter,\n RemoteSharePointKnowledgeSource,\n RemoteSharePointKnowledgeSourceParameters,\n RescoringOptions,\n SearchableField,\n SearchField,\n SearchFieldDataType,\n SearchIndex,\n SearchIndexFieldReference,\n SearchIndexKnowledgeSource,\n SearchIndexKnowledgeSourceParameters,\n SearchIndexPermissionFilterOption,\n SemanticConfiguration,\n SemanticField,\n SemanticPrioritizedFields,\n SemanticSearch,\n SimpleField,\n VectorSearch,\n VectorSearchAlgorithmConfiguration,\n VectorSearchCompression,\n VectorSearchCompressionRescoreStorageMethod,\n VectorSearchProfile,\n VectorSearchVectorizer,\n WebKnowledgeSource,\n)\n\nfrom .blobmanager import BlobManager\nfrom .embeddings import OpenAIEmbeddings\nfrom .listfilestrategy import File\nfrom .strategy import SearchInfo\nfrom .textsplitter import Chunk\n\nlogger = logging.getLogger(\"scripts\")\n\n\nclass Section:\n \"\"\"\n A section of a page that is stored in a search service. These sections are used as context by Azure OpenAI service\n \"\"\"\n\n def __init__(self, chunk: Chunk, content: File, category: Optional[str] = None):\n self.chunk = chunk # content comes from here\n self.content = content # sourcepage and sourcefile come from here\n self.category = category\n # this also needs images which will become the images field\n\n\nclass SearchManager:\n \"\"\"\n Class to manage a search service. It can create indexes, and update or remove sections stored in these indexes\n To learn more, please visit https://learn.microsoft.com/azure/search/search-what-is-azure-search\n \"\"\"\n\n def __init__(\n self,\n search_info: SearchInfo,\n search_analyzer_name: Optional[str] = None,\n use_acls: bool = False,\n use_parent_index_projection: bool = False,\n embeddings: Optional[OpenAIEmbeddings] = None,\n field_name_embedding: Optional[str] = None,\n search_images: bool = False,\n enforce_access_control: bool = False,\n use_web_source: bool = False,\n use_sharepoint_source: bool = False,\n ):\n self.search_info = search_info\n self.search_analyzer_name = search_analyzer_name\n self.use_acls = use_acls\n self.use_parent_index_projection = use_parent_index_projection\n self.embeddings = embeddings\n self.embedding_dimensions = self.embeddings.open_ai_dimensions if self.embeddings else None\n self.field_name_embedding = field_name_embedding\n self.search_images = search_images\n self.enforce_access_control = enforce_access_control\n self.use_web_source = use_web_source\n self.use_sharepoint_source = use_sharepoint_source\n\n async def create_index(self):\n logger.info(\"Checking whether search index %s exists...\", self.search_info.index_name)\n\n async with self.search_info.create_search_index_client() as search_index_client:\n embedding_field = None\n images_field = None\n text_vector_search_profile = None\n text_vector_algorithm = None\n text_vector_compression = None\n image_vector_search_profile = None\n image_vector_algorithm = None\n permission_filter_option = None\n\n if self.embeddings:\n if self.embedding_dimensions is None:\n raise ValueError(\n \"Embedding dimensions must be set in order to add an embedding field to the search index\"\n )\n if self.field_name_embedding is None:\n raise ValueError(\n \"Embedding field must be set in order to add an embedding field to the search index\"\n )\n\n text_vectorizer = None\n if self.embeddings.azure_endpoint and self.embeddings.azure_deployment_name:\n text_vectorizer = AzureOpenAIVectorizer(\n vectorizer_name=f\"{self.embeddings.open_ai_model_name}-vectorizer\",\n parameters=AzureOpenAIVectorizerParameters(\n resource_url=self.embeddings.azure_endpoint,\n deployment_name=self.embeddings.azure_deployment_name,\n model_name=self.embeddings.open_ai_model_name,\n ),\n )\n\n text_vector_algorithm = HnswAlgorithmConfiguration(\n name=\"hnsw_config\",\n parameters=HnswParameters(metric=\"cosine\"),\n )\n text_vector_compression = BinaryQuantizationCompression(\n compression_name=f\"{self.field_name_embedding}-compression\",\n truncation_dimension=1024, # should this be a parameter? maybe not yet?\n rescoring_options=RescoringOptions(\n enable_rescoring=True,\n default_oversampling=10,\n rescore_storage_method=VectorSearchCompressionRescoreStorageMethod.PRESERVE_ORIGINALS,\n ),\n )\n text_vector_search_profile = VectorSearchProfile(\n name=f\"{self.field_name_embedding}-profile\",\n algorithm_configuration_name=text_vector_algorithm.name,\n compression_name=text_vector_compression.compression_name,\n **({\"vectorizer_name\": text_vectorizer.vectorizer_name if text_vectorizer else None}),\n )\n\n embedding_field = SearchField(\n name=self.field_name_embedding,\n type=SearchFieldDataType.Collection(SearchFieldDataType.Single),\n hidden=True,\n searchable=True,\n filterable=False,\n sortable=False,\n facetable=False,\n vector_search_dimensions=self.embedding_dimensions,\n vector_search_profile_name=f\"{self.field_name_embedding}-profile\",\n stored=False,\n )\n\n if self.search_images:\n if not self.search_info.azure_vision_endpoint:\n raise ValueError(\"Azure AI Vision endpoint must be provided to use image embeddings\")\n image_vector_algorithm = HnswAlgorithmConfiguration(\n name=\"images_hnsw_config\",\n parameters=HnswParameters(metric=\"cosine\"),\n )\n\n # Create the AI Vision vectorizer for image embeddings\n image_vectorizer = AIServicesVisionVectorizer(\n vectorizer_name=\"images-vision-vectorizer\",\n ai_services_vision_parameters=AIServicesVisionParameters(\n resource_uri=self.search_info.azure_vision_endpoint,\n model_version=\"2023-04-15\",\n ),\n )\n\n image_vector_search_profile = VectorSearchProfile(\n name=\"images_embedding_profile\",\n algorithm_configuration_name=image_vector_algorithm.name,\n vectorizer_name=image_vectorizer.vectorizer_name,\n )\n images_field = SearchField(\n name=\"images\",\n type=SearchFieldDataType.Collection(SearchFieldDataType.ComplexType),\n fields=[\n SearchField(\n name=\"embedding\",\n type=SearchFieldDataType.Collection(SearchFieldDataType.Single),\n searchable=True,\n stored=False,\n vector_search_dimensions=1024,\n vector_search_profile_name=image_vector_search_profile.name,\n ),\n SearchField(\n name=\"url\",\n type=SearchFieldDataType.String,\n searchable=False,\n filterable=True,\n sortable=False,\n facetable=True,\n ),\n SearchField(\n name=\"description\",\n type=SearchFieldDataType.String,\n searchable=True,\n filterable=False,\n sortable=False,\n facetable=False,\n ),\n SearchField(\n name=\"boundingbox\",\n type=SearchFieldDataType.Collection(SearchFieldDataType.Double),\n searchable=False,\n filterable=False,\n sortable=False,\n facetable=False,\n ),\n ],\n )\n\n if self.use_acls:\n oids_field = SearchField(\n name=\"oids\",\n type=SearchFieldDataType.Collection(SearchFieldDataType.String),\n filterable=True,\n permission_filter=PermissionFilter.USER_IDS,\n )\n groups_field = SearchField(\n name=\"groups\",\n type=SearchFieldDataType.Collection(SearchFieldDataType.String),\n filterable=True,\n permission_filter=PermissionFilter.GROUP_IDS,\n )\n\n if self.search_info.index_name not in [name async for name in search_index_client.list_index_names()]:\n logger.info(\"Creating new search index %s\", self.search_info.index_name)\n fields = [\n (\n SimpleField(name=\"id\", type=\"Edm.String\", key=True)\n if not self.use_parent_index_projection\n else SearchField(\n name=\"id\",\n type=\"Edm.String\",\n key=True,\n sortable=True,\n filterable=True,\n facetable=True,\n analyzer_name=\"keyword\",\n )\n ),\n SearchableField(\n name=\"content\",\n type=\"Edm.String\",\n analyzer_name=self.search_analyzer_name,\n ),\n SimpleField(name=\"category\", type=\"Edm.String\", filterable=True, facetable=True),\n SimpleField(\n name=\"sourcepage\",\n type=\"Edm.String\",\n filterable=True,\n facetable=True,\n ),\n SimpleField(\n name=\"sourcefile\",\n type=\"Edm.String\",\n filterable=True,\n facetable=True,\n ),\n SimpleField(\n name=\"storageUrl\",\n type=\"Edm.String\",\n filterable=True,\n facetable=False,\n ),\n ]\n if self.use_acls:\n fields.append(oids_field)\n fields.append(groups_field)\n permission_filter_option = (\n SearchIndexPermissionFilterOption.ENABLED\n if self.enforce_access_control\n else SearchIndexPermissionFilterOption.DISABLED\n )\n\n if self.use_parent_index_projection:\n logger.info(\"Including parent_id field for parent/child index projection support in new index\")\n fields.append(SearchableField(name=\"parent_id\", type=\"Edm.String\", filterable=True))\n\n vectorizers: list[VectorSearchVectorizer] = []\n vector_search_profiles = []\n vector_algorithms: list[VectorSearchAlgorithmConfiguration] = []\n vector_compressions: list[VectorSearchCompression] = []\n if embedding_field:\n logger.info(\"Including %s field for text vectors in new index\", embedding_field.name)\n fields.append(embedding_field)\n if text_vectorizer is not None:\n vectorizers.append(text_vectorizer)\n if (\n text_vector_search_profile is None\n or text_vector_algorithm is None\n or text_vector_compression is None\n ):\n raise ValueError(\"Text vector search profile, algorithm and compression must be set\")\n vector_search_profiles.append(text_vector_search_profile)\n vector_algorithms.append(text_vector_algorithm)\n vector_compressions.append(text_vector_compression)\n\n if images_field:\n logger.info(\"Including %s field for image descriptions and vectors in new index\", images_field.name)\n fields.append(images_field)\n if image_vector_search_profile is None or image_vector_algorithm is None:\n raise ValueError(\"Image search profile and algorithm must be set\")\n vector_search_profiles.append(image_vector_search_profile)\n vector_algorithms.append(image_vector_algorithm)\n # Add image vectorizer to vectorizers list\n vectorizers.append(image_vectorizer)\n\n index = SearchIndex(\n name=self.search_info.index_name,\n fields=fields,\n semantic_search=SemanticSearch(\n default_configuration_name=\"default\",\n configurations=[\n SemanticConfiguration(\n name=\"default\",\n prioritized_fields=SemanticPrioritizedFields(\n title_field=SemanticField(field_name=\"sourcepage\"),\n content_fields=[SemanticField(field_name=\"content\")],\n ),\n )\n ],\n ),\n vector_search=VectorSearch(\n profiles=vector_search_profiles,\n algorithms=vector_algorithms,\n compressions=vector_compressions,\n vectorizers=vectorizers,\n ),\n permission_filter_option=permission_filter_option,\n )\n\n await search_index_client.create_index(index)\n else:\n logger.info(\"Search index %s already exists\", self.search_info.index_name)\n existing_index = await search_index_client.get_index(self.search_info.index_name)\n if not any(field.name == \"storageUrl\" for field in existing_index.fields):\n logger.info(\"Adding storageUrl field to index %s\", self.search_info.index_name)\n existing_index.fields.append(\n SimpleField(\n name=\"storageUrl\",\n type=\"Edm.String\",\n filterable=True,\n facetable=False,\n ),\n )\n await search_index_client.create_or_update_index(existing_index)\n\n if embedding_field and not any(\n field.name == self.field_name_embedding for field in existing_index.fields\n ):\n logger.info(\"Adding %s field for text embeddings\", self.field_name_embedding)\n embedding_field.stored = True\n existing_index.fields.append(embedding_field)\n if existing_index.vector_search is None:\n raise ValueError(\"Vector search is not enabled for the existing index\")\n if text_vectorizer is not None:\n if existing_index.vector_search.vectorizers is None:\n existing_index.vector_search.vectorizers = []\n existing_index.vector_search.vectorizers.append(text_vectorizer)\n if (\n text_vector_search_profile is None\n or text_vector_algorithm is None\n or text_vector_compression is None\n ):\n raise ValueError(\"Text vector search profile, algorithm and compression must be set\")\n if existing_index.vector_search.profiles is None:\n existing_index.vector_search.profiles = []\n existing_index.vector_search.profiles.append(text_vector_search_profile)\n if existing_index.vector_search.algorithms is None:\n existing_index.vector_search.algorithms = []\n # existing_index.vector_search.algorithms.append(text_vector_algorithm)\n if existing_index.vector_search.compressions is None:\n existing_index.vector_search.compressions = []\n existing_index.vector_search.compressions.append(text_vector_compression)\n await search_index_client.create_or_update_index(existing_index)\n\n if (\n images_field\n and images_field.fields\n and not any(field.name == \"images\" for field in existing_index.fields)\n ):\n logger.info(\"Adding %s field for image embeddings\", images_field.name)\n images_field.fields[0].stored = True\n existing_index.fields.append(images_field)\n if image_vector_search_profile is None or image_vector_algorithm is None:\n raise ValueError(\"Image vector search profile and algorithm must be set\")\n if existing_index.vector_search is None:\n raise ValueError(\"Image vector search is not enabled for the existing index\")\n if existing_index.vector_search.profiles is None:\n existing_index.vector_search.profiles = []\n existing_index.vector_search.profiles.append(image_vector_search_profile)\n if existing_index.vector_search.algorithms is None:\n existing_index.vector_search.algorithms = []\n existing_index.vector_search.algorithms.append(image_vector_algorithm)\n if existing_index.vector_search.vectorizers is None:\n existing_index.vector_search.vectorizers = []\n existing_index.vector_search.vectorizers.append(image_vectorizer)\n await search_index_client.create_or_update_index(existing_index)\n\n if existing_index.semantic_search:\n if not existing_index.semantic_search.default_configuration_name:\n logger.info(\"Adding default semantic configuration to index %s\", self.search_info.index_name)\n existing_index.semantic_search.default_configuration_name = \"default\"\n\n if existing_index.semantic_search.configurations:\n existing_semantic_config = existing_index.semantic_search.configurations[0]\n if (\n existing_semantic_config.prioritized_fields\n and existing_semantic_config.prioritized_fields.title_field\n and not existing_semantic_config.prioritized_fields.title_field.field_name == \"sourcepage\"\n ):\n logger.info(\"Updating semantic configuration for index %s\", self.search_info.index_name)\n existing_semantic_config.prioritized_fields.title_field = SemanticField(\n field_name=\"sourcepage\"\n )\n\n if existing_index.vector_search is not None and (\n existing_index.vector_search.vectorizers is None\n or len(existing_index.vector_search.vectorizers) == 0\n ):\n if (\n self.embeddings is not None\n and self.embeddings.azure_endpoint\n and self.embeddings.azure_deployment_name\n ):\n logger.info(\"Adding vectorizer to search index %s\", self.search_info.index_name)\n existing_index.vector_search.vectorizers = [\n AzureOpenAIVectorizer(\n vectorizer_name=f\"{self.search_info.index_name}-vectorizer\",\n parameters=AzureOpenAIVectorizerParameters(\n resource_url=self.embeddings.azure_endpoint,\n deployment_name=self.embeddings.azure_deployment_name,\n model_name=self.embeddings.open_ai_model_name,\n ),\n )\n ]\n await search_index_client.create_or_update_index(existing_index)\n\n else:\n logger.info(\n \"Can't add vectorizer to search index %s since no Azure OpenAI embeddings service is defined\",\n self.search_info,\n )\n\n if self.use_acls:\n if self.enforce_access_control:\n logger.info(\"Enabling permission filtering on index %s\", self.search_info.index_name)\n existing_index.permission_filter_option = SearchIndexPermissionFilterOption.ENABLED\n else:\n logger.info(\"Disabling permission filtering on index %s\", self.search_info.index_name)\n existing_index.permission_filter_option = SearchIndexPermissionFilterOption.DISABLED\n\n existing_oids_field = next((field for field in existing_index.fields if field.name == \"oids\"), None)\n if existing_oids_field:\n existing_oids_field.permission_filter = PermissionFilter.USER_IDS\n else:\n existing_index.fields.append(oids_field)\n existing_groups_field = next(\n (field for field in existing_index.fields if field.name == \"groups\"), None\n )\n if existing_groups_field:\n existing_groups_field.permission_filter = PermissionFilter.GROUP_IDS\n else:\n existing_index.fields.append(groups_field)\n\n await search_index_client.create_or_update_index(existing_index)\n\n if self.search_info.use_agentic_knowledgebase and self.search_info.knowledgebase_name:\n await self.create_knowledgebase()\n\n async def create_knowledgebase(self):\n \"\"\"Creates one or more Knowledge Bases in the search index based on desired knowledge sources.\"\"\"\n if self.search_info.knowledgebase_name:\n field_names = [\"id\", \"sourcepage\", \"sourcefile\", \"content\", \"category\"]\n if self.use_acls:\n field_names.extend([\"oids\", \"groups\"])\n if self.search_images:\n field_names.append(\"images/url\")\n\n # Create field references using the new SDK pattern\n source_data_fields = [SearchIndexFieldReference(name=field) for field in field_names]\n\n async with self.search_info.create_search_index_client() as search_index_client:\n search_index_knowledge_source = SearchIndexKnowledgeSource(\n name=self.search_info.index_name, # Use the same name for convenience\n description=\"Default knowledge source using the main search index\",\n search_index_parameters=SearchIndexKnowledgeSourceParameters(\n search_index_name=self.search_info.index_name,\n source_data_fields=source_data_fields,\n ),\n )\n await search_index_client.create_or_update_knowledge_source(\n knowledge_source=search_index_knowledge_source\n )\n\n knowledge_source_refs: dict[str, KnowledgeSourceReference] = {\n \"index\": KnowledgeSourceReference(name=search_index_knowledge_source.name)\n }\n\n if self.use_web_source:\n logger.info(\"Adding web knowledge source to the knowledge base\")\n web_knowledge_source = WebKnowledgeSource(\n name=\"web\"\n # We do not specify a description here, since the default description is quite detailed already\n )\n await search_index_client.create_or_update_knowledge_source(knowledge_source=web_knowledge_source)\n knowledge_source_refs[\"web\"] = KnowledgeSourceReference(name=web_knowledge_source.name)\n\n if self.use_sharepoint_source:\n logger.info(\"Adding SharePoint knowledge source to the knowledge base\")\n sharepoint_knowledge_source = RemoteSharePointKnowledgeSource(\n name=\"sharepoint\",\n description=\"SharePoint knowledge source\",\n remote_share_point_parameters=RemoteSharePointKnowledgeSourceParameters(),\n )\n await search_index_client.create_or_update_knowledge_source(\n knowledge_source=sharepoint_knowledge_source\n )\n knowledge_source_refs[\"sharepoint\"] = KnowledgeSourceReference(\n name=sharepoint_knowledge_source.name\n )\n\n # Build the set of knowledge bases that should exist based on optional sources\n base_knowledgebase_name = self.search_info.knowledgebase_name\n knowledge_bases_to_upsert: list[tuple[str, list[KnowledgeSourceReference]]] = [\n (base_knowledgebase_name, [knowledge_source_refs[\"index\"]])\n ]\n\n if \"web\" in knowledge_source_refs:\n knowledge_bases_to_upsert.append(\n (\n f\"{base_knowledgebase_name}-with-web\",\n [knowledge_source_refs[\"index\"], knowledge_source_refs[\"web\"]],\n )\n )\n if \"sharepoint\" in knowledge_source_refs:\n knowledge_bases_to_upsert.append(\n (\n f\"{base_knowledgebase_name}-with-sp\",\n [knowledge_source_refs[\"index\"], knowledge_source_refs[\"sharepoint\"]],\n )\n )\n if \"web\" in knowledge_source_refs and \"sharepoint\" in knowledge_source_refs:\n knowledge_bases_to_upsert.append(\n (\n f\"{base_knowledgebase_name}-with-web-and-sp\",\n [\n knowledge_source_refs[\"index\"],\n knowledge_source_refs[\"web\"],\n knowledge_source_refs[\"sharepoint\"],\n ],\n )\n )\n\n created_kb_names: list[str] = []\n for kb_name, sources in knowledge_bases_to_upsert:\n logger.info(\"Creating (or updating) knowledge base '%s'...\", kb_name)\n await search_index_client.create_or_update_knowledge_base(\n knowledge_base=KnowledgeBase(\n name=kb_name,\n knowledge_sources=sources,\n models=[\n KnowledgeBaseAzureOpenAIModel(\n azure_open_ai_parameters=AzureOpenAIVectorizerParameters(\n resource_url=self.search_info.azure_openai_endpoint,\n deployment_name=self.search_info.azure_openai_knowledgebase_deployment,\n model_name=self.search_info.azure_openai_knowledgebase_model,\n )\n )\n ],\n output_mode=KnowledgeRetrievalOutputMode.ANSWER_SYNTHESIS,\n )\n )\n created_kb_names.append(kb_name)\n\n if created_kb_names:\n logger.info(\n \"Knowledge bases created successfully: %s\",\n \", \".join(created_kb_names),\n )\n\n async def update_content(self, sections: list[Section], url: Optional[str] = None):\n MAX_BATCH_SIZE = 1000\n section_batches = [sections[i : i + MAX_BATCH_SIZE] for i in range(0, len(sections), MAX_BATCH_SIZE)]\n\n async with self.search_info.create_search_client() as search_client:\n for batch_index, batch in enumerate(section_batches):\n documents = []\n for section_index, section in enumerate(batch):\n image_fields = {}\n if self.search_images:\n image_fields = {\n \"images\": [\n {\n \"url\": image.url,\n \"description\": image.description,\n \"boundingbox\": image.bbox,\n \"embedding\": image.embedding,\n }\n for image in section.chunk.images\n ]\n }\n document = {\n \"id\": f\"{section.content.filename_to_id()}-page-{section_index + batch_index * MAX_BATCH_SIZE}\",\n \"content\": section.chunk.text,\n \"category\": section.category,\n \"sourcepage\": BlobManager.sourcepage_from_file_page(\n filename=section.content.filename(), page=section.chunk.page_num\n ),\n \"sourcefile\": section.content.filename(),\n **image_fields,\n **section.content.acls,\n }\n documents.append(document)\n if url:\n for document in documents:\n document[\"storageUrl\"] = url\n if self.embeddings:\n if self.field_name_embedding is None:\n raise ValueError(\"Embedding field name must be set\")\n embeddings = await self.embeddings.create_embeddings(\n texts=[section.chunk.text for section in batch]\n )\n for i, document in enumerate(documents):\n document[self.field_name_embedding] = embeddings[i]\n logger.info(\n \"Uploading batch %d with %d sections to search index '%s'\",\n batch_index + 1,\n len(documents),\n self.search_info.index_name,\n )\n await search_client.upload_documents(documents)\n\n async def remove_content(self, path: Optional[str] = None, only_oid: Optional[str] = None):\n logger.info(\n \"Removing sections from '{%s or ''}' from search index '%s'\", path, self.search_info.index_name\n )\n async with self.search_info.create_search_client() as search_client:\n while True:\n filter = None\n if path is not None:\n # Replace ' with '' to escape the single quote for the filter\n # https://learn.microsoft.com/azure/search/query-odata-filter-orderby-syntax#escaping-special-characters-in-string-constants\n path_for_filter = os.path.basename(path).replace(\"'\", \"''\")\n filter = f\"sourcefile eq '{path_for_filter}'\"\n max_results = 1000\n result = await search_client.search(\n search_text=\"\", filter=filter, top=max_results, include_total_count=True\n )\n result_count = await result.get_count()\n if result_count == 0:\n break\n documents_to_remove = []\n async for document in result:\n # If only_oid is set, only remove documents that have only this oid\n if not only_oid or document.get(\"oids\") == [only_oid]:\n documents_to_remove.append({\"id\": document[\"id\"]})\n if len(documents_to_remove) == 0:\n if result_count < max_results:\n break\n else:\n continue\n removed_docs = await search_client.delete_documents(documents_to_remove)\n logger.info(\"Removed %d sections from index\", len(removed_docs))\n # It can take a few seconds for search results to reflect changes, so wait a bit\n await asyncio.sleep(2)\n" }, { "path": "app/backend/prepdocslib/servicesetup.py", "content": "\"\"\"Shared service setup helpers.\"\"\"\n\nimport logging\nimport os\nfrom collections.abc import Awaitable, Callable\nfrom enum import Enum\nfrom typing import Optional\n\nfrom azure.core.credentials import AzureKeyCredential\nfrom azure.core.credentials_async import AsyncTokenCredential\nfrom azure.identity.aio import get_bearer_token_provider\nfrom openai import AsyncOpenAI\n\nfrom .blobmanager import BlobManager\nfrom .csvparser import CsvParser\nfrom .embeddings import ImageEmbeddings, OpenAIEmbeddings\nfrom .figureprocessor import FigureProcessor, MediaDescriptionStrategy\nfrom .fileprocessor import FileProcessor\nfrom .htmlparser import LocalHTMLParser\nfrom .jsonparser import JsonParser\nfrom .parser import Parser\nfrom .pdfparser import DocumentAnalysisParser, LocalPdfParser\nfrom .strategy import SearchInfo\nfrom .textparser import TextParser\nfrom .textsplitter import SentenceTextSplitter, SimpleTextSplitter\n\nlogger = logging.getLogger(\"scripts\")\n\n\ndef clean_key_if_exists(key: Optional[str]) -> Optional[str]:\n \"\"\"Remove leading and trailing whitespace from a key if it exists. If the key is empty, return None.\"\"\"\n if key is not None and key.strip() != \"\":\n return key.strip()\n return None\n\n\nclass OpenAIHost(str, Enum):\n \"\"\"Supported OpenAI hosting styles.\n\n OPENAI: Public OpenAI API.\n AZURE: Standard Azure OpenAI (service name becomes endpoint).\n AZURE_CUSTOM: A fully custom endpoint URL (for Network Isolation / APIM).\n LOCAL: A locally hosted OpenAI-compatible endpoint (no key required).\n \"\"\"\n\n OPENAI = \"openai\"\n AZURE = \"azure\"\n AZURE_CUSTOM = \"azure_custom\"\n LOCAL = \"local\"\n\n\ndef setup_search_info(\n search_service: str,\n index_name: str,\n azure_credential: AsyncTokenCredential,\n use_agentic_knowledgebase: Optional[bool] = None,\n azure_openai_endpoint: Optional[str] = None,\n knowledgebase_name: Optional[str] = None,\n azure_openai_knowledgebase_deployment: Optional[str] = None,\n azure_openai_knowledgebase_model: Optional[str] = None,\n search_key: Optional[str] = None,\n azure_vision_endpoint: Optional[str] = None,\n) -> SearchInfo:\n \"\"\"Setup search service information.\"\"\"\n search_creds: AsyncTokenCredential | AzureKeyCredential = (\n azure_credential if search_key is None else AzureKeyCredential(search_key)\n )\n if use_agentic_knowledgebase and azure_openai_knowledgebase_deployment is None:\n raise ValueError(\"Azure OpenAI deployment for Knowledge Base must be specified for agentic retrieval.\")\n\n return SearchInfo(\n endpoint=f\"https://{search_service}.search.windows.net/\",\n credential=search_creds,\n index_name=index_name,\n knowledgebase_name=knowledgebase_name,\n use_agentic_knowledgebase=use_agentic_knowledgebase,\n azure_openai_endpoint=azure_openai_endpoint,\n azure_openai_knowledgebase_model=azure_openai_knowledgebase_model,\n azure_openai_knowledgebase_deployment=azure_openai_knowledgebase_deployment,\n azure_vision_endpoint=azure_vision_endpoint,\n )\n\n\ndef setup_openai_client(\n openai_host: OpenAIHost,\n azure_credential: AsyncTokenCredential,\n azure_openai_api_key: Optional[str] = None,\n azure_openai_service: Optional[str] = None,\n azure_openai_custom_url: Optional[str] = None,\n openai_api_key: Optional[str] = None,\n openai_organization: Optional[str] = None,\n) -> tuple[AsyncOpenAI, Optional[str]]:\n openai_client: AsyncOpenAI\n azure_openai_endpoint: Optional[str] = None\n\n if openai_host in [OpenAIHost.AZURE, OpenAIHost.AZURE_CUSTOM]:\n base_url: Optional[str] = None\n api_key_or_token: Optional[str | Callable[[], Awaitable[str]]] = None\n if openai_host == OpenAIHost.AZURE_CUSTOM:\n logger.info(\"OPENAI_HOST is azure_custom, setting up Azure OpenAI custom client\")\n if not azure_openai_custom_url:\n raise ValueError(\"AZURE_OPENAI_CUSTOM_URL must be set when OPENAI_HOST is azure_custom\")\n base_url = azure_openai_custom_url\n else:\n logger.info(\"OPENAI_HOST is azure, setting up Azure OpenAI client\")\n if not azure_openai_service:\n raise ValueError(\"AZURE_OPENAI_SERVICE must be set when OPENAI_HOST is azure\")\n azure_openai_endpoint = f\"https://{azure_openai_service}.openai.azure.com\"\n base_url = f\"{azure_openai_endpoint}/openai/v1\"\n if azure_openai_api_key:\n logger.info(\"AZURE_OPENAI_API_KEY_OVERRIDE found, using as api_key for Azure OpenAI client\")\n api_key_or_token = azure_openai_api_key\n else:\n logger.info(\"Using Azure credential (passwordless authentication) for Azure OpenAI client\")\n api_key_or_token = get_bearer_token_provider(\n azure_credential, \"https://cognitiveservices.azure.com/.default\"\n )\n openai_client = AsyncOpenAI(\n base_url=base_url,\n api_key=api_key_or_token,\n )\n elif openai_host == OpenAIHost.LOCAL:\n logger.info(\"OPENAI_HOST is local, setting up local OpenAI client for OPENAI_BASE_URL with no key\")\n openai_client = AsyncOpenAI(\n base_url=os.environ[\"OPENAI_BASE_URL\"],\n api_key=\"no-key-required\",\n )\n else:\n logger.info(\n \"OPENAI_HOST is not azure, setting up OpenAI client using OPENAI_API_KEY and OPENAI_ORGANIZATION environment variables\"\n )\n if openai_api_key is None:\n raise ValueError(\"OpenAI key is required when using the non-Azure OpenAI API\")\n openai_client = AsyncOpenAI(\n api_key=openai_api_key,\n organization=openai_organization,\n )\n return openai_client, azure_openai_endpoint\n\n\ndef setup_image_embeddings_service(\n azure_credential: AsyncTokenCredential,\n vision_endpoint: Optional[str],\n use_multimodal: bool,\n) -> ImageEmbeddings | None:\n image_embeddings_service: Optional[ImageEmbeddings] = None\n if use_multimodal:\n if vision_endpoint is None:\n raise ValueError(\"An Azure AI Vision endpoint must be provided to use multimodal features.\")\n image_embeddings_service = ImageEmbeddings(\n endpoint=vision_endpoint,\n token_provider=get_bearer_token_provider(azure_credential, \"https://cognitiveservices.azure.com/.default\"),\n )\n return image_embeddings_service\n\n\ndef setup_embeddings_service(\n openai_host: OpenAIHost,\n open_ai_client: AsyncOpenAI,\n emb_model_name: str,\n emb_model_dimensions: int,\n azure_openai_deployment: Optional[str] = None,\n azure_openai_endpoint: Optional[str] = None,\n disable_batch: bool = False,\n) -> OpenAIEmbeddings:\n if openai_host in [OpenAIHost.AZURE, OpenAIHost.AZURE_CUSTOM]:\n if azure_openai_endpoint is None:\n raise ValueError(\"Azure OpenAI endpoint must be provided when using Azure OpenAI embeddings\")\n if azure_openai_deployment is None:\n raise ValueError(\"Azure OpenAI deployment must be provided when using Azure OpenAI embeddings\")\n\n return OpenAIEmbeddings(\n open_ai_client=open_ai_client,\n open_ai_model_name=emb_model_name,\n open_ai_dimensions=emb_model_dimensions,\n disable_batch=disable_batch,\n azure_deployment_name=azure_openai_deployment,\n azure_endpoint=azure_openai_endpoint,\n )\n\n\ndef setup_blob_manager(\n azure_credential: AsyncTokenCredential | str,\n storage_account: str,\n storage_container: str,\n storage_resource_group: Optional[str] = None,\n subscription_id: Optional[str] = None,\n storage_key: Optional[str] = None,\n image_storage_container: Optional[str] = None,\n) -> BlobManager:\n \"\"\"Create a BlobManager instance for document or figure storage.\n\n The optional resource group and subscription are retained for parity with\n local ingestion (used for diagnostic operations) but not required by\n Azure Functions.\n The optional image storage container is used for the multimodal ingestion feature.\n \"\"\"\n endpoint = f\"https://{storage_account}.blob.core.windows.net\"\n storage_credential: AsyncTokenCredential | str = azure_credential if storage_key is None else storage_key\n\n return BlobManager(\n endpoint=endpoint,\n container=storage_container,\n account=storage_account,\n credential=storage_credential,\n resource_group=storage_resource_group,\n subscription_id=subscription_id,\n image_container=image_storage_container,\n )\n\n\ndef setup_figure_processor(\n *,\n credential: AsyncTokenCredential | None,\n use_multimodal: bool,\n use_content_understanding: bool,\n content_understanding_endpoint: str | None,\n openai_client: object | None,\n openai_model: str | None,\n openai_deployment: str | None,\n) -> FigureProcessor | None:\n \"\"\"Create a FigureProcessor based on feature flags.\n\n Priority order:\n 1. use_multimodal -> MediaDescriptionStrategy.OPENAI\n 2. else if use_content_understanding and endpoint -> CONTENTUNDERSTANDING\n 3. else -> return None (no figure description)\n \"\"\"\n if use_multimodal:\n return FigureProcessor(\n credential=credential,\n strategy=MediaDescriptionStrategy.OPENAI,\n openai_client=openai_client,\n openai_model=openai_model,\n openai_deployment=openai_deployment,\n )\n if use_content_understanding and content_understanding_endpoint:\n return FigureProcessor(\n credential=credential,\n strategy=MediaDescriptionStrategy.CONTENTUNDERSTANDING,\n content_understanding_endpoint=content_understanding_endpoint,\n )\n return None\n\n\ndef build_file_processors(\n *,\n azure_credential: AsyncTokenCredential,\n document_intelligence_service: str | None,\n document_intelligence_key: str | None = None,\n use_local_pdf_parser: bool = False,\n use_local_html_parser: bool = False,\n process_figures: bool = False,\n) -> dict[str, FileProcessor]:\n sentence_text_splitter = SentenceTextSplitter()\n\n doc_int_parser: Optional[DocumentAnalysisParser] = None\n # check if Azure Document Intelligence credentials are provided\n if document_intelligence_service:\n credential: AsyncTokenCredential | AzureKeyCredential\n if document_intelligence_key:\n credential = AzureKeyCredential(document_intelligence_key)\n else:\n credential = azure_credential\n doc_int_parser = DocumentAnalysisParser(\n endpoint=f\"https://{document_intelligence_service}.cognitiveservices.azure.com/\",\n credential=credential,\n process_figures=process_figures,\n )\n\n pdf_parser: Optional[Parser] = None\n if use_local_pdf_parser or document_intelligence_service is None:\n pdf_parser = LocalPdfParser()\n elif doc_int_parser is not None:\n pdf_parser = doc_int_parser\n else:\n logger.warning(\"No PDF parser available\")\n\n html_parser: Optional[Parser] = None\n if use_local_html_parser or document_intelligence_service is None:\n html_parser = LocalHTMLParser()\n elif doc_int_parser is not None:\n html_parser = doc_int_parser\n else:\n logger.warning(\"No HTML parser available\")\n\n # These file formats can always be parsed:\n file_processors = {\n \".json\": FileProcessor(JsonParser(), SimpleTextSplitter()),\n \".md\": FileProcessor(TextParser(), sentence_text_splitter),\n \".txt\": FileProcessor(TextParser(), sentence_text_splitter),\n \".csv\": FileProcessor(CsvParser(), sentence_text_splitter),\n }\n # These require either a Python package or Document Intelligence\n if pdf_parser is not None:\n file_processors.update({\".pdf\": FileProcessor(pdf_parser, sentence_text_splitter)})\n if html_parser is not None:\n file_processors.update({\".html\": FileProcessor(html_parser, sentence_text_splitter)})\n # These file formats require Document Intelligence\n if doc_int_parser is not None:\n file_processors.update(\n {\n \".docx\": FileProcessor(doc_int_parser, sentence_text_splitter),\n \".pptx\": FileProcessor(doc_int_parser, sentence_text_splitter),\n \".xlsx\": FileProcessor(doc_int_parser, sentence_text_splitter),\n \".png\": FileProcessor(doc_int_parser, sentence_text_splitter),\n \".jpg\": FileProcessor(doc_int_parser, sentence_text_splitter),\n \".jpeg\": FileProcessor(doc_int_parser, sentence_text_splitter),\n \".tiff\": FileProcessor(doc_int_parser, sentence_text_splitter),\n \".bmp\": FileProcessor(doc_int_parser, sentence_text_splitter),\n \".heic\": FileProcessor(doc_int_parser, sentence_text_splitter),\n }\n )\n return file_processors\n\n\ndef select_processor_for_filename(file_name: str, file_processors: dict[str, FileProcessor]) -> FileProcessor:\n \"\"\"Select the appropriate file processor for a given filename.\n\n Args:\n file_name: Name of the file to process\n file_processors: Dictionary mapping file extensions to FileProcessor instances\n\n Returns:\n FileProcessor instance for the file\n\n Raises:\n ValueError: If the file extension is not supported\n \"\"\"\n file_ext = os.path.splitext(file_name)[1].lower()\n file_processor = file_processors.get(file_ext)\n if not file_processor:\n raise ValueError(f\"Unsupported file type: {file_name}\")\n return file_processor\n" }, { "path": "app/backend/prepdocslib/strategy.py", "content": "from abc import ABC\nfrom enum import Enum\nfrom typing import Optional\n\nfrom azure.core.credentials import AzureKeyCredential\nfrom azure.core.credentials_async import AsyncTokenCredential\nfrom azure.search.documents.aio import SearchClient\nfrom azure.search.documents.indexes.aio import SearchIndexClient, SearchIndexerClient\n\nUSER_AGENT = \"azure-search-chat-demo/1.0.0\"\n\n\nclass SearchInfo:\n \"\"\"\n Class representing a connection to a search service\n To learn more, please visit https://learn.microsoft.com/azure/search/search-what-is-azure-search\n \"\"\"\n\n def __init__(\n self,\n endpoint: str,\n credential: AsyncTokenCredential | AzureKeyCredential,\n index_name: str,\n use_agentic_knowledgebase: Optional[bool] = False,\n knowledgebase_name: Optional[str] = None,\n azure_openai_knowledgebase_model: Optional[str] = None,\n azure_openai_knowledgebase_deployment: Optional[str] = None,\n azure_openai_endpoint: Optional[str] = None,\n azure_vision_endpoint: Optional[str] = None,\n ):\n self.endpoint = endpoint\n self.credential = credential\n self.index_name = index_name\n self.knowledgebase_name = knowledgebase_name\n self.use_agentic_knowledgebase = use_agentic_knowledgebase\n self.azure_openai_knowledgebase_model = azure_openai_knowledgebase_model\n self.azure_openai_knowledgebase_deployment = azure_openai_knowledgebase_deployment\n self.azure_openai_endpoint = azure_openai_endpoint\n self.azure_vision_endpoint = azure_vision_endpoint\n\n def create_search_client(self) -> SearchClient:\n return SearchClient(endpoint=self.endpoint, index_name=self.index_name, credential=self.credential)\n\n def create_search_index_client(self) -> SearchIndexClient:\n return SearchIndexClient(endpoint=self.endpoint, credential=self.credential)\n\n def create_search_indexer_client(self) -> SearchIndexerClient:\n return SearchIndexerClient(endpoint=self.endpoint, credential=self.credential)\n\n\nclass DocumentAction(Enum):\n Add = 0\n Remove = 1\n RemoveAll = 2\n\n\nclass Strategy(ABC):\n \"\"\"\n Abstract strategy for ingesting documents into a search service. It has a single setup step to perform any required initialization, and then a run step that actually ingests documents into the search service.\n \"\"\"\n\n async def setup(self):\n raise NotImplementedError\n\n async def run(self):\n raise NotImplementedError\n" }, { "path": "app/backend/prepdocslib/textparser.py", "content": "import re\nfrom collections.abc import AsyncGenerator\nfrom typing import IO\n\nfrom .page import Page\nfrom .parser import Parser\n\n\ndef cleanup_data(data: str) -> str:\n \"\"\"Cleans up the given content using regexes\n Args:\n data: (str): The data to clean up.\n Returns:\n str: The cleaned up data.\n \"\"\"\n # match two or more newlines and replace them with one new line\n output = re.sub(r\"\\n{2,}\", \"\\n\", data)\n # match two or more spaces that are not newlines and replace them with one space\n output = re.sub(r\"[^\\S\\n]{2,}\", \" \", output)\n\n return output.strip()\n\n\nclass TextParser(Parser):\n \"\"\"Parses simple text into a Page object.\"\"\"\n\n async def parse(self, content: IO) -> AsyncGenerator[Page, None]:\n data = content.read()\n decoded_data = data.decode(\"utf-8\")\n text = cleanup_data(decoded_data)\n yield Page(0, 0, text=text)\n" }, { "path": "app/backend/prepdocslib/textprocessor.py", "content": "\"\"\"Utilities for processing document text and combining it with figure descriptions.\"\"\"\n\nimport logging\n\nfrom .figureprocessor import build_figure_markup\nfrom .listfilestrategy import File\nfrom .page import Page\nfrom .searchmanager import Section\nfrom .textsplitter import TextSplitter\n\nlogger = logging.getLogger(\"scripts\")\n\n\ndef combine_text_with_figures(page: \"Page\") -> None:\n \"\"\"Replace figure placeholders in page text with full description markup.\"\"\"\n for image in page.images:\n if image.description and image.placeholder in page.text:\n figure_markup = build_figure_markup(image, image.description)\n page.text = page.text.replace(image.placeholder, figure_markup)\n logger.info(\"Replaced placeholder for figure %s with description markup\", image.figure_id)\n elif not image.description:\n logger.debug(\"No description for figure %s; keeping placeholder\", image.figure_id)\n elif image.placeholder not in page.text:\n logger.warning(\"Placeholder not found for figure %s in page %d\", image.figure_id, page.page_num)\n\n\ndef process_text(\n pages: list[\"Page\"],\n file: \"File\",\n splitter: \"TextSplitter\",\n category: str | None = None,\n) -> list[\"Section\"]:\n \"\"\"Process document text and figures into searchable sections.\n Combines text with figure descriptions, splits into chunks, and\n associates figures with their containing sections.\n \"\"\"\n # Step 1: Combine text with figures on each page\n for page in pages:\n combine_text_with_figures(page)\n\n # Step 2: Split combined text into chunks\n logger.info(\"Splitting '%s' into sections\", file.filename())\n sections = [Section(chunk, content=file, category=category) for chunk in splitter.split_pages(pages)]\n\n # Step 3: Add images back to each section based on page number\n for section in sections:\n section.chunk.images = [\n image for page in pages if page.page_num == section.chunk.page_num for image in page.images\n ]\n\n return sections\n" }, { "path": "app/backend/prepdocslib/textsplitter.py", "content": "import logging\nimport re\nfrom abc import ABC\nfrom collections.abc import Generator\nfrom dataclasses import dataclass, field\nfrom typing import Optional\n\nimport tiktoken\n\nfrom .page import Chunk, Page\n\nlogger = logging.getLogger(\"scripts\")\n\n\nclass TextSplitter(ABC):\n \"\"\"\n Splits a list of pages into smaller chunks.\n :param pages: The pages to split\n :return: A generator of Chunk\n \"\"\"\n\n def split_pages(self, pages: list[Page]) -> Generator[Chunk, None, None]:\n if False: # pragma: no cover - this is necessary for mypy to type check\n yield\n\n\nENCODING_MODEL = \"text-embedding-ada-002\"\n\nSTANDARD_WORD_BREAKS = [\",\", \";\", \":\", \" \", \"(\", \")\", \"[\", \"]\", \"{\", \"}\", \"\\t\", \"\\n\"]\n\n# See W3C document https://www.w3.org/TR/jlreq/#cl-01\nCJK_WORD_BREAKS = [\n \"、\",\n \"，\",\n \"；\",\n \"：\",\n \"（\",\n \"）\",\n \"【\",\n \"】\",\n \"「\",\n \"」\",\n \"『\",\n \"』\",\n \"〔\",\n \"〕\",\n \"〈\",\n \"〉\",\n \"《\",\n \"》\",\n \"〖\",\n \"〗\",\n \"〘\",\n \"〙\",\n \"〚\",\n \"〛\",\n \"〝\",\n \"〞\",\n \"〟\",\n \"〰\",\n \"–\",\n \"—\",\n \"‘\",\n \"’\",\n \"‚\",\n \"‛\",\n \"“\",\n \"”\",\n \"„\",\n \"‟\",\n \"‹\",\n \"›\",\n]\n\nSTANDARD_SENTENCE_ENDINGS = [\".\", \"!\", \"?\"]\n\n# See CL05 and CL06, based on JIS X 4051:2004\n# https://www.w3.org/TR/jlreq/#cl-04\nCJK_SENTENCE_ENDINGS = [\"。\", \"！\", \"？\", \"‼\", \"⁇\", \"⁈\", \"⁉\"]\n\n# NB: text-embedding-3-XX is the same BPE as text-embedding-ada-002\nbpe = tiktoken.encoding_for_model(ENCODING_MODEL)\n\nDEFAULT_OVERLAP_PERCENT = 10 # See semantic search article for 10% overlap performance\nDEFAULT_SECTION_LENGTH = 1000 # Roughly 400-500 tokens for English\n\n\ndef _safe_concat(a: str, b: str) -> str:\n \"\"\"Concatenate two non-empty segments, inserting a space only when both sides\n end/start with alphanumerics and no natural boundary exists.\n\n Rules:\n - Both input strings are expected to be non-empty\n - Preserve existing whitespace if either side already provides a boundary.\n - Do not insert a space after a closing HTML tag marker '>'.\n - If both boundary characters are alphanumeric, insert a single space.\n - Otherwise concatenate directly.\n \"\"\"\n assert a and b, \"_safe_concat expects non-empty strings\"\n a_last = a[-1]\n b_first = b[0]\n if a_last.isspace() or b_first.isspace(): # pre-existing boundary\n return a + b\n if a_last == \">\": # HTML tag end acts as a boundary\n return a + b\n if a_last.isalnum() and b_first.isalnum(): # need explicit separator\n return a + \" \" + b\n return a + b\n\n\ndef _normalize_chunk(text: str, max_chars: int) -> str:\n \"\"\"Normalize a non-figure chunk that may slightly exceed max_chars.\n\n Allows overflow for any chunk containing a
tag (figures are atomic),\n trims leading spaces if they alone cause minor overflow, and as a final step\n removes a trailing space/newline when within a small tolerance (<=3 chars over).\n \"\"\"\n lower = text.lower()\n if \" max_chars:\n trimmed = trimmed[1:]\n if len(trimmed) > max_chars and len(trimmed) <= max_chars + 3:\n if trimmed.endswith(\" \") or trimmed.endswith(\"\\n\"):\n trimmed = trimmed.rstrip()\n return trimmed\n\n\n@dataclass\nclass _ChunkBuilder:\n \"\"\"Accumulates sentence-like spans for a single page until size limits are reached.\n\n Responsibilities:\n - Track appended text fragments and their approximate token length.\n - Decide if a new span can be added without exceeding character or token thresholds.\n - Flush accumulated content into an output list as a `Chunk`.\n - Allow a figure block to be force-appended (even if it overflows) so that headings + figure stay together.\n\n Notes:\n - Character limit is soft (exact enforcement + later normalization); token limit is hard.\n - Token counts are computed by the caller and passed to `add`; this class stays agnostic of the encoder.\n \"\"\"\n\n page_num: int\n max_chars: int\n max_tokens: int\n parts: list[str] = field(default_factory=list)\n token_len: int = 0\n\n def can_fit(self, text: str, token_count: int) -> bool:\n if not self.parts: # always allow first span\n return token_count <= self.max_tokens and len(text) <= self.max_chars\n # Character + token constraints\n return (len(\"\".join(self.parts)) + len(text) <= self.max_chars) and (\n self.token_len + token_count <= self.max_tokens\n )\n\n def add(self, text: str, token_count: int) -> bool:\n if not self.can_fit(text, token_count):\n return False\n self.parts.append(text)\n self.token_len += token_count\n return True\n\n def force_append(self, text: str):\n self.parts.append(text)\n\n def flush_into(self, out: list[Chunk]):\n if self.parts:\n chunk = \"\".join(self.parts)\n if chunk.strip():\n out.append(Chunk(page_num=self.page_num, text=chunk))\n self.parts.clear()\n self.token_len = 0\n\n # Convenience helpers for readability at call sites\n def has_content(self) -> bool:\n return bool(self.parts)\n\n def append_figure_and_flush(self, figure_text: str, out: list[Chunk]):\n \"\"\"Append a figure (allowed to overflow) to current accumulation and flush in one step.\"\"\"\n self.force_append(figure_text)\n self.flush_into(out)\n\n\nclass SentenceTextSplitter(TextSplitter):\n \"\"\"\n Class that splits pages into smaller chunks. This is required because embedding models may not be able to analyze an entire page at once\n \"\"\"\n\n def __init__(self, max_tokens_per_section: int = 500):\n self.sentence_endings = STANDARD_SENTENCE_ENDINGS + CJK_SENTENCE_ENDINGS\n self.word_breaks = STANDARD_WORD_BREAKS + CJK_WORD_BREAKS\n self.max_section_length = DEFAULT_SECTION_LENGTH\n self.sentence_search_limit = 100\n self.max_tokens_per_section = max_tokens_per_section\n self.section_overlap = int(self.max_section_length * DEFAULT_OVERLAP_PERCENT / 100)\n # Always-on semantic overlap percent (duplicated suffix of previous chunk), applied:\n # - Between chunks on the same page.\n # - Across page boundary ONLY if semantic continuation heuristics pass.\n self.semantic_overlap_percent = 10\n\n def _find_split_pos(self, text: str) -> tuple[int, bool]:\n \"\"\"Find a good split position near midpoint.\n\n Returns (index, use_overlap_fallback).\n\n Priority:\n 1. Sentence-ending punctuation near midpoint (scan outward within central third).\n 2. Word-break character near midpoint (space / punctuation) within same window.\n 3. Fallback: caller should use midpoint + overlap strategy.\n \"\"\"\n length = len(text)\n if length < 4:\n return -1, True\n mid = length // 2\n window_limit = length // 3 # defines central region scan boundary\n\n # 1. Sentence endings\n pos = 0\n while mid - pos > window_limit:\n left = mid - pos\n right = mid + pos\n if left >= 0 and text[left] in self.sentence_endings:\n return left, False\n if right < length and text[right] in self.sentence_endings:\n return right, False\n pos += 1\n\n # 2. Word breaks\n pos = 0\n while mid - pos > window_limit:\n left = mid - pos\n right = mid + pos\n if left >= 0 and text[left] in self.word_breaks:\n return left, False\n if right < length and text[right] in self.word_breaks:\n return right, False\n pos += 1\n\n # 3. Fallback\n return -1, True\n\n def split_page_by_max_tokens(self, page_num: int, text: str) -> Generator[Chunk, None, None]:\n \"\"\"Recursively split plain text by token count.\n\n Boundary preference order when an oversized span is encountered:\n 1. Sentence-ending punctuation near midpoint.\n 2. Word-break character near midpoint (space/punctuation) to avoid mid-word cuts.\n 3. Midpoint split with symmetric overlap (DEFAULT_OVERLAP_PERCENT).\n \"\"\"\n tokens = bpe.encode(text)\n if len(tokens) <= self.max_tokens_per_section:\n yield Chunk(page_num=page_num, text=text)\n return\n\n split_pos, use_overlap = self._find_split_pos(text)\n if not use_overlap and split_pos > 0:\n first_half = text[: split_pos + 1]\n second_half = text[split_pos + 1 :]\n else:\n middle = len(text) // 2\n overlap = int(len(text) * (DEFAULT_OVERLAP_PERCENT / 100))\n first_half = text[: middle + overlap]\n second_half = text[middle - overlap :]\n\n yield from self.split_page_by_max_tokens(page_num, first_half)\n yield from self.split_page_by_max_tokens(page_num, second_half)\n\n def _is_heading_like(self, line: str) -> bool:\n \"\"\"Heuristic heading detector used to suppress cross-page semantic overlap when a new section starts.\"\"\"\n line_str = line.strip()\n if not line_str:\n return False\n if line_str.startswith(\"#\"):\n return True\n # Short Title Case or ALL CAPS lines (limited word count) often represent headings\n if len(line_str) <= 80 and (line_str.isupper() or (line_str.istitle() and len(line_str.split()) <= 12)):\n return True\n import re as _re\n\n # Numbered / roman numeral list or section forms: '1. ', 'II) ', 'III. '\n if _re.match(r\"^(?:\\d+|[IVXLCM]+)[.)]\\s\", line_str):\n return True\n if line_str.startswith((\"- \", \"* \", \"• \")):\n return True\n return False\n\n def _should_cross_page_overlap(self, prev: Chunk, nxt: Chunk) -> bool:\n if not prev or not nxt:\n return False\n if \" Chunk:\n \"\"\"Return a modified copy of prev_chunk whose text has an appended semantic\n overlap prefix from next_chunk; next_chunk itself is left unchanged so it\n continues to start at its natural sentence boundary.\n\n Strategy:\n - Take ~semantic_overlap_percent tail size (in chars) from the START of next_chunk.\n - Extend that region forward to the first sentence-ending (preferred) or word break\n so we end on a natural boundary (avoid chopping mid-word/mid-sentence).\n - Refuse overlap if either chunk contains a
to avoid duplicating figures.\n - Enforce hard token + soft char limits; shrink overlap if necessary.\n \"\"\"\n if not prev_chunk or not next_chunk:\n return prev_chunk\n if \" 20: # fallback boundary after some progress\n boundary_found = True\n break\n if not boundary_found:\n # Trim trailing partial word if we stopped without boundary\n while prefix and prefix[-1].isalnum() and len(prefix) > target:\n prefix = prefix[:-1]\n\n # Avoid appending text that already exists at end (rare but possible due to prior operations)\n if prev_chunk.text.endswith(prefix):\n return prev_chunk\n\n candidate = prev_chunk.text + prefix\n max_chars = int(self.max_section_length * 1.2)\n if len(candidate) > max_chars or len(bpe.encode(candidate)) > self.max_tokens_per_section:\n # Attempt to shrink prefix at word / sentence boundaries from its start\n shrink = prefix\n while shrink and (\n len(prev_chunk.text + shrink) > max_chars\n or len(bpe.encode(prev_chunk.text + shrink)) > self.max_tokens_per_section\n ):\n cut_index = 1\n for i, ch in enumerate(shrink):\n if ch in self.word_breaks or ch in self.sentence_endings:\n cut_index = i + 1\n break\n shrink = shrink[:-cut_index] if cut_index < len(shrink) else \"\"\n if not shrink:\n return prev_chunk\n candidate = prev_chunk.text + shrink\n if len(candidate) > max_chars or len(bpe.encode(candidate)) > self.max_tokens_per_section:\n return prev_chunk\n return Chunk(page_num=prev_chunk.page_num, text=candidate)\n\n def split_pages(self, pages: list[Page]) -> Generator[Chunk, None, None]:\n \"\"\"Split each page into semantic chunks using token-aware accumulation with atomic figures.\n\n Strategy (per page):\n 1. Extract balanced
...
blocks as atomic \"figure\" blocks.\n 2. Treat intervening text as \"text\" blocks.\n 3. For text blocks, break into sentence-ish spans (using sentence ending chars) and accumulate\n until adding the next span would exceed either character or token limit. Flush when needed.\n 4. When a figure block arrives:\n - If there is accumulated text (builder), append the figure even if this exceeds token limit and flush.\n - If no accumulated text, emit the figure as its own chunk.\n 5. Ignore token limits for any chunk that contains a figure (never split figures).\n This avoids partial/duplicated figures and keeps headings with their following figure when space permits.\n \"\"\"\n figure_regex = re.compile(r\"\", re.IGNORECASE | re.DOTALL)\n previous_chunk: Optional[Chunk] = None\n\n for page in pages:\n raw = page.text or \"\"\n if not raw.strip():\n continue\n\n # Build ordered list of blocks: (type, text)\n blocks: list[tuple[str, str]] = []\n last = 0\n for m in figure_regex.finditer(raw):\n if m.start() > last:\n blocks.append((\"text\", raw[last : m.start()]))\n blocks.append((\"figure\", m.group()))\n last = m.end()\n if last < len(raw):\n blocks.append((\"text\", raw[last:]))\n\n page_chunks: list[Chunk] = []\n builder = _ChunkBuilder(\n page_num=page.page_num,\n max_chars=self.max_section_length,\n max_tokens=self.max_tokens_per_section,\n )\n\n for btype, btext in blocks:\n if btype == \"figure\":\n if builder.has_content():\n # Append figure to existing text (allow overflow) and flush\n builder.append_figure_and_flush(btext, page_chunks)\n else:\n # Emit figure standalone\n if btext.strip():\n page_chunks.append(Chunk(page_num=page.page_num, text=btext))\n continue\n\n # Process text block: split into sentence-like spans\n spans: list[str] = []\n current_chars: list[str] = []\n for ch in btext:\n current_chars.append(ch)\n if ch in self.sentence_endings:\n spans.append(\"\".join(current_chars))\n current_chars = []\n if current_chars: # remaining tail\n spans.append(\"\".join(current_chars))\n\n for span in spans:\n span_tokens = len(bpe.encode(span))\n # If a single span itself exceeds token limit (rare, very long sentence), split it directly\n if span_tokens > self.max_tokens_per_section:\n builder.flush_into(page_chunks)\n for chunk in self.split_page_by_max_tokens(page.page_num, span):\n page_chunks.append(chunk)\n continue\n if not builder.add(span, span_tokens):\n # Flush and retry (guaranteed to fit because span_tokens <= limit)\n builder.flush_into(page_chunks)\n if not builder.add(span, span_tokens):\n page_chunks.append(Chunk(page_num=page.page_num, text=span))\n\n # Flush any trailing builder content\n builder.flush_into(page_chunks)\n\n # Attempt cross-page merge with previous_chunk (look-behind) if semantic continuation\n if previous_chunk and page_chunks:\n prev_last_char = previous_chunk.text.rstrip()[-1:] if previous_chunk.text.rstrip() else \"\"\n first_new = page_chunks[0]\n first_new_stripped = first_new.text.lstrip()\n first_char = first_new_stripped[:1]\n if (\n prev_last_char\n and prev_last_char not in self.sentence_endings\n and not first_new_stripped.startswith(\"#\")\n and first_char\n and first_char.islower()\n and \" bool:\n combined = candidate + first_new_text\n if len(combined) > max_chars:\n return False\n if len(bpe.encode(combined)) > self.max_tokens_per_section:\n return False\n return True\n\n move_fragment = fragment_full\n if not fits(move_fragment):\n # Hard trim path: fragment begins after the last sentence-ending punctuation\n # of the previous chunk. Reduce to remaining character budget, then iteratively\n # shrink until token constraints are satisfied.\n remaining_chars = max_chars - len(first_new_text) # always > 0 given builder invariants\n move_fragment = move_fragment[:remaining_chars]\n while (\n move_fragment\n and len(bpe.encode(move_fragment + first_new_text)) > self.max_tokens_per_section\n ):\n move_fragment = (\n move_fragment[:-50] if len(move_fragment) > 50 else move_fragment[:-1]\n )\n leftover_fragment = fragment_full[len(move_fragment) :]\n # Prepend the allowed fragment\n if move_fragment:\n page_chunks[0] = Chunk(\n page_num=page_chunks[0].page_num,\n text=_safe_concat(move_fragment, first_new_text),\n )\n # Adjust previous_chunk retained portion\n if retained.strip():\n previous_chunk = Chunk(page_num=previous_chunk.page_num, text=retained)\n else:\n previous_chunk = None\n # Insert leftover fragment as its own chunk (split if needed) BEFORE modified first_new\n if leftover_fragment.strip():\n # Ensure leftover respects limits by splitting if needed\n leftover_pages = list(\n self.split_page_by_max_tokens(page_chunks[0].page_num, leftover_fragment)\n )\n # Insert these before current first chunk\n page_chunks = leftover_pages + page_chunks\n\n # Normalize chunks (non-figure) that barely exceed char limit due to added boundary space\n max_chars = int(self.max_section_length * 1.2)\n if previous_chunk:\n previous_chunk = Chunk(\n page_num=previous_chunk.page_num, text=_normalize_chunk(previous_chunk.text, max_chars)\n )\n if page_chunks:\n page_chunks = [\n Chunk(page_num=chunk.page_num, text=_normalize_chunk(chunk.text, max_chars))\n for chunk in page_chunks\n ]\n\n # Apply semantic overlap duplication (append style). We append a small\n # prefix of the NEXT chunk onto the PREVIOUS chunk, keeping natural starts.\n if self.semantic_overlap_percent > 0:\n # Cross-page overlap: modify previous_chunk (look-ahead to first new chunk)\n if previous_chunk and page_chunks and self._should_cross_page_overlap(previous_chunk, page_chunks[0]):\n previous_chunk = self._append_overlap(previous_chunk, page_chunks[0])\n\n # Intra-page overlaps\n if len(page_chunks) > 1:\n for i in range(1, len(page_chunks)):\n prev_c = page_chunks[i - 1]\n curr_c = page_chunks[i]\n if \" Generator[Chunk, None, None]:\n all_text = \"\".join(page.text for page in pages)\n if len(all_text.strip()) == 0:\n return\n\n length = len(all_text)\n if length <= self.max_object_length:\n yield Chunk(page_num=0, text=all_text)\n return\n\n # its too big, so we need to split it\n for i in range(0, length, self.max_object_length):\n yield Chunk(page_num=i // self.max_object_length, text=all_text[i : i + self.max_object_length])\n return\n" }, { "path": "app/backend/requirements.in", "content": "azure-functions>=1.24.0\nazure-identity\nquart\nquart-cors\nopenai>=1.109.1\ntiktoken\ntenacity\nazure-ai-documentintelligence==1.0.2\nazure-cognitiveservices-speech\nazure-cosmos\nazure-search-documents==11.7.0b2\nazure-storage-blob\nazure-storage-file-datalake\nuvicorn\naiohttp\nazure-monitor-opentelemetry\nopentelemetry-instrumentation-asgi\nopentelemetry-instrumentation-httpx\nopentelemetry-instrumentation-aiohttp-client\nopentelemetry-instrumentation-openai\nmsal\ncryptography\nPyJWT\nPillow\ntypes-Pillow\npypdf\nPyMuPDF\nbeautifulsoup4\ntypes-beautifulsoup4\nmsgraph-sdk\npython-dotenv\njinja2\nrich\ntyping-extensions\n" }, { "path": "app/backend/requirements.txt", "content": "# This file was autogenerated by uv via the following command:\n# uv pip compile requirements.in -o requirements.txt --python-version 3.10\naiofiles==24.1.0\n # via quart\naiohappyeyeballs==2.6.1\n # via aiohttp\naiohttp==3.13.3\n # via\n # -r requirements.in\n # microsoft-kiota-authentication-azure\naiosignal==1.4.0\n # via aiohttp\nannotated-types==0.7.0\n # via pydantic\nanyio==4.4.0\n # via\n # httpx\n # openai\nasgiref==3.10.0\n # via opentelemetry-instrumentation-asgi\nattrs==25.3.0\n # via aiohttp\nazure-ai-documentintelligence==1.0.2\n # via -r requirements.in\nazure-cognitiveservices-speech==1.40.0\n # via -r requirements.in\nazure-common==1.1.28\n # via azure-search-documents\nazure-core==1.38.0\n # via\n # azure-ai-documentintelligence\n # azure-core-tracing-opentelemetry\n # azure-cosmos\n # azure-identity\n # azure-monitor-opentelemetry\n # azure-monitor-opentelemetry-exporter\n # azure-search-documents\n # azure-storage-blob\n # azure-storage-file-datalake\n # microsoft-kiota-authentication-azure\n # msrest\nazure-core-tracing-opentelemetry==1.0.0b11\n # via azure-monitor-opentelemetry\nazure-cosmos==4.9.0\n # via -r requirements.in\nazure-functions==1.24.0\n # via -r requirements.in\nazure-identity==1.17.1\n # via\n # -r requirements.in\n # azure-monitor-opentelemetry-exporter\n # msgraph-sdk\nazure-monitor-opentelemetry==1.8.4\n # via -r requirements.in\nazure-monitor-opentelemetry-exporter==1.0.0b46\n # via azure-monitor-opentelemetry\nazure-search-documents==11.7.0b2\n # via -r requirements.in\nazure-storage-blob==12.22.0\n # via\n # -r requirements.in\n # azure-storage-file-datalake\nazure-storage-file-datalake==12.16.0\n # via -r requirements.in\nbeautifulsoup4==4.14.3\n # via -r requirements.in\nblinker==1.9.0\n # via\n # flask\n # quart\ncertifi==2026.1.4\n # via\n # httpcore\n # httpx\n # msrest\n # requests\ncffi==2.0.0\n # via cryptography\ncharset-normalizer==3.3.2\n # via requests\nclick==8.3.1\n # via\n # flask\n # quart\n # uvicorn\ncryptography==46.0.5\n # via\n # -r requirements.in\n # azure-identity\n # azure-storage-blob\n # msal\n # pyjwt\ndistro==1.9.0\n # via openai\nflask==3.1.3\n # via quart\nfrozenlist==1.8.0\n # via\n # aiohttp\n # aiosignal\nh11==0.16.0\n # via\n # httpcore\n # hypercorn\n # uvicorn\n # wsproto\nh2==4.3.0\n # via\n # httpx\n # hypercorn\nhpack==4.1.0\n # via h2\nhttpcore==1.0.9\n # via httpx\nhttpx[http2]==0.28.1\n # via\n # microsoft-kiota-http\n # msgraph-core\n # openai\nhypercorn==0.18.0\n # via quart\nhyperframe==6.1.0\n # via h2\nidna==3.10\n # via\n # anyio\n # httpx\n # requests\n # yarl\nimportlib-metadata==8.0.0\n # via opentelemetry-api\nisodate==0.7.2\n # via\n # azure-ai-documentintelligence\n # azure-search-documents\n # azure-storage-blob\n # azure-storage-file-datalake\n # msrest\nitsdangerous==2.2.0\n # via\n # flask\n # quart\njinja2==3.1.6\n # via\n # -r requirements.in\n # flask\n # quart\njiter==0.11.0\n # via openai\nmarkdown-it-py==3.0.0\n # via rich\nmarkupsafe==3.0.3\n # via\n # flask\n # jinja2\n # quart\n # werkzeug\nmdurl==0.1.2\n # via markdown-it-py\nmicrosoft-kiota-abstractions==1.9.3\n # via\n # microsoft-kiota-authentication-azure\n # microsoft-kiota-http\n # microsoft-kiota-serialization-form\n # microsoft-kiota-serialization-json\n # microsoft-kiota-serialization-multipart\n # microsoft-kiota-serialization-text\n # msgraph-core\nmicrosoft-kiota-authentication-azure==1.9.3\n # via msgraph-core\nmicrosoft-kiota-http==1.9.3\n # via msgraph-core\nmicrosoft-kiota-serialization-form==1.9.3\n # via msgraph-sdk\nmicrosoft-kiota-serialization-json==1.9.3\n # via msgraph-sdk\nmicrosoft-kiota-serialization-multipart==1.9.3\n # via msgraph-sdk\nmicrosoft-kiota-serialization-text==1.9.3\n # via msgraph-sdk\nmsal==1.33.0\n # via\n # -r requirements.in\n # azure-identity\n # msal-extensions\nmsal-extensions==1.3.1\n # via azure-identity\nmsgraph-core==1.3.3\n # via msgraph-sdk\nmsgraph-sdk==1.45.0\n # via -r requirements.in\nmsrest==0.7.1\n # via azure-monitor-opentelemetry-exporter\nmultidict==6.7.0\n # via\n # aiohttp\n # yarl\noauthlib==3.3.1\n # via requests-oauthlib\nopenai==2.6.1\n # via -r requirements.in\nopentelemetry-api==1.39.0\n # via\n # azure-core-tracing-opentelemetry\n # azure-monitor-opentelemetry-exporter\n # microsoft-kiota-abstractions\n # microsoft-kiota-authentication-azure\n # microsoft-kiota-http\n # opentelemetry-instrumentation\n # opentelemetry-instrumentation-aiohttp-client\n # opentelemetry-instrumentation-asgi\n # opentelemetry-instrumentation-dbapi\n # opentelemetry-instrumentation-django\n # opentelemetry-instrumentation-fastapi\n # opentelemetry-instrumentation-flask\n # opentelemetry-instrumentation-httpx\n # opentelemetry-instrumentation-openai\n # opentelemetry-instrumentation-psycopg2\n # opentelemetry-instrumentation-requests\n # opentelemetry-instrumentation-urllib\n # opentelemetry-instrumentation-urllib3\n # opentelemetry-instrumentation-wsgi\n # opentelemetry-sdk\n # opentelemetry-semantic-conventions\nopentelemetry-instrumentation==0.60b0\n # via\n # opentelemetry-instrumentation-aiohttp-client\n # opentelemetry-instrumentation-asgi\n # opentelemetry-instrumentation-dbapi\n # opentelemetry-instrumentation-django\n # opentelemetry-instrumentation-fastapi\n # opentelemetry-instrumentation-flask\n # opentelemetry-instrumentation-httpx\n # opentelemetry-instrumentation-openai\n # opentelemetry-instrumentation-psycopg2\n # opentelemetry-instrumentation-requests\n # opentelemetry-instrumentation-urllib\n # opentelemetry-instrumentation-urllib3\n # opentelemetry-instrumentation-wsgi\nopentelemetry-instrumentation-aiohttp-client==0.60b0\n # via -r requirements.in\nopentelemetry-instrumentation-asgi==0.60b0\n # via\n # -r requirements.in\n # opentelemetry-instrumentation-fastapi\nopentelemetry-instrumentation-dbapi==0.60b0\n # via opentelemetry-instrumentation-psycopg2\nopentelemetry-instrumentation-django==0.60b0\n # via azure-monitor-opentelemetry\nopentelemetry-instrumentation-fastapi==0.60b0\n # via azure-monitor-opentelemetry\nopentelemetry-instrumentation-flask==0.60b0\n # via azure-monitor-opentelemetry\nopentelemetry-instrumentation-httpx==0.60b0\n # via -r requirements.in\nopentelemetry-instrumentation-openai==0.51.1\n # via -r requirements.in\nopentelemetry-instrumentation-psycopg2==0.60b0\n # via azure-monitor-opentelemetry\nopentelemetry-instrumentation-requests==0.60b0\n # via azure-monitor-opentelemetry\nopentelemetry-instrumentation-urllib==0.60b0\n # via azure-monitor-opentelemetry\nopentelemetry-instrumentation-urllib3==0.60b0\n # via azure-monitor-opentelemetry\nopentelemetry-instrumentation-wsgi==0.60b0\n # via\n # opentelemetry-instrumentation-django\n # opentelemetry-instrumentation-flask\nopentelemetry-resource-detector-azure==0.1.5\n # via azure-monitor-opentelemetry\nopentelemetry-sdk==1.39.0\n # via\n # azure-monitor-opentelemetry\n # azure-monitor-opentelemetry-exporter\n # microsoft-kiota-abstractions\n # microsoft-kiota-authentication-azure\n # microsoft-kiota-http\n # opentelemetry-resource-detector-azure\nopentelemetry-semantic-conventions==0.60b0\n # via\n # opentelemetry-instrumentation\n # opentelemetry-instrumentation-aiohttp-client\n # opentelemetry-instrumentation-asgi\n # opentelemetry-instrumentation-dbapi\n # opentelemetry-instrumentation-django\n # opentelemetry-instrumentation-fastapi\n # opentelemetry-instrumentation-flask\n # opentelemetry-instrumentation-httpx\n # opentelemetry-instrumentation-openai\n # opentelemetry-instrumentation-requests\n # opentelemetry-instrumentation-urllib\n # opentelemetry-instrumentation-urllib3\n # opentelemetry-instrumentation-wsgi\n # opentelemetry-sdk\nopentelemetry-semantic-conventions-ai==0.4.13\n # via opentelemetry-instrumentation-openai\nopentelemetry-util-http==0.60b0\n # via\n # opentelemetry-instrumentation-aiohttp-client\n # opentelemetry-instrumentation-asgi\n # opentelemetry-instrumentation-django\n # opentelemetry-instrumentation-fastapi\n # opentelemetry-instrumentation-flask\n # opentelemetry-instrumentation-httpx\n # opentelemetry-instrumentation-requests\n # opentelemetry-instrumentation-urllib\n # opentelemetry-instrumentation-urllib3\n # opentelemetry-instrumentation-wsgi\npackaging==24.1\n # via\n # opentelemetry-instrumentation\n # opentelemetry-instrumentation-flask\npillow==12.1.1\n # via -r requirements.in\npriority==2.0.0\n # via hypercorn\npropcache==0.2.0\n # via\n # aiohttp\n # yarl\npsutil==7.1.2\n # via azure-monitor-opentelemetry-exporter\npycparser==2.22\n # via cffi\npydantic==2.12.5\n # via openai\npydantic-core==2.41.5\n # via pydantic\npygments==2.19.2\n # via rich\npyjwt[crypto]==2.11.0\n # via\n # -r requirements.in\n # msal\npymupdf==1.26.0\n # via -r requirements.in\npypdf==6.8.0\n # via -r requirements.in\npython-dotenv==1.1.1\n # via -r requirements.in\nquart==0.20.0\n # via\n # -r requirements.in\n # quart-cors\nquart-cors==0.7.0\n # via -r requirements.in\nregex==2025.7.34\n # via tiktoken\nrequests==2.32.4\n # via\n # azure-core\n # msal\n # msrest\n # requests-oauthlib\n # tiktoken\nrequests-oauthlib==2.0.0\n # via msrest\nrich==14.1.0\n # via -r requirements.in\nsniffio==1.3.1\n # via\n # anyio\n # openai\nsoupsieve==2.7\n # via beautifulsoup4\nstd-uritemplate==2.0.8\n # via microsoft-kiota-abstractions\ntenacity==9.1.2\n # via -r requirements.in\ntiktoken==0.12.0\n # via -r requirements.in\ntqdm==4.66.5\n # via openai\ntypes-beautifulsoup4==4.12.0.20250516\n # via -r requirements.in\ntypes-html5lib==1.1.11.20251014\n # via types-beautifulsoup4\ntypes-pillow==10.2.0.20240822\n # via -r requirements.in\ntyping-extensions==4.15.0\n # via\n # -r requirements.in\n # azure-ai-documentintelligence\n # azure-core\n # azure-cosmos\n # azure-identity\n # azure-search-documents\n # azure-storage-blob\n # azure-storage-file-datalake\n # beautifulsoup4\n # openai\n # opentelemetry-api\n # opentelemetry-sdk\n # opentelemetry-semantic-conventions\n # pydantic\n # pydantic-core\n # typing-inspection\ntyping-inspection==0.4.2\n # via pydantic\nurllib3==2.6.3\n # via requests\nuvicorn==0.30.6\n # via -r requirements.in\nwerkzeug==3.1.6\n # via\n # azure-functions\n # flask\n # quart\nwrapt==1.16.0\n # via\n # opentelemetry-instrumentation\n # opentelemetry-instrumentation-aiohttp-client\n # opentelemetry-instrumentation-dbapi\n # opentelemetry-instrumentation-httpx\n # opentelemetry-instrumentation-urllib3\nwsproto==1.3.2\n # via hypercorn\nyarl==1.17.2\n # via aiohttp\nzipp==3.23.0\n # via importlib-metadata\n" }, { "path": "app/backend/setup_cloud_ingestion.py", "content": "\"\"\"Script to setup cloud ingestion for Azure AI Search.\"\"\"\n\nimport asyncio\nimport logging\nimport os\n\nfrom azure.core.credentials_async import AsyncTokenCredential\nfrom azure.identity.aio import AzureDeveloperCliCredential\nfrom openai import AsyncOpenAI\nfrom rich.logging import RichHandler\n\nfrom load_azd_env import load_azd_env\nfrom prepdocslib.blobmanager import BlobManager\nfrom prepdocslib.cloudingestionstrategy import CloudIngestionStrategy\nfrom prepdocslib.listfilestrategy import LocalListFileStrategy\nfrom prepdocslib.servicesetup import (\n OpenAIHost,\n clean_key_if_exists,\n setup_blob_manager,\n setup_embeddings_service,\n setup_openai_client,\n setup_search_info,\n)\nfrom prepdocslib.strategy import DocumentAction\n\nlogger = logging.getLogger(\"scripts\")\n\n\nasync def setup_cloud_ingestion_strategy(\n azure_credential: AsyncTokenCredential,\n document_action: DocumentAction = DocumentAction.Add,\n) -> tuple[CloudIngestionStrategy, AsyncOpenAI, AsyncTokenCredential, BlobManager]:\n \"\"\"Setup the cloud ingestion strategy with all required services.\"\"\"\n\n # Get environment variables\n search_service = os.environ[\"AZURE_SEARCH_SERVICE\"]\n index_name = os.environ[\"AZURE_SEARCH_INDEX\"]\n search_user_assigned_identity_resource_id = os.environ[\"AZURE_SEARCH_USER_ASSIGNED_IDENTITY_RESOURCE_ID\"]\n storage_container = os.environ[\"AZURE_STORAGE_CONTAINER\"]\n subscription_id = os.environ[\"AZURE_SUBSCRIPTION_ID\"]\n image_storage_container = os.environ.get(\"AZURE_IMAGESTORAGE_CONTAINER\")\n search_embedding_field = os.environ[\"AZURE_SEARCH_FIELD_NAME_EMBEDDING\"]\n\n # Cloud ingestion storage account (ADLS Gen2 when ACLs enabled, standard blob otherwise)\n # Fallback to AZURE_STORAGE_ACCOUNT is for legacy deployments only - may be removed in future\n storage_account = os.getenv(\"AZURE_CLOUD_INGESTION_STORAGE_ACCOUNT\") or os.environ[\"AZURE_STORAGE_ACCOUNT\"]\n storage_resource_group = (\n os.getenv(\"AZURE_CLOUD_INGESTION_STORAGE_RESOURCE_GROUP\") or os.environ[\"AZURE_STORAGE_RESOURCE_GROUP\"]\n )\n\n # Cloud ingestion specific endpoints\n document_extractor_uri = os.environ[\"DOCUMENT_EXTRACTOR_SKILL_ENDPOINT\"]\n document_extractor_resource_id = os.environ[\"DOCUMENT_EXTRACTOR_SKILL_AUTH_RESOURCE_ID\"]\n figure_processor_uri = os.environ[\"FIGURE_PROCESSOR_SKILL_ENDPOINT\"]\n figure_processor_resource_id = os.environ[\"FIGURE_PROCESSOR_SKILL_AUTH_RESOURCE_ID\"]\n text_processor_uri = os.environ[\"TEXT_PROCESSOR_SKILL_ENDPOINT\"]\n text_processor_resource_id = os.environ[\"TEXT_PROCESSOR_SKILL_AUTH_RESOURCE_ID\"]\n\n # Feature flags\n use_multimodal = os.getenv(\"USE_MULTIMODAL\", \"\").lower() == \"true\"\n use_acls = os.getenv(\"USE_CLOUD_INGESTION_ACLS\", \"\").lower() == \"true\"\n enforce_access_control = os.getenv(\"AZURE_ENFORCE_ACCESS_CONTROL\", \"\").lower() == \"true\"\n use_web_source = os.getenv(\"USE_WEB_SOURCE\", \"\").lower() == \"true\"\n\n # Warn if access control is enforced but ACL extraction is not enabled\n if enforce_access_control and not use_acls:\n logger.warning(\n \"AZURE_ENFORCE_ACCESS_CONTROL is enabled but USE_CLOUD_INGESTION_ACLS is not. \"\n \"Documents will be indexed without ACLs, so access control filtering will not work. \"\n \"Either set USE_CLOUD_INGESTION_ACLS=true to extract ACLs from ADLS Gen2, \"\n \"or manually set ACLs using scripts/manageacl.py after ingestion.\"\n )\n\n # Setup search info\n search_info = setup_search_info(\n search_service=search_service,\n index_name=index_name,\n azure_credential=azure_credential,\n azure_vision_endpoint=os.getenv(\"AZURE_VISION_ENDPOINT\"),\n )\n\n # Setup blob manager\n blob_manager = setup_blob_manager(\n azure_credential=azure_credential,\n storage_account=storage_account,\n storage_container=storage_container,\n storage_resource_group=storage_resource_group,\n subscription_id=subscription_id,\n storage_key=None,\n image_storage_container=image_storage_container,\n )\n\n # Setup OpenAI embeddings\n OPENAI_HOST = OpenAIHost(os.environ[\"OPENAI_HOST\"])\n openai_client, azure_openai_endpoint = setup_openai_client(\n openai_host=OPENAI_HOST,\n azure_credential=azure_credential,\n azure_openai_service=os.getenv(\"AZURE_OPENAI_SERVICE\"),\n azure_openai_custom_url=os.getenv(\"AZURE_OPENAI_CUSTOM_URL\"),\n azure_openai_api_key=os.getenv(\"AZURE_OPENAI_API_KEY_OVERRIDE\"),\n openai_api_key=clean_key_if_exists(os.getenv(\"OPENAI_API_KEY\")),\n openai_organization=os.getenv(\"OPENAI_ORGANIZATION\"),\n )\n\n emb_model_dimensions = 1536\n if os.getenv(\"AZURE_OPENAI_EMB_DIMENSIONS\"):\n emb_model_dimensions = int(os.environ[\"AZURE_OPENAI_EMB_DIMENSIONS\"])\n\n openai_embeddings_service = setup_embeddings_service(\n OPENAI_HOST,\n openai_client,\n emb_model_name=os.environ[\"AZURE_OPENAI_EMB_MODEL_NAME\"],\n emb_model_dimensions=emb_model_dimensions,\n azure_openai_deployment=os.getenv(\"AZURE_OPENAI_EMB_DEPLOYMENT\"),\n azure_openai_endpoint=azure_openai_endpoint,\n disable_batch=False,\n )\n\n # Create a list file strategy for uploading files from the data folder\n list_file_strategy = LocalListFileStrategy(path_pattern=\"data/*\", enable_global_documents=False)\n\n # Create the cloud ingestion strategy\n ingestion_strategy = CloudIngestionStrategy(\n list_file_strategy=list_file_strategy,\n blob_manager=blob_manager,\n search_info=search_info,\n embeddings=openai_embeddings_service,\n search_field_name_embedding=search_embedding_field,\n document_extractor_uri=document_extractor_uri,\n document_extractor_auth_resource_id=document_extractor_resource_id,\n figure_processor_uri=figure_processor_uri,\n figure_processor_auth_resource_id=figure_processor_resource_id,\n text_processor_uri=text_processor_uri,\n text_processor_auth_resource_id=text_processor_resource_id,\n subscription_id=subscription_id,\n document_action=document_action,\n search_analyzer_name=os.getenv(\"AZURE_SEARCH_ANALYZER_NAME\"),\n use_acls=use_acls,\n use_multimodal=use_multimodal,\n enforce_access_control=enforce_access_control,\n use_web_source=use_web_source,\n search_user_assigned_identity_resource_id=search_user_assigned_identity_resource_id,\n )\n\n return ingestion_strategy, openai_client, azure_credential, blob_manager\n\n\nasync def main():\n \"\"\"Main function to setup cloud ingestion.\"\"\"\n load_azd_env()\n\n # Check if cloud ingestion is enabled\n use_cloud_ingestion = os.getenv(\"USE_CLOUD_INGESTION\", \"\").lower() == \"true\"\n if not use_cloud_ingestion:\n logger.info(\"Cloud ingestion is not enabled. Skipping setup.\")\n return\n\n # Setup logging\n logging.basicConfig(format=\"%(message)s\", datefmt=\"[%X]\", handlers=[RichHandler(rich_tracebacks=True)])\n logger.setLevel(logging.INFO)\n\n logger.info(\"Setting up cloud ingestion...\")\n\n # Use the current user identity to connect to Azure services\n if tenant_id := os.getenv(\"AZURE_TENANT_ID\"):\n logger.info(\"Connecting to Azure services using the azd credential for tenant %s\", tenant_id)\n azd_credential = AzureDeveloperCliCredential(tenant_id=tenant_id, process_timeout=60)\n else:\n logger.info(\"Connecting to Azure services using the azd credential for home tenant\")\n azd_credential = AzureDeveloperCliCredential(process_timeout=60)\n\n try:\n ingestion_strategy, openai_client, credential, blob_manager = await setup_cloud_ingestion_strategy(\n azure_credential=azd_credential,\n document_action=DocumentAction.Add,\n )\n\n # Setup the indexer, skillset, and data source\n logger.info(\"Setting up indexer, skillset, and data source...\")\n await ingestion_strategy.setup()\n logger.info(\"Triggering initial indexing run...\")\n await ingestion_strategy.run()\n\n finally:\n await blob_manager.close_clients()\n await openai_client.close()\n await azd_credential.close()\n\n\nif __name__ == \"__main__\":\n asyncio.run(main())\n" }, { "path": "app/frontend/.npmrc", "content": "engine-strict=true\nfund=false\n# Required because react-helmet-async declares peer deps for @types/react <19.0.0,\n# but works fine with React 19. See https://github.com/staylor/react-helmet-async/issues/238\nlegacy-peer-deps=true\n" }, { "path": "app/frontend/.nvmrc", "content": "22.0.0\n" }, { "path": "app/frontend/.prettierignore", "content": "# Ignore JSON\n**/*.json\n" }, { "path": "app/frontend/.prettierrc.json", "content": "{\n \"tabWidth\": 4,\n \"printWidth\": 160,\n \"arrowParens\": \"avoid\",\n \"trailingComma\": \"none\"\n}\n" }, { "path": "app/frontend/index.html", "content": "\n\n \n \n \n \n Azure OpenAI + AI Search\n \n \n
\n \n \n\n" }, { "path": "app/frontend/package.json", "content": "{\n \"name\": \"frontend\",\n \"private\": true,\n \"version\": \"0.0.0\",\n \"type\": \"module\",\n \"engines\": {\n \"node\": \">=20.0.0\"\n },\n \"scripts\": {\n \"dev\": \"vite --host 127.0.0.1\",\n \"build\": \"tsc && vite build\",\n \"preview\": \"vite preview\",\n \"format\": \"prettier --write .\"\n },\n \"dependencies\": {\n \"@azure/msal-browser\": \"^4.16.0\",\n \"@azure/msal-react\": \"^3.0.16\",\n \"@fluentui/react-components\": \"^9.73.1\",\n \"@fluentui/react-icons\": \"^2.0.319\",\n \"@fluentui/react-table\": \"^9.19.9\",\n \"@react-spring/web\": \"^10.0.3\",\n \"dompurify\": \"^3.3.2\",\n \"i18next\": \"^24.2.0\",\n \"i18next-browser-languagedetector\": \"^8.0.2\",\n \"i18next-http-backend\": \"^3.0.1\",\n \"idb\": \"^8.0.0\",\n \"ndjson-readablestream\": \"^1.4.0\",\n \"react\": \"^19.2.4\",\n \"react-dom\": \"^19.2.4\",\n \"react-helmet-async\": \"^2.0.5\",\n \"react-i18next\": \"^15.4.1\",\n \"react-markdown\": \"^9.0.1\",\n \"react-router-dom\": \"^7.12.0\",\n \"react-syntax-highlighter\": \"^16.1.0\",\n \"rehype-raw\": \"^7.0.0\",\n \"remark-gfm\": \"^4.0.0\",\n \"scheduler\": \"^0.27.0\"\n },\n \"devDependencies\": {\n \"@types/dom-speech-recognition\": \"^0.0.7\",\n \"@types/dompurify\": \"^3.0.5\",\n \"@types/react\": \"^19.2.13\",\n \"@types/react-dom\": \"^19.2.3\",\n \"@types/react-syntax-highlighter\": \"^15.5.13\",\n \"@vitejs/plugin-react\": \"^4.3.3\",\n \"prettier\": \"^3.3.3\",\n \"rollup-plugin-visualizer\": \"^5.12.0\",\n \"typescript\": \"^5.6.3\",\n \"vite\": \"^6.0.0\"\n }\n}\n" }, { "path": "app/frontend/src/api/api.ts", "content": "const BACKEND_URI = \"\";\n\nimport { ChatAppResponse, ChatAppResponseOrError, ChatAppRequest, Config, SimpleAPIResponse, HistoryListApiResponse, HistoryApiResponse } from \"./models\";\nimport { useLogin, getToken, isUsingAppServicesLogin } from \"../authConfig\";\n\nexport async function getHeaders(idToken: string | undefined): Promise> {\n // If using login and not using app services, add the id token of the logged in account as the authorization\n if (useLogin && !isUsingAppServicesLogin) {\n if (idToken) {\n return { Authorization: `Bearer ${idToken}` };\n }\n }\n\n return {};\n}\n\nexport async function configApi(): Promise {\n const response = await fetch(`${BACKEND_URI}/config`, {\n method: \"GET\"\n });\n\n return (await response.json()) as Config;\n}\n\nexport async function chatApi(request: ChatAppRequest, shouldStream: boolean, idToken: string | undefined, signal: AbortSignal): Promise {\n let url = `${BACKEND_URI}/chat`;\n if (shouldStream) {\n url += \"/stream\";\n }\n const headers = await getHeaders(idToken);\n return await fetch(url, {\n method: \"POST\",\n headers: { ...headers, \"Content-Type\": \"application/json\" },\n body: JSON.stringify(request),\n signal: signal\n });\n}\n\nexport async function getSpeechApi(text: string): Promise {\n return await fetch(\"/speech\", {\n method: \"POST\",\n headers: {\n \"Content-Type\": \"application/json\"\n },\n body: JSON.stringify({\n text: text\n })\n })\n .then(response => {\n if (response.status == 200) {\n return response.blob();\n } else if (response.status == 400) {\n console.log(\"Speech synthesis is not enabled.\");\n return null;\n } else {\n console.error(\"Unable to get speech synthesis.\");\n return null;\n }\n })\n .then(blob => (blob ? URL.createObjectURL(blob) : null));\n}\n\nexport function getCitationFilePath(citation: string): string {\n // If there are parentheses at end of citation, remove part in parentheses\n const cleanedCitation = citation.replace(/\\s*\\(.*?\\)\\s*$/, \"\").trim();\n return `${BACKEND_URI}/content/${cleanedCitation}`;\n}\n\nexport async function uploadFileApi(request: FormData, idToken: string): Promise {\n const response = await fetch(\"/upload\", {\n method: \"POST\",\n headers: await getHeaders(idToken),\n body: request\n });\n\n if (!response.ok) {\n throw new Error(`Uploading files failed: ${response.statusText}`);\n }\n\n const dataResponse: SimpleAPIResponse = await response.json();\n return dataResponse;\n}\n\nexport async function deleteUploadedFileApi(filename: string, idToken: string): Promise {\n const headers = await getHeaders(idToken);\n const response = await fetch(\"/delete_uploaded\", {\n method: \"POST\",\n headers: { ...headers, \"Content-Type\": \"application/json\" },\n body: JSON.stringify({ filename })\n });\n\n if (!response.ok) {\n throw new Error(`Deleting file failed: ${response.statusText}`);\n }\n\n const dataResponse: SimpleAPIResponse = await response.json();\n return dataResponse;\n}\n\nexport async function listUploadedFilesApi(idToken: string): Promise {\n const response = await fetch(`/list_uploaded`, {\n method: \"GET\",\n headers: await getHeaders(idToken)\n });\n\n if (!response.ok) {\n throw new Error(`Listing files failed: ${response.statusText}`);\n }\n\n const dataResponse: string[] = await response.json();\n return dataResponse;\n}\n\nexport async function postChatHistoryApi(item: any, idToken: string): Promise {\n const headers = await getHeaders(idToken);\n const response = await fetch(\"/chat_history\", {\n method: \"POST\",\n headers: { ...headers, \"Content-Type\": \"application/json\" },\n body: JSON.stringify(item)\n });\n\n if (!response.ok) {\n throw new Error(`Posting chat history failed: ${response.statusText}`);\n }\n\n const dataResponse: any = await response.json();\n return dataResponse;\n}\n\nexport async function getChatHistoryListApi(count: number, continuationToken: string | undefined, idToken: string): Promise {\n const headers = await getHeaders(idToken);\n let url = `${BACKEND_URI}/chat_history/sessions?count=${count}`;\n if (continuationToken) {\n url += `&continuationToken=${continuationToken}`;\n }\n\n const response = await fetch(url.toString(), {\n method: \"GET\",\n headers: { ...headers, \"Content-Type\": \"application/json\" }\n });\n\n if (!response.ok) {\n throw new Error(`Getting chat histories failed: ${response.statusText}`);\n }\n\n const dataResponse: HistoryListApiResponse = await response.json();\n return dataResponse;\n}\n\nexport async function getChatHistoryApi(id: string, idToken: string): Promise {\n const headers = await getHeaders(idToken);\n const response = await fetch(`/chat_history/sessions/${id}`, {\n method: \"GET\",\n headers: { ...headers, \"Content-Type\": \"application/json\" }\n });\n\n if (!response.ok) {\n throw new Error(`Getting chat history failed: ${response.statusText}`);\n }\n\n const dataResponse: HistoryApiResponse = await response.json();\n return dataResponse;\n}\n\nexport async function deleteChatHistoryApi(id: string, idToken: string): Promise {\n const headers = await getHeaders(idToken);\n const response = await fetch(`/chat_history/sessions/${id}`, {\n method: \"DELETE\",\n headers: { ...headers, \"Content-Type\": \"application/json\" }\n });\n\n if (!response.ok) {\n throw new Error(`Deleting chat history failed: ${response.statusText}`);\n }\n}\n" }, { "path": "app/frontend/src/api/index.ts", "content": "export * from \"./api\";\nexport * from \"./models\";\n" }, { "path": "app/frontend/src/api/models.ts", "content": "export const enum RetrievalMode {\n Hybrid = \"hybrid\",\n Vectors = \"vectors\",\n Text = \"text\"\n}\n\nexport type ChatAppRequestOverrides = {\n retrieval_mode?: RetrievalMode;\n semantic_ranker?: boolean;\n semantic_captions?: boolean;\n query_rewriting?: boolean;\n reasoning_effort?: string;\n include_category?: string;\n exclude_category?: string;\n seed?: number;\n top?: number;\n retrieval_reasoning_effort?: string;\n temperature?: number;\n minimum_search_score?: number;\n minimum_reranker_score?: number;\n prompt_template?: string;\n prompt_template_prefix?: string;\n prompt_template_suffix?: string;\n suggest_followup_questions?: boolean;\n send_text_sources: boolean;\n send_image_sources: boolean;\n search_text_embeddings: boolean;\n search_image_embeddings: boolean;\n language: string;\n use_agentic_knowledgebase: boolean;\n use_web_source?: boolean;\n use_sharepoint_source?: boolean;\n};\n\nexport type ResponseMessage = {\n content: string;\n role: string;\n};\n\nexport type Thoughts = {\n title: string;\n description: any; // It can be any output from the api\n props?: { [key: string]: any };\n};\n\nexport type ActivityDetail = {\n id?: number;\n number?: number;\n type?: string;\n label?: string;\n source?: string;\n query?: string;\n};\n\nexport type ExternalResultMetadata = {\n id?: string;\n title?: string;\n url?: string;\n snippet?: string;\n activity?: ActivityDetail;\n};\n\nexport type CitationActivityDetail = {\n id?: string;\n number?: number;\n type?: string;\n source?: string;\n query?: string;\n};\n\nexport type DataPoints = {\n text: string[];\n images: string[];\n citations: string[];\n citation_activity_details?: Record;\n external_results_metadata?: ExternalResultMetadata[];\n};\n\nexport type ResponseContext = {\n data_points: DataPoints;\n followup_questions: string[] | null;\n thoughts: Thoughts[];\n answer?: string;\n};\n\nexport type ChatAppResponseOrError = {\n message: ResponseMessage;\n delta: ResponseMessage;\n context: ResponseContext;\n session_state: any;\n error?: string;\n};\n\nexport type ChatAppResponse = {\n message: ResponseMessage;\n delta: ResponseMessage;\n context: ResponseContext;\n session_state: any;\n};\n\nexport type ChatAppRequestContext = {\n overrides?: ChatAppRequestOverrides;\n};\n\nexport type ChatAppRequest = {\n messages: ResponseMessage[];\n context?: ChatAppRequestContext;\n session_state: any;\n};\n\nexport type Config = {\n defaultReasoningEffort: string;\n defaultRetrievalReasoningEffort: string;\n showMultimodalOptions: boolean;\n showSemanticRankerOption: boolean;\n showQueryRewritingOption: boolean;\n showReasoningEffortOption: boolean;\n streamingEnabled: boolean;\n showVectorOption: boolean;\n showUserUpload: boolean;\n showLanguagePicker: boolean;\n showSpeechInput: boolean;\n showSpeechOutputBrowser: boolean;\n showSpeechOutputAzure: boolean;\n showChatHistoryBrowser: boolean;\n showChatHistoryCosmos: boolean;\n showAgenticRetrievalOption: boolean;\n ragSearchTextEmbeddings: boolean;\n ragSearchImageEmbeddings: boolean;\n ragSendTextSources: boolean;\n ragSendImageSources: boolean;\n webSourceEnabled: boolean;\n sharepointSourceEnabled: boolean;\n};\n\nexport type SimpleAPIResponse = {\n message?: string;\n};\n\nexport interface SpeechConfig {\n speechUrls: (string | null)[];\n setSpeechUrls: (urls: (string | null)[]) => void;\n audio: HTMLAudioElement;\n isPlaying: boolean;\n setIsPlaying: (isPlaying: boolean) => void;\n}\n\nexport type HistoryListApiResponse = {\n sessions: {\n id: string;\n entra_oid: string;\n title: string;\n timestamp: number;\n }[];\n continuation_token?: string;\n};\n\nexport type HistoryApiResponse = {\n id: string;\n entra_oid: string;\n answers: any;\n};\n" }, { "path": "app/frontend/src/authConfig.ts", "content": "// Refactored from https://github.com/Azure-Samples/ms-identity-javascript-react-tutorial/blob/main/1-Authentication/1-sign-in/SPA/src/authConfig.js\n\nimport { IPublicClientApplication } from \"@azure/msal-browser\";\n\nconst appServicesAuthTokenUrl = \".auth/me\";\nconst appServicesAuthTokenRefreshUrl = \".auth/refresh\";\nconst appServicesAuthLogoutUrl = \".auth/logout?post_logout_redirect_uri=/\";\n\ninterface AppServicesToken {\n id_token: string;\n access_token: string;\n user_claims: Record;\n expires_on: string;\n}\n\ninterface AuthSetup {\n // Set to true if login elements should be shown in the UI\n useLogin: boolean;\n // Set to true if access control is enforced by the application\n requireAccessControl: boolean;\n // Set to true if the application allows unauthenticated access (only applies for documents without access control)\n enableUnauthenticatedAccess: boolean;\n /**\n * Configuration object to be passed to MSAL instance on creation.\n * For a full list of MSAL.js configuration parameters, visit:\n * https://github.com/AzureAD/microsoft-authentication-library-for-js/blob/dev/lib/msal-browser/docs/configuration.md\n */\n msalConfig: {\n auth: {\n clientId: string; // Client app id used for login\n authority: string; // Directory to use for login https://learn.microsoft.com/entra/identity-platform/msal-client-application-configuration#authority\n redirectUri: string; // Points to window.location.origin. You must register this URI on Azure Portal/App Registration.\n postLogoutRedirectUri: string; // Indicates the page to navigate after logout.\n navigateToLoginRequestUrl: boolean; // If \"true\", will navigate back to the original request location before processing the auth code response.\n };\n cache: {\n cacheLocation: string; // Configures cache location. \"sessionStorage\" is more secure, but \"localStorage\" gives you SSO between tabs.\n storeAuthStateInCookie: boolean; // Set this to \"true\" if you are having issues on IE11 or Edge\n };\n };\n loginRequest: {\n /**\n * Scopes you add here will be prompted for user consent during sign-in.\n * By default, MSAL.js will add OIDC scopes (openid, profile, email) to any login request.\n * For more information about OIDC scopes, visit:\n * https://learn.microsoft.com/entra/identity-platform/permissions-consent-overview#openid-connect-scopes\n */\n scopes: Array;\n };\n tokenRequest: {\n scopes: Array;\n };\n}\n\n// Fetch the auth setup JSON data from the API if not already cached\nasync function fetchAuthSetup(): Promise {\n const response = await fetch(\"/auth_setup\");\n if (!response.ok) {\n throw new Error(`auth setup response was not ok: ${response.status}`);\n }\n return await response.json();\n}\n\nconst authSetup = await fetchAuthSetup();\n\nexport const useLogin = authSetup.useLogin;\n\nexport const requireAccessControl = authSetup.requireAccessControl;\n\nexport const enableUnauthenticatedAccess = authSetup.enableUnauthenticatedAccess;\n\nexport const requireLogin = requireAccessControl && !enableUnauthenticatedAccess;\n\n/**\n * Configuration object to be passed to MSAL instance on creation.\n * For a full list of MSAL.js configuration parameters, visit:\n * https://github.com/AzureAD/microsoft-authentication-library-for-js/blob/dev/lib/msal-browser/docs/configuration.md\n */\nexport const msalConfig = authSetup.msalConfig;\n\n/**\n * Scopes you add here will be prompted for user consent during sign-in.\n * By default, MSAL.js will add OIDC scopes (openid, profile, email) to any login request.\n * For more information about OIDC scopes, visit:\n * https://learn.microsoft.com/entra/identity-platform/permissions-consent-overview#openid-connect-scopes\n */\nexport const loginRequest = authSetup.loginRequest;\n\nconst tokenRequest = authSetup.tokenRequest;\n\n// Build an absolute redirect URI using the current window's location and the relative redirect URI from auth setup\nexport const getRedirectUri = () => {\n return window.location.origin + authSetup.msalConfig.auth.redirectUri;\n};\n\n// Cache the app services token if it's available\n// https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Operators/this#global_context\ndeclare global {\n var cachedAppServicesToken: AppServicesToken | null;\n}\nglobalThis.cachedAppServicesToken = null;\n\n/**\n * Retrieves an access token if the user is logged in using app services authentication.\n * Checks if the current token is expired and fetches a new token if necessary.\n * Returns null if the app doesn't support app services authentication.\n *\n * @returns {Promise} A promise that resolves to an AppServicesToken if the user is authenticated, or null if authentication is not supported or fails.\n */\nconst getAppServicesToken = (): Promise => {\n const checkNotExpired = (appServicesToken: AppServicesToken) => {\n const currentDate = new Date();\n const expiresOnDate = new Date(appServicesToken.expires_on);\n return expiresOnDate > currentDate;\n };\n\n if (globalThis.cachedAppServicesToken && checkNotExpired(globalThis.cachedAppServicesToken)) {\n return Promise.resolve(globalThis.cachedAppServicesToken);\n }\n\n const getAppServicesTokenFromMe: () => Promise = () => {\n return fetch(appServicesAuthTokenUrl).then(r => {\n if (r.ok) {\n return r.json().then(json => {\n if (json.length > 0) {\n return {\n id_token: json[0][\"id_token\"] as string,\n access_token: json[0][\"access_token\"] as string,\n user_claims: json[0][\"user_claims\"].reduce((acc: Record, item: Record) => {\n acc[item.typ] = item.val;\n return acc;\n }, {}) as Record,\n expires_on: json[0][\"expires_on\"] as string\n } as AppServicesToken;\n }\n\n return null;\n });\n }\n\n return null;\n });\n };\n\n return getAppServicesTokenFromMe().then(token => {\n if (token) {\n if (checkNotExpired(token)) {\n globalThis.cachedAppServicesToken = token;\n return token;\n }\n\n return fetch(appServicesAuthTokenRefreshUrl).then(r => {\n if (r.ok) {\n return getAppServicesTokenFromMe();\n }\n return null;\n });\n }\n\n return null;\n });\n};\n\nexport const isUsingAppServicesLogin = (await getAppServicesToken()) != null;\n\n// Sign out of app services\n// Learn more at https://learn.microsoft.com/azure/app-service/configure-authentication-customize-sign-in-out#sign-out-of-a-session\nexport const appServicesLogout = () => {\n window.location.href = appServicesAuthLogoutUrl;\n};\n\n/**\n * Determines if the user is logged in either via the MSAL public client application or the app services login.\n * @param {IPublicClientApplication | undefined} client - The MSAL public client application instance, or undefined if not available.\n * @returns {Promise} A promise that resolves to true if the user is logged in, false otherwise.\n */\nexport const checkLoggedIn = async (client: IPublicClientApplication | undefined): Promise => {\n if (client) {\n const activeAccount = client.getActiveAccount();\n if (activeAccount) {\n return true;\n }\n }\n\n const appServicesToken = await getAppServicesToken();\n if (appServicesToken) {\n return true;\n }\n\n return false;\n};\n\n// Get an access token for use with the API server.\n// ID token received when logging in may not be used for this purpose because it has the incorrect audience\n// Use the access token from app services login if available\nexport const getToken = async (client: IPublicClientApplication): Promise => {\n const appServicesToken = await getAppServicesToken();\n if (appServicesToken) {\n return Promise.resolve(appServicesToken.access_token);\n }\n\n return client\n .acquireTokenSilent({\n ...tokenRequest,\n redirectUri: getRedirectUri()\n })\n .then(r => r.accessToken)\n .catch(error => {\n console.log(error);\n return undefined;\n });\n};\n\n/**\n * Retrieves the username of the active account.\n * If no active account is found, attempts to retrieve the username from the app services login token if available.\n * @param {IPublicClientApplication} client - The MSAL public client application instance.\n * @returns {Promise} The username of the active account, or null if no username is found.\n */\nexport const getUsername = async (client: IPublicClientApplication): Promise => {\n const activeAccount = client.getActiveAccount();\n if (activeAccount) {\n return activeAccount.username;\n }\n\n const appServicesToken = await getAppServicesToken();\n if (appServicesToken?.user_claims) {\n return appServicesToken.user_claims.preferred_username;\n }\n\n return null;\n};\n\n/**\n * Retrieves the token claims of the active account.\n * If no active account is found, attempts to retrieve the token claims from the app services login token if available.\n * @param {IPublicClientApplication} client - The MSAL public client application instance.\n * @returns {Promise | undefined>} A promise that resolves to the token claims of the active account, the user claims from the app services login token, or undefined if no claims are found.\n */\nexport const getTokenClaims = async (client: IPublicClientApplication): Promise | undefined> => {\n const activeAccount = client.getActiveAccount();\n if (activeAccount) {\n return activeAccount.idTokenClaims;\n }\n\n const appServicesToken = await getAppServicesToken();\n if (appServicesToken) {\n return appServicesToken.user_claims;\n }\n\n return undefined;\n};\n" }, { "path": "app/frontend/src/components/AnalysisPanel/AgentPlan.tsx", "content": "import React from \"react\";\nimport { Light as SyntaxHighlighter } from \"react-syntax-highlighter\";\nimport json from \"react-syntax-highlighter/dist/esm/languages/hljs/json\";\nimport { a11yLight } from \"react-syntax-highlighter/dist/esm/styles/hljs\";\n\nimport { getCitationFilePath } from \"../../api\";\nimport { QueryPlanStep, getStepLabel } from \"./agentPlanUtils\";\nimport styles from \"./AnalysisPanel.module.css\";\nimport { TokenUsage, TokenUsageGraph } from \"./TokenUsageGraph\";\n\nSyntaxHighlighter.registerLanguage(\"json\", json);\n\nconst renderDetail = (step: QueryPlanStep) => {\n switch (step.type) {\n case \"modelQueryPlanning\": {\n const usage: TokenUsage = {\n prompt_tokens: step.input_tokens ?? 0,\n completion_tokens: step.output_tokens ?? 0,\n reasoning_tokens: 0,\n total_tokens: (step.input_tokens ?? 0) + (step.output_tokens ?? 0)\n };\n\n return ;\n }\n case \"searchIndex\": {\n const search = step.search_index_arguments?.search ?? \"—\";\n return (\n <>\n
\n Source: {step.knowledge_source_name ?? \"search index\"}\n
\n
\n Search: {search}\n
\n \n );\n }\n case \"web\": {\n const webSearch = step.web_arguments?.search ?? \"—\";\n return (\n <>\n
\n Source: {step.knowledge_source_name ?? \"web\"}\n
\n
\n Search:\n {webSearch}\n
\n \n );\n }\n case \"remoteSharePoint\": {\n const sharepointSearch = step.remote_share_point_arguments?.search ?? \"—\";\n return (\n <>\n
\n Source: {step.knowledge_source_name ?? \"SharePoint\"}\n
\n
\n Search: \n {sharepointSearch}\n
\n \n );\n }\n case \"agenticReasoning\": {\n const usage: TokenUsage = {\n prompt_tokens: 0,\n completion_tokens: step.reasoning_tokens ?? 0,\n reasoning_tokens: step.reasoning_tokens ?? 0,\n total_tokens: step.reasoning_tokens ?? 0\n };\n\n return (\n <>\n \n
\n This step uses Azure AI Search models, so the token capacity does not affect the deployed model.\n
\n \n );\n }\n case \"modelAnswerSynthesis\": {\n const usage: TokenUsage = {\n prompt_tokens: step.input_tokens ?? 0,\n completion_tokens: step.output_tokens ?? 0,\n reasoning_tokens: 0,\n total_tokens: (step.input_tokens ?? 0) + (step.output_tokens ?? 0)\n };\n\n return ;\n }\n default:\n return (\n \n {JSON.stringify(step, null, 2)}\n \n );\n }\n};\n\ninterface Props {\n queryPlan: QueryPlanStep[];\n onEffortExtracted?: (effort: string | undefined) => void;\n onCitationClicked?: (citationFilePath: string) => void;\n results?: any[];\n}\n\nexport const AgentPlan: React.FC = ({ queryPlan, onEffortExtracted, onCitationClicked, results }) => {\n // Helper to get search query for a step\n const getStepQuery = (step: QueryPlanStep): string | undefined => {\n if (step.search_index_arguments?.search) return step.search_index_arguments.search;\n if (step.web_arguments?.search) return step.web_arguments.search;\n if (step.remote_share_point_arguments?.search) return step.remote_share_point_arguments.search;\n return undefined;\n };\n\n // Helper to get results for a specific step\n const getResultsForStep = (step: QueryPlanStep): any[] => {\n if (!results || results.length === 0) return [];\n const stepQuery = getStepQuery(step);\n if (!stepQuery) return [];\n\n // Filter by both query and step type, then de-duplicate by filename\n const filtered = results.filter(result => result.activity?.query === stepQuery && result.type == step.type);\n const uniqueMap = new Map(filtered.map(r => [r.sourcepage || r.web_url || r.url, r]));\n return Array.from(uniqueMap.values());\n };\n\n const stepNumberLookup = React.useMemo(() => {\n const lookup: Record = {};\n queryPlan.forEach((step, index) => {\n if (step != null && step.id !== undefined && step.id !== null) {\n lookup[String(step.id)] = index + 1;\n }\n });\n return lookup;\n }, [queryPlan]);\n\n const iterations = React.useMemo(() => {\n if (!queryPlan || queryPlan.length === 0) {\n return [] as QueryPlanStep[][];\n }\n\n const planningIndices = queryPlan.reduce((indices, step, index) => {\n if (step.type === \"modelQueryPlanning\") {\n indices.push(index);\n }\n return indices;\n }, []);\n\n if (planningIndices.length <= 1) {\n return [queryPlan];\n }\n\n const iterationsList: QueryPlanStep[][] = [];\n const prePlanningSteps = planningIndices[0] > 0 ? queryPlan.slice(0, planningIndices[0]) : [];\n\n planningIndices.forEach((planningIndex, idx) => {\n const nextPlanningIndex = planningIndices[idx + 1] ?? queryPlan.length;\n const iterationSteps = queryPlan.slice(planningIndex, nextPlanningIndex);\n\n if (idx === 0 && prePlanningSteps.length > 0) {\n iterationsList.push([...prePlanningSteps, ...iterationSteps]);\n } else if (iterationSteps.length > 0) {\n iterationsList.push(iterationSteps);\n }\n });\n\n return iterationsList;\n }, [queryPlan]);\n\n React.useEffect(() => {\n // Extract effort from first agentic reasoning step\n const agenticStep = queryPlan.find(step => step.type === \"agenticReasoning\");\n const effort = agenticStep?.retrieval_reasoning_effort?.kind;\n if (onEffortExtracted) {\n onEffortExtracted(effort);\n }\n }, [queryPlan, onEffortExtracted]);\n\n if (iterations.length === 0) {\n return null;\n }\n\n return (\n
\n {iterations.map((iterationSteps, iterationIndex) => {\n const hasMultipleIterations = iterations.length > 1;\n const headerLabel = hasMultipleIterations ? `Iteration ${iterationIndex + 1} Execution steps` : \"Execution steps\";\n\n return (\n
\n
{headerLabel}
\n \n \n \n \n \n \n \n \n \n {iterationSteps.map(step => {\n const stepId = step?.id;\n const stepNumber = stepId !== undefined ? stepNumberLookup[String(stepId)] : undefined;\n const stepResults = getResultsForStep(step);\n\n return (\n \n \n \n \n \n );\n })}\n \n
StepDetailsElapsed MS
\n
\n {stepNumber && {`Step ${stepNumber}:`}}\n {getStepLabel(step)}\n
\n 		\n {renderDetail(step)}\n {(step.type === \"searchIndex\" || step.type === \"remoteSharePoint\" || step.type === \"web\") &&\n (stepResults.length > 0 ? (\n
\n {stepResults.map((result, idx) => {\n // Handle different result types\n if (result.type === \"remoteSharePoint\" && result.web_url) {\n return (\n \n );\n } else if (result.url) {\n // Web result\n return (\n \n );\n } else if (result.sourcepage) {\n // Document result - make it clickable to open citation tab\n const path = getCitationFilePath(result.sourcepage);\n return (\n
\n {\n e.preventDefault();\n if (onCitationClicked) {\n onCitationClicked(path);\n }\n }}\n style={{ cursor: \"pointer\" }}\n >\n {result.sourcepage}\n \n
\n );\n }\n return null;\n })}\n
\n ) : (\n
No results found
\n ))}\n 		{step.elapsed_ms ?? \"—\"}
\n
\n );\n })}\n
\n );\n};\n" }, { "path": "app/frontend/src/components/AnalysisPanel/AnalysisPanel.module.css", "content": ".thoughtProcess {\n font-family: source-code-pro, Menlo, Monaco, Consolas, \"Courier New\", monospace;\n word-wrap: break-word;\n padding-top: 0.75em;\n padding-bottom: 0.75em;\n}\n\n.tList {\n padding: 1.25em 1.25em 0 1.25em;\n display: block; /* allow it to shrink within container */\n background: #e9e9e9;\n width: 100%;\n box-sizing: border-box;\n overflow-wrap: anywhere; /* aggressive wrapping for very long tokens/urls */\n}\n\n.tListItem {\n list-style: none;\n margin: auto;\n margin-left: 1.25em;\n min-height: 3.125em;\n border-left: 0.0625em solid #123bb6;\n padding: 0 0 1.875em 1.875em;\n position: relative;\n}\n\n.tListItem:last-child {\n border-left: 0;\n}\n\n.tListItem::before {\n position: absolute;\n left: -18px;\n top: -5px;\n content: \" \";\n border: 8px solid #d1dbfa;\n border-radius: 500%;\n background: #123bb6;\n height: 20px;\n width: 20px;\n}\n\n.tStep {\n color: #123bb6;\n position: relative;\n font-size: 0.875em;\n margin-bottom: 0.5em;\n}\n\n.tCodeBlock {\n max-height: 18.75em;\n overflow: auto;\n white-space: pre-wrap; /* allow wrapping */\n word-break: break-word;\n overflow-wrap: anywhere;\n width: 100%;\n box-sizing: border-box;\n}\n\n.tPropRow {\n flex-wrap: wrap;\n gap: 5px;\n max-width: 100%;\n margin-bottom: 0.5em;\n}\n\n.tProp {\n display: inline-block;\n background-color: #d7d7d7;\n font-size: 0.95em;\n padding: 0.1875em 0.625em;\n border-radius: 0.625em;\n margin-bottom: 0.3em;\n word-break: break-word;\n white-space: normal;\n overflow-wrap: anywhere;\n}\n\n.citationImg {\n height: 28.125rem;\n max-width: 100%;\n object-fit: contain;\n}\n\n.header {\n color: #123bb6;\n position: relative;\n font-size: 0.875em;\n margin-bottom: 0.5em;\n}\n\n.reasoningEffort {\n font-size: 14px;\n margin-bottom: 8px;\n}\n\n.tokenUsageGraph {\n padding: 6px 6px;\n width: 100%;\n box-sizing: border-box;\n}\n\n.tokenBar {\n min-height: 28px;\n height: auto;\n margin-bottom: 8px;\n padding: 6px 12px;\n font-size: inherit;\n display: flex;\n flex-wrap: wrap;\n align-items: center;\n background-color: #d7d7d7;\n white-space: normal;\n overflow: hidden;\n word-break: break-word;\n overflow-wrap: anywhere;\n min-width: 0;\n box-sizing: border-box;\n}\n\n/* Adjust tokenLabel to allow bar-specific text color overrides */\n.tokenLabel {\n font-size: inherit;\n padding-right: 4px;\n flex: 1 1 auto;\n word-break: break-word;\n overflow-wrap: anywhere;\n}\n\n.primaryBarContainer {\n width: 100%;\n display: flex;\n gap: 8px;\n padding: 4px 0;\n flex-wrap: nowrap;\n}\n\n.primaryBarContainer .tokenBar {\n margin-bottom: 0;\n}\n\n.promptBar {\n background-color: #a82424;\n color: #ffffff; /* White text for contrast */\n}\n\n.reasoningBar {\n background-color: #265e29;\n color: #ffffff;\n}\n\n.outputBar {\n background-color: #12579b;\n color: #ffffff;\n min-width: 120px;\n}\n\n.totalBar {\n background-color: #424242;\n color: #ffffff;\n}\n\n.secondaryTotalBar {\n background-color: #6d6d6d;\n color: #ffffff;\n}\n\n.segmentWrapper {\n margin-top: 16px;\n padding-top: 4px;\n}\n\n.segmentWrapperFirst {\n margin-top: 0;\n}\n\n.standaloneTotalBar {\n margin-top: 10px;\n}\n\n.groupedTotalBar {\n margin-top: 2px;\n}\n\n.subqueriesTable,\n.subqueriesTable th,\n.subqueriesTable td,\n.subqueriesTable tr {\n background: #fff;\n}\n\n.iterationSection {\n margin-top: 1.5em;\n}\n\n.iterationSection:first-of-type {\n margin-top: 0;\n}\n\n.stepHeaderCell {\n display: flex;\n flex-direction: column;\n align-items: flex-start;\n gap: 0.25em;\n}\n\n.stepNumberText {\n font-weight: 600;\n color: #123bb6;\n}\n\n.stepLabel {\n font-weight: 600;\n}\n\n.stepCitations {\n margin-top: 0.75em;\n display: flex;\n flex-direction: column;\n gap: 0.375em;\n}\n\n.stepResults {\n margin-top: 0.5em;\n margin-bottom: 0.5em;\n display: flex;\n flex-direction: column;\n gap: 0.25em;\n}\n\n.noResults {\n margin-top: 0.5em;\n font-size: 0.85em;\n color: #666;\n font-style: italic;\n}\n\n.stepResult {\n display: inline-block;\n font-size: 0.85em;\n word-break: break-word;\n overflow-wrap: anywhere;\n}\n\n.stepResult a {\n font-weight: 500;\n line-height: 1.5em;\n text-align: center;\n border-radius: 0.25em;\n padding: 0em 0.5em;\n background: #d1dbfa;\n color: #123bb6;\n text-decoration: none;\n cursor: pointer;\n display: inline-block;\n}\n\n.stepResult a:hover {\n text-decoration: underline;\n}\n\n.stepCitationEntry {\n display: inline-flex;\n flex-wrap: wrap;\n align-items: center;\n gap: 0.25em;\n white-space: normal;\n word-break: break-word;\n overflow-wrap: anywhere;\n max-width: 100%;\n}\n\n.stepCitationEntry a {\n white-space: normal;\n word-break: break-word;\n overflow-wrap: anywhere;\n text-align: left;\n max-width: 100%;\n}\n" }, { "path": "app/frontend/src/components/AnalysisPanel/AnalysisPanel.tsx", "content": "import { useMsal } from \"@azure/msal-react\";\nimport { Tab, TabList, SelectTabData, SelectTabEvent } from \"@fluentui/react-components\";\nimport { useEffect, useState } from \"react\";\nimport { useTranslation } from \"react-i18next\";\n\nimport { ChatAppResponse, getHeaders } from \"../../api\";\nimport { getToken, useLogin } from \"../../authConfig\";\nimport { MarkdownViewer } from \"../MarkdownViewer\";\nimport { SupportingContent } from \"../SupportingContent\";\nimport styles from \"./AnalysisPanel.module.css\";\nimport { AnalysisPanelTabs } from \"./AnalysisPanelTabs\";\nimport { ThoughtProcess } from \"./ThoughtProcess\";\n\ninterface Props {\n className: string;\n activeTab: AnalysisPanelTabs;\n onActiveTabChanged: (tab: AnalysisPanelTabs) => void;\n activeCitation: string | undefined;\n citationHeight: string;\n answer: ChatAppResponse;\n onCitationClicked?: (citationFilePath: string) => void;\n}\n\nexport const AnalysisPanel = ({ answer, activeTab, activeCitation, citationHeight, className, onActiveTabChanged, onCitationClicked }: Props) => {\n const isDisabledThoughtProcessTab: boolean = !answer.context.thoughts;\n const dataPoints = answer.context.data_points;\n const hasSupportingContent = Boolean(\n dataPoints &&\n ((dataPoints.text && dataPoints.text.length > 0) ||\n (dataPoints.images && dataPoints.images.length > 0) ||\n (dataPoints.external_results_metadata && dataPoints.external_results_metadata.length > 0))\n );\n const isDisabledSupportingContentTab: boolean = !hasSupportingContent;\n const isDisabledCitationTab: boolean = !activeCitation;\n const [citation, setCitation] = useState(\"\");\n\n const client = useLogin ? useMsal().instance : undefined;\n const { t } = useTranslation();\n\n const fetchCitation = async () => {\n const token = client ? await getToken(client) : undefined;\n if (activeCitation) {\n // Get hash from the URL as it may contain #page=N\n // which helps browser PDF renderer jump to correct page N\n const originalHash = activeCitation.indexOf(\"#\") ? activeCitation.split(\"#\")[1] : \"\";\n const response = await fetch(activeCitation, {\n method: \"GET\",\n headers: await getHeaders(token)\n });\n const citationContent = await response.blob();\n let citationObjectUrl = URL.createObjectURL(citationContent);\n // Add hash back to the new blob URL\n if (originalHash) {\n citationObjectUrl += \"#\" + originalHash;\n }\n setCitation(citationObjectUrl);\n }\n };\n useEffect(() => {\n fetchCitation();\n }, []);\n\n const renderFileViewer = () => {\n if (!activeCitation) {\n return null;\n }\n\n const fileExtension = activeCitation.split(\".\").pop()?.toLowerCase();\n switch (fileExtension) {\n case \"png\":\n return \"Citation;\n case \"md\":\n return ;\n default:\n return